summaryrefslogtreecommitdiff
path: root/arch/powerpc/include/asm/hw_irq.h
blob: 317659fdeacf2b1bfd2305a7890ffc0d2e7d76e1 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
 */
#ifndef _ASM_POWERPC_HW_IRQ_H
#define _ASM_POWERPC_HW_IRQ_H

#ifdef __KERNEL__

#include <linux/errno.h>
#include <linux/compiler.h>
#include <asm/ptrace.h>
#include <asm/processor.h>

#ifdef CONFIG_PPC64

/*
 * PACA flags in paca->irq_happened.
 *
 * This bits are set when interrupts occur while soft-disabled
 * and allow a proper replay.
 *
 * The PACA_IRQ_HARD_DIS is set whenever we hard disable. It is almost
 * always in synch with the MSR[EE] state, except:
 * - A window in interrupt entry, where hardware disables MSR[EE] and that
 *   must be "reconciled" with the soft mask state.
 * - NMI interrupts that hit in awkward places, until they fix the state.
 * - When local irqs are being enabled and state is being fixed up.
 * - When returning from an interrupt there are some windows where this
 *   can become out of synch, but gets fixed before the RFI or before
 *   executing the next user instruction (see arch/powerpc/kernel/interrupt.c).
 */
#define PACA_IRQ_HARD_DIS	0x01
#define PACA_IRQ_DBELL		0x02
#define PACA_IRQ_EE		0x04
#define PACA_IRQ_DEC		0x08 /* Or FIT */
#define PACA_IRQ_HMI		0x10
#define PACA_IRQ_PMI		0x20
#define PACA_IRQ_REPLAYING	0x40

/*
 * Some soft-masked interrupts must be hard masked until they are replayed
 * (e.g., because the soft-masked handler does not clear the exception).
 * Interrupt replay itself must remain hard masked too.
 */
#ifdef CONFIG_PPC_BOOK3S
#define PACA_IRQ_MUST_HARD_MASK	(PACA_IRQ_EE|PACA_IRQ_PMI|PACA_IRQ_REPLAYING)
#else
#define PACA_IRQ_MUST_HARD_MASK	(PACA_IRQ_EE|PACA_IRQ_REPLAYING)
#endif

#endif /* CONFIG_PPC64 */

/*
 * flags for paca->irq_soft_mask
 */
#define IRQS_ENABLED		0
#define IRQS_DISABLED		1 /* local_irq_disable() interrupts */
#define IRQS_PMI_DISABLED	2
#define IRQS_ALL_DISABLED	(IRQS_DISABLED | IRQS_PMI_DISABLED)

#ifndef __ASSEMBLY__

static inline void __hard_irq_enable(void)
{
	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
		wrtee(MSR_EE);
	else if (IS_ENABLED(CONFIG_PPC_8xx))
		wrtspr(SPRN_EIE);
	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
		__mtmsrd(MSR_EE | MSR_RI, 1);
	else
		mtmsr(mfmsr() | MSR_EE);
}

static inline void __hard_irq_disable(void)
{
	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
		wrtee(0);
	else if (IS_ENABLED(CONFIG_PPC_8xx))
		wrtspr(SPRN_EID);
	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
		__mtmsrd(MSR_RI, 1);
	else
		mtmsr(mfmsr() & ~MSR_EE);
}

static inline void __hard_EE_RI_disable(void)
{
	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
		wrtee(0);
	else if (IS_ENABLED(CONFIG_PPC_8xx))
		wrtspr(SPRN_NRI);
	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
		__mtmsrd(0, 1);
	else
		mtmsr(mfmsr() & ~(MSR_EE | MSR_RI));
}

static inline void __hard_RI_enable(void)
{
	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
		return;

	if (IS_ENABLED(CONFIG_PPC_8xx))
		wrtspr(SPRN_EID);
	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
		__mtmsrd(MSR_RI, 1);
	else
		mtmsr(mfmsr() | MSR_RI);
}

#ifdef CONFIG_PPC64
#include <asm/paca.h>

static inline notrace unsigned long irq_soft_mask_return(void)
{
	unsigned long flags;

	asm volatile(
		"lbz %0,%1(13)"
		: "=r" (flags)
		: "i" (offsetof(struct paca_struct, irq_soft_mask)));

	return flags;
}

/*
 * The "memory" clobber acts as both a compiler barrier
 * for the critical section and as a clobber because
 * we changed paca->irq_soft_mask
 */
static inline notrace void irq_soft_mask_set(unsigned long mask)
{
	/*
	 * The irq mask must always include the STD bit if any are set.
	 *
	 * and interrupts don't get replayed until the standard
	 * interrupt (local_irq_disable()) is unmasked.
	 *
	 * Other masks must only provide additional masking beyond
	 * the standard, and they are also not replayed until the
	 * standard interrupt becomes unmasked.
	 *
	 * This could be changed, but it will require partial
	 * unmasks to be replayed, among other things. For now, take
	 * the simple approach.
	 */
	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
		WARN_ON(mask && !(mask & IRQS_DISABLED));

	asm volatile(
		"stb %0,%1(13)"
		:
		: "r" (mask),
		  "i" (offsetof(struct paca_struct, irq_soft_mask))
		: "memory");
}

static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask)
{
	unsigned long flags = irq_soft_mask_return();

	irq_soft_mask_set(mask);

	return flags;
}

static inline notrace unsigned long irq_soft_mask_or_return(unsigned long mask)
{
	unsigned long flags = irq_soft_mask_return();

	irq_soft_mask_set(flags | mask);

	return flags;
}

static inline notrace unsigned long irq_soft_mask_andc_return(unsigned long mask)
{
	unsigned long flags = irq_soft_mask_return();

	irq_soft_mask_set(flags & ~mask);

	return flags;
}

static inline unsigned long arch_local_save_flags(void)
{
	return irq_soft_mask_return();
}

static inline void arch_local_irq_disable(void)
{
	irq_soft_mask_set(IRQS_DISABLED);
}

extern void arch_local_irq_restore(unsigned long);

static inline void arch_local_irq_enable(void)
{
	arch_local_irq_restore(IRQS_ENABLED);
}

static inline unsigned long arch_local_irq_save(void)
{
	return irq_soft_mask_or_return(IRQS_DISABLED);
}

static inline bool arch_irqs_disabled_flags(unsigned long flags)
{
	return flags & IRQS_DISABLED;
}

static inline bool arch_irqs_disabled(void)
{
	return arch_irqs_disabled_flags(arch_local_save_flags());
}

static inline void set_pmi_irq_pending(void)
{
	/*
	 * Invoked from PMU callback functions to set PMI bit in the paca.
	 * This has to be called with irq's disabled (via hard_irq_disable()).
	 */
	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
		WARN_ON_ONCE(mfmsr() & MSR_EE);

	get_paca()->irq_happened |= PACA_IRQ_PMI;
}

static inline void clear_pmi_irq_pending(void)
{
	/*
	 * Invoked from PMU callback functions to clear the pending PMI bit
	 * in the paca.
	 */
	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
		WARN_ON_ONCE(mfmsr() & MSR_EE);

	get_paca()->irq_happened &= ~PACA_IRQ_PMI;
}

static inline bool pmi_irq_pending(void)
{
	/*
	 * Invoked from PMU callback functions to check if there is a pending
	 * PMI bit in the paca.
	 */
	if (get_paca()->irq_happened & PACA_IRQ_PMI)
		return true;

	return false;
}

#ifdef CONFIG_PPC_BOOK3S
/*
 * To support disabling and enabling of irq with PMI, set of
 * new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore()
 * functions are added. These macros are implemented using generic
 * linux local_irq_* code from include/linux/irqflags.h.
 */
#define raw_local_irq_pmu_save(flags)					\
	do {								\
		typecheck(unsigned long, flags);			\
		flags = irq_soft_mask_or_return(IRQS_DISABLED |	\
				IRQS_PMI_DISABLED);			\
	} while(0)

#define raw_local_irq_pmu_restore(flags)				\
	do {								\
		typecheck(unsigned long, flags);			\
		arch_local_irq_restore(flags);				\
	} while(0)

#ifdef CONFIG_TRACE_IRQFLAGS
#define powerpc_local_irq_pmu_save(flags)			\
	 do {							\
		raw_local_irq_pmu_save(flags);			\
		if (!raw_irqs_disabled_flags(flags))		\
			trace_hardirqs_off();			\
	} while(0)
#define powerpc_local_irq_pmu_restore(flags)			\
	do {							\
		if (!raw_irqs_disabled_flags(flags))		\
			trace_hardirqs_on();			\
		raw_local_irq_pmu_restore(flags);		\
	} while(0)
#else
#define powerpc_local_irq_pmu_save(flags)			\
	do {							\
		raw_local_irq_pmu_save(flags);			\
	} while(0)
#define powerpc_local_irq_pmu_restore(flags)			\
	do {							\
		raw_local_irq_pmu_restore(flags);		\
	} while (0)
#endif  /* CONFIG_TRACE_IRQFLAGS */

#endif /* CONFIG_PPC_BOOK3S */

#define hard_irq_disable()	do {					\
	unsigned long flags;						\
	__hard_irq_disable();						\
	flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED);		\
	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;			\
	if (!arch_irqs_disabled_flags(flags)) {				\
		asm volatile("std%X0 %1,%0" : "=m" (local_paca->saved_r1) \
					    : "r" (current_stack_pointer)); \
		trace_hardirqs_off();					\
	}								\
} while(0)

static inline bool __lazy_irq_pending(u8 irq_happened)
{
	return !!(irq_happened & ~PACA_IRQ_HARD_DIS);
}

/*
 * Check if a lazy IRQ is pending. Should be called with IRQs hard disabled.
 */
static inline bool lazy_irq_pending(void)
{
	return __lazy_irq_pending(get_paca()->irq_happened);
}

/*
 * Check if a lazy IRQ is pending, with no debugging checks.
 * Should be called with IRQs hard disabled.
 * For use in RI disabled code or other constrained situations.
 */
static inline bool lazy_irq_pending_nocheck(void)
{
	return __lazy_irq_pending(local_paca->irq_happened);
}

bool power_pmu_wants_prompt_pmi(void);

/*
 * This is called by asynchronous interrupts to check whether to
 * conditionally re-enable hard interrupts after having cleared
 * the source of the interrupt. They are kept disabled if there
 * is a different soft-masked interrupt pending that requires hard
 * masking.
 */
static inline bool should_hard_irq_enable(struct pt_regs *regs)
{
	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
		WARN_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
		WARN_ON(!(get_paca()->irq_happened & PACA_IRQ_HARD_DIS));
		WARN_ON(mfmsr() & MSR_EE);
	}

	if (!IS_ENABLED(CONFIG_PERF_EVENTS))
		return false;
	/*
	 * If the PMU is not running, there is not much reason to enable
	 * MSR[EE] in irq handlers because any interrupts would just be
	 * soft-masked.
	 *
	 * TODO: Add test for 64e
	 */
	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
		if (!power_pmu_wants_prompt_pmi())
			return false;
		/*
		 * If PMIs are disabled then IRQs should be disabled as well,
		 * so we shouldn't see this condition, check for it just in
		 * case because we are about to enable PMIs.
		 */
		if (WARN_ON_ONCE(regs->softe & IRQS_PMI_DISABLED))
			return false;
	}

	if (get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK)
		return false;

	return true;
}

/*
 * Do the hard enabling, only call this if should_hard_irq_enable is true.
 * This allows PMI interrupts to profile irq handlers.
 */
static inline void do_hard_irq_enable(void)
{
	/*
	 * Asynch interrupts come in with IRQS_ALL_DISABLED,
	 * PACA_IRQ_HARD_DIS, and MSR[EE]=0.
	 */
	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
		irq_soft_mask_andc_return(IRQS_PMI_DISABLED);
	get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
	__hard_irq_enable();
}

static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
{
	return (regs->softe & IRQS_DISABLED);
}

extern bool prep_irq_for_idle(void);
extern bool prep_irq_for_idle_irqsoff(void);
extern void irq_set_pending_from_srr1(unsigned long srr1);

#define fini_irq_for_idle_irqsoff() trace_hardirqs_off();

extern void force_external_irq_replay(void);

static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
{
	regs->softe = val;
}
#else /* CONFIG_PPC64 */

static inline notrace unsigned long irq_soft_mask_return(void)
{
	return 0;
}

static inline unsigned long arch_local_save_flags(void)
{
	return mfmsr();
}

static inline void arch_local_irq_restore(unsigned long flags)
{
	if (IS_ENABLED(CONFIG_BOOKE))
		wrtee(flags);
	else
		mtmsr(flags);
}

static inline unsigned long arch_local_irq_save(void)
{
	unsigned long flags = arch_local_save_flags();

	if (IS_ENABLED(CONFIG_BOOKE))
		wrtee(0);
	else if (IS_ENABLED(CONFIG_PPC_8xx))
		wrtspr(SPRN_EID);
	else
		mtmsr(flags & ~MSR_EE);

	return flags;
}

static inline void arch_local_irq_disable(void)
{
	__hard_irq_disable();
}

static inline void arch_local_irq_enable(void)
{
	__hard_irq_enable();
}

static inline bool arch_irqs_disabled_flags(unsigned long flags)
{
	return (flags & MSR_EE) == 0;
}

static inline bool arch_irqs_disabled(void)
{
	return arch_irqs_disabled_flags(arch_local_save_flags());
}

#define hard_irq_disable()		arch_local_irq_disable()

static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
{
	return !(regs->msr & MSR_EE);
}

static __always_inline bool should_hard_irq_enable(struct pt_regs *regs)
{
	return false;
}

static inline void do_hard_irq_enable(void)
{
	BUILD_BUG();
}

static inline void clear_pmi_irq_pending(void) { }
static inline void set_pmi_irq_pending(void) { }
static inline bool pmi_irq_pending(void) { return false; }

static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
{
}
#endif /* CONFIG_PPC64 */

static inline unsigned long mtmsr_isync_irqsafe(unsigned long msr)
{
#ifdef CONFIG_PPC64
	if (arch_irqs_disabled()) {
		/*
		 * With soft-masking, MSR[EE] can change from 1 to 0
		 * asynchronously when irqs are disabled, and we don't want to
		 * set MSR[EE] back to 1 here if that has happened. A race-free
		 * way to do this is ensure EE is already 0. Another way it
		 * could be done is with a RESTART_TABLE handler, but that's
		 * probably overkill here.
		 */
		msr &= ~MSR_EE;
		mtmsr_isync(msr);
		irq_soft_mask_set(IRQS_ALL_DISABLED);
		local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
	} else
#endif
		mtmsr_isync(msr);

	return msr;
}


#define ARCH_IRQ_INIT_FLAGS	IRQ_NOREQUEST

#endif  /* __ASSEMBLY__ */
#endif	/* __KERNEL__ */
#endif	/* _ASM_POWERPC_HW_IRQ_H */