1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _X86_POSTED_INTR_H
#define _X86_POSTED_INTR_H
#include <asm/cmpxchg.h>
#include <asm/rwonce.h>
#include <asm/irq_vectors.h>
#include <linux/bitmap.h>
#define POSTED_INTR_ON 0
#define POSTED_INTR_SN 1
#define PID_TABLE_ENTRY_VALID 1
#define NR_PIR_VECTORS 256
#define NR_PIR_WORDS (NR_PIR_VECTORS / BITS_PER_LONG)
/* Posted-Interrupt Descriptor */
struct pi_desc {
unsigned long pir[NR_PIR_WORDS]; /* Posted interrupt requested */
union {
struct {
u16 notifications; /* Suppress and outstanding bits */
u8 nv;
u8 rsvd_2;
u32 ndst;
};
u64 control;
};
u32 rsvd[6];
} __aligned(64);
/*
* De-multiplexing posted interrupts is on the performance path, the code
* below is written to optimize the cache performance based on the following
* considerations:
* 1.Posted interrupt descriptor (PID) fits in a cache line that is frequently
* accessed by both CPU and IOMMU.
* 2.During software processing of posted interrupts, the CPU needs to do
* natural width read and xchg for checking and clearing posted interrupt
* request (PIR), a 256 bit field within the PID.
* 3.On the other side, the IOMMU does atomic swaps of the entire PID cache
* line when posting interrupts and setting control bits.
* 4.The CPU can access the cache line a magnitude faster than the IOMMU.
* 5.Each time the IOMMU does interrupt posting to the PIR will evict the PID
* cache line. The cache line states after each operation are as follows,
* assuming a 64-bit kernel:
* CPU IOMMU PID Cache line state
* ---------------------------------------------------------------
*...read64 exclusive
*...lock xchg64 modified
*... post/atomic swap invalid
*...-------------------------------------------------------------
*
* To reduce L1 data cache miss, it is important to avoid contention with
* IOMMU's interrupt posting/atomic swap. Therefore, a copy of PIR is used
* when processing posted interrupts in software, e.g. to dispatch interrupt
* handlers for posted MSIs, or to move interrupts from the PIR to the vIRR
* in KVM.
*
* In addition, the code is trying to keep the cache line state consistent
* as much as possible. e.g. when making a copy and clearing the PIR
* (assuming non-zero PIR bits are present in the entire PIR), it does:
* read, read, read, read, xchg, xchg, xchg, xchg
* instead of:
* read, xchg, read, xchg, read, xchg, read, xchg
*/
static __always_inline bool pi_harvest_pir(unsigned long *pir,
unsigned long *pir_vals)
{
unsigned long pending = 0;
int i;
for (i = 0; i < NR_PIR_WORDS; i++) {
pir_vals[i] = READ_ONCE(pir[i]);
pending |= pir_vals[i];
}
if (!pending)
return false;
for (i = 0; i < NR_PIR_WORDS; i++) {
if (!pir_vals[i])
continue;
pir_vals[i] = arch_xchg(&pir[i], 0);
}
return true;
}
static inline bool pi_test_and_set_on(struct pi_desc *pi_desc)
{
return test_and_set_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
}
static inline bool pi_test_and_clear_on(struct pi_desc *pi_desc)
{
return test_and_clear_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
}
static inline bool pi_test_and_clear_sn(struct pi_desc *pi_desc)
{
return test_and_clear_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
}
static inline bool pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
{
return test_and_set_bit(vector, pi_desc->pir);
}
static inline bool pi_is_pir_empty(struct pi_desc *pi_desc)
{
return bitmap_empty(pi_desc->pir, NR_VECTORS);
}
static inline void pi_set_sn(struct pi_desc *pi_desc)
{
set_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
}
static inline void pi_set_on(struct pi_desc *pi_desc)
{
set_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
}
static inline void pi_clear_on(struct pi_desc *pi_desc)
{
clear_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
}
static inline void pi_clear_sn(struct pi_desc *pi_desc)
{
clear_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
}
static inline bool pi_test_on(struct pi_desc *pi_desc)
{
return test_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
}
static inline bool pi_test_sn(struct pi_desc *pi_desc)
{
return test_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
}
static inline bool pi_test_pir(int vector, struct pi_desc *pi_desc)
{
return test_bit(vector, (unsigned long *)pi_desc->pir);
}
/* Non-atomic helpers */
static inline void __pi_set_sn(struct pi_desc *pi_desc)
{
pi_desc->notifications |= BIT(POSTED_INTR_SN);
}
static inline void __pi_clear_sn(struct pi_desc *pi_desc)
{
pi_desc->notifications &= ~BIT(POSTED_INTR_SN);
}
#ifdef CONFIG_X86_POSTED_MSI
/*
* Not all external vectors are subject to interrupt remapping, e.g. IOMMU's
* own interrupts. Here we do not distinguish them since those vector bits in
* PIR will always be zero.
*/
static inline bool pi_pending_this_cpu(unsigned int vector)
{
struct pi_desc *pid = this_cpu_ptr(&posted_msi_pi_desc);
if (WARN_ON_ONCE(vector > NR_VECTORS || vector < FIRST_EXTERNAL_VECTOR))
return false;
return test_bit(vector, pid->pir);
}
extern void intel_posted_msi_init(void);
#else
static inline bool pi_pending_this_cpu(unsigned int vector) { return false; }
static inline void intel_posted_msi_init(void) {};
#endif /* X86_POSTED_MSI */
#endif /* _X86_POSTED_INTR_H */
|
