summaryrefslogtreecommitdiff
path: root/arch/riscv/mm/context.c
blob: 217fd4de6134224655db81759f0b35c73948e46d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2012 Regents of the University of California
 * Copyright (C) 2017 SiFive
 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
 */

#include <linux/bitops.h>
#include <linux/cpumask.h>
#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/static_key.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>

#ifdef CONFIG_MMU

DEFINE_STATIC_KEY_FALSE(use_asid_allocator);

static unsigned long asid_bits;
static unsigned long num_asids;
unsigned long asid_mask;

static atomic_long_t current_version;

static DEFINE_RAW_SPINLOCK(context_lock);
static cpumask_t context_tlb_flush_pending;
static unsigned long *context_asid_map;

static DEFINE_PER_CPU(atomic_long_t, active_context);
static DEFINE_PER_CPU(unsigned long, reserved_context);

static bool check_update_reserved_context(unsigned long cntx,
					  unsigned long newcntx)
{
	int cpu;
	bool hit = false;

	/*
	 * Iterate over the set of reserved CONTEXT looking for a match.
	 * If we find one, then we can update our mm to use new CONTEXT
	 * (i.e. the same CONTEXT in the current_version) but we can't
	 * exit the loop early, since we need to ensure that all copies
	 * of the old CONTEXT are updated to reflect the mm. Failure to do
	 * so could result in us missing the reserved CONTEXT in a future
	 * version.
	 */
	for_each_possible_cpu(cpu) {
		if (per_cpu(reserved_context, cpu) == cntx) {
			hit = true;
			per_cpu(reserved_context, cpu) = newcntx;
		}
	}

	return hit;
}

static void __flush_context(void)
{
	int i;
	unsigned long cntx;

	/* Must be called with context_lock held */
	lockdep_assert_held(&context_lock);

	/* Update the list of reserved ASIDs and the ASID bitmap. */
	bitmap_zero(context_asid_map, num_asids);

	/* Mark already active ASIDs as used */
	for_each_possible_cpu(i) {
		cntx = atomic_long_xchg_relaxed(&per_cpu(active_context, i), 0);
		/*
		 * If this CPU has already been through a rollover, but
		 * hasn't run another task in the meantime, we must preserve
		 * its reserved CONTEXT, as this is the only trace we have of
		 * the process it is still running.
		 */
		if (cntx == 0)
			cntx = per_cpu(reserved_context, i);

		__set_bit(cntx & asid_mask, context_asid_map);
		per_cpu(reserved_context, i) = cntx;
	}

	/* Mark ASID #0 as used because it is used at boot-time */
	__set_bit(0, context_asid_map);

	/* Queue a TLB invalidation for each CPU on next context-switch */
	cpumask_setall(&context_tlb_flush_pending);
}

static unsigned long __new_context(struct mm_struct *mm)
{
	static u32 cur_idx = 1;
	unsigned long cntx = atomic_long_read(&mm->context.id);
	unsigned long asid, ver = atomic_long_read(&current_version);

	/* Must be called with context_lock held */
	lockdep_assert_held(&context_lock);

	if (cntx != 0) {
		unsigned long newcntx = ver | (cntx & asid_mask);

		/*
		 * If our current CONTEXT was active during a rollover, we
		 * can continue to use it and this was just a false alarm.
		 */
		if (check_update_reserved_context(cntx, newcntx))
			return newcntx;

		/*
		 * We had a valid CONTEXT in a previous life, so try to
		 * re-use it if possible.
		 */
		if (!__test_and_set_bit(cntx & asid_mask, context_asid_map))
			return newcntx;
	}

	/*
	 * Allocate a free ASID. If we can't find one then increment
	 * current_version and flush all ASIDs.
	 */
	asid = find_next_zero_bit(context_asid_map, num_asids, cur_idx);
	if (asid != num_asids)
		goto set_asid;

	/* We're out of ASIDs, so increment current_version */
	ver = atomic_long_add_return_relaxed(num_asids, &current_version);

	/* Flush everything  */
	__flush_context();

	/* We have more ASIDs than CPUs, so this will always succeed */
	asid = find_next_zero_bit(context_asid_map, num_asids, 1);

set_asid:
	__set_bit(asid, context_asid_map);
	cur_idx = asid;
	return asid | ver;
}

static void set_mm_asid(struct mm_struct *mm, unsigned int cpu)
{
	unsigned long flags;
	bool need_flush_tlb = false;
	unsigned long cntx, old_active_cntx;

	cntx = atomic_long_read(&mm->context.id);

	/*
	 * If our active_context is non-zero and the context matches the
	 * current_version, then we update the active_context entry with a
	 * relaxed cmpxchg.
	 *
	 * Following is how we handle racing with a concurrent rollover:
	 *
	 * - We get a zero back from the cmpxchg and end up waiting on the
	 *   lock. Taking the lock synchronises with the rollover and so
	 *   we are forced to see the updated verion.
	 *
	 * - We get a valid context back from the cmpxchg then we continue
	 *   using old ASID because __flush_context() would have marked ASID
	 *   of active_context as used and next context switch we will
	 *   allocate new context.
	 */
	old_active_cntx = atomic_long_read(&per_cpu(active_context, cpu));
	if (old_active_cntx &&
	    ((cntx & ~asid_mask) == atomic_long_read(&current_version)) &&
	    atomic_long_cmpxchg_relaxed(&per_cpu(active_context, cpu),
					old_active_cntx, cntx))
		goto switch_mm_fast;

	raw_spin_lock_irqsave(&context_lock, flags);

	/* Check that our ASID belongs to the current_version. */
	cntx = atomic_long_read(&mm->context.id);
	if ((cntx & ~asid_mask) != atomic_long_read(&current_version)) {
		cntx = __new_context(mm);
		atomic_long_set(&mm->context.id, cntx);
	}

	if (cpumask_test_and_clear_cpu(cpu, &context_tlb_flush_pending))
		need_flush_tlb = true;

	atomic_long_set(&per_cpu(active_context, cpu), cntx);

	raw_spin_unlock_irqrestore(&context_lock, flags);

switch_mm_fast:
	csr_write(CSR_SATP, virt_to_pfn(mm->pgd) |
		  ((cntx & asid_mask) << SATP_ASID_SHIFT) |
		  satp_mode);

	if (need_flush_tlb)
		local_flush_tlb_all();
}

static void set_mm_noasid(struct mm_struct *mm)
{
	/* Switch the page table and blindly nuke entire local TLB */
	csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | satp_mode);
	local_flush_tlb_all();
}

static inline void set_mm(struct mm_struct *prev,
			  struct mm_struct *next, unsigned int cpu)
{
	/*
	 * The mm_cpumask indicates which harts' TLBs contain the virtual
	 * address mapping of the mm. Compared to noasid, using asid
	 * can't guarantee that stale TLB entries are invalidated because
	 * the asid mechanism wouldn't flush TLB for every switch_mm for
	 * performance. So when using asid, keep all CPUs footmarks in
	 * cpumask() until mm reset.
	 */
	cpumask_set_cpu(cpu, mm_cpumask(next));
	if (static_branch_unlikely(&use_asid_allocator)) {
		set_mm_asid(next, cpu);
	} else {
		cpumask_clear_cpu(cpu, mm_cpumask(prev));
		set_mm_noasid(next);
	}
}

static int __init asids_init(void)
{
	unsigned long old;

	/* Figure-out number of ASID bits in HW */
	old = csr_read(CSR_SATP);
	asid_bits = old | (SATP_ASID_MASK << SATP_ASID_SHIFT);
	csr_write(CSR_SATP, asid_bits);
	asid_bits = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT)  & SATP_ASID_MASK;
	asid_bits = fls_long(asid_bits);
	csr_write(CSR_SATP, old);

	/*
	 * In the process of determining number of ASID bits (above)
	 * we polluted the TLB of current HART so let's do TLB flushed
	 * to remove unwanted TLB enteries.
	 */
	local_flush_tlb_all();

	/* Pre-compute ASID details */
	if (asid_bits) {
		num_asids = 1 << asid_bits;
		asid_mask = num_asids - 1;
	}

	/*
	 * Use ASID allocator only if number of HW ASIDs are
	 * at-least twice more than CPUs
	 */
	if (num_asids > (2 * num_possible_cpus())) {
		atomic_long_set(&current_version, num_asids);

		context_asid_map = bitmap_zalloc(num_asids, GFP_KERNEL);
		if (!context_asid_map)
			panic("Failed to allocate bitmap for %lu ASIDs\n",
			      num_asids);

		__set_bit(0, context_asid_map);

		static_branch_enable(&use_asid_allocator);

		pr_info("ASID allocator using %lu bits (%lu entries)\n",
			asid_bits, num_asids);
	} else {
		pr_info("ASID allocator disabled (%lu bits)\n", asid_bits);
	}

	return 0;
}
early_initcall(asids_init);
#else
static inline void set_mm(struct mm_struct *prev,
			  struct mm_struct *next, unsigned int cpu)
{
	/* Nothing to do here when there is no MMU */
}
#endif

/*
 * When necessary, performs a deferred icache flush for the given MM context,
 * on the local CPU.  RISC-V has no direct mechanism for instruction cache
 * shoot downs, so instead we send an IPI that informs the remote harts they
 * need to flush their local instruction caches.  To avoid pathologically slow
 * behavior in a common case (a bunch of single-hart processes on a many-hart
 * machine, ie 'make -j') we avoid the IPIs for harts that are not currently
 * executing a MM context and instead schedule a deferred local instruction
 * cache flush to be performed before execution resumes on each hart.  This
 * actually performs that local instruction cache flush, which implicitly only
 * refers to the current hart.
 *
 * The "cpu" argument must be the current local CPU number.
 */
static inline void flush_icache_deferred(struct mm_struct *mm, unsigned int cpu)
{
#ifdef CONFIG_SMP
	cpumask_t *mask = &mm->context.icache_stale_mask;

	if (cpumask_test_cpu(cpu, mask)) {
		cpumask_clear_cpu(cpu, mask);
		/*
		 * Ensure the remote hart's writes are visible to this hart.
		 * This pairs with a barrier in flush_icache_mm.
		 */
		smp_mb();
		local_flush_icache_all();
	}

#endif
}

void switch_mm(struct mm_struct *prev, struct mm_struct *next,
	struct task_struct *task)
{
	unsigned int cpu;

	if (unlikely(prev == next))
		return;

	/*
	 * Mark the current MM context as inactive, and the next as
	 * active.  This is at least used by the icache flushing
	 * routines in order to determine who should be flushed.
	 */
	cpu = smp_processor_id();

	set_mm(prev, next, cpu);

	flush_icache_deferred(next, cpu);
}