summaryrefslogtreecommitdiff
path: root/arch/arm/crypto/blake2s-core.S
blob: 86345751bbf3a3d8a7e3af72684985833cf6734b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * BLAKE2s digest algorithm, ARM scalar implementation
 *
 * Copyright 2020 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	// Registers used to hold message words temporarily.  There aren't
	// enough ARM registers to hold the whole message block, so we have to
	// load the words on-demand.
	M_0		.req	r12
	M_1		.req	r14

// The BLAKE2s initialization vector
.Lblake2s_IV:
	.word	0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A
	.word	0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19

.macro __ldrd		a, b, src, offset
#if __LINUX_ARM_ARCH__ >= 6
	ldrd		\a, \b, [\src, #\offset]
#else
	ldr		\a, [\src, #\offset]
	ldr		\b, [\src, #\offset + 4]
#endif
.endm

.macro __strd		a, b, dst, offset
#if __LINUX_ARM_ARCH__ >= 6
	strd		\a, \b, [\dst, #\offset]
#else
	str		\a, [\dst, #\offset]
	str		\b, [\dst, #\offset + 4]
#endif
.endm

.macro _le32_bswap	a, tmp
#ifdef __ARMEB__
	rev_l		\a, \tmp
#endif
.endm

.macro _le32_bswap_8x	a, b, c, d, e, f, g, h,  tmp
	_le32_bswap	\a, \tmp
	_le32_bswap	\b, \tmp
	_le32_bswap	\c, \tmp
	_le32_bswap	\d, \tmp
	_le32_bswap	\e, \tmp
	_le32_bswap	\f, \tmp
	_le32_bswap	\g, \tmp
	_le32_bswap	\h, \tmp
.endm

// Execute a quarter-round of BLAKE2s by mixing two columns or two diagonals.
// (a0, b0, c0, d0) and (a1, b1, c1, d1) give the registers containing the two
// columns/diagonals.  s0-s1 are the word offsets to the message words the first
// column/diagonal needs, and likewise s2-s3 for the second column/diagonal.
// M_0 and M_1 are free to use, and the message block can be found at sp + 32.
//
// Note that to save instructions, the rotations don't happen when the
// pseudocode says they should, but rather they are delayed until the values are
// used.  See the comment above _blake2s_round().
.macro _blake2s_quarterround  a0, b0, c0, d0,  a1, b1, c1, d1,  s0, s1, s2, s3

	ldr		M_0, [sp, #32 + 4 * \s0]
	ldr		M_1, [sp, #32 + 4 * \s2]

	// a += b + m[blake2s_sigma[r][2*i + 0]];
	add		\a0, \a0, \b0, ror #brot
	add		\a1, \a1, \b1, ror #brot
	add		\a0, \a0, M_0
	add		\a1, \a1, M_1

	// d = ror32(d ^ a, 16);
	eor		\d0, \a0, \d0, ror #drot
	eor		\d1, \a1, \d1, ror #drot

	// c += d;
	add		\c0, \c0, \d0, ror #16
	add		\c1, \c1, \d1, ror #16

	// b = ror32(b ^ c, 12);
	eor		\b0, \c0, \b0, ror #brot
	eor		\b1, \c1, \b1, ror #brot

	ldr		M_0, [sp, #32 + 4 * \s1]
	ldr		M_1, [sp, #32 + 4 * \s3]

	// a += b + m[blake2s_sigma[r][2*i + 1]];
	add		\a0, \a0, \b0, ror #12
	add		\a1, \a1, \b1, ror #12
	add		\a0, \a0, M_0
	add		\a1, \a1, M_1

	// d = ror32(d ^ a, 8);
	eor		\d0, \a0, \d0, ror#16
	eor		\d1, \a1, \d1, ror#16

	// c += d;
	add		\c0, \c0, \d0, ror#8
	add		\c1, \c1, \d1, ror#8

	// b = ror32(b ^ c, 7);
	eor		\b0, \c0, \b0, ror#12
	eor		\b1, \c1, \b1, ror#12
.endm

// Execute one round of BLAKE2s by updating the state matrix v[0..15].  v[0..9]
// are in r0..r9.  The stack pointer points to 8 bytes of scratch space for
// spilling v[8..9], then to v[9..15], then to the message block.  r10-r12 and
// r14 are free to use.  The macro arguments s0-s15 give the order in which the
// message words are used in this round.
//
// All rotates are performed using the implicit rotate operand accepted by the
// 'add' and 'eor' instructions.  This is faster than using explicit rotate
// instructions.  To make this work, we allow the values in the second and last
// rows of the BLAKE2s state matrix (rows 'b' and 'd') to temporarily have the
// wrong rotation amount.  The rotation amount is then fixed up just in time
// when the values are used.  'brot' is the number of bits the values in row 'b'
// need to be rotated right to arrive at the correct values, and 'drot'
// similarly for row 'd'.  (brot, drot) start out as (0, 0) but we make it such
// that they end up as (7, 8) after every round.
.macro	_blake2s_round	s0, s1, s2, s3, s4, s5, s6, s7, \
			s8, s9, s10, s11, s12, s13, s14, s15

	// Mix first two columns:
	// (v[0], v[4], v[8], v[12]) and (v[1], v[5], v[9], v[13]).
	__ldrd		r10, r11, sp, 16	// load v[12] and v[13]
	_blake2s_quarterround	r0, r4, r8, r10,  r1, r5, r9, r11, \
				\s0, \s1, \s2, \s3
	__strd		r8, r9, sp, 0
	__strd		r10, r11, sp, 16

	// Mix second two columns:
	// (v[2], v[6], v[10], v[14]) and (v[3], v[7], v[11], v[15]).
	__ldrd		r8, r9, sp, 8		// load v[10] and v[11]
	__ldrd		r10, r11, sp, 24	// load v[14] and v[15]
	_blake2s_quarterround	r2, r6, r8, r10,  r3, r7, r9, r11, \
				\s4, \s5, \s6, \s7
	str		r10, [sp, #24]		// store v[14]
	// v[10], v[11], and v[15] are used below, so no need to store them yet.

	.set brot, 7
	.set drot, 8

	// Mix first two diagonals:
	// (v[0], v[5], v[10], v[15]) and (v[1], v[6], v[11], v[12]).
	ldr		r10, [sp, #16]		// load v[12]
	_blake2s_quarterround	r0, r5, r8, r11,  r1, r6, r9, r10, \
				\s8, \s9, \s10, \s11
	__strd		r8, r9, sp, 8
	str		r11, [sp, #28]
	str		r10, [sp, #16]

	// Mix second two diagonals:
	// (v[2], v[7], v[8], v[13]) and (v[3], v[4], v[9], v[14]).
	__ldrd		r8, r9, sp, 0		// load v[8] and v[9]
	__ldrd		r10, r11, sp, 20	// load v[13] and v[14]
	_blake2s_quarterround	r2, r7, r8, r10,  r3, r4, r9, r11, \
				\s12, \s13, \s14, \s15
	__strd		r10, r11, sp, 20
.endm

//
// void blake2s_compress_arch(struct blake2s_state *state,
//			      const u8 *block, size_t nblocks, u32 inc);
//
// Only the first three fields of struct blake2s_state are used:
//	u32 h[8];	(inout)
//	u32 t[2];	(inout)
//	u32 f[2];	(in)
//
	.align		5
ENTRY(blake2s_compress_arch)
	push		{r0-r2,r4-r11,lr}	// keep this an even number

.Lnext_block:
	// r0 is 'state'
	// r1 is 'block'
	// r3 is 'inc'

	// Load and increment the counter t[0..1].
	__ldrd		r10, r11, r0, 32
	adds		r10, r10, r3
	adc		r11, r11, #0
	__strd		r10, r11, r0, 32

	// _blake2s_round is very short on registers, so copy the message block
	// to the stack to save a register during the rounds.  This also has the
	// advantage that misalignment only needs to be dealt with in one place.
	sub		sp, sp, #64
	mov		r12, sp
	tst		r1, #3
	bne		.Lcopy_block_misaligned
	ldmia		r1!, {r2-r9}
	_le32_bswap_8x	r2, r3, r4, r5, r6, r7, r8, r9,  r14
	stmia		r12!, {r2-r9}
	ldmia		r1!, {r2-r9}
	_le32_bswap_8x	r2, r3, r4, r5, r6, r7, r8, r9,  r14
	stmia		r12, {r2-r9}
.Lcopy_block_done:
	str		r1, [sp, #68]		// Update message pointer

	// Calculate v[8..15].  Push v[9..15] onto the stack, and leave space
	// for spilling v[8..9].  Leave v[8..9] in r8-r9.
	mov		r14, r0			// r14 = state
	adr		r12, .Lblake2s_IV
	ldmia		r12!, {r8-r9}		// load IV[0..1]
	__ldrd		r0, r1, r14, 40		// load f[0..1]
	ldm		r12, {r2-r7}		// load IV[3..7]
	eor		r4, r4, r10		// v[12] = IV[4] ^ t[0]
	eor		r5, r5, r11		// v[13] = IV[5] ^ t[1]
	eor		r6, r6, r0		// v[14] = IV[6] ^ f[0]
	eor		r7, r7, r1		// v[15] = IV[7] ^ f[1]
	push		{r2-r7}			// push v[9..15]
	sub		sp, sp, #8		// leave space for v[8..9]

	// Load h[0..7] == v[0..7].
	ldm		r14, {r0-r7}

	// Execute the rounds.  Each round is provided the order in which it
	// needs to use the message words.
	.set brot, 0
	.set drot, 0
	_blake2s_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	_blake2s_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3
	_blake2s_round	11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4
	_blake2s_round	7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8
	_blake2s_round	9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13
	_blake2s_round	2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9
	_blake2s_round	12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11
	_blake2s_round	13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10
	_blake2s_round	6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5
	_blake2s_round	10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0

	// Fold the final state matrix into the hash chaining value:
	//
	//	for (i = 0; i < 8; i++)
	//		h[i] ^= v[i] ^ v[i + 8];
	//
	ldr		r14, [sp, #96]		// r14 = &h[0]
	add		sp, sp, #8		// v[8..9] are already loaded.
	pop		{r10-r11}		// load v[10..11]
	eor		r0, r0, r8
	eor		r1, r1, r9
	eor		r2, r2, r10
	eor		r3, r3, r11
	ldm		r14, {r8-r11}		// load h[0..3]
	eor		r0, r0, r8
	eor		r1, r1, r9
	eor		r2, r2, r10
	eor		r3, r3, r11
	stmia		r14!, {r0-r3}		// store new h[0..3]
	ldm		r14, {r0-r3}		// load old h[4..7]
	pop		{r8-r11}		// load v[12..15]
	eor		r0, r0, r4, ror #brot
	eor		r1, r1, r5, ror #brot
	eor		r2, r2, r6, ror #brot
	eor		r3, r3, r7, ror #brot
	eor		r0, r0, r8, ror #drot
	eor		r1, r1, r9, ror #drot
	eor		r2, r2, r10, ror #drot
	eor		r3, r3, r11, ror #drot
	  add		sp, sp, #64		// skip copy of message block
	stm		r14, {r0-r3}		// store new h[4..7]

	// Advance to the next block, if there is one.  Note that if there are
	// multiple blocks, then 'inc' (the counter increment amount) must be
	// 64.  So we can simply set it to 64 without re-loading it.
	ldm		sp, {r0, r1, r2}	// load (state, block, nblocks)
	mov		r3, #64			// set 'inc'
	subs		r2, r2, #1		// nblocks--
	str		r2, [sp, #8]
	bne		.Lnext_block		// nblocks != 0?

	pop		{r0-r2,r4-r11,pc}

	// The next message block (pointed to by r1) isn't 4-byte aligned, so it
	// can't be loaded using ldmia.  Copy it to the stack buffer (pointed to
	// by r12) using an alternative method.  r2-r9 are free to use.
.Lcopy_block_misaligned:
	mov		r2, #64
1:
#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
	ldr		r3, [r1], #4
	_le32_bswap	r3, r4
#else
	ldrb		r3, [r1, #0]
	ldrb		r4, [r1, #1]
	ldrb		r5, [r1, #2]
	ldrb		r6, [r1, #3]
	add		r1, r1, #4
	orr		r3, r3, r4, lsl #8
	orr		r3, r3, r5, lsl #16
	orr		r3, r3, r6, lsl #24
#endif
	subs		r2, r2, #4
	str		r3, [r12], #4
	bne		1b
	b		.Lcopy_block_done
ENDPROC(blake2s_compress_arch)