diff options
Diffstat (limited to 'arch/powerpc/lib/crypto/poly1305-p10le_64.S')
| -rw-r--r-- | arch/powerpc/lib/crypto/poly1305-p10le_64.S | 1075 |
1 files changed, 0 insertions, 1075 deletions
diff --git a/arch/powerpc/lib/crypto/poly1305-p10le_64.S b/arch/powerpc/lib/crypto/poly1305-p10le_64.S deleted file mode 100644 index a3c1987f1ecd..000000000000 --- a/arch/powerpc/lib/crypto/poly1305-p10le_64.S +++ /dev/null @@ -1,1075 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -# -# Accelerated poly1305 implementation for ppc64le. -# -# Copyright 2023- IBM Corp. All rights reserved -# -#=================================================================================== -# Written by Danny Tsen <dtsen@us.ibm.com> -# -# Poly1305 - this version mainly using vector/VSX/Scalar -# - 26 bits limbs -# - Handle multiple 64 byte blcok. -# -# Block size 16 bytes -# key = (r, s) -# clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF -# p = 2^130 - 5 -# a += m -# a = (r + a) % p -# a += s -# -# Improve performance by breaking down polynominal to the sum of products with -# h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r -# -# 07/22/21 - this revison based on the above sum of products. Setup r^4, r^3, r^2, r and s3, s2, s1, s0 -# to 9 vectors for multiplications. -# -# setup r^4, r^3, r^2, r vectors -# vs [r^1, r^3, r^2, r^4] -# vs0 = [r0,.....] -# vs1 = [r1,.....] -# vs2 = [r2,.....] -# vs3 = [r3,.....] -# vs4 = [r4,.....] -# vs5 = [r1*5,...] -# vs6 = [r2*5,...] -# vs7 = [r2*5,...] -# vs8 = [r4*5,...] -# -# Each word in a vector consists a member of a "r/s" in [a * r/s]. -# -# r0, r4*5, r3*5, r2*5, r1*5; -# r1, r0, r4*5, r3*5, r2*5; -# r2, r1, r0, r4*5, r3*5; -# r3, r2, r1, r0, r4*5; -# r4, r3, r2, r1, r0 ; -# -# -# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) -# k = 32 bytes key -# r3 = k (r, s) -# r4 = mlen -# r5 = m -# -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> -#include <asm/asm-compat.h> -#include <linux/linkage.h> - -.machine "any" - -.text - -.macro SAVE_GPR GPR OFFSET FRAME - std \GPR,\OFFSET(\FRAME) -.endm - -.macro SAVE_VRS VRS OFFSET FRAME - li 16, \OFFSET - stvx \VRS, 16, \FRAME -.endm - -.macro SAVE_VSX VSX OFFSET FRAME - li 16, \OFFSET - stxvx \VSX, 16, \FRAME -.endm - -.macro RESTORE_GPR GPR OFFSET FRAME - ld \GPR,\OFFSET(\FRAME) -.endm - -.macro RESTORE_VRS VRS OFFSET FRAME - li 16, \OFFSET - lvx \VRS, 16, \FRAME -.endm - -.macro RESTORE_VSX VSX OFFSET FRAME - li 16, \OFFSET - lxvx \VSX, 16, \FRAME -.endm - -.macro SAVE_REGS - mflr 0 - std 0, 16(1) - stdu 1,-752(1) - - SAVE_GPR 14, 112, 1 - SAVE_GPR 15, 120, 1 - SAVE_GPR 16, 128, 1 - SAVE_GPR 17, 136, 1 - SAVE_GPR 18, 144, 1 - SAVE_GPR 19, 152, 1 - SAVE_GPR 20, 160, 1 - SAVE_GPR 21, 168, 1 - SAVE_GPR 22, 176, 1 - SAVE_GPR 23, 184, 1 - SAVE_GPR 24, 192, 1 - SAVE_GPR 25, 200, 1 - SAVE_GPR 26, 208, 1 - SAVE_GPR 27, 216, 1 - SAVE_GPR 28, 224, 1 - SAVE_GPR 29, 232, 1 - SAVE_GPR 30, 240, 1 - SAVE_GPR 31, 248, 1 - - addi 9, 1, 256 - SAVE_VRS 20, 0, 9 - SAVE_VRS 21, 16, 9 - SAVE_VRS 22, 32, 9 - SAVE_VRS 23, 48, 9 - SAVE_VRS 24, 64, 9 - SAVE_VRS 25, 80, 9 - SAVE_VRS 26, 96, 9 - SAVE_VRS 27, 112, 9 - SAVE_VRS 28, 128, 9 - SAVE_VRS 29, 144, 9 - SAVE_VRS 30, 160, 9 - SAVE_VRS 31, 176, 9 - - SAVE_VSX 14, 192, 9 - SAVE_VSX 15, 208, 9 - SAVE_VSX 16, 224, 9 - SAVE_VSX 17, 240, 9 - SAVE_VSX 18, 256, 9 - SAVE_VSX 19, 272, 9 - SAVE_VSX 20, 288, 9 - SAVE_VSX 21, 304, 9 - SAVE_VSX 22, 320, 9 - SAVE_VSX 23, 336, 9 - SAVE_VSX 24, 352, 9 - SAVE_VSX 25, 368, 9 - SAVE_VSX 26, 384, 9 - SAVE_VSX 27, 400, 9 - SAVE_VSX 28, 416, 9 - SAVE_VSX 29, 432, 9 - SAVE_VSX 30, 448, 9 - SAVE_VSX 31, 464, 9 -.endm # SAVE_REGS - -.macro RESTORE_REGS - addi 9, 1, 256 - RESTORE_VRS 20, 0, 9 - RESTORE_VRS 21, 16, 9 - RESTORE_VRS 22, 32, 9 - RESTORE_VRS 23, 48, 9 - RESTORE_VRS 24, 64, 9 - RESTORE_VRS 25, 80, 9 - RESTORE_VRS 26, 96, 9 - RESTORE_VRS 27, 112, 9 - RESTORE_VRS 28, 128, 9 - RESTORE_VRS 29, 144, 9 - RESTORE_VRS 30, 160, 9 - RESTORE_VRS 31, 176, 9 - - RESTORE_VSX 14, 192, 9 - RESTORE_VSX 15, 208, 9 - RESTORE_VSX 16, 224, 9 - RESTORE_VSX 17, 240, 9 - RESTORE_VSX 18, 256, 9 - RESTORE_VSX 19, 272, 9 - RESTORE_VSX 20, 288, 9 - RESTORE_VSX 21, 304, 9 - RESTORE_VSX 22, 320, 9 - RESTORE_VSX 23, 336, 9 - RESTORE_VSX 24, 352, 9 - RESTORE_VSX 25, 368, 9 - RESTORE_VSX 26, 384, 9 - RESTORE_VSX 27, 400, 9 - RESTORE_VSX 28, 416, 9 - RESTORE_VSX 29, 432, 9 - RESTORE_VSX 30, 448, 9 - RESTORE_VSX 31, 464, 9 - - RESTORE_GPR 14, 112, 1 - RESTORE_GPR 15, 120, 1 - RESTORE_GPR 16, 128, 1 - RESTORE_GPR 17, 136, 1 - RESTORE_GPR 18, 144, 1 - RESTORE_GPR 19, 152, 1 - RESTORE_GPR 20, 160, 1 - RESTORE_GPR 21, 168, 1 - RESTORE_GPR 22, 176, 1 - RESTORE_GPR 23, 184, 1 - RESTORE_GPR 24, 192, 1 - RESTORE_GPR 25, 200, 1 - RESTORE_GPR 26, 208, 1 - RESTORE_GPR 27, 216, 1 - RESTORE_GPR 28, 224, 1 - RESTORE_GPR 29, 232, 1 - RESTORE_GPR 30, 240, 1 - RESTORE_GPR 31, 248, 1 - - addi 1, 1, 752 - ld 0, 16(1) - mtlr 0 -.endm # RESTORE_REGS - -# -# p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5; -# p[1] = a0*r1 + a1*r0 + a2*r4*5 + a3*r3*5 + a4*r2*5; -# p[2] = a0*r2 + a1*r1 + a2*r0 + a3*r4*5 + a4*r3*5; -# p[3] = a0*r3 + a1*r2 + a2*r1 + a3*r0 + a4*r4*5; -# p[4] = a0*r4 + a1*r3 + a2*r2 + a3*r1 + a4*r0 ; -# -# [r^2, r^3, r^1, r^4] -# [m3, m2, m4, m1] -# -# multiply odd and even words -.macro mul_odd - vmulouw 14, 4, 26 - vmulouw 10, 5, 3 - vmulouw 11, 6, 2 - vmulouw 12, 7, 1 - vmulouw 13, 8, 0 - vmulouw 15, 4, 27 - vaddudm 14, 14, 10 - vaddudm 14, 14, 11 - vmulouw 10, 5, 26 - vmulouw 11, 6, 3 - vaddudm 14, 14, 12 - vaddudm 14, 14, 13 # x0 - vaddudm 15, 15, 10 - vaddudm 15, 15, 11 - vmulouw 12, 7, 2 - vmulouw 13, 8, 1 - vaddudm 15, 15, 12 - vaddudm 15, 15, 13 # x1 - vmulouw 16, 4, 28 - vmulouw 10, 5, 27 - vmulouw 11, 6, 26 - vaddudm 16, 16, 10 - vaddudm 16, 16, 11 - vmulouw 12, 7, 3 - vmulouw 13, 8, 2 - vaddudm 16, 16, 12 - vaddudm 16, 16, 13 # x2 - vmulouw 17, 4, 29 - vmulouw 10, 5, 28 - vmulouw 11, 6, 27 - vaddudm 17, 17, 10 - vaddudm 17, 17, 11 - vmulouw 12, 7, 26 - vmulouw 13, 8, 3 - vaddudm 17, 17, 12 - vaddudm 17, 17, 13 # x3 - vmulouw 18, 4, 30 - vmulouw 10, 5, 29 - vmulouw 11, 6, 28 - vaddudm 18, 18, 10 - vaddudm 18, 18, 11 - vmulouw 12, 7, 27 - vmulouw 13, 8, 26 - vaddudm 18, 18, 12 - vaddudm 18, 18, 13 # x4 -.endm - -.macro mul_even - vmuleuw 9, 4, 26 - vmuleuw 10, 5, 3 - vmuleuw 11, 6, 2 - vmuleuw 12, 7, 1 - vmuleuw 13, 8, 0 - vaddudm 14, 14, 9 - vaddudm 14, 14, 10 - vaddudm 14, 14, 11 - vaddudm 14, 14, 12 - vaddudm 14, 14, 13 # x0 - - vmuleuw 9, 4, 27 - vmuleuw 10, 5, 26 - vmuleuw 11, 6, 3 - vmuleuw 12, 7, 2 - vmuleuw 13, 8, 1 - vaddudm 15, 15, 9 - vaddudm 15, 15, 10 - vaddudm 15, 15, 11 - vaddudm 15, 15, 12 - vaddudm 15, 15, 13 # x1 - - vmuleuw 9, 4, 28 - vmuleuw 10, 5, 27 - vmuleuw 11, 6, 26 - vmuleuw 12, 7, 3 - vmuleuw 13, 8, 2 - vaddudm 16, 16, 9 - vaddudm 16, 16, 10 - vaddudm 16, 16, 11 - vaddudm 16, 16, 12 - vaddudm 16, 16, 13 # x2 - - vmuleuw 9, 4, 29 - vmuleuw 10, 5, 28 - vmuleuw 11, 6, 27 - vmuleuw 12, 7, 26 - vmuleuw 13, 8, 3 - vaddudm 17, 17, 9 - vaddudm 17, 17, 10 - vaddudm 17, 17, 11 - vaddudm 17, 17, 12 - vaddudm 17, 17, 13 # x3 - - vmuleuw 9, 4, 30 - vmuleuw 10, 5, 29 - vmuleuw 11, 6, 28 - vmuleuw 12, 7, 27 - vmuleuw 13, 8, 26 - vaddudm 18, 18, 9 - vaddudm 18, 18, 10 - vaddudm 18, 18, 11 - vaddudm 18, 18, 12 - vaddudm 18, 18, 13 # x4 -.endm - -# -# poly1305_setup_r -# -# setup r^4, r^3, r^2, r vectors -# [r, r^3, r^2, r^4] -# vs0 = [r0,...] -# vs1 = [r1,...] -# vs2 = [r2,...] -# vs3 = [r3,...] -# vs4 = [r4,...] -# vs5 = [r4*5,...] -# vs6 = [r3*5,...] -# vs7 = [r2*5,...] -# vs8 = [r1*5,...] -# -# r0, r4*5, r3*5, r2*5, r1*5; -# r1, r0, r4*5, r3*5, r2*5; -# r2, r1, r0, r4*5, r3*5; -# r3, r2, r1, r0, r4*5; -# r4, r3, r2, r1, r0 ; -# -.macro poly1305_setup_r - - # save r - xxlor 26, 58, 58 - xxlor 27, 59, 59 - xxlor 28, 60, 60 - xxlor 29, 61, 61 - xxlor 30, 62, 62 - - xxlxor 31, 31, 31 - -# [r, r^3, r^2, r^4] - # compute r^2 - vmr 4, 26 - vmr 5, 27 - vmr 6, 28 - vmr 7, 29 - vmr 8, 30 - bl do_mul # r^2 r^1 - xxpermdi 58, 58, 36, 0x3 # r0 - xxpermdi 59, 59, 37, 0x3 # r1 - xxpermdi 60, 60, 38, 0x3 # r2 - xxpermdi 61, 61, 39, 0x3 # r3 - xxpermdi 62, 62, 40, 0x3 # r4 - xxpermdi 36, 36, 36, 0x3 - xxpermdi 37, 37, 37, 0x3 - xxpermdi 38, 38, 38, 0x3 - xxpermdi 39, 39, 39, 0x3 - xxpermdi 40, 40, 40, 0x3 - vspltisb 13, 2 - vsld 9, 27, 13 - vsld 10, 28, 13 - vsld 11, 29, 13 - vsld 12, 30, 13 - vaddudm 0, 9, 27 - vaddudm 1, 10, 28 - vaddudm 2, 11, 29 - vaddudm 3, 12, 30 - - bl do_mul # r^4 r^3 - vmrgow 26, 26, 4 - vmrgow 27, 27, 5 - vmrgow 28, 28, 6 - vmrgow 29, 29, 7 - vmrgow 30, 30, 8 - vspltisb 13, 2 - vsld 9, 27, 13 - vsld 10, 28, 13 - vsld 11, 29, 13 - vsld 12, 30, 13 - vaddudm 0, 9, 27 - vaddudm 1, 10, 28 - vaddudm 2, 11, 29 - vaddudm 3, 12, 30 - - # r^2 r^4 - xxlor 0, 58, 58 - xxlor 1, 59, 59 - xxlor 2, 60, 60 - xxlor 3, 61, 61 - xxlor 4, 62, 62 - xxlor 5, 32, 32 - xxlor 6, 33, 33 - xxlor 7, 34, 34 - xxlor 8, 35, 35 - - vspltw 9, 26, 3 - vspltw 10, 26, 2 - vmrgow 26, 10, 9 - vspltw 9, 27, 3 - vspltw 10, 27, 2 - vmrgow 27, 10, 9 - vspltw 9, 28, 3 - vspltw 10, 28, 2 - vmrgow 28, 10, 9 - vspltw 9, 29, 3 - vspltw 10, 29, 2 - vmrgow 29, 10, 9 - vspltw 9, 30, 3 - vspltw 10, 30, 2 - vmrgow 30, 10, 9 - - vsld 9, 27, 13 - vsld 10, 28, 13 - vsld 11, 29, 13 - vsld 12, 30, 13 - vaddudm 0, 9, 27 - vaddudm 1, 10, 28 - vaddudm 2, 11, 29 - vaddudm 3, 12, 30 -.endm - -SYM_FUNC_START_LOCAL(do_mul) - mul_odd - - # do reduction ( h %= p ) - # carry reduction - vspltisb 9, 2 - vsrd 10, 14, 31 - vsrd 11, 17, 31 - vand 7, 17, 25 - vand 4, 14, 25 - vaddudm 18, 18, 11 - vsrd 12, 18, 31 - vaddudm 15, 15, 10 - - vsrd 11, 15, 31 - vand 8, 18, 25 - vand 5, 15, 25 - vaddudm 4, 4, 12 - vsld 10, 12, 9 - vaddudm 6, 16, 11 - - vsrd 13, 6, 31 - vand 6, 6, 25 - vaddudm 4, 4, 10 - vsrd 10, 4, 31 - vaddudm 7, 7, 13 - - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 5, 5, 10 - vaddudm 8, 8, 11 - blr -SYM_FUNC_END(do_mul) - -# -# init key -# -.macro do_poly1305_init - addis 10, 2, rmask@toc@ha - addi 10, 10, rmask@toc@l - - ld 11, 0(10) - ld 12, 8(10) - - li 14, 16 - li 15, 32 - addis 10, 2, cnum@toc@ha - addi 10, 10, cnum@toc@l - lvx 25, 0, 10 # v25 - mask - lvx 31, 14, 10 # v31 = 1a - lvx 19, 15, 10 # v19 = 1 << 24 - lxv 24, 48(10) # vs24 - lxv 25, 64(10) # vs25 - - # initialize - # load key from r3 to vectors - ld 9, 24(3) - ld 10, 32(3) - and. 9, 9, 11 - and. 10, 10, 12 - - # break 26 bits - extrdi 14, 9, 26, 38 - extrdi 15, 9, 26, 12 - extrdi 16, 9, 12, 0 - mtvsrdd 58, 0, 14 - insrdi 16, 10, 14, 38 - mtvsrdd 59, 0, 15 - extrdi 17, 10, 26, 24 - mtvsrdd 60, 0, 16 - extrdi 18, 10, 24, 0 - mtvsrdd 61, 0, 17 - mtvsrdd 62, 0, 18 - - # r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5 - li 9, 5 - mtvsrdd 36, 0, 9 - vmulouw 0, 27, 4 # v0 = rr0 - vmulouw 1, 28, 4 # v1 = rr1 - vmulouw 2, 29, 4 # v2 = rr2 - vmulouw 3, 30, 4 # v3 = rr3 -.endm - -# -# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) -# k = 32 bytes key -# r3 = k (r, s) -# r4 = mlen -# r5 = m -# -SYM_FUNC_START(poly1305_p10le_4blocks) -.align 5 - cmpdi 5, 64 - blt Out_no_poly1305 - - SAVE_REGS - - do_poly1305_init - - li 21, 0 # counter to message - - poly1305_setup_r - - # load previous H state - # break/convert r6 to 26 bits - ld 9, 0(3) - ld 10, 8(3) - ld 19, 16(3) - sldi 19, 19, 24 - mtvsrdd 41, 0, 19 - extrdi 14, 9, 26, 38 - extrdi 15, 9, 26, 12 - extrdi 16, 9, 12, 0 - mtvsrdd 36, 0, 14 - insrdi 16, 10, 14, 38 - mtvsrdd 37, 0, 15 - extrdi 17, 10, 26, 24 - mtvsrdd 38, 0, 16 - extrdi 18, 10, 24, 0 - mtvsrdd 39, 0, 17 - mtvsrdd 40, 0, 18 - vor 8, 8, 9 - - # input m1 m2 - add 20, 4, 21 - xxlor 49, 24, 24 - xxlor 50, 25, 25 - lxvw4x 43, 0, 20 - addi 17, 20, 16 - lxvw4x 44, 0, 17 - vperm 14, 11, 12, 17 - vperm 15, 11, 12, 18 - vand 9, 14, 25 # a0 - vsrd 10, 14, 31 # >> 26 - vsrd 11, 10, 31 # 12 bits left - vand 10, 10, 25 # a1 - vspltisb 13, 12 - vand 16, 15, 25 - vsld 12, 16, 13 - vor 11, 11, 12 - vand 11, 11, 25 # a2 - vspltisb 13, 14 - vsrd 12, 15, 13 # >> 14 - vsrd 13, 12, 31 # >> 26, a4 - vand 12, 12, 25 # a3 - - vaddudm 20, 4, 9 - vaddudm 21, 5, 10 - vaddudm 22, 6, 11 - vaddudm 23, 7, 12 - vaddudm 24, 8, 13 - - # m3 m4 - addi 17, 17, 16 - lxvw4x 43, 0, 17 - addi 17, 17, 16 - lxvw4x 44, 0, 17 - vperm 14, 11, 12, 17 - vperm 15, 11, 12, 18 - vand 9, 14, 25 # a0 - vsrd 10, 14, 31 # >> 26 - vsrd 11, 10, 31 # 12 bits left - vand 10, 10, 25 # a1 - vspltisb 13, 12 - vand 16, 15, 25 - vsld 12, 16, 13 - vspltisb 13, 14 - vor 11, 11, 12 - vand 11, 11, 25 # a2 - vsrd 12, 15, 13 # >> 14 - vsrd 13, 12, 31 # >> 26, a4 - vand 12, 12, 25 # a3 - - # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] - vmrgow 4, 9, 20 - vmrgow 5, 10, 21 - vmrgow 6, 11, 22 - vmrgow 7, 12, 23 - vmrgow 8, 13, 24 - vaddudm 8, 8, 19 - - addi 5, 5, -64 # len -= 64 - addi 21, 21, 64 # offset += 64 - - li 9, 64 - divdu 31, 5, 9 - - cmpdi 31, 0 - ble Skip_block_loop - - mtctr 31 - -# h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r -# Rewrite the polynominal sum of product as follows, -# h1 = (h0 + m1) * r^2, h2 = (h0 + m2) * r^2 -# h3 = (h1 + m3) * r^2, h4 = (h2 + m4) * r^2 --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2 -# .... Repeat -# h5 = (h3 + m5) * r^2, h6 = (h4 + m6) * r^2 --> -# h7 = (h5 + m7) * r^2, h8 = (h6 + m8) * r^1 --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r -# -loop_4blocks: - - # Multiply odd words and even words - mul_odd - mul_even - # carry reduction - vspltisb 9, 2 - vsrd 10, 14, 31 - vsrd 11, 17, 31 - vand 7, 17, 25 - vand 4, 14, 25 - vaddudm 18, 18, 11 - vsrd 12, 18, 31 - vaddudm 15, 15, 10 - - vsrd 11, 15, 31 - vand 8, 18, 25 - vand 5, 15, 25 - vaddudm 4, 4, 12 - vsld 10, 12, 9 - vaddudm 6, 16, 11 - - vsrd 13, 6, 31 - vand 6, 6, 25 - vaddudm 4, 4, 10 - vsrd 10, 4, 31 - vaddudm 7, 7, 13 - - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 5, 5, 10 - vaddudm 8, 8, 11 - - # input m1 m2 m3 m4 - add 20, 4, 21 - xxlor 49, 24, 24 - xxlor 50, 25, 25 - lxvw4x 43, 0, 20 - addi 17, 20, 16 - lxvw4x 44, 0, 17 - vperm 14, 11, 12, 17 - vperm 15, 11, 12, 18 - addi 17, 17, 16 - lxvw4x 43, 0, 17 - addi 17, 17, 16 - lxvw4x 44, 0, 17 - vperm 17, 11, 12, 17 - vperm 18, 11, 12, 18 - - vand 20, 14, 25 # a0 - vand 9, 17, 25 # a0 - vsrd 21, 14, 31 # >> 26 - vsrd 22, 21, 31 # 12 bits left - vsrd 10, 17, 31 # >> 26 - vsrd 11, 10, 31 # 12 bits left - - vand 21, 21, 25 # a1 - vand 10, 10, 25 # a1 - - vspltisb 13, 12 - vand 16, 15, 25 - vsld 23, 16, 13 - vor 22, 22, 23 - vand 22, 22, 25 # a2 - vand 16, 18, 25 - vsld 12, 16, 13 - vor 11, 11, 12 - vand 11, 11, 25 # a2 - vspltisb 13, 14 - vsrd 23, 15, 13 # >> 14 - vsrd 24, 23, 31 # >> 26, a4 - vand 23, 23, 25 # a3 - vsrd 12, 18, 13 # >> 14 - vsrd 13, 12, 31 # >> 26, a4 - vand 12, 12, 25 # a3 - - vaddudm 4, 4, 20 - vaddudm 5, 5, 21 - vaddudm 6, 6, 22 - vaddudm 7, 7, 23 - vaddudm 8, 8, 24 - - # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] - vmrgow 4, 9, 4 - vmrgow 5, 10, 5 - vmrgow 6, 11, 6 - vmrgow 7, 12, 7 - vmrgow 8, 13, 8 - vaddudm 8, 8, 19 - - addi 5, 5, -64 # len -= 64 - addi 21, 21, 64 # offset += 64 - - bdnz loop_4blocks - -Skip_block_loop: - xxlor 58, 0, 0 - xxlor 59, 1, 1 - xxlor 60, 2, 2 - xxlor 61, 3, 3 - xxlor 62, 4, 4 - xxlor 32, 5, 5 - xxlor 33, 6, 6 - xxlor 34, 7, 7 - xxlor 35, 8, 8 - - # Multiply odd words and even words - mul_odd - mul_even - - # Sum the products. - xxpermdi 41, 31, 46, 0 - xxpermdi 42, 31, 47, 0 - vaddudm 4, 14, 9 - xxpermdi 36, 31, 36, 3 - vaddudm 5, 15, 10 - xxpermdi 37, 31, 37, 3 - xxpermdi 43, 31, 48, 0 - vaddudm 6, 16, 11 - xxpermdi 38, 31, 38, 3 - xxpermdi 44, 31, 49, 0 - vaddudm 7, 17, 12 - xxpermdi 39, 31, 39, 3 - xxpermdi 45, 31, 50, 0 - vaddudm 8, 18, 13 - xxpermdi 40, 31, 40, 3 - - # carry reduction - vspltisb 9, 2 - vsrd 10, 4, 31 - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 8, 8, 11 - vsrd 12, 8, 31 - vaddudm 5, 5, 10 - - vsrd 11, 5, 31 - vand 8, 8, 25 - vand 5, 5, 25 - vaddudm 4, 4, 12 - vsld 10, 12, 9 - vaddudm 6, 6, 11 - - vsrd 13, 6, 31 - vand 6, 6, 25 - vaddudm 4, 4, 10 - vsrd 10, 4, 31 - vaddudm 7, 7, 13 - - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 5, 5, 10 - vsrd 10, 5, 31 - vand 5, 5, 25 - vaddudm 6, 6, 10 - vaddudm 8, 8, 11 - - b do_final_update - -do_final_update: - # combine 26 bit limbs - # v4, v5, v6, v7 and v8 are 26 bit vectors - vsld 5, 5, 31 - vor 20, 4, 5 - vspltisb 11, 12 - vsrd 12, 6, 11 - vsld 6, 6, 31 - vsld 6, 6, 31 - vor 20, 20, 6 - vspltisb 11, 14 - vsld 7, 7, 11 - vor 21, 7, 12 - mfvsrld 16, 40 # save last 2 bytes - vsld 8, 8, 11 - vsld 8, 8, 31 - vor 21, 21, 8 - mfvsrld 17, 52 - mfvsrld 19, 53 - srdi 16, 16, 24 - - std 17, 0(3) - std 19, 8(3) - stw 16, 16(3) - -Out_loop: - li 3, 0 - - RESTORE_REGS - - blr - -Out_no_poly1305: - li 3, 0 - blr -SYM_FUNC_END(poly1305_p10le_4blocks) - -# -# ======================================================================= -# The following functions implement 64 x 64 bits multiplication poly1305. -# -SYM_FUNC_START_LOCAL(Poly1305_init_64) - # mask 0x0FFFFFFC0FFFFFFC - # mask 0x0FFFFFFC0FFFFFFF - addis 10, 2, rmask@toc@ha - addi 10, 10, rmask@toc@l - ld 11, 0(10) - ld 12, 8(10) - - # initialize - # load key from r3 - ld 9, 24(3) - ld 10, 32(3) - and. 9, 9, 11 # cramp mask r0 - and. 10, 10, 12 # cramp mask r1 - - srdi 21, 10, 2 - add 19, 21, 10 # s1: r19 - (r1 >> 2) *5 - - # setup r and s - li 25, 0 - mtvsrdd 32+0, 9, 19 # r0, s1 - mtvsrdd 32+1, 10, 9 # r1, r0 - mtvsrdd 32+2, 19, 25 # s1 - mtvsrdd 32+3, 9, 25 # r0 - - blr -SYM_FUNC_END(Poly1305_init_64) - -# Poly1305_mult -# v6 = (h0, h1), v8 = h2 -# v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0 -# -# Output: v7, v10, v11 -# -SYM_FUNC_START_LOCAL(Poly1305_mult) - # - # d0 = h0 * r0 + h1 * s1 - vmsumudm 7, 6, 0, 9 # h0 * r0, h1 * s1 - - # d1 = h0 * r1 + h1 * r0 + h2 * s1 - vmsumudm 11, 6, 1, 9 # h0 * r1, h1 * r0 - vmsumudm 10, 8, 2, 11 # d1 += h2 * s1 - - # d2 = r0 - vmsumudm 11, 8, 3, 9 # d2 = h2 * r0 - blr -SYM_FUNC_END(Poly1305_mult) - -# -# carry reduction -# h %=p -# -# Input: v7, v10, v11 -# Output: r27, r28, r29 -# -SYM_FUNC_START_LOCAL(Carry_reduction) - mfvsrld 27, 32+7 - mfvsrld 28, 32+10 - mfvsrld 29, 32+11 - mfvsrd 20, 32+7 # h0.h - mfvsrd 21, 32+10 # h1.h - - addc 28, 28, 20 - adde 29, 29, 21 - srdi 22, 29, 0x2 - sldi 23, 22, 0x2 - add 23, 23, 22 # (h2 & 3) * 5 - addc 27, 27, 23 # h0 - addze 28, 28 # h1 - andi. 29, 29, 0x3 # h2 - blr -SYM_FUNC_END(Carry_reduction) - -# -# poly1305 multiplication -# h *= r, h %= p -# d0 = h0 * r0 + h1 * s1 -# d1 = h0 * r1 + h1 * r0 + h2 * s1 -# d2 = h0 * r0 -# -# -# unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit) -# - no highbit if final leftover block (highbit = 0) -# -SYM_FUNC_START(poly1305_64s) - cmpdi 5, 0 - ble Out_no_poly1305_64 - - mflr 0 - std 0, 16(1) - stdu 1,-400(1) - - SAVE_GPR 14, 112, 1 - SAVE_GPR 15, 120, 1 - SAVE_GPR 16, 128, 1 - SAVE_GPR 17, 136, 1 - SAVE_GPR 18, 144, 1 - SAVE_GPR 19, 152, 1 - SAVE_GPR 20, 160, 1 - SAVE_GPR 21, 168, 1 - SAVE_GPR 22, 176, 1 - SAVE_GPR 23, 184, 1 - SAVE_GPR 24, 192, 1 - SAVE_GPR 25, 200, 1 - SAVE_GPR 26, 208, 1 - SAVE_GPR 27, 216, 1 - SAVE_GPR 28, 224, 1 - SAVE_GPR 29, 232, 1 - SAVE_GPR 30, 240, 1 - SAVE_GPR 31, 248, 1 - - # Init poly1305 - bl Poly1305_init_64 - - li 25, 0 # offset to inp and outp - - add 11, 25, 4 - - # load h - # h0, h1, h2? - ld 27, 0(3) - ld 28, 8(3) - lwz 29, 16(3) - - li 30, 16 - divdu 31, 5, 30 - - mtctr 31 - - mr 24, 6 # highbit - -Loop_block_64: - vxor 9, 9, 9 - - ld 20, 0(11) - ld 21, 8(11) - addi 11, 11, 16 - - addc 27, 27, 20 - adde 28, 28, 21 - adde 29, 29, 24 - - li 22, 0 - mtvsrdd 32+6, 27, 28 # h0, h1 - mtvsrdd 32+8, 29, 22 # h2 - - bl Poly1305_mult - - bl Carry_reduction - - bdnz Loop_block_64 - - std 27, 0(3) - std 28, 8(3) - stw 29, 16(3) - - li 3, 0 - - RESTORE_GPR 14, 112, 1 - RESTORE_GPR 15, 120, 1 - RESTORE_GPR 16, 128, 1 - RESTORE_GPR 17, 136, 1 - RESTORE_GPR 18, 144, 1 - RESTORE_GPR 19, 152, 1 - RESTORE_GPR 20, 160, 1 - RESTORE_GPR 21, 168, 1 - RESTORE_GPR 22, 176, 1 - RESTORE_GPR 23, 184, 1 - RESTORE_GPR 24, 192, 1 - RESTORE_GPR 25, 200, 1 - RESTORE_GPR 26, 208, 1 - RESTORE_GPR 27, 216, 1 - RESTORE_GPR 28, 224, 1 - RESTORE_GPR 29, 232, 1 - RESTORE_GPR 30, 240, 1 - RESTORE_GPR 31, 248, 1 - - addi 1, 1, 400 - ld 0, 16(1) - mtlr 0 - - blr - -Out_no_poly1305_64: - li 3, 0 - blr -SYM_FUNC_END(poly1305_64s) - -# -# Input: r3 = h, r4 = s, r5 = mac -# mac = h + s -# -SYM_FUNC_START(poly1305_emit_64) - ld 10, 0(3) - ld 11, 8(3) - ld 12, 16(3) - - # compare modulus - # h + 5 + (-p) - mr 6, 10 - mr 7, 11 - mr 8, 12 - addic. 6, 6, 5 - addze 7, 7 - addze 8, 8 - srdi 9, 8, 2 # overflow? - cmpdi 9, 0 - beq Skip_h64 - mr 10, 6 - mr 11, 7 - mr 12, 8 - -Skip_h64: - ld 6, 0(4) - ld 7, 8(4) - addc 10, 10, 6 - adde 11, 11, 7 - addze 12, 12 - - std 10, 0(5) - std 11, 8(5) - blr -SYM_FUNC_END(poly1305_emit_64) - -SYM_DATA_START_LOCAL(RMASK) -.align 5 -rmask: -.byte 0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f -cnum: -.long 0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000 -.long 0x1a, 0x00, 0x1a, 0x00 -.long 0x01000000, 0x01000000, 0x01000000, 0x01000000 -.long 0x00010203, 0x04050607, 0x10111213, 0x14151617 -.long 0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f -SYM_DATA_END(RMASK) |