diff options
Diffstat (limited to 'arch/arm64/crypto/sha2-ce-glue.c')
-rw-r--r-- | arch/arm64/crypto/sha2-ce-glue.c | 91 |
1 files changed, 69 insertions, 22 deletions
diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 604a01a4ede6..0a44d2e7ee1f 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -10,7 +10,7 @@ #include <asm/unaligned.h> #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> -#include <crypto/sha.h> +#include <crypto/sha2.h> #include <crypto/sha256_base.h> #include <linux/cpufeature.h> #include <linux/crypto.h> @@ -19,14 +19,35 @@ MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha224"); +MODULE_ALIAS_CRYPTO("sha256"); struct sha256_ce_state { struct sha256_state sst; u32 finalize; }; -asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, - int blocks); +extern const u32 sha256_ce_offsetof_count; +extern const u32 sha256_ce_offsetof_finalize; + +asmlinkage int __sha256_ce_transform(struct sha256_ce_state *sst, u8 const *src, + int blocks); + +static void sha256_ce_transform(struct sha256_state *sst, u8 const *src, + int blocks) +{ + while (blocks) { + int rem; + + kernel_neon_begin(); + rem = __sha256_ce_transform(container_of(sst, + struct sha256_ce_state, + sst), src, blocks); + kernel_neon_end(); + src += (blocks - rem) * SHA256_BLOCK_SIZE; + blocks = rem; + } +} const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, sst.count); @@ -35,6 +56,12 @@ const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state, asmlinkage void sha256_block_data_order(u32 *digest, u8 const *src, int blocks); +static void sha256_arm64_transform(struct sha256_state *sst, u8 const *src, + int blocks) +{ + sha256_block_data_order(sst->state, src, blocks); +} + static int sha256_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -42,13 +69,10 @@ static int sha256_ce_update(struct shash_desc *desc, const u8 *data, if (!crypto_simd_usable()) return sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha256_block_data_order); + sha256_arm64_transform); sctx->finalize = 0; - kernel_neon_begin(); - sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha2_ce_transform); - kernel_neon_end(); + sha256_base_do_update(desc, data, len, sha256_ce_transform); return 0; } @@ -62,9 +86,8 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, if (!crypto_simd_usable()) { if (len) sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha256_block_data_order); - sha256_base_do_finalize(desc, - (sha256_block_fn *)sha256_block_data_order); + sha256_arm64_transform); + sha256_base_do_finalize(desc, sha256_arm64_transform); return sha256_base_finish(desc, out); } @@ -74,13 +97,9 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, */ sctx->finalize = finalize; - kernel_neon_begin(); - sha256_base_do_update(desc, data, len, - (sha256_block_fn *)sha2_ce_transform); + sha256_base_do_update(desc, data, len, sha256_ce_transform); if (!finalize) - sha256_base_do_finalize(desc, - (sha256_block_fn *)sha2_ce_transform); - kernel_neon_end(); + sha256_base_do_finalize(desc, sha256_ce_transform); return sha256_base_finish(desc, out); } @@ -89,24 +108,48 @@ static int sha256_ce_final(struct shash_desc *desc, u8 *out) struct sha256_ce_state *sctx = shash_desc_ctx(desc); if (!crypto_simd_usable()) { - sha256_base_do_finalize(desc, - (sha256_block_fn *)sha256_block_data_order); + sha256_base_do_finalize(desc, sha256_arm64_transform); return sha256_base_finish(desc, out); } sctx->finalize = 0; - kernel_neon_begin(); - sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); - kernel_neon_end(); + sha256_base_do_finalize(desc, sha256_ce_transform); return sha256_base_finish(desc, out); } +static int sha256_ce_digest(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + sha256_base_init(desc); + return sha256_ce_finup(desc, data, len, out); +} + +static int sha256_ce_export(struct shash_desc *desc, void *out) +{ + struct sha256_ce_state *sctx = shash_desc_ctx(desc); + + memcpy(out, &sctx->sst, sizeof(struct sha256_state)); + return 0; +} + +static int sha256_ce_import(struct shash_desc *desc, const void *in) +{ + struct sha256_ce_state *sctx = shash_desc_ctx(desc); + + memcpy(&sctx->sst, in, sizeof(struct sha256_state)); + sctx->finalize = 0; + return 0; +} + static struct shash_alg algs[] = { { .init = sha224_base_init, .update = sha256_ce_update, .final = sha256_ce_final, .finup = sha256_ce_finup, + .export = sha256_ce_export, + .import = sha256_ce_import, .descsize = sizeof(struct sha256_ce_state), + .statesize = sizeof(struct sha256_state), .digestsize = SHA224_DIGEST_SIZE, .base = { .cra_name = "sha224", @@ -120,7 +163,11 @@ static struct shash_alg algs[] = { { .update = sha256_ce_update, .final = sha256_ce_final, .finup = sha256_ce_finup, + .digest = sha256_ce_digest, + .export = sha256_ce_export, + .import = sha256_ce_import, .descsize = sizeof(struct sha256_ce_state), + .statesize = sizeof(struct sha256_state), .digestsize = SHA256_DIGEST_SIZE, .base = { .cra_name = "sha256", |