summaryrefslogtreecommitdiff
path: root/crypto
diff options
context:
space:
mode:
Diffstat (limited to 'crypto')
-rw-r--r--crypto/Kconfig39
-rw-r--r--crypto/Makefile8
-rw-r--r--crypto/ahash.c43
-rw-r--r--crypto/crc32.c65
-rw-r--r--crypto/crc32c.c66
-rw-r--r--crypto/cryptd.c6
-rw-r--r--crypto/crypto_engine.c55
-rw-r--r--crypto/deflate.c7
-rw-r--r--crypto/hash_info.c63
-rw-r--r--crypto/hkdf.c2
-rw-r--r--crypto/jitterentropy-kcapi.c9
-rw-r--r--crypto/jitterentropy.c2
-rw-r--r--crypto/krb5/selftest.c1
-rw-r--r--crypto/pcrypt.c7
-rw-r--r--crypto/sha1.c201
-rw-r--r--crypto/sha1_generic.c87
-rw-r--r--crypto/sha256.c371
-rw-r--r--crypto/sha512.c354
-rw-r--r--crypto/sha512_generic.c217
-rw-r--r--crypto/testmgr.c139
-rw-r--r--crypto/wp512.c121
-rw-r--r--crypto/zstd.c356
22 files changed, 1216 insertions, 1003 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig
index e9fee7818e27..23bd98981ae8 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -176,16 +176,33 @@ config CRYPTO_USER
config CRYPTO_SELFTESTS
bool "Enable cryptographic self-tests"
- depends on DEBUG_KERNEL
+ depends on EXPERT
help
Enable the cryptographic self-tests.
The cryptographic self-tests run at boot time, or at algorithm
registration time if algorithms are dynamically loaded later.
- This is primarily intended for developer use. It should not be
- enabled in production kernels, unless you are trying to use these
- tests to fulfill a FIPS testing requirement.
+ There are two main use cases for these tests:
+
+ - Development and pre-release testing. In this case, also enable
+ CRYPTO_SELFTESTS_FULL to get the full set of tests. All crypto code
+ in the kernel is expected to pass the full set of tests.
+
+ - Production kernels, to help prevent buggy drivers from being used
+ and/or meet FIPS 140-3 pre-operational testing requirements. In
+ this case, enable CRYPTO_SELFTESTS but not CRYPTO_SELFTESTS_FULL.
+
+config CRYPTO_SELFTESTS_FULL
+ bool "Enable the full set of cryptographic self-tests"
+ depends on CRYPTO_SELFTESTS
+ help
+ Enable the full set of cryptographic self-tests for each algorithm.
+
+ The full set of tests should be enabled for development and
+ pre-release testing, but not in production kernels.
+
+ All crypto code in the kernel is expected to pass the full tests.
config CRYPTO_NULL
tristate "Null algorithms"
@@ -969,15 +986,16 @@ config CRYPTO_SHA1
select CRYPTO_HASH
select CRYPTO_LIB_SHA1
help
- SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3)
+ SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3), including
+ HMAC support.
config CRYPTO_SHA256
tristate "SHA-224 and SHA-256"
select CRYPTO_HASH
select CRYPTO_LIB_SHA256
- select CRYPTO_LIB_SHA256_GENERIC
help
- SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC 10118-3)
+ SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC
+ 10118-3), including HMAC support.
This is required for IPsec AH (XFRM_AH) and IPsec ESP (XFRM_ESP).
Used by the btrfs filesystem, Ceph, NFS, and SMB.
@@ -985,8 +1003,10 @@ config CRYPTO_SHA256
config CRYPTO_SHA512
tristate "SHA-384 and SHA-512"
select CRYPTO_HASH
+ select CRYPTO_LIB_SHA512
help
- SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC 10118-3)
+ SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC
+ 10118-3), including HMAC support.
config CRYPTO_SHA3
tristate "SHA-3"
@@ -1403,9 +1423,6 @@ config CRYPTO_USER_API_ENABLE_OBSOLETE
endmenu
-config CRYPTO_HASH_INFO
- bool
-
if !KMSAN # avoid false positives from assembly
if ARM
source "arch/arm/crypto/Kconfig"
diff --git a/crypto/Makefile b/crypto/Makefile
index 017df3a2e4bb..6c5d59369dac 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -75,10 +75,9 @@ obj-$(CONFIG_CRYPTO_NULL) += crypto_null.o
obj-$(CONFIG_CRYPTO_MD4) += md4.o
obj-$(CONFIG_CRYPTO_MD5) += md5.o
obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o
-obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o
+obj-$(CONFIG_CRYPTO_SHA1) += sha1.o
obj-$(CONFIG_CRYPTO_SHA256) += sha256.o
-CFLAGS_sha256.o += -DARCH=$(ARCH)
-obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o
+obj-$(CONFIG_CRYPTO_SHA512) += sha512.o
obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o
obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o
obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o
@@ -154,10 +153,8 @@ obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o
obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
obj-$(CONFIG_CRYPTO_CRC32C) += crc32c-cryptoapi.o
crc32c-cryptoapi-y := crc32c.o
-CFLAGS_crc32c.o += -DARCH=$(ARCH)
obj-$(CONFIG_CRYPTO_CRC32) += crc32-cryptoapi.o
crc32-cryptoapi-y := crc32.o
-CFLAGS_crc32.o += -DARCH=$(ARCH)
obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
obj-$(CONFIG_CRYPTO_KRB5ENC) += krb5enc.o
obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o
@@ -205,7 +202,6 @@ obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o
obj-$(CONFIG_XOR_BLOCKS) += xor.o
obj-$(CONFIG_ASYNC_CORE) += async_tx/
obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
-obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
crypto_simd-y := simd.o
obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o
diff --git a/crypto/ahash.c b/crypto/ahash.c
index e10bc2659ae4..a227793d2c5b 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -29,19 +29,6 @@
#define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e
-struct crypto_hash_walk {
- const char *data;
-
- unsigned int offset;
- unsigned int flags;
-
- struct page *pg;
- unsigned int entrylen;
-
- unsigned int total;
- struct scatterlist *sg;
-};
-
static int ahash_def_finup(struct ahash_request *req);
static inline bool crypto_ahash_block_only(struct crypto_ahash *tfm)
@@ -112,8 +99,8 @@ static int hash_walk_new_entry(struct crypto_hash_walk *walk)
return hash_walk_next(walk);
}
-static int crypto_hash_walk_first(struct ahash_request *req,
- struct crypto_hash_walk *walk)
+int crypto_hash_walk_first(struct ahash_request *req,
+ struct crypto_hash_walk *walk)
{
walk->total = req->nbytes;
walk->entrylen = 0;
@@ -133,8 +120,9 @@ static int crypto_hash_walk_first(struct ahash_request *req,
return hash_walk_new_entry(walk);
}
+EXPORT_SYMBOL_GPL(crypto_hash_walk_first);
-static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
+int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
{
if ((walk->flags & CRYPTO_AHASH_REQ_VIRT))
return err;
@@ -160,11 +148,7 @@ static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
return hash_walk_new_entry(walk);
}
-
-static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk)
-{
- return !(walk->entrylen | walk->total);
-}
+EXPORT_SYMBOL_GPL(crypto_hash_walk_done);
/*
* For an ahash tfm that is using an shash algorithm (instead of an ahash
@@ -347,6 +331,12 @@ static int ahash_do_req_chain(struct ahash_request *req,
if (crypto_ahash_statesize(tfm) > HASH_MAX_STATESIZE)
return -ENOSYS;
+ if (!crypto_ahash_need_fallback(tfm))
+ return -ENOSYS;
+
+ if (crypto_hash_no_export_core(tfm))
+ return -ENOSYS;
+
{
u8 state[HASH_MAX_STATESIZE];
@@ -600,12 +590,14 @@ static void ahash_def_finup_done2(void *data, int err)
static int ahash_def_finup_finish1(struct ahash_request *req, int err)
{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+
if (err)
goto out;
req->base.complete = ahash_def_finup_done2;
- err = crypto_ahash_final(req);
+ err = crypto_ahash_alg(tfm)->final(req);
if (err == -EINPROGRESS || err == -EBUSY)
return err;
@@ -952,6 +944,10 @@ static int ahash_prepare_alg(struct ahash_alg *alg)
base->cra_reqsize > MAX_SYNC_HASH_REQSIZE)
return -EINVAL;
+ if (base->cra_flags & CRYPTO_ALG_NEED_FALLBACK &&
+ base->cra_flags & CRYPTO_ALG_NO_FALLBACK)
+ return -EINVAL;
+
err = hash_prepare_alg(&alg->halg);
if (err)
return err;
@@ -960,7 +956,8 @@ static int ahash_prepare_alg(struct ahash_alg *alg)
base->cra_flags |= CRYPTO_ALG_TYPE_AHASH;
if ((base->cra_flags ^ CRYPTO_ALG_REQ_VIRT) &
- (CRYPTO_ALG_ASYNC | CRYPTO_ALG_REQ_VIRT))
+ (CRYPTO_ALG_ASYNC | CRYPTO_ALG_REQ_VIRT) &&
+ !(base->cra_flags & CRYPTO_ALG_NO_FALLBACK))
base->cra_flags |= CRYPTO_ALG_NEED_FALLBACK;
if (!alg->setkey)
diff --git a/crypto/crc32.c b/crypto/crc32.c
index cc371d42601f..489cbed9422e 100644
--- a/crypto/crc32.c
+++ b/crypto/crc32.c
@@ -59,29 +59,12 @@ static int crc32_update(struct shash_desc *desc, const u8 *data,
{
u32 *crcp = shash_desc_ctx(desc);
- *crcp = crc32_le_base(*crcp, data, len);
- return 0;
-}
-
-static int crc32_update_arch(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- u32 *crcp = shash_desc_ctx(desc);
-
*crcp = crc32_le(*crcp, data, len);
return 0;
}
/* No final XOR 0xFFFFFFFF, like crc32_le */
-static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len,
- u8 *out)
-{
- put_unaligned_le32(crc32_le_base(*crcp, data, len), out);
- return 0;
-}
-
-static int __crc32_finup_arch(u32 *crcp, const u8 *data, unsigned int len,
- u8 *out)
+static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out)
{
put_unaligned_le32(crc32_le(*crcp, data, len), out);
return 0;
@@ -93,12 +76,6 @@ static int crc32_finup(struct shash_desc *desc, const u8 *data,
return __crc32_finup(shash_desc_ctx(desc), data, len, out);
}
-static int crc32_finup_arch(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- return __crc32_finup_arch(shash_desc_ctx(desc), data, len, out);
-}
-
static int crc32_final(struct shash_desc *desc, u8 *out)
{
u32 *crcp = shash_desc_ctx(desc);
@@ -113,13 +90,7 @@ static int crc32_digest(struct shash_desc *desc, const u8 *data,
return __crc32_finup(crypto_shash_ctx(desc->tfm), data, len, out);
}
-static int crc32_digest_arch(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- return __crc32_finup_arch(crypto_shash_ctx(desc->tfm), data, len, out);
-}
-
-static struct shash_alg algs[] = {{
+static struct shash_alg alg = {
.setkey = crc32_setkey,
.init = crc32_init,
.update = crc32_update,
@@ -130,46 +101,23 @@ static struct shash_alg algs[] = {{
.digestsize = CHKSUM_DIGEST_SIZE,
.base.cra_name = "crc32",
- .base.cra_driver_name = "crc32-generic",
+ .base.cra_driver_name = "crc32-lib",
.base.cra_priority = 100,
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.base.cra_blocksize = CHKSUM_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(u32),
.base.cra_module = THIS_MODULE,
.base.cra_init = crc32_cra_init,
-}, {
- .setkey = crc32_setkey,
- .init = crc32_init,
- .update = crc32_update_arch,
- .final = crc32_final,
- .finup = crc32_finup_arch,
- .digest = crc32_digest_arch,
- .descsize = sizeof(u32),
- .digestsize = CHKSUM_DIGEST_SIZE,
-
- .base.cra_name = "crc32",
- .base.cra_driver_name = "crc32-" __stringify(ARCH),
- .base.cra_priority = 150,
- .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
- .base.cra_blocksize = CHKSUM_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(u32),
- .base.cra_module = THIS_MODULE,
- .base.cra_init = crc32_cra_init,
-}};
-
-static int num_algs;
+};
static int __init crc32_mod_init(void)
{
- /* register the arch flavor only if it differs from the generic one */
- num_algs = 1 + ((crc32_optimizations() & CRC32_LE_OPTIMIZATION) != 0);
-
- return crypto_register_shashes(algs, num_algs);
+ return crypto_register_shash(&alg);
}
static void __exit crc32_mod_fini(void)
{
- crypto_unregister_shashes(algs, num_algs);
+ crypto_unregister_shash(&alg);
}
module_init(crc32_mod_init);
@@ -179,4 +127,3 @@ MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
MODULE_DESCRIPTION("CRC32 calculations wrapper for lib/crc32");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("crc32");
-MODULE_ALIAS_CRYPTO("crc32-generic");
diff --git a/crypto/crc32c.c b/crypto/crc32c.c
index e5377898414a..1eff54dde2f7 100644
--- a/crypto/crc32c.c
+++ b/crypto/crc32c.c
@@ -85,15 +85,6 @@ static int chksum_update(struct shash_desc *desc, const u8 *data,
{
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
- ctx->crc = crc32c_base(ctx->crc, data, length);
- return 0;
-}
-
-static int chksum_update_arch(struct shash_desc *desc, const u8 *data,
- unsigned int length)
-{
- struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
-
ctx->crc = crc32c(ctx->crc, data, length);
return 0;
}
@@ -108,13 +99,6 @@ static int chksum_final(struct shash_desc *desc, u8 *out)
static int __chksum_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out)
{
- put_unaligned_le32(~crc32c_base(*crcp, data, len), out);
- return 0;
-}
-
-static int __chksum_finup_arch(u32 *crcp, const u8 *data, unsigned int len,
- u8 *out)
-{
put_unaligned_le32(~crc32c(*crcp, data, len), out);
return 0;
}
@@ -127,14 +111,6 @@ static int chksum_finup(struct shash_desc *desc, const u8 *data,
return __chksum_finup(&ctx->crc, data, len, out);
}
-static int chksum_finup_arch(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
-
- return __chksum_finup_arch(&ctx->crc, data, len, out);
-}
-
static int chksum_digest(struct shash_desc *desc, const u8 *data,
unsigned int length, u8 *out)
{
@@ -143,14 +119,6 @@ static int chksum_digest(struct shash_desc *desc, const u8 *data,
return __chksum_finup(&mctx->key, data, length, out);
}
-static int chksum_digest_arch(struct shash_desc *desc, const u8 *data,
- unsigned int length, u8 *out)
-{
- struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
-
- return __chksum_finup_arch(&mctx->key, data, length, out);
-}
-
static int crc32c_cra_init(struct crypto_tfm *tfm)
{
struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
@@ -159,7 +127,7 @@ static int crc32c_cra_init(struct crypto_tfm *tfm)
return 0;
}
-static struct shash_alg algs[] = {{
+static struct shash_alg alg = {
.digestsize = CHKSUM_DIGEST_SIZE,
.setkey = chksum_setkey,
.init = chksum_init,
@@ -170,46 +138,23 @@ static struct shash_alg algs[] = {{
.descsize = sizeof(struct chksum_desc_ctx),
.base.cra_name = "crc32c",
- .base.cra_driver_name = "crc32c-generic",
+ .base.cra_driver_name = "crc32c-lib",
.base.cra_priority = 100,
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.base.cra_blocksize = CHKSUM_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct chksum_ctx),
.base.cra_module = THIS_MODULE,
.base.cra_init = crc32c_cra_init,
-}, {
- .digestsize = CHKSUM_DIGEST_SIZE,
- .setkey = chksum_setkey,
- .init = chksum_init,
- .update = chksum_update_arch,
- .final = chksum_final,
- .finup = chksum_finup_arch,
- .digest = chksum_digest_arch,
- .descsize = sizeof(struct chksum_desc_ctx),
-
- .base.cra_name = "crc32c",
- .base.cra_driver_name = "crc32c-" __stringify(ARCH),
- .base.cra_priority = 150,
- .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
- .base.cra_blocksize = CHKSUM_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct chksum_ctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_init = crc32c_cra_init,
-}};
-
-static int num_algs;
+};
static int __init crc32c_mod_init(void)
{
- /* register the arch flavor only if it differs from the generic one */
- num_algs = 1 + ((crc32_optimizations() & CRC32C_OPTIMIZATION) != 0);
-
- return crypto_register_shashes(algs, num_algs);
+ return crypto_register_shash(&alg);
}
static void __exit crc32c_mod_fini(void)
{
- crypto_unregister_shashes(algs, num_algs);
+ crypto_unregister_shash(&alg);
}
module_init(crc32c_mod_init);
@@ -219,4 +164,3 @@ MODULE_AUTHOR("Clay Haapala <chaapala@cisco.com>");
MODULE_DESCRIPTION("CRC32c (Castagnoli) calculations wrapper for lib/crc32c");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("crc32c");
-MODULE_ALIAS_CRYPTO("crc32c-generic");
diff --git a/crypto/cryptd.c b/crypto/cryptd.c
index 5bb6f8d88cc2..efff54e707cb 100644
--- a/crypto/cryptd.c
+++ b/crypto/cryptd.c
@@ -34,6 +34,7 @@ MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth");
static struct workqueue_struct *cryptd_wq;
struct cryptd_cpu_queue {
+ local_lock_t bh_lock;
struct crypto_queue queue;
struct work_struct work;
};
@@ -110,6 +111,7 @@ static int cryptd_init_queue(struct cryptd_queue *queue,
cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
+ local_lock_init(&cpu_queue->bh_lock);
}
pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen);
return 0;
@@ -135,6 +137,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue,
refcount_t *refcnt;
local_bh_disable();
+ local_lock_nested_bh(&queue->cpu_queue->bh_lock);
cpu_queue = this_cpu_ptr(queue->cpu_queue);
err = crypto_enqueue_request(&cpu_queue->queue, request);
@@ -151,6 +154,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue,
refcount_inc(refcnt);
out:
+ local_unlock_nested_bh(&queue->cpu_queue->bh_lock);
local_bh_enable();
return err;
@@ -169,8 +173,10 @@ static void cryptd_queue_worker(struct work_struct *work)
* Only handle one request at a time to avoid hogging crypto workqueue.
*/
local_bh_disable();
+ __local_lock_nested_bh(&cpu_queue->bh_lock);
backlog = crypto_get_backlog(&cpu_queue->queue);
req = crypto_dequeue_request(&cpu_queue->queue);
+ __local_unlock_nested_bh(&cpu_queue->bh_lock);
local_bh_enable();
if (!req)
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c
index 445d3c113ee1..18e1689efe12 100644
--- a/crypto/crypto_engine.c
+++ b/crypto/crypto_engine.c
@@ -74,7 +74,6 @@ static void crypto_pump_requests(struct crypto_engine *engine,
struct crypto_engine_alg *alg;
struct crypto_engine_op *op;
unsigned long flags;
- bool was_busy = false;
int ret;
spin_lock_irqsave(&engine->queue_lock, flags);
@@ -83,12 +82,6 @@ static void crypto_pump_requests(struct crypto_engine *engine,
if (!engine->retry_support && engine->cur_req)
goto out;
- /* If another context is idling then defer */
- if (engine->idling) {
- kthread_queue_work(engine->kworker, &engine->pump_requests);
- goto out;
- }
-
/* Check if the engine queue is idle */
if (!crypto_queue_len(&engine->queue) || !engine->running) {
if (!engine->busy)
@@ -102,15 +95,6 @@ static void crypto_pump_requests(struct crypto_engine *engine,
}
engine->busy = false;
- engine->idling = true;
- spin_unlock_irqrestore(&engine->queue_lock, flags);
-
- if (engine->unprepare_crypt_hardware &&
- engine->unprepare_crypt_hardware(engine))
- dev_err(engine->dev, "failed to unprepare crypt hardware\n");
-
- spin_lock_irqsave(&engine->queue_lock, flags);
- engine->idling = false;
goto out;
}
@@ -129,22 +113,11 @@ start_request:
if (!engine->retry_support)
engine->cur_req = async_req;
- if (engine->busy)
- was_busy = true;
- else
+ if (!engine->busy)
engine->busy = true;
spin_unlock_irqrestore(&engine->queue_lock, flags);
- /* Until here we get the request need to be encrypted successfully */
- if (!was_busy && engine->prepare_crypt_hardware) {
- ret = engine->prepare_crypt_hardware(engine);
- if (ret) {
- dev_err(engine->dev, "failed to prepare crypt hardware\n");
- goto req_err_1;
- }
- }
-
alg = container_of(async_req->tfm->__crt_alg,
struct crypto_engine_alg, base);
op = &alg->op;
@@ -195,17 +168,6 @@ retry:
out:
spin_unlock_irqrestore(&engine->queue_lock, flags);
- /*
- * Batch requests is possible only if
- * hardware can enqueue multiple requests
- */
- if (engine->do_batch_requests) {
- ret = engine->do_batch_requests(engine);
- if (ret)
- dev_err(engine->dev, "failed to do batch requests: %d\n",
- ret);
- }
-
return;
}
@@ -462,12 +424,6 @@ EXPORT_SYMBOL_GPL(crypto_engine_stop);
* crypto-engine queue.
* @dev: the device attached with one hardware engine
* @retry_support: whether hardware has support for retry mechanism
- * @cbk_do_batch: pointer to a callback function to be invoked when executing
- * a batch of requests.
- * This has the form:
- * callback(struct crypto_engine *engine)
- * where:
- * engine: the crypto engine structure.
* @rt: whether this queue is set to run as a realtime task
* @qlen: maximum size of the crypto-engine queue
*
@@ -476,7 +432,6 @@ EXPORT_SYMBOL_GPL(crypto_engine_stop);
*/
struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
bool retry_support,
- int (*cbk_do_batch)(struct crypto_engine *engine),
bool rt, int qlen)
{
struct crypto_engine *engine;
@@ -492,14 +447,8 @@ struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
engine->rt = rt;
engine->running = false;
engine->busy = false;
- engine->idling = false;
engine->retry_support = retry_support;
engine->priv_data = dev;
- /*
- * Batch requests is possible only if
- * hardware has support for retry mechanism.
- */
- engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
snprintf(engine->name, sizeof(engine->name),
"%s-engine", dev_name(dev));
@@ -534,7 +483,7 @@ EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
*/
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
{
- return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
+ return crypto_engine_alloc_init_and_set(dev, false, rt,
CRYPTO_ENGINE_MAX_QLEN);
}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
diff --git a/crypto/deflate.c b/crypto/deflate.c
index fe8e4ad0fee1..21404515dc77 100644
--- a/crypto/deflate.c
+++ b/crypto/deflate.c
@@ -48,9 +48,14 @@ static void *deflate_alloc_stream(void)
return ctx;
}
+static void deflate_free_stream(void *ctx)
+{
+ kvfree(ctx);
+}
+
static struct crypto_acomp_streams deflate_streams = {
.alloc_ctx = deflate_alloc_stream,
- .cfree_ctx = kvfree,
+ .free_ctx = deflate_free_stream,
};
static int deflate_compress_one(struct acomp_req *req,
diff --git a/crypto/hash_info.c b/crypto/hash_info.c
deleted file mode 100644
index 9a467638c971..000000000000
--- a/crypto/hash_info.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Hash Info: Hash algorithms information
- *
- * Copyright (c) 2013 Dmitry Kasatkin <d.kasatkin@samsung.com>
- */
-
-#include <linux/export.h>
-#include <crypto/hash_info.h>
-
-const char *const hash_algo_name[HASH_ALGO__LAST] = {
- [HASH_ALGO_MD4] = "md4",
- [HASH_ALGO_MD5] = "md5",
- [HASH_ALGO_SHA1] = "sha1",
- [HASH_ALGO_RIPE_MD_160] = "rmd160",
- [HASH_ALGO_SHA256] = "sha256",
- [HASH_ALGO_SHA384] = "sha384",
- [HASH_ALGO_SHA512] = "sha512",
- [HASH_ALGO_SHA224] = "sha224",
- [HASH_ALGO_RIPE_MD_128] = "rmd128",
- [HASH_ALGO_RIPE_MD_256] = "rmd256",
- [HASH_ALGO_RIPE_MD_320] = "rmd320",
- [HASH_ALGO_WP_256] = "wp256",
- [HASH_ALGO_WP_384] = "wp384",
- [HASH_ALGO_WP_512] = "wp512",
- [HASH_ALGO_TGR_128] = "tgr128",
- [HASH_ALGO_TGR_160] = "tgr160",
- [HASH_ALGO_TGR_192] = "tgr192",
- [HASH_ALGO_SM3_256] = "sm3",
- [HASH_ALGO_STREEBOG_256] = "streebog256",
- [HASH_ALGO_STREEBOG_512] = "streebog512",
- [HASH_ALGO_SHA3_256] = "sha3-256",
- [HASH_ALGO_SHA3_384] = "sha3-384",
- [HASH_ALGO_SHA3_512] = "sha3-512",
-};
-EXPORT_SYMBOL_GPL(hash_algo_name);
-
-const int hash_digest_size[HASH_ALGO__LAST] = {
- [HASH_ALGO_MD4] = MD5_DIGEST_SIZE,
- [HASH_ALGO_MD5] = MD5_DIGEST_SIZE,
- [HASH_ALGO_SHA1] = SHA1_DIGEST_SIZE,
- [HASH_ALGO_RIPE_MD_160] = RMD160_DIGEST_SIZE,
- [HASH_ALGO_SHA256] = SHA256_DIGEST_SIZE,
- [HASH_ALGO_SHA384] = SHA384_DIGEST_SIZE,
- [HASH_ALGO_SHA512] = SHA512_DIGEST_SIZE,
- [HASH_ALGO_SHA224] = SHA224_DIGEST_SIZE,
- [HASH_ALGO_RIPE_MD_128] = RMD128_DIGEST_SIZE,
- [HASH_ALGO_RIPE_MD_256] = RMD256_DIGEST_SIZE,
- [HASH_ALGO_RIPE_MD_320] = RMD320_DIGEST_SIZE,
- [HASH_ALGO_WP_256] = WP256_DIGEST_SIZE,
- [HASH_ALGO_WP_384] = WP384_DIGEST_SIZE,
- [HASH_ALGO_WP_512] = WP512_DIGEST_SIZE,
- [HASH_ALGO_TGR_128] = TGR128_DIGEST_SIZE,
- [HASH_ALGO_TGR_160] = TGR160_DIGEST_SIZE,
- [HASH_ALGO_TGR_192] = TGR192_DIGEST_SIZE,
- [HASH_ALGO_SM3_256] = SM3256_DIGEST_SIZE,
- [HASH_ALGO_STREEBOG_256] = STREEBOG256_DIGEST_SIZE,
- [HASH_ALGO_STREEBOG_512] = STREEBOG512_DIGEST_SIZE,
- [HASH_ALGO_SHA3_256] = SHA3_256_DIGEST_SIZE,
- [HASH_ALGO_SHA3_384] = SHA3_384_DIGEST_SIZE,
- [HASH_ALGO_SHA3_512] = SHA3_512_DIGEST_SIZE,
-};
-EXPORT_SYMBOL_GPL(hash_digest_size);
diff --git a/crypto/hkdf.c b/crypto/hkdf.c
index f24c2a8d4df9..82d1b32ca6ce 100644
--- a/crypto/hkdf.c
+++ b/crypto/hkdf.c
@@ -566,7 +566,7 @@ static int __init crypto_hkdf_module_init(void)
static void __exit crypto_hkdf_module_exit(void) {}
-module_init(crypto_hkdf_module_init);
+late_initcall(crypto_hkdf_module_init);
module_exit(crypto_hkdf_module_exit);
MODULE_LICENSE("GPL");
diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c
index c24d4ff2b4a8..1266eb790708 100644
--- a/crypto/jitterentropy-kcapi.c
+++ b/crypto/jitterentropy-kcapi.c
@@ -144,7 +144,7 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl,
* Inject the data from the previous loop into the pool. This data is
* not considered to contain any entropy, but it stirs the pool a bit.
*/
- ret = crypto_shash_update(desc, intermediary, sizeof(intermediary));
+ ret = crypto_shash_update(hash_state_desc, intermediary, sizeof(intermediary));
if (ret)
goto err;
@@ -157,11 +157,12 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl,
* conditioning operation to have an identical amount of input data
* according to section 3.1.5.
*/
- if (!stuck) {
- ret = crypto_shash_update(hash_state_desc, (u8 *)&time,
- sizeof(__u64));
+ if (stuck) {
+ time = 0;
}
+ ret = crypto_shash_update(hash_state_desc, (u8 *)&time, sizeof(__u64));
+
err:
shash_desc_zero(desc);
memzero_explicit(intermediary, sizeof(intermediary));
diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c
index 3b390bd6c119..3f93cdc9a7af 100644
--- a/crypto/jitterentropy.c
+++ b/crypto/jitterentropy.c
@@ -145,6 +145,7 @@ struct rand_data {
*/
#define JENT_ENTROPY_SAFETY_FACTOR 64
+#include <linux/array_size.h>
#include <linux/fips.h>
#include <linux/minmax.h>
#include "jitterentropy.h"
@@ -178,7 +179,6 @@ static const unsigned int jent_apt_cutoff_lookup[15] = {
static const unsigned int jent_apt_cutoff_permanent_lookup[15] = {
355, 447, 479, 494, 502, 507, 510, 512,
512, 512, 512, 512, 512, 512, 512 };
-#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
static void jent_apt_init(struct rand_data *ec, unsigned int osr)
{
diff --git a/crypto/krb5/selftest.c b/crypto/krb5/selftest.c
index 2a81a6315a0d..4519c572d37e 100644
--- a/crypto/krb5/selftest.c
+++ b/crypto/krb5/selftest.c
@@ -152,6 +152,7 @@ static int krb5_test_one_prf(const struct krb5_prf_test *test)
out:
clear_buf(&result);
+ clear_buf(&prf);
clear_buf(&octet);
clear_buf(&key);
return ret;
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
index c33d29a523e0..c3a9d4f2995c 100644
--- a/crypto/pcrypt.c
+++ b/crypto/pcrypt.c
@@ -178,7 +178,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req)
static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
{
- int cpu, cpu_index;
+ int cpu_index;
struct aead_instance *inst = aead_alg_instance(tfm);
struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst);
struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
@@ -187,10 +187,7 @@ static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) %
cpumask_weight(cpu_online_mask);
- ctx->cb_cpu = cpumask_first(cpu_online_mask);
- for (cpu = 0; cpu < cpu_index; cpu++)
- ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
-
+ ctx->cb_cpu = cpumask_nth(cpu_index, cpu_online_mask);
cipher = crypto_spawn_aead(&ictx->spawn);
if (IS_ERR(cipher))
diff --git a/crypto/sha1.c b/crypto/sha1.c
new file mode 100644
index 000000000000..ecef4bf2d9c0
--- /dev/null
+++ b/crypto/sha1.c
@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Crypto API support for SHA-1 and HMAC-SHA1
+ *
+ * Copyright (c) Alan Smithee.
+ * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
+ * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/internal/hash.h>
+#include <crypto/sha1.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+/*
+ * Export and import functions. crypto_shash wants a particular format that
+ * matches that used by some legacy drivers. It currently is the same as the
+ * library SHA context, except the value in bytecount must be block-aligned and
+ * the remainder must be stored in an extra u8 appended to the struct.
+ */
+
+#define SHA1_SHASH_STATE_SIZE (sizeof(struct sha1_ctx) + 1)
+static_assert(sizeof(struct sha1_ctx) == sizeof(struct sha1_state));
+static_assert(offsetof(struct sha1_ctx, state) == offsetof(struct sha1_state, state));
+static_assert(offsetof(struct sha1_ctx, bytecount) == offsetof(struct sha1_state, count));
+static_assert(offsetof(struct sha1_ctx, buf) == offsetof(struct sha1_state, buffer));
+
+static int __crypto_sha1_export(const struct sha1_ctx *ctx0, void *out)
+{
+ struct sha1_ctx ctx = *ctx0;
+ unsigned int partial;
+ u8 *p = out;
+
+ partial = ctx.bytecount % SHA1_BLOCK_SIZE;
+ ctx.bytecount -= partial;
+ memcpy(p, &ctx, sizeof(ctx));
+ p += sizeof(ctx);
+ *p = partial;
+ return 0;
+}
+
+static int __crypto_sha1_import(struct sha1_ctx *ctx, const void *in)
+{
+ const u8 *p = in;
+
+ memcpy(ctx, p, sizeof(*ctx));
+ p += sizeof(*ctx);
+ ctx->bytecount += *p;
+ return 0;
+}
+
+const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = {
+ 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
+ 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
+ 0xaf, 0xd8, 0x07, 0x09
+};
+EXPORT_SYMBOL_GPL(sha1_zero_message_hash);
+
+#define SHA1_CTX(desc) ((struct sha1_ctx *)shash_desc_ctx(desc))
+
+static int crypto_sha1_init(struct shash_desc *desc)
+{
+ sha1_init(SHA1_CTX(desc));
+ return 0;
+}
+
+static int crypto_sha1_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ sha1_update(SHA1_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_sha1_final(struct shash_desc *desc, u8 *out)
+{
+ sha1_final(SHA1_CTX(desc), out);
+ return 0;
+}
+
+static int crypto_sha1_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len, u8 *out)
+{
+ sha1(data, len, out);
+ return 0;
+}
+
+static int crypto_sha1_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha1_export(SHA1_CTX(desc), out);
+}
+
+static int crypto_sha1_import(struct shash_desc *desc, const void *in)
+{
+ return __crypto_sha1_import(SHA1_CTX(desc), in);
+}
+
+#define HMAC_SHA1_KEY(tfm) ((struct hmac_sha1_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA1_CTX(desc) ((struct hmac_sha1_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha1_setkey(struct crypto_shash *tfm,
+ const u8 *raw_key, unsigned int keylen)
+{
+ hmac_sha1_preparekey(HMAC_SHA1_KEY(tfm), raw_key, keylen);
+ return 0;
+}
+
+static int crypto_hmac_sha1_init(struct shash_desc *desc)
+{
+ hmac_sha1_init(HMAC_SHA1_CTX(desc), HMAC_SHA1_KEY(desc->tfm));
+ return 0;
+}
+
+static int crypto_hmac_sha1_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ hmac_sha1_update(HMAC_SHA1_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_hmac_sha1_final(struct shash_desc *desc, u8 *out)
+{
+ hmac_sha1_final(HMAC_SHA1_CTX(desc), out);
+ return 0;
+}
+
+static int crypto_hmac_sha1_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len, u8 *out)
+{
+ hmac_sha1(HMAC_SHA1_KEY(desc->tfm), data, len, out);
+ return 0;
+}
+
+static int crypto_hmac_sha1_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha1_export(&HMAC_SHA1_CTX(desc)->sha_ctx, out);
+}
+
+static int crypto_hmac_sha1_import(struct shash_desc *desc, const void *in)
+{
+ struct hmac_sha1_ctx *ctx = HMAC_SHA1_CTX(desc);
+
+ ctx->ostate = HMAC_SHA1_KEY(desc->tfm)->ostate;
+ return __crypto_sha1_import(&ctx->sha_ctx, in);
+}
+
+static struct shash_alg algs[] = {
+ {
+ .base.cra_name = "sha1",
+ .base.cra_driver_name = "sha1-lib",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = SHA1_BLOCK_SIZE,
+ .base.cra_module = THIS_MODULE,
+ .digestsize = SHA1_DIGEST_SIZE,
+ .init = crypto_sha1_init,
+ .update = crypto_sha1_update,
+ .final = crypto_sha1_final,
+ .digest = crypto_sha1_digest,
+ .export = crypto_sha1_export,
+ .import = crypto_sha1_import,
+ .descsize = sizeof(struct sha1_ctx),
+ .statesize = SHA1_SHASH_STATE_SIZE,
+ },
+ {
+ .base.cra_name = "hmac(sha1)",
+ .base.cra_driver_name = "hmac-sha1-lib",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = SHA1_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct hmac_sha1_key),
+ .base.cra_module = THIS_MODULE,
+ .digestsize = SHA1_DIGEST_SIZE,
+ .setkey = crypto_hmac_sha1_setkey,
+ .init = crypto_hmac_sha1_init,
+ .update = crypto_hmac_sha1_update,
+ .final = crypto_hmac_sha1_final,
+ .digest = crypto_hmac_sha1_digest,
+ .export = crypto_hmac_sha1_export,
+ .import = crypto_hmac_sha1_import,
+ .descsize = sizeof(struct hmac_sha1_ctx),
+ .statesize = SHA1_SHASH_STATE_SIZE,
+ },
+};
+
+static int __init crypto_sha1_mod_init(void)
+{
+ return crypto_register_shashes(algs, ARRAY_SIZE(algs));
+}
+module_init(crypto_sha1_mod_init);
+
+static void __exit crypto_sha1_mod_exit(void)
+{
+ crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+}
+module_exit(crypto_sha1_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Crypto API support for SHA-1 and HMAC-SHA1");
+
+MODULE_ALIAS_CRYPTO("sha1");
+MODULE_ALIAS_CRYPTO("sha1-lib");
+MODULE_ALIAS_CRYPTO("hmac(sha1)");
+MODULE_ALIAS_CRYPTO("hmac-sha1-lib");
diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c
deleted file mode 100644
index 024e8043bab0..000000000000
--- a/crypto/sha1_generic.c
+++ /dev/null
@@ -1,87 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Cryptographic API.
- *
- * SHA1 Secure Hash Algorithm.
- *
- * Derived from cryptoapi implementation, adapted for in-place
- * scatterlist interface.
- *
- * Copyright (c) Alan Smithee.
- * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
- * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
- */
-#include <crypto/internal/hash.h>
-#include <crypto/sha1.h>
-#include <crypto/sha1_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/string.h>
-
-const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = {
- 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
- 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
- 0xaf, 0xd8, 0x07, 0x09
-};
-EXPORT_SYMBOL_GPL(sha1_zero_message_hash);
-
-static void sha1_generic_block_fn(struct sha1_state *sst, u8 const *src,
- int blocks)
-{
- u32 temp[SHA1_WORKSPACE_WORDS];
-
- while (blocks--) {
- sha1_transform(sst->state, src, temp);
- src += SHA1_BLOCK_SIZE;
- }
- memzero_explicit(temp, sizeof(temp));
-}
-
-static int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- return sha1_base_do_update_blocks(desc, data, len,
- sha1_generic_block_fn);
-}
-
-static int crypto_sha1_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- sha1_base_do_finup(desc, data, len, sha1_generic_block_fn);
- return sha1_base_finish(desc, out);
-}
-
-static struct shash_alg alg = {
- .digestsize = SHA1_DIGEST_SIZE,
- .init = sha1_base_init,
- .update = crypto_sha1_update,
- .finup = crypto_sha1_finup,
- .descsize = SHA1_STATE_SIZE,
- .base = {
- .cra_name = "sha1",
- .cra_driver_name= "sha1-generic",
- .cra_priority = 100,
- .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
- .cra_blocksize = SHA1_BLOCK_SIZE,
- .cra_module = THIS_MODULE,
- }
-};
-
-static int __init sha1_generic_mod_init(void)
-{
- return crypto_register_shash(&alg);
-}
-
-static void __exit sha1_generic_mod_fini(void)
-{
- crypto_unregister_shash(&alg);
-}
-
-module_init(sha1_generic_mod_init);
-module_exit(sha1_generic_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm");
-
-MODULE_ALIAS_CRYPTO("sha1");
-MODULE_ALIAS_CRYPTO("sha1-generic");
diff --git a/crypto/sha256.c b/crypto/sha256.c
index 4aeb213bab11..052806559f06 100644
--- a/crypto/sha256.c
+++ b/crypto/sha256.c
@@ -1,17 +1,57 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * Crypto API wrapper for the SHA-256 and SHA-224 library functions
+ * Crypto API support for SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256
*
* Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
* Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
* SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com>
+ * Copyright 2025 Google LLC
*/
#include <crypto/internal/hash.h>
-#include <crypto/internal/sha2.h>
+#include <crypto/sha2.h>
#include <linux/kernel.h>
#include <linux/module.h>
+/*
+ * Export and import functions. crypto_shash wants a particular format that
+ * matches that used by some legacy drivers. It currently is the same as the
+ * library SHA context, except the value in bytecount must be block-aligned and
+ * the remainder must be stored in an extra u8 appended to the struct.
+ */
+
+#define SHA256_SHASH_STATE_SIZE 105
+static_assert(offsetof(struct __sha256_ctx, state) == 0);
+static_assert(offsetof(struct __sha256_ctx, bytecount) == 32);
+static_assert(offsetof(struct __sha256_ctx, buf) == 40);
+static_assert(sizeof(struct __sha256_ctx) + 1 == SHA256_SHASH_STATE_SIZE);
+
+static int __crypto_sha256_export(const struct __sha256_ctx *ctx0, void *out)
+{
+ struct __sha256_ctx ctx = *ctx0;
+ unsigned int partial;
+ u8 *p = out;
+
+ partial = ctx.bytecount % SHA256_BLOCK_SIZE;
+ ctx.bytecount -= partial;
+ memcpy(p, &ctx, sizeof(ctx));
+ p += sizeof(ctx);
+ *p = partial;
+ return 0;
+}
+
+static int __crypto_sha256_import(struct __sha256_ctx *ctx, const void *in)
+{
+ const u8 *p = in;
+
+ memcpy(ctx, p, sizeof(*ctx));
+ p += sizeof(*ctx);
+ ctx->bytecount += *p;
+ return 0;
+}
+
+/* SHA-224 */
+
const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = {
0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
@@ -20,6 +60,46 @@ const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = {
};
EXPORT_SYMBOL_GPL(sha224_zero_message_hash);
+#define SHA224_CTX(desc) ((struct sha224_ctx *)shash_desc_ctx(desc))
+
+static int crypto_sha224_init(struct shash_desc *desc)
+{
+ sha224_init(SHA224_CTX(desc));
+ return 0;
+}
+
+static int crypto_sha224_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ sha224_update(SHA224_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_sha224_final(struct shash_desc *desc, u8 *out)
+{
+ sha224_final(SHA224_CTX(desc), out);
+ return 0;
+}
+
+static int crypto_sha224_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len, u8 *out)
+{
+ sha224(data, len, out);
+ return 0;
+}
+
+static int crypto_sha224_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha256_export(&SHA224_CTX(desc)->ctx, out);
+}
+
+static int crypto_sha224_import(struct shash_desc *desc, const void *in)
+{
+ return __crypto_sha256_import(&SHA224_CTX(desc)->ctx, in);
+}
+
+/* SHA-256 */
+
const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = {
0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
@@ -28,256 +108,241 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = {
};
EXPORT_SYMBOL_GPL(sha256_zero_message_hash);
+#define SHA256_CTX(desc) ((struct sha256_ctx *)shash_desc_ctx(desc))
+
static int crypto_sha256_init(struct shash_desc *desc)
{
- sha256_block_init(shash_desc_ctx(desc));
+ sha256_init(SHA256_CTX(desc));
+ return 0;
+}
+
+static int crypto_sha256_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ sha256_update(SHA256_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_sha256_final(struct shash_desc *desc, u8 *out)
+{
+ sha256_final(SHA256_CTX(desc), out);
return 0;
}
-static inline int crypto_sha256_update(struct shash_desc *desc, const u8 *data,
- unsigned int len, bool force_generic)
+static int crypto_sha256_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len, u8 *out)
{
- struct crypto_sha256_state *sctx = shash_desc_ctx(desc);
- int remain = len % SHA256_BLOCK_SIZE;
+ sha256(data, len, out);
+ return 0;
+}
- sctx->count += len - remain;
- sha256_choose_blocks(sctx->state, data, len / SHA256_BLOCK_SIZE,
- force_generic, !force_generic);
- return remain;
+static int crypto_sha256_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha256_export(&SHA256_CTX(desc)->ctx, out);
}
-static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 *data,
- unsigned int len)
+static int crypto_sha256_import(struct shash_desc *desc, const void *in)
{
- return crypto_sha256_update(desc, data, len, true);
+ return __crypto_sha256_import(&SHA256_CTX(desc)->ctx, in);
}
-static int crypto_sha256_update_lib(struct shash_desc *desc, const u8 *data,
- unsigned int len)
+/* HMAC-SHA224 */
+
+#define HMAC_SHA224_KEY(tfm) ((struct hmac_sha224_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA224_CTX(desc) ((struct hmac_sha224_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha224_setkey(struct crypto_shash *tfm,
+ const u8 *raw_key, unsigned int keylen)
{
- sha256_update(shash_desc_ctx(desc), data, len);
+ hmac_sha224_preparekey(HMAC_SHA224_KEY(tfm), raw_key, keylen);
return 0;
}
-static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data,
- unsigned int len)
+static int crypto_hmac_sha224_init(struct shash_desc *desc)
{
- return crypto_sha256_update(desc, data, len, false);
+ hmac_sha224_init(HMAC_SHA224_CTX(desc), HMAC_SHA224_KEY(desc->tfm));
+ return 0;
}
-static int crypto_sha256_final_lib(struct shash_desc *desc, u8 *out)
+static int crypto_hmac_sha224_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
{
- sha256_final(shash_desc_ctx(desc), out);
+ hmac_sha224_update(HMAC_SHA224_CTX(desc), data, len);
return 0;
}
-static __always_inline int crypto_sha256_finup(struct shash_desc *desc,
- const u8 *data,
- unsigned int len, u8 *out,
- bool force_generic)
+static int crypto_hmac_sha224_final(struct shash_desc *desc, u8 *out)
{
- struct crypto_sha256_state *sctx = shash_desc_ctx(desc);
- unsigned int remain = len;
- u8 *buf;
-
- if (len >= SHA256_BLOCK_SIZE)
- remain = crypto_sha256_update(desc, data, len, force_generic);
- sctx->count += remain;
- buf = memcpy(sctx + 1, data + len - remain, remain);
- sha256_finup(sctx, buf, remain, out,
- crypto_shash_digestsize(desc->tfm), force_generic,
- !force_generic);
+ hmac_sha224_final(HMAC_SHA224_CTX(desc), out);
return 0;
}
-static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int crypto_hmac_sha224_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len,
+ u8 *out)
{
- return crypto_sha256_finup(desc, data, len, out, true);
+ hmac_sha224(HMAC_SHA224_KEY(desc->tfm), data, len, out);
+ return 0;
}
-static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int crypto_hmac_sha224_export(struct shash_desc *desc, void *out)
{
- return crypto_sha256_finup(desc, data, len, out, false);
+ return __crypto_sha256_export(&HMAC_SHA224_CTX(desc)->ctx.sha_ctx, out);
}
-static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int crypto_hmac_sha224_import(struct shash_desc *desc, const void *in)
{
- crypto_sha256_init(desc);
- return crypto_sha256_finup_generic(desc, data, len, out);
+ struct hmac_sha224_ctx *ctx = HMAC_SHA224_CTX(desc);
+
+ ctx->ctx.ostate = HMAC_SHA224_KEY(desc->tfm)->key.ostate;
+ return __crypto_sha256_import(&ctx->ctx.sha_ctx, in);
}
-static int crypto_sha256_digest_lib(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+/* HMAC-SHA256 */
+
+#define HMAC_SHA256_KEY(tfm) ((struct hmac_sha256_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA256_CTX(desc) ((struct hmac_sha256_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha256_setkey(struct crypto_shash *tfm,
+ const u8 *raw_key, unsigned int keylen)
{
- sha256(data, len, out);
+ hmac_sha256_preparekey(HMAC_SHA256_KEY(tfm), raw_key, keylen);
return 0;
}
-static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int crypto_hmac_sha256_init(struct shash_desc *desc)
{
- crypto_sha256_init(desc);
- return crypto_sha256_finup_arch(desc, data, len, out);
+ hmac_sha256_init(HMAC_SHA256_CTX(desc), HMAC_SHA256_KEY(desc->tfm));
+ return 0;
}
-static int crypto_sha224_init(struct shash_desc *desc)
+static int crypto_hmac_sha256_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
{
- sha224_block_init(shash_desc_ctx(desc));
+ hmac_sha256_update(HMAC_SHA256_CTX(desc), data, len);
return 0;
}
-static int crypto_sha224_final_lib(struct shash_desc *desc, u8 *out)
+static int crypto_hmac_sha256_final(struct shash_desc *desc, u8 *out)
{
- sha224_final(shash_desc_ctx(desc), out);
+ hmac_sha256_final(HMAC_SHA256_CTX(desc), out);
return 0;
}
-static int crypto_sha256_import_lib(struct shash_desc *desc, const void *in)
+static int crypto_hmac_sha256_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len,
+ u8 *out)
{
- struct sha256_state *sctx = shash_desc_ctx(desc);
- const u8 *p = in;
-
- memcpy(sctx, p, sizeof(*sctx));
- p += sizeof(*sctx);
- sctx->count += *p;
+ hmac_sha256(HMAC_SHA256_KEY(desc->tfm), data, len, out);
return 0;
}
-static int crypto_sha256_export_lib(struct shash_desc *desc, void *out)
+static int crypto_hmac_sha256_export(struct shash_desc *desc, void *out)
{
- struct sha256_state *sctx0 = shash_desc_ctx(desc);
- struct sha256_state sctx = *sctx0;
- unsigned int partial;
- u8 *p = out;
+ return __crypto_sha256_export(&HMAC_SHA256_CTX(desc)->ctx.sha_ctx, out);
+}
- partial = sctx.count % SHA256_BLOCK_SIZE;
- sctx.count -= partial;
- memcpy(p, &sctx, sizeof(sctx));
- p += sizeof(sctx);
- *p = partial;
- return 0;
+static int crypto_hmac_sha256_import(struct shash_desc *desc, const void *in)
+{
+ struct hmac_sha256_ctx *ctx = HMAC_SHA256_CTX(desc);
+
+ ctx->ctx.ostate = HMAC_SHA256_KEY(desc->tfm)->key.ostate;
+ return __crypto_sha256_import(&ctx->ctx.sha_ctx, in);
}
+/* Algorithm definitions */
+
static struct shash_alg algs[] = {
{
- .base.cra_name = "sha256",
- .base.cra_driver_name = "sha256-generic",
- .base.cra_priority = 100,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SHA256_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
- .digestsize = SHA256_DIGEST_SIZE,
- .init = crypto_sha256_init,
- .update = crypto_sha256_update_generic,
- .finup = crypto_sha256_finup_generic,
- .digest = crypto_sha256_digest_generic,
- .descsize = sizeof(struct crypto_sha256_state),
- },
- {
.base.cra_name = "sha224",
- .base.cra_driver_name = "sha224-generic",
- .base.cra_priority = 100,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
+ .base.cra_driver_name = "sha224-lib",
+ .base.cra_priority = 300,
.base.cra_blocksize = SHA224_BLOCK_SIZE,
.base.cra_module = THIS_MODULE,
.digestsize = SHA224_DIGEST_SIZE,
.init = crypto_sha224_init,
- .update = crypto_sha256_update_generic,
- .finup = crypto_sha256_finup_generic,
- .descsize = sizeof(struct crypto_sha256_state),
+ .update = crypto_sha224_update,
+ .final = crypto_sha224_final,
+ .digest = crypto_sha224_digest,
+ .export = crypto_sha224_export,
+ .import = crypto_sha224_import,
+ .descsize = sizeof(struct sha224_ctx),
+ .statesize = SHA256_SHASH_STATE_SIZE,
},
{
.base.cra_name = "sha256",
.base.cra_driver_name = "sha256-lib",
+ .base.cra_priority = 300,
.base.cra_blocksize = SHA256_BLOCK_SIZE,
.base.cra_module = THIS_MODULE,
.digestsize = SHA256_DIGEST_SIZE,
.init = crypto_sha256_init,
- .update = crypto_sha256_update_lib,
- .final = crypto_sha256_final_lib,
- .digest = crypto_sha256_digest_lib,
- .descsize = sizeof(struct sha256_state),
- .statesize = sizeof(struct crypto_sha256_state) +
- SHA256_BLOCK_SIZE + 1,
- .import = crypto_sha256_import_lib,
- .export = crypto_sha256_export_lib,
+ .update = crypto_sha256_update,
+ .final = crypto_sha256_final,
+ .digest = crypto_sha256_digest,
+ .export = crypto_sha256_export,
+ .import = crypto_sha256_import,
+ .descsize = sizeof(struct sha256_ctx),
+ .statesize = SHA256_SHASH_STATE_SIZE,
},
{
- .base.cra_name = "sha224",
- .base.cra_driver_name = "sha224-lib",
+ .base.cra_name = "hmac(sha224)",
+ .base.cra_driver_name = "hmac-sha224-lib",
+ .base.cra_priority = 300,
.base.cra_blocksize = SHA224_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct hmac_sha224_key),
.base.cra_module = THIS_MODULE,
.digestsize = SHA224_DIGEST_SIZE,
- .init = crypto_sha224_init,
- .update = crypto_sha256_update_lib,
- .final = crypto_sha224_final_lib,
- .descsize = sizeof(struct sha256_state),
- .statesize = sizeof(struct crypto_sha256_state) +
- SHA256_BLOCK_SIZE + 1,
- .import = crypto_sha256_import_lib,
- .export = crypto_sha256_export_lib,
+ .setkey = crypto_hmac_sha224_setkey,
+ .init = crypto_hmac_sha224_init,
+ .update = crypto_hmac_sha224_update,
+ .final = crypto_hmac_sha224_final,
+ .digest = crypto_hmac_sha224_digest,
+ .export = crypto_hmac_sha224_export,
+ .import = crypto_hmac_sha224_import,
+ .descsize = sizeof(struct hmac_sha224_ctx),
+ .statesize = SHA256_SHASH_STATE_SIZE,
},
{
- .base.cra_name = "sha256",
- .base.cra_driver_name = "sha256-" __stringify(ARCH),
+ .base.cra_name = "hmac(sha256)",
+ .base.cra_driver_name = "hmac-sha256-lib",
.base.cra_priority = 300,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
.base.cra_blocksize = SHA256_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct hmac_sha256_key),
.base.cra_module = THIS_MODULE,
.digestsize = SHA256_DIGEST_SIZE,
- .init = crypto_sha256_init,
- .update = crypto_sha256_update_arch,
- .finup = crypto_sha256_finup_arch,
- .digest = crypto_sha256_digest_arch,
- .descsize = sizeof(struct crypto_sha256_state),
- },
- {
- .base.cra_name = "sha224",
- .base.cra_driver_name = "sha224-" __stringify(ARCH),
- .base.cra_priority = 300,
- .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .base.cra_blocksize = SHA224_BLOCK_SIZE,
- .base.cra_module = THIS_MODULE,
- .digestsize = SHA224_DIGEST_SIZE,
- .init = crypto_sha224_init,
- .update = crypto_sha256_update_arch,
- .finup = crypto_sha256_finup_arch,
- .descsize = sizeof(struct crypto_sha256_state),
+ .setkey = crypto_hmac_sha256_setkey,
+ .init = crypto_hmac_sha256_init,
+ .update = crypto_hmac_sha256_update,
+ .final = crypto_hmac_sha256_final,
+ .digest = crypto_hmac_sha256_digest,
+ .export = crypto_hmac_sha256_export,
+ .import = crypto_hmac_sha256_import,
+ .descsize = sizeof(struct hmac_sha256_ctx),
+ .statesize = SHA256_SHASH_STATE_SIZE,
},
};
-static unsigned int num_algs;
-
static int __init crypto_sha256_mod_init(void)
{
- /* register the arch flavours only if they differ from generic */
- num_algs = ARRAY_SIZE(algs);
- BUILD_BUG_ON(ARRAY_SIZE(algs) <= 2);
- if (!sha256_is_arch_optimized())
- num_algs -= 2;
return crypto_register_shashes(algs, ARRAY_SIZE(algs));
}
module_init(crypto_sha256_mod_init);
static void __exit crypto_sha256_mod_exit(void)
{
- crypto_unregister_shashes(algs, num_algs);
+ crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
}
module_exit(crypto_sha256_mod_exit);
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Crypto API wrapper for the SHA-256 and SHA-224 library functions");
+MODULE_DESCRIPTION("Crypto API support for SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256");
-MODULE_ALIAS_CRYPTO("sha256");
-MODULE_ALIAS_CRYPTO("sha256-generic");
-MODULE_ALIAS_CRYPTO("sha256-" __stringify(ARCH));
MODULE_ALIAS_CRYPTO("sha224");
-MODULE_ALIAS_CRYPTO("sha224-generic");
-MODULE_ALIAS_CRYPTO("sha224-" __stringify(ARCH));
+MODULE_ALIAS_CRYPTO("sha224-lib");
+MODULE_ALIAS_CRYPTO("sha256");
+MODULE_ALIAS_CRYPTO("sha256-lib");
+MODULE_ALIAS_CRYPTO("hmac(sha224)");
+MODULE_ALIAS_CRYPTO("hmac-sha224-lib");
+MODULE_ALIAS_CRYPTO("hmac(sha256)");
+MODULE_ALIAS_CRYPTO("hmac-sha256-lib");
diff --git a/crypto/sha512.c b/crypto/sha512.c
new file mode 100644
index 000000000000..fb1c520978ef
--- /dev/null
+++ b/crypto/sha512.c
@@ -0,0 +1,354 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Crypto API support for SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512
+ *
+ * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
+ * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
+ * Copyright (c) 2003 Kyle McMartin <kyle@debian.org>
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/internal/hash.h>
+#include <crypto/sha2.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+/*
+ * Export and import functions. crypto_shash wants a particular format that
+ * matches that used by some legacy drivers. It currently is the same as the
+ * library SHA context, except the value in bytecount_lo must be block-aligned
+ * and the remainder must be stored in an extra u8 appended to the struct.
+ */
+
+#define SHA512_SHASH_STATE_SIZE 209
+static_assert(offsetof(struct __sha512_ctx, state) == 0);
+static_assert(offsetof(struct __sha512_ctx, bytecount_lo) == 64);
+static_assert(offsetof(struct __sha512_ctx, bytecount_hi) == 72);
+static_assert(offsetof(struct __sha512_ctx, buf) == 80);
+static_assert(sizeof(struct __sha512_ctx) + 1 == SHA512_SHASH_STATE_SIZE);
+
+static int __crypto_sha512_export(const struct __sha512_ctx *ctx0, void *out)
+{
+ struct __sha512_ctx ctx = *ctx0;
+ unsigned int partial;
+ u8 *p = out;
+
+ partial = ctx.bytecount_lo % SHA512_BLOCK_SIZE;
+ ctx.bytecount_lo -= partial;
+ memcpy(p, &ctx, sizeof(ctx));
+ p += sizeof(ctx);
+ *p = partial;
+ return 0;
+}
+
+static int __crypto_sha512_import(struct __sha512_ctx *ctx, const void *in)
+{
+ const u8 *p = in;
+
+ memcpy(ctx, p, sizeof(*ctx));
+ p += sizeof(*ctx);
+ ctx->bytecount_lo += *p;
+ return 0;
+}
+
+/* SHA-384 */
+
+const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] = {
+ 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38,
+ 0x4c, 0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a,
+ 0x21, 0xfd, 0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43,
+ 0x4c, 0x0c, 0xc7, 0xbf, 0x63, 0xf6, 0xe1, 0xda,
+ 0x27, 0x4e, 0xde, 0xbf, 0xe7, 0x6f, 0x65, 0xfb,
+ 0xd5, 0x1a, 0xd2, 0xf1, 0x48, 0x98, 0xb9, 0x5b
+};
+EXPORT_SYMBOL_GPL(sha384_zero_message_hash);
+
+#define SHA384_CTX(desc) ((struct sha384_ctx *)shash_desc_ctx(desc))
+
+static int crypto_sha384_init(struct shash_desc *desc)
+{
+ sha384_init(SHA384_CTX(desc));
+ return 0;
+}
+
+static int crypto_sha384_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ sha384_update(SHA384_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_sha384_final(struct shash_desc *desc, u8 *out)
+{
+ sha384_final(SHA384_CTX(desc), out);
+ return 0;
+}
+
+static int crypto_sha384_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len, u8 *out)
+{
+ sha384(data, len, out);
+ return 0;
+}
+
+static int crypto_sha384_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha512_export(&SHA384_CTX(desc)->ctx, out);
+}
+
+static int crypto_sha384_import(struct shash_desc *desc, const void *in)
+{
+ return __crypto_sha512_import(&SHA384_CTX(desc)->ctx, in);
+}
+
+/* SHA-512 */
+
+const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] = {
+ 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd,
+ 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07,
+ 0xd6, 0x20, 0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc,
+ 0x83, 0xf4, 0xa9, 0x21, 0xd3, 0x6c, 0xe9, 0xce,
+ 0x47, 0xd0, 0xd1, 0x3c, 0x5d, 0x85, 0xf2, 0xb0,
+ 0xff, 0x83, 0x18, 0xd2, 0x87, 0x7e, 0xec, 0x2f,
+ 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41, 0x7a, 0x81,
+ 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda, 0x3e
+};
+EXPORT_SYMBOL_GPL(sha512_zero_message_hash);
+
+#define SHA512_CTX(desc) ((struct sha512_ctx *)shash_desc_ctx(desc))
+
+static int crypto_sha512_init(struct shash_desc *desc)
+{
+ sha512_init(SHA512_CTX(desc));
+ return 0;
+}
+
+static int crypto_sha512_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ sha512_update(SHA512_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_sha512_final(struct shash_desc *desc, u8 *out)
+{
+ sha512_final(SHA512_CTX(desc), out);
+ return 0;
+}
+
+static int crypto_sha512_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len, u8 *out)
+{
+ sha512(data, len, out);
+ return 0;
+}
+
+static int crypto_sha512_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha512_export(&SHA512_CTX(desc)->ctx, out);
+}
+
+static int crypto_sha512_import(struct shash_desc *desc, const void *in)
+{
+ return __crypto_sha512_import(&SHA512_CTX(desc)->ctx, in);
+}
+
+/* HMAC-SHA384 */
+
+#define HMAC_SHA384_KEY(tfm) ((struct hmac_sha384_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA384_CTX(desc) ((struct hmac_sha384_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha384_setkey(struct crypto_shash *tfm,
+ const u8 *raw_key, unsigned int keylen)
+{
+ hmac_sha384_preparekey(HMAC_SHA384_KEY(tfm), raw_key, keylen);
+ return 0;
+}
+
+static int crypto_hmac_sha384_init(struct shash_desc *desc)
+{
+ hmac_sha384_init(HMAC_SHA384_CTX(desc), HMAC_SHA384_KEY(desc->tfm));
+ return 0;
+}
+
+static int crypto_hmac_sha384_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ hmac_sha384_update(HMAC_SHA384_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_hmac_sha384_final(struct shash_desc *desc, u8 *out)
+{
+ hmac_sha384_final(HMAC_SHA384_CTX(desc), out);
+ return 0;
+}
+
+static int crypto_hmac_sha384_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len,
+ u8 *out)
+{
+ hmac_sha384(HMAC_SHA384_KEY(desc->tfm), data, len, out);
+ return 0;
+}
+
+static int crypto_hmac_sha384_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha512_export(&HMAC_SHA384_CTX(desc)->ctx.sha_ctx, out);
+}
+
+static int crypto_hmac_sha384_import(struct shash_desc *desc, const void *in)
+{
+ struct hmac_sha384_ctx *ctx = HMAC_SHA384_CTX(desc);
+
+ ctx->ctx.ostate = HMAC_SHA384_KEY(desc->tfm)->key.ostate;
+ return __crypto_sha512_import(&ctx->ctx.sha_ctx, in);
+}
+
+/* HMAC-SHA512 */
+
+#define HMAC_SHA512_KEY(tfm) ((struct hmac_sha512_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA512_CTX(desc) ((struct hmac_sha512_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha512_setkey(struct crypto_shash *tfm,
+ const u8 *raw_key, unsigned int keylen)
+{
+ hmac_sha512_preparekey(HMAC_SHA512_KEY(tfm), raw_key, keylen);
+ return 0;
+}
+
+static int crypto_hmac_sha512_init(struct shash_desc *desc)
+{
+ hmac_sha512_init(HMAC_SHA512_CTX(desc), HMAC_SHA512_KEY(desc->tfm));
+ return 0;
+}
+
+static int crypto_hmac_sha512_update(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ hmac_sha512_update(HMAC_SHA512_CTX(desc), data, len);
+ return 0;
+}
+
+static int crypto_hmac_sha512_final(struct shash_desc *desc, u8 *out)
+{
+ hmac_sha512_final(HMAC_SHA512_CTX(desc), out);
+ return 0;
+}
+
+static int crypto_hmac_sha512_digest(struct shash_desc *desc,
+ const u8 *data, unsigned int len,
+ u8 *out)
+{
+ hmac_sha512(HMAC_SHA512_KEY(desc->tfm), data, len, out);
+ return 0;
+}
+
+static int crypto_hmac_sha512_export(struct shash_desc *desc, void *out)
+{
+ return __crypto_sha512_export(&HMAC_SHA512_CTX(desc)->ctx.sha_ctx, out);
+}
+
+static int crypto_hmac_sha512_import(struct shash_desc *desc, const void *in)
+{
+ struct hmac_sha512_ctx *ctx = HMAC_SHA512_CTX(desc);
+
+ ctx->ctx.ostate = HMAC_SHA512_KEY(desc->tfm)->key.ostate;
+ return __crypto_sha512_import(&ctx->ctx.sha_ctx, in);
+}
+
+/* Algorithm definitions */
+
+static struct shash_alg algs[] = {
+ {
+ .base.cra_name = "sha384",
+ .base.cra_driver_name = "sha384-lib",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = SHA384_BLOCK_SIZE,
+ .base.cra_module = THIS_MODULE,
+ .digestsize = SHA384_DIGEST_SIZE,
+ .init = crypto_sha384_init,
+ .update = crypto_sha384_update,
+ .final = crypto_sha384_final,
+ .digest = crypto_sha384_digest,
+ .export = crypto_sha384_export,
+ .import = crypto_sha384_import,
+ .descsize = sizeof(struct sha384_ctx),
+ .statesize = SHA512_SHASH_STATE_SIZE,
+ },
+ {
+ .base.cra_name = "sha512",
+ .base.cra_driver_name = "sha512-lib",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = SHA512_BLOCK_SIZE,
+ .base.cra_module = THIS_MODULE,
+ .digestsize = SHA512_DIGEST_SIZE,
+ .init = crypto_sha512_init,
+ .update = crypto_sha512_update,
+ .final = crypto_sha512_final,
+ .digest = crypto_sha512_digest,
+ .export = crypto_sha512_export,
+ .import = crypto_sha512_import,
+ .descsize = sizeof(struct sha512_ctx),
+ .statesize = SHA512_SHASH_STATE_SIZE,
+ },
+ {
+ .base.cra_name = "hmac(sha384)",
+ .base.cra_driver_name = "hmac-sha384-lib",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = SHA384_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct hmac_sha384_key),
+ .base.cra_module = THIS_MODULE,
+ .digestsize = SHA384_DIGEST_SIZE,
+ .setkey = crypto_hmac_sha384_setkey,
+ .init = crypto_hmac_sha384_init,
+ .update = crypto_hmac_sha384_update,
+ .final = crypto_hmac_sha384_final,
+ .digest = crypto_hmac_sha384_digest,
+ .export = crypto_hmac_sha384_export,
+ .import = crypto_hmac_sha384_import,
+ .descsize = sizeof(struct hmac_sha384_ctx),
+ .statesize = SHA512_SHASH_STATE_SIZE,
+ },
+ {
+ .base.cra_name = "hmac(sha512)",
+ .base.cra_driver_name = "hmac-sha512-lib",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = SHA512_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct hmac_sha512_key),
+ .base.cra_module = THIS_MODULE,
+ .digestsize = SHA512_DIGEST_SIZE,
+ .setkey = crypto_hmac_sha512_setkey,
+ .init = crypto_hmac_sha512_init,
+ .update = crypto_hmac_sha512_update,
+ .final = crypto_hmac_sha512_final,
+ .digest = crypto_hmac_sha512_digest,
+ .export = crypto_hmac_sha512_export,
+ .import = crypto_hmac_sha512_import,
+ .descsize = sizeof(struct hmac_sha512_ctx),
+ .statesize = SHA512_SHASH_STATE_SIZE,
+ },
+};
+
+static int __init crypto_sha512_mod_init(void)
+{
+ return crypto_register_shashes(algs, ARRAY_SIZE(algs));
+}
+module_init(crypto_sha512_mod_init);
+
+static void __exit crypto_sha512_mod_exit(void)
+{
+ crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+}
+module_exit(crypto_sha512_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Crypto API support for SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512");
+
+MODULE_ALIAS_CRYPTO("sha384");
+MODULE_ALIAS_CRYPTO("sha384-lib");
+MODULE_ALIAS_CRYPTO("sha512");
+MODULE_ALIAS_CRYPTO("sha512-lib");
+MODULE_ALIAS_CRYPTO("hmac(sha384)");
+MODULE_ALIAS_CRYPTO("hmac-sha384-lib");
+MODULE_ALIAS_CRYPTO("hmac(sha512)");
+MODULE_ALIAS_CRYPTO("hmac-sha512-lib");
diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c
deleted file mode 100644
index 7368173f545e..000000000000
--- a/crypto/sha512_generic.c
+++ /dev/null
@@ -1,217 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* SHA-512 code by Jean-Luc Cooke <jlcooke@certainkey.com>
- *
- * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
- * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
- * Copyright (c) 2003 Kyle McMartin <kyle@debian.org>
- */
-#include <crypto/internal/hash.h>
-#include <crypto/sha2.h>
-#include <crypto/sha512_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/unaligned.h>
-
-const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] = {
- 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38,
- 0x4c, 0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a,
- 0x21, 0xfd, 0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43,
- 0x4c, 0x0c, 0xc7, 0xbf, 0x63, 0xf6, 0xe1, 0xda,
- 0x27, 0x4e, 0xde, 0xbf, 0xe7, 0x6f, 0x65, 0xfb,
- 0xd5, 0x1a, 0xd2, 0xf1, 0x48, 0x98, 0xb9, 0x5b
-};
-EXPORT_SYMBOL_GPL(sha384_zero_message_hash);
-
-const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] = {
- 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd,
- 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07,
- 0xd6, 0x20, 0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc,
- 0x83, 0xf4, 0xa9, 0x21, 0xd3, 0x6c, 0xe9, 0xce,
- 0x47, 0xd0, 0xd1, 0x3c, 0x5d, 0x85, 0xf2, 0xb0,
- 0xff, 0x83, 0x18, 0xd2, 0x87, 0x7e, 0xec, 0x2f,
- 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41, 0x7a, 0x81,
- 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda, 0x3e
-};
-EXPORT_SYMBOL_GPL(sha512_zero_message_hash);
-
-static inline u64 Ch(u64 x, u64 y, u64 z)
-{
- return z ^ (x & (y ^ z));
-}
-
-static inline u64 Maj(u64 x, u64 y, u64 z)
-{
- return (x & y) | (z & (x | y));
-}
-
-static const u64 sha512_K[80] = {
- 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL,
- 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL,
- 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL,
- 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL,
- 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL,
- 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL,
- 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL,
- 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL,
- 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL,
- 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL,
- 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL,
- 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL,
- 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL,
- 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL,
- 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL,
- 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL,
- 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL,
- 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL,
- 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL,
- 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL,
- 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL,
- 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL,
- 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL,
- 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL,
- 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL,
- 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
- 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL,
-};
-
-#define e0(x) (ror64(x,28) ^ ror64(x,34) ^ ror64(x,39))
-#define e1(x) (ror64(x,14) ^ ror64(x,18) ^ ror64(x,41))
-#define s0(x) (ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7))
-#define s1(x) (ror64(x,19) ^ ror64(x,61) ^ (x >> 6))
-
-static inline void LOAD_OP(int I, u64 *W, const u8 *input)
-{
- W[I] = get_unaligned_be64((__u64 *)input + I);
-}
-
-static inline void BLEND_OP(int I, u64 *W)
-{
- W[I & 15] += s1(W[(I-2) & 15]) + W[(I-7) & 15] + s0(W[(I-15) & 15]);
-}
-
-static void
-sha512_transform(u64 *state, const u8 *input)
-{
- u64 a, b, c, d, e, f, g, h, t1, t2;
-
- int i;
- u64 W[16];
-
- /* load the state into our registers */
- a=state[0]; b=state[1]; c=state[2]; d=state[3];
- e=state[4]; f=state[5]; g=state[6]; h=state[7];
-
- /* now iterate */
- for (i=0; i<80; i+=8) {
- if (!(i & 8)) {
- int j;
-
- if (i < 16) {
- /* load the input */
- for (j = 0; j < 16; j++)
- LOAD_OP(i + j, W, input);
- } else {
- for (j = 0; j < 16; j++) {
- BLEND_OP(i + j, W);
- }
- }
- }
-
- t1 = h + e1(e) + Ch(e,f,g) + sha512_K[i ] + W[(i & 15)];
- t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
- t1 = g + e1(d) + Ch(d,e,f) + sha512_K[i+1] + W[(i & 15) + 1];
- t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
- t1 = f + e1(c) + Ch(c,d,e) + sha512_K[i+2] + W[(i & 15) + 2];
- t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
- t1 = e + e1(b) + Ch(b,c,d) + sha512_K[i+3] + W[(i & 15) + 3];
- t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
- t1 = d + e1(a) + Ch(a,b,c) + sha512_K[i+4] + W[(i & 15) + 4];
- t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
- t1 = c + e1(h) + Ch(h,a,b) + sha512_K[i+5] + W[(i & 15) + 5];
- t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
- t1 = b + e1(g) + Ch(g,h,a) + sha512_K[i+6] + W[(i & 15) + 6];
- t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
- t1 = a + e1(f) + Ch(f,g,h) + sha512_K[i+7] + W[(i & 15) + 7];
- t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
- }
-
- state[0] += a; state[1] += b; state[2] += c; state[3] += d;
- state[4] += e; state[5] += f; state[6] += g; state[7] += h;
-}
-
-void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src,
- int blocks)
-{
- do {
- sha512_transform(sst->state, src);
- src += SHA512_BLOCK_SIZE;
- } while (--blocks);
-}
-EXPORT_SYMBOL_GPL(sha512_generic_block_fn);
-
-static int crypto_sha512_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- return sha512_base_do_update_blocks(desc, data, len,
- sha512_generic_block_fn);
-}
-
-static int crypto_sha512_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *hash)
-{
- sha512_base_do_finup(desc, data, len, sha512_generic_block_fn);
- return sha512_base_finish(desc, hash);
-}
-
-static struct shash_alg sha512_algs[2] = { {
- .digestsize = SHA512_DIGEST_SIZE,
- .init = sha512_base_init,
- .update = crypto_sha512_update,
- .finup = crypto_sha512_finup,
- .descsize = SHA512_STATE_SIZE,
- .base = {
- .cra_name = "sha512",
- .cra_driver_name = "sha512-generic",
- .cra_priority = 100,
- .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .cra_blocksize = SHA512_BLOCK_SIZE,
- .cra_module = THIS_MODULE,
- }
-}, {
- .digestsize = SHA384_DIGEST_SIZE,
- .init = sha384_base_init,
- .update = crypto_sha512_update,
- .finup = crypto_sha512_finup,
- .descsize = SHA512_STATE_SIZE,
- .base = {
- .cra_name = "sha384",
- .cra_driver_name = "sha384-generic",
- .cra_priority = 100,
- .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .cra_blocksize = SHA384_BLOCK_SIZE,
- .cra_module = THIS_MODULE,
- }
-} };
-
-static int __init sha512_generic_mod_init(void)
-{
- return crypto_register_shashes(sha512_algs, ARRAY_SIZE(sha512_algs));
-}
-
-static void __exit sha512_generic_mod_fini(void)
-{
- crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs));
-}
-
-module_init(sha512_generic_mod_init);
-module_exit(sha512_generic_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA-512 and SHA-384 Secure Hash Algorithms");
-
-MODULE_ALIAS_CRYPTO("sha384");
-MODULE_ALIAS_CRYPTO("sha384-generic");
-MODULE_ALIAS_CRYPTO("sha512");
-MODULE_ALIAS_CRYPTO("sha512-generic");
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 72005074a5c2..ee33ba21ae2b 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -45,6 +45,7 @@ static bool notests;
module_param(notests, bool, 0644);
MODULE_PARM_DESC(notests, "disable all crypto self-tests");
+#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
static bool noslowtests;
module_param(noslowtests, bool, 0644);
MODULE_PARM_DESC(noslowtests, "disable slow crypto self-tests");
@@ -52,6 +53,10 @@ MODULE_PARM_DESC(noslowtests, "disable slow crypto self-tests");
static unsigned int fuzz_iterations = 100;
module_param(fuzz_iterations, uint, 0644);
MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
+#else
+#define noslowtests 1
+#define fuzz_iterations 0
+#endif
#ifndef CONFIG_CRYPTO_SELFTESTS
@@ -319,9 +324,9 @@ struct testvec_config {
/*
* The following are the lists of testvec_configs to test for each algorithm
- * type when the fast crypto self-tests are enabled. They aim to provide good
- * test coverage, while keeping the test time much shorter than the full tests
- * so that the fast tests can be used to fulfill FIPS 140 testing requirements.
+ * type when the "fast" crypto self-tests are enabled. They aim to provide good
+ * test coverage, while keeping the test time much shorter than the "full" tests
+ * so that the "fast" tests can be enabled in a wider range of circumstances.
*/
/* Configs for skciphers and aeads */
@@ -1183,14 +1188,18 @@ static void generate_random_testvec_config(struct rnd_state *rng,
static void crypto_disable_simd_for_test(void)
{
+#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
migrate_disable();
__this_cpu_write(crypto_simd_disabled_for_test, true);
+#endif
}
static void crypto_reenable_simd_for_test(void)
{
+#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
__this_cpu_write(crypto_simd_disabled_for_test, false);
migrate_enable();
+#endif
}
/*
@@ -3541,59 +3550,6 @@ static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
return err;
}
-static int alg_test_crc32c(const struct alg_test_desc *desc,
- const char *driver, u32 type, u32 mask)
-{
- struct crypto_shash *tfm;
- __le32 val;
- int err;
-
- err = alg_test_hash(desc, driver, type, mask);
- if (err)
- return err;
-
- tfm = crypto_alloc_shash(driver, type, mask);
- if (IS_ERR(tfm)) {
- if (PTR_ERR(tfm) == -ENOENT) {
- /*
- * This crc32c implementation is only available through
- * ahash API, not the shash API, so the remaining part
- * of the test is not applicable to it.
- */
- return 0;
- }
- printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "
- "%ld\n", driver, PTR_ERR(tfm));
- return PTR_ERR(tfm);
- }
- driver = crypto_shash_driver_name(tfm);
-
- do {
- SHASH_DESC_ON_STACK(shash, tfm);
- u32 *ctx = (u32 *)shash_desc_ctx(shash);
-
- shash->tfm = tfm;
-
- *ctx = 420553207;
- err = crypto_shash_final(shash, (u8 *)&val);
- if (err) {
- printk(KERN_ERR "alg: crc32c: Operation failed for "
- "%s: %d\n", driver, err);
- break;
- }
-
- if (val != cpu_to_le32(~420553207)) {
- pr_err("alg: crc32c: Test failed for %s: %u\n",
- driver, le32_to_cpu(val));
- err = -EINVAL;
- }
- } while (0);
-
- crypto_free_shash(tfm);
-
- return err;
-}
-
static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,
u32 type, u32 mask)
{
@@ -4228,19 +4184,21 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "authenc(hmac(sha1),cbc(aes))",
+ .generic_driver = "authenc(hmac-sha1-lib,cbc(aes-generic))",
.test = alg_test_aead,
- .fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
}
}, {
.alg = "authenc(hmac(sha1),cbc(des))",
+ .generic_driver = "authenc(hmac-sha1-lib,cbc(des-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha1_des_cbc_tv_temp)
}
}, {
.alg = "authenc(hmac(sha1),cbc(des3_ede))",
+ .generic_driver = "authenc(hmac-sha1-lib,cbc(des3_ede-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
@@ -4248,9 +4206,9 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha1),ctr(aes))",
.test = alg_test_null,
- .fips_allowed = 1,
}, {
.alg = "authenc(hmac(sha1),ecb(cipher_null))",
+ .generic_driver = "authenc(hmac-sha1-lib,ecb-cipher_null)",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
@@ -4258,21 +4216,23 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
.test = alg_test_null,
- .fips_allowed = 1,
}, {
.alg = "authenc(hmac(sha224),cbc(des))",
+ .generic_driver = "authenc(hmac-sha224-lib,cbc(des-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha224_des_cbc_tv_temp)
}
}, {
.alg = "authenc(hmac(sha224),cbc(des3_ede))",
+ .generic_driver = "authenc(hmac-sha224-lib,cbc(des3_ede-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
}
}, {
.alg = "authenc(hmac(sha256),cbc(aes))",
+ .generic_driver = "authenc(hmac-sha256-lib,cbc(aes-generic))",
.test = alg_test_aead,
.fips_allowed = 1,
.suite = {
@@ -4280,12 +4240,14 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "authenc(hmac(sha256),cbc(des))",
+ .generic_driver = "authenc(hmac-sha256-lib,cbc(des-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha256_des_cbc_tv_temp)
}
}, {
.alg = "authenc(hmac(sha256),cbc(des3_ede))",
+ .generic_driver = "authenc(hmac-sha256-lib,cbc(des3_ede-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
@@ -4296,6 +4258,7 @@ static const struct alg_test_desc alg_test_descs[] = {
.fips_allowed = 1,
}, {
.alg = "authenc(hmac(sha256),cts(cbc(aes)))",
+ .generic_driver = "authenc(hmac-sha256-lib,cts(cbc(aes-generic)))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(krb5_test_aes128_cts_hmac_sha256_128)
@@ -4306,12 +4269,14 @@ static const struct alg_test_desc alg_test_descs[] = {
.fips_allowed = 1,
}, {
.alg = "authenc(hmac(sha384),cbc(des))",
+ .generic_driver = "authenc(hmac-sha384-lib,cbc(des-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha384_des_cbc_tv_temp)
}
}, {
.alg = "authenc(hmac(sha384),cbc(des3_ede))",
+ .generic_driver = "authenc(hmac-sha384-lib,cbc(des3_ede-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
@@ -4322,6 +4287,7 @@ static const struct alg_test_desc alg_test_descs[] = {
.fips_allowed = 1,
}, {
.alg = "authenc(hmac(sha384),cts(cbc(aes)))",
+ .generic_driver = "authenc(hmac-sha384-lib,cts(cbc(aes-generic)))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(krb5_test_aes256_cts_hmac_sha384_192)
@@ -4332,6 +4298,7 @@ static const struct alg_test_desc alg_test_descs[] = {
.fips_allowed = 1,
}, {
.alg = "authenc(hmac(sha512),cbc(aes))",
+ .generic_driver = "authenc(hmac-sha512-lib,cbc(aes-generic))",
.fips_allowed = 1,
.test = alg_test_aead,
.suite = {
@@ -4339,12 +4306,14 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "authenc(hmac(sha512),cbc(des))",
+ .generic_driver = "authenc(hmac-sha512-lib,cbc(des-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha512_des_cbc_tv_temp)
}
}, {
.alg = "authenc(hmac(sha512),cbc(des3_ede))",
+ .generic_driver = "authenc(hmac-sha512-lib,cbc(des3_ede-generic))",
.test = alg_test_aead,
.suite = {
.aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
@@ -4546,6 +4515,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "crc32",
+ .generic_driver = "crc32-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -4553,7 +4523,8 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "crc32c",
- .test = alg_test_crc32c,
+ .generic_driver = "crc32c-lib",
+ .test = alg_test_hash,
.fips_allowed = 1,
.suite = {
.hash = __VECS(crc32c_tv_template)
@@ -4738,6 +4709,7 @@ static const struct alg_test_desc alg_test_descs[] = {
*/
.alg = "drbg_nopr_hmac_sha384",
.test = alg_test_null,
+ .fips_allowed = 1
}, {
.alg = "drbg_nopr_hmac_sha512",
.test = alg_test_drbg,
@@ -4756,6 +4728,7 @@ static const struct alg_test_desc alg_test_descs[] = {
/* covered by drbg_nopr_sha256 test */
.alg = "drbg_nopr_sha384",
.test = alg_test_null,
+ .fips_allowed = 1
}, {
.alg = "drbg_nopr_sha512",
.fips_allowed = 1,
@@ -4787,6 +4760,7 @@ static const struct alg_test_desc alg_test_descs[] = {
/* covered by drbg_pr_hmac_sha256 test */
.alg = "drbg_pr_hmac_sha384",
.test = alg_test_null,
+ .fips_allowed = 1
}, {
.alg = "drbg_pr_hmac_sha512",
.test = alg_test_null,
@@ -4802,6 +4776,7 @@ static const struct alg_test_desc alg_test_descs[] = {
/* covered by drbg_pr_sha256 test */
.alg = "drbg_pr_sha384",
.test = alg_test_null,
+ .fips_allowed = 1
}, {
.alg = "drbg_pr_sha512",
.fips_allowed = 1,
@@ -5000,6 +4975,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",
+ .generic_driver = "essiv(authenc(hmac-sha256-lib,cbc(aes-generic)),sha256-lib)",
.test = alg_test_aead,
.fips_allowed = 1,
.suite = {
@@ -5007,6 +4983,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "essiv(cbc(aes),sha256)",
+ .generic_driver = "essiv(cbc(aes-generic),sha256-lib)",
.test = alg_test_skcipher,
.fips_allowed = 1,
.suite = {
@@ -5099,13 +5076,14 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "hmac(sha1)",
+ .generic_driver = "hmac-sha1-lib",
.test = alg_test_hash,
- .fips_allowed = 1,
.suite = {
.hash = __VECS(hmac_sha1_tv_template)
}
}, {
.alg = "hmac(sha224)",
+ .generic_driver = "hmac-sha224-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -5113,6 +5091,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "hmac(sha256)",
+ .generic_driver = "hmac-sha256-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -5148,6 +5127,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "hmac(sha384)",
+ .generic_driver = "hmac-sha384-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -5155,6 +5135,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "hmac(sha512)",
+ .generic_driver = "hmac-sha512-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -5310,6 +5291,36 @@ static const struct alg_test_desc alg_test_descs[] = {
.cipher = __VECS(fcrypt_pcbc_tv_template)
}
}, {
+#if IS_ENABLED(CONFIG_CRYPTO_PHMAC_S390)
+ .alg = "phmac(sha224)",
+ .test = alg_test_hash,
+ .fips_allowed = 1,
+ .suite = {
+ .hash = __VECS(hmac_sha224_tv_template)
+ }
+ }, {
+ .alg = "phmac(sha256)",
+ .test = alg_test_hash,
+ .fips_allowed = 1,
+ .suite = {
+ .hash = __VECS(hmac_sha256_tv_template)
+ }
+ }, {
+ .alg = "phmac(sha384)",
+ .test = alg_test_hash,
+ .fips_allowed = 1,
+ .suite = {
+ .hash = __VECS(hmac_sha384_tv_template)
+ }
+ }, {
+ .alg = "phmac(sha512)",
+ .test = alg_test_hash,
+ .fips_allowed = 1,
+ .suite = {
+ .hash = __VECS(hmac_sha512_tv_template)
+ }
+ }, {
+#endif
.alg = "pkcs1(rsa,none)",
.test = alg_test_sig,
.suite = {
@@ -5435,13 +5446,14 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "sha1",
+ .generic_driver = "sha1-lib",
.test = alg_test_hash,
- .fips_allowed = 1,
.suite = {
.hash = __VECS(sha1_tv_template)
}
}, {
.alg = "sha224",
+ .generic_driver = "sha224-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -5449,6 +5461,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "sha256",
+ .generic_driver = "sha256-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -5484,6 +5497,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "sha384",
+ .generic_driver = "sha384-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@@ -5491,6 +5505,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "sha512",
+ .generic_driver = "sha512-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
diff --git a/crypto/wp512.c b/crypto/wp512.c
index 41f13d490333..229b189a7988 100644
--- a/crypto/wp512.c
+++ b/crypto/wp512.c
@@ -21,10 +21,10 @@
*/
#include <crypto/internal/hash.h>
#include <linux/init.h>
+#include <linux/kernel.h>
#include <linux/module.h>
-#include <linux/mm.h>
-#include <asm/byteorder.h>
-#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
#define WP512_DIGEST_SIZE 64
#define WP384_DIGEST_SIZE 48
@@ -37,9 +37,6 @@
struct wp512_ctx {
u8 bitLength[WP512_LENGTHBYTES];
- u8 buffer[WP512_BLOCK_SIZE];
- int bufferBits;
- int bufferPos;
u64 hash[WP512_DIGEST_SIZE/8];
};
@@ -779,16 +776,16 @@ static const u64 rc[WHIRLPOOL_ROUNDS] = {
* The core Whirlpool transform.
*/
-static __no_kmsan_checks void wp512_process_buffer(struct wp512_ctx *wctx) {
+static __no_kmsan_checks void wp512_process_buffer(struct wp512_ctx *wctx,
+ const u8 *buffer) {
int i, r;
u64 K[8]; /* the round key */
u64 block[8]; /* mu(buffer) */
u64 state[8]; /* the cipher state */
u64 L[8];
- const __be64 *buffer = (const __be64 *)wctx->buffer;
for (i = 0; i < 8; i++)
- block[i] = be64_to_cpu(buffer[i]);
+ block[i] = get_unaligned_be64(buffer + i * 8);
state[0] = block[0] ^ (K[0] = wctx->hash[0]);
state[1] = block[1] ^ (K[1] = wctx->hash[1]);
@@ -991,8 +988,6 @@ static int wp512_init(struct shash_desc *desc) {
int i;
memset(wctx->bitLength, 0, 32);
- wctx->bufferBits = wctx->bufferPos = 0;
- wctx->buffer[0] = 0;
for (i = 0; i < 8; i++) {
wctx->hash[i] = 0L;
}
@@ -1000,84 +995,54 @@ static int wp512_init(struct shash_desc *desc) {
return 0;
}
-static int wp512_update(struct shash_desc *desc, const u8 *source,
- unsigned int len)
+static void wp512_add_length(u8 *bitLength, u64 value)
{
- struct wp512_ctx *wctx = shash_desc_ctx(desc);
- int sourcePos = 0;
- unsigned int bits_len = len * 8; // convert to number of bits
- int sourceGap = (8 - ((int)bits_len & 7)) & 7;
- int bufferRem = wctx->bufferBits & 7;
+ u32 carry;
int i;
- u32 b, carry;
- u8 *buffer = wctx->buffer;
- u8 *bitLength = wctx->bitLength;
- int bufferBits = wctx->bufferBits;
- int bufferPos = wctx->bufferPos;
- u64 value = bits_len;
for (i = 31, carry = 0; i >= 0 && (carry != 0 || value != 0ULL); i--) {
carry += bitLength[i] + ((u32)value & 0xff);
bitLength[i] = (u8)carry;
carry >>= 8;
value >>= 8;
}
- while (bits_len > 8) {
- b = ((source[sourcePos] << sourceGap) & 0xff) |
- ((source[sourcePos + 1] & 0xff) >> (8 - sourceGap));
- buffer[bufferPos++] |= (u8)(b >> bufferRem);
- bufferBits += 8 - bufferRem;
- if (bufferBits == WP512_BLOCK_SIZE * 8) {
- wp512_process_buffer(wctx);
- bufferBits = bufferPos = 0;
- }
- buffer[bufferPos] = b << (8 - bufferRem);
- bufferBits += bufferRem;
- bits_len -= 8;
- sourcePos++;
- }
- if (bits_len > 0) {
- b = (source[sourcePos] << sourceGap) & 0xff;
- buffer[bufferPos] |= b >> bufferRem;
- } else {
- b = 0;
- }
- if (bufferRem + bits_len < 8) {
- bufferBits += bits_len;
- } else {
- bufferPos++;
- bufferBits += 8 - bufferRem;
- bits_len -= 8 - bufferRem;
- if (bufferBits == WP512_BLOCK_SIZE * 8) {
- wp512_process_buffer(wctx);
- bufferBits = bufferPos = 0;
- }
- buffer[bufferPos] = b << (8 - bufferRem);
- bufferBits += (int)bits_len;
- }
+}
- wctx->bufferBits = bufferBits;
- wctx->bufferPos = bufferPos;
+static int wp512_update(struct shash_desc *desc, const u8 *source,
+ unsigned int len)
+{
+ struct wp512_ctx *wctx = shash_desc_ctx(desc);
+ unsigned int remain = len % WP512_BLOCK_SIZE;
+ u64 bits_len = (len - remain) * 8ull;
+ u8 *bitLength = wctx->bitLength;
- return 0;
+ wp512_add_length(bitLength, bits_len);
+ do {
+ wp512_process_buffer(wctx, source);
+ source += WP512_BLOCK_SIZE;
+ bits_len -= WP512_BLOCK_SIZE * 8;
+ } while (bits_len);
+
+ return remain;
}
-static int wp512_final(struct shash_desc *desc, u8 *out)
+static int wp512_finup(struct shash_desc *desc, const u8 *src,
+ unsigned int bufferPos, u8 *out)
{
struct wp512_ctx *wctx = shash_desc_ctx(desc);
int i;
- u8 *buffer = wctx->buffer;
u8 *bitLength = wctx->bitLength;
- int bufferBits = wctx->bufferBits;
- int bufferPos = wctx->bufferPos;
__be64 *digest = (__be64 *)out;
+ u8 buffer[WP512_BLOCK_SIZE];
- buffer[bufferPos] |= 0x80U >> (bufferBits & 7);
+ wp512_add_length(bitLength, bufferPos * 8);
+ memcpy(buffer, src, bufferPos);
+ buffer[bufferPos] = 0x80U;
bufferPos++;
if (bufferPos > WP512_BLOCK_SIZE - WP512_LENGTHBYTES) {
if (bufferPos < WP512_BLOCK_SIZE)
memset(&buffer[bufferPos], 0, WP512_BLOCK_SIZE - bufferPos);
- wp512_process_buffer(wctx);
+ wp512_process_buffer(wctx, buffer);
bufferPos = 0;
}
if (bufferPos < WP512_BLOCK_SIZE - WP512_LENGTHBYTES)
@@ -1086,31 +1051,32 @@ static int wp512_final(struct shash_desc *desc, u8 *out)
bufferPos = WP512_BLOCK_SIZE - WP512_LENGTHBYTES;
memcpy(&buffer[WP512_BLOCK_SIZE - WP512_LENGTHBYTES],
bitLength, WP512_LENGTHBYTES);
- wp512_process_buffer(wctx);
+ wp512_process_buffer(wctx, buffer);
+ memzero_explicit(buffer, sizeof(buffer));
for (i = 0; i < WP512_DIGEST_SIZE/8; i++)
digest[i] = cpu_to_be64(wctx->hash[i]);
- wctx->bufferBits = bufferBits;
- wctx->bufferPos = bufferPos;
return 0;
}
-static int wp384_final(struct shash_desc *desc, u8 *out)
+static int wp384_finup(struct shash_desc *desc, const u8 *src,
+ unsigned int len, u8 *out)
{
u8 D[64];
- wp512_final(desc, D);
+ wp512_finup(desc, src, len, D);
memcpy(out, D, WP384_DIGEST_SIZE);
memzero_explicit(D, WP512_DIGEST_SIZE);
return 0;
}
-static int wp256_final(struct shash_desc *desc, u8 *out)
+static int wp256_finup(struct shash_desc *desc, const u8 *src,
+ unsigned int len, u8 *out)
{
u8 D[64];
- wp512_final(desc, D);
+ wp512_finup(desc, src, len, D);
memcpy(out, D, WP256_DIGEST_SIZE);
memzero_explicit(D, WP512_DIGEST_SIZE);
@@ -1121,11 +1087,12 @@ static struct shash_alg wp_algs[3] = { {
.digestsize = WP512_DIGEST_SIZE,
.init = wp512_init,
.update = wp512_update,
- .final = wp512_final,
+ .finup = wp512_finup,
.descsize = sizeof(struct wp512_ctx),
.base = {
.cra_name = "wp512",
.cra_driver_name = "wp512-generic",
+ .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
.cra_blocksize = WP512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@@ -1133,11 +1100,12 @@ static struct shash_alg wp_algs[3] = { {
.digestsize = WP384_DIGEST_SIZE,
.init = wp512_init,
.update = wp512_update,
- .final = wp384_final,
+ .finup = wp384_finup,
.descsize = sizeof(struct wp512_ctx),
.base = {
.cra_name = "wp384",
.cra_driver_name = "wp384-generic",
+ .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
.cra_blocksize = WP512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@@ -1145,11 +1113,12 @@ static struct shash_alg wp_algs[3] = { {
.digestsize = WP256_DIGEST_SIZE,
.init = wp512_init,
.update = wp512_update,
- .final = wp256_final,
+ .finup = wp256_finup,
.descsize = sizeof(struct wp512_ctx),
.base = {
.cra_name = "wp256",
.cra_driver_name = "wp256-generic",
+ .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
.cra_blocksize = WP512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
diff --git a/crypto/zstd.c b/crypto/zstd.c
index 7570e11b4ee6..c2a19cb0879d 100644
--- a/crypto/zstd.c
+++ b/crypto/zstd.c
@@ -12,188 +12,304 @@
#include <linux/net.h>
#include <linux/vmalloc.h>
#include <linux/zstd.h>
-#include <crypto/internal/scompress.h>
+#include <crypto/internal/acompress.h>
+#include <crypto/scatterwalk.h>
-#define ZSTD_DEF_LEVEL 3
+#define ZSTD_DEF_LEVEL 3
+#define ZSTD_MAX_WINDOWLOG 18
+#define ZSTD_MAX_SIZE BIT(ZSTD_MAX_WINDOWLOG)
struct zstd_ctx {
zstd_cctx *cctx;
zstd_dctx *dctx;
- void *cwksp;
- void *dwksp;
+ size_t wksp_size;
+ zstd_parameters params;
+ u8 wksp[] __aligned(8);
};
-static zstd_parameters zstd_params(void)
-{
- return zstd_get_params(ZSTD_DEF_LEVEL, 0);
-}
+static DEFINE_MUTEX(zstd_stream_lock);
-static int zstd_comp_init(struct zstd_ctx *ctx)
+static void *zstd_alloc_stream(void)
{
- int ret = 0;
- const zstd_parameters params = zstd_params();
- const size_t wksp_size = zstd_cctx_workspace_bound(&params.cParams);
+ zstd_parameters params;
+ struct zstd_ctx *ctx;
+ size_t wksp_size;
- ctx->cwksp = vzalloc(wksp_size);
- if (!ctx->cwksp) {
- ret = -ENOMEM;
- goto out;
- }
+ params = zstd_get_params(ZSTD_DEF_LEVEL, ZSTD_MAX_SIZE);
- ctx->cctx = zstd_init_cctx(ctx->cwksp, wksp_size);
- if (!ctx->cctx) {
- ret = -EINVAL;
- goto out_free;
- }
-out:
- return ret;
-out_free:
- vfree(ctx->cwksp);
- goto out;
+ wksp_size = max_t(size_t,
+ zstd_cstream_workspace_bound(&params.cParams),
+ zstd_dstream_workspace_bound(ZSTD_MAX_SIZE));
+ if (!wksp_size)
+ return ERR_PTR(-EINVAL);
+
+ ctx = kvmalloc(sizeof(*ctx) + wksp_size, GFP_KERNEL);
+ if (!ctx)
+ return ERR_PTR(-ENOMEM);
+
+ ctx->params = params;
+ ctx->wksp_size = wksp_size;
+
+ return ctx;
}
-static int zstd_decomp_init(struct zstd_ctx *ctx)
+static void zstd_free_stream(void *ctx)
+{
+ kvfree(ctx);
+}
+
+static struct crypto_acomp_streams zstd_streams = {
+ .alloc_ctx = zstd_alloc_stream,
+ .free_ctx = zstd_free_stream,
+};
+
+static int zstd_init(struct crypto_acomp *acomp_tfm)
{
int ret = 0;
- const size_t wksp_size = zstd_dctx_workspace_bound();
- ctx->dwksp = vzalloc(wksp_size);
- if (!ctx->dwksp) {
- ret = -ENOMEM;
- goto out;
- }
+ mutex_lock(&zstd_stream_lock);
+ ret = crypto_acomp_alloc_streams(&zstd_streams);
+ mutex_unlock(&zstd_stream_lock);
- ctx->dctx = zstd_init_dctx(ctx->dwksp, wksp_size);
- if (!ctx->dctx) {
- ret = -EINVAL;
- goto out_free;
- }
-out:
return ret;
-out_free:
- vfree(ctx->dwksp);
- goto out;
}
-static void zstd_comp_exit(struct zstd_ctx *ctx)
+static void zstd_exit(struct crypto_acomp *acomp_tfm)
{
- vfree(ctx->cwksp);
- ctx->cwksp = NULL;
- ctx->cctx = NULL;
+ crypto_acomp_free_streams(&zstd_streams);
}
-static void zstd_decomp_exit(struct zstd_ctx *ctx)
+static int zstd_compress_one(struct acomp_req *req, struct zstd_ctx *ctx,
+ const void *src, void *dst, unsigned int *dlen)
{
- vfree(ctx->dwksp);
- ctx->dwksp = NULL;
- ctx->dctx = NULL;
-}
+ unsigned int out_len;
-static int __zstd_init(void *ctx)
-{
- int ret;
+ ctx->cctx = zstd_init_cctx(ctx->wksp, ctx->wksp_size);
+ if (!ctx->cctx)
+ return -EINVAL;
- ret = zstd_comp_init(ctx);
- if (ret)
- return ret;
- ret = zstd_decomp_init(ctx);
- if (ret)
- zstd_comp_exit(ctx);
- return ret;
+ out_len = zstd_compress_cctx(ctx->cctx, dst, req->dlen, src, req->slen,
+ &ctx->params);
+ if (zstd_is_error(out_len))
+ return -EINVAL;
+
+ *dlen = out_len;
+
+ return 0;
}
-static void *zstd_alloc_ctx(void)
+static int zstd_compress(struct acomp_req *req)
{
- int ret;
+ struct crypto_acomp_stream *s;
+ unsigned int pos, scur, dcur;
+ unsigned int total_out = 0;
+ bool data_available = true;
+ zstd_out_buffer outbuf;
+ struct acomp_walk walk;
+ zstd_in_buffer inbuf;
struct zstd_ctx *ctx;
+ size_t pending_bytes;
+ size_t num_bytes;
+ int ret;
- ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
- if (!ctx)
- return ERR_PTR(-ENOMEM);
+ s = crypto_acomp_lock_stream_bh(&zstd_streams);
+ ctx = s->ctx;
- ret = __zstd_init(ctx);
- if (ret) {
- kfree(ctx);
- return ERR_PTR(ret);
+ ret = acomp_walk_virt(&walk, req, true);
+ if (ret)
+ goto out;
+
+ ctx->cctx = zstd_init_cstream(&ctx->params, 0, ctx->wksp, ctx->wksp_size);
+ if (!ctx->cctx) {
+ ret = -EINVAL;
+ goto out;
}
- return ctx;
-}
+ do {
+ dcur = acomp_walk_next_dst(&walk);
+ if (!dcur) {
+ ret = -ENOSPC;
+ goto out;
+ }
-static void __zstd_exit(void *ctx)
-{
- zstd_comp_exit(ctx);
- zstd_decomp_exit(ctx);
-}
+ outbuf.pos = 0;
+ outbuf.dst = (u8 *)walk.dst.virt.addr;
+ outbuf.size = dcur;
-static void zstd_free_ctx(void *ctx)
-{
- __zstd_exit(ctx);
- kfree_sensitive(ctx);
-}
+ do {
+ scur = acomp_walk_next_src(&walk);
+ if (dcur == req->dlen && scur == req->slen) {
+ ret = zstd_compress_one(req, ctx, walk.src.virt.addr,
+ walk.dst.virt.addr, &total_out);
+ acomp_walk_done_src(&walk, scur);
+ acomp_walk_done_dst(&walk, dcur);
+ goto out;
+ }
-static int __zstd_compress(const u8 *src, unsigned int slen,
- u8 *dst, unsigned int *dlen, void *ctx)
-{
- size_t out_len;
- struct zstd_ctx *zctx = ctx;
- const zstd_parameters params = zstd_params();
+ if (scur) {
+ inbuf.pos = 0;
+ inbuf.src = walk.src.virt.addr;
+ inbuf.size = scur;
+ } else {
+ data_available = false;
+ break;
+ }
- out_len = zstd_compress_cctx(zctx->cctx, dst, *dlen, src, slen, &params);
- if (zstd_is_error(out_len))
- return -EINVAL;
- *dlen = out_len;
- return 0;
-}
+ num_bytes = zstd_compress_stream(ctx->cctx, &outbuf, &inbuf);
+ if (ZSTD_isError(num_bytes)) {
+ ret = -EIO;
+ goto out;
+ }
-static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src,
- unsigned int slen, u8 *dst, unsigned int *dlen,
- void *ctx)
-{
- return __zstd_compress(src, slen, dst, dlen, ctx);
+ pending_bytes = zstd_flush_stream(ctx->cctx, &outbuf);
+ if (ZSTD_isError(pending_bytes)) {
+ ret = -EIO;
+ goto out;
+ }
+ acomp_walk_done_src(&walk, inbuf.pos);
+ } while (dcur != outbuf.pos);
+
+ total_out += outbuf.pos;
+ acomp_walk_done_dst(&walk, dcur);
+ } while (data_available);
+
+ pos = outbuf.pos;
+ num_bytes = zstd_end_stream(ctx->cctx, &outbuf);
+ if (ZSTD_isError(num_bytes))
+ ret = -EIO;
+ else
+ total_out += (outbuf.pos - pos);
+
+out:
+ if (ret)
+ req->dlen = 0;
+ else
+ req->dlen = total_out;
+
+ crypto_acomp_unlock_stream_bh(s);
+
+ return ret;
}
-static int __zstd_decompress(const u8 *src, unsigned int slen,
- u8 *dst, unsigned int *dlen, void *ctx)
+static int zstd_decompress_one(struct acomp_req *req, struct zstd_ctx *ctx,
+ const void *src, void *dst, unsigned int *dlen)
{
size_t out_len;
- struct zstd_ctx *zctx = ctx;
- out_len = zstd_decompress_dctx(zctx->dctx, dst, *dlen, src, slen);
+ ctx->dctx = zstd_init_dctx(ctx->wksp, ctx->wksp_size);
+ if (!ctx->dctx)
+ return -EINVAL;
+
+ out_len = zstd_decompress_dctx(ctx->dctx, dst, req->dlen, src, req->slen);
if (zstd_is_error(out_len))
return -EINVAL;
+
*dlen = out_len;
+
return 0;
}
-static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
- unsigned int slen, u8 *dst, unsigned int *dlen,
- void *ctx)
+static int zstd_decompress(struct acomp_req *req)
{
- return __zstd_decompress(src, slen, dst, dlen, ctx);
-}
+ struct crypto_acomp_stream *s;
+ unsigned int total_out = 0;
+ unsigned int scur, dcur;
+ zstd_out_buffer outbuf;
+ struct acomp_walk walk;
+ zstd_in_buffer inbuf;
+ struct zstd_ctx *ctx;
+ size_t pending_bytes;
+ int ret;
-static struct scomp_alg scomp = {
- .alloc_ctx = zstd_alloc_ctx,
- .free_ctx = zstd_free_ctx,
- .compress = zstd_scompress,
- .decompress = zstd_sdecompress,
- .base = {
- .cra_name = "zstd",
- .cra_driver_name = "zstd-scomp",
- .cra_module = THIS_MODULE,
+ s = crypto_acomp_lock_stream_bh(&zstd_streams);
+ ctx = s->ctx;
+
+ ret = acomp_walk_virt(&walk, req, true);
+ if (ret)
+ goto out;
+
+ ctx->dctx = zstd_init_dstream(ZSTD_MAX_SIZE, ctx->wksp, ctx->wksp_size);
+ if (!ctx->dctx) {
+ ret = -EINVAL;
+ goto out;
}
+
+ do {
+ scur = acomp_walk_next_src(&walk);
+ if (scur) {
+ inbuf.pos = 0;
+ inbuf.size = scur;
+ inbuf.src = walk.src.virt.addr;
+ } else {
+ break;
+ }
+
+ do {
+ dcur = acomp_walk_next_dst(&walk);
+ if (dcur == req->dlen && scur == req->slen) {
+ ret = zstd_decompress_one(req, ctx, walk.src.virt.addr,
+ walk.dst.virt.addr, &total_out);
+ acomp_walk_done_dst(&walk, dcur);
+ acomp_walk_done_src(&walk, scur);
+ goto out;
+ }
+
+ if (!dcur) {
+ ret = -ENOSPC;
+ goto out;
+ }
+
+ outbuf.pos = 0;
+ outbuf.dst = (u8 *)walk.dst.virt.addr;
+ outbuf.size = dcur;
+
+ pending_bytes = zstd_decompress_stream(ctx->dctx, &outbuf, &inbuf);
+ if (ZSTD_isError(pending_bytes)) {
+ ret = -EIO;
+ goto out;
+ }
+
+ total_out += outbuf.pos;
+
+ acomp_walk_done_dst(&walk, outbuf.pos);
+ } while (inbuf.pos != scur);
+
+ acomp_walk_done_src(&walk, scur);
+ } while (ret == 0);
+
+out:
+ if (ret)
+ req->dlen = 0;
+ else
+ req->dlen = total_out;
+
+ crypto_acomp_unlock_stream_bh(s);
+
+ return ret;
+}
+
+static struct acomp_alg zstd_acomp = {
+ .base = {
+ .cra_name = "zstd",
+ .cra_driver_name = "zstd-generic",
+ .cra_flags = CRYPTO_ALG_REQ_VIRT,
+ .cra_module = THIS_MODULE,
+ },
+ .init = zstd_init,
+ .exit = zstd_exit,
+ .compress = zstd_compress,
+ .decompress = zstd_decompress,
};
static int __init zstd_mod_init(void)
{
- return crypto_register_scomp(&scomp);
+ return crypto_register_acomp(&zstd_acomp);
}
static void __exit zstd_mod_fini(void)
{
- crypto_unregister_scomp(&scomp);
+ crypto_unregister_acomp(&zstd_acomp);
}
module_init(zstd_mod_init);
generated by cgit (git 2.34.1) at 2025-08-01 17:42:13 +0000