diff options
Diffstat (limited to 'crypto/testmgr.c')
| -rw-r--r-- | crypto/testmgr.c | 1242 |
1 files changed, 1079 insertions, 163 deletions
diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 8386038d67c7..c9e67c2bd725 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -37,6 +37,7 @@ #include <crypto/akcipher.h> #include <crypto/kpp.h> #include <crypto/acompress.h> +#include <crypto/internal/simd.h> #include "internal.h" @@ -44,6 +45,9 @@ static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); +static bool panic_on_fail; +module_param(panic_on_fail, bool, 0444); + #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS static bool noextratests; module_param(noextratests, bool, 0644); @@ -52,6 +56,9 @@ MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); static unsigned int fuzz_iterations = 100; module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); + +DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test); +EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test); #endif #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS @@ -121,6 +128,7 @@ struct kpp_test_suite { struct alg_test_desc { const char *alg; + const char *generic_driver; int (*test)(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask); int fips_allowed; /* set if alg is allowed in fips mode */ @@ -230,12 +238,14 @@ enum finalization_type { * @offset * @flush_type: for hashes, whether an update() should be done now vs. * continuing to accumulate data + * @nosimd: if doing the pending update(), do it with SIMD disabled? */ struct test_sg_division { unsigned int proportion_of_total; unsigned int offset; bool offset_relative_to_alignmask; enum flush_type flush_type; + bool nosimd; }; /** @@ -255,6 +265,7 @@ struct test_sg_division { * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to * the @iv_offset * @finalization_type: what finalization function to use for hashes + * @nosimd: execute with SIMD disabled? Requires !CRYPTO_TFM_REQ_MAY_SLEEP. */ struct testvec_config { const char *name; @@ -265,6 +276,7 @@ struct testvec_config { unsigned int iv_offset; bool iv_offset_relative_to_alignmask; enum finalization_type finalization_type; + bool nosimd; }; #define TESTVEC_CONFIG_NAMELEN 192 @@ -416,8 +428,11 @@ static unsigned int count_test_sg_divisions(const struct test_sg_division *divs) return ndivs; } +#define SGDIVS_HAVE_FLUSHES BIT(0) +#define SGDIVS_HAVE_NOSIMD BIT(1) + static bool valid_sg_divisions(const struct test_sg_division *divs, - unsigned int count, bool *any_flushes_ret) + unsigned int count, int *flags_ret) { unsigned int total = 0; unsigned int i; @@ -428,7 +443,9 @@ static bool valid_sg_divisions(const struct test_sg_division *divs, return false; total += divs[i].proportion_of_total; if (divs[i].flush_type != FLUSH_TYPE_NONE) - *any_flushes_ret = true; + *flags_ret |= SGDIVS_HAVE_FLUSHES; + if (divs[i].nosimd) + *flags_ret |= SGDIVS_HAVE_NOSIMD; } return total == TEST_SG_TOTAL && memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL; @@ -441,19 +458,18 @@ static bool valid_sg_divisions(const struct test_sg_division *divs, */ static bool valid_testvec_config(const struct testvec_config *cfg) { - bool any_flushes = false; + int flags = 0; if (cfg->name == NULL) return false; if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs), - &any_flushes)) + &flags)) return false; if (cfg->dst_divs[0].proportion_of_total) { if (!valid_sg_divisions(cfg->dst_divs, - ARRAY_SIZE(cfg->dst_divs), - &any_flushes)) + ARRAY_SIZE(cfg->dst_divs), &flags)) return false; } else { if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs))) @@ -466,7 +482,12 @@ static bool valid_testvec_config(const struct testvec_config *cfg) MAX_ALGAPI_ALIGNMASK + 1) return false; - if (any_flushes && cfg->finalization_type == FINALIZATION_TYPE_DIGEST) + if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) && + cfg->finalization_type == FINALIZATION_TYPE_DIGEST) + return false; + + if ((cfg->nosimd || (flags & SGDIVS_HAVE_NOSIMD)) && + (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) return false; return true; @@ -725,15 +746,101 @@ static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls, } #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS + +/* Generate a random length in range [0, max_len], but prefer smaller values */ +static unsigned int generate_random_length(unsigned int max_len) +{ + unsigned int len = prandom_u32() % (max_len + 1); + + switch (prandom_u32() % 4) { + case 0: + return len % 64; + case 1: + return len % 256; + case 2: + return len % 1024; + default: + return len; + } +} + +/* Sometimes make some random changes to the given data buffer */ +static void mutate_buffer(u8 *buf, size_t count) +{ + size_t num_flips; + size_t i; + size_t pos; + + /* Sometimes flip some bits */ + if (prandom_u32() % 4 == 0) { + num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count * 8); + for (i = 0; i < num_flips; i++) { + pos = prandom_u32() % (count * 8); + buf[pos / 8] ^= 1 << (pos % 8); + } + } + + /* Sometimes flip some bytes */ + if (prandom_u32() % 4 == 0) { + num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count); + for (i = 0; i < num_flips; i++) + buf[prandom_u32() % count] ^= 0xff; + } +} + +/* Randomly generate 'count' bytes, but sometimes make them "interesting" */ +static void generate_random_bytes(u8 *buf, size_t count) +{ + u8 b; + u8 increment; + size_t i; + + if (count == 0) + return; + + switch (prandom_u32() % 8) { /* Choose a generation strategy */ + case 0: + case 1: + /* All the same byte, plus optional mutations */ + switch (prandom_u32() % 4) { + case 0: + b = 0x00; + break; + case 1: + b = 0xff; + break; + default: + b = (u8)prandom_u32(); + break; + } + memset(buf, b, count); + mutate_buffer(buf, count); + break; + case 2: + /* Ascending or descending bytes, plus optional mutations */ + increment = (u8)prandom_u32(); + b = (u8)prandom_u32(); + for (i = 0; i < count; i++, b += increment) + buf[i] = b; + mutate_buffer(buf, count); + break; + default: + /* Fully random bytes */ + for (i = 0; i < count; i++) + buf[i] = (u8)prandom_u32(); + } +} + static char *generate_random_sgl_divisions(struct test_sg_division *divs, size_t max_divs, char *p, char *end, - bool gen_flushes) + bool gen_flushes, u32 req_flags) { struct test_sg_division *div = divs; unsigned int remaining = TEST_SG_TOTAL; do { unsigned int this_len; + const char *flushtype_str; if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0) this_len = remaining; @@ -762,11 +869,31 @@ static char *generate_random_sgl_divisions(struct test_sg_division *divs, } } + if (div->flush_type != FLUSH_TYPE_NONE && + !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && + prandom_u32() % 2 == 0) + div->nosimd = true; + + switch (div->flush_type) { + case FLUSH_TYPE_FLUSH: + if (div->nosimd) + flushtype_str = "<flush,nosimd>"; + else + flushtype_str = "<flush>"; + break; + case FLUSH_TYPE_REIMPORT: + if (div->nosimd) + flushtype_str = "<reimport,nosimd>"; + else + flushtype_str = "<reimport>"; + break; + default: + flushtype_str = ""; + break; + } + BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */ - p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", - div->flush_type == FLUSH_TYPE_NONE ? "" : - div->flush_type == FLUSH_TYPE_FLUSH ? - "<flush> " : "<reimport> ", + p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str, this_len / 100, this_len % 100, div->offset_relative_to_alignmask ? "alignmask" : "", @@ -816,18 +943,26 @@ static void generate_random_testvec_config(struct testvec_config *cfg, break; } + if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && + prandom_u32() % 2 == 0) { + cfg->nosimd = true; + p += scnprintf(p, end - p, " nosimd"); + } + p += scnprintf(p, end - p, " src_divs=["); p = generate_random_sgl_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs), p, end, (cfg->finalization_type != - FINALIZATION_TYPE_DIGEST)); + FINALIZATION_TYPE_DIGEST), + cfg->req_flags); p += scnprintf(p, end - p, "]"); if (!cfg->inplace && prandom_u32() % 2 == 0) { p += scnprintf(p, end - p, " dst_divs=["); p = generate_random_sgl_divisions(cfg->dst_divs, ARRAY_SIZE(cfg->dst_divs), - p, end, false); + p, end, false, + cfg->req_flags); p += scnprintf(p, end - p, "]"); } @@ -838,21 +973,100 @@ static void generate_random_testvec_config(struct testvec_config *cfg, WARN_ON_ONCE(!valid_testvec_config(cfg)); } -#endif /* CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + +static void crypto_disable_simd_for_test(void) +{ + preempt_disable(); + __this_cpu_write(crypto_simd_disabled_for_test, true); +} + +static void crypto_reenable_simd_for_test(void) +{ + __this_cpu_write(crypto_simd_disabled_for_test, false); + preempt_enable(); +} + +/* + * Given an algorithm name, build the name of the generic implementation of that + * algorithm, assuming the usual naming convention. Specifically, this appends + * "-generic" to every part of the name that is not a template name. Examples: + * + * aes => aes-generic + * cbc(aes) => cbc(aes-generic) + * cts(cbc(aes)) => cts(cbc(aes-generic)) + * rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic) + * + * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long + */ +static int build_generic_driver_name(const char *algname, + char driver_name[CRYPTO_MAX_ALG_NAME]) +{ + const char *in = algname; + char *out = driver_name; + size_t len = strlen(algname); + + if (len >= CRYPTO_MAX_ALG_NAME) + goto too_long; + do { + const char *in_saved = in; + + while (*in && *in != '(' && *in != ')' && *in != ',') + *out++ = *in++; + if (*in != '(' && in > in_saved) { + len += 8; + if (len >= CRYPTO_MAX_ALG_NAME) + goto too_long; + memcpy(out, "-generic", 8); + out += 8; + } + } while ((*out++ = *in++) != '\0'); + return 0; + +too_long: + pr_err("alg: generic driver name for \"%s\" would be too long\n", + algname); + return -ENAMETOOLONG; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static void crypto_disable_simd_for_test(void) +{ +} + +static void crypto_reenable_simd_for_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + +static int do_ahash_op(int (*op)(struct ahash_request *req), + struct ahash_request *req, + struct crypto_wait *wait, bool nosimd) +{ + int err; + + if (nosimd) + crypto_disable_simd_for_test(); + + err = op(req); + + if (nosimd) + crypto_reenable_simd_for_test(); + + return crypto_wait_req(err, wait); +} static int check_nonfinal_hash_op(const char *op, int err, u8 *result, unsigned int digestsize, - const char *driver, unsigned int vec_num, + const char *driver, const char *vec_name, const struct testvec_config *cfg) { if (err) { - pr_err("alg: hash: %s %s() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); + pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, op, err, vec_name, cfg->name); return err; } if (!testmgr_is_poison(result, digestsize)) { - pr_err("alg: hash: %s %s() used result buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } return 0; @@ -860,7 +1074,7 @@ static int check_nonfinal_hash_op(const char *op, int err, static int test_hash_vec_cfg(const char *driver, const struct hash_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct ahash_request *req, struct test_sglist *tsgl, @@ -885,11 +1099,18 @@ static int test_hash_vec_cfg(const char *driver, if (vec->ksize) { err = crypto_ahash_setkey(tfm, vec->key, vec->ksize); if (err) { - pr_err("alg: hash: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, + if (err == vec->setkey_error) + return 0; + pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, crypto_ahash_get_flags(tfm)); return err; } + if (vec->setkey_error) { + pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); + return -EINVAL; + } } /* Build the scatterlist for the source data */ @@ -899,8 +1120,8 @@ static int test_hash_vec_cfg(const char *driver, err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, &input, divs); if (err) { - pr_err("alg: hash: %s: error preparing scatterlist for test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return err; } @@ -909,17 +1130,26 @@ static int test_hash_vec_cfg(const char *driver, testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); testmgr_poison(result, digestsize + TESTMGR_POISON_LEN); - if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST) { + if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST || + vec->digest_error) { /* Just using digest() */ ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize); - err = crypto_wait_req(crypto_ahash_digest(req), &wait); + err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s digest() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + if (err == vec->digest_error) + return 0; + pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, err, + cfg->name); return err; } + if (vec->digest_error) { + pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, cfg->name); + return -EINVAL; + } goto result_ready; } @@ -927,9 +1157,9 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, NULL, result, 0); - err = crypto_wait_req(crypto_ahash_init(req), &wait); + err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd); err = check_nonfinal_hash_op("init", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; @@ -943,10 +1173,11 @@ static int test_hash_vec_cfg(const char *driver, crypto_req_done, &wait); ahash_request_set_crypt(req, pending_sgl, result, pending_len); - err = crypto_wait_req(crypto_ahash_update(req), &wait); + err = do_ahash_op(crypto_ahash_update, req, &wait, + divs[i]->nosimd); err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; pending_sgl = NULL; @@ -959,13 +1190,13 @@ static int test_hash_vec_cfg(const char *driver, err = crypto_ahash_export(req, hashstate); err = check_nonfinal_hash_op("export", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; if (!testmgr_is_poison(hashstate + statesize, TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s export() overran state buffer on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EOVERFLOW; } @@ -973,7 +1204,7 @@ static int test_hash_vec_cfg(const char *driver, err = crypto_ahash_import(req, hashstate); err = check_nonfinal_hash_op("import", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; } @@ -986,23 +1217,23 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_crypt(req, pending_sgl, result, pending_len); if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) { /* finish with update() and final() */ - err = crypto_wait_req(crypto_ahash_update(req), &wait); + err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd); err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; - err = crypto_wait_req(crypto_ahash_final(req), &wait); + err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s final() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); return err; } } else { /* finish with finup() */ - err = crypto_wait_req(crypto_ahash_finup(req), &wait); + err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s finup() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); return err; } } @@ -1010,13 +1241,13 @@ static int test_hash_vec_cfg(const char *driver, result_ready: /* Check that the algorithm produced the correct digest */ if (memcmp(result, vec->digest, digestsize) != 0) { - pr_err("alg: hash: %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EINVAL; } if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s overran result buffer on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EOVERFLOW; } @@ -1027,11 +1258,14 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, unsigned int vec_num, struct ahash_request *req, struct test_sglist *tsgl, u8 *hashstate) { + char vec_name[16]; unsigned int i; int err; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { - err = test_hash_vec_cfg(driver, vec, vec_num, + err = test_hash_vec_cfg(driver, vec, vec_name, &default_hash_testvec_configs[i], req, tsgl, hashstate); if (err) @@ -1046,7 +1280,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, vec, vec_num, &cfg, + err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, req, tsgl, hashstate); if (err) return err; @@ -1056,9 +1290,168 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate a hash test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_hash_testvec(struct crypto_shash *tfm, + struct hash_testvec *vec, + unsigned int maxkeysize, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + SHASH_DESC_ON_STACK(desc, tfm); + + /* Data */ + vec->psize = generate_random_length(maxdatasize); + generate_random_bytes((u8 *)vec->plaintext, vec->psize); + + /* + * Key: length in range [1, maxkeysize], but usually choose maxkeysize. + * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0. + */ + vec->setkey_error = 0; + vec->ksize = 0; + if (maxkeysize) { + vec->ksize = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->ksize = 1 + (prandom_u32() % maxkeysize); + generate_random_bytes((u8 *)vec->key, vec->ksize); + + vec->setkey_error = crypto_shash_setkey(tfm, vec->key, + vec->ksize); + /* If the key couldn't be set, no need to continue to digest. */ + if (vec->setkey_error) + goto done; + } + + /* Digest */ + desc->tfm = tfm; + vec->digest_error = crypto_shash_digest(desc, vec->plaintext, + vec->psize, (u8 *)vec->digest); +done: + snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"", + vec->psize, vec->ksize); +} + +/* + * Test the hash algorithm represented by @req against the corresponding generic + * implementation, if one is available. + */ +static int test_hash_vs_generic_impl(const char *driver, + const char *generic_driver, + unsigned int maxkeysize, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + const unsigned int digestsize = crypto_ahash_digestsize(tfm); + const unsigned int blocksize = crypto_ahash_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_shash *generic_tfm = NULL; + unsigned int i; + struct hash_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_shash(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + /* Check the algorithm properties for consistency. */ + + if (digestsize != crypto_shash_digestsize(generic_tfm)) { + pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n", + driver, digestsize, + crypto_shash_digestsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_shash_blocksize(generic_tfm)) { + pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, crypto_shash_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + vec.key = kmalloc(maxkeysize, GFP_KERNEL); + vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL); + vec.digest = kmalloc(digestsize, GFP_KERNEL); + if (!vec.key || !vec.plaintext || !vec.digest) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_hash_testvec(generic_tfm, &vec, + maxkeysize, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg, + req, tsgl, hashstate); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.plaintext); + kfree(vec.digest); + crypto_free_shash(generic_tfm); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_hash_vs_generic_impl(const char *driver, + const char *generic_driver, + unsigned int maxkeysize, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int __alg_test_hash(const struct hash_testvec *vecs, unsigned int num_vecs, const char *driver, - u32 type, u32 mask) + u32 type, u32 mask, + const char *generic_driver, unsigned int maxkeysize) { struct crypto_ahash *tfm; struct ahash_request *req = NULL; @@ -1106,7 +1499,8 @@ static int __alg_test_hash(const struct hash_testvec *vecs, if (err) goto out; } - err = 0; + err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, + tsgl, hashstate); out: kfree(hashstate); if (tsgl) { @@ -1124,6 +1518,7 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, const struct hash_testvec *template = desc->suite.hash.vecs; unsigned int tcount = desc->suite.hash.count; unsigned int nr_unkeyed, nr_keyed; + unsigned int maxkeysize = 0; int err; /* @@ -1142,23 +1537,27 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, "unkeyed ones must come first\n", desc->alg); return -EINVAL; } + maxkeysize = max_t(unsigned int, maxkeysize, + template[nr_unkeyed + nr_keyed].ksize); } err = 0; if (nr_unkeyed) { - err = __alg_test_hash(template, nr_unkeyed, driver, type, mask); + err = __alg_test_hash(template, nr_unkeyed, driver, type, mask, + desc->generic_driver, maxkeysize); template += nr_unkeyed; } if (!err && nr_keyed) - err = __alg_test_hash(template, nr_keyed, driver, type, mask); + err = __alg_test_hash(template, nr_keyed, driver, type, mask, + desc->generic_driver, maxkeysize); return err; } static int test_aead_vec_cfg(const char *driver, int enc, const struct aead_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct aead_request *req, struct cipher_test_sglists *tsgls) @@ -1175,6 +1574,7 @@ static int test_aead_vec_cfg(const char *driver, int enc, cfg->iv_offset + (cfg->iv_offset_relative_to_alignmask ? alignmask : 0); struct kvec input[2]; + int expected_error; int err; /* Set the key */ @@ -1183,26 +1583,33 @@ static int test_aead_vec_cfg(const char *driver, int enc, else crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); err = crypto_aead_setkey(tfm, vec->key, vec->klen); - if (err) { - if (vec->fail) /* expectedly failed to set key? */ - return 0; - pr_err("alg: aead: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, crypto_aead_get_flags(tfm)); + if (err && err != vec->setkey_error) { + pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_aead_get_flags(tfm)); return err; } - if (vec->fail) { - pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %u\n", - driver, vec_num); + if (!err && vec->setkey_error) { + pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); return -EINVAL; } /* Set the authentication tag size */ err = crypto_aead_setauthsize(tfm, authsize); - if (err) { - pr_err("alg: aead: %s setauthsize failed with err %d on test vector %u\n", - driver, err, vec_num); + if (err && err != vec->setauthsize_error) { + pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n", + driver, vec_name, vec->setauthsize_error, err); return err; } + if (!err && vec->setauthsize_error) { + pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setauthsize_error); + return -EINVAL; + } + + if (vec->setkey_error || vec->setauthsize_error) + return 0; /* The IV must be copied to a buffer, as the algorithm may modify it */ if (WARN_ON(ivsize > MAX_IVLEN)) @@ -1224,8 +1631,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, vec->plen), input, 2); if (err) { - pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1235,23 +1642,12 @@ static int test_aead_vec_cfg(const char *driver, int enc, aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr, enc ? vec->plen : vec->clen, iv); aead_request_set_ad(req, vec->alen); - err = crypto_wait_req(enc ? crypto_aead_encrypt(req) : - crypto_aead_decrypt(req), &wait); - - aead_request_set_tfm(req, tfm); /* TODO: get rid of this */ - - if (err) { - if (err == -EBADMSG && vec->novrfy) - return 0; - pr_err("alg: aead: %s %s failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); - return err; - } - if (vec->novrfy) { - pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); - return -EINVAL; - } + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = crypto_wait_req(err, &wait); /* Check that the algorithm didn't overwrite things it shouldn't have */ if (req->cryptlen != (enc ? vec->plen : vec->clen) || @@ -1263,8 +1659,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, req->base.complete != crypto_req_done || req->base.flags != req_flags || req->base.data != &wait) { - pr_err("alg: aead: %s %s corrupted request struct on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); if (req->cryptlen != (enc ? vec->plen : vec->clen)) pr_err("alg: aead: changed 'req->cryptlen'\n"); if (req->assoclen != vec->alen) @@ -1286,14 +1682,29 @@ static int test_aead_vec_cfg(const char *driver, int enc, return -EINVAL; } if (is_test_sglist_corrupted(&tsgls->src)) { - pr_err("alg: aead: %s %s corrupted src sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } if (tsgls->dst.sgl_ptr != tsgls->src.sgl && is_test_sglist_corrupted(&tsgls->dst)) { - pr_err("alg: aead: %s %s corrupted dst sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); + return -EINVAL; + } + + /* Check for success or failure */ + expected_error = vec->novrfy ? -EBADMSG : vec->crypt_error; + if (err) { + if (err == expected_error) + return 0; + pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, expected_error, err, cfg->name); + return err; + } + if (expected_error) { + pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, expected_error, cfg->name); return -EINVAL; } @@ -1302,13 +1713,13 @@ static int test_aead_vec_cfg(const char *driver, int enc, enc ? vec->clen : vec->plen, vec->alen, enc || !cfg->inplace); if (err == -EOVERFLOW) { - pr_err("alg: aead: %s %s overran dst buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } if (err) { - pr_err("alg: aead: %s %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1320,14 +1731,17 @@ static int test_aead_vec(const char *driver, int enc, struct aead_request *req, struct cipher_test_sglists *tsgls) { + char vec_name[16]; unsigned int i; int err; if (enc && vec->novrfy) return 0; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_aead_vec_cfg(driver, enc, vec, vec_num, + err = test_aead_vec_cfg(driver, enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -1342,7 +1756,7 @@ static int test_aead_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, enc, vec, vec_num, + err = test_aead_vec_cfg(driver, enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -1352,6 +1766,226 @@ static int test_aead_vec(const char *driver, int enc, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate an AEAD test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_aead_testvec(struct aead_request *req, + struct aead_testvec *vec, + unsigned int maxkeysize, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const unsigned int ivsize = crypto_aead_ivsize(tfm); + unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize; + unsigned int authsize; + unsigned int total_len; + int i; + struct scatterlist src[2], dst; + u8 iv[MAX_IVLEN]; + DECLARE_CRYPTO_WAIT(wait); + + /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ + vec->klen = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->klen = prandom_u32() % (maxkeysize + 1); + generate_random_bytes((u8 *)vec->key, vec->klen); + vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen); + + /* IV */ + generate_random_bytes((u8 *)vec->iv, ivsize); + + /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */ + authsize = maxauthsize; + if (prandom_u32() % 4 == 0) + authsize = prandom_u32() % (maxauthsize + 1); + if (WARN_ON(authsize > maxdatasize)) + authsize = maxdatasize; + maxdatasize -= authsize; + vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize); + + /* Plaintext and associated data */ + total_len = generate_random_length(maxdatasize); + if (prandom_u32() % 4 == 0) + vec->alen = 0; + else + vec->alen = generate_random_length(total_len); + vec->plen = total_len - vec->alen; + generate_random_bytes((u8 *)vec->assoc, vec->alen); + generate_random_bytes((u8 *)vec->ptext, vec->plen); + + vec->clen = vec->plen + authsize; + + /* + * If the key or authentication tag size couldn't be set, no need to + * continue to encrypt. + */ + if (vec->setkey_error || vec->setauthsize_error) + goto done; + + /* Ciphertext */ + sg_init_table(src, 2); + i = 0; + if (vec->alen) + sg_set_buf(&src[i++], vec->assoc, vec->alen); + if (vec->plen) + sg_set_buf(&src[i++], vec->ptext, vec->plen); + sg_init_one(&dst, vec->ctext, vec->alen + vec->clen); + memcpy(iv, vec->iv, ivsize); + aead_request_set_callback(req, 0, crypto_req_done, &wait); + aead_request_set_crypt(req, src, &dst, vec->plen, iv); + aead_request_set_ad(req, vec->alen); + vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req), &wait); + if (vec->crypt_error == 0) + memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen); +done: + snprintf(name, max_namelen, + "\"random: alen=%u plen=%u authsize=%u klen=%u\"", + vec->alen, vec->plen, authsize, vec->klen); +} + +/* + * Test the AEAD algorithm represented by @req against the corresponding generic + * implementation, if one is available. + */ +static int test_aead_vs_generic_impl(const char *driver, + const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const unsigned int ivsize = crypto_aead_ivsize(tfm); + const unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize; + const unsigned int blocksize = crypto_aead_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_aead_alg(tfm)->base.cra_name; + const char *generic_driver = test_desc->generic_driver; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_aead *generic_tfm = NULL; + struct aead_request *generic_req = NULL; + unsigned int maxkeysize; + unsigned int i; + struct aead_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_aead(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL); + if (!generic_req) { + err = -ENOMEM; + goto out; + } + + /* Check the algorithm properties for consistency. */ + + if (maxauthsize != crypto_aead_alg(generic_tfm)->maxauthsize) { + pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n", + driver, maxauthsize, + crypto_aead_alg(generic_tfm)->maxauthsize); + err = -EINVAL; + goto out; + } + + if (ivsize != crypto_aead_ivsize(generic_tfm)) { + pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n", + driver, ivsize, crypto_aead_ivsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_aead_blocksize(generic_tfm)) { + pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, crypto_aead_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + maxkeysize = 0; + for (i = 0; i < test_desc->suite.aead.count; i++) + maxkeysize = max_t(unsigned int, maxkeysize, + test_desc->suite.aead.vecs[i].klen); + + vec.key = kmalloc(maxkeysize, GFP_KERNEL); + vec.iv = kmalloc(ivsize, GFP_KERNEL); + vec.assoc = kmalloc(maxdatasize, GFP_KERNEL); + vec.ptext = kmalloc(maxdatasize, GFP_KERNEL); + vec.ctext = kmalloc(maxdatasize, GFP_KERNEL); + if (!vec.key || !vec.iv || !vec.assoc || !vec.ptext || !vec.ctext) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_aead_testvec(generic_req, &vec, + maxkeysize, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg, + req, tsgls); + if (err) + goto out; + err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg, + req, tsgls); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.iv); + kfree(vec.assoc); + kfree(vec.ptext); + kfree(vec.ctext); + crypto_free_aead(generic_tfm); + aead_request_free(generic_req); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_aead_vs_generic_impl(const char *driver, + const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int test_aead(const char *driver, int enc, const struct aead_test_suite *suite, struct aead_request *req, @@ -1411,6 +2045,10 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, goto out; err = test_aead(driver, DECRYPT, suite, req, tsgls); + if (err) + goto out; + + err = test_aead_vs_generic_impl(driver, desc, req, tsgls); out: free_cipher_test_sglists(tsgls); aead_request_free(req); @@ -1462,13 +2100,20 @@ static int test_cipher(struct crypto_cipher *tfm, int enc, ret = crypto_cipher_setkey(tfm, template[i].key, template[i].klen); - if (template[i].fail == !ret) { - printk(KERN_ERR "alg: cipher: setkey failed " - "on test %d for %s: flags=%x\n", j, - algo, crypto_cipher_get_flags(tfm)); + if (ret) { + if (ret == template[i].setkey_error) + continue; + pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n", + algo, j, template[i].setkey_error, ret, + crypto_cipher_get_flags(tfm)); goto out; - } else if (ret) - continue; + } + if (template[i].setkey_error) { + pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n", + algo, j, template[i].setkey_error); + ret = -EINVAL; + goto out; + } for (k = 0; k < template[i].len; k += crypto_cipher_blocksize(tfm)) { @@ -1500,7 +2145,7 @@ out_nobuf: static int test_skcipher_vec_cfg(const char *driver, int enc, const struct cipher_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct skcipher_request *req, struct cipher_test_sglists *tsgls) @@ -1526,15 +2171,16 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); err = crypto_skcipher_setkey(tfm, vec->key, vec->klen); if (err) { - if (vec->fail) /* expectedly failed to set key? */ + if (err == vec->setkey_error) return 0; - pr_err("alg: skcipher: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, crypto_skcipher_get_flags(tfm)); + pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_skcipher_get_flags(tfm)); return err; } - if (vec->fail) { - pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %u\n", - driver, vec_num); + if (vec->setkey_error) { + pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); return -EINVAL; } @@ -1550,8 +2196,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, memset(iv, 0, ivsize); } else { if (vec->generates_iv) { - pr_err("alg: skcipher: %s has ivsize=0 but test vector %u generates IV!\n", - driver, vec_num); + pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n", + driver, vec_name); return -EINVAL; } iv = NULL; @@ -1563,8 +2209,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, err = build_cipher_test_sglists(tsgls, cfg, alignmask, vec->len, vec->len, &input, 1); if (err) { - pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1573,13 +2219,12 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait); skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr, vec->len, iv); - err = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) : - crypto_skcipher_decrypt(req), &wait); - if (err) { - pr_err("alg: skcipher: %s %s failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); - return err; - } + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = crypto_wait_req(err, &wait); /* Check that the algorithm didn't overwrite things it shouldn't have */ if (req->cryptlen != vec->len || @@ -1590,8 +2235,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, req->base.complete != crypto_req_done || req->base.flags != req_flags || req->base.data != &wait) { - pr_err("alg: skcipher: %s %s corrupted request struct on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); if (req->cryptlen != vec->len) pr_err("alg: skcipher: changed 'req->cryptlen'\n"); if (req->iv != iv) @@ -1611,14 +2256,28 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, return -EINVAL; } if (is_test_sglist_corrupted(&tsgls->src)) { - pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } if (tsgls->dst.sgl_ptr != tsgls->src.sgl && is_test_sglist_corrupted(&tsgls->dst)) { - pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); + return -EINVAL; + } + + /* Check for success or failure */ + if (err) { + if (err == vec->crypt_error) + return 0; + pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, vec->crypt_error, err, cfg->name); + return err; + } + if (vec->crypt_error) { + pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, vec->crypt_error, cfg->name); return -EINVAL; } @@ -1626,20 +2285,20 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext, vec->len, 0, true); if (err == -EOVERFLOW) { - pr_err("alg: skcipher: %s %s overran dst buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } if (err) { - pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } /* If applicable, check that the algorithm generated the correct IV */ if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) { - pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); hexdump(iv, ivsize); return -EINVAL; } @@ -1653,14 +2312,17 @@ static int test_skcipher_vec(const char *driver, int enc, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { + char vec_name[16]; unsigned int i; int err; if (fips_enabled && vec->fips_skip) return 0; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_skcipher_vec_cfg(driver, enc, vec, vec_num, + err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -1675,7 +2337,7 @@ static int test_skcipher_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_skcipher_vec_cfg(driver, enc, vec, vec_num, + err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -1685,6 +2347,186 @@ static int test_skcipher_vec(const char *driver, int enc, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate a symmetric cipher test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_cipher_testvec(struct skcipher_request *req, + struct cipher_testvec *vec, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const unsigned int maxkeysize = tfm->keysize; + const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct scatterlist src, dst; + u8 iv[MAX_IVLEN]; + DECLARE_CRYPTO_WAIT(wait); + + /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ + vec->klen = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->klen = prandom_u32() % (maxkeysize + 1); + generate_random_bytes((u8 *)vec->key, vec->klen); + vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen); + + /* IV */ + generate_random_bytes((u8 *)vec->iv, ivsize); + + /* Plaintext */ + vec->len = generate_random_length(maxdatasize); + generate_random_bytes((u8 *)vec->ptext, vec->len); + + /* If the key couldn't be set, no need to continue to encrypt. */ + if (vec->setkey_error) + goto done; + + /* Ciphertext */ + sg_init_one(&src, vec->ptext, vec->len); + sg_init_one(&dst, vec->ctext, vec->len); + memcpy(iv, vec->iv, ivsize); + skcipher_request_set_callback(req, 0, crypto_req_done, &wait); + skcipher_request_set_crypt(req, &src, &dst, vec->len, iv); + vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); +done: + snprintf(name, max_namelen, "\"random: len=%u klen=%u\"", + vec->len, vec->klen); +} + +/* + * Test the skcipher algorithm represented by @req against the corresponding + * generic implementation, if one is available. + */ +static int test_skcipher_vs_generic_impl(const char *driver, + const char *generic_driver, + struct skcipher_request *req, + struct cipher_test_sglists *tsgls) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + const unsigned int blocksize = crypto_skcipher_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_skcipher_alg(tfm)->base.cra_name; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_skcipher *generic_tfm = NULL; + struct skcipher_request *generic_req = NULL; + unsigned int i; + struct cipher_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + /* Keywrap isn't supported here yet as it handles its IV differently. */ + if (strncmp(algname, "kw(", 3) == 0) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL); + if (!generic_req) { + err = -ENOMEM; + goto out; + } + + /* Check the algorithm properties for consistency. */ + + if (tfm->keysize != generic_tfm->keysize) { + pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n", + driver, tfm->keysize, generic_tfm->keysize); + err = -EINVAL; + goto out; + } + + if (ivsize != crypto_skcipher_ivsize(generic_tfm)) { + pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n", + driver, ivsize, crypto_skcipher_ivsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_skcipher_blocksize(generic_tfm)) { + pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, + crypto_skcipher_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + vec.key = kmalloc(tfm->keysize, GFP_KERNEL); + vec.iv = kmalloc(ivsize, GFP_KERNEL); + vec.ptext = kmalloc(maxdatasize, GFP_KERNEL); + vec.ctext = kmalloc(maxdatasize, GFP_KERNEL); + if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_cipher_testvec(generic_req, &vec, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, + &cfg, req, tsgls); + if (err) + goto out; + err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, + &cfg, req, tsgls); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.iv); + kfree(vec.ptext); + kfree(vec.ctext); + crypto_free_skcipher(generic_tfm); + skcipher_request_free(generic_req); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_skcipher_vs_generic_impl(const char *driver, + const char *generic_driver, + struct skcipher_request *req, + struct cipher_test_sglists *tsgls) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int test_skcipher(const char *driver, int enc, const struct cipher_test_suite *suite, struct skcipher_request *req, @@ -1744,6 +2586,11 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, goto out; err = test_skcipher(driver, DECRYPT, suite, req, tsgls); + if (err) + goto out; + + err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req, + tsgls); out: free_cipher_test_sglists(tsgls); skcipher_request_free(req); @@ -2179,7 +3026,6 @@ static int alg_test_crc32c(const struct alg_test_desc *desc, u32 *ctx = (u32 *)shash_desc_ctx(shash); shash->tfm = tfm; - shash->flags = 0; *ctx = 420553207; err = crypto_shash_final(shash, (u8 *)&val); @@ -2493,6 +3339,12 @@ static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver, return err; } +static u8 *test_pack_u32(u8 *dst, u32 val) +{ + memcpy(dst, &val, sizeof(val)); + return dst + sizeof(val); +} + static int test_akcipher_one(struct crypto_akcipher *tfm, const struct akcipher_testvec *vecs) { @@ -2503,10 +3355,11 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, struct crypto_wait wait; unsigned int out_len_max, out_len = 0; int err = -ENOMEM; - struct scatterlist src, dst, src_tab[2]; + struct scatterlist src, dst, src_tab[3]; const char *m, *c; unsigned int m_size, c_size; const char *op; + u8 *key, *ptr; if (testmgr_alloc_buf(xbuf)) return err; @@ -2517,22 +3370,29 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, crypto_init_wait(&wait); + key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len, + GFP_KERNEL); + if (!key) + goto free_xbuf; + memcpy(key, vecs->key, vecs->key_len); + ptr = key + vecs->key_len; + ptr = test_pack_u32(ptr, vecs->algo); + ptr = test_pack_u32(ptr, vecs->param_len); + memcpy(ptr, vecs->params, vecs->param_len); + if (vecs->public_key_vec) - err = crypto_akcipher_set_pub_key(tfm, vecs->key, - vecs->key_len); + err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len); else - err = crypto_akcipher_set_priv_key(tfm, vecs->key, - vecs->key_len); + err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len); if (err) goto free_req; - err = -ENOMEM; - out_len_max = crypto_akcipher_maxsize(tfm); - /* * First run test which do not require a private key, such as * encrypt or verify. */ + err = -ENOMEM; + out_len_max = crypto_akcipher_maxsize(tfm); outbuf_enc = kzalloc(out_len_max, GFP_KERNEL); if (!outbuf_enc) goto free_req; @@ -2558,12 +3418,20 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, goto free_all; memcpy(xbuf[0], m, m_size); - sg_init_table(src_tab, 2); + sg_init_table(src_tab, 3); sg_set_buf(&src_tab[0], xbuf[0], 8); sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8); - sg_init_one(&dst, outbuf_enc, out_len_max); - akcipher_request_set_crypt(req, src_tab, &dst, m_size, - out_len_max); + if (vecs->siggen_sigver_test) { + if (WARN_ON(c_size > PAGE_SIZE)) + goto free_all; + memcpy(xbuf[1], c, c_size); + sg_set_buf(&src_tab[2], xbuf[1], c_size); + akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size); + } else { + sg_init_one(&dst, outbuf_enc, out_len_max); + akcipher_request_set_crypt(req, src_tab, &dst, m_size, + out_len_max); + } akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &wait); @@ -2576,18 +3444,21 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, pr_err("alg: akcipher: %s test failed. err %d\n", op, err); goto free_all; } - if (req->dst_len != c_size) { - pr_err("alg: akcipher: %s test failed. Invalid output len\n", - op); - err = -EINVAL; - goto free_all; - } - /* verify that encrypted message is equal to expected */ - if (memcmp(c, outbuf_enc, c_size)) { - pr_err("alg: akcipher: %s test failed. Invalid output\n", op); - hexdump(outbuf_enc, c_size); - err = -EINVAL; - goto free_all; + if (!vecs->siggen_sigver_test) { + if (req->dst_len != c_size) { + pr_err("alg: akcipher: %s test failed. Invalid output len\n", + op); + err = -EINVAL; + goto free_all; + } + /* verify that encrypted message is equal to expected */ + if (memcmp(c, outbuf_enc, c_size) != 0) { + pr_err("alg: akcipher: %s test failed. Invalid output\n", + op); + hexdump(outbuf_enc, c_size); + err = -EINVAL; + goto free_all; + } } /* @@ -2642,6 +3513,7 @@ free_all: kfree(outbuf_enc); free_req: akcipher_request_free(req); + kfree(key); free_xbuf: testmgr_free_buf(xbuf); return err; @@ -2699,12 +3571,14 @@ static int alg_test_null(const struct alg_test_desc *desc, static const struct alg_test_desc alg_test_descs[] = { { .alg = "adiantum(xchacha12,aes)", + .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)", .test = alg_test_skcipher, .suite = { .cipher = __VECS(adiantum_xchacha12_aes_tv_template) }, }, { .alg = "adiantum(xchacha20,aes)", + .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)", .test = alg_test_skcipher, .suite = { .cipher = __VECS(adiantum_xchacha20_aes_tv_template) @@ -2921,6 +3795,12 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + /* Same as cbc(sm4) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "cbc(psm4)", + .test = alg_test_null, + }, { .alg = "cbc(serpent)", .test = alg_test_skcipher, .suite = { @@ -2947,6 +3827,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ccm(aes)", + .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3055,6 +3936,13 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + + /* Same as ctr(sm4) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "ctr(psm4)", + .test = alg_test_null, + }, { .alg = "ctr(serpent)", .test = alg_test_skcipher, .suite = { @@ -3080,6 +3968,13 @@ static const struct alg_test_desc alg_test_descs[] = { .cipher = __VECS(cts_mode_tv_template) } }, { + /* Same as cts(cbc((aes)) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "cts(cbc(paes))", + .test = alg_test_null, + .fips_allowed = 1, + }, { .alg = "deflate", .test = alg_test_comp, .fips_allowed = 1, @@ -3358,7 +4253,14 @@ static const struct alg_test_desc alg_test_descs[] = { .kpp = __VECS(ecdh_tv_template) } }, { + .alg = "ecrdsa", + .test = alg_test_akcipher, + .suite = { + .akcipher = __VECS(ecrdsa_tv_template) + } + }, { .alg = "gcm(aes)", + .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3477,30 +4379,35 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "lrw(aes)", + .generic_driver = "lrw(ecb(aes-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(aes_lrw_tv_template) } }, { .alg = "lrw(camellia)", + .generic_driver = "lrw(ecb(camellia-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(camellia_lrw_tv_template) } }, { .alg = "lrw(cast6)", + .generic_driver = "lrw(ecb(cast6-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(cast6_lrw_tv_template) } }, { .alg = "lrw(serpent)", + .generic_driver = "lrw(ecb(serpent-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(serpent_lrw_tv_template) } }, { .alg = "lrw(twofish)", + .generic_driver = "lrw(ecb(twofish-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(tf_lrw_tv_template) @@ -3625,6 +4532,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4106(gcm(aes))", + .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3632,6 +4540,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4309(ccm(aes))", + .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3639,6 +4548,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4543(gcm(aes))", + .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(aes_gcm_rfc4543_tv_template) @@ -3835,6 +4745,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, }, { .alg = "xts(aes)", + .generic_driver = "xts(ecb(aes-generic))", .test = alg_test_skcipher, .fips_allowed = 1, .suite = { @@ -3842,12 +4753,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "xts(camellia)", + .generic_driver = "xts(ecb(camellia-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(camellia_xts_tv_template) } }, { .alg = "xts(cast6)", + .generic_driver = "xts(ecb(cast6-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(cast6_xts_tv_template) @@ -3861,12 +4774,14 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "xts(serpent)", + .generic_driver = "xts(ecb(serpent-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(serpent_xts_tv_template) } }, { .alg = "xts(twofish)", + .generic_driver = "xts(ecb(twofish-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(tf_xts_tv_template) @@ -4020,8 +4935,9 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) type, mask); test_done: - if (fips_enabled && rc) - panic("%s: %s alg self test failed in fips mode!\n", driver, alg); + if (rc && (fips_enabled || panic_on_fail)) + panic("alg: self-tests for %s (%s) failed in %s mode!\n", + driver, alg, fips_enabled ? "fips" : "panic_on_fail"); if (fips_enabled && !rc) pr_info("alg: self-tests for %s (%s) passed\n", driver, alg); |