diff options
Diffstat (limited to 'crypto/crypto_engine.c')
| -rw-r--r-- | crypto/crypto_engine.c | 416 |
1 files changed, 294 insertions, 122 deletions
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index 992e8d8dcdd9..18e1689efe12 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -1,25 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Handle async block request by crypto hardware engine. * * Copyright (C) 2016 Linaro, Inc. * * Author: Baolin Wang <baolin.wang@linaro.org> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * */ +#include <crypto/internal/aead.h> +#include <crypto/internal/akcipher.h> +#include <crypto/internal/engine.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/kpp.h> +#include <crypto/internal/skcipher.h> #include <linux/err.h> #include <linux/delay.h> -#include <crypto/engine.h> +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/module.h> #include <uapi/linux/sched/types.h> #include "internal.h" #define CRYPTO_ENGINE_MAX_QLEN 10 +struct crypto_engine_alg { + struct crypto_alg base; + struct crypto_engine_op op; +}; + /** * crypto_finalize_request - finalize one request if the request is done * @engine: the hardware engine @@ -27,33 +35,25 @@ * @err: error number */ static void crypto_finalize_request(struct crypto_engine *engine, - struct crypto_async_request *req, int err) + struct crypto_async_request *req, int err) { unsigned long flags; - bool finalize_cur_req = false; - int ret; - struct crypto_engine_ctx *enginectx; - spin_lock_irqsave(&engine->queue_lock, flags); - if (engine->cur_req == req) - finalize_cur_req = true; - spin_unlock_irqrestore(&engine->queue_lock, flags); - - if (finalize_cur_req) { - enginectx = crypto_tfm_ctx(req->tfm); - if (engine->cur_req_prepared && - enginectx->op.unprepare_request) { - ret = enginectx->op.unprepare_request(engine, req); - if (ret) - dev_err(engine->dev, "failed to unprepare request\n"); - } + /* + * If hardware cannot enqueue more requests + * and retry mechanism is not supported + * make sure we are completing the current request + */ + if (!engine->retry_support) { spin_lock_irqsave(&engine->queue_lock, flags); - engine->cur_req = NULL; - engine->cur_req_prepared = false; + if (engine->cur_req == req) { + engine->cur_req = NULL; + } spin_unlock_irqrestore(&engine->queue_lock, flags); } - req->complete(req, err); + lockdep_assert_in_softirq(); + crypto_request_complete(req, err); kthread_queue_work(engine->kworker, &engine->pump_requests); } @@ -71,23 +71,17 @@ static void crypto_pump_requests(struct crypto_engine *engine, bool in_kthread) { struct crypto_async_request *async_req, *backlog; + struct crypto_engine_alg *alg; + struct crypto_engine_op *op; unsigned long flags; - bool was_busy = false; int ret; - struct crypto_engine_ctx *enginectx; spin_lock_irqsave(&engine->queue_lock, flags); /* Make sure we are not already running a request */ - if (engine->cur_req) + if (!engine->retry_support && engine->cur_req) goto out; - /* If another context is idling then defer */ - if (engine->idling) { - kthread_queue_work(engine->kworker, &engine->pump_requests); - goto out; - } - /* Check if the engine queue is idle */ if (!crypto_queue_len(&engine->queue) || !engine->running) { if (!engine->busy) @@ -101,73 +95,80 @@ static void crypto_pump_requests(struct crypto_engine *engine, } engine->busy = false; - engine->idling = true; - spin_unlock_irqrestore(&engine->queue_lock, flags); - - if (engine->unprepare_crypt_hardware && - engine->unprepare_crypt_hardware(engine)) - dev_err(engine->dev, "failed to unprepare crypt hardware\n"); - - spin_lock_irqsave(&engine->queue_lock, flags); - engine->idling = false; goto out; } +start_request: /* Get the fist request from the engine queue to handle */ backlog = crypto_get_backlog(&engine->queue); async_req = crypto_dequeue_request(&engine->queue); if (!async_req) goto out; - engine->cur_req = async_req; - if (backlog) - backlog->complete(backlog, -EINPROGRESS); + /* + * If hardware doesn't support the retry mechanism, + * keep track of the request we are processing now. + * We'll need it on completion (crypto_finalize_request). + */ + if (!engine->retry_support) + engine->cur_req = async_req; - if (engine->busy) - was_busy = true; - else + if (!engine->busy) engine->busy = true; spin_unlock_irqrestore(&engine->queue_lock, flags); - /* Until here we get the request need to be encrypted successfully */ - if (!was_busy && engine->prepare_crypt_hardware) { - ret = engine->prepare_crypt_hardware(engine); - if (ret) { - dev_err(engine->dev, "failed to prepare crypt hardware\n"); - goto req_err; + alg = container_of(async_req->tfm->__crt_alg, + struct crypto_engine_alg, base); + op = &alg->op; + ret = op->do_one_request(engine, async_req); + + /* Request unsuccessfully executed by hardware */ + if (ret < 0) { + /* + * If hardware queue is full (-ENOSPC), requeue request + * regardless of backlog flag. + * Otherwise, unprepare and complete the request. + */ + if (!engine->retry_support || + (ret != -ENOSPC)) { + dev_err(engine->dev, + "Failed to do one request from queue: %d\n", + ret); + goto req_err_1; } + spin_lock_irqsave(&engine->queue_lock, flags); + /* + * If hardware was unable to execute request, enqueue it + * back in front of crypto-engine queue, to keep the order + * of requests. + */ + crypto_enqueue_request_head(&engine->queue, async_req); + + kthread_queue_work(engine->kworker, &engine->pump_requests); + goto out; } - enginectx = crypto_tfm_ctx(async_req->tfm); + goto retry; - if (enginectx->op.prepare_request) { - ret = enginectx->op.prepare_request(engine, async_req); - if (ret) { - dev_err(engine->dev, "failed to prepare request: %d\n", - ret); - goto req_err; - } - engine->cur_req_prepared = true; - } - if (!enginectx->op.do_one_request) { - dev_err(engine->dev, "failed to do request\n"); - ret = -EINVAL; - goto req_err; - } - ret = enginectx->op.do_one_request(engine, async_req); - if (ret) { - dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret); - goto req_err; - } - return; +req_err_1: + crypto_request_complete(async_req, ret); -req_err: - crypto_finalize_request(engine, async_req, ret); +retry: + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + /* If retry mechanism is supported, send new requests to engine */ + if (engine->retry_support) { + spin_lock_irqsave(&engine->queue_lock, flags); + goto start_request; + } return; out: spin_unlock_irqrestore(&engine->queue_lock, flags); + + return; } static void crypto_pump_work(struct kthread_work *work) @@ -182,6 +183,7 @@ static void crypto_pump_work(struct kthread_work *work) * crypto_transfer_request - transfer the new request into the engine queue * @engine: the hardware engine * @req: the request need to be listed into the engine queue + * @need_pump: indicates whether queue the pump of request to kthread_work */ static int crypto_transfer_request(struct crypto_engine *engine, struct crypto_async_request *req, @@ -219,20 +221,6 @@ static int crypto_transfer_request_to_engine(struct crypto_engine *engine, } /** - * crypto_transfer_ablkcipher_request_to_engine - transfer one ablkcipher_request - * to list into the engine queue - * @engine: the hardware engine - * @req: the request need to be listed into the engine queue - * TODO: Remove this function when skcipher conversion is finished - */ -int crypto_transfer_ablkcipher_request_to_engine(struct crypto_engine *engine, - struct ablkcipher_request *req) -{ - return crypto_transfer_request_to_engine(engine, &req->base); -} -EXPORT_SYMBOL_GPL(crypto_transfer_ablkcipher_request_to_engine); - -/** * crypto_transfer_aead_request_to_engine - transfer one aead_request * to list into the engine queue * @engine: the hardware engine @@ -272,32 +260,30 @@ int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine, EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine); /** - * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request - * to list into the engine queue + * crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list + * into the engine queue * @engine: the hardware engine * @req: the request need to be listed into the engine queue */ -int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine, - struct skcipher_request *req) +int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine, + struct kpp_request *req) { return crypto_transfer_request_to_engine(engine, &req->base); } -EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine); +EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine); /** - * crypto_finalize_ablkcipher_request - finalize one ablkcipher_request if - * the request is done + * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request + * to list into the engine queue * @engine: the hardware engine - * @req: the request need to be finalized - * @err: error number - * TODO: Remove this function when skcipher conversion is finished + * @req: the request need to be listed into the engine queue */ -void crypto_finalize_ablkcipher_request(struct crypto_engine *engine, - struct ablkcipher_request *req, int err) +int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine, + struct skcipher_request *req) { - return crypto_finalize_request(engine, &req->base, err); + return crypto_transfer_request_to_engine(engine, &req->base); } -EXPORT_SYMBOL_GPL(crypto_finalize_ablkcipher_request); +EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine); /** * crypto_finalize_aead_request - finalize one aead_request if @@ -342,6 +328,19 @@ void crypto_finalize_hash_request(struct crypto_engine *engine, EXPORT_SYMBOL_GPL(crypto_finalize_hash_request); /** + * crypto_finalize_kpp_request - finalize one kpp_request if the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +void crypto_finalize_kpp_request(struct crypto_engine *engine, + struct kpp_request *req, int err) +{ + return crypto_finalize_request(engine, &req->base, err); +} +EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request); + +/** * crypto_finalize_skcipher_request - finalize one skcipher_request if * the request is done * @engine: the hardware engine @@ -420,17 +419,21 @@ int crypto_engine_stop(struct crypto_engine *engine) EXPORT_SYMBOL_GPL(crypto_engine_stop); /** - * crypto_engine_alloc_init - allocate crypto hardware engine structure and - * initialize it. + * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure + * and initialize it by setting the maximum number of entries in the software + * crypto-engine queue. * @dev: the device attached with one hardware engine + * @retry_support: whether hardware has support for retry mechanism * @rt: whether this queue is set to run as a realtime task + * @qlen: maximum size of the crypto-engine queue * * This must be called from context that can sleep. * Return: the crypto engine structure on success, else NULL. */ -struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) +struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, + bool retry_support, + bool rt, int qlen) { - struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; struct crypto_engine *engine; if (!dev) @@ -444,16 +447,16 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) engine->rt = rt; engine->running = false; engine->busy = false; - engine->idling = false; - engine->cur_req_prepared = false; + engine->retry_support = retry_support; engine->priv_data = dev; + snprintf(engine->name, sizeof(engine->name), "%s-engine", dev_name(dev)); - crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN); + crypto_init_queue(&engine->queue, qlen); spin_lock_init(&engine->queue_lock); - engine->kworker = kthread_create_worker(0, "%s", engine->name); + engine->kworker = kthread_run_worker(0, "%s", engine->name); if (IS_ERR(engine->kworker)) { dev_err(dev, "failed to create crypto request pump task\n"); return NULL; @@ -462,32 +465,201 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) if (engine->rt) { dev_info(dev, "will run requests pump with realtime priority\n"); - sched_setscheduler(engine->kworker->task, SCHED_FIFO, ¶m); + sched_set_fifo(engine->kworker->task); } return engine; } +EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set); + +/** + * crypto_engine_alloc_init - allocate crypto hardware engine structure and + * initialize it. + * @dev: the device attached with one hardware engine + * @rt: whether this queue is set to run as a realtime task + * + * This must be called from context that can sleep. + * Return: the crypto engine structure on success, else NULL. + */ +struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) +{ + return crypto_engine_alloc_init_and_set(dev, false, rt, + CRYPTO_ENGINE_MAX_QLEN); +} EXPORT_SYMBOL_GPL(crypto_engine_alloc_init); /** * crypto_engine_exit - free the resources of hardware engine when exit * @engine: the hardware engine need to be freed - * - * Return 0 for success. */ -int crypto_engine_exit(struct crypto_engine *engine) +void crypto_engine_exit(struct crypto_engine *engine) { int ret; ret = crypto_engine_stop(engine); if (ret) - return ret; + return; kthread_destroy_worker(engine->kworker); +} +EXPORT_SYMBOL_GPL(crypto_engine_exit); + +int crypto_engine_register_aead(struct aead_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + return crypto_register_aead(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_aead); + +void crypto_engine_unregister_aead(struct aead_engine_alg *alg) +{ + crypto_unregister_aead(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_aead); + +int crypto_engine_register_aeads(struct aead_engine_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_engine_register_aead(&algs[i]); + if (ret) + goto err; + } return 0; + +err: + crypto_engine_unregister_aeads(algs, i); + + return ret; } -EXPORT_SYMBOL_GPL(crypto_engine_exit); +EXPORT_SYMBOL_GPL(crypto_engine_register_aeads); + +void crypto_engine_unregister_aeads(struct aead_engine_alg *algs, int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_engine_unregister_aead(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_aeads); + +int crypto_engine_register_ahash(struct ahash_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + return crypto_register_ahash(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_ahash); + +void crypto_engine_unregister_ahash(struct ahash_engine_alg *alg) +{ + crypto_unregister_ahash(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahash); + +int crypto_engine_register_ahashes(struct ahash_engine_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_engine_register_ahash(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + crypto_engine_unregister_ahashes(algs, i); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_engine_register_ahashes); + +void crypto_engine_unregister_ahashes(struct ahash_engine_alg *algs, + int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_engine_unregister_ahash(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahashes); + +int crypto_engine_register_akcipher(struct akcipher_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + return crypto_register_akcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_akcipher); + +void crypto_engine_unregister_akcipher(struct akcipher_engine_alg *alg) +{ + crypto_unregister_akcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_akcipher); + +int crypto_engine_register_kpp(struct kpp_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + return crypto_register_kpp(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_kpp); + +void crypto_engine_unregister_kpp(struct kpp_engine_alg *alg) +{ + crypto_unregister_kpp(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_kpp); + +int crypto_engine_register_skcipher(struct skcipher_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + return crypto_register_skcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_skcipher); + +void crypto_engine_unregister_skcipher(struct skcipher_engine_alg *alg) +{ + return crypto_unregister_skcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_skcipher); + +int crypto_engine_register_skciphers(struct skcipher_engine_alg *algs, + int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_engine_register_skcipher(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + crypto_engine_unregister_skciphers(algs, i); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_engine_register_skciphers); + +void crypto_engine_unregister_skciphers(struct skcipher_engine_alg *algs, + int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_engine_unregister_skcipher(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_skciphers); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Crypto hardware engine framework"); |