From 108586eba094b318e6a831f977f4ddcc403a15da Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 25 Jul 2022 12:09:28 +0800 Subject: crypto: sahara - don't sleep when in softirq Function of sahara_aes_crypt maybe could be called by function of crypto_skcipher_encrypt during the rx softirq, so it is not allowed to use mutex lock. Fixes: c0c3c89ae347 ("crypto: sahara - replace tasklets with...") Signed-off-by: Zhengchao Shao Signed-off-by: Herbert Xu --- drivers/crypto/sahara.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 457084b344c1..b07ae4ba165e 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -26,10 +26,10 @@ #include #include #include -#include #include #include #include +#include #define SHA_BUFFER_LEN PAGE_SIZE #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE @@ -196,7 +196,7 @@ struct sahara_dev { void __iomem *regs_base; struct clk *clk_ipg; struct clk *clk_ahb; - struct mutex queue_mutex; + spinlock_t queue_spinlock; struct task_struct *kthread; struct completion dma_completion; @@ -642,9 +642,9 @@ static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode) rctx->mode = mode; - mutex_lock(&dev->queue_mutex); + spin_lock_bh(&dev->queue_spinlock); err = crypto_enqueue_request(&dev->queue, &req->base); - mutex_unlock(&dev->queue_mutex); + spin_unlock_bh(&dev->queue_spinlock); wake_up_process(dev->kthread); @@ -1043,10 +1043,10 @@ static int sahara_queue_manage(void *data) do { __set_current_state(TASK_INTERRUPTIBLE); - mutex_lock(&dev->queue_mutex); + spin_lock_bh(&dev->queue_spinlock); backlog = crypto_get_backlog(&dev->queue); async_req = crypto_dequeue_request(&dev->queue); - mutex_unlock(&dev->queue_mutex); + spin_unlock_bh(&dev->queue_spinlock); if (backlog) backlog->complete(backlog, -EINPROGRESS); @@ -1092,9 +1092,9 @@ static int sahara_sha_enqueue(struct ahash_request *req, int last) rctx->first = 1; } - mutex_lock(&dev->queue_mutex); + spin_lock_bh(&dev->queue_spinlock); ret = crypto_enqueue_request(&dev->queue, &req->base); - mutex_unlock(&dev->queue_mutex); + spin_unlock_bh(&dev->queue_spinlock); wake_up_process(dev->kthread); @@ -1449,7 +1449,7 @@ static int sahara_probe(struct platform_device *pdev) crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); - mutex_init(&dev->queue_mutex); + spin_lock_init(&dev->queue_spinlock); dev_ptr = dev; -- cgit From 908f24270d9ccbe120b91e7029b372f3dcd18290 Mon Sep 17 00:00:00 2001 From: Srinivas Kerekare Date: Mon, 25 Jul 2022 11:40:09 +0100 Subject: crypto: qat - add check to validate firmware images The function qat_uclo_check_image() validates the MMP and AE firmware images. If the QAT device supports firmware authentication (indicated by the handle to firmware loader), the input signed binary MMP and AE images are validated by parsing the following information: - Header length - Full size of the binary - Type of binary image (MMP or AE Firmware) Firmware binaries use RSA3K for signing and verification. The header length for the RSA3k is 0x384 bytes. All the size field values in the binary are quantified as DWORDS (1 DWORD = 4bytes). On an invalid value the function prints an error message and returns with an error code "EINVAL". Signed-off-by: Srinivas Kerekare Reviewed-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_uclo.h | 3 +- drivers/crypto/qat/qat_common/qat_uclo.c | 56 +++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 4b36869bf460..69482abdb8b9 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -86,7 +86,8 @@ ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ ICP_QAT_CSS_SIGNATURE_LEN(handle)) -#define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000 +#define ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN 0x40000 +#define ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN 0x30000 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) #define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 0fe5a474aa45..b7f7869ef8b2 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1367,6 +1367,48 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, } } +static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + unsigned int fw_type) +{ + char *fw_type_name = fw_type ? "MMP" : "AE"; + unsigned int css_dword_size = sizeof(u32); + + if (handle->chip_info->fw_auth) { + struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; + unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); + + if ((css_hdr->header_len * css_dword_size) != header_len) + goto err; + if ((css_hdr->size * css_dword_size) != size) + goto err; + if (fw_type != css_hdr->fw_type) + goto err; + if (size <= header_len) + goto err; + size -= header_len; + } + + if (fw_type == CSS_AE_FIRMWARE) { + if (size < sizeof(struct icp_qat_simg_ae_mode *) + + ICP_QAT_SIMG_AE_INIT_SEQ_LEN) + goto err; + if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) + goto err; + } else if (fw_type == CSS_MMP_FIRMWARE) { + if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) + goto err; + } else { + pr_err("QAT: Unsupported firmware type\n"); + return -EINVAL; + } + return 0; + +err: + pr_err("QAT: Invalid %s firmware image\n", fw_type_name); + return -EINVAL; +} + static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, char *image, unsigned int size, struct icp_qat_fw_auth_desc **desc) @@ -1379,7 +1421,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_simg_ae_mode *simg_ae_mode; struct icp_firml_dram_desc img_desc; - if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { + if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)) { pr_err("QAT: error, input image size overflow %d\n", size); return -EINVAL; } @@ -1547,6 +1589,11 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_fw_auth_desc *desc = NULL; int status = 0; + int ret; + + ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE); + if (ret) + return ret; if (handle->chip_info->fw_auth) { status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); @@ -2018,8 +2065,15 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) struct icp_qat_fw_auth_desc *desc = NULL; struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; + int ret; for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { + ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf, + simg_hdr[i].simg_len, + CSS_AE_FIRMWARE); + if (ret) + return ret; + if (qat_uclo_map_auth_fw(handle, (char *)simg_hdr[i].simg_buf, (unsigned int) -- cgit From 7033b937e21b12629d920e7864c20c46bc4ccf39 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 25 Jul 2022 11:36:34 -0700 Subject: crypto: lib - create utils module and move __crypto_memneq into it As requested at https://lore.kernel.org/r/YtEgzHuuMts0YBCz@gondor.apana.org.au, move __crypto_memneq into lib/crypto/ and put it under a new tristate. The tristate is CRYPTO_LIB_UTILS, and it builds a module libcryptoutils. As more crypto library utilities are being added, this creates a single place for them to go without cluttering up the main lib directory. The module's main file will be lib/crypto/utils.c. However, leave memneq.c as its own file because of its nonstandard license. Signed-off-by: Eric Biggers Reviewed-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- lib/Kconfig | 3 - lib/Makefile | 1 - lib/crypto/Kconfig | 5 +- lib/crypto/Makefile | 3 + lib/crypto/memneq.c | 179 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/memneq.c | 176 --------------------------------------------------- 7 files changed, 187 insertions(+), 182 deletions(-) create mode 100644 lib/crypto/memneq.c delete mode 100644 lib/memneq.c diff --git a/crypto/Kconfig b/crypto/Kconfig index bb427a835e44..b1ccf873779d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -15,7 +15,7 @@ source "crypto/async_tx/Kconfig" # menuconfig CRYPTO tristate "Cryptographic API" - select LIB_MEMNEQ + select CRYPTO_LIB_UTILS help This option provides the core Cryptographic API. diff --git a/lib/Kconfig b/lib/Kconfig index dc1ab2ed1dc6..6f113ade49ca 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -127,9 +127,6 @@ config TRACE_MMIO_ACCESS source "lib/crypto/Kconfig" -config LIB_MEMNEQ - bool - config CRC_CCITT tristate "CRC-CCITT functions" help diff --git a/lib/Makefile b/lib/Makefile index c95212141928..0eebd502e09d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -253,7 +253,6 @@ obj-$(CONFIG_DIMLIB) += dim/ obj-$(CONFIG_SIGNATURE) += digsig.o lib-$(CONFIG_CLZ_TAB) += clz_tab.o -lib-$(CONFIG_LIB_MEMNEQ) += memneq.o obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 9ff549f63540..b09d9d6546cb 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -2,6 +2,9 @@ menu "Crypto library routines" +config CRYPTO_LIB_UTILS + tristate + config CRYPTO_LIB_AES tristate @@ -71,7 +74,7 @@ config CRYPTO_LIB_CURVE25519 tristate "Curve25519 scalar multiplication library" depends on CRYPTO_ARCH_HAVE_LIB_CURVE25519 || !CRYPTO_ARCH_HAVE_LIB_CURVE25519 select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n - select LIB_MEMNEQ + select CRYPTO_LIB_UTILS help Enable the Curve25519 library interface. This interface may be fulfilled by either the generic implementation or an arch-specific diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 919cbb2c220d..b956b3bae26a 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o +libcryptoutils-y := memneq.o + # chacha is used by the /dev/random driver which is always builtin obj-y += chacha.o obj-$(CONFIG_CRYPTO_LIB_CHACHA_GENERIC) += libchacha.o diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c new file mode 100644 index 000000000000..f20983184284 --- /dev/null +++ b/lib/crypto/memneq.c @@ -0,0 +1,179 @@ +/* + * Constant-time equality testing of memory regions. + * + * Authors: + * + * James Yonan + * Daniel Borkmann + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * The full GNU General Public License is included in this distribution + * in the file called LICENSE.GPL. + * + * BSD LICENSE + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of OpenVPN Technologies nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ + +/* Generic path for arbitrary size */ +static inline unsigned long +__crypto_memneq_generic(const void *a, const void *b, size_t size) +{ + unsigned long neq = 0; + +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + while (size >= sizeof(unsigned long)) { + neq |= get_unaligned((unsigned long *)a) ^ + get_unaligned((unsigned long *)b); + OPTIMIZER_HIDE_VAR(neq); + a += sizeof(unsigned long); + b += sizeof(unsigned long); + size -= sizeof(unsigned long); + } +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + while (size > 0) { + neq |= *(unsigned char *)a ^ *(unsigned char *)b; + OPTIMIZER_HIDE_VAR(neq); + a += 1; + b += 1; + size -= 1; + } + return neq; +} + +/* Loop-free fast-path for frequently used 16-byte size */ +static inline unsigned long __crypto_memneq_16(const void *a, const void *b) +{ + unsigned long neq = 0; + +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (sizeof(unsigned long) == 8) { + neq |= get_unaligned((unsigned long *)a) ^ + get_unaligned((unsigned long *)b); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned long *)(a + 8)) ^ + get_unaligned((unsigned long *)(b + 8)); + OPTIMIZER_HIDE_VAR(neq); + } else if (sizeof(unsigned int) == 4) { + neq |= get_unaligned((unsigned int *)a) ^ + get_unaligned((unsigned int *)b); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned int *)(a + 4)) ^ + get_unaligned((unsigned int *)(b + 4)); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned int *)(a + 8)) ^ + get_unaligned((unsigned int *)(b + 8)); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned int *)(a + 12)) ^ + get_unaligned((unsigned int *)(b + 12)); + OPTIMIZER_HIDE_VAR(neq); + } else +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + { + neq |= *(unsigned char *)(a) ^ *(unsigned char *)(b); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+1) ^ *(unsigned char *)(b+1); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+2) ^ *(unsigned char *)(b+2); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+3) ^ *(unsigned char *)(b+3); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+4) ^ *(unsigned char *)(b+4); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+5) ^ *(unsigned char *)(b+5); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+6) ^ *(unsigned char *)(b+6); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+7) ^ *(unsigned char *)(b+7); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+8) ^ *(unsigned char *)(b+8); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+9) ^ *(unsigned char *)(b+9); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+10) ^ *(unsigned char *)(b+10); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+11) ^ *(unsigned char *)(b+11); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+12) ^ *(unsigned char *)(b+12); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+13) ^ *(unsigned char *)(b+13); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+14) ^ *(unsigned char *)(b+14); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+15) ^ *(unsigned char *)(b+15); + OPTIMIZER_HIDE_VAR(neq); + } + + return neq; +} + +/* Compare two areas of memory without leaking timing information, + * and with special optimizations for common sizes. Users should + * not call this function directly, but should instead use + * crypto_memneq defined in crypto/algapi.h. + */ +noinline unsigned long __crypto_memneq(const void *a, const void *b, + size_t size) +{ + switch (size) { + case 16: + return __crypto_memneq_16(a, b); + default: + return __crypto_memneq_generic(a, b, size); + } +} +EXPORT_SYMBOL(__crypto_memneq); + +#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ + +MODULE_LICENSE("GPL"); diff --git a/lib/memneq.c b/lib/memneq.c deleted file mode 100644 index fb11608b1ec1..000000000000 --- a/lib/memneq.c +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Constant-time equality testing of memory regions. - * - * Authors: - * - * James Yonan - * Daniel Borkmann - * - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * BSD LICENSE - * - * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of OpenVPN Technologies nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include - -#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ - -/* Generic path for arbitrary size */ -static inline unsigned long -__crypto_memneq_generic(const void *a, const void *b, size_t size) -{ - unsigned long neq = 0; - -#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) - while (size >= sizeof(unsigned long)) { - neq |= get_unaligned((unsigned long *)a) ^ - get_unaligned((unsigned long *)b); - OPTIMIZER_HIDE_VAR(neq); - a += sizeof(unsigned long); - b += sizeof(unsigned long); - size -= sizeof(unsigned long); - } -#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ - while (size > 0) { - neq |= *(unsigned char *)a ^ *(unsigned char *)b; - OPTIMIZER_HIDE_VAR(neq); - a += 1; - b += 1; - size -= 1; - } - return neq; -} - -/* Loop-free fast-path for frequently used 16-byte size */ -static inline unsigned long __crypto_memneq_16(const void *a, const void *b) -{ - unsigned long neq = 0; - -#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS - if (sizeof(unsigned long) == 8) { - neq |= get_unaligned((unsigned long *)a) ^ - get_unaligned((unsigned long *)b); - OPTIMIZER_HIDE_VAR(neq); - neq |= get_unaligned((unsigned long *)(a + 8)) ^ - get_unaligned((unsigned long *)(b + 8)); - OPTIMIZER_HIDE_VAR(neq); - } else if (sizeof(unsigned int) == 4) { - neq |= get_unaligned((unsigned int *)a) ^ - get_unaligned((unsigned int *)b); - OPTIMIZER_HIDE_VAR(neq); - neq |= get_unaligned((unsigned int *)(a + 4)) ^ - get_unaligned((unsigned int *)(b + 4)); - OPTIMIZER_HIDE_VAR(neq); - neq |= get_unaligned((unsigned int *)(a + 8)) ^ - get_unaligned((unsigned int *)(b + 8)); - OPTIMIZER_HIDE_VAR(neq); - neq |= get_unaligned((unsigned int *)(a + 12)) ^ - get_unaligned((unsigned int *)(b + 12)); - OPTIMIZER_HIDE_VAR(neq); - } else -#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ - { - neq |= *(unsigned char *)(a) ^ *(unsigned char *)(b); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+1) ^ *(unsigned char *)(b+1); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+2) ^ *(unsigned char *)(b+2); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+3) ^ *(unsigned char *)(b+3); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+4) ^ *(unsigned char *)(b+4); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+5) ^ *(unsigned char *)(b+5); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+6) ^ *(unsigned char *)(b+6); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+7) ^ *(unsigned char *)(b+7); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+8) ^ *(unsigned char *)(b+8); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+9) ^ *(unsigned char *)(b+9); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+10) ^ *(unsigned char *)(b+10); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+11) ^ *(unsigned char *)(b+11); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+12) ^ *(unsigned char *)(b+12); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+13) ^ *(unsigned char *)(b+13); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+14) ^ *(unsigned char *)(b+14); - OPTIMIZER_HIDE_VAR(neq); - neq |= *(unsigned char *)(a+15) ^ *(unsigned char *)(b+15); - OPTIMIZER_HIDE_VAR(neq); - } - - return neq; -} - -/* Compare two areas of memory without leaking timing information, - * and with special optimizations for common sizes. Users should - * not call this function directly, but should instead use - * crypto_memneq defined in crypto/algapi.h. - */ -noinline unsigned long __crypto_memneq(const void *a, const void *b, - size_t size) -{ - switch (size) { - case 16: - return __crypto_memneq_16(a, b); - default: - return __crypto_memneq_generic(a, b, size); - } -} -EXPORT_SYMBOL(__crypto_memneq); - -#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ -- cgit From 6e78ad0bb45dd20b3c1a56c72a32e1d82f98b422 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 25 Jul 2022 11:36:35 -0700 Subject: crypto: lib - move __crypto_xor into utils CRYPTO_LIB_CHACHA depends on CRYPTO for __crypto_xor, defined in crypto/algapi.c. This is a layering violation because the dependencies should only go in the other direction (crypto/ => lib/crypto/). Also the correct dependency would be CRYPTO_ALGAPI, not CRYPTO. Fix this by moving __crypto_xor into the utils module in lib/crypto/. Note that CRYPTO_LIB_CHACHA_GENERIC selected XOR_BLOCKS, which is unrelated and unnecessary. It was perhaps thought that XOR_BLOCKS was needed for __crypto_xor, but that's not the case. Signed-off-by: Eric Biggers Reviewed-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- crypto/algapi.c | 71 ------------------------------------------ lib/crypto/Kconfig | 3 +- lib/crypto/Makefile | 2 +- lib/crypto/memneq.c | 2 -- lib/crypto/utils.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 76 deletions(-) create mode 100644 lib/crypto/utils.c diff --git a/crypto/algapi.c b/crypto/algapi.c index d1c99288af3e..5c69ff8e8fa5 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -997,77 +997,6 @@ void crypto_inc(u8 *a, unsigned int size) } EXPORT_SYMBOL_GPL(crypto_inc); -void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) -{ - int relalign = 0; - - if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - int size = sizeof(unsigned long); - int d = (((unsigned long)dst ^ (unsigned long)src1) | - ((unsigned long)dst ^ (unsigned long)src2)) & - (size - 1); - - relalign = d ? 1 << __ffs(d) : size; - - /* - * If we care about alignment, process as many bytes as - * needed to advance dst and src to values whose alignments - * equal their relative alignment. This will allow us to - * process the remainder of the input using optimal strides. - */ - while (((unsigned long)dst & (relalign - 1)) && len > 0) { - *dst++ = *src1++ ^ *src2++; - len--; - } - } - - while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { - if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - u64 l = get_unaligned((u64 *)src1) ^ - get_unaligned((u64 *)src2); - put_unaligned(l, (u64 *)dst); - } else { - *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; - } - dst += 8; - src1 += 8; - src2 += 8; - len -= 8; - } - - while (len >= 4 && !(relalign & 3)) { - if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - u32 l = get_unaligned((u32 *)src1) ^ - get_unaligned((u32 *)src2); - put_unaligned(l, (u32 *)dst); - } else { - *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; - } - dst += 4; - src1 += 4; - src2 += 4; - len -= 4; - } - - while (len >= 2 && !(relalign & 1)) { - if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - u16 l = get_unaligned((u16 *)src1) ^ - get_unaligned((u16 *)src2); - put_unaligned(l, (u16 *)dst); - } else { - *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; - } - dst += 2; - src1 += 2; - src2 += 2; - len -= 2; - } - - while (len--) - *dst++ = *src1++ ^ *src2++; -} -EXPORT_SYMBOL_GPL(__crypto_xor); - unsigned int crypto_alg_extsize(struct crypto_alg *alg) { return alg->cra_ctxsize + diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index b09d9d6546cb..7e9683e9f5c6 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -36,7 +36,7 @@ config CRYPTO_ARCH_HAVE_LIB_CHACHA config CRYPTO_LIB_CHACHA_GENERIC tristate - select XOR_BLOCKS + select CRYPTO_LIB_UTILS help This symbol can be depended upon by arch implementations of the ChaCha library interface that require the generic code as a @@ -46,7 +46,6 @@ config CRYPTO_LIB_CHACHA_GENERIC config CRYPTO_LIB_CHACHA tristate "ChaCha library interface" - depends on CRYPTO depends on CRYPTO_ARCH_HAVE_LIB_CHACHA || !CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC if CRYPTO_ARCH_HAVE_LIB_CHACHA=n help diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index b956b3bae26a..c852f067ab06 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o -libcryptoutils-y := memneq.o +libcryptoutils-y := memneq.o utils.o # chacha is used by the /dev/random driver which is always builtin obj-y += chacha.o diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c index f20983184284..d1e8c86fbb0f 100644 --- a/lib/crypto/memneq.c +++ b/lib/crypto/memneq.c @@ -175,5 +175,3 @@ noinline unsigned long __crypto_memneq(const void *a, const void *b, EXPORT_SYMBOL(__crypto_memneq); #endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ - -MODULE_LICENSE("GPL"); diff --git a/lib/crypto/utils.c b/lib/crypto/utils.c new file mode 100644 index 000000000000..53230ab1b195 --- /dev/null +++ b/lib/crypto/utils.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto library utility functions + * + * Copyright (c) 2006 Herbert Xu + */ + +#include +#include +#include + +/* + * XOR @len bytes from @src1 and @src2 together, writing the result to @dst + * (which may alias one of the sources). Don't call this directly; call + * crypto_xor() or crypto_xor_cpy() instead. + */ +void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) +{ + int relalign = 0; + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + int size = sizeof(unsigned long); + int d = (((unsigned long)dst ^ (unsigned long)src1) | + ((unsigned long)dst ^ (unsigned long)src2)) & + (size - 1); + + relalign = d ? 1 << __ffs(d) : size; + + /* + * If we care about alignment, process as many bytes as + * needed to advance dst and src to values whose alignments + * equal their relative alignment. This will allow us to + * process the remainder of the input using optimal strides. + */ + while (((unsigned long)dst & (relalign - 1)) && len > 0) { + *dst++ = *src1++ ^ *src2++; + len--; + } + } + + while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + u64 l = get_unaligned((u64 *)src1) ^ + get_unaligned((u64 *)src2); + put_unaligned(l, (u64 *)dst); + } else { + *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; + } + dst += 8; + src1 += 8; + src2 += 8; + len -= 8; + } + + while (len >= 4 && !(relalign & 3)) { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + u32 l = get_unaligned((u32 *)src1) ^ + get_unaligned((u32 *)src2); + put_unaligned(l, (u32 *)dst); + } else { + *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; + } + dst += 4; + src1 += 4; + src2 += 4; + len -= 4; + } + + while (len >= 2 && !(relalign & 1)) { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + u16 l = get_unaligned((u16 *)src1) ^ + get_unaligned((u16 *)src2); + put_unaligned(l, (u16 *)dst); + } else { + *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; + } + dst += 2; + src1 += 2; + src2 += 2; + len -= 2; + } + + while (len--) + *dst++ = *src1++ ^ *src2++; +} +EXPORT_SYMBOL_GPL(__crypto_xor); + +MODULE_LICENSE("GPL"); -- cgit From 4a772c40006ad488a14acedd0cc83c6f574f16fc Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 25 Jul 2022 11:36:36 -0700 Subject: crypto: lib - remove __HAVE_ARCH_CRYPTO_MEMNEQ No architecture actually defines this, so it's unneeded. Signed-off-by: Eric Biggers Reviewed-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- lib/crypto/memneq.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c index d1e8c86fbb0f..243d8677cc51 100644 --- a/lib/crypto/memneq.c +++ b/lib/crypto/memneq.c @@ -63,8 +63,6 @@ #include #include -#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ - /* Generic path for arbitrary size */ static inline unsigned long __crypto_memneq_generic(const void *a, const void *b, size_t size) @@ -173,5 +171,3 @@ noinline unsigned long __crypto_memneq(const void *a, const void *b, } } EXPORT_SYMBOL(__crypto_memneq); - -#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ -- cgit From d74f9340097a881869c4c22ca376654cc2516ecc Mon Sep 17 00:00:00 2001 From: Ye Weihua Date: Thu, 28 Jul 2022 10:07:58 +0800 Subject: crypto: hisilicon/zip - fix mismatch in get/set sgl_sge_nr KASAN reported this Bug: [17619.659757] BUG: KASAN: global-out-of-bounds in param_get_int+0x34/0x60 [17619.673193] Read of size 4 at addr fffff01332d7ed00 by task read_all/1507958 ... [17619.698934] The buggy address belongs to the variable: [17619.708371] sgl_sge_nr+0x0/0xffffffffffffa300 [hisi_zip] There is a mismatch in hisi_zip when get/set the variable sgl_sge_nr. The type of sgl_sge_nr is u16, and get/set sgl_sge_nr by param_get/set_int. Replacing param_get/set_int to param_get/set_ushort can fix this bug. Fixes: f081fda293ffb ("crypto: hisilicon - add sgl_sge_nr module param for zip") Signed-off-by: Ye Weihua Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_crypto.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index ad35434a3fdb..06a2d6e81ae9 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -123,12 +123,12 @@ static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) return -EINVAL; - return param_set_int(val, kp); + return param_set_ushort(val, kp); } static const struct kernel_param_ops sgl_sge_nr_ops = { .set = sgl_sge_nr_set, - .get = param_get_int, + .get = param_get_ushort, }; static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; -- cgit From 36cb6494429bd64b27b7ff8b4af56f8e526da2b4 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Thu, 28 Jul 2022 18:22:20 +0800 Subject: hwrng: core - let sleep be interrupted when unregistering hwrng MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are two deadlock scenarios that need addressing, which cause problems when the computer goes to sleep, the interface is set down, and hwrng_unregister() is called. When the deadlock is hit, sleep is delayed for tens of seconds, causing it to fail. These scenarios are: 1) The hwrng kthread can't be stopped while it's sleeping, because it uses msleep_interruptible() which does not react to kthread_stop. 2) A normal user thread can't be interrupted by hwrng_unregister() while it's sleeping, because hwrng_unregister() is called from elsewhere. We solve both issues by add a completion object called dying that fulfils waiters once we have started the process in hwrng_unregister. At the same time, we should cleanup a common and useless dmesg splat in the same area. Cc: Reported-by: Gregory Erwin Fixes: fcd09c90c3c5 ("ath9k: use hw_random API instead of directly dumping into random.c") Link: https://lore.kernel.org/all/CAO+Okf6ZJC5-nTE_EJUGQtd8JiCkiEHytGgDsFGTEjs0c00giw@mail.gmail.com/ Link: https://lore.kernel.org/lkml/CAO+Okf5k+C+SE6pMVfPf-d8MfVPVq4PO7EY8Hys_DVXtent3HA@mail.gmail.com/ Link: https://bugs.archlinux.org/task/75138 Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu Acked-by: Toke Høiland-Jørgensen Acked-by: Kalle Valo Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 19 +++++++++++++++---- drivers/net/wireless/ath/ath9k/rng.c | 3 ++- include/linux/hw_random.h | 3 +++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 16f227b995e8..d7045dfaf16c 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -507,16 +507,17 @@ static int hwrng_fillfn(void *unused) rng->quality = current_quality; /* obsolete */ quality = rng->quality; mutex_unlock(&reading_mutex); + + if (rc <= 0) + hwrng_msleep(rng, 10000); + put_rng(rng); if (!quality) break; - if (rc <= 0) { - pr_warn("hwrng: no data available\n"); - msleep_interruptible(10000); + if (rc <= 0) continue; - } /* If we cannot credit at least one bit of entropy, * keep track of the remainder for the next iteration @@ -570,6 +571,7 @@ int hwrng_register(struct hwrng *rng) init_completion(&rng->cleanup_done); complete(&rng->cleanup_done); + init_completion(&rng->dying); if (!current_rng || (!cur_rng_set_by_user && rng->quality > current_rng->quality)) { @@ -617,6 +619,7 @@ void hwrng_unregister(struct hwrng *rng) old_rng = current_rng; list_del(&rng->list); + complete_all(&rng->dying); if (current_rng == rng) { err = enable_best_rng(); if (err) { @@ -685,6 +688,14 @@ void devm_hwrng_unregister(struct device *dev, struct hwrng *rng) } EXPORT_SYMBOL_GPL(devm_hwrng_unregister); +long hwrng_msleep(struct hwrng *rng, unsigned int msecs) +{ + unsigned long timeout = msecs_to_jiffies(msecs) + 1; + + return wait_for_completion_interruptible_timeout(&rng->dying, timeout); +} +EXPORT_SYMBOL_GPL(hwrng_msleep); + static int __init hwrng_modinit(void) { int ret; diff --git a/drivers/net/wireless/ath/ath9k/rng.c b/drivers/net/wireless/ath/ath9k/rng.c index cb5414265a9b..58c0ab01771b 100644 --- a/drivers/net/wireless/ath/ath9k/rng.c +++ b/drivers/net/wireless/ath/ath9k/rng.c @@ -83,7 +83,8 @@ static int ath9k_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) if (!wait || !max || likely(bytes_read) || fail_stats > 110) break; - msleep_interruptible(ath9k_rng_delay_get(++fail_stats)); + if (hwrng_msleep(rng, ath9k_rng_delay_get(++fail_stats))) + break; } if (wait && !bytes_read && max) diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index aa1d4da03538..77c2885c4c13 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -50,6 +50,7 @@ struct hwrng { struct list_head list; struct kref ref; struct completion cleanup_done; + struct completion dying; }; struct device; @@ -61,4 +62,6 @@ extern int devm_hwrng_register(struct device *dev, struct hwrng *rng); extern void hwrng_unregister(struct hwrng *rng); extern void devm_hwrng_unregister(struct device *dve, struct hwrng *rng); +extern long hwrng_msleep(struct hwrng *rng, unsigned int msecs); + #endif /* LINUX_HWRANDOM_H_ */ -- cgit From 882aa6525cabcfa0cea61e1a19c9af4c543118ac Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 29 Jul 2022 17:35:31 +0800 Subject: crypto: qcom-rng - Fix qcom_rng_of_match unused warning Module device tables need to be declared as maybe_unused because they will be unused when built-in and the corresponding option is also disabled. This patch adds the maybe_unused attributes to OF and ACPI. This also allows us to remove the ifdef around the ACPI data structure. Reported-by: kernel test robot Signed-off-by: Herbert Xu Reviewed-by: Vinod Koul Signed-off-by: Herbert Xu --- drivers/crypto/qcom-rng.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 031b5f701a0a..72dd1a4ebac4 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -201,15 +202,13 @@ static int qcom_rng_remove(struct platform_device *pdev) return 0; } -#if IS_ENABLED(CONFIG_ACPI) -static const struct acpi_device_id qcom_rng_acpi_match[] = { +static const struct acpi_device_id __maybe_unused qcom_rng_acpi_match[] = { { .id = "QCOM8160", .driver_data = 1 }, {} }; MODULE_DEVICE_TABLE(acpi, qcom_rng_acpi_match); -#endif -static const struct of_device_id qcom_rng_of_match[] = { +static const struct of_device_id __maybe_unused qcom_rng_of_match[] = { { .compatible = "qcom,prng", .data = (void *)0}, { .compatible = "qcom,prng-ee", .data = (void *)1}, {} -- cgit From 042b4b169c6fb9d4df268d66282d7302dd73d37b Mon Sep 17 00:00:00 2001 From: James Cowgill Date: Mon, 1 Aug 2022 20:04:18 +0000 Subject: hwrng: arm-smccc-trng - fix NO_ENTROPY handling The SMCCC_RET_TRNG_NO_ENTROPY switch arm is never used because the NO_ENTROPY return value is negative and negative values are handled above the switch by immediately returning. Fix by handling errors using a default arm in the switch. Fixes: 0888d04b47a1 ("hwrng: Add Arm SMCCC TRNG based driver") Signed-off-by: James Cowgill Signed-off-by: Herbert Xu --- drivers/char/hw_random/arm_smccc_trng.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/char/hw_random/arm_smccc_trng.c b/drivers/char/hw_random/arm_smccc_trng.c index b24ac39a903b..e34c3ea692b6 100644 --- a/drivers/char/hw_random/arm_smccc_trng.c +++ b/drivers/char/hw_random/arm_smccc_trng.c @@ -71,8 +71,6 @@ static int smccc_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) MAX_BITS_PER_CALL); arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND, bits, &res); - if ((int)res.a0 < 0) - return (int)res.a0; switch ((int)res.a0) { case SMCCC_RET_SUCCESS: @@ -88,6 +86,8 @@ static int smccc_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) return copied; cond_resched(); break; + default: + return -EIO; } } -- cgit From 00278564a60e11df8bcca0ececd8b2f55434e406 Mon Sep 17 00:00:00 2001 From: Zhuo Chen Date: Tue, 2 Aug 2022 11:29:37 +0800 Subject: crypto: hisilicon - Remove pci_aer_clear_nonfatal_status() call Calls to pci_cleanup_aer_uncorrect_error_status() have already been removed after commit 62b36c3ea664 ("PCI/AER: Remove pci_cleanup_aer_uncorrect_error_status() calls"). But in commit 6c6dd5802c2d ("crypto: hisilicon/qm - add controller reset interface") pci_aer_clear_nonfatal_status() was used again, so remove it in this patch. note: pci_cleanup_aer_uncorrect_error_status() was renamed to pci_aer_clear_nonfatal_status() in commit 894020fdd88c ("PCI/AER: Rationalize error status register clearing") Signed-off-by: Zhuo Chen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index ad83c194d664..9701434a8ef3 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -5466,8 +5466,6 @@ pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) if (pdev->is_virtfn) return PCI_ERS_RESULT_RECOVERED; - pci_aer_clear_nonfatal_status(pdev); - /* reset pcie device controller */ ret = qm_controller_reset(qm); if (ret) { -- cgit From 7433d2fda2f0d1fcbfc72d3b36f908ad94aebf6d Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Tue, 2 Aug 2022 07:48:20 +0000 Subject: crypto: sun8i-ce - using the pm_runtime_resume_and_get to simplify the code Using pm_runtime_resume_and_get() to instade of pm_runtime_get_sync and pm_runtime_put_noidle. Reported-by: Zeal Robot Signed-off-by: ye xingchen Acked-by: Jernej Skrabec Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c index 19cd2e52f89d..c4b0a8b58842 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -54,11 +54,9 @@ static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wa goto err_dst; } - err = pm_runtime_get_sync(ce->dev); - if (err < 0) { - pm_runtime_put_noidle(ce->dev); + err = pm_runtime_resume_and_get(ce->dev); + if (err < 0) goto err_pm; - } mutex_lock(&ce->rnglock); chan = &ce->chanlist[flow]; -- cgit From 56fae4304c8eea2dfb9037ded1aa774acb110d69 Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Wed, 3 Aug 2022 22:47:55 +0200 Subject: crypto: keembay-ocs - Drop obsolete dependency on COMPILE_TEST Since commit 0166dc11be91 ("of: make CONFIG_OF user selectable"), it is possible to test-build any driver which depends on OF on any architecture by explicitly selecting OF. Therefore depending on COMPILE_TEST as an alternative is no longer needed. It is actually better to always build such drivers with OF enabled, so that the test builds are closer to how each driver will actually be built on its intended target. Building them without OF may not test much as the compiler will optimize out potentially large parts of the code. In the worst case, this could even pop false positive warnings. Dropping COMPILE_TEST here improves the quality of our testing and avoids wasting time on non-existent issues. Signed-off-by: Jean Delvare Cc: Declan Murphy Cc: Daniele Alessandrelli Cc: Mark Gross Cc: Herbert Xu Cc: Prabhjot Khurana Cc: "David S. Miller" Signed-off-by: Herbert Xu --- drivers/crypto/keembay/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig index 7942b48dd55a..1cd62f9c3e3a 100644 --- a/drivers/crypto/keembay/Kconfig +++ b/drivers/crypto/keembay/Kconfig @@ -42,7 +42,7 @@ config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS config CRYPTO_DEV_KEEMBAY_OCS_ECC tristate "Support for Intel Keem Bay OCS ECC HW acceleration" depends on ARCH_KEEMBAY || COMPILE_TEST - depends on OF || COMPILE_TEST + depends on OF depends on HAS_IOMEM select CRYPTO_ECDH select CRYPTO_ENGINE @@ -64,7 +64,7 @@ config CRYPTO_DEV_KEEMBAY_OCS_HCU select CRYPTO_ENGINE depends on HAS_IOMEM depends on ARCH_KEEMBAY || COMPILE_TEST - depends on OF || COMPILE_TEST + depends on OF help Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash Control Unit (HCU) hardware acceleration for use with Crypto API. -- cgit From b3b9fdf1a9be4266b01a2063b1f37cdc20806e3b Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Wed, 10 Aug 2022 01:49:15 +0300 Subject: crypto: ccp - Add a quirk to firmware update A quirk for fixing the committed TCB version, when upgrading from a firmware version earlier than 1.50. This is a known issue, and the documented workaround is to load the firmware twice. Currently, this issue requires the following workaround: sudo modprobe -r kvm_amd sudo modprobe -r ccp sudo modprobe ccp sudo modprobe kvm_amd Implement this workaround inside kernel by checking whether the API version is less than 1.50, and if so, download the firmware twice. This addresses the TCB version issue. Link: https://lore.kernel.org/all/de02389f-249d-f565-1136-4af3655fab2a@profian.com/ Reported-by: Harald Hoyer Signed-off-by: Jarkko Sakkinen Acked-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 9f588c9728f8..b292641c8a99 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -744,6 +744,11 @@ static int sev_update_firmware(struct device *dev) struct page *p; u64 data_size; + if (!sev_version_greater_or_equal(0, 15)) { + dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n"); + return -1; + } + if (sev_get_firmware(dev, &firmware) == -ENOENT) { dev_dbg(dev, "No SEV firmware file present\n"); return -1; @@ -776,6 +781,14 @@ static int sev_update_firmware(struct device *dev) data->len = firmware->size; ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error); + + /* + * A quirk for fixing the committed TCB version, when upgrading from + * earlier firmware version than 1.50. + */ + if (!ret && !sev_version_greater_or_equal(1, 50)) + ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error); + if (ret) dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error); else @@ -1285,8 +1298,7 @@ void sev_pci_init(void) if (sev_get_api_version()) goto err; - if (sev_version_greater_or_equal(0, 15) && - sev_update_firmware(sev->dev) == 0) + if (sev_update_firmware(sev->dev) == 0) sev_get_api_version(); /* If an init_ex_path is provided rely on INIT_EX for PSP initialization -- cgit From 66c8137f75315d9b354c63d3aa215fe9d83a9004 Mon Sep 17 00:00:00 2001 From: Dong Chuanjian Date: Thu, 11 Aug 2022 15:17:33 +0800 Subject: crypto: drbg - remove unnecessary (void*) conversions remove unnecessary void* type casting v2: Turn assignments less than 75 characters into one line. Signed-off-by: Dong Chuanjian Signed-off-by: Herbert Xu --- crypto/drbg.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 177983b6ae38..982d4ca4526d 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1703,7 +1703,7 @@ static int drbg_init_hash_kernel(struct drbg_state *drbg) static int drbg_fini_hash_kernel(struct drbg_state *drbg) { - struct sdesc *sdesc = (struct sdesc *)drbg->priv_data; + struct sdesc *sdesc = drbg->priv_data; if (sdesc) { crypto_free_shash(sdesc->shash.tfm); kfree_sensitive(sdesc); @@ -1715,7 +1715,7 @@ static int drbg_fini_hash_kernel(struct drbg_state *drbg) static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg, const unsigned char *key) { - struct sdesc *sdesc = (struct sdesc *)drbg->priv_data; + struct sdesc *sdesc = drbg->priv_data; crypto_shash_setkey(sdesc->shash.tfm, key, drbg_statelen(drbg)); } @@ -1723,7 +1723,7 @@ static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg, static int drbg_kcapi_hash(struct drbg_state *drbg, unsigned char *outval, const struct list_head *in) { - struct sdesc *sdesc = (struct sdesc *)drbg->priv_data; + struct sdesc *sdesc = drbg->priv_data; struct drbg_string *input = NULL; crypto_shash_init(&sdesc->shash); @@ -1818,8 +1818,7 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) static void drbg_kcapi_symsetkey(struct drbg_state *drbg, const unsigned char *key) { - struct crypto_cipher *tfm = - (struct crypto_cipher *)drbg->priv_data; + struct crypto_cipher *tfm = drbg->priv_data; crypto_cipher_setkey(tfm, key, (drbg_keylen(drbg))); } @@ -1827,8 +1826,7 @@ static void drbg_kcapi_symsetkey(struct drbg_state *drbg, static int drbg_kcapi_sym(struct drbg_state *drbg, unsigned char *outval, const struct drbg_string *in) { - struct crypto_cipher *tfm = - (struct crypto_cipher *)drbg->priv_data; + struct crypto_cipher *tfm = drbg->priv_data; /* there is only component in *in */ BUG_ON(in->len < drbg_blocklen(drbg)); -- cgit From 450df3ecef4df9d94f7ca0d68527944418b2d1ed Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 11 Aug 2022 20:07:05 +0800 Subject: crypto: cavium - Fix comment typo The double `the' is duplicated in the comment, remove one. Signed-off-by: Jason Wang Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cpt_hw_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/cavium/cpt/cpt_hw_types.h b/drivers/crypto/cavium/cpt/cpt_hw_types.h index 8ec6edc69f3f..ae4791a8ec4a 100644 --- a/drivers/crypto/cavium/cpt/cpt_hw_types.h +++ b/drivers/crypto/cavium/cpt/cpt_hw_types.h @@ -396,7 +396,7 @@ union cptx_vqx_misc_ena_w1s { * Word0 * reserved_20_63:44 [63:20] Reserved. * dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add - * to the CPT instruction doorbell count. Readback value is the the + * to the CPT instruction doorbell count. Readback value is the * current number of pending doorbell requests. If counter overflows * CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to * zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF], -- cgit From bc9d6dac098bd4f6671970e0ba6c247e3a8c4029 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 11 Aug 2022 20:13:49 +0800 Subject: crypto: api - Fix comment typo The double `to' is duplicated in the comment, remove one. Signed-off-by: Jason Wang Signed-off-by: Herbert Xu --- crypto/api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/api.c b/crypto/api.c index 69508ae9345e..ab4b5e2b0756 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -321,7 +321,7 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask) /* * If the internal flag is set for a cipher, require a caller to - * to invoke the cipher with the internal flag to use that cipher. + * invoke the cipher with the internal flag to use that cipher. * Also, if a caller wants to allocate a cipher that may or may * not be an internal cipher, use type | CRYPTO_ALG_INTERNAL and * !(mask & CRYPTO_ALG_INTERNAL). -- cgit From 5a4c2936669736f2e915fe6135a668e9e079de34 Mon Sep 17 00:00:00 2001 From: Lucas Segarra Fernandez Date: Fri, 12 Aug 2022 16:16:02 +0200 Subject: crypto: testmgr - extend acomp tests for NULL destination buffer Acomp API supports NULL destination buffer for compression and decompression requests. In such cases allocation is performed by API. Add test cases for crypto_acomp_compress() and crypto_acomp_decompress() with dst buffer allocated by API. Tests will only run if CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y. Signed-off-by: Lucas Segarra Fernandez Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- crypto/testmgr.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 5349ffee6bbd..bf905c1e89ed 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3417,6 +3417,21 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS + crypto_init_wait(&wait); + sg_init_one(&src, input_vec, ilen); + acomp_request_set_params(req, &src, NULL, ilen, 0); + + ret = crypto_wait_req(crypto_acomp_compress(req), &wait); + if (ret) { + pr_err("alg: acomp: compression failed on NULL dst buffer test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } +#endif + kfree(input_vec); acomp_request_free(req); } @@ -3478,6 +3493,20 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS + crypto_init_wait(&wait); + acomp_request_set_params(req, &src, NULL, ilen, 0); + + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); + if (ret) { + pr_err("alg: acomp: decompression failed on NULL dst buffer test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } +#endif + kfree(input_vec); acomp_request_free(req); } -- cgit From 4f336045276b26c1620d0cb64d4af39ec508f436 Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 13 Aug 2022 17:57:52 +0800 Subject: crypto: hisilicon/zip - optimization for performance 1.Remove some useless steps during doing requests. 2.Adjust the possibility of branch prediction. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_crypto.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 06a2d6e81ae9..6b3f8da150ad 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -183,7 +183,7 @@ static int add_comp_head(struct scatterlist *dst, u8 req_type) int ret; ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); - if (ret != head_size) { + if (unlikely(ret != head_size)) { pr_err("the head size of buffer is wrong (%d)!\n", ret); return -ENOMEM; } @@ -193,11 +193,11 @@ static int add_comp_head(struct scatterlist *dst, u8 req_type) static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) { - if (!acomp_req->src || !acomp_req->slen) + if (unlikely(!acomp_req->src || !acomp_req->slen)) return -EINVAL; - if (req_type == HZIP_ALG_TYPE_GZIP && - acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT) + if (unlikely(req_type == HZIP_ALG_TYPE_GZIP && + acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) return -EINVAL; switch (req_type) { @@ -230,6 +230,8 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, } set_bit(req_id, req_q->req_bitmap); + write_unlock(&req_q->req_lock); + req_cache = q + req_id; req_cache->req_id = req_id; req_cache->req = req; @@ -242,8 +244,6 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, req_cache->dskip = 0; } - write_unlock(&req_q->req_lock); - return req_cache; } @@ -254,7 +254,6 @@ static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, write_lock(&req_q->req_lock); clear_bit(req->req_id, req_q->req_bitmap); - memset(req, 0, sizeof(struct hisi_zip_req)); write_unlock(&req_q->req_lock); } @@ -339,7 +338,7 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, struct hisi_zip_sqe zip_sqe; int ret; - if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) + if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)) return -EINVAL; req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, @@ -365,7 +364,7 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, /* send command to start a task */ atomic64_inc(&dfx->send_cnt); ret = hisi_qp_send(qp, &zip_sqe); - if (ret < 0) { + if (unlikely(ret < 0)) { atomic64_inc(&dfx->send_busy_cnt); ret = -EAGAIN; dev_dbg_ratelimited(dev, "failed to send request!\n"); @@ -417,7 +416,7 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) atomic64_inc(&dfx->recv_cnt); status = ops->get_status(sqe); - if (status != 0 && status != HZIP_NC_ERR) { + if (unlikely(status != 0 && status != HZIP_NC_ERR)) { dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, sqe->produced); @@ -450,7 +449,7 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) /* let's output compression head now */ head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); - if (head_size < 0) { + if (unlikely(head_size < 0)) { dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", head_size); return head_size; @@ -461,7 +460,7 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) { + if (unlikely(ret != -EINPROGRESS)) { dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); hisi_zip_remove_req(qp_ctx, req); } @@ -478,7 +477,7 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) int head_size, ret; head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); - if (head_size < 0) { + if (unlikely(head_size < 0)) { dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", head_size); return head_size; @@ -489,7 +488,7 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) { + if (unlikely(ret != -EINPROGRESS)) { dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", ret); hisi_zip_remove_req(qp_ctx, req); -- cgit From 6d9a899557c8751372da99d137eaaa9cdbe81f41 Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 13 Aug 2022 18:19:39 +0800 Subject: crypto: hisilicon/zip - some misc cleanup Some cleanup for code: 1. Change names for easy to understand. 2. Unify the variables type. 3. Use the right return value. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip.h | 2 +- drivers/crypto/hisilicon/zip/zip_crypto.c | 30 +++++++++++++++--------------- drivers/crypto/hisilicon/zip/zip_main.c | 10 +++++++--- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 3dfd3bac5a33..f289656e9ac0 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -81,7 +81,7 @@ struct hisi_zip_sqe { u32 rsvd1[4]; }; -int zip_create_qps(struct hisi_qp **qps, int ctx_num, int node); +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 6b3f8da150ad..a6c914d527eb 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -135,7 +135,7 @@ static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); -static u16 get_extra_field_size(const u8 *start) +static u32 get_extra_field_size(const u8 *start) { return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; } @@ -167,7 +167,7 @@ static u32 __get_gzip_head_size(const u8 *src) return size; } -static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) +static u32 __maybe_unused get_gzip_head_size(struct scatterlist *sgl) { char buf[HZIP_GZIP_HEAD_BUF]; @@ -497,7 +497,7 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return ret; } -static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, +static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, int alg_type, int req_type) { struct device *dev = &qp->qm->pdev->dev; @@ -505,7 +505,7 @@ static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, qp->req_type = req_type; qp->alg_type = alg_type; - qp->qp_ctx = ctx; + qp->qp_ctx = qp_ctx; ret = hisi_qm_start_qp(qp, 0); if (ret < 0) { @@ -513,15 +513,15 @@ static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, return ret; } - ctx->qp = qp; + qp_ctx->qp = qp; return 0; } -static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) +static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) { - hisi_qm_stop_qp(ctx->qp); - hisi_qm_free_qps(&ctx->qp, 1); + hisi_qm_stop_qp(qp_ctx->qp); + hisi_qm_free_qps(&qp_ctx->qp, 1); } static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { @@ -593,7 +593,7 @@ static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) { int i; - for (i = 1; i >= 0; i--) + for (i = 0; i < HZIP_CTX_Q_NUM; i++) hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); } @@ -612,7 +612,7 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) if (i == 0) return ret; - goto err_free_loop0; + goto err_free_comp_q; } rwlock_init(&req_q->req_lock); @@ -621,19 +621,19 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) if (!req_q->q) { ret = -ENOMEM; if (i == 0) - goto err_free_bitmap; + goto err_free_comp_bitmap; else - goto err_free_loop1; + goto err_free_decomp_bitmap; } } return 0; -err_free_loop1: +err_free_decomp_bitmap: bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); -err_free_loop0: +err_free_comp_q: kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); -err_free_bitmap: +err_free_comp_bitmap: bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); return ret; } diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index c3303d99acac..04c8a4c65d77 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -586,8 +586,9 @@ static ssize_t hisi_zip_ctrl_debug_write(struct file *filp, return len; tbuf[len] = '\0'; - if (kstrtoul(tbuf, 0, &val)) - return -EFAULT; + ret = kstrtoul(tbuf, 0, &val); + if (ret) + return ret; ret = hisi_qm_get_dfx_access(qm); if (ret) @@ -976,7 +977,10 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) qm->err_ini = &hisi_zip_err_ini; qm->err_ini->err_info_init(qm); - hisi_zip_set_user_domain_and_cache(qm); + ret = hisi_zip_set_user_domain_and_cache(qm); + if (ret) + return ret; + hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); -- cgit From 582b05bba481d5798ef884f1396285ab47e426e1 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 13 Aug 2022 18:34:21 +0800 Subject: crypto: hisilicon/hpre - change return type of hpre_cluster_inqry_write() hpre_cluster_inqry_write() always returns 0. So change the type of hpre_cluster_inqry_write() to void. Signed-off-by: Weili Qian Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 9d529df0eab9..8e0e87cede6b 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -708,7 +708,7 @@ static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file) return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT); } -static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) +static void hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) { struct hisi_qm *qm = hpre_file_to_qm(file); int cluster_index = file->index - HPRE_CLUSTER_CTRL; @@ -716,8 +716,6 @@ static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) HPRE_CLSTR_ADDR_INTRVL; writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY); - - return 0; } static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf, @@ -792,9 +790,7 @@ static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf, goto err_input; break; case HPRE_CLUSTER_CTRL: - ret = hpre_cluster_inqry_write(file, val); - if (ret) - goto err_input; + hpre_cluster_inqry_write(file, val); break; default: ret = -EINVAL; -- cgit From 116be08f6e4e385733d42360a33c3d883d2dd702 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 13 Aug 2022 18:34:52 +0800 Subject: crypto: hisilicon/qm - fix missing destroy qp_idr In the function hisi_qm_memory_init(), if resource alloc fails after idr_init, the initialized qp_idr needs to be destroyed. Signed-off-by: Weili Qian Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 9701434a8ef3..728dcee0b1ff 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -6139,8 +6139,8 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) GFP_ATOMIC); dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); if (!qm->qdma.va) { - ret = -ENOMEM; - goto err_alloc_qdma; + ret = -ENOMEM; + goto err_destroy_idr; } QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); @@ -6156,7 +6156,8 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) err_alloc_qp_array: dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); -err_alloc_qdma: +err_destroy_idr: + idr_destroy(&qm->qp_idr); kfree(qm->factor); return ret; -- cgit From 1129d2d533195993aa0b0d0cd1c868950e01770d Mon Sep 17 00:00:00 2001 From: Junchong Pan Date: Sat, 13 Aug 2022 18:35:15 +0800 Subject: crypto: hisilicon/qm - remove unneeded data storage The dump_show() is used to output hardware information for error locating. It is not need to apply for memory to temporarily store the converted data. It can directly output the data. Therefore, remove some unnecessary code. Signed-off-by: Junchong Pan Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 80 +++++++++++-------------------------------- 1 file changed, 20 insertions(+), 60 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 728dcee0b1ff..927d9fa7b6b2 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1857,39 +1857,19 @@ static void qm_ctx_free(struct hisi_qm *qm, size_t ctx_size, kfree(ctx_addr); } -static int dump_show(struct hisi_qm *qm, void *info, +static void dump_show(struct hisi_qm *qm, void *info, unsigned int info_size, char *info_name) { struct device *dev = &qm->pdev->dev; - u8 *info_buf, *info_curr = info; + u8 *info_curr = info; u32 i; #define BYTE_PER_DW 4 - info_buf = kzalloc(info_size, GFP_KERNEL); - if (!info_buf) - return -ENOMEM; - - for (i = 0; i < info_size; i++, info_curr++) { - if (i % BYTE_PER_DW == 0) - info_buf[i + 3UL] = *info_curr; - else if (i % BYTE_PER_DW == 1) - info_buf[i + 1UL] = *info_curr; - else if (i % BYTE_PER_DW == 2) - info_buf[i - 1] = *info_curr; - else if (i % BYTE_PER_DW == 3) - info_buf[i - 3] = *info_curr; - } - dev_info(dev, "%s DUMP\n", info_name); - for (i = 0; i < info_size; i += BYTE_PER_DW) { + for (i = 0; i < info_size; i += BYTE_PER_DW, info_curr += BYTE_PER_DW) { pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW, - info_buf[i], info_buf[i + 1UL], - info_buf[i + 2UL], info_buf[i + 3UL]); + *(info_curr + 3), *(info_curr + 2), *(info_curr + 1), *(info_curr)); } - - kfree(info_buf); - - return 0; } static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id) @@ -1929,23 +1909,18 @@ static int qm_sqc_dump(struct hisi_qm *qm, const char *s) if (qm->sqc) { sqc_curr = qm->sqc + qp_id; - ret = dump_show(qm, sqc_curr, sizeof(*sqc), - "SOFT SQC"); - if (ret) - dev_info(dev, "Show soft sqc failed!\n"); + dump_show(qm, sqc_curr, sizeof(*sqc), "SOFT SQC"); } up_read(&qm->qps_lock); - goto err_free_ctx; + goto free_ctx; } - ret = dump_show(qm, sqc, sizeof(*sqc), "SQC"); - if (ret) - dev_info(dev, "Show hw sqc failed!\n"); + dump_show(qm, sqc, sizeof(*sqc), "SQC"); -err_free_ctx: +free_ctx: qm_ctx_free(qm, sizeof(*sqc), sqc, &sqc_dma); - return ret; + return 0; } static int qm_cqc_dump(struct hisi_qm *qm, const char *s) @@ -1975,23 +1950,18 @@ static int qm_cqc_dump(struct hisi_qm *qm, const char *s) if (qm->cqc) { cqc_curr = qm->cqc + qp_id; - ret = dump_show(qm, cqc_curr, sizeof(*cqc), - "SOFT CQC"); - if (ret) - dev_info(dev, "Show soft cqc failed!\n"); + dump_show(qm, cqc_curr, sizeof(*cqc), "SOFT CQC"); } up_read(&qm->qps_lock); - goto err_free_ctx; + goto free_ctx; } - ret = dump_show(qm, cqc, sizeof(*cqc), "CQC"); - if (ret) - dev_info(dev, "Show hw cqc failed!\n"); + dump_show(qm, cqc, sizeof(*cqc), "CQC"); -err_free_ctx: +free_ctx: qm_ctx_free(qm, sizeof(*cqc), cqc, &cqc_dma); - return ret; + return 0; } static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size, @@ -2015,9 +1985,7 @@ static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size, if (ret) goto err_free_ctx; - ret = dump_show(qm, xeqc, size, name); - if (ret) - dev_info(dev, "Show hw %s failed!\n", name); + dump_show(qm, xeqc, size, name); err_free_ctx: qm_ctx_free(qm, size, xeqc, &xeqc_dma); @@ -2066,7 +2034,6 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, static int qm_sq_dump(struct hisi_qm *qm, char *s) { - struct device *dev = &qm->pdev->dev; void *sqe, *sqe_curr; struct hisi_qp *qp; u32 qp_id, sqe_id; @@ -2086,18 +2053,15 @@ static int qm_sq_dump(struct hisi_qm *qm, char *s) memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK, qm->debug.sqe_mask_len); - ret = dump_show(qm, sqe_curr, qm->sqe_size, "SQE"); - if (ret) - dev_info(dev, "Show sqe failed!\n"); + dump_show(qm, sqe_curr, qm->sqe_size, "SQE"); kfree(sqe); - return ret; + return 0; } static int qm_cq_dump(struct hisi_qm *qm, char *s) { - struct device *dev = &qm->pdev->dev; struct qm_cqe *cqe_curr; struct hisi_qp *qp; u32 qp_id, cqe_id; @@ -2109,11 +2073,9 @@ static int qm_cq_dump(struct hisi_qm *qm, char *s) qp = &qm->qp_array[qp_id]; cqe_curr = qp->cqe + cqe_id; - ret = dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE"); - if (ret) - dev_info(dev, "Show cqe failed!\n"); + dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE"); - return ret; + return 0; } static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, @@ -2150,9 +2112,7 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, goto err_unlock; } - ret = dump_show(qm, xeqe, size, name); - if (ret) - dev_info(dev, "Show %s failed!\n", name); + dump_show(qm, xeqe, size, name); err_unlock: up_read(&qm->qps_lock); -- cgit From 6a088a2cbcaf40747cf2881df47f4f5d65acd7ab Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 13 Aug 2022 18:35:45 +0800 Subject: crypto: hisilicon/qm - remove unneeded hardware cache write back Data in the hardware cache needs to be written back to the memory before the queue memory is released. Currently, the queue memory is applied for when the driver is loaded and released when the driver is removed. Therefore, the hardware cache does not need to be written back when process puts queue. Signed-off-by: Weili Qian Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 927d9fa7b6b2..b2e7abff1b8a 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3246,7 +3246,6 @@ static void hisi_qm_uacce_put_queue(struct uacce_queue *q) { struct hisi_qp *qp = q->priv; - hisi_qm_cache_wb(qp->qm); hisi_qm_release_qp(qp); } -- cgit From aa031b8f702e7941b4c86022348a366c335d389a Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 13 Aug 2022 18:04:31 -0500 Subject: crypto: x86/sha512 - load based on CPU features x86 optimized crypto modules built as modules rather than built-in to the kernel end up as .ko files in the filesystem, e.g., in /usr/lib/modules. If the filesystem itself is a module, these might not be available when the crypto API is initialized, resulting in the generic implementation being used (e.g., sha512_transform rather than sha512_transform_avx2). In one test case, CPU utilization in the sha512 function dropped from 15.34% to 7.18% after forcing loading of the optimized module. Add module aliases for this x86 optimized crypto module based on CPU feature bits so udev gets a chance to load them later in the boot process when the filesystems are all running. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/x86/crypto/sha512_ssse3_glue.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index 30e70f4fe2f7..6d3b85e53d0e 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -36,6 +36,7 @@ #include #include #include +#include #include asmlinkage void sha512_transform_ssse3(struct sha512_state *state, @@ -284,6 +285,13 @@ static int register_sha512_avx2(void) ARRAY_SIZE(sha512_avx2_algs)); return 0; } +static const struct x86_cpu_id module_cpu_ids[] = { + X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), + X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), + X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); static void unregister_sha512_avx2(void) { @@ -294,6 +302,8 @@ static void unregister_sha512_avx2(void) static int __init sha512_ssse3_mod_init(void) { + if (!x86_match_cpu(module_cpu_ids)) + return -ENODEV; if (register_sha512_ssse3()) goto fail; -- cgit From a76bd86a85cac9feddc66d38019f943d054f0218 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 13 Aug 2022 18:14:43 -0500 Subject: crypto: testmgr - don't generate WARN for missing modules This userspace command: modprobe tcrypt or modprobe tcrypt mode=0 runs all the tcrypt test cases numbered <200 (i.e., all the test cases calling tcrypt_test() and returning return values). Tests are sparsely numbered from 0 to 1000. For example: modprobe tcrypt mode=12 tests sha512, and modprobe tcrypt mode=152 tests rfc4543(gcm(aes))) - AES-GCM as GMAC The test manager generates WARNING crashdumps every time it attempts a test using an algorithm that is not available (not built-in to the kernel or available as a module): alg: skcipher: failed to allocate transform for ecb(arc4): -2 ------------[ cut here ]----------- alg: self-tests for ecb(arc4) (ecb(arc4)) failed (rc=-2) WARNING: CPU: 9 PID: 4618 at crypto/testmgr.c:5777 alg_test+0x30b/0x510 [50 more lines....] ---[ end trace 0000000000000000 ]--- If the kernel is compiled with CRYPTO_USER_API_ENABLE_OBSOLETE disabled (the default), then these algorithms are not compiled into the kernel or made into modules and trigger WARNINGs: arc4 tea xtea khazad anubis xeta seed Additionally, any other algorithms that are not enabled in .config will generate WARNINGs. In RHEL 9.0, for example, the default selection of algorithms leads to 16 WARNING dumps. One attempt to fix this was by modifying tcrypt_test() to check crypto_has_alg() and immediately return 0 if crypto_has_alg() fails, rather than proceed and return a non-zero error value that causes the caller (alg_test() in crypto/testmgr.c) to invoke WARN(). That knocks out too many algorithms, though; some combinations like ctr(des3_ede) would work. Instead, change the condition on the WARN to ignore a return value is ENOENT, which is the value returned when the algorithm or combination of algorithms doesn't exist. Add a pr_warn to communicate that information in case the WARN is skipped. This approach allows algorithm tests to work that are combinations, not provided by one driver, like ctr(blowfish). Result - no more WARNINGs: modprobe tcrypt [ 115.541765] tcrypt: testing md5 [ 115.556415] tcrypt: testing sha1 [ 115.570463] tcrypt: testing ecb(des) [ 115.585303] cryptomgr: alg: skcipher: failed to allocate transform for ecb(des): -2 [ 115.593037] cryptomgr: alg: self-tests for ecb(des) using ecb(des) failed (rc=-2) [ 115.593038] tcrypt: testing cbc(des) [ 115.610641] cryptomgr: alg: skcipher: failed to allocate transform for cbc(des): -2 [ 115.618359] cryptomgr: alg: self-tests for cbc(des) using cbc(des) failed (rc=-2) ... Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- crypto/testmgr.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index bf905c1e89ed..2ad4bcc58617 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5830,8 +5830,11 @@ test_done: driver, alg, fips_enabled ? "fips" : "panic_on_fail"); } - WARN(1, "alg: self-tests for %s (%s) failed (rc=%d)", - driver, alg, rc); + pr_warn("alg: self-tests for %s using %s failed (rc=%d)", + alg, driver, rc); + WARN(rc != -ENOENT, + "alg: self-tests for %s using %s failed (rc=%d)", + alg, driver, rc); } else { if (fips_enabled) pr_info("alg: self-tests for %s (%s) passed\n", -- cgit From 90cb3ca2fa4f5c4c3fbeb9014584b69b1ba26242 Mon Sep 17 00:00:00 2001 From: Tuo Cao <91tuocao@gmail.com> Date: Sun, 14 Aug 2022 21:00:54 +0800 Subject: crypto: artpec6 - move spin_lock_bh to spin_lock in tasklet it is unnecessary to call spin_lock_bh in a tasklet. Signed-off-by: Tuo Cao <91tuocao@gmail.com> Signed-off-by: Herbert Xu --- drivers/crypto/axis/artpec6_crypto.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 9ad188cffd0d..b4820594ab80 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -2091,7 +2091,7 @@ static void artpec6_crypto_task(unsigned long data) return; } - spin_lock_bh(&ac->queue_lock); + spin_lock(&ac->queue_lock); list_for_each_entry_safe(req, n, &ac->pending, list) { struct artpec6_crypto_dma_descriptors *dma = req->dma; @@ -2128,7 +2128,7 @@ static void artpec6_crypto_task(unsigned long data) artpec6_crypto_process_queue(ac, &complete_in_progress); - spin_unlock_bh(&ac->queue_lock); + spin_unlock(&ac->queue_lock); /* Perform the completion callbacks without holding the queue lock * to allow new request submissions from the callbacks. -- cgit From 47d35bf22b6913aa9fa2389431377d4189102a15 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Sun, 14 Aug 2022 19:03:59 +0200 Subject: hwrng: imx-rngc - use KBUILD_MODNAME as driver name Use KBUILD_MODNAME instead of hard coding the driver name. Signed-off-by: Martin Kaiser Signed-off-by: Herbert Xu --- drivers/char/hw_random/imx-rngc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index b05d676ca814..78c10fa4c79e 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -355,7 +355,7 @@ MODULE_DEVICE_TABLE(of, imx_rngc_dt_ids); static struct platform_driver imx_rngc_driver = { .driver = { - .name = "imx_rngc", + .name = KBUILD_MODNAME, .pm = &imx_rngc_pm_ops, .of_match_table = imx_rngc_dt_ids, }, -- cgit From 6363d81b78c00d98f6d92b04acf65b4a18013690 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sun, 14 Aug 2022 23:29:15 -0500 Subject: crypto: tcrypt - remove mode=1000 The lists of algothms checked for existence by modprobe tcrypt mode=1000 generates three bogus errors: modprobe tcrypt mode=1000 console log: tcrypt: alg rot13 not found tcrypt: alg cts not found tcrypt: alg arc4 not found rot13 is not an algorithm in the crypto API or tested. cts is a wrapper, not a base algorithm. arc4 is named ecb(arc4), not arc4. Also, the list is missing numerous algorithms that are tested by other test modes: blake2b-512 blake2s-256 crct10dif xxhash64 ghash cast5 sm4 ansi_prng Several of the algorithms are only available if CONFIG_CRYPTO_USER_API_ENABLE_OBSOLETE is enabled: arc4 khazad seed tea, xtea, xeta Rather that fix that list, remove test mode=1000 entirely. It seems to have limited utility, and a web search shows no discussion of anybody using it. Suggested-by: Ard Biesheuvel Signed-off-by: Robert Elliott Reviewed-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 59eb8ec36664..e85f623c3c54 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -66,17 +66,6 @@ static u32 num_mb = 8; static unsigned int klen; static char *tvmem[TVMEMSIZE]; -static const char *check[] = { - "des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3", - "blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes", - "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea", - "khazad", "wp512", "wp384", "wp256", "xeta", "fcrypt", - "camellia", "seed", "rmd160", "aria", - "lzo", "lzo-rle", "cts", "sha3-224", "sha3-256", "sha3-384", - "sha3-512", "streebog256", "streebog512", - NULL -}; - static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 }; static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 }; @@ -1454,18 +1443,6 @@ static void test_cipher_speed(const char *algo, int enc, unsigned int secs, false); } -static void test_available(void) -{ - const char **name = check; - - while (*name) { - printk("alg %s ", *name); - printk(crypto_has_alg(*name, 0, 0) ? - "found\n" : "not found\n"); - name++; - } -} - static inline int tcrypt_test(const char *alg) { int ret; @@ -2859,10 +2836,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0, speed_template_8_32, num_mb); break; - - case 1000: - test_available(); - break; } return ret; -- cgit From 6a2bc448423cea44e7dba0f72d7c82ae04ab201e Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Mon, 15 Aug 2022 21:37:42 +0200 Subject: hwrng: imx-rngc - use devm_clk_get_enabled Use the new devm_clk_get_enabled function to get our clock. We don't have to disable and unprepare the clock ourselves any more in error paths and in the remove function. Signed-off-by: Martin Kaiser Signed-off-by: Herbert Xu --- drivers/char/hw_random/imx-rngc.c | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 78c10fa4c79e..9b243356d4db 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -245,7 +245,7 @@ static int imx_rngc_probe(struct platform_device *pdev) if (IS_ERR(rngc->base)) return PTR_ERR(rngc->base); - rngc->clk = devm_clk_get(&pdev->dev, NULL); + rngc->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(rngc->clk)) { dev_err(&pdev->dev, "Can not get rng_clk\n"); return PTR_ERR(rngc->clk); @@ -255,26 +255,20 @@ static int imx_rngc_probe(struct platform_device *pdev) if (irq < 0) return irq; - ret = clk_prepare_enable(rngc->clk); - if (ret) - return ret; - ver_id = readl(rngc->base + RNGC_VER_ID); rng_type = ver_id >> RNGC_TYPE_SHIFT; /* * This driver supports only RNGC and RNGB. (There's a different * driver for RNGA.) */ - if (rng_type != RNGC_TYPE_RNGC && rng_type != RNGC_TYPE_RNGB) { - ret = -ENODEV; - goto err; - } + if (rng_type != RNGC_TYPE_RNGC && rng_type != RNGC_TYPE_RNGB) + return -ENODEV; ret = devm_request_irq(&pdev->dev, irq, imx_rngc_irq, 0, pdev->name, (void *)rngc); if (ret) { dev_err(rngc->dev, "Can't get interrupt working.\n"); - goto err; + return ret; } init_completion(&rngc->rng_op_done); @@ -294,14 +288,14 @@ static int imx_rngc_probe(struct platform_device *pdev) ret = imx_rngc_self_test(rngc); if (ret) { dev_err(rngc->dev, "self test failed\n"); - goto err; + return ret; } } ret = hwrng_register(&rngc->rng); if (ret) { dev_err(&pdev->dev, "hwrng registration failed\n"); - goto err; + return ret; } dev_info(&pdev->dev, @@ -309,11 +303,6 @@ static int imx_rngc_probe(struct platform_device *pdev) rng_type == RNGC_TYPE_RNGB ? 'B' : 'C', (ver_id >> RNGC_VER_MAJ_SHIFT) & 0xff, ver_id & 0xff); return 0; - -err: - clk_disable_unprepare(rngc->clk); - - return ret; } static int __exit imx_rngc_remove(struct platform_device *pdev) @@ -322,8 +311,6 @@ static int __exit imx_rngc_remove(struct platform_device *pdev) hwrng_unregister(&rngc->rng); - clk_disable_unprepare(rngc->clk); - return 0; } -- cgit From d5eb916d889f12f27275e2049b9c4df43b7cfa13 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Mon, 15 Aug 2022 21:37:43 +0200 Subject: hwrng: imx-rngc - use devres for hwrng registration Replace hwrng_register with devm_hwrng_register and let devres unregister our hwrng when the device is removed. It's possible to do this now that devres also handles clock disable+uprepare. When we had to disable+unprepare the clock ourselves, we had to unregister the hwrng before this and couldn't use devres. There's nothing left to do for imx_rngc_remove, this function can go. Signed-off-by: Martin Kaiser Signed-off-by: Herbert Xu --- drivers/char/hw_random/imx-rngc.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 9b243356d4db..f5d07b472d8a 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -292,7 +292,7 @@ static int imx_rngc_probe(struct platform_device *pdev) } } - ret = hwrng_register(&rngc->rng); + ret = devm_hwrng_register(&pdev->dev, &rngc->rng); if (ret) { dev_err(&pdev->dev, "hwrng registration failed\n"); return ret; @@ -305,15 +305,6 @@ static int imx_rngc_probe(struct platform_device *pdev) return 0; } -static int __exit imx_rngc_remove(struct platform_device *pdev) -{ - struct imx_rngc *rngc = platform_get_drvdata(pdev); - - hwrng_unregister(&rngc->rng); - - return 0; -} - static int __maybe_unused imx_rngc_suspend(struct device *dev) { struct imx_rngc *rngc = dev_get_drvdata(dev); @@ -346,7 +337,6 @@ static struct platform_driver imx_rngc_driver = { .pm = &imx_rngc_pm_ops, .of_match_table = imx_rngc_dt_ids, }, - .remove = __exit_p(imx_rngc_remove), }; module_platform_driver_probe(imx_rngc_driver, imx_rngc_probe); -- cgit From d8da2da21fdb1f5964c11c00f0cc84fb0edf31d0 Mon Sep 17 00:00:00 2001 From: Jacky Li Date: Tue, 16 Aug 2022 19:32:08 +0000 Subject: crypto: ccp - Initialize PSP when reading psp data file failed Currently the OS fails the PSP initialization when the file specified at 'init_ex_path' does not exist or has invalid content. However the SEV spec just requires users to allocate 32KB of 0xFF in the file, which can be taken care of by the OS easily. To improve the robustness during the PSP init, leverage the retry mechanism and continue the init process: Before the first INIT_EX call, if the content is invalid or missing, continue the process by feeding those contents into PSP instead of aborting. PSP will then override it with 32KB 0xFF and return SEV_RET_SECURE_DATA_INVALID status code. In the second INIT_EX call, this 32KB 0xFF content will then be fed and PSP will write the valid data to the file. In order to do this, sev_read_init_ex_file should only be called once for the first INIT_EX call. Calling it again for the second INIT_EX call will cause the invalid file content overwriting the valid 32KB 0xFF data provided by PSP in the first INIT_EX call. Co-developed-by: Peter Gonda Signed-off-by: Peter Gonda Signed-off-by: Jacky Li Reported-by: Alper Gun Acked-by: David Rientjes Acked-by: Tom Lendacky Signed-off-by: Herbert Xu --- .../virt/kvm/x86/amd-memory-encryption.rst | 5 ++- drivers/crypto/ccp/sev-dev.c | 36 +++++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index 2d307811978c..935aaeb97fe6 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -89,9 +89,8 @@ context. In a typical workflow, this command should be the first command issued. The firmware can be initialized either by using its own non-volatile storage or the OS can manage the NV storage for the firmware using the module parameter -``init_ex_path``. The file specified by ``init_ex_path`` must exist. To create -a new NV storage file allocate the file with 32KB bytes of 0xFF as required by -the SEV spec. +``init_ex_path``. If the file specified by ``init_ex_path`` does not exist or +is invalid, the OS will create or override the file with output from PSP. Returns: 0 on success, -negative on error diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index b292641c8a99..8512101f0bdf 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -211,18 +211,24 @@ static int sev_read_init_ex_file(void) if (IS_ERR(fp)) { int ret = PTR_ERR(fp); - dev_err(sev->dev, - "SEV: could not open %s for read, error %d\n", - init_ex_path, ret); + if (ret == -ENOENT) { + dev_info(sev->dev, + "SEV: %s does not exist and will be created later.\n", + init_ex_path); + ret = 0; + } else { + dev_err(sev->dev, + "SEV: could not open %s for read, error %d\n", + init_ex_path, ret); + } return ret; } nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL); if (nread != NV_LENGTH) { - dev_err(sev->dev, - "SEV: failed to read %u bytes to non volatile memory area, ret %ld\n", + dev_info(sev->dev, + "SEV: could not read %u bytes to non volatile memory area, ret %ld\n", NV_LENGTH, nread); - return -EIO; } dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread); @@ -410,17 +416,12 @@ static int __sev_init_locked(int *error) static int __sev_init_ex_locked(int *error) { struct sev_data_init_ex data; - int ret; memset(&data, 0, sizeof(data)); data.length = sizeof(data); data.nv_address = __psp_pa(sev_init_ex_buffer); data.nv_len = NV_LENGTH; - ret = sev_read_init_ex_file(); - if (ret) - return ret; - if (sev_es_tmr) { /* * Do not include the encryption mask on the physical @@ -439,7 +440,7 @@ static int __sev_platform_init_locked(int *error) { struct psp_device *psp = psp_master; struct sev_device *sev; - int rc, psp_ret = -1; + int rc = 0, psp_ret = -1; int (*init_function)(int *error); if (!psp || !psp->sev_data) @@ -450,8 +451,15 @@ static int __sev_platform_init_locked(int *error) if (sev->state == SEV_STATE_INIT) return 0; - init_function = sev_init_ex_buffer ? __sev_init_ex_locked : - __sev_init_locked; + if (sev_init_ex_buffer) { + init_function = __sev_init_ex_locked; + rc = sev_read_init_ex_file(); + if (rc) + return rc; + } else { + init_function = __sev_init_locked; + } + rc = init_function(&psp_ret); if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) { /* -- cgit From efb4b01c1c993d245e6608076684ff2162cf9dc6 Mon Sep 17 00:00:00 2001 From: Jacky Li Date: Tue, 16 Aug 2022 19:32:09 +0000 Subject: crypto: ccp - Fail the PSP initialization when writing psp data file failed Currently the OS continues the PSP initialization when there is a write failure to the init_ex_file. Therefore, the userspace would be told that SEV is properly INIT'd even though the psp data file is not updated. This is problematic because later when asked for the SEV data, the OS won't be able to provide it. Fixes: 3d725965f836 ("crypto: ccp - Add SEV_INIT_EX support") Reported-by: Peter Gonda Reported-by: kernel test robot Signed-off-by: Jacky Li Acked-by: David Rientjes Acked-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 8512101f0bdf..06fc7156c04f 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -237,7 +237,7 @@ static int sev_read_init_ex_file(void) return 0; } -static void sev_write_init_ex_file(void) +static int sev_write_init_ex_file(void) { struct sev_device *sev = psp_master->sev_data; struct file *fp; @@ -247,14 +247,16 @@ static void sev_write_init_ex_file(void) lockdep_assert_held(&sev_cmd_mutex); if (!sev_init_ex_buffer) - return; + return 0; fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600); if (IS_ERR(fp)) { + int ret = PTR_ERR(fp); + dev_err(sev->dev, - "SEV: could not open file for write, error %ld\n", - PTR_ERR(fp)); - return; + "SEV: could not open file for write, error %d\n", + ret); + return ret; } nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset); @@ -265,18 +267,20 @@ static void sev_write_init_ex_file(void) dev_err(sev->dev, "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n", NV_LENGTH, nwrite); - return; + return -EIO; } dev_dbg(sev->dev, "SEV: write successful to NV file\n"); + + return 0; } -static void sev_write_init_ex_file_if_required(int cmd_id) +static int sev_write_init_ex_file_if_required(int cmd_id) { lockdep_assert_held(&sev_cmd_mutex); if (!sev_init_ex_buffer) - return; + return 0; /* * Only a few platform commands modify the SPI/NV area, but none of the @@ -291,10 +295,10 @@ static void sev_write_init_ex_file_if_required(int cmd_id) case SEV_CMD_PEK_GEN: break; default: - return; + return 0; } - sev_write_init_ex_file(); + return sev_write_init_ex_file(); } static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) @@ -367,7 +371,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) cmd, reg & PSP_CMDRESP_ERR_MASK); ret = -EIO; } else { - sev_write_init_ex_file_if_required(cmd); + ret = sev_write_init_ex_file_if_required(cmd); } print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data, -- cgit From 108713a713c7e4b7d07e6cd9b808503d5bb7089b Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Thu, 18 Aug 2022 11:59:52 +0800 Subject: crypto: aspeed - Add HACE hash driver Hash and Crypto Engine (HACE) is designed to accelerate the throughput of hash data digest, encryption, and decryption. Basically, HACE can be divided into two independently engines - Hash Engine and Crypto Engine. This patch aims to add HACE hash engine driver for hash accelerator. Signed-off-by: Neal Liu Signed-off-by: Johnny Huang Reviewed-by: Dhananjay Phadke Signed-off-by: Herbert Xu --- MAINTAINERS | 7 + drivers/crypto/Kconfig | 1 + drivers/crypto/Makefile | 1 + drivers/crypto/aspeed/Kconfig | 32 + drivers/crypto/aspeed/Makefile | 6 + drivers/crypto/aspeed/aspeed-hace-hash.c | 1389 ++++++++++++++++++++++++++++++ drivers/crypto/aspeed/aspeed-hace.c | 206 +++++ drivers/crypto/aspeed/aspeed-hace.h | 186 ++++ 8 files changed, 1828 insertions(+) create mode 100644 drivers/crypto/aspeed/Kconfig create mode 100644 drivers/crypto/aspeed/Makefile create mode 100644 drivers/crypto/aspeed/aspeed-hace-hash.c create mode 100644 drivers/crypto/aspeed/aspeed-hace.c create mode 100644 drivers/crypto/aspeed/aspeed-hace.h diff --git a/MAINTAINERS b/MAINTAINERS index 8a5012ba6ff9..164f67e59e5f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3210,6 +3210,13 @@ S: Maintained F: Documentation/devicetree/bindings/usb/aspeed,ast2600-udc.yaml F: drivers/usb/gadget/udc/aspeed_udc.c +ASPEED CRYPTO DRIVER +M: Neal Liu +L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers) +S: Maintained +F: Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml +F: drivers/crypto/aspeed/ + ASUS NOTEBOOKS AND EEEPC ACPI/WMI EXTRAS DRIVERS M: Corentin Chary L: acpi4asus-user@lists.sourceforge.net diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 3e6aa319920b..f6fe1945edd6 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -818,5 +818,6 @@ config CRYPTO_DEV_SA2UL acceleration for cryptographic algorithms on these devices. source "drivers/crypto/keembay/Kconfig" +source "drivers/crypto/aspeed/Kconfig" endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index f81703a86b98..116de173a66c 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_ALLWINNER) += allwinner/ +obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed/ obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o diff --git a/drivers/crypto/aspeed/Kconfig b/drivers/crypto/aspeed/Kconfig new file mode 100644 index 000000000000..03e3fc61498e --- /dev/null +++ b/drivers/crypto/aspeed/Kconfig @@ -0,0 +1,32 @@ +config CRYPTO_DEV_ASPEED + tristate "Support for Aspeed cryptographic engine driver" + depends on ARCH_ASPEED + help + Hash and Crypto Engine (HACE) is designed to accelerate the + throughput of hash data digest, encryption and decryption. + + Select y here to have support for the cryptographic driver + available on Aspeed SoC. + +config CRYPTO_DEV_ASPEED_DEBUG + bool "Enable Aspeed crypto debug messages" + depends on CRYPTO_DEV_ASPEED + help + Print Aspeed crypto debugging messages if you use this + option to ask for those messages. + Avoid enabling this option for production build to + minimize driver timing. + +config CRYPTO_DEV_ASPEED_HACE_HASH + bool "Enable Aspeed Hash & Crypto Engine (HACE) hash" + depends on CRYPTO_DEV_ASPEED + select CRYPTO_ENGINE + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_HMAC + help + Select here to enable Aspeed Hash & Crypto Engine (HACE) + hash driver. + Supports multiple message digest standards, including + SHA-1, SHA-224, SHA-256, SHA-384, SHA-512, and so on. diff --git a/drivers/crypto/aspeed/Makefile b/drivers/crypto/aspeed/Makefile new file mode 100644 index 000000000000..8bc8d4fed5a9 --- /dev/null +++ b/drivers/crypto/aspeed/Makefile @@ -0,0 +1,6 @@ +obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed_crypto.o +aspeed_crypto-objs := aspeed-hace.o \ + $(hace-hash-y) + +obj-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) += aspeed-hace-hash.o +hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace-hash.o diff --git a/drivers/crypto/aspeed/aspeed-hace-hash.c b/drivers/crypto/aspeed/aspeed-hace-hash.c new file mode 100644 index 000000000000..0a44ffc0e13b --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace-hash.c @@ -0,0 +1,1389 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include "aspeed-hace.h" + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG +#define AHASH_DBG(h, fmt, ...) \ + dev_info((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define AHASH_DBG(h, fmt, ...) \ + dev_dbg((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +/* Initialization Vectors for SHA-family */ +static const __be32 sha1_iv[8] = { + cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), + cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), + cpu_to_be32(SHA1_H4), 0, 0, 0 +}; + +static const __be32 sha224_iv[8] = { + cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), + cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), + cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), + cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), +}; + +static const __be32 sha256_iv[8] = { + cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), + cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), + cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), + cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), +}; + +static const __be64 sha384_iv[8] = { + cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), + cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), + cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), + cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7) +}; + +static const __be64 sha512_iv[8] = { + cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), + cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), + cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), + cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7) +}; + +static const __be32 sha512_224_iv[16] = { + cpu_to_be32(0xC8373D8CUL), cpu_to_be32(0xA24D5419UL), + cpu_to_be32(0x6699E173UL), cpu_to_be32(0xD6D4DC89UL), + cpu_to_be32(0xAEB7FA1DUL), cpu_to_be32(0x829CFF32UL), + cpu_to_be32(0x14D59D67UL), cpu_to_be32(0xCF9F2F58UL), + cpu_to_be32(0x692B6D0FUL), cpu_to_be32(0xA84DD47BUL), + cpu_to_be32(0x736FE377UL), cpu_to_be32(0x4289C404UL), + cpu_to_be32(0xA8859D3FUL), cpu_to_be32(0xC8361D6AUL), + cpu_to_be32(0xADE61211UL), cpu_to_be32(0xA192D691UL) +}; + +static const __be32 sha512_256_iv[16] = { + cpu_to_be32(0x94213122UL), cpu_to_be32(0x2CF72BFCUL), + cpu_to_be32(0xA35F559FUL), cpu_to_be32(0xC2644CC8UL), + cpu_to_be32(0x6BB89323UL), cpu_to_be32(0x51B1536FUL), + cpu_to_be32(0x19773896UL), cpu_to_be32(0xBDEA4059UL), + cpu_to_be32(0xE23E2896UL), cpu_to_be32(0xE3FF8EA8UL), + cpu_to_be32(0x251E5EBEUL), cpu_to_be32(0x92398653UL), + cpu_to_be32(0xFC99012BUL), cpu_to_be32(0xAAB8852CUL), + cpu_to_be32(0xDC2DB70EUL), cpu_to_be32(0xA22CC581UL) +}; + +/* The purpose of this padding is to ensure that the padded message is a + * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). + * The bit "1" is appended at the end of the message followed by + * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or + * 128 bits block (SHA384/SHA512) equals to the message length in bits + * is appended. + * + * For SHA1/SHA224/SHA256, padlen is calculated as followed: + * - if message length < 56 bytes then padlen = 56 - message length + * - else padlen = 64 + 56 - message length + * + * For SHA384/SHA512, padlen is calculated as followed: + * - if message length < 112 bytes then padlen = 112 - message length + * - else padlen = 128 + 112 - message length + */ +static void aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev, + struct aspeed_sham_reqctx *rctx) +{ + unsigned int index, padlen; + __be64 bits[2]; + + AHASH_DBG(hace_dev, "rctx flags:0x%x\n", (u32)rctx->flags); + + switch (rctx->flags & SHA_FLAGS_MASK) { + case SHA_FLAGS_SHA1: + case SHA_FLAGS_SHA224: + case SHA_FLAGS_SHA256: + bits[0] = cpu_to_be64(rctx->digcnt[0] << 3); + index = rctx->bufcnt & 0x3f; + padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); + *(rctx->buffer + rctx->bufcnt) = 0x80; + memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1); + memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 8); + rctx->bufcnt += padlen + 8; + break; + default: + bits[1] = cpu_to_be64(rctx->digcnt[0] << 3); + bits[0] = cpu_to_be64(rctx->digcnt[1] << 3 | + rctx->digcnt[0] >> 61); + index = rctx->bufcnt & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); + *(rctx->buffer + rctx->bufcnt) = 0x80; + memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1); + memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 16); + rctx->bufcnt += padlen + 16; + break; + } +} + +/* + * Prepare DMA buffer before hardware engine + * processing. + */ +static int aspeed_ahash_dma_prepare(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + int length, remain; + + length = rctx->total + rctx->bufcnt; + remain = length % rctx->block_size; + + AHASH_DBG(hace_dev, "length:0x%x, remain:0x%x\n", length, remain); + + if (rctx->bufcnt) + memcpy(hash_engine->ahash_src_addr, rctx->buffer, rctx->bufcnt); + + if (rctx->total + rctx->bufcnt < ASPEED_CRYPTO_SRC_DMA_BUF_LEN) { + scatterwalk_map_and_copy(hash_engine->ahash_src_addr + + rctx->bufcnt, rctx->src_sg, + rctx->offset, rctx->total - remain, 0); + rctx->offset += rctx->total - remain; + + } else { + dev_warn(hace_dev->dev, "Hash data length is too large\n"); + return -EINVAL; + } + + scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg, + rctx->offset, remain, 0); + + rctx->bufcnt = remain; + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + return -ENOMEM; + } + + hash_engine->src_length = length - remain; + hash_engine->src_dma = hash_engine->ahash_src_dma_addr; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return 0; +} + +/* + * Prepare DMA buffer as SG list buffer before + * hardware engine processing. + */ +static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct aspeed_sg_list *src_list; + struct scatterlist *s; + int length, remain, sg_len, i; + int rc = 0; + + remain = (rctx->total + rctx->bufcnt) % rctx->block_size; + length = rctx->total + rctx->bufcnt - remain; + + AHASH_DBG(hace_dev, "%s:0x%x, %s:0x%x, %s:0x%x, %s:0x%x\n", + "rctx total", rctx->total, "bufcnt", rctx->bufcnt, + "length", length, "remain", remain); + + sg_len = dma_map_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); + if (!sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + rc = -ENOMEM; + goto end; + } + + src_list = (struct aspeed_sg_list *)hash_engine->ahash_src_addr; + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + rc = -ENOMEM; + goto free_src_sg; + } + + if (rctx->bufcnt != 0) { + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, + rctx->buffer, + rctx->block_size * 2, + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; + } + + src_list[0].phy_addr = rctx->buffer_dma_addr; + src_list[0].len = rctx->bufcnt; + length -= src_list[0].len; + + /* Last sg list */ + if (length == 0) + src_list[0].len |= HASH_SG_LAST_LIST; + + src_list[0].phy_addr = cpu_to_le32(src_list[0].phy_addr); + src_list[0].len = cpu_to_le32(src_list[0].len); + src_list++; + } + + if (length != 0) { + for_each_sg(rctx->src_sg, s, sg_len, i) { + src_list[i].phy_addr = sg_dma_address(s); + + if (length > sg_dma_len(s)) { + src_list[i].len = sg_dma_len(s); + length -= sg_dma_len(s); + + } else { + /* Last sg list */ + src_list[i].len = length; + src_list[i].len |= HASH_SG_LAST_LIST; + length = 0; + } + + src_list[i].phy_addr = cpu_to_le32(src_list[i].phy_addr); + src_list[i].len = cpu_to_le32(src_list[i].len); + } + } + + if (length != 0) { + rc = -EINVAL; + goto free_rctx_buffer; + } + + rctx->offset = rctx->total - remain; + hash_engine->src_length = rctx->total + rctx->bufcnt - remain; + hash_engine->src_dma = hash_engine->ahash_src_dma_addr; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return 0; + +free_rctx_buffer: + if (rctx->bufcnt != 0) + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, DMA_TO_DEVICE); +free_rctx_digest: + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); +free_src_sg: + dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); +end: + return rc; +} + +static int aspeed_ahash_complete(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + + AHASH_DBG(hace_dev, "\n"); + + hash_engine->flags &= ~CRYPTO_FLAGS_BUSY; + + crypto_finalize_hash_request(hace_dev->crypt_engine_hash, req, 0); + + return 0; +} + +/* + * Copy digest to the corresponding request result. + * This function will be called at final() stage. + */ +static int aspeed_ahash_transfer(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, DMA_TO_DEVICE); + + memcpy(req->result, rctx->digest, rctx->digsize); + + return aspeed_ahash_complete(hace_dev); +} + +/* + * Trigger hardware engines to do the math. + */ +static int aspeed_hace_ahash_trigger(struct aspeed_hace_dev *hace_dev, + aspeed_hace_fn_t resume) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "src_dma:0x%x, digest_dma:0x%x, length:0x%x\n", + hash_engine->src_dma, hash_engine->digest_dma, + hash_engine->src_length); + + rctx->cmd |= HASH_CMD_INT_ENABLE; + hash_engine->resume = resume; + + ast_hace_write(hace_dev, hash_engine->src_dma, ASPEED_HACE_HASH_SRC); + ast_hace_write(hace_dev, hash_engine->digest_dma, + ASPEED_HACE_HASH_DIGEST_BUFF); + ast_hace_write(hace_dev, hash_engine->digest_dma, + ASPEED_HACE_HASH_KEY_BUFF); + ast_hace_write(hace_dev, hash_engine->src_length, + ASPEED_HACE_HASH_DATA_LEN); + + /* Memory barrier to ensure all data setup before engine starts */ + mb(); + + ast_hace_write(hace_dev, rctx->cmd, ASPEED_HACE_HASH_CMD); + + return -EINPROGRESS; +} + +/* + * HMAC resume aims to do the second pass produces + * the final HMAC code derived from the inner hash + * result and the outer key. + */ +static int aspeed_ahash_hmac_resume(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + int rc = 0; + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, DMA_TO_DEVICE); + + /* o key pad + hash sum 1 */ + memcpy(rctx->buffer, bctx->opad, rctx->block_size); + memcpy(rctx->buffer + rctx->block_size, rctx->digest, rctx->digsize); + + rctx->bufcnt = rctx->block_size + rctx->digsize; + rctx->digcnt[0] = rctx->block_size + rctx->digsize; + + aspeed_ahash_fill_padding(hace_dev, rctx); + memcpy(rctx->digest, rctx->sha_iv, rctx->ivsize); + + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + rc = -ENOMEM; + goto end; + } + + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, rctx->buffer, + rctx->block_size * 2, + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; + } + + hash_engine->src_dma = rctx->buffer_dma_addr; + hash_engine->src_length = rctx->bufcnt; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer); + +free_rctx_digest: + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); +end: + return rc; +} + +static int aspeed_ahash_req_final(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + int rc = 0; + + AHASH_DBG(hace_dev, "\n"); + + aspeed_ahash_fill_padding(hace_dev, rctx); + + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, + rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + rc = -ENOMEM; + goto end; + } + + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, + rctx->buffer, + rctx->block_size * 2, + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; + } + + hash_engine->src_dma = rctx->buffer_dma_addr; + hash_engine->src_length = rctx->bufcnt; + hash_engine->digest_dma = rctx->digest_dma_addr; + + if (rctx->flags & SHA_FLAGS_HMAC) + return aspeed_hace_ahash_trigger(hace_dev, + aspeed_ahash_hmac_resume); + + return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer); + +free_rctx_digest: + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); +end: + return rc; +} + +static int aspeed_ahash_update_resume_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); + + if (rctx->bufcnt != 0) + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, + DMA_TO_DEVICE); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg, rctx->offset, + rctx->total - rctx->offset, 0); + + rctx->bufcnt = rctx->total - rctx->offset; + rctx->cmd &= ~HASH_CMD_HASH_SRC_SG_CTRL; + + if (rctx->flags & SHA_FLAGS_FINUP) + return aspeed_ahash_req_final(hace_dev); + + return aspeed_ahash_complete(hace_dev); +} + +static int aspeed_ahash_update_resume(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + if (rctx->flags & SHA_FLAGS_FINUP) + return aspeed_ahash_req_final(hace_dev); + + return aspeed_ahash_complete(hace_dev); +} + +static int aspeed_ahash_req_update(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + aspeed_hace_fn_t resume; + int ret; + + AHASH_DBG(hace_dev, "\n"); + + if (hace_dev->version == AST2600_VERSION) { + rctx->cmd |= HASH_CMD_HASH_SRC_SG_CTRL; + resume = aspeed_ahash_update_resume_sg; + + } else { + resume = aspeed_ahash_update_resume; + } + + ret = hash_engine->dma_prepare(hace_dev); + if (ret) + return ret; + + return aspeed_hace_ahash_trigger(hace_dev, resume); +} + +static int aspeed_hace_hash_handle_queue(struct aspeed_hace_dev *hace_dev, + struct ahash_request *req) +{ + return crypto_transfer_hash_request_to_engine( + hace_dev->crypt_engine_hash, req); +} + +static int aspeed_ahash_do_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_engine_hash *hash_engine; + int ret = 0; + + hash_engine = &hace_dev->hash_engine; + hash_engine->flags |= CRYPTO_FLAGS_BUSY; + + if (rctx->op == SHA_OP_UPDATE) + ret = aspeed_ahash_req_update(hace_dev); + else if (rctx->op == SHA_OP_FINAL) + ret = aspeed_ahash_req_final(hace_dev); + + if (ret != -EINPROGRESS) + return ret; + + return 0; +} + +static int aspeed_ahash_prepare_request(struct crypto_engine *engine, + void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_engine_hash *hash_engine; + + hash_engine = &hace_dev->hash_engine; + hash_engine->req = req; + + if (hace_dev->version == AST2600_VERSION) + hash_engine->dma_prepare = aspeed_ahash_dma_prepare_sg; + else + hash_engine->dma_prepare = aspeed_ahash_dma_prepare; + + return 0; +} + +static int aspeed_sham_update(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes); + + rctx->total = req->nbytes; + rctx->src_sg = req->src; + rctx->offset = 0; + rctx->src_nents = sg_nents(req->src); + rctx->op = SHA_OP_UPDATE; + + rctx->digcnt[0] += rctx->total; + if (rctx->digcnt[0] < rctx->total) + rctx->digcnt[1]++; + + if (rctx->bufcnt + rctx->total < rctx->block_size) { + scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, + rctx->src_sg, rctx->offset, + rctx->total, 0); + rctx->bufcnt += rctx->total; + + return 0; + } + + return aspeed_hace_hash_handle_queue(hace_dev, req); +} + +static int aspeed_sham_shash_digest(struct crypto_shash *tfm, u32 flags, + const u8 *data, unsigned int len, u8 *out) +{ + SHASH_DESC_ON_STACK(shash, tfm); + + shash->tfm = tfm; + + return crypto_shash_digest(shash, data, len, out); +} + +static int aspeed_sham_final(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "req->nbytes:%d, rctx->total:%d\n", + req->nbytes, rctx->total); + rctx->op = SHA_OP_FINAL; + + return aspeed_hace_hash_handle_queue(hace_dev, req); +} + +static int aspeed_sham_finup(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + int rc1, rc2; + + AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes); + + rctx->flags |= SHA_FLAGS_FINUP; + + rc1 = aspeed_sham_update(req); + if (rc1 == -EINPROGRESS || rc1 == -EBUSY) + return rc1; + + /* + * final() has to be always called to cleanup resources + * even if update() failed, except EINPROGRESS + */ + rc2 = aspeed_sham_final(req); + + return rc1 ? : rc2; +} + +static int aspeed_sham_init(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + AHASH_DBG(hace_dev, "%s: digest size:%d\n", + crypto_tfm_alg_name(&tfm->base), + crypto_ahash_digestsize(tfm)); + + rctx->cmd = HASH_CMD_ACC_MODE; + rctx->flags = 0; + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA1 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA1; + rctx->digsize = SHA1_DIGEST_SIZE; + rctx->block_size = SHA1_BLOCK_SIZE; + rctx->sha_iv = sha1_iv; + rctx->ivsize = 32; + memcpy(rctx->digest, sha1_iv, rctx->ivsize); + break; + case SHA224_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA224 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA224; + rctx->digsize = SHA224_DIGEST_SIZE; + rctx->block_size = SHA224_BLOCK_SIZE; + rctx->sha_iv = sha224_iv; + rctx->ivsize = 32; + memcpy(rctx->digest, sha224_iv, rctx->ivsize); + break; + case SHA256_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA256 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA256; + rctx->digsize = SHA256_DIGEST_SIZE; + rctx->block_size = SHA256_BLOCK_SIZE; + rctx->sha_iv = sha256_iv; + rctx->ivsize = 32; + memcpy(rctx->digest, sha256_iv, rctx->ivsize); + break; + case SHA384_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA384 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA384; + rctx->digsize = SHA384_DIGEST_SIZE; + rctx->block_size = SHA384_BLOCK_SIZE; + rctx->sha_iv = (const __be32 *)sha384_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha384_iv, rctx->ivsize); + break; + case SHA512_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA512 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA512; + rctx->digsize = SHA512_DIGEST_SIZE; + rctx->block_size = SHA512_BLOCK_SIZE; + rctx->sha_iv = (const __be32 *)sha512_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha512_iv, rctx->ivsize); + break; + default: + dev_warn(tctx->hace_dev->dev, "digest size %d not support\n", + crypto_ahash_digestsize(tfm)); + return -EINVAL; + } + + rctx->bufcnt = 0; + rctx->total = 0; + rctx->digcnt[0] = 0; + rctx->digcnt[1] = 0; + + /* HMAC init */ + if (tctx->flags & SHA_FLAGS_HMAC) { + rctx->digcnt[0] = rctx->block_size; + rctx->bufcnt = rctx->block_size; + memcpy(rctx->buffer, bctx->ipad, rctx->block_size); + rctx->flags |= SHA_FLAGS_HMAC; + } + + return 0; +} + +static int aspeed_sha512s_init(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + AHASH_DBG(hace_dev, "digest size: %d\n", crypto_ahash_digestsize(tfm)); + + rctx->cmd = HASH_CMD_ACC_MODE; + rctx->flags = 0; + + switch (crypto_ahash_digestsize(tfm)) { + case SHA224_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA512_224 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA512_224; + rctx->digsize = SHA224_DIGEST_SIZE; + rctx->block_size = SHA512_BLOCK_SIZE; + rctx->sha_iv = sha512_224_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha512_224_iv, rctx->ivsize); + break; + case SHA256_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA512_256 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA512_256; + rctx->digsize = SHA256_DIGEST_SIZE; + rctx->block_size = SHA512_BLOCK_SIZE; + rctx->sha_iv = sha512_256_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha512_256_iv, rctx->ivsize); + break; + default: + dev_warn(tctx->hace_dev->dev, "digest size %d not support\n", + crypto_ahash_digestsize(tfm)); + return -EINVAL; + } + + rctx->bufcnt = 0; + rctx->total = 0; + rctx->digcnt[0] = 0; + rctx->digcnt[1] = 0; + + /* HMAC init */ + if (tctx->flags & SHA_FLAGS_HMAC) { + rctx->digcnt[0] = rctx->block_size; + rctx->bufcnt = rctx->block_size; + memcpy(rctx->buffer, bctx->ipad, rctx->block_size); + rctx->flags |= SHA_FLAGS_HMAC; + } + + return 0; +} + +static int aspeed_sham_digest(struct ahash_request *req) +{ + return aspeed_sham_init(req) ? : aspeed_sham_finup(req); +} + +static int aspeed_sham_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + int ds = crypto_shash_digestsize(bctx->shash); + int bs = crypto_shash_blocksize(bctx->shash); + int err = 0; + int i; + + AHASH_DBG(hace_dev, "%s: keylen:%d\n", crypto_tfm_alg_name(&tfm->base), + keylen); + + if (keylen > bs) { + err = aspeed_sham_shash_digest(bctx->shash, + crypto_shash_get_flags(bctx->shash), + key, keylen, bctx->ipad); + if (err) + return err; + keylen = ds; + + } else { + memcpy(bctx->ipad, key, keylen); + } + + memset(bctx->ipad + keylen, 0, bs - keylen); + memcpy(bctx->opad, bctx->ipad, bs); + + for (i = 0; i < bs; i++) { + bctx->ipad[i] ^= HMAC_IPAD_VALUE; + bctx->opad[i] ^= HMAC_OPAD_VALUE; + } + + return err; +} + +static int aspeed_sham_cra_init(struct crypto_tfm *tfm) +{ + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm); + struct aspeed_hace_alg *ast_alg; + + ast_alg = container_of(alg, struct aspeed_hace_alg, alg.ahash); + tctx->hace_dev = ast_alg->hace_dev; + tctx->flags = 0; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct aspeed_sham_reqctx)); + + if (ast_alg->alg_base) { + /* hmac related */ + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + tctx->flags |= SHA_FLAGS_HMAC; + bctx->shash = crypto_alloc_shash(ast_alg->alg_base, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(bctx->shash)) { + dev_warn(ast_alg->hace_dev->dev, + "base driver '%s' could not be loaded.\n", + ast_alg->alg_base); + return PTR_ERR(bctx->shash); + } + } + + tctx->enginectx.op.do_one_request = aspeed_ahash_do_request; + tctx->enginectx.op.prepare_request = aspeed_ahash_prepare_request; + tctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void aspeed_sham_cra_exit(struct crypto_tfm *tfm) +{ + struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(tfm)); + + if (tctx->flags & SHA_FLAGS_HMAC) { + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + crypto_free_shash(bctx->shash); + } +} + +static int aspeed_sham_export(struct ahash_request *req, void *out) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int aspeed_sham_import(struct ahash_request *req, const void *in) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +struct aspeed_hace_alg aspeed_ahash_algs[] = { + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha1", + .cra_driver_name = "aspeed-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "aspeed-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "aspeed-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha1", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "aspeed-hmac-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha224", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "aspeed-hmac-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha256", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "aspeed-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, +}; + +struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "aspeed-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "aspeed-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha512_224", + .cra_driver_name = "aspeed-sha512_224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha512_256", + .cra_driver_name = "aspeed-sha512_256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha384", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "aspeed-hmac-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha512", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "aspeed-hmac-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha512_224", + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha512_224)", + .cra_driver_name = "aspeed-hmac-sha512_224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha512_256", + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha512_256)", + .cra_driver_name = "aspeed-hmac-sha512_256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, +}; + +void aspeed_unregister_hace_hash_algs(struct aspeed_hace_dev *hace_dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs); i++) + crypto_unregister_ahash(&aspeed_ahash_algs[i].alg.ahash); + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs_g6); i++) + crypto_unregister_ahash(&aspeed_ahash_algs_g6[i].alg.ahash); +} + +void aspeed_register_hace_hash_algs(struct aspeed_hace_dev *hace_dev) +{ + int rc, i; + + AHASH_DBG(hace_dev, "\n"); + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs); i++) { + aspeed_ahash_algs[i].hace_dev = hace_dev; + rc = crypto_register_ahash(&aspeed_ahash_algs[i].alg.ahash); + if (rc) { + AHASH_DBG(hace_dev, "Failed to register %s\n", + aspeed_ahash_algs[i].alg.ahash.halg.base.cra_name); + } + } + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs_g6); i++) { + aspeed_ahash_algs_g6[i].hace_dev = hace_dev; + rc = crypto_register_ahash(&aspeed_ahash_algs_g6[i].alg.ahash); + if (rc) { + AHASH_DBG(hace_dev, "Failed to register %s\n", + aspeed_ahash_algs_g6[i].alg.ahash.halg.base.cra_name); + } + } +} diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c new file mode 100644 index 000000000000..23a66f481b62 --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "aspeed-hace.h" + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG +#define HACE_DBG(d, fmt, ...) \ + dev_info((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define HACE_DBG(d, fmt, ...) \ + dev_dbg((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +/* HACE interrupt service routine */ +static irqreturn_t aspeed_hace_irq(int irq, void *dev) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)dev; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + u32 sts; + + sts = ast_hace_read(hace_dev, ASPEED_HACE_STS); + ast_hace_write(hace_dev, sts, ASPEED_HACE_STS); + + HACE_DBG(hace_dev, "irq status: 0x%x\n", sts); + + if (sts & HACE_HASH_ISR) { + if (hash_engine->flags & CRYPTO_FLAGS_BUSY) + tasklet_schedule(&hash_engine->done_task); + else + dev_warn(hace_dev->dev, "HASH no active requests.\n"); + } + + return IRQ_HANDLED; +} + +static void aspeed_hace_hash_done_task(unsigned long data) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)data; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + + hash_engine->resume(hace_dev); +} + +static void aspeed_hace_register(struct aspeed_hace_dev *hace_dev) +{ +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH + aspeed_register_hace_hash_algs(hace_dev); +#endif +} + +static void aspeed_hace_unregister(struct aspeed_hace_dev *hace_dev) +{ +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH + aspeed_unregister_hace_hash_algs(hace_dev); +#endif +} + +static const struct of_device_id aspeed_hace_of_matches[] = { + { .compatible = "aspeed,ast2500-hace", .data = (void *)5, }, + { .compatible = "aspeed,ast2600-hace", .data = (void *)6, }, + {}, +}; + +static int aspeed_hace_probe(struct platform_device *pdev) +{ + const struct of_device_id *hace_dev_id; + struct aspeed_engine_hash *hash_engine; + struct aspeed_hace_dev *hace_dev; + struct resource *res; + int rc; + + hace_dev = devm_kzalloc(&pdev->dev, sizeof(struct aspeed_hace_dev), + GFP_KERNEL); + if (!hace_dev) + return -ENOMEM; + + hace_dev_id = of_match_device(aspeed_hace_of_matches, &pdev->dev); + if (!hace_dev_id) { + dev_err(&pdev->dev, "Failed to match hace dev id\n"); + return -EINVAL; + } + + hace_dev->dev = &pdev->dev; + hace_dev->version = (unsigned long)hace_dev_id->data; + hash_engine = &hace_dev->hash_engine; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + + platform_set_drvdata(pdev, hace_dev); + + hace_dev->regs = devm_ioremap_resource(&pdev->dev, res); + if (!hace_dev->regs) { + dev_err(&pdev->dev, "Failed to map resources\n"); + return -ENOMEM; + } + + /* Get irq number and register it */ + hace_dev->irq = platform_get_irq(pdev, 0); + if (!hace_dev->irq) { + dev_err(&pdev->dev, "Failed to get interrupt\n"); + return -ENXIO; + } + + rc = devm_request_irq(&pdev->dev, hace_dev->irq, aspeed_hace_irq, 0, + dev_name(&pdev->dev), hace_dev); + if (rc) { + dev_err(&pdev->dev, "Failed to request interrupt\n"); + return rc; + } + + /* Get clk and enable it */ + hace_dev->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(hace_dev->clk)) { + dev_err(&pdev->dev, "Failed to get clk\n"); + return -ENODEV; + } + + rc = clk_prepare_enable(hace_dev->clk); + if (rc) { + dev_err(&pdev->dev, "Failed to enable clock 0x%x\n", rc); + return rc; + } + + /* Initialize crypto hardware engine structure for hash */ + hace_dev->crypt_engine_hash = crypto_engine_alloc_init(hace_dev->dev, + true); + if (!hace_dev->crypt_engine_hash) { + rc = -ENOMEM; + goto clk_exit; + } + + rc = crypto_engine_start(hace_dev->crypt_engine_hash); + if (rc) + goto err_engine_hash_start; + + tasklet_init(&hash_engine->done_task, aspeed_hace_hash_done_task, + (unsigned long)hace_dev); + + /* Allocate DMA buffer for hash engine input used */ + hash_engine->ahash_src_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_HASH_SRC_DMA_BUF_LEN, + &hash_engine->ahash_src_dma_addr, + GFP_KERNEL); + if (!hash_engine->ahash_src_addr) { + dev_err(&pdev->dev, "Failed to allocate dma buffer\n"); + rc = -ENOMEM; + goto err_engine_hash_start; + } + + aspeed_hace_register(hace_dev); + + dev_info(&pdev->dev, "Aspeed Crypto Accelerator successfully registered\n"); + + return 0; + +err_engine_hash_start: + crypto_engine_exit(hace_dev->crypt_engine_hash); +clk_exit: + clk_disable_unprepare(hace_dev->clk); + + return rc; +} + +static int aspeed_hace_remove(struct platform_device *pdev) +{ + struct aspeed_hace_dev *hace_dev = platform_get_drvdata(pdev); + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + + aspeed_hace_unregister(hace_dev); + + crypto_engine_exit(hace_dev->crypt_engine_hash); + + tasklet_kill(&hash_engine->done_task); + + clk_disable_unprepare(hace_dev->clk); + + return 0; +} + +MODULE_DEVICE_TABLE(of, aspeed_hace_of_matches); + +static struct platform_driver aspeed_hace_driver = { + .probe = aspeed_hace_probe, + .remove = aspeed_hace_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = aspeed_hace_of_matches, + }, +}; + +module_platform_driver(aspeed_hace_driver); + +MODULE_AUTHOR("Neal Liu "); +MODULE_DESCRIPTION("Aspeed HACE driver Crypto Accelerator"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/aspeed/aspeed-hace.h b/drivers/crypto/aspeed/aspeed-hace.h new file mode 100644 index 000000000000..3494ff22f69d --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __ASPEED_HACE_H__ +#define __ASPEED_HACE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/***************************** + * * + * HACE register definitions * + * * + * ***************************/ + +#define ASPEED_HACE_STS 0x1C /* HACE Status Register */ +#define ASPEED_HACE_HASH_SRC 0x20 /* Hash Data Source Base Address Register */ +#define ASPEED_HACE_HASH_DIGEST_BUFF 0x24 /* Hash Digest Write Buffer Base Address Register */ +#define ASPEED_HACE_HASH_KEY_BUFF 0x28 /* Hash HMAC Key Buffer Base Address Register */ +#define ASPEED_HACE_HASH_DATA_LEN 0x2C /* Hash Data Length Register */ +#define ASPEED_HACE_HASH_CMD 0x30 /* Hash Engine Command Register */ + +/* interrupt status reg */ +#define HACE_HASH_ISR BIT(9) +#define HACE_HASH_BUSY BIT(0) + +/* hash cmd reg */ +#define HASH_CMD_MBUS_REQ_SYNC_EN BIT(20) +#define HASH_CMD_HASH_SRC_SG_CTRL BIT(18) +#define HASH_CMD_SHA512_224 (0x3 << 10) +#define HASH_CMD_SHA512_256 (0x2 << 10) +#define HASH_CMD_SHA384 (0x1 << 10) +#define HASH_CMD_SHA512 (0) +#define HASH_CMD_INT_ENABLE BIT(9) +#define HASH_CMD_HMAC (0x1 << 7) +#define HASH_CMD_ACC_MODE (0x2 << 7) +#define HASH_CMD_HMAC_KEY (0x3 << 7) +#define HASH_CMD_SHA1 (0x2 << 4) +#define HASH_CMD_SHA224 (0x4 << 4) +#define HASH_CMD_SHA256 (0x5 << 4) +#define HASH_CMD_SHA512_SER (0x6 << 4) +#define HASH_CMD_SHA_SWAP (0x2 << 2) + +#define HASH_SG_LAST_LIST BIT(31) + +#define CRYPTO_FLAGS_BUSY BIT(1) + +#define SHA_OP_UPDATE 1 +#define SHA_OP_FINAL 2 + +#define SHA_FLAGS_SHA1 BIT(0) +#define SHA_FLAGS_SHA224 BIT(1) +#define SHA_FLAGS_SHA256 BIT(2) +#define SHA_FLAGS_SHA384 BIT(3) +#define SHA_FLAGS_SHA512 BIT(4) +#define SHA_FLAGS_SHA512_224 BIT(5) +#define SHA_FLAGS_SHA512_256 BIT(6) +#define SHA_FLAGS_HMAC BIT(8) +#define SHA_FLAGS_FINUP BIT(9) +#define SHA_FLAGS_MASK (0xff) + +#define ASPEED_CRYPTO_SRC_DMA_BUF_LEN 0xa000 +#define ASPEED_CRYPTO_DST_DMA_BUF_LEN 0xa000 +#define ASPEED_CRYPTO_GCM_TAG_OFFSET 0x9ff0 +#define ASPEED_HASH_SRC_DMA_BUF_LEN 0xa000 +#define ASPEED_HASH_QUEUE_LENGTH 50 + +struct aspeed_hace_dev; + +typedef int (*aspeed_hace_fn_t)(struct aspeed_hace_dev *); + +struct aspeed_sg_list { + __le32 len; + __le32 phy_addr; +}; + +struct aspeed_engine_hash { + struct tasklet_struct done_task; + unsigned long flags; + struct ahash_request *req; + + /* input buffer */ + void *ahash_src_addr; + dma_addr_t ahash_src_dma_addr; + + dma_addr_t src_dma; + dma_addr_t digest_dma; + + size_t src_length; + + /* callback func */ + aspeed_hace_fn_t resume; + aspeed_hace_fn_t dma_prepare; +}; + +struct aspeed_sha_hmac_ctx { + struct crypto_shash *shash; + u8 ipad[SHA512_BLOCK_SIZE]; + u8 opad[SHA512_BLOCK_SIZE]; +}; + +struct aspeed_sham_ctx { + struct crypto_engine_ctx enginectx; + + struct aspeed_hace_dev *hace_dev; + unsigned long flags; /* hmac flag */ + + struct aspeed_sha_hmac_ctx base[0]; +}; + +struct aspeed_sham_reqctx { + unsigned long flags; /* final update flag should no use*/ + unsigned long op; /* final or update */ + u32 cmd; /* trigger cmd */ + + /* walk state */ + struct scatterlist *src_sg; + int src_nents; + unsigned int offset; /* offset in current sg */ + unsigned int total; /* per update length */ + + size_t digsize; + size_t block_size; + size_t ivsize; + const __be32 *sha_iv; + + /* remain data buffer */ + u8 buffer[SHA512_BLOCK_SIZE * 2]; + dma_addr_t buffer_dma_addr; + size_t bufcnt; /* buffer counter */ + + /* output buffer */ + u8 digest[SHA512_DIGEST_SIZE] __aligned(64); + dma_addr_t digest_dma_addr; + u64 digcnt[2]; +}; + +struct aspeed_hace_dev { + void __iomem *regs; + struct device *dev; + int irq; + struct clk *clk; + unsigned long version; + + struct crypto_engine *crypt_engine_hash; + + struct aspeed_engine_hash hash_engine; +}; + +struct aspeed_hace_alg { + struct aspeed_hace_dev *hace_dev; + + const char *alg_base; + + union { + struct skcipher_alg skcipher; + struct ahash_alg ahash; + } alg; +}; + +enum aspeed_version { + AST2500_VERSION = 5, + AST2600_VERSION +}; + +#define ast_hace_write(hace, val, offset) \ + writel((val), (hace)->regs + (offset)) +#define ast_hace_read(hace, offset) \ + readl((hace)->regs + (offset)) + +void aspeed_register_hace_hash_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_unregister_hace_hash_algs(struct aspeed_hace_dev *hace_dev); + +#endif -- cgit From dffc3c566be3da314fdc98432dd166abb35aee03 Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Thu, 18 Aug 2022 11:59:53 +0800 Subject: dt-bindings: clock: Add AST2500/AST2600 HACE reset definition Add HACE reset bit definition for AST2500/AST2600. Signed-off-by: Neal Liu Signed-off-by: Johnny Huang Acked-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- include/dt-bindings/clock/aspeed-clock.h | 1 + include/dt-bindings/clock/ast2600-clock.h | 1 + 2 files changed, 2 insertions(+) diff --git a/include/dt-bindings/clock/aspeed-clock.h b/include/dt-bindings/clock/aspeed-clock.h index 9ff4f6e4558c..06d568382c77 100644 --- a/include/dt-bindings/clock/aspeed-clock.h +++ b/include/dt-bindings/clock/aspeed-clock.h @@ -52,5 +52,6 @@ #define ASPEED_RESET_I2C 7 #define ASPEED_RESET_AHB 8 #define ASPEED_RESET_CRT1 9 +#define ASPEED_RESET_HACE 10 #endif diff --git a/include/dt-bindings/clock/ast2600-clock.h b/include/dt-bindings/clock/ast2600-clock.h index 62b9520a00fd..d8b0db2f7a7d 100644 --- a/include/dt-bindings/clock/ast2600-clock.h +++ b/include/dt-bindings/clock/ast2600-clock.h @@ -111,6 +111,7 @@ #define ASPEED_RESET_PCIE_RC_O 19 #define ASPEED_RESET_PCIE_RC_OEN 18 #define ASPEED_RESET_PCI_DP 5 +#define ASPEED_RESET_HACE 4 #define ASPEED_RESET_AHB 1 #define ASPEED_RESET_SDRAM 0 -- cgit From a1a2990e6fea186bf0d3e7fada8645a1a85bd12a Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Thu, 18 Aug 2022 11:59:54 +0800 Subject: ARM: dts: aspeed: Add HACE device controller node Add hace node to device tree for AST2500/AST2600. Signed-off-by: Neal Liu Signed-off-by: Johnny Huang Reviewed-by: Dhananjay Phadke Signed-off-by: Herbert Xu --- arch/arm/boot/dts/aspeed-g5.dtsi | 8 ++++++++ arch/arm/boot/dts/aspeed-g6.dtsi | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/arch/arm/boot/dts/aspeed-g5.dtsi b/arch/arm/boot/dts/aspeed-g5.dtsi index c89092c3905b..04f98d1dbb97 100644 --- a/arch/arm/boot/dts/aspeed-g5.dtsi +++ b/arch/arm/boot/dts/aspeed-g5.dtsi @@ -262,6 +262,14 @@ quality = <100>; }; + hace: crypto@1e6e3000 { + compatible = "aspeed,ast2500-hace"; + reg = <0x1e6e3000 0x100>; + interrupts = <4>; + clocks = <&syscon ASPEED_CLK_GATE_YCLK>; + resets = <&syscon ASPEED_RESET_HACE>; + }; + gfx: display@1e6e6000 { compatible = "aspeed,ast2500-gfx", "syscon"; reg = <0x1e6e6000 0x1000>; diff --git a/arch/arm/boot/dts/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed-g6.dtsi index 6660564855ff..095cf8d03616 100644 --- a/arch/arm/boot/dts/aspeed-g6.dtsi +++ b/arch/arm/boot/dts/aspeed-g6.dtsi @@ -323,6 +323,14 @@ #size-cells = <1>; ranges; + hace: crypto@1e6d0000 { + compatible = "aspeed,ast2600-hace"; + reg = <0x1e6d0000 0x200>; + interrupts = ; + clocks = <&syscon ASPEED_CLK_GATE_YCLK>; + resets = <&syscon ASPEED_RESET_HACE>; + }; + syscon: syscon@1e6e2000 { compatible = "aspeed,ast2600-scu", "syscon", "simple-mfd"; reg = <0x1e6e2000 0x1000>; -- cgit From c3708e6562694872e34c2aa41b6c949e98ee5945 Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Thu, 18 Aug 2022 11:59:55 +0800 Subject: dt-bindings: crypto: add documentation for aspeed hace Add device tree binding documentation for the Aspeed Hash and Crypto Engines (HACE) Controller. Signed-off-by: Neal Liu Signed-off-by: Johnny Huang Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- .../bindings/crypto/aspeed,ast2500-hace.yaml | 53 ++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml diff --git a/Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml b/Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml new file mode 100644 index 000000000000..a772d232de09 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/aspeed,ast2500-hace.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ASPEED HACE hash and crypto Hardware Accelerator Engines + +maintainers: + - Neal Liu + +description: | + The Hash and Crypto Engine (HACE) is designed to accelerate the throughput + of hash data digest, encryption, and decryption. Basically, HACE can be + divided into two independently engines - Hash Engine and Crypto Engine. + +properties: + compatible: + enum: + - aspeed,ast2500-hace + - aspeed,ast2600-hace + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + resets: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - interrupts + - resets + +additionalProperties: false + +examples: + - | + #include + hace: crypto@1e6d0000 { + compatible = "aspeed,ast2600-hace"; + reg = <0x1e6d0000 0x200>; + interrupts = <4>; + clocks = <&syscon ASPEED_CLK_GATE_YCLK>; + resets = <&syscon ASPEED_RESET_HACE>; + }; -- cgit From 62f58b1637b7c8df34afaa548cb4b03d31c0764b Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Thu, 18 Aug 2022 11:59:56 +0800 Subject: crypto: aspeed - add HACE crypto driver Add HACE crypto driver to support symmetric-key encryption and decryption with multiple modes of operation. Signed-off-by: Neal Liu Signed-off-by: Johnny Huang Reviewed-by: Dhananjay Phadke Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/Kconfig | 17 + drivers/crypto/aspeed/Makefile | 7 +- drivers/crypto/aspeed/aspeed-hace-crypto.c | 1135 ++++++++++++++++++++++++++++ drivers/crypto/aspeed/aspeed-hace.c | 84 +- drivers/crypto/aspeed/aspeed-hace.h | 112 +++ 5 files changed, 1352 insertions(+), 3 deletions(-) create mode 100644 drivers/crypto/aspeed/aspeed-hace-crypto.c diff --git a/drivers/crypto/aspeed/Kconfig b/drivers/crypto/aspeed/Kconfig index 03e3fc61498e..001bf2e09a72 100644 --- a/drivers/crypto/aspeed/Kconfig +++ b/drivers/crypto/aspeed/Kconfig @@ -30,3 +30,20 @@ config CRYPTO_DEV_ASPEED_HACE_HASH hash driver. Supports multiple message digest standards, including SHA-1, SHA-224, SHA-256, SHA-384, SHA-512, and so on. + +config CRYPTO_DEV_ASPEED_HACE_CRYPTO + bool "Enable Aspeed Hash & Crypto Engine (HACE) crypto" + depends on CRYPTO_DEV_ASPEED + select CRYPTO_ENGINE + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CFB + select CRYPTO_OFB + select CRYPTO_CTR + help + Select here to enable Aspeed Hash & Crypto Engine (HACE) + crypto driver. + Supports AES/DES symmetric-key encryption and decryption + with ECB/CBC/CFB/OFB/CTR options. diff --git a/drivers/crypto/aspeed/Makefile b/drivers/crypto/aspeed/Makefile index 8bc8d4fed5a9..421e2ca9c53e 100644 --- a/drivers/crypto/aspeed/Makefile +++ b/drivers/crypto/aspeed/Makefile @@ -1,6 +1,9 @@ obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed_crypto.o -aspeed_crypto-objs := aspeed-hace.o \ - $(hace-hash-y) +aspeed_crypto-objs := aspeed-hace.o \ + $(hace-hash-y) \ + $(hace-crypto-y) obj-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) += aspeed-hace-hash.o hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace-hash.o +obj-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) += aspeed-hace-crypto.o +hace-crypto-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) := aspeed-hace-crypto.o diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c new file mode 100644 index 000000000000..ba6158f5cf18 --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -0,0 +1,1135 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include "aspeed-hace.h" + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO_DEBUG +#define CIPHER_DBG(h, fmt, ...) \ + dev_info((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define CIPHER_DBG(h, fmt, ...) \ + dev_dbg((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +static int aspeed_crypto_do_fallback(struct skcipher_request *areq) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + + if (rctx->enc_cmd & HACE_CMD_ENCRYPT) + err = crypto_skcipher_encrypt(&rctx->fallback_req); + else + err = crypto_skcipher_decrypt(&rctx->fallback_req); + + return err; +} + +static bool aspeed_crypto_need_fallback(struct skcipher_request *areq) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq); + + if (areq->cryptlen == 0) + return true; + + if ((rctx->enc_cmd & HACE_CMD_DES_SELECT) && + !IS_ALIGNED(areq->cryptlen, DES_BLOCK_SIZE)) + return true; + + if ((!(rctx->enc_cmd & HACE_CMD_DES_SELECT)) && + !IS_ALIGNED(areq->cryptlen, AES_BLOCK_SIZE)) + return true; + + return false; +} + +static int aspeed_hace_crypto_handle_queue(struct aspeed_hace_dev *hace_dev, + struct skcipher_request *req) +{ + if (hace_dev->version == AST2500_VERSION && + aspeed_crypto_need_fallback(req)) { + CIPHER_DBG(hace_dev, "SW fallback\n"); + return aspeed_crypto_do_fallback(req); + } + + return crypto_transfer_skcipher_request_to_engine( + hace_dev->crypt_engine_crypto, req); +} + +static int aspeed_crypto_do_request(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = skcipher_request_cast(areq); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + struct aspeed_engine_crypto *crypto_engine; + int rc; + + crypto_engine = &hace_dev->crypto_engine; + crypto_engine->req = req; + crypto_engine->flags |= CRYPTO_FLAGS_BUSY; + + rc = ctx->start(hace_dev); + + if (rc != -EINPROGRESS) + return -EIO; + + return 0; +} + +static int aspeed_sk_complete(struct aspeed_hace_dev *hace_dev, int err) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) { + if (rctx->enc_cmd & HACE_CMD_DES_SELECT) + memcpy(req->iv, crypto_engine->cipher_ctx + + DES_KEY_SIZE, DES_KEY_SIZE); + else + memcpy(req->iv, crypto_engine->cipher_ctx, + AES_BLOCK_SIZE); + } + + crypto_engine->flags &= ~CRYPTO_FLAGS_BUSY; + + crypto_finalize_skcipher_request(hace_dev->crypt_engine_crypto, req, + err); + + return err; +} + +static int aspeed_sk_transfer_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct device *dev = hace_dev->dev; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, rctx->dst_nents, DMA_FROM_DEVICE); + } + + return aspeed_sk_complete(hace_dev, 0); +} + +static int aspeed_sk_transfer(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *out_sg; + int nbytes = 0; + int rc = 0; + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + out_sg = req->dst; + + /* Copy output buffer to dst scatter-gather lists */ + nbytes = sg_copy_from_buffer(out_sg, rctx->dst_nents, + crypto_engine->cipher_addr, req->cryptlen); + if (!nbytes) { + dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n", + "nbytes", nbytes, "cryptlen", req->cryptlen); + rc = -EINVAL; + } + + CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n", + "nbytes", nbytes, "req->cryptlen", req->cryptlen, + "nb_out_sg", rctx->dst_nents, + "cipher addr", crypto_engine->cipher_addr); + + return aspeed_sk_complete(hace_dev, rc); +} + +static int aspeed_sk_start(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *in_sg; + int nbytes; + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + in_sg = req->src; + + nbytes = sg_copy_to_buffer(in_sg, rctx->src_nents, + crypto_engine->cipher_addr, req->cryptlen); + + CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n", + "nbytes", nbytes, "req->cryptlen", req->cryptlen, + "nb_in_sg", rctx->src_nents, + "cipher addr", crypto_engine->cipher_addr); + + if (!nbytes) { + dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n", + "nbytes", nbytes, "cryptlen", req->cryptlen); + return -EINVAL; + } + + crypto_engine->resume = aspeed_sk_transfer; + + /* Trigger engines */ + ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr, + ASPEED_HACE_SRC); + ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr, + ASPEED_HACE_DEST); + ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN); + ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD); + + return -EINPROGRESS; +} + +static int aspeed_sk_start_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_sg_list *src_list, *dst_list; + dma_addr_t src_dma_addr, dst_dma_addr; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *s; + int src_sg_len; + int dst_sg_len; + int total, i; + int rc; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + rctx->enc_cmd |= HACE_CMD_DES_SG_CTRL | HACE_CMD_SRC_SG_CTRL | + HACE_CMD_AES_KEY_HW_EXP | HACE_CMD_MBUS_REQ_SYNC_EN; + + /* BIDIRECTIONAL */ + if (req->dst == req->src) { + src_sg_len = dma_map_sg(hace_dev->dev, req->src, + rctx->src_nents, DMA_BIDIRECTIONAL); + dst_sg_len = src_sg_len; + if (!src_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + return -EINVAL; + } + + } else { + src_sg_len = dma_map_sg(hace_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (!src_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + return -EINVAL; + } + + dst_sg_len = dma_map_sg(hace_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (!dst_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() dst error\n"); + rc = -EINVAL; + goto free_req_src; + } + } + + src_list = (struct aspeed_sg_list *)crypto_engine->cipher_addr; + src_dma_addr = crypto_engine->cipher_dma_addr; + total = req->cryptlen; + + for_each_sg(req->src, s, src_sg_len, i) { + src_list[i].phy_addr = sg_dma_address(s); + + if (total > sg_dma_len(s)) { + src_list[i].len = sg_dma_len(s); + total -= src_list[i].len; + + } else { + /* last sg list */ + src_list[i].len = total; + src_list[i].len |= BIT(31); + total = 0; + } + + src_list[i].phy_addr = cpu_to_le32(src_list[i].phy_addr); + src_list[i].len = cpu_to_le32(src_list[i].len); + } + + if (total != 0) { + rc = -EINVAL; + goto free_req; + } + + if (req->dst == req->src) { + dst_list = src_list; + dst_dma_addr = src_dma_addr; + + } else { + dst_list = (struct aspeed_sg_list *)crypto_engine->dst_sg_addr; + dst_dma_addr = crypto_engine->dst_sg_dma_addr; + total = req->cryptlen; + + for_each_sg(req->dst, s, dst_sg_len, i) { + dst_list[i].phy_addr = sg_dma_address(s); + + if (total > sg_dma_len(s)) { + dst_list[i].len = sg_dma_len(s); + total -= dst_list[i].len; + + } else { + /* last sg list */ + dst_list[i].len = total; + dst_list[i].len |= BIT(31); + total = 0; + } + + dst_list[i].phy_addr = cpu_to_le32(dst_list[i].phy_addr); + dst_list[i].len = cpu_to_le32(dst_list[i].len); + + } + + dst_list[dst_sg_len].phy_addr = 0; + dst_list[dst_sg_len].len = 0; + } + + if (total != 0) { + rc = -EINVAL; + goto free_req; + } + + crypto_engine->resume = aspeed_sk_transfer_sg; + + /* Memory barrier to ensure all data setup before engine starts */ + mb(); + + /* Trigger engines */ + ast_hace_write(hace_dev, src_dma_addr, ASPEED_HACE_SRC); + ast_hace_write(hace_dev, dst_dma_addr, ASPEED_HACE_DEST); + ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN); + ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD); + + return -EINPROGRESS; + +free_req: + if (req->dst == req->src) { + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, + DMA_BIDIRECTIONAL); + + } else { + dma_unmap_sg(hace_dev->dev, req->dst, rctx->dst_nents, + DMA_TO_DEVICE); + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, + DMA_TO_DEVICE); + } + + return rc; + +free_req_src: + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + + return rc; +} + +static int aspeed_hace_skcipher_trigger(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct crypto_skcipher *cipher; + struct aspeed_cipher_ctx *ctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + cipher = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(cipher); + + /* enable interrupt */ + rctx->enc_cmd |= HACE_CMD_ISR_EN; + + rctx->dst_nents = sg_nents(req->dst); + rctx->src_nents = sg_nents(req->src); + + ast_hace_write(hace_dev, crypto_engine->cipher_ctx_dma, + ASPEED_HACE_CONTEXT); + + if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) { + if (rctx->enc_cmd & HACE_CMD_DES_SELECT) + memcpy(crypto_engine->cipher_ctx + DES_BLOCK_SIZE, + req->iv, DES_BLOCK_SIZE); + else + memcpy(crypto_engine->cipher_ctx, req->iv, + AES_BLOCK_SIZE); + } + + if (hace_dev->version == AST2600_VERSION) { + memcpy(crypto_engine->cipher_ctx + 16, ctx->key, ctx->key_len); + + return aspeed_sk_start_sg(hace_dev); + } + + memcpy(crypto_engine->cipher_ctx + 16, ctx->key, AES_MAX_KEYLENGTH); + + return aspeed_sk_start(hace_dev); +} + +static int aspeed_des_crypt(struct skcipher_request *req, u32 cmd) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK; + + CIPHER_DBG(hace_dev, "\n"); + + if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) { + if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) + return -EINVAL; + } + + rctx->enc_cmd = cmd | HACE_CMD_DES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE | + HACE_CMD_DES | HACE_CMD_CONTEXT_LOAD_ENABLE | + HACE_CMD_CONTEXT_SAVE_ENABLE; + + return aspeed_hace_crypto_handle_queue(hace_dev, req); +} + +static int aspeed_des_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + int rc; + + CIPHER_DBG(hace_dev, "keylen: %d bits\n", keylen); + + if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) { + dev_warn(hace_dev->dev, "invalid keylen: %d bits\n", keylen); + return -EINVAL; + } + + if (keylen == DES_KEY_SIZE) { + rc = crypto_des_verify_key(tfm, key); + if (rc) + return rc; + + } else if (keylen == DES3_EDE_KEY_SIZE) { + rc = crypto_des3_ede_verify_key(tfm, key); + if (rc) + return rc; + } + + memcpy(ctx->key, key, keylen); + ctx->key_len = keylen; + + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int aspeed_tdes_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ofb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ofb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cfb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cfb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_des_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ofb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ofb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cfb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cfb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_aes_crypt(struct skcipher_request *req, u32 cmd) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK; + + if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) { + if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) + return -EINVAL; + } + + CIPHER_DBG(hace_dev, "%s\n", + (cmd & HACE_CMD_ENCRYPT) ? "encrypt" : "decrypt"); + + cmd |= HACE_CMD_AES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE | + HACE_CMD_CONTEXT_LOAD_ENABLE | HACE_CMD_CONTEXT_SAVE_ENABLE; + + switch (ctx->key_len) { + case AES_KEYSIZE_128: + cmd |= HACE_CMD_AES128; + break; + case AES_KEYSIZE_192: + cmd |= HACE_CMD_AES192; + break; + case AES_KEYSIZE_256: + cmd |= HACE_CMD_AES256; + break; + default: + return -EINVAL; + } + + rctx->enc_cmd = cmd; + + return aspeed_hace_crypto_handle_queue(hace_dev, req); +} + +static int aspeed_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + struct crypto_aes_ctx gen_aes_key; + + CIPHER_DBG(hace_dev, "keylen: %d bits\n", (keylen * 8)); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + if (ctx->hace_dev->version == AST2500_VERSION) { + aes_expandkey(&gen_aes_key, key, keylen); + memcpy(ctx->key, gen_aes_key.key_enc, AES_MAX_KEYLENGTH); + + } else { + memcpy(ctx->key, key, keylen); + } + + ctx->key_len = keylen; + + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int aspeed_aes_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR); +} + +static int aspeed_aes_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR); +} + +static int aspeed_aes_ofb_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB); +} + +static int aspeed_aes_ofb_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB); +} + +static int aspeed_aes_cfb_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB); +} + +static int aspeed_aes_cfb_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB); +} + +static int aspeed_aes_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC); +} + +static int aspeed_aes_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC); +} + +static int aspeed_aes_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB); +} + +static int aspeed_aes_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB); +} + +static int aspeed_crypto_cra_init(struct crypto_skcipher *tfm) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + const char *name = crypto_tfm_alg_name(&tfm->base); + struct aspeed_hace_alg *crypto_alg; + + + crypto_alg = container_of(alg, struct aspeed_hace_alg, alg.skcipher); + ctx->hace_dev = crypto_alg->hace_dev; + ctx->start = aspeed_hace_skcipher_trigger; + + CIPHER_DBG(ctx->hace_dev, "%s\n", name); + + ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + dev_err(ctx->hace_dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(ctx->fallback_tfm)); + return PTR_ERR(ctx->fallback_tfm); + } + + crypto_skcipher_set_reqsize(tfm, sizeof(struct aspeed_cipher_reqctx) + + crypto_skcipher_reqsize(ctx->fallback_tfm)); + + ctx->enginectx.op.do_one_request = aspeed_crypto_do_request; + ctx->enginectx.op.prepare_request = NULL; + ctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void aspeed_crypto_cra_exit(struct crypto_skcipher *tfm) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + + CIPHER_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(&tfm->base)); + crypto_free_skcipher(ctx->fallback_tfm); +} + +struct aspeed_hace_alg aspeed_crypto_algs[] = { + { + .alg.skcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ecb_encrypt, + .decrypt = aspeed_aes_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "aspeed-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_cbc_encrypt, + .decrypt = aspeed_aes_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "aspeed-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_cfb_encrypt, + .decrypt = aspeed_aes_cfb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cfb(aes)", + .cra_driver_name = "aspeed-cfb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ofb_encrypt, + .decrypt = aspeed_aes_ofb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ofb(aes)", + .cra_driver_name = "aspeed-ofb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ecb_encrypt, + .decrypt = aspeed_des_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "aspeed-ecb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_cbc_encrypt, + .decrypt = aspeed_des_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "aspeed-cbc-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_cfb_encrypt, + .decrypt = aspeed_des_cfb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cfb(des)", + .cra_driver_name = "aspeed-cfb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ofb_encrypt, + .decrypt = aspeed_des_ofb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ofb(des)", + .cra_driver_name = "aspeed-ofb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ecb_encrypt, + .decrypt = aspeed_tdes_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "aspeed-ecb-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_cbc_encrypt, + .decrypt = aspeed_tdes_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "aspeed-cbc-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_cfb_encrypt, + .decrypt = aspeed_tdes_cfb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cfb(des3_ede)", + .cra_driver_name = "aspeed-cfb-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ofb_encrypt, + .decrypt = aspeed_tdes_ofb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ofb(des3_ede)", + .cra_driver_name = "aspeed-ofb-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, +}; + +struct aspeed_hace_alg aspeed_crypto_algs_g6[] = { + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ctr_encrypt, + .decrypt = aspeed_aes_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "aspeed-ctr-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ctr_encrypt, + .decrypt = aspeed_des_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(des)", + .cra_driver_name = "aspeed-ctr-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ctr_encrypt, + .decrypt = aspeed_tdes_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(des3_ede)", + .cra_driver_name = "aspeed-ctr-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + +}; + +void aspeed_unregister_hace_crypto_algs(struct aspeed_hace_dev *hace_dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++) + crypto_unregister_skcipher(&aspeed_crypto_algs[i].alg.skcipher); + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++) + crypto_unregister_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher); +} + +void aspeed_register_hace_crypto_algs(struct aspeed_hace_dev *hace_dev) +{ + int rc, i; + + CIPHER_DBG(hace_dev, "\n"); + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++) { + aspeed_crypto_algs[i].hace_dev = hace_dev; + rc = crypto_register_skcipher(&aspeed_crypto_algs[i].alg.skcipher); + if (rc) { + CIPHER_DBG(hace_dev, "Failed to register %s\n", + aspeed_crypto_algs[i].alg.skcipher.base.cra_name); + } + } + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++) { + aspeed_crypto_algs_g6[i].hace_dev = hace_dev; + rc = crypto_register_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher); + if (rc) { + CIPHER_DBG(hace_dev, "Failed to register %s\n", + aspeed_crypto_algs_g6[i].alg.skcipher.base.cra_name); + } + } +} diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c index 23a66f481b62..4fefc9e69d72 100644 --- a/drivers/crypto/aspeed/aspeed-hace.c +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -25,6 +25,7 @@ static irqreturn_t aspeed_hace_irq(int irq, void *dev) { struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)dev; + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; u32 sts; @@ -40,9 +41,24 @@ static irqreturn_t aspeed_hace_irq(int irq, void *dev) dev_warn(hace_dev->dev, "HASH no active requests.\n"); } + if (sts & HACE_CRYPTO_ISR) { + if (crypto_engine->flags & CRYPTO_FLAGS_BUSY) + tasklet_schedule(&crypto_engine->done_task); + else + dev_warn(hace_dev->dev, "CRYPTO no active requests.\n"); + } + return IRQ_HANDLED; } +static void aspeed_hace_crypto_done_task(unsigned long data) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)data; + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + + crypto_engine->resume(hace_dev); +} + static void aspeed_hace_hash_done_task(unsigned long data) { struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)data; @@ -56,6 +72,9 @@ static void aspeed_hace_register(struct aspeed_hace_dev *hace_dev) #ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH aspeed_register_hace_hash_algs(hace_dev); #endif +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO + aspeed_register_hace_crypto_algs(hace_dev); +#endif } static void aspeed_hace_unregister(struct aspeed_hace_dev *hace_dev) @@ -63,6 +82,9 @@ static void aspeed_hace_unregister(struct aspeed_hace_dev *hace_dev) #ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH aspeed_unregister_hace_hash_algs(hace_dev); #endif +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO + aspeed_unregister_hace_crypto_algs(hace_dev); +#endif } static const struct of_device_id aspeed_hace_of_matches[] = { @@ -73,6 +95,7 @@ static const struct of_device_id aspeed_hace_of_matches[] = { static int aspeed_hace_probe(struct platform_device *pdev) { + struct aspeed_engine_crypto *crypto_engine; const struct of_device_id *hace_dev_id; struct aspeed_engine_hash *hash_engine; struct aspeed_hace_dev *hace_dev; @@ -93,6 +116,7 @@ static int aspeed_hace_probe(struct platform_device *pdev) hace_dev->dev = &pdev->dev; hace_dev->version = (unsigned long)hace_dev_id->data; hash_engine = &hace_dev->hash_engine; + crypto_engine = &hace_dev->crypto_engine; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -146,6 +170,21 @@ static int aspeed_hace_probe(struct platform_device *pdev) tasklet_init(&hash_engine->done_task, aspeed_hace_hash_done_task, (unsigned long)hace_dev); + /* Initialize crypto hardware engine structure for crypto */ + hace_dev->crypt_engine_crypto = crypto_engine_alloc_init(hace_dev->dev, + true); + if (!hace_dev->crypt_engine_crypto) { + rc = -ENOMEM; + goto err_engine_hash_start; + } + + rc = crypto_engine_start(hace_dev->crypt_engine_crypto); + if (rc) + goto err_engine_crypto_start; + + tasklet_init(&crypto_engine->done_task, aspeed_hace_crypto_done_task, + (unsigned long)hace_dev); + /* Allocate DMA buffer for hash engine input used */ hash_engine->ahash_src_addr = dmam_alloc_coherent(&pdev->dev, @@ -155,7 +194,45 @@ static int aspeed_hace_probe(struct platform_device *pdev) if (!hash_engine->ahash_src_addr) { dev_err(&pdev->dev, "Failed to allocate dma buffer\n"); rc = -ENOMEM; - goto err_engine_hash_start; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine context used */ + crypto_engine->cipher_ctx = + dmam_alloc_coherent(&pdev->dev, + PAGE_SIZE, + &crypto_engine->cipher_ctx_dma, + GFP_KERNEL); + if (!crypto_engine->cipher_ctx) { + dev_err(&pdev->dev, "Failed to allocate cipher ctx dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine input used */ + crypto_engine->cipher_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_CRYPTO_SRC_DMA_BUF_LEN, + &crypto_engine->cipher_dma_addr, + GFP_KERNEL); + if (!crypto_engine->cipher_addr) { + dev_err(&pdev->dev, "Failed to allocate cipher addr dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine output used */ + if (hace_dev->version == AST2600_VERSION) { + crypto_engine->dst_sg_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_CRYPTO_DST_DMA_BUF_LEN, + &crypto_engine->dst_sg_dma_addr, + GFP_KERNEL); + if (!crypto_engine->dst_sg_addr) { + dev_err(&pdev->dev, "Failed to allocate dst_sg dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } } aspeed_hace_register(hace_dev); @@ -164,6 +241,8 @@ static int aspeed_hace_probe(struct platform_device *pdev) return 0; +err_engine_crypto_start: + crypto_engine_exit(hace_dev->crypt_engine_crypto); err_engine_hash_start: crypto_engine_exit(hace_dev->crypt_engine_hash); clk_exit: @@ -175,13 +254,16 @@ clk_exit: static int aspeed_hace_remove(struct platform_device *pdev) { struct aspeed_hace_dev *hace_dev = platform_get_drvdata(pdev); + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; aspeed_hace_unregister(hace_dev); crypto_engine_exit(hace_dev->crypt_engine_hash); + crypto_engine_exit(hace_dev->crypt_engine_crypto); tasklet_kill(&hash_engine->done_task); + tasklet_kill(&crypto_engine->done_task); clk_disable_unprepare(hace_dev->clk); diff --git a/drivers/crypto/aspeed/aspeed-hace.h b/drivers/crypto/aspeed/aspeed-hace.h index 3494ff22f69d..f2cde23b56ae 100644 --- a/drivers/crypto/aspeed/aspeed-hace.h +++ b/drivers/crypto/aspeed/aspeed-hace.h @@ -7,9 +7,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -24,15 +27,75 @@ * HACE register definitions * * * * ***************************/ +#define ASPEED_HACE_SRC 0x00 /* Crypto Data Source Base Address Register */ +#define ASPEED_HACE_DEST 0x04 /* Crypto Data Destination Base Address Register */ +#define ASPEED_HACE_CONTEXT 0x08 /* Crypto Context Buffer Base Address Register */ +#define ASPEED_HACE_DATA_LEN 0x0C /* Crypto Data Length Register */ +#define ASPEED_HACE_CMD 0x10 /* Crypto Engine Command Register */ + +/* G5 */ +#define ASPEED_HACE_TAG 0x18 /* HACE Tag Register */ +/* G6 */ +#define ASPEED_HACE_GCM_ADD_LEN 0x14 /* Crypto AES-GCM Additional Data Length Register */ +#define ASPEED_HACE_GCM_TAG_BASE_ADDR 0x18 /* Crypto AES-GCM Tag Write Buff Base Address Reg */ #define ASPEED_HACE_STS 0x1C /* HACE Status Register */ + #define ASPEED_HACE_HASH_SRC 0x20 /* Hash Data Source Base Address Register */ #define ASPEED_HACE_HASH_DIGEST_BUFF 0x24 /* Hash Digest Write Buffer Base Address Register */ #define ASPEED_HACE_HASH_KEY_BUFF 0x28 /* Hash HMAC Key Buffer Base Address Register */ #define ASPEED_HACE_HASH_DATA_LEN 0x2C /* Hash Data Length Register */ #define ASPEED_HACE_HASH_CMD 0x30 /* Hash Engine Command Register */ +/* crypto cmd */ +#define HACE_CMD_SINGLE_DES 0 +#define HACE_CMD_TRIPLE_DES BIT(17) +#define HACE_CMD_AES_SELECT 0 +#define HACE_CMD_DES_SELECT BIT(16) +#define HACE_CMD_ISR_EN BIT(12) +#define HACE_CMD_CONTEXT_SAVE_ENABLE (0) +#define HACE_CMD_CONTEXT_SAVE_DISABLE BIT(9) +#define HACE_CMD_AES (0) +#define HACE_CMD_DES (0) +#define HACE_CMD_RC4 BIT(8) +#define HACE_CMD_DECRYPT (0) +#define HACE_CMD_ENCRYPT BIT(7) + +#define HACE_CMD_ECB (0x0 << 4) +#define HACE_CMD_CBC (0x1 << 4) +#define HACE_CMD_CFB (0x2 << 4) +#define HACE_CMD_OFB (0x3 << 4) +#define HACE_CMD_CTR (0x4 << 4) +#define HACE_CMD_OP_MODE_MASK (0x7 << 4) + +#define HACE_CMD_AES128 (0x0 << 2) +#define HACE_CMD_AES192 (0x1 << 2) +#define HACE_CMD_AES256 (0x2 << 2) +#define HACE_CMD_OP_CASCADE (0x3) +#define HACE_CMD_OP_INDEPENDENT (0x1) + +/* G5 */ +#define HACE_CMD_RI_WO_DATA_ENABLE (0) +#define HACE_CMD_RI_WO_DATA_DISABLE BIT(11) +#define HACE_CMD_CONTEXT_LOAD_ENABLE (0) +#define HACE_CMD_CONTEXT_LOAD_DISABLE BIT(10) +/* G6 */ +#define HACE_CMD_AES_KEY_FROM_OTP BIT(24) +#define HACE_CMD_GHASH_TAG_XOR_EN BIT(23) +#define HACE_CMD_GHASH_PAD_LEN_INV BIT(22) +#define HACE_CMD_GCM_TAG_ADDR_SEL BIT(21) +#define HACE_CMD_MBUS_REQ_SYNC_EN BIT(20) +#define HACE_CMD_DES_SG_CTRL BIT(19) +#define HACE_CMD_SRC_SG_CTRL BIT(18) +#define HACE_CMD_CTR_IV_AES_96 (0x1 << 14) +#define HACE_CMD_CTR_IV_DES_32 (0x1 << 14) +#define HACE_CMD_CTR_IV_AES_64 (0x2 << 14) +#define HACE_CMD_CTR_IV_AES_32 (0x3 << 14) +#define HACE_CMD_AES_KEY_HW_EXP BIT(13) +#define HACE_CMD_GCM (0x5 << 4) + /* interrupt status reg */ +#define HACE_CRYPTO_ISR BIT(12) #define HACE_HASH_ISR BIT(9) #define HACE_HASH_BUSY BIT(0) @@ -77,6 +140,9 @@ #define ASPEED_HASH_SRC_DMA_BUF_LEN 0xa000 #define ASPEED_HASH_QUEUE_LENGTH 50 +#define HACE_CMD_IV_REQUIRE (HACE_CMD_CBC | HACE_CMD_CFB | \ + HACE_CMD_OFB | HACE_CMD_CTR) + struct aspeed_hace_dev; typedef int (*aspeed_hace_fn_t)(struct aspeed_hace_dev *); @@ -147,6 +213,48 @@ struct aspeed_sham_reqctx { u64 digcnt[2]; }; +struct aspeed_engine_crypto { + struct tasklet_struct done_task; + unsigned long flags; + struct skcipher_request *req; + + /* context buffer */ + void *cipher_ctx; + dma_addr_t cipher_ctx_dma; + + /* input buffer, could be single/scatter-gather lists */ + void *cipher_addr; + dma_addr_t cipher_dma_addr; + + /* output buffer, only used in scatter-gather lists */ + void *dst_sg_addr; + dma_addr_t dst_sg_dma_addr; + + /* callback func */ + aspeed_hace_fn_t resume; +}; + +struct aspeed_cipher_ctx { + struct crypto_engine_ctx enginectx; + + struct aspeed_hace_dev *hace_dev; + int key_len; + u8 key[AES_MAX_KEYLENGTH]; + + /* callback func */ + aspeed_hace_fn_t start; + + struct crypto_skcipher *fallback_tfm; +}; + +struct aspeed_cipher_reqctx { + int enc_cmd; + int src_nents; + int dst_nents; + + struct skcipher_request fallback_req; /* keep at the end */ +}; + struct aspeed_hace_dev { void __iomem *regs; struct device *dev; @@ -155,8 +263,10 @@ struct aspeed_hace_dev { unsigned long version; struct crypto_engine *crypt_engine_hash; + struct crypto_engine *crypt_engine_crypto; struct aspeed_engine_hash hash_engine; + struct aspeed_engine_crypto crypto_engine; }; struct aspeed_hace_alg { @@ -182,5 +292,7 @@ enum aspeed_version { void aspeed_register_hace_hash_algs(struct aspeed_hace_dev *hace_dev); void aspeed_unregister_hace_hash_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_register_hace_crypto_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_unregister_hace_crypto_algs(struct aspeed_hace_dev *hace_dev); #endif -- cgit From dd4f8ee7ed95e01c3e81870b7552799a06b7c7b6 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Thu, 18 Aug 2022 22:59:54 +0200 Subject: crypto: core - move from strlcpy with unused retval to strscpy Follow the advice of the below link and prefer 'strscpy' in this subsystem. Conversion is 1:1 because the return value is not used. Generated by a coccinelle script. Link: https://lore.kernel.org/r/CAHk-=wgfRnXz0W3D37d01q3JFkr_i_uTL=V6A6G1oUZcprmknw@mail.gmail.com/ Signed-off-by: Wolfram Sang Signed-off-by: Herbert Xu --- crypto/api.c | 2 +- crypto/essiv.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/api.c b/crypto/api.c index ab4b5e2b0756..64f2d365a8e9 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -114,7 +114,7 @@ struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask) larval->alg.cra_priority = -1; larval->alg.cra_destroy = crypto_larval_destroy; - strlcpy(larval->alg.cra_name, name, CRYPTO_MAX_ALG_NAME); + strscpy(larval->alg.cra_name, name, CRYPTO_MAX_ALG_NAME); init_completion(&larval->completion); return larval; diff --git a/crypto/essiv.c b/crypto/essiv.c index 8bcc5bdcb2a9..e33369df9034 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -543,7 +543,7 @@ static int essiv_create(struct crypto_template *tmpl, struct rtattr **tb) } /* record the driver name so we can instantiate this exact algo later */ - strlcpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name, + strscpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME); /* Instance fields */ -- cgit From 28855860057ac0d51607a0b1386cb6c2cd1ad0d4 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Thu, 18 Aug 2022 23:00:03 +0200 Subject: crypto: drivers - move from strlcpy with unused retval to strscpy Follow the advice of the below link and prefer 'strscpy' in this subsystem. Conversion is 1:1 because the return value is not used. Generated by a coccinelle script. Link: https://lore.kernel.org/r/CAHk-=wgfRnXz0W3D37d01q3JFkr_i_uTL=V6A6G1oUZcprmknw@mail.gmail.com/ Signed-off-by: Wolfram Sang Acked-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c | 6 +++--- drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c | 4 ++-- drivers/crypto/qat/qat_common/adf_cfg.c | 6 +++--- drivers/crypto/qat/qat_common/adf_ctl_drv.c | 2 +- drivers/crypto/qat/qat_common/adf_transport_debug.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index 40b482198ebc..23c6edc70914 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -97,7 +97,7 @@ static int dev_supports_eng_type(struct otx_cpt_eng_grps *eng_grps, static void set_ucode_filename(struct otx_cpt_ucode *ucode, const char *filename) { - strlcpy(ucode->filename, filename, OTX_CPT_UCODE_NAME_LENGTH); + strscpy(ucode->filename, filename, OTX_CPT_UCODE_NAME_LENGTH); } static char *get_eng_type_str(int eng_type) @@ -138,7 +138,7 @@ static int get_ucode_type(struct otx_cpt_ucode_hdr *ucode_hdr, int *ucode_type) u32 i, val = 0; u8 nn; - strlcpy(tmp_ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ); + strscpy(tmp_ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ); for (i = 0; i < strlen(tmp_ver_str); i++) tmp_ver_str[i] = tolower(tmp_ver_str[i]); @@ -1328,7 +1328,7 @@ static ssize_t ucode_load_store(struct device *dev, eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr); err_msg = "Invalid engine group format"; - strlcpy(tmp_buf, buf, OTX_CPT_UCODE_NAME_LENGTH); + strscpy(tmp_buf, buf, OTX_CPT_UCODE_NAME_LENGTH); start = tmp_buf; has_se = has_ie = has_ae = false; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index f10050fead16..1577986677f6 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -68,7 +68,7 @@ static int is_2nd_ucode_used(struct otx2_cpt_eng_grp_info *eng_grp) static void set_ucode_filename(struct otx2_cpt_ucode *ucode, const char *filename) { - strlcpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH); + strscpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH); } static char *get_eng_type_str(int eng_type) @@ -126,7 +126,7 @@ static int get_ucode_type(struct device *dev, int i, val = 0; u8 nn; - strlcpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ); + strscpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ); for (i = 0; i < strlen(tmp_ver_str); i++) tmp_ver_str[i] = tolower(tmp_ver_str[i]); diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index e61b3e13db3b..1931e5b37f2b 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -251,13 +251,13 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, return -ENOMEM; INIT_LIST_HEAD(&key_val->list); - strlcpy(key_val->key, key, sizeof(key_val->key)); + strscpy(key_val->key, key, sizeof(key_val->key)); if (type == ADF_DEC) { snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, "%ld", (*((long *)val))); } else if (type == ADF_STR) { - strlcpy(key_val->val, (char *)val, sizeof(key_val->val)); + strscpy(key_val->val, (char *)val, sizeof(key_val->val)); } else if (type == ADF_HEX) { snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, "0x%lx", (unsigned long)val); @@ -315,7 +315,7 @@ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) if (!sec) return -ENOMEM; - strlcpy(sec->name, name, sizeof(sec->name)); + strscpy(sec->name, name, sizeof(sec->name)); INIT_LIST_HEAD(&sec->param_head); down_write(&cfg->lock); list_add_tail(&sec->list, &cfg->sec_list); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index e8ac932bbaab..508c18edd692 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -363,7 +363,7 @@ static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, dev_info.num_logical_accel = hw_data->num_logical_accel; dev_info.banks_per_accel = hw_data->num_banks / hw_data->num_logical_accel; - strlcpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); + strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); dev_info.instance_id = hw_data->instance_id; dev_info.type = hw_data->dev_class->type; dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number; diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index e69e5907f595..08bca1c506c0 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -96,7 +96,7 @@ int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) if (!ring_debug) return -ENOMEM; - strlcpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); + strscpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); snprintf(entry_name, sizeof(entry_name), "ring_%02d", ring->ring_number); -- cgit From 545665ad1e84eb8f047018a2f607e78cef29c7fa Mon Sep 17 00:00:00 2001 From: Jack Wang Date: Fri, 19 Aug 2022 08:07:49 +0200 Subject: crypto: gemini - Fix error check for dma_map_sg dma_map_sg return 0 on error. Cc: Corentin Labbe Cc: Hans Ulli Kroll Cc: Linus Walleij Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Jack Wang Reviewed-by: Linus Walleij Signed-off-by: Herbert Xu --- drivers/crypto/gemini/sl3516-ce-cipher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c index 14d0d83d388d..34fea8aa91b6 100644 --- a/drivers/crypto/gemini/sl3516-ce-cipher.c +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -149,7 +149,7 @@ static int sl3516_ce_cipher(struct skcipher_request *areq) if (areq->src == areq->dst) { nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL); - if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { + if (!nr_sgs || nr_sgs > MAXDESC / 2) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; @@ -158,14 +158,14 @@ static int sl3516_ce_cipher(struct skcipher_request *areq) } else { nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { + if (!nr_sgs || nr_sgs > MAXDESC / 2) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; } nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE); - if (nr_sgd <= 0 || nr_sgd > MAXDESC) { + if (!nr_sgd || nr_sgd > MAXDESC) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); err = -EINVAL; goto theend_sgs; -- cgit From 66f0b6b7d839b49e19a856a90b70656dccc4b844 Mon Sep 17 00:00:00 2001 From: Jack Wang Date: Fri, 19 Aug 2022 08:07:50 +0200 Subject: crypto: sahara - Fix error check for dma_map_sg dma_map_sg return 0 on error, it returns the number of DMA address segments mapped (this may be shorter than passed in if some elements of the scatter/gather list are physically or virtually adjacent and an IOMMU maps them with a single entry). Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Jack Wang Signed-off-by: Herbert Xu --- drivers/crypto/sahara.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index b07ae4ba165e..7ab20fb95166 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -487,13 +487,13 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); - if (ret != dev->nb_in_sg) { + if (!ret) { dev_err(dev->device, "couldn't map in sg\n"); goto unmap_in; } ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, DMA_FROM_DEVICE); - if (ret != dev->nb_out_sg) { + if (!ret) { dev_err(dev->device, "couldn't map out sg\n"); goto unmap_out; } -- cgit From 417f62f6402c985dde661c6b09734fddb33d9b1a Mon Sep 17 00:00:00 2001 From: Jack Wang Date: Fri, 19 Aug 2022 08:07:51 +0200 Subject: crypto: qce - Fix dma_map_sg error check dma_map_sg return 0 on error, fix the error check and return -EIO to caller. Cc: Thara Gopinath Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linux-arm-msm@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Jack Wang Signed-off-by: Herbert Xu --- drivers/crypto/qce/aead.c | 4 ++-- drivers/crypto/qce/sha.c | 8 +++++--- drivers/crypto/qce/skcipher.c | 8 ++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c index 97a530171f07..6eb4d2e35629 100644 --- a/drivers/crypto/qce/aead.c +++ b/drivers/crypto/qce/aead.c @@ -450,8 +450,8 @@ qce_aead_async_req_handle(struct crypto_async_request *async_req) if (ret) return ret; dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); - if (dst_nents < 0) { - ret = dst_nents; + if (!dst_nents) { + ret = -EIO; goto error_free; } diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 59159f5e64e5..37bafd7aeb79 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -97,14 +97,16 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) } ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); - if (ret < 0) - return ret; + if (!ret) + return -EIO; sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); ret = dma_map_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); - if (ret < 0) + if (!ret) { + ret = -EIO; goto error_unmap_src; + } ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents, &rctx->result_sg, 1, qce_ahash_done, async_req); diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index 3d27cd5210ef..5b493fdc1e74 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -124,15 +124,15 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req) rctx->dst_sg = rctx->dst_tbl.sgl; dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); - if (dst_nents < 0) { - ret = dst_nents; + if (!dst_nents) { + ret = -EIO; goto error_free; } if (diff_dst) { src_nents = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); - if (src_nents < 0) { - ret = src_nents; + if (!src_nents) { + ret = -EIO; goto error_unmap_dst; } rctx->src_sg = req->src; -- cgit From 45fa321e7de604146603e24ee5bf3c0b766efe46 Mon Sep 17 00:00:00 2001 From: Jack Wang Date: Fri, 19 Aug 2022 08:07:52 +0200 Subject: crypto: amlogic - Fix dma_map_sg error check dma_map_sg return 0 on error. Cc: Corentin Labbe Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linux-amlogic@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Jack Wang Signed-off-by: Herbert Xu --- drivers/crypto/amlogic/amlogic-gxl-cipher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c index e79514fce731..af017a087ebf 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-cipher.c +++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c @@ -177,7 +177,7 @@ static int meson_cipher(struct skcipher_request *areq) if (areq->src == areq->dst) { nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL); - if (nr_sgs < 0) { + if (!nr_sgs) { dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs); err = -EINVAL; goto theend; @@ -186,14 +186,14 @@ static int meson_cipher(struct skcipher_request *areq) } else { nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); - if (nr_sgs < 0 || nr_sgs > MAXDESC - 3) { + if (!nr_sgs || nr_sgs > MAXDESC - 3) { dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs); err = -EINVAL; goto theend; } nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE); - if (nr_sgd < 0 || nr_sgd > MAXDESC - 3) { + if (!nr_sgd || nr_sgd > MAXDESC - 3) { dev_err(mc->dev, "Invalid SG count %d\n", nr_sgd); err = -EINVAL; goto theend; -- cgit From 2b02187bdb0bb75000850bd0309e70eb8664159e Mon Sep 17 00:00:00 2001 From: Jack Wang Date: Fri, 19 Aug 2022 08:07:53 +0200 Subject: crypto: allwinner - Fix dma_map_sg error check dma_map_sg return 0 on error. Cc: Corentin Labbe Cc: Herbert Xu Cc: "David S. Miller" Cc: Chen-Yu Tsai Cc: Jernej Skrabec Cc: Samuel Holland Cc: Dan Carpenter Cc: Minghao Chi Cc: Peng Wu Cc: Alexey Khoroshilov Cc: linux-crypto@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-sunxi@lists.linux.dev Cc: linux-kernel@vger.kernel.org Signed-off-by: Jack Wang Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c | 6 +++--- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 2 +- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c | 4 ++-- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index 74b4e910a38d..be7f46faef7e 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -208,7 +208,7 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req if (areq->src == areq->dst) { nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL); - if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + if (!nr_sgs || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend_iv; @@ -216,13 +216,13 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req nr_sgd = nr_sgs; } else { nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + if (!nr_sgs || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend_iv; } nr_sgd = dma_map_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); - if (nr_sgd <= 0 || nr_sgd > MAX_SG) { + if (!nr_sgd || nr_sgd > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); err = -EINVAL; goto theend_sgs; diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 8b5b9b9d04c3..0e6843ec197f 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -389,7 +389,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) cet->t_asym_ctl = 0; nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + if (!nr_sgs || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 910d6751644c..fdcc98cdecaa 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -232,13 +232,13 @@ static int sun8i_ss_cipher(struct skcipher_request *areq) nr_sgd = nr_sgs; } else { nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 8) { + if (!nr_sgs || nr_sgs > 8) { dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend_iv; } nr_sgd = dma_map_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE); - if (nr_sgd <= 0 || nr_sgd > 8) { + if (!nr_sgd || nr_sgd > 8) { dev_err(ss->dev, "Invalid sg number %d\n", nr_sgd); err = -EINVAL; goto theend_sgs; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 36a82b22953c..fcb8c41cc957 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -527,7 +527,7 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) rctx->method = ss->variant->alg_hash[algt->ss_algo_id]; nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + if (!nr_sgs || nr_sgs > MAX_SG) { dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; -- cgit From 9b32fed8d6715207a0a2cd42f48a147d203a7576 Mon Sep 17 00:00:00 2001 From: Jack Wang Date: Fri, 19 Aug 2022 08:07:54 +0200 Subject: crypto: ccree - Fix dma_map_sg error check dma_map_sg return 0 on error, and dma_map_error is not supposed to use here. Cc: Gilad Ben-Yossef Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Jack Wang Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_buffer_mgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index 6140e4927322..9efd88f871d1 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -274,7 +274,7 @@ static int cc_map_sg(struct device *dev, struct scatterlist *sg, } ret = dma_map_sg(dev, sg, *nents, direction); - if (dma_mapping_error(dev, ret)) { + if (!ret) { *nents = 0; dev_err(dev, "dma_map_sg() sg buffer failed %d\n", ret); return -ENOMEM; -- cgit From d03e89b3eba46121e8cbf2753b02be407810991b Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 19 Aug 2022 07:46:18 +0000 Subject: crypto: hisilicon/qm - no judgment in the back process Judgment should not be added in the back process. So clean it. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index b2e7abff1b8a..c180ad33d2f8 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1357,11 +1357,9 @@ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, return 0; back_sqc_cqc: - for (i = SQC_VFT; i <= CQC_VFT; i++) { - ret = qm_set_vft_common(qm, i, fun_num, 0, 0); - if (ret) - return ret; - } + for (i = SQC_VFT; i <= CQC_VFT; i++) + qm_set_vft_common(qm, i, fun_num, 0, 0); + return ret; } -- cgit From e45f710b42afd7e67276234853d2de19faf46362 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:35 -0500 Subject: crypto: Kconfig - move mips entries to a submenu Move CPU-specific crypto/Kconfig entries to arch/xxx/crypto/Kconfig and create a submenu for them under the Crypto API menu. Suggested-by: Eric Biggers Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/mips/crypto/Kconfig | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 59 ++++------------------------------------------- 2 files changed, 64 insertions(+), 55 deletions(-) create mode 100644 arch/mips/crypto/Kconfig diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig new file mode 100644 index 000000000000..7c07611e2322 --- /dev/null +++ b/arch/mips/crypto/Kconfig @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (mips)" + +config CRYPTO_CRC32_MIPS + tristate "CRC32c and CRC32 CRC algorithm (MIPS)" + depends on MIPS_CRC_SUPPORT + select CRYPTO_HASH + help + CRC32c and CRC32 CRC algorithms implemented using mips crypto + instructions, when available. + +config CRYPTO_POLY1305_MIPS + tristate "Poly1305 authenticator algorithm (MIPS optimized)" + depends on MIPS + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + +config CRYPTO_MD5_OCTEON + tristate "MD5 digest algorithm (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_MD5 + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) implemented + using OCTEON crypto instructions, when available. + +config CRYPTO_SHA1_OCTEON + tristate "SHA1 digest algorithm (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using OCTEON crypto instructions, when available. + +config CRYPTO_SHA256_OCTEON + tristate "SHA224 and SHA256 digest algorithm (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-256 secure hash standard (DFIPS 180-2) implemented + using OCTEON crypto instructions, when available. + +config CRYPTO_SHA512_OCTEON + tristate "SHA384 and SHA512 digest algorithms (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using OCTEON crypto instructions, when available. + +config CRYPTO_CHACHA_MIPS + tristate "ChaCha stream cipher algorithms (MIPS 32r2 optimized)" + depends on CPU_MIPS32_R2 + select CRYPTO_SKCIPHER + select CRYPTO_ARCH_HAVE_LIB_CHACHA + +endmenu diff --git a/crypto/Kconfig b/crypto/Kconfig index b1ccf873779d..85fa86f334f2 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -699,14 +699,6 @@ config CRYPTO_CRC32_PCLMUL which will enable any routine to use the CRC-32-IEEE 802.3 checksum and gain better performance as compared with the table implementation. -config CRYPTO_CRC32_MIPS - tristate "CRC32c and CRC32 CRC algorithm (MIPS)" - depends on MIPS_CRC_SUPPORT - select CRYPTO_HASH - help - CRC32c and CRC32 CRC algorithms implemented using mips crypto - instructions, when available. - config CRYPTO_CRC32_S390 tristate "CRC-32 algorithms" depends on S390 @@ -841,11 +833,6 @@ config CRYPTO_POLY1305_X86_64 in IETF protocols. This is the x86_64 assembler implementation using SIMD instructions. -config CRYPTO_POLY1305_MIPS - tristate "Poly1305 authenticator algorithm (MIPS optimized)" - depends on MIPS - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - config CRYPTO_MD4 tristate "MD4 digest algorithm" select CRYPTO_HASH @@ -858,15 +845,6 @@ config CRYPTO_MD5 help MD5 message digest algorithm (RFC1321). -config CRYPTO_MD5_OCTEON - tristate "MD5 digest algorithm (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_MD5 - select CRYPTO_HASH - help - MD5 message digest algorithm (RFC1321) implemented - using OCTEON crypto instructions, when available. - config CRYPTO_MD5_PPC tristate "MD5 digest algorithm (PPC)" depends on PPC @@ -961,15 +939,6 @@ config CRYPTO_SHA512_S390 It is available as of z10. -config CRYPTO_SHA1_OCTEON - tristate "SHA1 digest algorithm (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. - config CRYPTO_SHA1_SPARC64 tristate "SHA1 digest algorithm (SPARC64)" depends on SPARC64 @@ -1025,15 +994,6 @@ config CRYPTO_SHA256_PPC_SPE SHA224 and SHA256 secure hash standard (DFIPS 180-2) implemented using powerpc SPE SIMD instruction set. -config CRYPTO_SHA256_OCTEON - tristate "SHA224 and SHA256 digest algorithm (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. - config CRYPTO_SHA256_SPARC64 tristate "SHA224 and SHA256 digest algorithm (SPARC64)" depends on SPARC64 @@ -1065,15 +1025,6 @@ config CRYPTO_SHA512 This code also includes SHA-384, a 384 bit hash with 192 bits of security against collision attacks. -config CRYPTO_SHA512_OCTEON - tristate "SHA384 and SHA512 digest algorithms (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. - config CRYPTO_SHA512_SPARC64 tristate "SHA384 and SHA512 digest algorithm (SPARC64)" depends on SPARC64 @@ -1611,12 +1562,6 @@ config CRYPTO_CHACHA20_X86_64 SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20, XChaCha20, and XChaCha12 stream ciphers. -config CRYPTO_CHACHA_MIPS - tristate "ChaCha stream cipher algorithms (MIPS 32r2 optimized)" - depends on CPU_MIPS32_R2 - select CRYPTO_SKCIPHER - select CRYPTO_ARCH_HAVE_LIB_CHACHA - config CRYPTO_CHACHA_S390 tristate "ChaCha20 stream cipher" depends on S390 @@ -2128,6 +2073,10 @@ config CRYPTO_STATS config CRYPTO_HASH_INFO bool +if MIPS +source "arch/mips/crypto/Kconfig" +endif + source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" source "certs/Kconfig" -- cgit From 6a490a4e8b4c015113045d045dc1ae94735211bb Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:36 -0500 Subject: crypto: Kconfig - move powerpc entries to a submenu Move CPU-specific crypto/Kconfig entries to arch/xxx/crypto/Kconfig and create a submenu for them under the Crypto API menu. Suggested-by: Eric Biggers Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Kconfig | 77 +++++++++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 76 ++------------------------------------------ 2 files changed, 80 insertions(+), 73 deletions(-) create mode 100644 arch/powerpc/crypto/Kconfig diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig new file mode 100644 index 000000000000..74f535940faa --- /dev/null +++ b/arch/powerpc/crypto/Kconfig @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (powerpc)" + +config CRYPTO_CRC32C_VPMSUM + tristate "CRC32c CRC algorithm (powerpc64)" + depends on PPC64 && ALTIVEC + select CRYPTO_HASH + select CRC32 + help + CRC32c algorithm implemented using vector polynomial multiply-sum + (vpmsum) instructions, introduced in POWER8. Enable on POWER8 + and newer processors for improved performance. + +config CRYPTO_CRCT10DIF_VPMSUM + tristate "CRC32T10DIF powerpc64 hardware acceleration" + depends on PPC64 && ALTIVEC && CRC_T10DIF + select CRYPTO_HASH + help + CRC10T10DIF algorithm implemented using vector polynomial + multiply-sum (vpmsum) instructions, introduced in POWER8. Enable on + POWER8 and newer processors for improved performance. + +config CRYPTO_VPMSUM_TESTER + tristate "Powerpc64 vpmsum hardware acceleration tester" + depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM + help + Stress test for CRC32c and CRC-T10DIF algorithms implemented with + POWER8 vpmsum instructions. + Unless you are testing these algorithms, you don't need this. + +config CRYPTO_MD5_PPC + tristate "MD5 digest algorithm (PPC)" + depends on PPC + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) implemented + in PPC assembler. + +config CRYPTO_SHA1_PPC + tristate "SHA1 digest algorithm (powerpc)" + depends on PPC + help + This is the powerpc hardware accelerated implementation of the + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). + +config CRYPTO_SHA1_PPC_SPE + tristate "SHA1 digest algorithm (PPC SPE)" + depends on PPC && SPE + help + SHA-1 secure hash standard (DFIPS 180-4) implemented + using powerpc SPE SIMD instruction set. + +config CRYPTO_SHA256_PPC_SPE + tristate "SHA224 and SHA256 digest algorithm (PPC SPE)" + depends on PPC && SPE + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA224 and SHA256 secure hash standard (DFIPS 180-2) + implemented using powerpc SPE SIMD instruction set. + +config CRYPTO_AES_PPC_SPE + tristate "AES cipher algorithms (PPC SPE)" + depends on PPC && SPE + select CRYPTO_SKCIPHER + help + AES cipher algorithms (FIPS-197). Additionally the acceleration + for popular block cipher modes ECB, CBC, CTR and XTS is supported. + This module should only be used for low power (router) devices + without hardware AES acceleration (e.g. caam crypto). It reduces the + size of the AES tables from 16KB to 8KB + 256 bytes and mitigates + timining attacks. Nevertheless it might be not as secure as other + architecture specific assembler implementations that work on 1KB + tables or 256 bytes S-boxes. + +endmenu diff --git a/crypto/Kconfig b/crypto/Kconfig index 85fa86f334f2..5299929144b6 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -658,17 +658,6 @@ config CRYPTO_CRC32C_INTEL gain performance compared with software implementation. Module will be crc32c-intel. -config CRYPTO_CRC32C_VPMSUM - tristate "CRC32c CRC algorithm (powerpc64)" - depends on PPC64 && ALTIVEC - select CRYPTO_HASH - select CRC32 - help - CRC32c algorithm implemented using vector polynomial multiply-sum - (vpmsum) instructions, introduced in POWER8. Enable on POWER8 - and newer processors for improved performance. - - config CRYPTO_CRC32C_SPARC64 tristate "CRC32c CRC algorithm (SPARC64)" depends on SPARC64 @@ -762,28 +751,11 @@ config CRYPTO_CRCT10DIF_PCLMUL 'crct10dif-pclmul' module, which is faster when computing the crct10dif checksum as compared with the generic table implementation. -config CRYPTO_CRCT10DIF_VPMSUM - tristate "CRC32T10DIF powerpc64 hardware acceleration" - depends on PPC64 && ALTIVEC && CRC_T10DIF - select CRYPTO_HASH - help - CRC10T10DIF algorithm implemented using vector polynomial - multiply-sum (vpmsum) instructions, introduced in POWER8. Enable on - POWER8 and newer processors for improved performance. - config CRYPTO_CRC64_ROCKSOFT tristate "Rocksoft Model CRC64 algorithm" depends on CRC64 select CRYPTO_HASH -config CRYPTO_VPMSUM_TESTER - tristate "Powerpc64 vpmsum hardware acceleration tester" - depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM - help - Stress test for CRC32c and CRC-T10DIF algorithms implemented with - POWER8 vpmsum instructions. - Unless you are testing these algorithms, you don't need this. - config CRYPTO_GHASH tristate "GHASH hash function" select CRYPTO_GF128MUL @@ -845,14 +817,6 @@ config CRYPTO_MD5 help MD5 message digest algorithm (RFC1321). -config CRYPTO_MD5_PPC - tristate "MD5 digest algorithm (PPC)" - depends on PPC - select CRYPTO_HASH - help - MD5 message digest algorithm (RFC1321) implemented - in PPC assembler. - config CRYPTO_MD5_SPARC64 tristate "MD5 digest algorithm (SPARC64)" depends on SPARC64 @@ -948,20 +912,6 @@ config CRYPTO_SHA1_SPARC64 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented using sparc64 crypto instructions, when available. -config CRYPTO_SHA1_PPC - tristate "SHA1 digest algorithm (powerpc)" - depends on PPC - help - This is the powerpc hardware accelerated implementation of the - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). - -config CRYPTO_SHA1_PPC_SPE - tristate "SHA1 digest algorithm (PPC SPE)" - depends on PPC && SPE - help - SHA-1 secure hash standard (DFIPS 180-4) implemented - using powerpc SPE SIMD instruction set. - config CRYPTO_SHA1_S390 tristate "SHA1 digest algorithm" depends on S390 @@ -985,15 +935,6 @@ config CRYPTO_SHA256 This code also includes SHA-224, a 224 bit hash with 112 bits of security against collision attacks. -config CRYPTO_SHA256_PPC_SPE - tristate "SHA224 and SHA256 digest algorithm (PPC SPE)" - depends on PPC && SPE - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA224 and SHA256 secure hash standard (DFIPS 180-2) - implemented using powerpc SPE SIMD instruction set. - config CRYPTO_SHA256_SPARC64 tristate "SHA224 and SHA256 digest algorithm (SPARC64)" depends on SPARC64 @@ -1235,20 +1176,6 @@ config CRYPTO_AES_SPARC64 for some popular block cipher mode is supported too, including ECB and CBC. -config CRYPTO_AES_PPC_SPE - tristate "AES cipher algorithms (PPC SPE)" - depends on PPC && SPE - select CRYPTO_SKCIPHER - help - AES cipher algorithms (FIPS-197). Additionally the acceleration - for popular block cipher modes ECB, CBC, CTR and XTS is supported. - This module should only be used for low power (router) devices - without hardware AES acceleration (e.g. caam crypto). It reduces the - size of the AES tables from 16KB to 8KB + 256 bytes and mitigates - timining attacks. Nevertheless it might be not as secure as other - architecture specific assembler implementations that work on 1KB - tables or 256 bytes S-boxes. - config CRYPTO_AES_S390 tristate "AES cipher algorithms" depends on S390 @@ -2076,6 +2003,9 @@ config CRYPTO_HASH_INFO if MIPS source "arch/mips/crypto/Kconfig" endif +if PPC +source "arch/powerpc/crypto/Kconfig" +endif source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" -- cgit From c9d24c97c89c1ba9b2d4ff8b721443d7471f87b6 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:37 -0500 Subject: crypto: Kconfig - move s390 entries to a submenu Move CPU-specific crypto/Kconfig entries to arch/xxx/crypto/Kconfig and create a submenu for them under the Crypto API menu. Suggested-by: Eric Biggers Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/s390/crypto/Kconfig | 120 +++++++++++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 118 ++-------------------------------------------- 2 files changed, 123 insertions(+), 115 deletions(-) create mode 100644 arch/s390/crypto/Kconfig diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig new file mode 100644 index 000000000000..ef0651d71e9d --- /dev/null +++ b/arch/s390/crypto/Kconfig @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (s390)" + +config CRYPTO_CRC32_S390 + tristate "CRC-32 algorithms" + depends on S390 + select CRYPTO_HASH + select CRC32 + help + Select this option if you want to use hardware accelerated + implementations of CRC algorithms. With this option, you + can optimize the computation of CRC-32 (IEEE 802.3 Ethernet) + and CRC-32C (Castagnoli). + + It is available with IBM z13 or later. + +config CRYPTO_SHA512_S390 + tristate "SHA384 and SHA512 digest algorithm" + depends on S390 + select CRYPTO_HASH + help + This is the s390 hardware accelerated implementation of the + SHA512 secure hash standard. + + It is available as of z10. + +config CRYPTO_SHA1_S390 + tristate "SHA1 digest algorithm" + depends on S390 + select CRYPTO_HASH + help + This is the s390 hardware accelerated implementation of the + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). + + It is available as of z990. + +config CRYPTO_SHA256_S390 + tristate "SHA256 digest algorithm" + depends on S390 + select CRYPTO_HASH + help + This is the s390 hardware accelerated implementation of the + SHA256 secure hash standard (DFIPS 180-2). + + It is available as of z9. + +config CRYPTO_SHA3_256_S390 + tristate "SHA3_224 and SHA3_256 digest algorithm" + depends on S390 + select CRYPTO_HASH + help + This is the s390 hardware accelerated implementation of the + SHA3_256 secure hash standard. + + It is available as of z14. + +config CRYPTO_SHA3_512_S390 + tristate "SHA3_384 and SHA3_512 digest algorithm" + depends on S390 + select CRYPTO_HASH + help + This is the s390 hardware accelerated implementation of the + SHA3_512 secure hash standard. + + It is available as of z14. + +config CRYPTO_GHASH_S390 + tristate "GHASH hash function" + depends on S390 + select CRYPTO_HASH + help + This is the s390 hardware accelerated implementation of GHASH, + the hash function used in GCM (Galois/Counter mode). + + It is available as of z196. + +config CRYPTO_AES_S390 + tristate "AES cipher algorithms" + depends on S390 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + help + This is the s390 hardware accelerated implementation of the + AES cipher algorithms (FIPS-197). + + As of z9 the ECB and CBC modes are hardware accelerated + for 128 bit keys. + As of z10 the ECB and CBC modes are hardware accelerated + for all AES key sizes. + As of z196 the CTR mode is hardware accelerated for all AES + key sizes and XTS mode is hardware accelerated for 256 and + 512 bit keys. + +config CRYPTO_DES_S390 + tristate "DES and Triple DES cipher algorithms" + depends on S390 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + help + This is the s390 hardware accelerated implementation of the + DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). + + As of z990 the ECB and CBC mode are hardware accelerated. + As of z196 the CTR mode is hardware accelerated. + +config CRYPTO_CHACHA_S390 + tristate "ChaCha20 stream cipher" + depends on S390 + select CRYPTO_SKCIPHER + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + This is the s390 SIMD implementation of the ChaCha20 stream + cipher (RFC 7539). + + It is available as of z13. + +endmenu diff --git a/crypto/Kconfig b/crypto/Kconfig index 5299929144b6..0eb6090e7562 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -688,19 +688,6 @@ config CRYPTO_CRC32_PCLMUL which will enable any routine to use the CRC-32-IEEE 802.3 checksum and gain better performance as compared with the table implementation. -config CRYPTO_CRC32_S390 - tristate "CRC-32 algorithms" - depends on S390 - select CRYPTO_HASH - select CRC32 - help - Select this option if you want to use hardware accelerated - implementations of CRC algorithms. With this option, you - can optimize the computation of CRC-32 (IEEE 802.3 Ethernet) - and CRC-32C (Castagnoli). - - It is available with IBM z13 or later. - config CRYPTO_XXHASH tristate "xxHash hash algorithm" select CRYPTO_HASH @@ -893,16 +880,6 @@ config CRYPTO_SHA512_SSSE3 Extensions version 1 (AVX1), or Advanced Vector Extensions version 2 (AVX2) instructions, when available. -config CRYPTO_SHA512_S390 - tristate "SHA384 and SHA512 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA512 secure hash standard. - - It is available as of z10. - config CRYPTO_SHA1_SPARC64 tristate "SHA1 digest algorithm (SPARC64)" depends on SPARC64 @@ -912,16 +889,6 @@ config CRYPTO_SHA1_SPARC64 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented using sparc64 crypto instructions, when available. -config CRYPTO_SHA1_S390 - tristate "SHA1 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). - - It is available as of z990. - config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH @@ -944,16 +911,6 @@ config CRYPTO_SHA256_SPARC64 SHA-256 secure hash standard (DFIPS 180-2) implemented using sparc64 crypto instructions, when available. -config CRYPTO_SHA256_S390 - tristate "SHA256 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA256 secure hash standard (DFIPS 180-2). - - It is available as of z9. - config CRYPTO_SHA512 tristate "SHA384 and SHA512 digest algorithms" select CRYPTO_HASH @@ -985,26 +942,6 @@ config CRYPTO_SHA3 References: http://keccak.noekeon.org/ -config CRYPTO_SHA3_256_S390 - tristate "SHA3_224 and SHA3_256 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA3_256 secure hash standard. - - It is available as of z14. - -config CRYPTO_SHA3_512_S390 - tristate "SHA3_384 and SHA3_512 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA3_512 secure hash standard. - - It is available as of z14. - config CRYPTO_SM3 tristate @@ -1065,16 +1002,6 @@ config CRYPTO_GHASH_CLMUL_NI_INTEL This is the x86_64 CLMUL-NI accelerated implementation of GHASH, the hash function used in GCM (Galois/Counter mode). -config CRYPTO_GHASH_S390 - tristate "GHASH hash function" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of GHASH, - the hash function used in GCM (Galois/Counter mode). - - It is available as of z196. - comment "Ciphers" config CRYPTO_AES @@ -1176,23 +1103,6 @@ config CRYPTO_AES_SPARC64 for some popular block cipher mode is supported too, including ECB and CBC. -config CRYPTO_AES_S390 - tristate "AES cipher algorithms" - depends on S390 - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER - help - This is the s390 hardware accelerated implementation of the - AES cipher algorithms (FIPS-197). - - As of z9 the ECB and CBC modes are hardware accelerated - for 128 bit keys. - As of z10 the ECB and CBC modes are hardware accelerated - for all AES key sizes. - As of z196 the CTR mode is hardware accelerated for all AES - key sizes and XTS mode is hardware accelerated for 256 and - 512 bit keys. - config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" depends on CRYPTO_USER_API_ENABLE_OBSOLETE @@ -1423,19 +1333,6 @@ config CRYPTO_DES3_EDE_X86_64 algorithm are provided; regular processing one input block and one that processes three blocks parallel. -config CRYPTO_DES_S390 - tristate "DES and Triple DES cipher algorithms" - depends on S390 - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER - select CRYPTO_LIB_DES - help - This is the s390 hardware accelerated implementation of the - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). - - As of z990 the ECB and CBC mode are hardware accelerated. - As of z196 the CTR mode is hardware accelerated. - config CRYPTO_FCRYPT tristate "FCrypt cipher algorithm" select CRYPTO_ALGAPI @@ -1489,18 +1386,6 @@ config CRYPTO_CHACHA20_X86_64 SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20, XChaCha20, and XChaCha12 stream ciphers. -config CRYPTO_CHACHA_S390 - tristate "ChaCha20 stream cipher" - depends on S390 - select CRYPTO_SKCIPHER - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - help - This is the s390 SIMD implementation of the ChaCha20 stream - cipher (RFC 7539). - - It is available as of z13. - config CRYPTO_SEED tristate "SEED cipher algorithm" depends on CRYPTO_USER_API_ENABLE_OBSOLETE @@ -2006,6 +1891,9 @@ endif if PPC source "arch/powerpc/crypto/Kconfig" endif +if S390 +source "arch/s390/crypto/Kconfig" +endif source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" -- cgit From 0e9f9ea6e21f7e0b2a25abf01140315e36e95d1d Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:38 -0500 Subject: crypto: Kconfig - move sparc entries to a submenu Move CPU-specific crypto/Kconfig entries to arch/xxx/crypto/Kconfig and create a submenu for them under the Crypto API menu. Suggested-by: Eric Biggers Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/sparc/crypto/Kconfig | 103 ++++++++++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 101 ++------------------------------------------- 2 files changed, 106 insertions(+), 98 deletions(-) create mode 100644 arch/sparc/crypto/Kconfig diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig new file mode 100644 index 000000000000..eaa2afc1d50a --- /dev/null +++ b/arch/sparc/crypto/Kconfig @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (sparc64)" + +config CRYPTO_DES_SPARC64 + tristate "DES and Triple DES EDE cipher algorithms (SPARC64)" + depends on SPARC64 + select CRYPTO_ALGAPI + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + help + DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3), + optimized using SPARC64 crypto opcodes. + +config CRYPTO_CRC32C_SPARC64 + tristate "CRC32c CRC algorithm (SPARC64)" + depends on SPARC64 + select CRYPTO_HASH + select CRC32 + help + CRC32c CRC algorithm implemented using sparc64 crypto instructions, + when available. + +config CRYPTO_MD5_SPARC64 + tristate "MD5 digest algorithm (SPARC64)" + depends on SPARC64 + select CRYPTO_MD5 + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) implemented + using sparc64 crypto instructions, when available. + +config CRYPTO_SHA1_SPARC64 + tristate "SHA1 digest algorithm (SPARC64)" + depends on SPARC64 + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using sparc64 crypto instructions, when available. + +config CRYPTO_SHA256_SPARC64 + tristate "SHA224 and SHA256 digest algorithm (SPARC64)" + depends on SPARC64 + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-256 secure hash standard (DFIPS 180-2) implemented + using sparc64 crypto instructions, when available. + +config CRYPTO_SHA512_SPARC64 + tristate "SHA384 and SHA512 digest algorithm (SPARC64)" + depends on SPARC64 + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using sparc64 crypto instructions, when available. + +config CRYPTO_AES_SPARC64 + tristate "AES cipher algorithms (SPARC64)" + depends on SPARC64 + select CRYPTO_SKCIPHER + help + Use SPARC64 crypto opcodes for AES algorithm. + + AES cipher algorithms (FIPS-197). AES uses the Rijndael + algorithm. + + Rijndael appears to be consistently a very good performer in + both hardware and software across a wide range of computing + environments regardless of its use in feedback or non-feedback + modes. Its key setup time is excellent, and its key agility is + good. Rijndael's very low memory requirements make it very well + suited for restricted-space environments, in which it also + demonstrates excellent performance. Rijndael's operations are + among the easiest to defend against power and timing attacks. + + The AES specifies three key sizes: 128, 192 and 256 bits + + See for more information. + + In addition to AES cipher algorithm support, the acceleration + for some popular block cipher mode is supported too, including + ECB and CBC. + +config CRYPTO_CAMELLIA_SPARC64 + tristate "Camellia cipher algorithm (SPARC64)" + depends on SPARC64 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + help + Camellia cipher algorithm module (SPARC64). + + Camellia is a symmetric key block cipher developed jointly + at NTT and Mitsubishi Electric Corporation. + + The Camellia specifies three key sizes: 128, 192 and 256 bits. + + See also: + + +endmenu diff --git a/crypto/Kconfig b/crypto/Kconfig index 0eb6090e7562..5ea3cdb975cd 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -658,15 +658,6 @@ config CRYPTO_CRC32C_INTEL gain performance compared with software implementation. Module will be crc32c-intel. -config CRYPTO_CRC32C_SPARC64 - tristate "CRC32c CRC algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_HASH - select CRC32 - help - CRC32c CRC algorithm implemented using sparc64 crypto instructions, - when available. - config CRYPTO_CRC32 tristate "CRC32 CRC algorithm" select CRYPTO_HASH @@ -804,15 +795,6 @@ config CRYPTO_MD5 help MD5 message digest algorithm (RFC1321). -config CRYPTO_MD5_SPARC64 - tristate "MD5 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_MD5 - select CRYPTO_HASH - help - MD5 message digest algorithm (RFC1321) implemented - using sparc64 crypto instructions, when available. - config CRYPTO_MICHAEL_MIC tristate "Michael MIC keyed digest algorithm" select CRYPTO_HASH @@ -880,15 +862,6 @@ config CRYPTO_SHA512_SSSE3 Extensions version 1 (AVX1), or Advanced Vector Extensions version 2 (AVX2) instructions, when available. -config CRYPTO_SHA1_SPARC64 - tristate "SHA1 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. - config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH @@ -902,15 +875,6 @@ config CRYPTO_SHA256 This code also includes SHA-224, a 224 bit hash with 112 bits of security against collision attacks. -config CRYPTO_SHA256_SPARC64 - tristate "SHA224 and SHA256 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. - config CRYPTO_SHA512 tristate "SHA384 and SHA512 digest algorithms" select CRYPTO_HASH @@ -923,15 +887,6 @@ config CRYPTO_SHA512 This code also includes SHA-384, a 384 bit hash with 192 bits of security against collision attacks. -config CRYPTO_SHA512_SPARC64 - tristate "SHA384 and SHA512 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. - config CRYPTO_SHA3 tristate "SHA3 digest algorithm" select CRYPTO_HASH @@ -1076,33 +1031,6 @@ config CRYPTO_AES_NI_INTEL ECB, CBC, LRW, XTS. The 64 bit version has additional acceleration for CTR and XCTR. -config CRYPTO_AES_SPARC64 - tristate "AES cipher algorithms (SPARC64)" - depends on SPARC64 - select CRYPTO_SKCIPHER - help - Use SPARC64 crypto opcodes for AES algorithm. - - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. - - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. - - The AES specifies three key sizes: 128, 192 and 256 bits - - See for more information. - - In addition to AES cipher algorithm support, the acceleration - for some popular block cipher mode is supported too, including - ECB and CBC. - config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" depends on CRYPTO_USER_API_ENABLE_OBSOLETE @@ -1233,22 +1161,6 @@ config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 See also: -config CRYPTO_CAMELLIA_SPARC64 - tristate "Camellia cipher algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER - help - Camellia cipher algorithm module (SPARC64). - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - - config CRYPTO_CAST_COMMON tristate help @@ -1309,16 +1221,6 @@ config CRYPTO_DES help DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). -config CRYPTO_DES_SPARC64 - tristate "DES and Triple DES EDE cipher algorithms (SPARC64)" - depends on SPARC64 - select CRYPTO_ALGAPI - select CRYPTO_LIB_DES - select CRYPTO_SKCIPHER - help - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3), - optimized using SPARC64 crypto opcodes. - config CRYPTO_DES3_EDE_X86_64 tristate "Triple DES EDE cipher algorithm (x86-64)" depends on X86 && 64BIT @@ -1894,6 +1796,9 @@ endif if S390 source "arch/s390/crypto/Kconfig" endif +if SPARC +source "arch/sparc/crypto/Kconfig" +endif source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" -- cgit From 28a936ef44e12b4d2b38f45ff767262763b60a20 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:39 -0500 Subject: crypto: Kconfig - move x86 entries to a submenu Move CPU-specific crypto/Kconfig entries to arch/xxx/crypto/Kconfig and create a submenu for them under the Crypto API menu. Suggested-by: Eric Biggers Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 500 ++++++++++++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 498 +---------------------------------------------- 2 files changed, 503 insertions(+), 495 deletions(-) create mode 100644 arch/x86/crypto/Kconfig diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig new file mode 100644 index 000000000000..04f4baea12a8 --- /dev/null +++ b/arch/x86/crypto/Kconfig @@ -0,0 +1,500 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (x86)" + +config CRYPTO_CURVE25519_X86 + tristate "x86_64 accelerated Curve25519 scalar multiplication library" + depends on X86 && 64BIT + select CRYPTO_LIB_CURVE25519_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + +config CRYPTO_AES_NI_INTEL + tristate "AES cipher algorithms (AES-NI)" + depends on X86 + select CRYPTO_AEAD + select CRYPTO_LIB_AES + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + help + Use Intel AES-NI instructions for AES algorithm. + + AES cipher algorithms (FIPS-197). AES uses the Rijndael + algorithm. + + Rijndael appears to be consistently a very good performer in + both hardware and software across a wide range of computing + environments regardless of its use in feedback or non-feedback + modes. Its key setup time is excellent, and its key agility is + good. Rijndael's very low memory requirements make it very well + suited for restricted-space environments, in which it also + demonstrates excellent performance. Rijndael's operations are + among the easiest to defend against power and timing attacks. + + The AES specifies three key sizes: 128, 192 and 256 bits + + See for more information. + + In addition to AES cipher algorithm support, the acceleration + for some popular block cipher mode is supported too, including + ECB, CBC, LRW, XTS. The 64 bit version has additional + acceleration for CTR and XCTR. + +config CRYPTO_BLOWFISH_X86_64 + tristate "Blowfish cipher algorithm (x86_64)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_BLOWFISH_COMMON + imply CRYPTO_CTR + help + Blowfish cipher algorithm (x86_64), by Bruce Schneier. + + This is a variable key length cipher which can use keys from 32 + bits to 448 bits in length. It's fast, simple and specifically + designed for use on "large microprocessors". + + See also: + + +config CRYPTO_CAMELLIA_X86_64 + tristate "Camellia cipher algorithm (x86_64)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + imply CRYPTO_CTR + help + Camellia cipher algorithm module (x86_64). + + Camellia is a symmetric key block cipher developed jointly + at NTT and Mitsubishi Electric Corporation. + + The Camellia specifies three key sizes: 128, 192 and 256 bits. + + See also: + + +config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 + tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_CAMELLIA_X86_64 + select CRYPTO_SIMD + imply CRYPTO_XTS + help + Camellia cipher algorithm module (x86_64/AES-NI/AVX). + + Camellia is a symmetric key block cipher developed jointly + at NTT and Mitsubishi Electric Corporation. + + The Camellia specifies three key sizes: 128, 192 and 256 bits. + + See also: + + +config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 + tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX2)" + depends on X86 && 64BIT + select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 + help + Camellia cipher algorithm module (x86_64/AES-NI/AVX2). + + Camellia is a symmetric key block cipher developed jointly + at NTT and Mitsubishi Electric Corporation. + + The Camellia specifies three key sizes: 128, 192 and 256 bits. + + See also: + + +config CRYPTO_CAST5_AVX_X86_64 + tristate "CAST5 (CAST-128) cipher algorithm (x86_64/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_CAST5 + select CRYPTO_CAST_COMMON + select CRYPTO_SIMD + imply CRYPTO_CTR + help + The CAST5 encryption algorithm (synonymous with CAST-128) is + described in RFC2144. + + This module provides the Cast5 cipher algorithm that processes + sixteen blocks parallel using the AVX instruction set. + +config CRYPTO_CAST6_AVX_X86_64 + tristate "CAST6 (CAST-256) cipher algorithm (x86_64/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_CAST6 + select CRYPTO_CAST_COMMON + select CRYPTO_SIMD + imply CRYPTO_XTS + imply CRYPTO_CTR + help + The CAST6 encryption algorithm (synonymous with CAST-256) is + described in RFC2612. + + This module provides the Cast6 cipher algorithm that processes + eight blocks parallel using the AVX instruction set. + +config CRYPTO_DES3_EDE_X86_64 + tristate "Triple DES EDE cipher algorithm (x86-64)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + imply CRYPTO_CTR + help + Triple DES EDE (FIPS 46-3) algorithm. + + This module provides implementation of the Triple DES EDE cipher + algorithm that is optimized for x86-64 processors. Two versions of + algorithm are provided; regular processing one input block and + one that processes three blocks parallel. + +config CRYPTO_SERPENT_SSE2_X86_64 + tristate "Serpent cipher algorithm (x86_64/SSE2)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SERPENT + select CRYPTO_SIMD + imply CRYPTO_CTR + help + Serpent cipher algorithm, by Anderson, Biham & Knudsen. + + Keys are allowed to be from 0 to 256 bits in length, in steps + of 8 bits. + + This module provides Serpent cipher algorithm that processes eight + blocks parallel using SSE2 instruction set. + + See also: + + +config CRYPTO_SERPENT_SSE2_586 + tristate "Serpent cipher algorithm (i586/SSE2)" + depends on X86 && !64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SERPENT + select CRYPTO_SIMD + imply CRYPTO_CTR + help + Serpent cipher algorithm, by Anderson, Biham & Knudsen. + + Keys are allowed to be from 0 to 256 bits in length, in steps + of 8 bits. + + This module provides Serpent cipher algorithm that processes four + blocks parallel using SSE2 instruction set. + + See also: + + +config CRYPTO_SERPENT_AVX_X86_64 + tristate "Serpent cipher algorithm (x86_64/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SERPENT + select CRYPTO_SIMD + imply CRYPTO_XTS + imply CRYPTO_CTR + help + Serpent cipher algorithm, by Anderson, Biham & Knudsen. + + Keys are allowed to be from 0 to 256 bits in length, in steps + of 8 bits. + + This module provides the Serpent cipher algorithm that processes + eight blocks parallel using the AVX instruction set. + + See also: + + +config CRYPTO_SERPENT_AVX2_X86_64 + tristate "Serpent cipher algorithm (x86_64/AVX2)" + depends on X86 && 64BIT + select CRYPTO_SERPENT_AVX_X86_64 + help + Serpent cipher algorithm, by Anderson, Biham & Knudsen. + + Keys are allowed to be from 0 to 256 bits in length, in steps + of 8 bits. + + This module provides Serpent cipher algorithm that processes 16 + blocks parallel using AVX2 instruction set. + + See also: + + +config CRYPTO_SM4_AESNI_AVX_X86_64 + tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_ALGAPI + select CRYPTO_SM4 + help + SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX). + + SM4 (GBT.32907-2016) is a cryptographic standard issued by the + Organization of State Commercial Administration of China (OSCCA) + as an authorized cryptographic algorithms for the use within China. + + This is SM4 optimized implementation using AES-NI/AVX/x86_64 + instruction set for block cipher. Through two affine transforms, + we can use the AES S-Box to simulate the SM4 S-Box to achieve the + effect of instruction acceleration. + + If unsure, say N. + +config CRYPTO_SM4_AESNI_AVX2_X86_64 + tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX2)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_ALGAPI + select CRYPTO_SM4 + select CRYPTO_SM4_AESNI_AVX_X86_64 + help + SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX2). + + SM4 (GBT.32907-2016) is a cryptographic standard issued by the + Organization of State Commercial Administration of China (OSCCA) + as an authorized cryptographic algorithms for the use within China. + + This is SM4 optimized implementation using AES-NI/AVX2/x86_64 + instruction set for block cipher. Through two affine transforms, + we can use the AES S-Box to simulate the SM4 S-Box to achieve the + effect of instruction acceleration. + + If unsure, say N. + +config CRYPTO_TWOFISH_586 + tristate "Twofish cipher algorithms (i586)" + depends on (X86 || UML_X86) && !64BIT + select CRYPTO_ALGAPI + select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR + help + Twofish cipher algorithm. + + Twofish was submitted as an AES (Advanced Encryption Standard) + candidate cipher by researchers at CounterPane Systems. It is a + 16 round block cipher supporting key sizes of 128, 192, and 256 + bits. + + See also: + + +config CRYPTO_TWOFISH_X86_64 + tristate "Twofish cipher algorithm (x86_64)" + depends on (X86 || UML_X86) && 64BIT + select CRYPTO_ALGAPI + select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR + help + Twofish cipher algorithm (x86_64). + + Twofish was submitted as an AES (Advanced Encryption Standard) + candidate cipher by researchers at CounterPane Systems. It is a + 16 round block cipher supporting key sizes of 128, 192, and 256 + bits. + + See also: + + +config CRYPTO_TWOFISH_X86_64_3WAY + tristate "Twofish cipher algorithm (x86_64, 3-way parallel)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_TWOFISH_COMMON + select CRYPTO_TWOFISH_X86_64 + help + Twofish cipher algorithm (x86_64, 3-way parallel). + + Twofish was submitted as an AES (Advanced Encryption Standard) + candidate cipher by researchers at CounterPane Systems. It is a + 16 round block cipher supporting key sizes of 128, 192, and 256 + bits. + + This module provides Twofish cipher algorithm that processes three + blocks parallel, utilizing resources of out-of-order CPUs better. + + See also: + + +config CRYPTO_TWOFISH_AVX_X86_64 + tristate "Twofish cipher algorithm (x86_64/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_TWOFISH_COMMON + select CRYPTO_TWOFISH_X86_64 + select CRYPTO_TWOFISH_X86_64_3WAY + imply CRYPTO_XTS + help + Twofish cipher algorithm (x86_64/AVX). + + Twofish was submitted as an AES (Advanced Encryption Standard) + candidate cipher by researchers at CounterPane Systems. It is a + 16 round block cipher supporting key sizes of 128, 192, and 256 + bits. + + This module provides the Twofish cipher algorithm that processes + eight blocks parallel using the AVX Instruction Set. + + See also: + + +config CRYPTO_CHACHA20_X86_64 + tristate "ChaCha stream cipher algorithms (x86_64/SSSE3/AVX2/AVX-512VL)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20, + XChaCha20, and XChaCha12 stream ciphers. + +config CRYPTO_AEGIS128_AESNI_SSE2 + tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" + depends on X86 && 64BIT + select CRYPTO_AEAD + select CRYPTO_SIMD + help + AESNI+SSE2 implementation of the AEGIS-128 dedicated AEAD algorithm. + +config CRYPTO_NHPOLY1305_SSE2 + tristate "NHPoly1305 hash function (x86_64 SSE2 implementation)" + depends on X86 && 64BIT + select CRYPTO_NHPOLY1305 + help + SSE2 optimized implementation of the hash function used by the + Adiantum encryption mode. + +config CRYPTO_NHPOLY1305_AVX2 + tristate "NHPoly1305 hash function (x86_64 AVX2 implementation)" + depends on X86 && 64BIT + select CRYPTO_NHPOLY1305 + help + AVX2 optimized implementation of the hash function used by the + Adiantum encryption mode. + +config CRYPTO_BLAKE2S_X86 + bool "BLAKE2s digest algorithm (x86 accelerated version)" + depends on X86 && 64BIT + select CRYPTO_LIB_BLAKE2S_GENERIC + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + +config CRYPTO_POLYVAL_CLMUL_NI + tristate "POLYVAL hash function (CLMUL-NI accelerated)" + depends on X86 && 64BIT + select CRYPTO_POLYVAL + help + This is the x86_64 CLMUL-NI accelerated implementation of POLYVAL. It is + used to efficiently implement HCTR2 on x86-64 processors that support + carry-less multiplication instructions. + +config CRYPTO_POLY1305_X86_64 + tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)" + depends on X86 && 64BIT + select CRYPTO_LIB_POLY1305_GENERIC + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm, RFC7539. + + Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. + It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use + in IETF protocols. This is the x86_64 assembler implementation using SIMD + instructions. + +config CRYPTO_SHA1_SSSE3 + tristate "SHA1 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" + depends on X86 && 64BIT + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using Supplemental SSE3 (SSSE3) instructions or Advanced Vector + Extensions (AVX/AVX2) or SHA-NI(SHA Extensions New Instructions), + when available. + +config CRYPTO_SHA256_SSSE3 + tristate "SHA256 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" + depends on X86 && 64BIT + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-256 secure hash standard (DFIPS 180-2) implemented + using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector + Extensions version 1 (AVX1), or Advanced Vector Extensions + version 2 (AVX2) instructions, or SHA-NI (SHA Extensions New + Instructions) when available. + +config CRYPTO_SHA512_SSSE3 + tristate "SHA512 digest algorithm (SSSE3/AVX/AVX2)" + depends on X86 && 64BIT + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector + Extensions version 1 (AVX1), or Advanced Vector Extensions + version 2 (AVX2) instructions, when available. + +config CRYPTO_SM3_AVX_X86_64 + tristate "SM3 digest algorithm (x86_64/AVX)" + depends on X86 && 64BIT + select CRYPTO_HASH + select CRYPTO_SM3 + help + SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). + It is part of the Chinese Commercial Cryptography suite. This is + SM3 optimized implementation using Advanced Vector Extensions (AVX) + when available. + + If unsure, say N. + +config CRYPTO_GHASH_CLMUL_NI_INTEL + tristate "GHASH hash function (CLMUL-NI accelerated)" + depends on X86 && 64BIT + select CRYPTO_CRYPTD + help + This is the x86_64 CLMUL-NI accelerated implementation of + GHASH, the hash function used in GCM (Galois/Counter mode). + +config CRYPTO_CRC32C_INTEL + tristate "CRC32c INTEL hardware acceleration" + depends on X86 + select CRYPTO_HASH + help + In Intel processor with SSE4.2 supported, the processor will + support CRC32C implementation using hardware accelerated CRC32 + instruction. This option will create 'crc32c-intel' module, + which will enable any routine to use the CRC32 instruction to + gain performance compared with software implementation. + Module will be crc32c-intel. + +config CRYPTO_CRC32_PCLMUL + tristate "CRC32 PCLMULQDQ hardware acceleration" + depends on X86 + select CRYPTO_HASH + select CRC32 + help + From Intel Westmere and AMD Bulldozer processor with SSE4.2 + and PCLMULQDQ supported, the processor will support + CRC32 PCLMULQDQ implementation using hardware accelerated PCLMULQDQ + instruction. This option will create 'crc32-pclmul' module, + which will enable any routine to use the CRC-32-IEEE 802.3 checksum + and gain better performance as compared with the table implementation. + +config CRYPTO_CRCT10DIF_PCLMUL + tristate "CRCT10DIF PCLMULQDQ hardware acceleration" + depends on X86 && 64BIT && CRC_T10DIF + select CRYPTO_HASH + help + For x86_64 processors with SSE4.2 and PCLMULQDQ supported, + CRC T10 DIF PCLMULQDQ computation can be hardware + accelerated PCLMULQDQ instruction. This option will create + 'crct10dif-pclmul' module, which is faster when computing the + crct10dif checksum as compared with the generic table implementation. + +endmenu diff --git a/crypto/Kconfig b/crypto/Kconfig index 5ea3cdb975cd..c249fdacba66 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -316,12 +316,6 @@ config CRYPTO_CURVE25519 select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC -config CRYPTO_CURVE25519_X86 - tristate "x86_64 accelerated Curve25519 scalar multiplication library" - depends on X86 && 64BIT - select CRYPTO_LIB_CURVE25519_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CURVE25519 - comment "Authenticated Encryption with Associated Data" config CRYPTO_CCM @@ -369,14 +363,6 @@ config CRYPTO_AEGIS128_SIMD depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON) default y -config CRYPTO_AEGIS128_AESNI_SSE2 - tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" - depends on X86 && 64BIT - select CRYPTO_AEAD - select CRYPTO_SIMD - help - AESNI+SSE2 implementation of the AEGIS-128 dedicated AEAD algorithm. - config CRYPTO_SEQIV tristate "Sequence Number IV Generator" select CRYPTO_AEAD @@ -514,22 +500,6 @@ config CRYPTO_NHPOLY1305 select CRYPTO_HASH select CRYPTO_LIB_POLY1305_GENERIC -config CRYPTO_NHPOLY1305_SSE2 - tristate "NHPoly1305 hash function (x86_64 SSE2 implementation)" - depends on X86 && 64BIT - select CRYPTO_NHPOLY1305 - help - SSE2 optimized implementation of the hash function used by the - Adiantum encryption mode. - -config CRYPTO_NHPOLY1305_AVX2 - tristate "NHPoly1305 hash function (x86_64 AVX2 implementation)" - depends on X86 && 64BIT - select CRYPTO_NHPOLY1305 - help - AVX2 optimized implementation of the hash function used by the - Adiantum encryption mode. - config CRYPTO_ADIANTUM tristate "Adiantum support" select CRYPTO_CHACHA20 @@ -646,18 +616,6 @@ config CRYPTO_CRC32C by iSCSI for header and data digests and by others. See Castagnoli93. Module will be crc32c. -config CRYPTO_CRC32C_INTEL - tristate "CRC32c INTEL hardware acceleration" - depends on X86 - select CRYPTO_HASH - help - In Intel processor with SSE4.2 supported, the processor will - support CRC32C implementation using hardware accelerated CRC32 - instruction. This option will create 'crc32c-intel' module, - which will enable any routine to use the CRC32 instruction to - gain performance compared with software implementation. - Module will be crc32c-intel. - config CRYPTO_CRC32 tristate "CRC32 CRC algorithm" select CRYPTO_HASH @@ -666,19 +624,6 @@ config CRYPTO_CRC32 CRC-32-IEEE 802.3 cyclic redundancy-check algorithm. Shash crypto api wrappers to crc32_le function. -config CRYPTO_CRC32_PCLMUL - tristate "CRC32 PCLMULQDQ hardware acceleration" - depends on X86 - select CRYPTO_HASH - select CRC32 - help - From Intel Westmere and AMD Bulldozer processor with SSE4.2 - and PCLMULQDQ supported, the processor will support - CRC32 PCLMULQDQ implementation using hardware accelerated PCLMULQDQ - instruction. This option will create 'crc32-pclmul' module, - which will enable any routine to use the CRC-32-IEEE 802.3 checksum - and gain better performance as compared with the table implementation. - config CRYPTO_XXHASH tristate "xxHash hash algorithm" select CRYPTO_HASH @@ -704,12 +649,6 @@ config CRYPTO_BLAKE2B See https://blake2.net for further information. -config CRYPTO_BLAKE2S_X86 - bool "BLAKE2s digest algorithm (x86 accelerated version)" - depends on X86 && 64BIT - select CRYPTO_LIB_BLAKE2S_GENERIC - select CRYPTO_ARCH_HAVE_LIB_BLAKE2S - config CRYPTO_CRCT10DIF tristate "CRCT10DIF algorithm" select CRYPTO_HASH @@ -718,17 +657,6 @@ config CRYPTO_CRCT10DIF a crypto transform. This allows for faster crc t10 diff transforms to be used if they are available. -config CRYPTO_CRCT10DIF_PCLMUL - tristate "CRCT10DIF PCLMULQDQ hardware acceleration" - depends on X86 && 64BIT && CRC_T10DIF - select CRYPTO_HASH - help - For x86_64 processors with SSE4.2 and PCLMULQDQ supported, - CRC T10 DIF PCLMULQDQ computation can be hardware - accelerated PCLMULQDQ instruction. This option will create - 'crct10dif-pclmul' module, which is faster when computing the - crct10dif checksum as compared with the generic table implementation. - config CRYPTO_CRC64_ROCKSOFT tristate "Rocksoft Model CRC64 algorithm" depends on CRC64 @@ -750,15 +678,6 @@ config CRYPTO_POLYVAL POLYVAL is the hash function used in HCTR2. It is not a general-purpose cryptographic hash function. -config CRYPTO_POLYVAL_CLMUL_NI - tristate "POLYVAL hash function (CLMUL-NI accelerated)" - depends on X86 && 64BIT - select CRYPTO_POLYVAL - help - This is the x86_64 CLMUL-NI accelerated implementation of POLYVAL. It is - used to efficiently implement HCTR2 on x86-64 processors that support - carry-less multiplication instructions. - config CRYPTO_POLY1305 tristate "Poly1305 authenticator algorithm" select CRYPTO_HASH @@ -770,19 +689,6 @@ config CRYPTO_POLY1305 It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use in IETF protocols. This is the portable C implementation of Poly1305. -config CRYPTO_POLY1305_X86_64 - tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)" - depends on X86 && 64BIT - select CRYPTO_LIB_POLY1305_GENERIC - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - help - Poly1305 authenticator algorithm, RFC7539. - - Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. - It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use - in IETF protocols. This is the x86_64 assembler implementation using SIMD - instructions. - config CRYPTO_MD4 tristate "MD4 digest algorithm" select CRYPTO_HASH @@ -828,40 +734,6 @@ config CRYPTO_SHA1 help SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). -config CRYPTO_SHA1_SSSE3 - tristate "SHA1 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" - depends on X86 && 64BIT - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions or Advanced Vector - Extensions (AVX/AVX2) or SHA-NI(SHA Extensions New Instructions), - when available. - -config CRYPTO_SHA256_SSSE3 - tristate "SHA256 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" - depends on X86 && 64BIT - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector - Extensions version 1 (AVX1), or Advanced Vector Extensions - version 2 (AVX2) instructions, or SHA-NI (SHA Extensions New - Instructions) when available. - -config CRYPTO_SHA512_SSSE3 - tristate "SHA512 digest algorithm (SSSE3/AVX/AVX2)" - depends on X86 && 64BIT - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector - Extensions version 1 (AVX1), or Advanced Vector Extensions - version 2 (AVX2) instructions, when available. - config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH @@ -912,19 +784,6 @@ config CRYPTO_SM3_GENERIC http://www.oscca.gov.cn/UpFile/20101222141857786.pdf https://datatracker.ietf.org/doc/html/draft-shen-sm3-hash -config CRYPTO_SM3_AVX_X86_64 - tristate "SM3 digest algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_HASH - select CRYPTO_SM3 - help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). - It is part of the Chinese Commercial Cryptography suite. This is - SM3 optimized implementation using Advanced Vector Extensions (AVX) - when available. - - If unsure, say N. - config CRYPTO_STREEBOG tristate "Streebog Hash Function" select CRYPTO_HASH @@ -949,14 +808,6 @@ config CRYPTO_WP512 See also: -config CRYPTO_GHASH_CLMUL_NI_INTEL - tristate "GHASH hash function (CLMUL-NI accelerated)" - depends on X86 && 64BIT - select CRYPTO_CRYPTD - help - This is the x86_64 CLMUL-NI accelerated implementation of - GHASH, the hash function used in GCM (Galois/Counter mode). - comment "Ciphers" config CRYPTO_AES @@ -999,38 +850,6 @@ config CRYPTO_AES_TI block. Interrupts are also disabled to avoid races where cachelines are evicted when the CPU is interrupted to do something else. -config CRYPTO_AES_NI_INTEL - tristate "AES cipher algorithms (AES-NI)" - depends on X86 - select CRYPTO_AEAD - select CRYPTO_LIB_AES - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER - select CRYPTO_SIMD - help - Use Intel AES-NI instructions for AES algorithm. - - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. - - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. - - The AES specifies three key sizes: 128, 192 and 256 bits - - See for more information. - - In addition to AES cipher algorithm support, the acceleration - for some popular block cipher mode is supported too, including - ECB, CBC, LRW, XTS. The 64 bit version has additional - acceleration for CTR and XCTR. - config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" depends on CRYPTO_USER_API_ENABLE_OBSOLETE @@ -1082,22 +901,6 @@ config CRYPTO_BLOWFISH_COMMON See also: -config CRYPTO_BLOWFISH_X86_64 - tristate "Blowfish cipher algorithm (x86_64)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_BLOWFISH_COMMON - imply CRYPTO_CTR - help - Blowfish cipher algorithm (x86_64), by Bruce Schneier. - - This is a variable key length cipher which can use keys from 32 - bits to 448 bits in length. It's fast, simple and specifically - designed for use on "large microprocessors". - - See also: - - config CRYPTO_CAMELLIA tristate "Camellia cipher algorithms" select CRYPTO_ALGAPI @@ -1112,55 +915,6 @@ config CRYPTO_CAMELLIA See also: -config CRYPTO_CAMELLIA_X86_64 - tristate "Camellia cipher algorithm (x86_64)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - imply CRYPTO_CTR - help - Camellia cipher algorithm module (x86_64). - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - - -config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 - tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_CAMELLIA_X86_64 - select CRYPTO_SIMD - imply CRYPTO_XTS - help - Camellia cipher algorithm module (x86_64/AES-NI/AVX). - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - - -config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 - tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX2)" - depends on X86 && 64BIT - select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 - help - Camellia cipher algorithm module (x86_64/AES-NI/AVX2). - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - - config CRYPTO_CAST_COMMON tristate help @@ -1175,21 +929,6 @@ config CRYPTO_CAST5 The CAST5 encryption algorithm (synonymous with CAST-128) is described in RFC2144. -config CRYPTO_CAST5_AVX_X86_64 - tristate "CAST5 (CAST-128) cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_CAST5 - select CRYPTO_CAST_COMMON - select CRYPTO_SIMD - imply CRYPTO_CTR - help - The CAST5 encryption algorithm (synonymous with CAST-128) is - described in RFC2144. - - This module provides the Cast5 cipher algorithm that processes - sixteen blocks parallel using the AVX instruction set. - config CRYPTO_CAST6 tristate "CAST6 (CAST-256) cipher algorithm" select CRYPTO_ALGAPI @@ -1198,22 +937,6 @@ config CRYPTO_CAST6 The CAST6 encryption algorithm (synonymous with CAST-256) is described in RFC2612. -config CRYPTO_CAST6_AVX_X86_64 - tristate "CAST6 (CAST-256) cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_CAST6 - select CRYPTO_CAST_COMMON - select CRYPTO_SIMD - imply CRYPTO_XTS - imply CRYPTO_CTR - help - The CAST6 encryption algorithm (synonymous with CAST-256) is - described in RFC2612. - - This module provides the Cast6 cipher algorithm that processes - eight blocks parallel using the AVX instruction set. - config CRYPTO_DES tristate "DES and Triple DES EDE cipher algorithms" select CRYPTO_ALGAPI @@ -1221,20 +944,6 @@ config CRYPTO_DES help DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). -config CRYPTO_DES3_EDE_X86_64 - tristate "Triple DES EDE cipher algorithm (x86-64)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_LIB_DES - imply CRYPTO_CTR - help - Triple DES EDE (FIPS 46-3) algorithm. - - This module provides implementation of the Triple DES EDE cipher - algorithm that is optimized for x86-64 processors. Two versions of - algorithm are provided; regular processing one input block and - one that processes three blocks parallel. - config CRYPTO_FCRYPT tristate "FCrypt cipher algorithm" select CRYPTO_ALGAPI @@ -1278,16 +987,6 @@ config CRYPTO_CHACHA20 reduced security margin but increased performance. It can be needed in some performance-sensitive scenarios. -config CRYPTO_CHACHA20_X86_64 - tristate "ChaCha stream cipher algorithms (x86_64/SSSE3/AVX2/AVX-512VL)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - help - SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20, - XChaCha20, and XChaCha12 stream ciphers. - config CRYPTO_SEED tristate "SEED cipher algorithm" depends on CRYPTO_USER_API_ENABLE_OBSOLETE @@ -1330,80 +1029,6 @@ config CRYPTO_SERPENT See also: -config CRYPTO_SERPENT_SSE2_X86_64 - tristate "Serpent cipher algorithm (x86_64/SSE2)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SERPENT - select CRYPTO_SIMD - imply CRYPTO_CTR - help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. - - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides Serpent cipher algorithm that processes eight - blocks parallel using SSE2 instruction set. - - See also: - - -config CRYPTO_SERPENT_SSE2_586 - tristate "Serpent cipher algorithm (i586/SSE2)" - depends on X86 && !64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SERPENT - select CRYPTO_SIMD - imply CRYPTO_CTR - help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. - - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides Serpent cipher algorithm that processes four - blocks parallel using SSE2 instruction set. - - See also: - - -config CRYPTO_SERPENT_AVX_X86_64 - tristate "Serpent cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SERPENT - select CRYPTO_SIMD - imply CRYPTO_XTS - imply CRYPTO_CTR - help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. - - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides the Serpent cipher algorithm that processes - eight blocks parallel using the AVX instruction set. - - See also: - - -config CRYPTO_SERPENT_AVX2_X86_64 - tristate "Serpent cipher algorithm (x86_64/AVX2)" - depends on X86 && 64BIT - select CRYPTO_SERPENT_AVX_X86_64 - help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. - - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides Serpent cipher algorithm that processes 16 - blocks parallel using AVX2 instruction set. - - See also: - - config CRYPTO_SM4 tristate @@ -1433,49 +1058,6 @@ config CRYPTO_SM4_GENERIC If unsure, say N. -config CRYPTO_SM4_AESNI_AVX_X86_64 - tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SIMD - select CRYPTO_ALGAPI - select CRYPTO_SM4 - help - SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX). - - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. - - This is SM4 optimized implementation using AES-NI/AVX/x86_64 - instruction set for block cipher. Through two affine transforms, - we can use the AES S-Box to simulate the SM4 S-Box to achieve the - effect of instruction acceleration. - - If unsure, say N. - -config CRYPTO_SM4_AESNI_AVX2_X86_64 - tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX2)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SIMD - select CRYPTO_ALGAPI - select CRYPTO_SM4 - select CRYPTO_SM4_AESNI_AVX_X86_64 - help - SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX2). - - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. - - This is SM4 optimized implementation using AES-NI/AVX2/x86_64 - instruction set for block cipher. Through two affine transforms, - we can use the AES S-Box to simulate the SM4 S-Box to achieve the - effect of instruction acceleration. - - If unsure, say N. - config CRYPTO_TEA tristate "TEA, XTEA and XETA cipher algorithms" depends on CRYPTO_USER_API_ENABLE_OBSOLETE @@ -1515,83 +1097,6 @@ config CRYPTO_TWOFISH_COMMON Common parts of the Twofish cipher algorithm shared by the generic c and the assembler implementations. -config CRYPTO_TWOFISH_586 - tristate "Twofish cipher algorithms (i586)" - depends on (X86 || UML_X86) && !64BIT - select CRYPTO_ALGAPI - select CRYPTO_TWOFISH_COMMON - imply CRYPTO_CTR - help - Twofish cipher algorithm. - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. - - See also: - - -config CRYPTO_TWOFISH_X86_64 - tristate "Twofish cipher algorithm (x86_64)" - depends on (X86 || UML_X86) && 64BIT - select CRYPTO_ALGAPI - select CRYPTO_TWOFISH_COMMON - imply CRYPTO_CTR - help - Twofish cipher algorithm (x86_64). - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. - - See also: - - -config CRYPTO_TWOFISH_X86_64_3WAY - tristate "Twofish cipher algorithm (x86_64, 3-way parallel)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_TWOFISH_COMMON - select CRYPTO_TWOFISH_X86_64 - help - Twofish cipher algorithm (x86_64, 3-way parallel). - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. - - This module provides Twofish cipher algorithm that processes three - blocks parallel, utilizing resources of out-of-order CPUs better. - - See also: - - -config CRYPTO_TWOFISH_AVX_X86_64 - tristate "Twofish cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SIMD - select CRYPTO_TWOFISH_COMMON - select CRYPTO_TWOFISH_X86_64 - select CRYPTO_TWOFISH_X86_64_3WAY - imply CRYPTO_XTS - help - Twofish cipher algorithm (x86_64/AVX). - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. - - This module provides the Twofish cipher algorithm that processes - eight blocks parallel using the AVX Instruction Set. - - See also: - - comment "Compression" config CRYPTO_DEFLATE @@ -1799,6 +1304,9 @@ endif if SPARC source "arch/sparc/crypto/Kconfig" endif +if X86 +source "arch/x86/crypto/Kconfig" +endif source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" -- cgit From 5530acc8b9bfcfa5f90909b940be39a84c350033 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:40 -0500 Subject: crypto: Kconfig - remove AES_ARM64 ref by SA2UL Remove the CRYPTO_AES_ARM64 selection by the TI security accelerator driver (SA2UL), which leads to this problem when running make allmodconfig for arm (32-bit): WARNING: unmet direct dependencies detected for CRYPTO_AES_ARM64 Depends on [n]: CRYPTO [=y] && ARM64 Selected by [m]: - CRYPTO_DEV_SA2UL [=m] && CRYPTO [=y] && CRYPTO_HW [=y] && (ARCH_K3 || COMPILE_TEST [=y]) Fixes: 7694b6ca649fe ("crypto: sa2ul - Add crypto driver") Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index f6fe1945edd6..72c58b21ba62 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -804,7 +804,6 @@ config CRYPTO_DEV_SA2UL depends on ARCH_K3 || COMPILE_TEST select ARM64_CRYPTO select CRYPTO_AES - select CRYPTO_AES_ARM64 select CRYPTO_ALGAPI select CRYPTO_AUTHENC select CRYPTO_SHA1 -- cgit From 4a329fecc9aaebb27a53fa7abfa53bbc2ee42f3f Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:41 -0500 Subject: crypto: Kconfig - submenus for arm and arm64 Move ARM- and ARM64-accelerated menus into a submenu under the Crypto API menu (paralleling all the architectures). Make each submenu always appear if the corresponding architecture is supported. Get rid of the ARM_CRYPTO and ARM64_CRYPTO symbols. The "ARM Accelerated" or "ARM64 Accelerated" entry disappears from: General setup ---> Platform selection ---> Kernel Features ---> Boot options ---> Power management options ---> CPU Power Management ---> [*] ACPI (Advanced Configuration and Power Interface) Support ---> [*] Virtualization ---> [*] ARM Accelerated Cryptographic Algorithms ---> (or) [*] ARM64 Accelerated Cryptographic Algorithms ---> ... -*- Cryptographic API ---> Library routines ---> Kernel hacking ---> and moves into the Cryptographic API menu, which now contains: ... Accelerated Cryptographic Algorithms for CPU (arm) ---> (or) Accelerated Cryptographic Algorithms for CPU (arm64) ---> [*] Hardware crypto devices ---> ... Suggested-by: Eric Biggers Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/arm/Kconfig | 4 ---- arch/arm/configs/exynos_defconfig | 1 - arch/arm/configs/milbeaut_m10v_defconfig | 1 - arch/arm/configs/multi_v7_defconfig | 1 - arch/arm/configs/omap2plus_defconfig | 1 - arch/arm/configs/pxa_defconfig | 1 - arch/arm/crypto/Kconfig | 11 ++--------- arch/arm64/Kconfig | 3 --- arch/arm64/configs/defconfig | 1 - arch/arm64/crypto/Kconfig | 11 ++--------- crypto/Kconfig | 6 ++++++ drivers/crypto/Kconfig | 1 - drivers/net/Kconfig | 2 -- 13 files changed, 10 insertions(+), 34 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 87badeae3181..048a4354c213 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1921,8 +1921,4 @@ config ARCH_HIBERNATION_POSSIBLE endmenu -if CRYPTO -source "arch/arm/crypto/Kconfig" -endif - source "arch/arm/Kconfig.assembler" diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index 1ce74f46e114..9d7f87cfe33a 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -32,7 +32,6 @@ CONFIG_KERNEL_MODE_NEON=y CONFIG_PM_DEBUG=y CONFIG_PM_ADVANCED_DEBUG=y CONFIG_ENERGY_MODEL=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 58810e98de3d..10503747924e 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -44,7 +44,6 @@ CONFIG_ARM_CPUIDLE=y CONFIG_VFP=y CONFIG_NEON=y CONFIG_KERNEL_MODE_NEON=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m CONFIG_CRYPTO_SHA2_ARM_CE=m diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index 12b35008571f..0b67ad28aa76 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -132,7 +132,6 @@ CONFIG_ARM_EXYNOS_CPUIDLE=y CONFIG_ARM_TEGRA_CPUIDLE=y CONFIG_ARM_QCOM_SPM_CPUIDLE=y CONFIG_KERNEL_MODE_NEON=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m CONFIG_CRYPTO_SHA2_ARM_CE=m diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index 99d015cf8919..e52e2dee4415 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -53,7 +53,6 @@ CONFIG_CPU_IDLE=y CONFIG_ARM_CPUIDLE=y CONFIG_KERNEL_MODE_NEON=y CONFIG_PM_DEBUG=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index 104a45722799..5a2c5358bbd9 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -34,7 +34,6 @@ CONFIG_CPUFREQ_DT=m CONFIG_ARM_PXA2xx_CPUFREQ=m CONFIG_CPU_IDLE=y CONFIG_ARM_CPUIDLE=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM=m CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 149a5bd6b88c..98b680da0495 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -1,13 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -menuconfig ARM_CRYPTO - bool "ARM Accelerated Cryptographic Algorithms" - depends on ARM - help - Say Y here to choose from a selection of cryptographic algorithms - implemented using ARM specific CPU features or instructions. - -if ARM_CRYPTO +menu "Accelerated Cryptographic Algorithms for CPU (arm)" config CRYPTO_SHA1_ARM tristate "SHA1 digest algorithm (ARM-asm)" @@ -170,4 +163,4 @@ config CRYPTO_CURVE25519_NEON select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_ARCH_HAVE_LIB_CURVE25519 -endif +endmenu diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 571cc234d0b3..91bf8e581dfe 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2215,6 +2215,3 @@ source "drivers/acpi/Kconfig" source "arch/arm64/kvm/Kconfig" -if CRYPTO -source "arch/arm64/crypto/Kconfig" -endif # CRYPTO diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index d5b2d2dd4904..e7a87f63d9d5 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -109,7 +109,6 @@ CONFIG_ACPI_APEI_MEMORY_FAILURE=y CONFIG_ACPI_APEI_EINJ=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=y -CONFIG_ARM64_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM64_CE=y CONFIG_CRYPTO_SHA2_ARM64_CE=y CONFIG_CRYPTO_SHA512_ARM64_CE=m diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 60db5bb2ddda..536511a5db56 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -1,13 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -menuconfig ARM64_CRYPTO - bool "ARM64 Accelerated Cryptographic Algorithms" - depends on ARM64 - help - Say Y here to choose from a selection of cryptographic algorithms - implemented using ARM64 specific CPU features or instructions. - -if ARM64_CRYPTO +menu "Accelerated Cryptographic Algorithms for CPU (arm64)" config CRYPTO_SHA256_ARM64 tristate "SHA-224/SHA-256 digest algorithm for arm64" @@ -138,4 +131,4 @@ config CRYPTO_AES_ARM64_BS select CRYPTO_AES_ARM64_NEON_BLK select CRYPTO_LIB_AES -endif +endmenu diff --git a/crypto/Kconfig b/crypto/Kconfig index c249fdacba66..0349b27075ab 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1292,6 +1292,12 @@ config CRYPTO_STATS config CRYPTO_HASH_INFO bool +if ARM +source "arch/arm/crypto/Kconfig" +endif +if ARM64 +source "arch/arm64/crypto/Kconfig" +endif if MIPS source "arch/mips/crypto/Kconfig" endif diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 72c58b21ba62..55e75fbb658e 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -802,7 +802,6 @@ source "drivers/crypto/amlogic/Kconfig" config CRYPTO_DEV_SA2UL tristate "Support for TI security accelerator" depends on ARCH_K3 || COMPILE_TEST - select ARM64_CRYPTO select CRYPTO_AES select CRYPTO_ALGAPI select CRYPTO_AUTHENC diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index 94c889802566..0e41d2295073 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -85,8 +85,6 @@ config WIREGUARD select CRYPTO_POLY1305_X86_64 if X86 && 64BIT select CRYPTO_BLAKE2S_X86 if X86 && 64BIT select CRYPTO_CURVE25519_X86 if X86 && 64BIT - select ARM_CRYPTO if ARM - select ARM64_CRYPTO if ARM64 select CRYPTO_CHACHA20_NEON if ARM || (ARM64 && KERNEL_MODE_NEON) select CRYPTO_POLY1305_NEON if ARM64 && KERNEL_MODE_NEON select CRYPTO_POLY1305_ARM if ARM -- cgit From 9e5647eb06529de4058c25e681fc36d00465927f Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:42 -0500 Subject: crypto: Kconfig - sort the arm64 entries Sort the arm64 entries so all like entries are together. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 107 +++++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 536511a5db56..c5d42f62d8bb 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -2,13 +2,24 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm64)" -config CRYPTO_SHA256_ARM64 - tristate "SHA-224/SHA-256 digest algorithm for arm64" +config CRYPTO_GHASH_ARM64_CE + tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" + depends on KERNEL_MODE_NEON select CRYPTO_HASH + select CRYPTO_GF128MUL + select CRYPTO_LIB_AES + select CRYPTO_AEAD -config CRYPTO_SHA512_ARM64 - tristate "SHA-384/SHA-512 digest algorithm for arm64" +config CRYPTO_NHPOLY1305_NEON + tristate "NHPoly1305 hash function using NEON instructions (for Adiantum)" + depends on KERNEL_MODE_NEON + select CRYPTO_NHPOLY1305 + +config CRYPTO_POLY1305_NEON + tristate "Poly1305 hash function using scalar or NEON instructions" + depends on KERNEL_MODE_NEON select CRYPTO_HASH + select CRYPTO_ARCH_HAVE_LIB_POLY1305 config CRYPTO_SHA1_ARM64_CE tristate "SHA-1 digest algorithm (ARMv8 Crypto Extensions)" @@ -16,12 +27,20 @@ config CRYPTO_SHA1_ARM64_CE select CRYPTO_HASH select CRYPTO_SHA1 +config CRYPTO_SHA256_ARM64 + tristate "SHA-224/SHA-256 digest algorithm for arm64" + select CRYPTO_HASH + config CRYPTO_SHA2_ARM64_CE tristate "SHA-224/SHA-256 digest algorithm (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA256_ARM64 +config CRYPTO_SHA512_ARM64 + tristate "SHA-384/SHA-512 digest algorithm for arm64" + select CRYPTO_HASH + config CRYPTO_SHA512_ARM64_CE tristate "SHA-384/SHA-512 digest algorithm (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON @@ -40,42 +59,11 @@ config CRYPTO_SM3_ARM64_CE select CRYPTO_HASH select CRYPTO_SM3 -config CRYPTO_SM4_ARM64_CE - tristate "SM4 symmetric cipher (ARMv8.2 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_ALGAPI - select CRYPTO_SM4 - -config CRYPTO_SM4_ARM64_CE_BLK - tristate "SM4 in ECB/CBC/CFB/CTR modes using ARMv8 Crypto Extensions" - depends on KERNEL_MODE_NEON - select CRYPTO_SKCIPHER - select CRYPTO_SM4 - -config CRYPTO_SM4_ARM64_NEON_BLK - tristate "SM4 in ECB/CBC/CFB/CTR modes using NEON instructions" - depends on KERNEL_MODE_NEON - select CRYPTO_SKCIPHER - select CRYPTO_SM4 - -config CRYPTO_GHASH_ARM64_CE - tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_GF128MUL - select CRYPTO_LIB_AES - select CRYPTO_AEAD - config CRYPTO_POLYVAL_ARM64_CE tristate "POLYVAL using ARMv8 Crypto Extensions (for HCTR2)" depends on KERNEL_MODE_NEON select CRYPTO_POLYVAL -config CRYPTO_CRCT10DIF_ARM64_CE - tristate "CRCT10DIF digest algorithm using PMULL instructions" - depends on KERNEL_MODE_NEON && CRC_T10DIF - select CRYPTO_HASH - config CRYPTO_AES_ARM64 tristate "AES core cipher using scalar instructions" select CRYPTO_AES @@ -86,14 +74,6 @@ config CRYPTO_AES_ARM64_CE select CRYPTO_ALGAPI select CRYPTO_LIB_AES -config CRYPTO_AES_ARM64_CE_CCM - tristate "AES in CCM mode using ARMv8 Crypto Extensions" - depends on ARM64 && KERNEL_MODE_NEON - select CRYPTO_ALGAPI - select CRYPTO_AES_ARM64_CE - select CRYPTO_AEAD - select CRYPTO_LIB_AES - config CRYPTO_AES_ARM64_CE_BLK tristate "AES in ECB/CBC/CTR/XTS/XCTR modes using ARMv8 Crypto Extensions" depends on KERNEL_MODE_NEON @@ -113,22 +93,43 @@ config CRYPTO_CHACHA20_NEON select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA -config CRYPTO_POLY1305_NEON - tristate "Poly1305 hash function using scalar or NEON instructions" +config CRYPTO_AES_ARM64_BS + tristate "AES in ECB/CBC/CTR/XTS modes using bit-sliced NEON algorithm" depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_ARCH_HAVE_LIB_POLY1305 + select CRYPTO_SKCIPHER + select CRYPTO_AES_ARM64_NEON_BLK + select CRYPTO_LIB_AES -config CRYPTO_NHPOLY1305_NEON - tristate "NHPoly1305 hash function using NEON instructions (for Adiantum)" +config CRYPTO_SM4_ARM64_CE + tristate "SM4 symmetric cipher (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON - select CRYPTO_NHPOLY1305 + select CRYPTO_ALGAPI + select CRYPTO_SM4 -config CRYPTO_AES_ARM64_BS - tristate "AES in ECB/CBC/CTR/XTS modes using bit-sliced NEON algorithm" +config CRYPTO_SM4_ARM64_CE_BLK + tristate "SM4 in ECB/CBC/CFB/CTR modes using ARMv8 Crypto Extensions" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER - select CRYPTO_AES_ARM64_NEON_BLK + select CRYPTO_SM4 + +config CRYPTO_SM4_ARM64_NEON_BLK + tristate "SM4 in ECB/CBC/CFB/CTR modes using NEON instructions" + depends on KERNEL_MODE_NEON + select CRYPTO_SKCIPHER + select CRYPTO_SM4 + +config CRYPTO_AES_ARM64_CE_CCM + tristate "AES in CCM mode using ARMv8 Crypto Extensions" + depends on ARM64 && KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_AES_ARM64_CE + select CRYPTO_AEAD select CRYPTO_LIB_AES +config CRYPTO_CRCT10DIF_ARM64_CE + tristate "CRCT10DIF digest algorithm using PMULL instructions" + depends on KERNEL_MODE_NEON && CRC_T10DIF + select CRYPTO_HASH + endmenu + -- cgit From 4a95d4ae98b1610ce1b1df2a36e8955c3f46623e Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:43 -0500 Subject: crypto: Kconfig - sort the arm entries Sort the arm entries so all like entries are together. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 111 ++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 55 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 98b680da0495..d73d19971b87 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -2,6 +2,53 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm)" +config CRYPTO_CURVE25519_NEON + tristate "NEON accelerated Curve25519 scalar multiplication library" + depends on KERNEL_MODE_NEON + select CRYPTO_LIB_CURVE25519_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + +config CRYPTO_GHASH_ARM_CE + tristate "PMULL-accelerated GHASH using NEON/ARMv8 Crypto Extensions" + depends on KERNEL_MODE_NEON + select CRYPTO_HASH + select CRYPTO_CRYPTD + select CRYPTO_GF128MUL + help + Use an implementation of GHASH (used by the GCM AEAD chaining mode) + that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) + that is part of the ARMv8 Crypto Extensions, or a slower variant that + uses the vmull.p8 instruction that is part of the basic NEON ISA. + +config CRYPTO_NHPOLY1305_NEON + tristate "NEON accelerated NHPoly1305 hash function (for Adiantum)" + depends on KERNEL_MODE_NEON + select CRYPTO_NHPOLY1305 + +config CRYPTO_POLY1305_ARM + tristate "Accelerated scalar and SIMD Poly1305 hash implementations" + select CRYPTO_HASH + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + +config CRYPTO_BLAKE2S_ARM + bool "BLAKE2s digest algorithm (ARM)" + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s digest algorithm optimized with ARM scalar instructions. This + is faster than the generic implementations of BLAKE2s and BLAKE2b, but + slower than the NEON implementation of BLAKE2b. (There is no NEON + implementation of BLAKE2s, since NEON doesn't really help with it.) + +config CRYPTO_BLAKE2B_NEON + tristate "BLAKE2b digest algorithm (ARM NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_BLAKE2B + help + BLAKE2b digest algorithm optimized with ARM NEON instructions. + On ARM processors that have NEON support but not the ARMv8 + Crypto Extensions, typically this BLAKE2b implementation is + much faster than SHA-2 and slightly faster than SHA-1. + config CRYPTO_SHA1_ARM tristate "SHA1 digest algorithm (ARM-asm)" select CRYPTO_SHA1 @@ -55,25 +102,6 @@ config CRYPTO_SHA512_ARM SHA-512 secure hash standard (DFIPS 180-2) implemented using optimized ARM assembler and NEON, when available. -config CRYPTO_BLAKE2S_ARM - bool "BLAKE2s digest algorithm (ARM)" - select CRYPTO_ARCH_HAVE_LIB_BLAKE2S - help - BLAKE2s digest algorithm optimized with ARM scalar instructions. This - is faster than the generic implementations of BLAKE2s and BLAKE2b, but - slower than the NEON implementation of BLAKE2b. (There is no NEON - implementation of BLAKE2s, since NEON doesn't really help with it.) - -config CRYPTO_BLAKE2B_NEON - tristate "BLAKE2b digest algorithm (ARM NEON)" - depends on KERNEL_MODE_NEON - select CRYPTO_BLAKE2B - help - BLAKE2b digest algorithm optimized with ARM NEON instructions. - On ARM processors that have NEON support but not the ARMv8 - Crypto Extensions, typically this BLAKE2b implementation is - much faster than SHA-2 and slightly faster than SHA-1. - config CRYPTO_AES_ARM tristate "Scalar AES cipher for ARM" select CRYPTO_ALGAPI @@ -118,49 +146,22 @@ config CRYPTO_AES_ARM_CE Use an implementation of AES in CBC, CTR and XTS modes that uses ARMv8 Crypto Extensions -config CRYPTO_GHASH_ARM_CE - tristate "PMULL-accelerated GHASH using NEON/ARMv8 Crypto Extensions" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_CRYPTD - select CRYPTO_GF128MUL - help - Use an implementation of GHASH (used by the GCM AEAD chaining mode) - that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) - that is part of the ARMv8 Crypto Extensions, or a slower variant that - uses the vmull.p8 instruction that is part of the basic NEON ISA. - -config CRYPTO_CRCT10DIF_ARM_CE - tristate "CRCT10DIF digest algorithm using PMULL instructions" - depends on KERNEL_MODE_NEON - depends on CRC_T10DIF - select CRYPTO_HASH - -config CRYPTO_CRC32_ARM_CE - tristate "CRC32(C) digest algorithm using CRC and/or PMULL instructions" - depends on KERNEL_MODE_NEON - depends on CRC32 - select CRYPTO_HASH - config CRYPTO_CHACHA20_NEON tristate "NEON and scalar accelerated ChaCha stream cipher algorithms" select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA -config CRYPTO_POLY1305_ARM - tristate "Accelerated scalar and SIMD Poly1305 hash implementations" - select CRYPTO_HASH - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - -config CRYPTO_NHPOLY1305_NEON - tristate "NEON accelerated NHPoly1305 hash function (for Adiantum)" +config CRYPTO_CRC32_ARM_CE + tristate "CRC32(C) digest algorithm using CRC and/or PMULL instructions" depends on KERNEL_MODE_NEON - select CRYPTO_NHPOLY1305 + depends on CRC32 + select CRYPTO_HASH -config CRYPTO_CURVE25519_NEON - tristate "NEON accelerated Curve25519 scalar multiplication library" +config CRYPTO_CRCT10DIF_ARM_CE + tristate "CRCT10DIF digest algorithm using PMULL instructions" depends on KERNEL_MODE_NEON - select CRYPTO_LIB_CURVE25519_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + depends on CRC_T10DIF + select CRYPTO_HASH endmenu + -- cgit From f1f142ad434883616c313bc93b9f443d496293db Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:44 -0500 Subject: crypto: Kconfig - add submenus Convert each comment section into a submenu: Cryptographic API Crypto core or helper Public-key cryptography Block ciphers Length-preserving ciphers and modes AEAD (authenticated encryption with associated data) ciphers Hashes, digests, and MACs CRCs (cyclic redundancy checks) Compression Random number generation Userspace interface That helps find entries (e.g., searching for a name like SHA512 doesn't just report the location is Main menu -> Cryptography API, leaving you to wade through 153 entries; it points you to the Digests page). Move entries so they fall into the correct submenus and are better sorted. Suggested-by: Eric Biggers Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- crypto/Kconfig | 1225 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 623 insertions(+), 602 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 0349b27075ab..e2e364cfa93e 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -21,7 +21,7 @@ menuconfig CRYPTO if CRYPTO -comment "Crypto core or helper" +menu "Crypto core or helper" config CRYPTO_FIPS bool "FIPS 200 compliance" @@ -235,7 +235,9 @@ config CRYPTO_SIMD config CRYPTO_ENGINE tristate -comment "Public-key cryptography" +endmenu + +menu "Public-key cryptography" config CRYPTO_RSA tristate "RSA algorithm" @@ -316,189 +318,265 @@ config CRYPTO_CURVE25519 select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC -comment "Authenticated Encryption with Associated Data" +endmenu -config CRYPTO_CCM - tristate "CCM support" - select CRYPTO_CTR - select CRYPTO_HASH - select CRYPTO_AEAD - select CRYPTO_MANAGER - help - Support for Counter with CBC MAC. Required for IPsec. +menu "Block ciphers" -config CRYPTO_GCM - tristate "GCM/GMAC support" - select CRYPTO_CTR - select CRYPTO_AEAD - select CRYPTO_GHASH - select CRYPTO_NULL - select CRYPTO_MANAGER +config CRYPTO_AES + tristate "AES cipher algorithms" + select CRYPTO_ALGAPI + select CRYPTO_LIB_AES help - Support for Galois/Counter Mode (GCM) and Galois Message - Authentication Code (GMAC). Required for IPSec. + AES cipher algorithms (FIPS-197). AES uses the Rijndael + algorithm. -config CRYPTO_CHACHA20POLY1305 - tristate "ChaCha20-Poly1305 AEAD support" - select CRYPTO_CHACHA20 - select CRYPTO_POLY1305 - select CRYPTO_AEAD - select CRYPTO_MANAGER - help - ChaCha20-Poly1305 AEAD support, RFC7539. + Rijndael appears to be consistently a very good performer in + both hardware and software across a wide range of computing + environments regardless of its use in feedback or non-feedback + modes. Its key setup time is excellent, and its key agility is + good. Rijndael's very low memory requirements make it very well + suited for restricted-space environments, in which it also + demonstrates excellent performance. Rijndael's operations are + among the easiest to defend against power and timing attacks. - Support for the AEAD wrapper using the ChaCha20 stream cipher combined - with the Poly1305 authenticator. It is defined in RFC7539 for use in - IETF protocols. + The AES specifies three key sizes: 128, 192 and 256 bits -config CRYPTO_AEGIS128 - tristate "AEGIS-128 AEAD algorithm" - select CRYPTO_AEAD - select CRYPTO_AES # for AES S-box tables + See for more information. + +config CRYPTO_AES_TI + tristate "Fixed time AES cipher" + select CRYPTO_ALGAPI + select CRYPTO_LIB_AES help - Support for the AEGIS-128 dedicated AEAD algorithm. + This is a generic implementation of AES that attempts to eliminate + data dependent latencies as much as possible without affecting + performance too much. It is intended for use by the generic CCM + and GCM drivers, and other CTR or CMAC/XCBC based modes that rely + solely on encryption (although decryption is supported as well, but + with a more dramatic performance hit) -config CRYPTO_AEGIS128_SIMD - bool "Support SIMD acceleration for AEGIS-128" - depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON) - default y + Instead of using 16 lookup tables of 1 KB each, (8 for encryption and + 8 for decryption), this implementation only uses just two S-boxes of + 256 bytes each, and attempts to eliminate data dependent latencies by + prefetching the entire table into the cache at the start of each + block. Interrupts are also disabled to avoid races where cachelines + are evicted when the CPU is interrupted to do something else. -config CRYPTO_SEQIV - tristate "Sequence Number IV Generator" - select CRYPTO_AEAD - select CRYPTO_SKCIPHER - select CRYPTO_NULL - select CRYPTO_RNG_DEFAULT - select CRYPTO_MANAGER +config CRYPTO_ANUBIS + tristate "Anubis cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_ALGAPI help - This IV generator generates an IV based on a sequence number by - xoring it with a salt. This algorithm is mainly useful for CTR + Anubis cipher algorithm. -config CRYPTO_ECHAINIV - tristate "Encrypted Chain IV Generator" - select CRYPTO_AEAD - select CRYPTO_NULL - select CRYPTO_RNG_DEFAULT - select CRYPTO_MANAGER - help - This IV generator generates an IV based on the encryption of - a sequence number xored with a salt. This is the default - algorithm for CBC. + Anubis is a variable key length cipher which can use keys from + 128 bits to 320 bits in length. It was evaluated as a entrant + in the NESSIE competition. -comment "Block modes" + See also: + + -config CRYPTO_CBC - tristate "CBC support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_ARIA + tristate "ARIA cipher algorithm" + select CRYPTO_ALGAPI help - CBC: Cipher Block Chaining mode - This block cipher algorithm is required for IPSec. + ARIA cipher algorithm (RFC5794). -config CRYPTO_CFB - tristate "CFB support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - help - CFB: Cipher FeedBack mode - This block cipher algorithm is required for TPM2 Cryptography. + ARIA is a standard encryption algorithm of the Republic of Korea. + The ARIA specifies three key sizes and rounds. + 128-bit: 12 rounds. + 192-bit: 14 rounds. + 256-bit: 16 rounds. -config CRYPTO_CTR - tristate "CTR support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - help - CTR: Counter mode - This block cipher algorithm is required for IPSec. + See also: + -config CRYPTO_CTS - tristate "CTS support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_BLOWFISH + tristate "Blowfish cipher algorithm" + select CRYPTO_ALGAPI + select CRYPTO_BLOWFISH_COMMON help - CTS: Cipher Text Stealing - This is the Cipher Text Stealing mode as described by - Section 8 of rfc2040 and referenced by rfc3962 - (rfc3962 includes errata information in its Appendix A) or - CBC-CS3 as defined by NIST in Sp800-38A addendum from Oct 2010. - This mode is required for Kerberos gss mechanism support - for AES encryption. + Blowfish cipher algorithm, by Bruce Schneier. - See: https://csrc.nist.gov/publications/detail/sp/800-38a/addendum/final + This is a variable key length cipher which can use keys from 32 + bits to 448 bits in length. It's fast, simple and specifically + designed for use on "large microprocessors". -config CRYPTO_ECB - tristate "ECB support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER + See also: + + +config CRYPTO_BLOWFISH_COMMON + tristate help - ECB: Electronic CodeBook mode - This is the simplest block cipher algorithm. It simply encrypts - the input block by block. + Common parts of the Blowfish cipher algorithm shared by the + generic c and the assembler implementations. -config CRYPTO_LRW - tristate "LRW support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - select CRYPTO_GF128MUL - select CRYPTO_ECB + See also: + + +config CRYPTO_CAMELLIA + tristate "Camellia cipher algorithms" + select CRYPTO_ALGAPI help - LRW: Liskov Rivest Wagner, a tweakable, non malleable, non movable - narrow block cipher mode for dm-crypt. Use it with cipher - specification string aes-lrw-benbi, the key must be 256, 320 or 384. - The first 128, 192 or 256 bits in the key are used for AES and the - rest is used to tie each cipher block to its logical position. + Camellia cipher algorithms module. -config CRYPTO_OFB - tristate "OFB support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER + Camellia is a symmetric key block cipher developed jointly + at NTT and Mitsubishi Electric Corporation. + + The Camellia specifies three key sizes: 128, 192 and 256 bits. + + See also: + + +config CRYPTO_CAST_COMMON + tristate help - OFB: the Output Feedback mode makes a block cipher into a synchronous - stream cipher. It generates keystream blocks, which are then XORed - with the plaintext blocks to get the ciphertext. Flipping a bit in the - ciphertext produces a flipped bit in the plaintext at the same - location. This property allows many error correcting codes to function - normally even when applied before encryption. + Common parts of the CAST cipher algorithms shared by the + generic c and the assembler implementations. -config CRYPTO_PCBC - tristate "PCBC support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_CAST5 + tristate "CAST5 (CAST-128) cipher algorithm" + select CRYPTO_ALGAPI + select CRYPTO_CAST_COMMON help - PCBC: Propagating Cipher Block Chaining mode - This block cipher algorithm is required for RxRPC. + The CAST5 encryption algorithm (synonymous with CAST-128) is + described in RFC2144. -config CRYPTO_XCTR - tristate - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_CAST6 + tristate "CAST6 (CAST-256) cipher algorithm" + select CRYPTO_ALGAPI + select CRYPTO_CAST_COMMON help - XCTR: XOR Counter mode. This blockcipher mode is a variant of CTR mode - using XORs and little-endian addition rather than big-endian arithmetic. - XCTR mode is used to implement HCTR2. + The CAST6 encryption algorithm (synonymous with CAST-256) is + described in RFC2612. -config CRYPTO_XTS - tristate "XTS support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - select CRYPTO_ECB +config CRYPTO_DES + tristate "DES and Triple DES EDE cipher algorithms" + select CRYPTO_ALGAPI + select CRYPTO_LIB_DES help - XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain, - key size 256, 384 or 512 bits. This implementation currently - can't handle a sectorsize which is not a multiple of 16 bytes. + DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). -config CRYPTO_KEYWRAP - tristate "Key wrapping support" +config CRYPTO_FCRYPT + tristate "FCrypt cipher algorithm" + select CRYPTO_ALGAPI select CRYPTO_SKCIPHER - select CRYPTO_MANAGER help - Support for key wrapping (NIST SP800-38F / RFC3394) without - padding. + FCrypt algorithm used by RxRPC. -config CRYPTO_NHPOLY1305 - tristate - select CRYPTO_HASH - select CRYPTO_LIB_POLY1305_GENERIC +config CRYPTO_KHAZAD + tristate "Khazad cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_ALGAPI + help + Khazad cipher algorithm. + + Khazad was a finalist in the initial NESSIE competition. It is + an algorithm optimized for 64-bit processors with good performance + on 32-bit processors. Khazad uses an 128 bit key size. + + See also: + + +config CRYPTO_SEED + tristate "SEED cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_ALGAPI + help + SEED cipher algorithm (RFC4269). + + SEED is a 128-bit symmetric key block cipher that has been + developed by KISA (Korea Information Security Agency) as a + national standard encryption algorithm of the Republic of Korea. + It is a 16 round block cipher with the key size of 128 bit. + + See also: + + +config CRYPTO_SERPENT + tristate "Serpent cipher algorithm" + select CRYPTO_ALGAPI + help + Serpent cipher algorithm, by Anderson, Biham & Knudsen. + + Keys are allowed to be from 0 to 256 bits in length, in steps + of 8 bits. + + See also: + + +config CRYPTO_SM4 + tristate + +config CRYPTO_SM4_GENERIC + tristate "SM4 cipher algorithm" + select CRYPTO_ALGAPI + select CRYPTO_SM4 + help + SM4 cipher algorithms (OSCCA GB/T 32907-2016). + + SM4 (GBT.32907-2016) is a cryptographic standard issued by the + Organization of State Commercial Administration of China (OSCCA) + as an authorized cryptographic algorithms for the use within China. + + SMS4 was originally created for use in protecting wireless + networks, and is mandated in the Chinese National Standard for + Wireless LAN WAPI (Wired Authentication and Privacy Infrastructure) + (GB.15629.11-2003). + + The latest SM4 standard (GBT.32907-2016) was proposed by OSCCA and + standardized through TC 260 of the Standardization Administration + of the People's Republic of China (SAC). + + The input, output, and key of SMS4 are each 128 bits. + + See also: + + If unsure, say N. + +config CRYPTO_TEA + tristate "TEA, XTEA and XETA cipher algorithms" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_ALGAPI + help + TEA cipher algorithm. + + Tiny Encryption Algorithm is a simple cipher that uses + many rounds for security. It is very fast and uses + little memory. + + Xtendend Tiny Encryption Algorithm is a modification to + the TEA algorithm to address a potential key weakness + in the TEA algorithm. + + Xtendend Encryption Tiny Algorithm is a mis-implementation + of the XTEA algorithm for compatibility purposes. + +config CRYPTO_TWOFISH + tristate "Twofish cipher algorithm" + select CRYPTO_ALGAPI + select CRYPTO_TWOFISH_COMMON + help + Twofish cipher algorithm. + + Twofish was submitted as an AES (Advanced Encryption Standard) + candidate cipher by researchers at CounterPane Systems. It is a + 16 round block cipher supporting key sizes of 128, 192, and 256 + bits. + + See also: + + +config CRYPTO_TWOFISH_COMMON + tristate + help + Common parts of the Twofish cipher algorithm shared by the + generic c and the assembler implementations. + +endmenu + +menu "Length-preserving ciphers and modes" config CRYPTO_ADIANTUM tristate "Adiantum support" @@ -524,580 +602,516 @@ config CRYPTO_ADIANTUM If unsure, say N. -config CRYPTO_HCTR2 - tristate "HCTR2 support" - select CRYPTO_XCTR - select CRYPTO_POLYVAL - select CRYPTO_MANAGER +config CRYPTO_ARC4 + tristate "ARC4 cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_SKCIPHER + select CRYPTO_LIB_ARC4 help - HCTR2 is a length-preserving encryption mode for storage encryption that - is efficient on processors with instructions to accelerate AES and - carryless multiplication, e.g. x86 processors with AES-NI and CLMUL, and - ARM processors with the ARMv8 crypto extensions. + ARC4 cipher algorithm. -config CRYPTO_ESSIV - tristate "ESSIV support for block encryption" - select CRYPTO_AUTHENC + ARC4 is a stream cipher using keys ranging from 8 bits to 2048 + bits in length. This algorithm is required for driver-based + WEP, but it should not be for other purposes because of the + weakness of the algorithm. + +config CRYPTO_CHACHA20 + tristate "ChaCha stream cipher algorithms" + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_SKCIPHER help - Encrypted salt-sector initialization vector (ESSIV) is an IV - generation method that is used in some cases by fscrypt and/or - dm-crypt. It uses the hash of the block encryption key as the - symmetric key for a block encryption pass applied to the input - IV, making low entropy IV sources more suitable for block - encryption. + The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms. - This driver implements a crypto API template that can be - instantiated either as an skcipher or as an AEAD (depending on the - type of the first template argument), and which defers encryption - and decryption requests to the encapsulated cipher after applying - ESSIV to the input IV. Note that in the AEAD case, it is assumed - that the keys are presented in the same format used by the authenc - template, and that the IV appears at the end of the authenticated - associated data (AAD) region (which is how dm-crypt uses it.) + ChaCha20 is a 256-bit high-speed stream cipher designed by Daniel J. + Bernstein and further specified in RFC7539 for use in IETF protocols. + This is the portable C implementation of ChaCha20. See also: + - Note that the use of ESSIV is not recommended for new deployments, - and so this only needs to be enabled when interoperability with - existing encrypted volumes of filesystems is required, or when - building for a particular system that requires it (e.g., when - the SoC in question has accelerated CBC but not XTS, making CBC - combined with ESSIV the only feasible mode for h/w accelerated - block encryption) + XChaCha20 is the application of the XSalsa20 construction to ChaCha20 + rather than to Salsa20. XChaCha20 extends ChaCha20's nonce length + from 64 bits (or 96 bits using the RFC7539 convention) to 192 bits, + while provably retaining ChaCha20's security. See also: + -comment "Hash modes" + XChaCha12 is XChaCha20 reduced to 12 rounds, with correspondingly + reduced security margin but increased performance. It can be needed + in some performance-sensitive scenarios. -config CRYPTO_CMAC - tristate "CMAC support" - select CRYPTO_HASH +config CRYPTO_CBC + tristate "CBC support" + select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - Cipher-based Message Authentication Code (CMAC) specified by - The National Institute of Standards and Technology (NIST). - - https://tools.ietf.org/html/rfc4493 - http://csrc.nist.gov/publications/nistpubs/800-38B/SP_800-38B.pdf + CBC: Cipher Block Chaining mode + This block cipher algorithm is required for IPSec. -config CRYPTO_HMAC - tristate "HMAC support" - select CRYPTO_HASH +config CRYPTO_CFB + tristate "CFB support" + select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - HMAC: Keyed-Hashing for Message Authentication (RFC2104). - This is required for IPSec. + CFB: Cipher FeedBack mode + This block cipher algorithm is required for TPM2 Cryptography. -config CRYPTO_XCBC - tristate "XCBC support" - select CRYPTO_HASH +config CRYPTO_CTR + tristate "CTR support" + select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - XCBC: Keyed-Hashing with encryption algorithm - https://www.ietf.org/rfc/rfc3566.txt - http://csrc.nist.gov/encryption/modes/proposedmodes/ - xcbc-mac/xcbc-mac-spec.pdf + CTR: Counter mode + This block cipher algorithm is required for IPSec. -config CRYPTO_VMAC - tristate "VMAC support" - select CRYPTO_HASH +config CRYPTO_CTS + tristate "CTS support" + select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - VMAC is a message authentication algorithm designed for - very high speed on 64-bit architectures. - - See also: - + CTS: Cipher Text Stealing + This is the Cipher Text Stealing mode as described by + Section 8 of rfc2040 and referenced by rfc3962 + (rfc3962 includes errata information in its Appendix A) or + CBC-CS3 as defined by NIST in Sp800-38A addendum from Oct 2010. + This mode is required for Kerberos gss mechanism support + for AES encryption. -comment "Digest" + See: https://csrc.nist.gov/publications/detail/sp/800-38a/addendum/final -config CRYPTO_CRC32C - tristate "CRC32c CRC algorithm" - select CRYPTO_HASH - select CRC32 +config CRYPTO_ECB + tristate "ECB support" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - Castagnoli, et al Cyclic Redundancy-Check Algorithm. Used - by iSCSI for header and data digests and by others. - See Castagnoli93. Module will be crc32c. + ECB: Electronic CodeBook mode + This is the simplest block cipher algorithm. It simply encrypts + the input block by block. -config CRYPTO_CRC32 - tristate "CRC32 CRC algorithm" - select CRYPTO_HASH - select CRC32 +config CRYPTO_HCTR2 + tristate "HCTR2 support" + select CRYPTO_XCTR + select CRYPTO_POLYVAL + select CRYPTO_MANAGER help - CRC-32-IEEE 802.3 cyclic redundancy-check algorithm. - Shash crypto api wrappers to crc32_le function. + HCTR2 is a length-preserving encryption mode for storage encryption that + is efficient on processors with instructions to accelerate AES and + carryless multiplication, e.g. x86 processors with AES-NI and CLMUL, and + ARM processors with the ARMv8 crypto extensions. -config CRYPTO_XXHASH - tristate "xxHash hash algorithm" - select CRYPTO_HASH - select XXHASH - help - xxHash non-cryptographic hash algorithm. Extremely fast, working at - speeds close to RAM limits. - -config CRYPTO_BLAKE2B - tristate "BLAKE2b digest algorithm" - select CRYPTO_HASH - help - Implementation of cryptographic hash function BLAKE2b (or just BLAKE2), - optimized for 64bit platforms and can produce digests of any size - between 1 to 64. The keyed hash is also implemented. - - This module provides the following algorithms: - - - blake2b-160 - - blake2b-256 - - blake2b-384 - - blake2b-512 - - See https://blake2.net for further information. - -config CRYPTO_CRCT10DIF - tristate "CRCT10DIF algorithm" - select CRYPTO_HASH - help - CRC T10 Data Integrity Field computation is being cast as - a crypto transform. This allows for faster crc t10 diff - transforms to be used if they are available. - -config CRYPTO_CRC64_ROCKSOFT - tristate "Rocksoft Model CRC64 algorithm" - depends on CRC64 - select CRYPTO_HASH - -config CRYPTO_GHASH - tristate "GHASH hash function" - select CRYPTO_GF128MUL - select CRYPTO_HASH +config CRYPTO_KEYWRAP + tristate "Key wrapping support" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - GHASH is the hash function used in GCM (Galois/Counter Mode). - It is not a general-purpose cryptographic hash function. + Support for key wrapping (NIST SP800-38F / RFC3394) without + padding. -config CRYPTO_POLYVAL - tristate +config CRYPTO_LRW + tristate "LRW support" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER select CRYPTO_GF128MUL - select CRYPTO_HASH + select CRYPTO_ECB help - POLYVAL is the hash function used in HCTR2. It is not a general-purpose - cryptographic hash function. + LRW: Liskov Rivest Wagner, a tweakable, non malleable, non movable + narrow block cipher mode for dm-crypt. Use it with cipher + specification string aes-lrw-benbi, the key must be 256, 320 or 384. + The first 128, 192 or 256 bits in the key are used for AES and the + rest is used to tie each cipher block to its logical position. -config CRYPTO_POLY1305 - tristate "Poly1305 authenticator algorithm" - select CRYPTO_HASH - select CRYPTO_LIB_POLY1305_GENERIC +config CRYPTO_OFB + tristate "OFB support" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - Poly1305 authenticator algorithm, RFC7539. - - Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. - It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use - in IETF protocols. This is the portable C implementation of Poly1305. + OFB: the Output Feedback mode makes a block cipher into a synchronous + stream cipher. It generates keystream blocks, which are then XORed + with the plaintext blocks to get the ciphertext. Flipping a bit in the + ciphertext produces a flipped bit in the plaintext at the same + location. This property allows many error correcting codes to function + normally even when applied before encryption. -config CRYPTO_MD4 - tristate "MD4 digest algorithm" - select CRYPTO_HASH +config CRYPTO_PCBC + tristate "PCBC support" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - MD4 message digest algorithm (RFC1320). + PCBC: Propagating Cipher Block Chaining mode + This block cipher algorithm is required for RxRPC. -config CRYPTO_MD5 - tristate "MD5 digest algorithm" - select CRYPTO_HASH +config CRYPTO_XCTR + tristate + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - MD5 message digest algorithm (RFC1321). + XCTR: XOR Counter mode. This blockcipher mode is a variant of CTR mode + using XORs and little-endian addition rather than big-endian arithmetic. + XCTR mode is used to implement HCTR2. -config CRYPTO_MICHAEL_MIC - tristate "Michael MIC keyed digest algorithm" - select CRYPTO_HASH +config CRYPTO_XTS + tristate "XTS support" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER + select CRYPTO_ECB help - Michael MIC is used for message integrity protection in TKIP - (IEEE 802.11i). This algorithm is required for TKIP, but it - should not be used for other purposes because of the weakness - of the algorithm. + XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain, + key size 256, 384 or 512 bits. This implementation currently + can't handle a sectorsize which is not a multiple of 16 bytes. -config CRYPTO_RMD160 - tristate "RIPEMD-160 digest algorithm" +config CRYPTO_NHPOLY1305 + tristate select CRYPTO_HASH - help - RIPEMD-160 (ISO/IEC 10118-3:2004). - - RIPEMD-160 is a 160-bit cryptographic hash function. It is intended - to be used as a secure replacement for the 128-bit hash functions - MD4, MD5 and its predecessor RIPEMD - (not to be confused with RIPEMD-128). - - It's speed is comparable to SHA1 and there are no known attacks - against RIPEMD-160. + select CRYPTO_LIB_POLY1305_GENERIC - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See +endmenu -config CRYPTO_SHA1 - tristate "SHA1 digest algorithm" - select CRYPTO_HASH - select CRYPTO_LIB_SHA1 - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). +menu "AEAD (authenticated encryption with associated data) ciphers" -config CRYPTO_SHA256 - tristate "SHA224 and SHA256 digest algorithm" - select CRYPTO_HASH - select CRYPTO_LIB_SHA256 +config CRYPTO_AEGIS128 + tristate "AEGIS-128 AEAD algorithm" + select CRYPTO_AEAD + select CRYPTO_AES # for AES S-box tables help - SHA256 secure hash standard (DFIPS 180-2). - - This version of SHA implements a 256 bit hash with 128 bits of - security against collision attacks. + Support for the AEGIS-128 dedicated AEAD algorithm. - This code also includes SHA-224, a 224 bit hash with 112 bits - of security against collision attacks. +config CRYPTO_AEGIS128_SIMD + bool "Support SIMD acceleration for AEGIS-128" + depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON) + default y -config CRYPTO_SHA512 - tristate "SHA384 and SHA512 digest algorithms" - select CRYPTO_HASH +config CRYPTO_CHACHA20POLY1305 + tristate "ChaCha20-Poly1305 AEAD support" + select CRYPTO_CHACHA20 + select CRYPTO_POLY1305 + select CRYPTO_AEAD + select CRYPTO_MANAGER help - SHA512 secure hash standard (DFIPS 180-2). - - This version of SHA implements a 512 bit hash with 256 bits of - security against collision attacks. + ChaCha20-Poly1305 AEAD support, RFC7539. - This code also includes SHA-384, a 384 bit hash with 192 bits - of security against collision attacks. + Support for the AEAD wrapper using the ChaCha20 stream cipher combined + with the Poly1305 authenticator. It is defined in RFC7539 for use in + IETF protocols. -config CRYPTO_SHA3 - tristate "SHA3 digest algorithm" +config CRYPTO_CCM + tristate "CCM support" + select CRYPTO_CTR select CRYPTO_HASH + select CRYPTO_AEAD + select CRYPTO_MANAGER help - SHA-3 secure hash standard (DFIPS 202). It's based on - cryptographic sponge function family called Keccak. - - References: - http://keccak.noekeon.org/ - -config CRYPTO_SM3 - tristate + Support for Counter with CBC MAC. Required for IPsec. -config CRYPTO_SM3_GENERIC - tristate "SM3 digest algorithm" - select CRYPTO_HASH - select CRYPTO_SM3 +config CRYPTO_GCM + tristate "GCM/GMAC support" + select CRYPTO_CTR + select CRYPTO_AEAD + select CRYPTO_GHASH + select CRYPTO_NULL + select CRYPTO_MANAGER help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). - It is part of the Chinese Commercial Cryptography suite. - - References: - http://www.oscca.gov.cn/UpFile/20101222141857786.pdf - https://datatracker.ietf.org/doc/html/draft-shen-sm3-hash + Support for Galois/Counter Mode (GCM) and Galois Message + Authentication Code (GMAC). Required for IPSec. -config CRYPTO_STREEBOG - tristate "Streebog Hash Function" - select CRYPTO_HASH +config CRYPTO_SEQIV + tristate "Sequence Number IV Generator" + select CRYPTO_AEAD + select CRYPTO_SKCIPHER + select CRYPTO_NULL + select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER help - Streebog Hash Function (GOST R 34.11-2012, RFC 6986) is one of the Russian - cryptographic standard algorithms (called GOST algorithms). - This setting enables two hash algorithms with 256 and 512 bits output. - - References: - https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf - https://tools.ietf.org/html/rfc6986 + This IV generator generates an IV based on a sequence number by + xoring it with a salt. This algorithm is mainly useful for CTR -config CRYPTO_WP512 - tristate "Whirlpool digest algorithms" - select CRYPTO_HASH +config CRYPTO_ECHAINIV + tristate "Encrypted Chain IV Generator" + select CRYPTO_AEAD + select CRYPTO_NULL + select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER help - Whirlpool hash algorithm 512, 384 and 256-bit hashes - - Whirlpool-512 is part of the NESSIE cryptographic primitives. - Whirlpool will be part of the ISO/IEC 10118-3:2003(E) standard - - See also: - - -comment "Ciphers" + This IV generator generates an IV based on the encryption of + a sequence number xored with a salt. This is the default + algorithm for CBC. -config CRYPTO_AES - tristate "AES cipher algorithms" - select CRYPTO_ALGAPI - select CRYPTO_LIB_AES +config CRYPTO_ESSIV + tristate "ESSIV support for block encryption" + select CRYPTO_AUTHENC help - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. - - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. + Encrypted salt-sector initialization vector (ESSIV) is an IV + generation method that is used in some cases by fscrypt and/or + dm-crypt. It uses the hash of the block encryption key as the + symmetric key for a block encryption pass applied to the input + IV, making low entropy IV sources more suitable for block + encryption. - The AES specifies three key sizes: 128, 192 and 256 bits + This driver implements a crypto API template that can be + instantiated either as an skcipher or as an AEAD (depending on the + type of the first template argument), and which defers encryption + and decryption requests to the encapsulated cipher after applying + ESSIV to the input IV. Note that in the AEAD case, it is assumed + that the keys are presented in the same format used by the authenc + template, and that the IV appears at the end of the authenticated + associated data (AAD) region (which is how dm-crypt uses it.) - See for more information. + Note that the use of ESSIV is not recommended for new deployments, + and so this only needs to be enabled when interoperability with + existing encrypted volumes of filesystems is required, or when + building for a particular system that requires it (e.g., when + the SoC in question has accelerated CBC but not XTS, making CBC + combined with ESSIV the only feasible mode for h/w accelerated + block encryption) -config CRYPTO_AES_TI - tristate "Fixed time AES cipher" - select CRYPTO_ALGAPI - select CRYPTO_LIB_AES - help - This is a generic implementation of AES that attempts to eliminate - data dependent latencies as much as possible without affecting - performance too much. It is intended for use by the generic CCM - and GCM drivers, and other CTR or CMAC/XCBC based modes that rely - solely on encryption (although decryption is supported as well, but - with a more dramatic performance hit) +endmenu - Instead of using 16 lookup tables of 1 KB each, (8 for encryption and - 8 for decryption), this implementation only uses just two S-boxes of - 256 bytes each, and attempts to eliminate data dependent latencies by - prefetching the entire table into the cache at the start of each - block. Interrupts are also disabled to avoid races where cachelines - are evicted when the CPU is interrupted to do something else. +menu "Hashes, digests, and MACs" -config CRYPTO_ANUBIS - tristate "Anubis cipher algorithm" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI +config CRYPTO_BLAKE2B + tristate "BLAKE2b digest algorithm" + select CRYPTO_HASH help - Anubis cipher algorithm. - - Anubis is a variable key length cipher which can use keys from - 128 bits to 320 bits in length. It was evaluated as a entrant - in the NESSIE competition. + Implementation of cryptographic hash function BLAKE2b (or just BLAKE2), + optimized for 64bit platforms and can produce digests of any size + between 1 to 64. The keyed hash is also implemented. - See also: - - + This module provides the following algorithms: -config CRYPTO_ARC4 - tristate "ARC4 cipher algorithm" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_SKCIPHER - select CRYPTO_LIB_ARC4 - help - ARC4 cipher algorithm. + - blake2b-160 + - blake2b-256 + - blake2b-384 + - blake2b-512 - ARC4 is a stream cipher using keys ranging from 8 bits to 2048 - bits in length. This algorithm is required for driver-based - WEP, but it should not be for other purposes because of the - weakness of the algorithm. + See https://blake2.net for further information. -config CRYPTO_BLOWFISH - tristate "Blowfish cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_BLOWFISH_COMMON +config CRYPTO_CMAC + tristate "CMAC support" + select CRYPTO_HASH + select CRYPTO_MANAGER help - Blowfish cipher algorithm, by Bruce Schneier. - - This is a variable key length cipher which can use keys from 32 - bits to 448 bits in length. It's fast, simple and specifically - designed for use on "large microprocessors". + Cipher-based Message Authentication Code (CMAC) specified by + The National Institute of Standards and Technology (NIST). - See also: - + https://tools.ietf.org/html/rfc4493 + http://csrc.nist.gov/publications/nistpubs/800-38B/SP_800-38B.pdf -config CRYPTO_BLOWFISH_COMMON - tristate +config CRYPTO_GHASH + tristate "GHASH hash function" + select CRYPTO_GF128MUL + select CRYPTO_HASH help - Common parts of the Blowfish cipher algorithm shared by the - generic c and the assembler implementations. - - See also: - + GHASH is the hash function used in GCM (Galois/Counter Mode). + It is not a general-purpose cryptographic hash function. -config CRYPTO_CAMELLIA - tristate "Camellia cipher algorithms" - select CRYPTO_ALGAPI +config CRYPTO_HMAC + tristate "HMAC support" + select CRYPTO_HASH + select CRYPTO_MANAGER help - Camellia cipher algorithms module. - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - + HMAC: Keyed-Hashing for Message Authentication (RFC2104). + This is required for IPSec. -config CRYPTO_CAST_COMMON - tristate +config CRYPTO_MD4 + tristate "MD4 digest algorithm" + select CRYPTO_HASH help - Common parts of the CAST cipher algorithms shared by the - generic c and the assembler implementations. + MD4 message digest algorithm (RFC1320). -config CRYPTO_CAST5 - tristate "CAST5 (CAST-128) cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_CAST_COMMON +config CRYPTO_MD5 + tristate "MD5 digest algorithm" + select CRYPTO_HASH help - The CAST5 encryption algorithm (synonymous with CAST-128) is - described in RFC2144. + MD5 message digest algorithm (RFC1321). -config CRYPTO_CAST6 - tristate "CAST6 (CAST-256) cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_CAST_COMMON +config CRYPTO_MICHAEL_MIC + tristate "Michael MIC keyed digest algorithm" + select CRYPTO_HASH help - The CAST6 encryption algorithm (synonymous with CAST-256) is - described in RFC2612. + Michael MIC is used for message integrity protection in TKIP + (IEEE 802.11i). This algorithm is required for TKIP, but it + should not be used for other purposes because of the weakness + of the algorithm. -config CRYPTO_DES - tristate "DES and Triple DES EDE cipher algorithms" - select CRYPTO_ALGAPI - select CRYPTO_LIB_DES +config CRYPTO_POLYVAL + tristate + select CRYPTO_GF128MUL + select CRYPTO_HASH help - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). + POLYVAL is the hash function used in HCTR2. It is not a general-purpose + cryptographic hash function. -config CRYPTO_FCRYPT - tristate "FCrypt cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER +config CRYPTO_POLY1305 + tristate "Poly1305 authenticator algorithm" + select CRYPTO_HASH + select CRYPTO_LIB_POLY1305_GENERIC help - FCrypt algorithm used by RxRPC. + Poly1305 authenticator algorithm, RFC7539. -config CRYPTO_KHAZAD - tristate "Khazad cipher algorithm" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI + Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. + It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use + in IETF protocols. This is the portable C implementation of Poly1305. + +config CRYPTO_RMD160 + tristate "RIPEMD-160 digest algorithm" + select CRYPTO_HASH help - Khazad cipher algorithm. + RIPEMD-160 (ISO/IEC 10118-3:2004). - Khazad was a finalist in the initial NESSIE competition. It is - an algorithm optimized for 64-bit processors with good performance - on 32-bit processors. Khazad uses an 128 bit key size. + RIPEMD-160 is a 160-bit cryptographic hash function. It is intended + to be used as a secure replacement for the 128-bit hash functions + MD4, MD5 and its predecessor RIPEMD + (not to be confused with RIPEMD-128). - See also: - + It's speed is comparable to SHA1 and there are no known attacks + against RIPEMD-160. -config CRYPTO_CHACHA20 - tristate "ChaCha stream cipher algorithms" - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_SKCIPHER + Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. + See + +config CRYPTO_SHA1 + tristate "SHA1 digest algorithm" + select CRYPTO_HASH + select CRYPTO_LIB_SHA1 help - The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms. + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). - ChaCha20 is a 256-bit high-speed stream cipher designed by Daniel J. - Bernstein and further specified in RFC7539 for use in IETF protocols. - This is the portable C implementation of ChaCha20. See also: - +config CRYPTO_SHA256 + tristate "SHA224 and SHA256 digest algorithm" + select CRYPTO_HASH + select CRYPTO_LIB_SHA256 + help + SHA256 secure hash standard (DFIPS 180-2). - XChaCha20 is the application of the XSalsa20 construction to ChaCha20 - rather than to Salsa20. XChaCha20 extends ChaCha20's nonce length - from 64 bits (or 96 bits using the RFC7539 convention) to 192 bits, - while provably retaining ChaCha20's security. See also: - + This version of SHA implements a 256 bit hash with 128 bits of + security against collision attacks. - XChaCha12 is XChaCha20 reduced to 12 rounds, with correspondingly - reduced security margin but increased performance. It can be needed - in some performance-sensitive scenarios. + This code also includes SHA-224, a 224 bit hash with 112 bits + of security against collision attacks. -config CRYPTO_SEED - tristate "SEED cipher algorithm" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI +config CRYPTO_SHA512 + tristate "SHA384 and SHA512 digest algorithms" + select CRYPTO_HASH help - SEED cipher algorithm (RFC4269). + SHA512 secure hash standard (DFIPS 180-2). - SEED is a 128-bit symmetric key block cipher that has been - developed by KISA (Korea Information Security Agency) as a - national standard encryption algorithm of the Republic of Korea. - It is a 16 round block cipher with the key size of 128 bit. + This version of SHA implements a 512 bit hash with 256 bits of + security against collision attacks. - See also: - + This code also includes SHA-384, a 384 bit hash with 192 bits + of security against collision attacks. -config CRYPTO_ARIA - tristate "ARIA cipher algorithm" - select CRYPTO_ALGAPI +config CRYPTO_SHA3 + tristate "SHA3 digest algorithm" + select CRYPTO_HASH help - ARIA cipher algorithm (RFC5794). + SHA-3 secure hash standard (DFIPS 202). It's based on + cryptographic sponge function family called Keccak. - ARIA is a standard encryption algorithm of the Republic of Korea. - The ARIA specifies three key sizes and rounds. - 128-bit: 12 rounds. - 192-bit: 14 rounds. - 256-bit: 16 rounds. + References: + http://keccak.noekeon.org/ - See also: - +config CRYPTO_SM3 + tristate -config CRYPTO_SERPENT - tristate "Serpent cipher algorithm" - select CRYPTO_ALGAPI +config CRYPTO_SM3_GENERIC + tristate "SM3 digest algorithm" + select CRYPTO_HASH + select CRYPTO_SM3 help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). + It is part of the Chinese Commercial Cryptography suite. - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. + References: + http://www.oscca.gov.cn/UpFile/20101222141857786.pdf + https://datatracker.ietf.org/doc/html/draft-shen-sm3-hash - See also: - +config CRYPTO_STREEBOG + tristate "Streebog Hash Function" + select CRYPTO_HASH + help + Streebog Hash Function (GOST R 34.11-2012, RFC 6986) is one of the Russian + cryptographic standard algorithms (called GOST algorithms). + This setting enables two hash algorithms with 256 and 512 bits output. -config CRYPTO_SM4 - tristate + References: + https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf + https://tools.ietf.org/html/rfc6986 -config CRYPTO_SM4_GENERIC - tristate "SM4 cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_SM4 +config CRYPTO_VMAC + tristate "VMAC support" + select CRYPTO_HASH + select CRYPTO_MANAGER help - SM4 cipher algorithms (OSCCA GB/T 32907-2016). - - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. + VMAC is a message authentication algorithm designed for + very high speed on 64-bit architectures. - SMS4 was originally created for use in protecting wireless - networks, and is mandated in the Chinese National Standard for - Wireless LAN WAPI (Wired Authentication and Privacy Infrastructure) - (GB.15629.11-2003). + See also: + - The latest SM4 standard (GBT.32907-2016) was proposed by OSCCA and - standardized through TC 260 of the Standardization Administration - of the People's Republic of China (SAC). +config CRYPTO_WP512 + tristate "Whirlpool digest algorithms" + select CRYPTO_HASH + help + Whirlpool hash algorithm 512, 384 and 256-bit hashes - The input, output, and key of SMS4 are each 128 bits. + Whirlpool-512 is part of the NESSIE cryptographic primitives. + Whirlpool will be part of the ISO/IEC 10118-3:2003(E) standard - See also: + See also: + - If unsure, say N. +config CRYPTO_XCBC + tristate "XCBC support" + select CRYPTO_HASH + select CRYPTO_MANAGER + help + XCBC: Keyed-Hashing with encryption algorithm + https://www.ietf.org/rfc/rfc3566.txt + http://csrc.nist.gov/encryption/modes/proposedmodes/ + xcbc-mac/xcbc-mac-spec.pdf -config CRYPTO_TEA - tristate "TEA, XTEA and XETA cipher algorithms" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI +config CRYPTO_XXHASH + tristate "xxHash hash algorithm" + select CRYPTO_HASH + select XXHASH help - TEA cipher algorithm. + xxHash non-cryptographic hash algorithm. Extremely fast, working at + speeds close to RAM limits. - Tiny Encryption Algorithm is a simple cipher that uses - many rounds for security. It is very fast and uses - little memory. +endmenu - Xtendend Tiny Encryption Algorithm is a modification to - the TEA algorithm to address a potential key weakness - in the TEA algorithm. +menu "CRCs (cyclic redundancy checks)" - Xtendend Encryption Tiny Algorithm is a mis-implementation - of the XTEA algorithm for compatibility purposes. +config CRYPTO_CRC32C + tristate "CRC32c CRC algorithm" + select CRYPTO_HASH + select CRC32 + help + Castagnoli, et al Cyclic Redundancy-Check Algorithm. Used + by iSCSI for header and data digests and by others. + See Castagnoli93. Module will be crc32c. -config CRYPTO_TWOFISH - tristate "Twofish cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_TWOFISH_COMMON +config CRYPTO_CRC32 + tristate "CRC32 CRC algorithm" + select CRYPTO_HASH + select CRC32 help - Twofish cipher algorithm. + CRC-32-IEEE 802.3 cyclic redundancy-check algorithm. + Shash crypto api wrappers to crc32_le function. - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. +config CRYPTO_CRCT10DIF + tristate "CRCT10DIF algorithm" + select CRYPTO_HASH + help + CRC T10 Data Integrity Field computation is being cast as + a crypto transform. This allows for faster crc t10 diff + transforms to be used if they are available. - See also: - +config CRYPTO_CRC64_ROCKSOFT + tristate "Rocksoft Model CRC64 algorithm" + depends on CRC64 + select CRYPTO_HASH -config CRYPTO_TWOFISH_COMMON - tristate - help - Common parts of the Twofish cipher algorithm shared by the - generic c and the assembler implementations. +endmenu -comment "Compression" +menu "Compression" config CRYPTO_DEFLATE tristate "Deflate compression algorithm" @@ -1156,7 +1170,9 @@ config CRYPTO_ZSTD help This is the zstd algorithm. -comment "Random Number Generation" +endmenu + +menu "Random number generation" config CRYPTO_ANSI_CPRNG tristate "Pseudo Random Number Generation for Cryptographic modules" @@ -1218,6 +1234,9 @@ config CRYPTO_KDF800108_CTR select CRYPTO_HMAC select CRYPTO_SHA256 +endmenu +menu "User-space interface" + config CRYPTO_USER_API tristate @@ -1289,6 +1308,8 @@ config CRYPTO_STATS - encrypt/decrypt/sign/verify numbers for asymmetric operations - generate/seed numbers for rng operations +endmenu + config CRYPTO_HASH_INFO bool -- cgit From 05b374652737706557d0360064b07cfbeccb93d2 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:45 -0500 Subject: crypto: Kconfig - simplify public-key entries Shorten menu titles and make them consistent: - acronym - name - architecture features in parenthesis - no suffixes like " algorithm", "support", or "hardware acceleration", or "optimized" Simplify help text descriptions, update references, and ensure that https references are still valid. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 7 ++++++- arch/x86/crypto/Kconfig | 7 ++++++- crypto/Kconfig | 55 ++++++++++++++++++++++++++++++------------------- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index d73d19971b87..4b062bf53fa2 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -3,10 +3,15 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm)" config CRYPTO_CURVE25519_NEON - tristate "NEON accelerated Curve25519 scalar multiplication library" + tristate "Public key crypto: Curve25519 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + help + Curve25519 algorithm + + Architecture: arm with + - NEON (Advanced SIMD) extensions config CRYPTO_GHASH_ARM_CE tristate "PMULL-accelerated GHASH using NEON/ARMv8 Crypto Extensions" diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 04f4baea12a8..76229ccb79fd 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -3,10 +3,15 @@ menu "Accelerated Cryptographic Algorithms for CPU (x86)" config CRYPTO_CURVE25519_X86 - tristate "x86_64 accelerated Curve25519 scalar multiplication library" + tristate "Public key crypto: Curve25519 (ADX)" depends on X86 && 64BIT select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + help + Curve25519 algorithm + + Architecture: x86_64 using: + - ADX (large integer arithmetic) config CRYPTO_AES_NI_INTEL tristate "AES cipher algorithms (AES-NI)" diff --git a/crypto/Kconfig b/crypto/Kconfig index e2e364cfa93e..1fda21abb0d1 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -240,51 +240,60 @@ endmenu menu "Public-key cryptography" config CRYPTO_RSA - tristate "RSA algorithm" + tristate "RSA (Rivest-Shamir-Adleman)" select CRYPTO_AKCIPHER select CRYPTO_MANAGER select MPILIB select ASN1 help - Generic implementation of the RSA public key algorithm. + RSA (Rivest-Shamir-Adleman) public key algorithm (RFC8017) config CRYPTO_DH - tristate "Diffie-Hellman algorithm" + tristate "DH (Diffie-Hellman)" select CRYPTO_KPP select MPILIB help - Generic implementation of the Diffie-Hellman algorithm. + DH (Diffie-Hellman) key exchange algorithm config CRYPTO_DH_RFC7919_GROUPS - bool "Support for RFC 7919 FFDHE group parameters" + bool "RFC 7919 FFDHE groups" depends on CRYPTO_DH select CRYPTO_RNG_DEFAULT help - Provide support for RFC 7919 FFDHE group parameters. If unsure, say N. + FFDHE (Finite-Field-based Diffie-Hellman Ephemeral) groups + defined in RFC7919. + + Support these finite-field groups in DH key exchanges: + - ffdhe2048, ffdhe3072, ffdhe4096, ffdhe6144, ffdhe8192 + + If unsure, say N. config CRYPTO_ECC tristate select CRYPTO_RNG_DEFAULT config CRYPTO_ECDH - tristate "ECDH algorithm" + tristate "ECDH (Elliptic Curve Diffie-Hellman)" select CRYPTO_ECC select CRYPTO_KPP help - Generic implementation of the ECDH algorithm + ECDH (Elliptic Curve Diffie-Hellman) key exchange algorithm + using curves P-192, P-256, and P-384 (FIPS 186) config CRYPTO_ECDSA - tristate "ECDSA (NIST P192, P256 etc.) algorithm" + tristate "ECDSA (Elliptic Curve Digital Signature Algorithm)" select CRYPTO_ECC select CRYPTO_AKCIPHER select ASN1 help - Elliptic Curve Digital Signature Algorithm (NIST P192, P256 etc.) - is A NIST cryptographic standard algorithm. Only signature verification - is implemented. + ECDSA (Elliptic Curve Digital Signature Algorithm) (FIPS 186, + ISO/IEC 14888-3) + using curves P-192, P-256, and P-384 + + Only signature verification is implemented. config CRYPTO_ECRDSA - tristate "EC-RDSA (GOST 34.10) algorithm" + tristate "EC-RDSA (Elliptic Curve Russian Digital Signature Algorithm)" select CRYPTO_ECC select CRYPTO_AKCIPHER select CRYPTO_STREEBOG @@ -292,31 +301,35 @@ config CRYPTO_ECRDSA select ASN1 help Elliptic Curve Russian Digital Signature Algorithm (GOST R 34.10-2012, - RFC 7091, ISO/IEC 14888-3:2018) is one of the Russian cryptographic - standard algorithms (called GOST algorithms). Only signature verification - is implemented. + RFC 7091, ISO/IEC 14888-3) + + One of the Russian cryptographic standard algorithms (called GOST + algorithms). Only signature verification is implemented. config CRYPTO_SM2 - tristate "SM2 algorithm" + tristate "SM2 (ShangMi 2)" select CRYPTO_SM3 select CRYPTO_AKCIPHER select CRYPTO_MANAGER select MPILIB select ASN1 help - Generic implementation of the SM2 public key algorithm. It was - published by State Encryption Management Bureau, China. + SM2 (ShangMi 2) public key algorithm + + Published by State Encryption Management Bureau, China, as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012. References: - https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02 + https://datatracker.ietf.org/doc/draft-shen-sm2-ecdsa/ http://www.oscca.gov.cn/sca/xxgk/2010-12/17/content_1002386.shtml http://www.gmbz.org.cn/main/bzlb.html config CRYPTO_CURVE25519 - tristate "Curve25519 algorithm" + tristate "Curve25519" select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC + help + Curve25519 elliptic curve (RFC7748) endmenu -- cgit From ec84348da449d96ce5be47f7d00221cb8374f462 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:46 -0500 Subject: crypto: Kconfig - simplify CRC entries Shorten menu titles and make them consistent: - acronym - name - architecture features in parenthesis - no suffixes like " algorithm", "support", or "hardware acceleration", or "optimized" Simplify help text descriptions, update references, and ensure that https references are still valid. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 17 +++++++++++++++-- arch/arm64/crypto/Kconfig | 7 ++++++- arch/mips/crypto/Kconfig | 7 ++++--- arch/powerpc/crypto/Kconfig | 28 +++++++++++++++++----------- arch/s390/crypto/Kconfig | 9 ++++----- arch/sparc/crypto/Kconfig | 7 ++++--- arch/x86/crypto/Kconfig | 36 ++++++++++++++++-------------------- crypto/Kconfig | 37 +++++++++++++++++++++++++------------ 8 files changed, 91 insertions(+), 57 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 4b062bf53fa2..75684521f581 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -157,16 +157,29 @@ config CRYPTO_CHACHA20_NEON select CRYPTO_ARCH_HAVE_LIB_CHACHA config CRYPTO_CRC32_ARM_CE - tristate "CRC32(C) digest algorithm using CRC and/or PMULL instructions" + tristate "CRC32C and CRC32" depends on KERNEL_MODE_NEON depends on CRC32 select CRYPTO_HASH + help + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + and CRC32 CRC algorithm (IEEE 802.3) + + Architecture: arm using: + - CRC and/or PMULL instructions + + Drivers: crc32-arm-ce and crc32c-arm-ce config CRYPTO_CRCT10DIF_ARM_CE - tristate "CRCT10DIF digest algorithm using PMULL instructions" + tristate "CRCT10DIF" depends on KERNEL_MODE_NEON depends on CRC_T10DIF select CRYPTO_HASH + help + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + Architecture: arm using: + - PMULL (Polynomial Multiply Long) instructions endmenu diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index c5d42f62d8bb..cfc934880c97 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -127,9 +127,14 @@ config CRYPTO_AES_ARM64_CE_CCM select CRYPTO_LIB_AES config CRYPTO_CRCT10DIF_ARM64_CE - tristate "CRCT10DIF digest algorithm using PMULL instructions" + tristate "CRCT10DIF (PMULL)" depends on KERNEL_MODE_NEON && CRC_T10DIF select CRYPTO_HASH + help + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + Architecture: arm64 using + - PMULL (Polynomial Multiply Long) instructions endmenu diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 7c07611e2322..8a40add80430 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -3,12 +3,13 @@ menu "Accelerated Cryptographic Algorithms for CPU (mips)" config CRYPTO_CRC32_MIPS - tristate "CRC32c and CRC32 CRC algorithm (MIPS)" + tristate "CRC32c and CRC32" depends on MIPS_CRC_SUPPORT select CRYPTO_HASH help - CRC32c and CRC32 CRC algorithms implemented using mips crypto - instructions, when available. + CRC32c and CRC32 CRC algorithms + + Architecture: mips config CRYPTO_POLY1305_MIPS tristate "Poly1305 authenticator algorithm (MIPS optimized)" diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 74f535940faa..d1c34e949ce1 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -3,30 +3,36 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)" config CRYPTO_CRC32C_VPMSUM - tristate "CRC32c CRC algorithm (powerpc64)" + tristate "CRC32c" depends on PPC64 && ALTIVEC select CRYPTO_HASH select CRC32 help - CRC32c algorithm implemented using vector polynomial multiply-sum - (vpmsum) instructions, introduced in POWER8. Enable on POWER8 - and newer processors for improved performance. + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + + Architecture: powerpc64 using + - AltiVec extensions + + Enable on POWER8 and newer processors for improved performance. config CRYPTO_CRCT10DIF_VPMSUM - tristate "CRC32T10DIF powerpc64 hardware acceleration" + tristate "CRC32T10DIF" depends on PPC64 && ALTIVEC && CRC_T10DIF select CRYPTO_HASH help - CRC10T10DIF algorithm implemented using vector polynomial - multiply-sum (vpmsum) instructions, introduced in POWER8. Enable on - POWER8 and newer processors for improved performance. + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + Architecture: powerpc64 using + - AltiVec extensions + + Enable on POWER8 and newer processors for improved performance. config CRYPTO_VPMSUM_TESTER - tristate "Powerpc64 vpmsum hardware acceleration tester" + tristate "CRC32c and CRC32T10DIF hardware acceleration tester" depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM help - Stress test for CRC32c and CRC-T10DIF algorithms implemented with - POWER8 vpmsum instructions. + Stress test for CRC32c and CRCT10DIF algorithms implemented with + powerpc64 AltiVec extensions (POWER8 vpmsum instructions). Unless you are testing these algorithms, you don't need this. config CRYPTO_MD5_PPC diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index ef0651d71e9d..5d12ecfaa337 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -3,15 +3,14 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" config CRYPTO_CRC32_S390 - tristate "CRC-32 algorithms" + tristate "CRC32c and CRC32" depends on S390 select CRYPTO_HASH select CRC32 help - Select this option if you want to use hardware accelerated - implementations of CRC algorithms. With this option, you - can optimize the computation of CRC-32 (IEEE 802.3 Ethernet) - and CRC-32C (Castagnoli). + CRC32c and CRC32 CRC algorithms + + Architecture: s390 It is available with IBM z13 or later. diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index eaa2afc1d50a..145debe629cd 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -13,13 +13,14 @@ config CRYPTO_DES_SPARC64 optimized using SPARC64 crypto opcodes. config CRYPTO_CRC32C_SPARC64 - tristate "CRC32c CRC algorithm (SPARC64)" + tristate "CRC32c" depends on SPARC64 select CRYPTO_HASH select CRC32 help - CRC32c CRC algorithm implemented using sparc64 crypto instructions, - when available. + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + + Architecture: sparc64 config CRYPTO_MD5_SPARC64 tristate "MD5 digest algorithm (SPARC64)" diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 76229ccb79fd..03f9a3a35e42 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -467,39 +467,35 @@ config CRYPTO_GHASH_CLMUL_NI_INTEL GHASH, the hash function used in GCM (Galois/Counter mode). config CRYPTO_CRC32C_INTEL - tristate "CRC32c INTEL hardware acceleration" + tristate "CRC32c (SSE4.2/PCLMULQDQ)" depends on X86 select CRYPTO_HASH help - In Intel processor with SSE4.2 supported, the processor will - support CRC32C implementation using hardware accelerated CRC32 - instruction. This option will create 'crc32c-intel' module, - which will enable any routine to use the CRC32 instruction to - gain performance compared with software implementation. - Module will be crc32c-intel. + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + + Architecture: x86 (32-bit and 64-bit) using: + - SSE4.2 (Streaming SIMD Extensions 4.2) CRC32 instruction + - PCLMULQDQ (carry-less multiplication) config CRYPTO_CRC32_PCLMUL - tristate "CRC32 PCLMULQDQ hardware acceleration" + tristate "CRC32 (PCLMULQDQ)" depends on X86 select CRYPTO_HASH select CRC32 help - From Intel Westmere and AMD Bulldozer processor with SSE4.2 - and PCLMULQDQ supported, the processor will support - CRC32 PCLMULQDQ implementation using hardware accelerated PCLMULQDQ - instruction. This option will create 'crc32-pclmul' module, - which will enable any routine to use the CRC-32-IEEE 802.3 checksum - and gain better performance as compared with the table implementation. + CRC32 CRC algorithm (IEEE 802.3) + + Architecture: x86 (32-bit and 64-bit) using: + - PCLMULQDQ (carry-less multiplication) config CRYPTO_CRCT10DIF_PCLMUL - tristate "CRCT10DIF PCLMULQDQ hardware acceleration" + tristate "CRCT10DIF (PCLMULQDQ)" depends on X86 && 64BIT && CRC_T10DIF select CRYPTO_HASH help - For x86_64 processors with SSE4.2 and PCLMULQDQ supported, - CRC T10 DIF PCLMULQDQ computation can be hardware - accelerated PCLMULQDQ instruction. This option will create - 'crct10dif-pclmul' module, which is faster when computing the - crct10dif checksum as compared with the generic table implementation. + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + Architecture: x86_64 using: + - PCLMULQDQ (carry-less multiplication) endmenu diff --git a/crypto/Kconfig b/crypto/Kconfig index 1fda21abb0d1..6dea21229376 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1093,34 +1093,47 @@ endmenu menu "CRCs (cyclic redundancy checks)" config CRYPTO_CRC32C - tristate "CRC32c CRC algorithm" + tristate "CRC32c" select CRYPTO_HASH select CRC32 help - Castagnoli, et al Cyclic Redundancy-Check Algorithm. Used - by iSCSI for header and data digests and by others. - See Castagnoli93. Module will be crc32c. + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + + A 32-bit CRC (cyclic redundancy check) with a polynomial defined + by G. Castagnoli, S. Braeuer and M. Herrman in "Optimization of Cyclic + Redundancy-Check Codes with 24 and 32 Parity Bits", IEEE Transactions + on Communications, Vol. 41, No. 6, June 1993, selected for use with + iSCSI. + + Used by btrfs, ext4, jbd2, NVMeoF/TCP, and iSCSI. config CRYPTO_CRC32 - tristate "CRC32 CRC algorithm" + tristate "CRC32" select CRYPTO_HASH select CRC32 help - CRC-32-IEEE 802.3 cyclic redundancy-check algorithm. - Shash crypto api wrappers to crc32_le function. + CRC32 CRC algorithm (IEEE 802.3) + + Used by RoCEv2 and f2fs. config CRYPTO_CRCT10DIF - tristate "CRCT10DIF algorithm" + tristate "CRCT10DIF" select CRYPTO_HASH help - CRC T10 Data Integrity Field computation is being cast as - a crypto transform. This allows for faster crc t10 diff - transforms to be used if they are available. + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + CRC algorithm used by the SCSI Block Commands standard. config CRYPTO_CRC64_ROCKSOFT - tristate "Rocksoft Model CRC64 algorithm" + tristate "CRC64 based on Rocksoft Model algorithm" depends on CRC64 select CRYPTO_HASH + help + CRC64 CRC algorithm based on the Rocksoft Model CRC Algorithm + + Used by the NVMe implementation of T10 DIF (BLK_DEV_INTEGRITY) + + See https://zlib.net/crc_v3.txt endmenu -- cgit From e3d2eadd06b39b69fbbc27de8e3ac2db022e8616 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:47 -0500 Subject: crypto: Kconfig - simplify aead entries Shorten menu titles and make them consistent: - acronym - name - architecture features in parenthesis - no suffixes like " algorithm", "support", or "hardware acceleration", or "optimized" Simplify help text descriptions, update references, and ensure that https references are still valid. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 8 ++++++-- crypto/Kconfig | 48 ++++++++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 03f9a3a35e42..93de2684b3dc 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -360,12 +360,16 @@ config CRYPTO_CHACHA20_X86_64 XChaCha20, and XChaCha12 stream ciphers. config CRYPTO_AEGIS128_AESNI_SSE2 - tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" + tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE2)" depends on X86 && 64BIT select CRYPTO_AEAD select CRYPTO_SIMD help - AESNI+SSE2 implementation of the AEGIS-128 dedicated AEAD algorithm. + AEGIS-128 AEAD algorithm + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - SSE2 (Streaming SIMD Extensions 2) config CRYPTO_NHPOLY1305_SSE2 tristate "NHPoly1305 hash function (x86_64 SSE2 implementation)" diff --git a/crypto/Kconfig b/crypto/Kconfig index 6dea21229376..5159a0efec84 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -779,49 +779,54 @@ endmenu menu "AEAD (authenticated encryption with associated data) ciphers" config CRYPTO_AEGIS128 - tristate "AEGIS-128 AEAD algorithm" + tristate "AEGIS-128" select CRYPTO_AEAD select CRYPTO_AES # for AES S-box tables help - Support for the AEGIS-128 dedicated AEAD algorithm. + AEGIS-128 AEAD algorithm config CRYPTO_AEGIS128_SIMD - bool "Support SIMD acceleration for AEGIS-128" + bool "AEGIS-128 (arm NEON, arm64 NEON)" depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON) default y + help + AEGIS-128 AEAD algorithm + + Architecture: arm or arm64 using: + - NEON (Advanced SIMD) extension config CRYPTO_CHACHA20POLY1305 - tristate "ChaCha20-Poly1305 AEAD support" + tristate "ChaCha20-Poly1305" select CRYPTO_CHACHA20 select CRYPTO_POLY1305 select CRYPTO_AEAD select CRYPTO_MANAGER help - ChaCha20-Poly1305 AEAD support, RFC7539. - - Support for the AEAD wrapper using the ChaCha20 stream cipher combined - with the Poly1305 authenticator. It is defined in RFC7539 for use in - IETF protocols. + ChaCha20 stream cipher and Poly1305 authenticator combined + mode (RFC8439) config CRYPTO_CCM - tristate "CCM support" + tristate "CCM (Counter with Cipher Block Chaining-Message Authentication Code)" select CRYPTO_CTR select CRYPTO_HASH select CRYPTO_AEAD select CRYPTO_MANAGER help - Support for Counter with CBC MAC. Required for IPsec. + CCM (Counter with Cipher Block Chaining-Message Authentication Code) + authenticated encryption mode (NIST SP800-38C) config CRYPTO_GCM - tristate "GCM/GMAC support" + tristate "GCM (Galois/Counter Mode) and GMAC (GCM Message Authentication Code)" select CRYPTO_CTR select CRYPTO_AEAD select CRYPTO_GHASH select CRYPTO_NULL select CRYPTO_MANAGER help - Support for Galois/Counter Mode (GCM) and Galois Message - Authentication Code (GMAC). Required for IPSec. + GCM (Galois/Counter Mode) authenticated encryption mode and GMAC + (GCM Message Authentication Code) (NIST SP800-38D) + + This is required for IPSec ESP (XFRM_ESP). config CRYPTO_SEQIV tristate "Sequence Number IV Generator" @@ -831,8 +836,12 @@ config CRYPTO_SEQIV select CRYPTO_RNG_DEFAULT select CRYPTO_MANAGER help + Sequence Number IV generator + This IV generator generates an IV based on a sequence number by - xoring it with a salt. This algorithm is mainly useful for CTR + xoring it with a salt. This algorithm is mainly useful for CTR. + + This is required for IPsec ESP (XFRM_ESP). config CRYPTO_ECHAINIV tristate "Encrypted Chain IV Generator" @@ -841,16 +850,19 @@ config CRYPTO_ECHAINIV select CRYPTO_RNG_DEFAULT select CRYPTO_MANAGER help + Encrypted Chain IV generator + This IV generator generates an IV based on the encryption of a sequence number xored with a salt. This is the default algorithm for CBC. config CRYPTO_ESSIV - tristate "ESSIV support for block encryption" + tristate "Encrypted Salt-Sector IV Generator" select CRYPTO_AUTHENC help - Encrypted salt-sector initialization vector (ESSIV) is an IV - generation method that is used in some cases by fscrypt and/or + Encrypted Salt-Sector IV generator + + This IV generator is used in some cases by fscrypt and/or dm-crypt. It uses the hash of the block encryption key as the symmetric key for a block encryption pass applied to the input IV, making low entropy IV sources more suitable for block -- cgit From 3f342a23257df99b792c1edb1236e85badc157de Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:48 -0500 Subject: crypto: Kconfig - simplify hash entries Shorten menu titles and make them consistent: - acronym - name - architecture features in parenthesis - no suffixes like " algorithm", "support", or "hardware acceleration", or "optimized" Simplify help text descriptions, update references, and ensure that https references are still valid. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 94 +++++++++++++++-------- arch/arm64/crypto/Kconfig | 77 ++++++++++++++++--- arch/mips/crypto/Kconfig | 34 +++++---- arch/powerpc/crypto/Kconfig | 30 +++++--- arch/s390/crypto/Kconfig | 42 ++++++----- arch/sparc/crypto/Kconfig | 28 ++++--- arch/x86/crypto/Kconfig | 101 ++++++++++++++----------- crypto/Kconfig | 176 ++++++++++++++++++++++++-------------------- 8 files changed, 366 insertions(+), 216 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 75684521f581..e64e9b8418d6 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -14,98 +14,134 @@ config CRYPTO_CURVE25519_NEON - NEON (Advanced SIMD) extensions config CRYPTO_GHASH_ARM_CE - tristate "PMULL-accelerated GHASH using NEON/ARMv8 Crypto Extensions" + tristate "Hash functions: GHASH (PMULL/NEON/ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_CRYPTD select CRYPTO_GF128MUL help + GCM GHASH function (NIST SP800-38D) + + Architecture: arm using + - PMULL (Polynomial Multiply Long) instructions + - NEON (Advanced SIMD) extensions + - ARMv8 Crypto Extensions + Use an implementation of GHASH (used by the GCM AEAD chaining mode) that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) that is part of the ARMv8 Crypto Extensions, or a slower variant that uses the vmull.p8 instruction that is part of the basic NEON ISA. config CRYPTO_NHPOLY1305_NEON - tristate "NEON accelerated NHPoly1305 hash function (for Adiantum)" + tristate "Hash functions: NHPoly1305 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_NHPOLY1305 + help + NHPoly1305 hash function (Adiantum) + + Architecture: arm using: + - NEON (Advanced SIMD) extensions config CRYPTO_POLY1305_ARM - tristate "Accelerated scalar and SIMD Poly1305 hash implementations" + tristate "Hash functions: Poly1305 (NEON)" select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm (RFC7539) + + Architecture: arm optionally using + - NEON (Advanced SIMD) extensions config CRYPTO_BLAKE2S_ARM - bool "BLAKE2s digest algorithm (ARM)" + bool "Hash functions: BLAKE2s" select CRYPTO_ARCH_HAVE_LIB_BLAKE2S help - BLAKE2s digest algorithm optimized with ARM scalar instructions. This - is faster than the generic implementations of BLAKE2s and BLAKE2b, but - slower than the NEON implementation of BLAKE2b. (There is no NEON - implementation of BLAKE2s, since NEON doesn't really help with it.) + BLAKE2s cryptographic hash function (RFC 7693) + + Architecture: arm + + This is faster than the generic implementations of BLAKE2s and + BLAKE2b, but slower than the NEON implementation of BLAKE2b. + There is no NEON implementation of BLAKE2s, since NEON doesn't + really help with it. config CRYPTO_BLAKE2B_NEON - tristate "BLAKE2b digest algorithm (ARM NEON)" + tristate "Hash functions: BLAKE2b (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_BLAKE2B help + BLAKE2b cryptographic hash function (RFC 7693) + + Architecture: arm using + - NEON (Advanced SIMD) extensions + BLAKE2b digest algorithm optimized with ARM NEON instructions. On ARM processors that have NEON support but not the ARMv8 Crypto Extensions, typically this BLAKE2b implementation is - much faster than SHA-2 and slightly faster than SHA-1. + much faster than the SHA-2 family and slightly faster than + SHA-1. config CRYPTO_SHA1_ARM - tristate "SHA1 digest algorithm (ARM-asm)" + tristate "Hash functions: SHA-1" select CRYPTO_SHA1 select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using optimized ARM assembler. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm config CRYPTO_SHA1_ARM_NEON - tristate "SHA1 digest algorithm (ARM NEON)" + tristate "Hash functions: SHA-1 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SHA1_ARM select CRYPTO_SHA1 select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using optimized ARM NEON assembly, when NEON instructions are - available. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm using + - NEON (Advanced SIMD) extensions config CRYPTO_SHA1_ARM_CE - tristate "SHA1 digest algorithm (ARM v8 Crypto Extensions)" + tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SHA1_ARM select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using special ARMv8 Crypto Extensions. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm using ARMv8 Crypto Extensions config CRYPTO_SHA2_ARM_CE - tristate "SHA-224/256 digest algorithm (ARM v8 Crypto Extensions)" + tristate "Hash functions: SHA-224 and SHA-256 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SHA256_ARM select CRYPTO_HASH help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using special ARMv8 Crypto Extensions. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm using + - ARMv8 Crypto Extensions config CRYPTO_SHA256_ARM - tristate "SHA-224/256 digest algorithm (ARM-asm and NEON)" + tristate "Hash functions: SHA-224 and SHA-256 (NEON)" select CRYPTO_HASH depends on !CPU_V7M help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using optimized ARM assembler and NEON, when available. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm using + - NEON (Advanced SIMD) extensions config CRYPTO_SHA512_ARM - tristate "SHA-384/512 digest algorithm (ARM-asm and NEON)" + tristate "Hash functions: SHA-384 and SHA-512 (NEON)" select CRYPTO_HASH depends on !CPU_V7M help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using optimized ARM assembler and NEON, when available. + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: arm using + - NEON (Advanced SIMD) extensions config CRYPTO_AES_ARM tristate "Scalar AES cipher for ARM" diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index cfc934880c97..709598f6d2e3 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -3,66 +3,119 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm64)" config CRYPTO_GHASH_ARM64_CE - tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" + tristate "Hash functions: GHASH (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_GF128MUL select CRYPTO_LIB_AES select CRYPTO_AEAD + help + GCM GHASH function (NIST SP800-38D) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_NHPOLY1305_NEON - tristate "NHPoly1305 hash function using NEON instructions (for Adiantum)" + tristate "Hash functions: NHPoly1305 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_NHPOLY1305 + help + NHPoly1305 hash function (Adiantum) + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions config CRYPTO_POLY1305_NEON - tristate "Poly1305 hash function using scalar or NEON instructions" + tristate "Hash functions: Poly1305 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm (RFC7539) + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions -config CRYPTO_SHA1_ARM64_CE - tristate "SHA-1 digest algorithm (ARMv8 Crypto Extensions)" +config CRYPTO_SHA1_ARM64 + tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA1 + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_SHA256_ARM64 - tristate "SHA-224/SHA-256 digest algorithm for arm64" + tristate "Hash functions: SHA-224 and SHA-256" select CRYPTO_HASH + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm64 config CRYPTO_SHA2_ARM64_CE - tristate "SHA-224/SHA-256 digest algorithm (ARMv8 Crypto Extensions)" + tristate "Hash functions: SHA-224 and SHA-256 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA256_ARM64 + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_SHA512_ARM64 - tristate "SHA-384/SHA-512 digest algorithm for arm64" + tristate "Hash functions: SHA-384 and SHA-512" select CRYPTO_HASH + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: arm64 config CRYPTO_SHA512_ARM64_CE - tristate "SHA-384/SHA-512 digest algorithm (ARMv8 Crypto Extensions)" + tristate "Hash functions: SHA-384 and SHA-512 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA512_ARM64 + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_SHA3_ARM64 - tristate "SHA3 digest algorithm (ARMv8.2 Crypto Extensions)" + tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA3 + help + SHA-3 secure hash algorithms (FIPS 202) + + Architecture: arm64 using: + - ARMv8.2 Crypto Extensions config CRYPTO_SM3_ARM64_CE - tristate "SM3 digest algorithm (ARMv8.2 Crypto Extensions)" + tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SM3 + help + SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) + + Architecture: arm64 using: + - ARMv8.2 Crypto Extensions config CRYPTO_POLYVAL_ARM64_CE - tristate "POLYVAL using ARMv8 Crypto Extensions (for HCTR2)" + tristate "Hash functions: POLYVAL (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_POLYVAL + help + POLYVAL hash function for HCTR2 + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_AES_ARM64 tristate "AES core cipher using scalar instructions" diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 8a40add80430..de162f69675c 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -12,45 +12,53 @@ config CRYPTO_CRC32_MIPS Architecture: mips config CRYPTO_POLY1305_MIPS - tristate "Poly1305 authenticator algorithm (MIPS optimized)" + tristate "Hash functions: Poly1305" depends on MIPS select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm (RFC7539) + + Architecture: mips config CRYPTO_MD5_OCTEON - tristate "MD5 digest algorithm (OCTEON)" + tristate "Digests: MD5 (OCTEON)" depends on CPU_CAVIUM_OCTEON select CRYPTO_MD5 select CRYPTO_HASH help - MD5 message digest algorithm (RFC1321) implemented - using OCTEON crypto instructions, when available. + MD5 message digest algorithm (RFC1321) + + Architecture: mips OCTEON using crypto instructions, when available config CRYPTO_SHA1_OCTEON - tristate "SHA1 digest algorithm (OCTEON)" + tristate "Hash functions: SHA-1 (OCTEON)" depends on CPU_CAVIUM_OCTEON select CRYPTO_SHA1 select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: mips OCTEON config CRYPTO_SHA256_OCTEON - tristate "SHA224 and SHA256 digest algorithm (OCTEON)" + tristate "Hash functions: SHA-224 and SHA-256 (OCTEON)" depends on CPU_CAVIUM_OCTEON select CRYPTO_SHA256 select CRYPTO_HASH help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: mips OCTEON using crypto instructions, when available config CRYPTO_SHA512_OCTEON - tristate "SHA384 and SHA512 digest algorithms (OCTEON)" + tristate "Hash functions: SHA-384 and SHA-512 (OCTEON)" depends on CPU_CAVIUM_OCTEON select CRYPTO_SHA512 select CRYPTO_HASH help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: mips OCTEON using crypto instructions, when available config CRYPTO_CHACHA_MIPS tristate "ChaCha stream cipher algorithms (MIPS 32r2 optimized)" diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index d1c34e949ce1..5a4770a029ef 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -36,35 +36,41 @@ config CRYPTO_VPMSUM_TESTER Unless you are testing these algorithms, you don't need this. config CRYPTO_MD5_PPC - tristate "MD5 digest algorithm (PPC)" + tristate "Digests: MD5" depends on PPC select CRYPTO_HASH help - MD5 message digest algorithm (RFC1321) implemented - in PPC assembler. + MD5 message digest algorithm (RFC1321) + + Architecture: powerpc config CRYPTO_SHA1_PPC - tristate "SHA1 digest algorithm (powerpc)" + tristate "Hash functions: SHA-1" depends on PPC help - This is the powerpc hardware accelerated implementation of the - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: powerpc config CRYPTO_SHA1_PPC_SPE - tristate "SHA1 digest algorithm (PPC SPE)" + tristate "Hash functions: SHA-1 (SPE)" depends on PPC && SPE help - SHA-1 secure hash standard (DFIPS 180-4) implemented - using powerpc SPE SIMD instruction set. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: powerpc using + - SPE (Signal Processing Engine) extensions config CRYPTO_SHA256_PPC_SPE - tristate "SHA224 and SHA256 digest algorithm (PPC SPE)" + tristate "Hash functions: SHA-224 and SHA-256 (SPE)" depends on PPC && SPE select CRYPTO_SHA256 select CRYPTO_HASH help - SHA224 and SHA256 secure hash standard (DFIPS 180-2) - implemented using powerpc SPE SIMD instruction set. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: powerpc using + - SPE (Signal Processing Engine) extensions config CRYPTO_AES_PPC_SPE tristate "AES cipher algorithms (PPC SPE)" diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 5d12ecfaa337..04cc3a6467ab 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -15,62 +15,68 @@ config CRYPTO_CRC32_S390 It is available with IBM z13 or later. config CRYPTO_SHA512_S390 - tristate "SHA384 and SHA512 digest algorithm" + tristate "Hash functions: SHA-384 and SHA-512" depends on S390 select CRYPTO_HASH help - This is the s390 hardware accelerated implementation of the - SHA512 secure hash standard. + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: s390 It is available as of z10. config CRYPTO_SHA1_S390 - tristate "SHA1 digest algorithm" + tristate "Hash functions: SHA-1" depends on S390 select CRYPTO_HASH help - This is the s390 hardware accelerated implementation of the - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: s390 It is available as of z990. config CRYPTO_SHA256_S390 - tristate "SHA256 digest algorithm" + tristate "Hash functions: SHA-224 and SHA-256" depends on S390 select CRYPTO_HASH help - This is the s390 hardware accelerated implementation of the - SHA256 secure hash standard (DFIPS 180-2). + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: s390 It is available as of z9. config CRYPTO_SHA3_256_S390 - tristate "SHA3_224 and SHA3_256 digest algorithm" + tristate "Hash functions: SHA3-224 and SHA3-256" depends on S390 select CRYPTO_HASH help - This is the s390 hardware accelerated implementation of the - SHA3_256 secure hash standard. + SHA3-224 and SHA3-256 secure hash algorithms (FIPS 202) + + Architecture: s390 It is available as of z14. config CRYPTO_SHA3_512_S390 - tristate "SHA3_384 and SHA3_512 digest algorithm" + tristate "Hash functions: SHA3-384 and SHA3-512" depends on S390 select CRYPTO_HASH help - This is the s390 hardware accelerated implementation of the - SHA3_512 secure hash standard. + SHA3-384 and SHA3-512 secure hash algorithms (FIPS 202) + + Architecture: s390 It is available as of z14. config CRYPTO_GHASH_S390 - tristate "GHASH hash function" + tristate "Hash functions: GHASH" depends on S390 select CRYPTO_HASH help - This is the s390 hardware accelerated implementation of GHASH, - the hash function used in GCM (Galois/Counter mode). + GCM GHASH hash function (NIST SP800-38D) + + Architecture: s390 It is available as of z196. diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index 145debe629cd..519348de6860 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -23,40 +23,44 @@ config CRYPTO_CRC32C_SPARC64 Architecture: sparc64 config CRYPTO_MD5_SPARC64 - tristate "MD5 digest algorithm (SPARC64)" + tristate "Digests: MD5" depends on SPARC64 select CRYPTO_MD5 select CRYPTO_HASH help - MD5 message digest algorithm (RFC1321) implemented - using sparc64 crypto instructions, when available. + MD5 message digest algorithm (RFC1321) + + Architecture: sparc64 using crypto instructions, when available config CRYPTO_SHA1_SPARC64 - tristate "SHA1 digest algorithm (SPARC64)" + tristate "Hash functions: SHA-1" depends on SPARC64 select CRYPTO_SHA1 select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: sparc64 config CRYPTO_SHA256_SPARC64 - tristate "SHA224 and SHA256 digest algorithm (SPARC64)" + tristate "Hash functions: SHA-224 and SHA-256" depends on SPARC64 select CRYPTO_SHA256 select CRYPTO_HASH help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: sparc64 using crypto instructions, when available config CRYPTO_SHA512_SPARC64 - tristate "SHA384 and SHA512 digest algorithm (SPARC64)" + tristate "Hash functions: SHA-384 and SHA-512" depends on SPARC64 select CRYPTO_SHA512 select CRYPTO_HASH help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: sparc64 using crypto instructions, when available config CRYPTO_AES_SPARC64 tristate "AES cipher algorithms (SPARC64)" diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 93de2684b3dc..fc24f4562700 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -372,103 +372,122 @@ config CRYPTO_AEGIS128_AESNI_SSE2 - SSE2 (Streaming SIMD Extensions 2) config CRYPTO_NHPOLY1305_SSE2 - tristate "NHPoly1305 hash function (x86_64 SSE2 implementation)" + tristate "Hash functions: NHPoly1305 (SSE2)" depends on X86 && 64BIT select CRYPTO_NHPOLY1305 help - SSE2 optimized implementation of the hash function used by the - Adiantum encryption mode. + NHPoly1305 hash function for Adiantum + + Architecture: x86_64 using: + - SSE2 (Streaming SIMD Extensions 2) config CRYPTO_NHPOLY1305_AVX2 - tristate "NHPoly1305 hash function (x86_64 AVX2 implementation)" + tristate "Hash functions: NHPoly1305 (AVX2)" depends on X86 && 64BIT select CRYPTO_NHPOLY1305 help - AVX2 optimized implementation of the hash function used by the - Adiantum encryption mode. + NHPoly1305 hash function for Adiantum + + Architecture: x86_64 using: + - AVX2 (Advanced Vector Extensions 2) config CRYPTO_BLAKE2S_X86 - bool "BLAKE2s digest algorithm (x86 accelerated version)" + bool "Hash functions: BLAKE2s (SSSE3/AVX-512)" depends on X86 && 64BIT select CRYPTO_LIB_BLAKE2S_GENERIC select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s cryptographic hash function (RFC 7693) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX-512 (Advanced Vector Extensions-512) config CRYPTO_POLYVAL_CLMUL_NI - tristate "POLYVAL hash function (CLMUL-NI accelerated)" + tristate "Hash functions: POLYVAL (CLMUL-NI)" depends on X86 && 64BIT select CRYPTO_POLYVAL help - This is the x86_64 CLMUL-NI accelerated implementation of POLYVAL. It is - used to efficiently implement HCTR2 on x86-64 processors that support - carry-less multiplication instructions. + POLYVAL hash function for HCTR2 + + Architecture: x86_64 using: + - CLMUL-NI (carry-less multiplication new instructions) config CRYPTO_POLY1305_X86_64 - tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)" + tristate "Hash functions: Poly1305 (SSE2/AVX2)" depends on X86 && 64BIT select CRYPTO_LIB_POLY1305_GENERIC select CRYPTO_ARCH_HAVE_LIB_POLY1305 help - Poly1305 authenticator algorithm, RFC7539. + Poly1305 authenticator algorithm (RFC7539) - Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. - It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use - in IETF protocols. This is the x86_64 assembler implementation using SIMD - instructions. + Architecture: x86_64 using: + - SSE2 (Streaming SIMD Extensions 2) + - AVX2 (Advanced Vector Extensions 2) config CRYPTO_SHA1_SSSE3 - tristate "SHA1 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" + tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)" depends on X86 && 64BIT select CRYPTO_SHA1 select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions or Advanced Vector - Extensions (AVX/AVX2) or SHA-NI(SHA Extensions New Instructions), - when available. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX (Advanced Vector Extensions) + - AVX2 (Advanced Vector Extensions 2) + - SHA-NI (SHA Extensions New Instructions) config CRYPTO_SHA256_SSSE3 - tristate "SHA256 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" + tristate "Hash functions: SHA-224 and SHA-256 (SSSE3/AVX/AVX2/SHA-NI)" depends on X86 && 64BIT select CRYPTO_SHA256 select CRYPTO_HASH help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector - Extensions version 1 (AVX1), or Advanced Vector Extensions - version 2 (AVX2) instructions, or SHA-NI (SHA Extensions New - Instructions) when available. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX (Advanced Vector Extensions) + - AVX2 (Advanced Vector Extensions 2) + - SHA-NI (SHA Extensions New Instructions) config CRYPTO_SHA512_SSSE3 - tristate "SHA512 digest algorithm (SSSE3/AVX/AVX2)" + tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)" depends on X86 && 64BIT select CRYPTO_SHA512 select CRYPTO_HASH help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector - Extensions version 1 (AVX1), or Advanced Vector Extensions - version 2 (AVX2) instructions, when available. + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX (Advanced Vector Extensions) + - AVX2 (Advanced Vector Extensions 2) config CRYPTO_SM3_AVX_X86_64 - tristate "SM3 digest algorithm (x86_64/AVX)" + tristate "Hash functions: SM3 (AVX)" depends on X86 && 64BIT select CRYPTO_HASH select CRYPTO_SM3 help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). - It is part of the Chinese Commercial Cryptography suite. This is - SM3 optimized implementation using Advanced Vector Extensions (AVX) - when available. + SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3 + + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) If unsure, say N. config CRYPTO_GHASH_CLMUL_NI_INTEL - tristate "GHASH hash function (CLMUL-NI accelerated)" + tristate "Hash functions: GHASH (CLMUL-NI)" depends on X86 && 64BIT select CRYPTO_CRYPTD help - This is the x86_64 CLMUL-NI accelerated implementation of - GHASH, the hash function used in GCM (Galois/Counter mode). + GCM GHASH hash function (NIST SP800-38D) + + Architecture: x86_64 using: + - CLMUL-NI (carry-less multiplication new instructions) config CRYPTO_CRC32C_INTEL tristate "CRC32c (SSE4.2/PCLMULQDQ)" diff --git a/crypto/Kconfig b/crypto/Kconfig index 5159a0efec84..0a385a7aa040 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -890,215 +890,233 @@ endmenu menu "Hashes, digests, and MACs" config CRYPTO_BLAKE2B - tristate "BLAKE2b digest algorithm" + tristate "BLAKE2b" select CRYPTO_HASH help - Implementation of cryptographic hash function BLAKE2b (or just BLAKE2), - optimized for 64bit platforms and can produce digests of any size - between 1 to 64. The keyed hash is also implemented. + BLAKE2b cryptographic hash function (RFC 7693) - This module provides the following algorithms: + BLAKE2b is optimized for 64-bit platforms and can produce digests + of any size between 1 and 64 bytes. The keyed hash is also implemented. + This module provides the following algorithms: - blake2b-160 - blake2b-256 - blake2b-384 - blake2b-512 + Used by the btrfs filesystem. + See https://blake2.net for further information. + config CRYPTO_BLAKE2S + tristate "BLAKE2s" + select CRYPTO_LIB_BLAKE2S_GENERIC + select CRYPTO_HASH + help + BLAKE2s cryptographic hash function (RFC 7693) + + BLAKE2s is optimized for 8 to 32-bit platforms and can produce + digests of any size between 1 and 32 bytes. The keyed hash is + also implemented. + + This module provides the following algorithms: + - blake2s-128 + - blake2s-160 + - blake2s-224 + - blake2s-256 + + Used by Wireguard. + + See https://blake2.net for further information. + config CRYPTO_CMAC - tristate "CMAC support" + tristate "CMAC (Cipher-based MAC)" select CRYPTO_HASH select CRYPTO_MANAGER help - Cipher-based Message Authentication Code (CMAC) specified by - The National Institute of Standards and Technology (NIST). - - https://tools.ietf.org/html/rfc4493 - http://csrc.nist.gov/publications/nistpubs/800-38B/SP_800-38B.pdf + CMAC (Cipher-based Message Authentication Code) authentication + mode (NIST SP800-38B and IETF RFC4493) config CRYPTO_GHASH - tristate "GHASH hash function" + tristate "GHASH" select CRYPTO_GF128MUL select CRYPTO_HASH help - GHASH is the hash function used in GCM (Galois/Counter Mode). - It is not a general-purpose cryptographic hash function. + GCM GHASH function (NIST SP800-38D) config CRYPTO_HMAC - tristate "HMAC support" + tristate "HMAC (Keyed-Hash MAC)" select CRYPTO_HASH select CRYPTO_MANAGER help - HMAC: Keyed-Hashing for Message Authentication (RFC2104). - This is required for IPSec. + HMAC (Keyed-Hash Message Authentication Code) (FIPS 198 and + RFC2104) + + This is required for IPsec AH (XFRM_AH) and IPsec ESP (XFRM_ESP). config CRYPTO_MD4 - tristate "MD4 digest algorithm" + tristate "MD4" select CRYPTO_HASH help - MD4 message digest algorithm (RFC1320). + MD4 message digest algorithm (RFC1320) config CRYPTO_MD5 - tristate "MD5 digest algorithm" + tristate "MD5" select CRYPTO_HASH help - MD5 message digest algorithm (RFC1321). + MD5 message digest algorithm (RFC1321) config CRYPTO_MICHAEL_MIC - tristate "Michael MIC keyed digest algorithm" + tristate "Michael MIC" select CRYPTO_HASH help - Michael MIC is used for message integrity protection in TKIP - (IEEE 802.11i). This algorithm is required for TKIP, but it - should not be used for other purposes because of the weakness - of the algorithm. + Michael MIC (Message Integrity Code) (IEEE 802.11i) + + Defined by the IEEE 802.11i TKIP (Temporal Key Integrity Protocol), + known as WPA (Wif-Fi Protected Access). + + This algorithm is required for TKIP, but it should not be used for + other purposes because of the weakness of the algorithm. config CRYPTO_POLYVAL tristate select CRYPTO_GF128MUL select CRYPTO_HASH help - POLYVAL is the hash function used in HCTR2. It is not a general-purpose + POLYVAL hash function for HCTR2 + + This is used in HCTR2. It is not a general-purpose cryptographic hash function. config CRYPTO_POLY1305 - tristate "Poly1305 authenticator algorithm" + tristate "Poly1305" select CRYPTO_HASH select CRYPTO_LIB_POLY1305_GENERIC help - Poly1305 authenticator algorithm, RFC7539. + Poly1305 authenticator algorithm (RFC7539) Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use in IETF protocols. This is the portable C implementation of Poly1305. config CRYPTO_RMD160 - tristate "RIPEMD-160 digest algorithm" + tristate "RIPEMD-160" select CRYPTO_HASH help - RIPEMD-160 (ISO/IEC 10118-3:2004). + RIPEMD-160 hash function (ISO/IEC 10118-3) RIPEMD-160 is a 160-bit cryptographic hash function. It is intended to be used as a secure replacement for the 128-bit hash functions MD4, MD5 and its predecessor RIPEMD (not to be confused with RIPEMD-128). - It's speed is comparable to SHA1 and there are no known attacks + Its speed is comparable to SHA-1 and there are no known attacks against RIPEMD-160. Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See + See https://homes.esat.kuleuven.be/~bosselae/ripemd160.html + for further information. config CRYPTO_SHA1 - tristate "SHA1 digest algorithm" + tristate "SHA-1" select CRYPTO_HASH select CRYPTO_LIB_SHA1 help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). + SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3) config CRYPTO_SHA256 - tristate "SHA224 and SHA256 digest algorithm" + tristate "SHA-224 and SHA-256" select CRYPTO_HASH select CRYPTO_LIB_SHA256 help - SHA256 secure hash standard (DFIPS 180-2). - - This version of SHA implements a 256 bit hash with 128 bits of - security against collision attacks. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) - This code also includes SHA-224, a 224 bit hash with 112 bits - of security against collision attacks. + This is required for IPsec AH (XFRM_AH) and IPsec ESP (XFRM_ESP). + Used by the btrfs filesystem, Ceph, NFS, and SMB. config CRYPTO_SHA512 - tristate "SHA384 and SHA512 digest algorithms" + tristate "SHA-384 and SHA-512" select CRYPTO_HASH help - SHA512 secure hash standard (DFIPS 180-2). - - This version of SHA implements a 512 bit hash with 256 bits of - security against collision attacks. - - This code also includes SHA-384, a 384 bit hash with 192 bits - of security against collision attacks. + SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) config CRYPTO_SHA3 - tristate "SHA3 digest algorithm" + tristate "SHA-3" select CRYPTO_HASH help - SHA-3 secure hash standard (DFIPS 202). It's based on - cryptographic sponge function family called Keccak. - - References: - http://keccak.noekeon.org/ + SHA-3 secure hash algorithms (FIPS 202, ISO/IEC 10118-3) config CRYPTO_SM3 tristate config CRYPTO_SM3_GENERIC - tristate "SM3 digest algorithm" + tristate "SM3 (ShangMi 3)" select CRYPTO_HASH select CRYPTO_SM3 help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). - It is part of the Chinese Commercial Cryptography suite. + SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012, ISO/IEC 10118-3) + + This is part of the Chinese Commercial Cryptography suite. References: http://www.oscca.gov.cn/UpFile/20101222141857786.pdf https://datatracker.ietf.org/doc/html/draft-shen-sm3-hash config CRYPTO_STREEBOG - tristate "Streebog Hash Function" + tristate "Streebog" select CRYPTO_HASH help - Streebog Hash Function (GOST R 34.11-2012, RFC 6986) is one of the Russian - cryptographic standard algorithms (called GOST algorithms). - This setting enables two hash algorithms with 256 and 512 bits output. + Streebog Hash Function (GOST R 34.11-2012, RFC 6986, ISO/IEC 10118-3) + + This is one of the Russian cryptographic standard algorithms (called + GOST algorithms). This setting enables two hash algorithms with + 256 and 512 bits output. References: https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf https://tools.ietf.org/html/rfc6986 config CRYPTO_VMAC - tristate "VMAC support" + tristate "VMAC" select CRYPTO_HASH select CRYPTO_MANAGER help VMAC is a message authentication algorithm designed for very high speed on 64-bit architectures. - See also: - + See https://fastcrypto.org/vmac for further information. config CRYPTO_WP512 - tristate "Whirlpool digest algorithms" + tristate "Whirlpool" select CRYPTO_HASH help - Whirlpool hash algorithm 512, 384 and 256-bit hashes + Whirlpool hash function (ISO/IEC 10118-3) + + 512, 384 and 256-bit hashes. Whirlpool-512 is part of the NESSIE cryptographic primitives. - Whirlpool will be part of the ISO/IEC 10118-3:2003(E) standard - See also: - + See https://web.archive.org/web/20171129084214/http://www.larc.usp.br/~pbarreto/WhirlpoolPage.html + for further information. config CRYPTO_XCBC - tristate "XCBC support" + tristate "XCBC-MAC (Extended Cipher Block Chaining MAC)" select CRYPTO_HASH select CRYPTO_MANAGER help - XCBC: Keyed-Hashing with encryption algorithm - https://www.ietf.org/rfc/rfc3566.txt - http://csrc.nist.gov/encryption/modes/proposedmodes/ - xcbc-mac/xcbc-mac-spec.pdf + XCBC-MAC (Extended Cipher Block Chaining Message Authentication + Code) (RFC3566) config CRYPTO_XXHASH - tristate "xxHash hash algorithm" + tristate "xxHash" select CRYPTO_HASH select XXHASH help - xxHash non-cryptographic hash algorithm. Extremely fast, working at - speeds close to RAM limits. + xxHash non-cryptographic hash algorithm + + Extremely fast, working at speeds close to RAM limits. + + Used by the btrfs filesystem. endmenu -- cgit From 9bc517155f41fbe387a4602e1860ab1ae0eae638 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:49 -0500 Subject: crypto: Kconfig - simplify userspace entries Shorten menu titles and make them consistent: - acronym - name - architecture features in parenthesis - no suffixes like " algorithm", "support", or "hardware acceleration", or "optimized" Simplify help text descriptions, update references, and ensure that https references are still valid. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- crypto/Kconfig | 65 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 0a385a7aa040..6621122984c0 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1291,60 +1291,72 @@ config CRYPTO_KDF800108_CTR select CRYPTO_SHA256 endmenu -menu "User-space interface" +menu "Userspace interface" config CRYPTO_USER_API tristate config CRYPTO_USER_API_HASH - tristate "User-space interface for hash algorithms" + tristate "Hash algorithms" depends on NET select CRYPTO_HASH select CRYPTO_USER_API help - This option enables the user-spaces interface for hash - algorithms. + Enable the userspace interface for hash algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_SKCIPHER - tristate "User-space interface for symmetric key cipher algorithms" + tristate "Symmetric key cipher algorithms" depends on NET select CRYPTO_SKCIPHER select CRYPTO_USER_API help - This option enables the user-spaces interface for symmetric - key cipher algorithms. + Enable the userspace interface for symmetric key cipher algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_RNG - tristate "User-space interface for random number generator algorithms" + tristate "RNG (random number generator) algorithms" depends on NET select CRYPTO_RNG select CRYPTO_USER_API help - This option enables the user-spaces interface for random - number generator algorithms. + Enable the userspace interface for RNG (random number generator) + algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_RNG_CAVP bool "Enable CAVP testing of DRBG" depends on CRYPTO_USER_API_RNG && CRYPTO_DRBG help - This option enables extra API for CAVP testing via the user-space - interface: resetting of DRBG entropy, and providing Additional Data. + Enable extra APIs in the userspace interface for NIST CAVP + (Cryptographic Algorithm Validation Program) testing: + - resetting DRBG entropy + - providing Additional Data + This should only be enabled for CAVP testing. You should say no unless you know what this is. config CRYPTO_USER_API_AEAD - tristate "User-space interface for AEAD cipher algorithms" + tristate "AEAD cipher algorithms" depends on NET select CRYPTO_AEAD select CRYPTO_SKCIPHER select CRYPTO_NULL select CRYPTO_USER_API help - This option enables the user-spaces interface for AEAD - cipher algorithms. + Enable the userspace interface for AEAD cipher algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_ENABLE_OBSOLETE - bool "Enable obsolete cryptographic algorithms for userspace" + bool "Obsolete cryptographic algorithms" depends on CRYPTO_USER_API default y help @@ -1353,16 +1365,21 @@ config CRYPTO_USER_API_ENABLE_OBSOLETE only useful for userspace clients that still rely on them. config CRYPTO_STATS - bool "Crypto usage statistics for User-space" + bool "Crypto usage statistics" depends on CRYPTO_USER help - This option enables the gathering of crypto stats. - This will collect: - - encrypt/decrypt size and numbers of symmeric operations - - compress/decompress size and numbers of compress operations - - size and numbers of hash operations - - encrypt/decrypt/sign/verify numbers for asymmetric operations - - generate/seed numbers for rng operations + Enable the gathering of crypto stats. + + This collects data sizes, numbers of requests, and numbers + of errors processed by: + - AEAD ciphers (encrypt, decrypt) + - asymmetric key ciphers (encrypt, decrypt, verify, sign) + - symmetric key ciphers (encrypt, decrypt) + - compression algorithms (compress, decompress) + - hash algorithms (hash) + - key-agreement protocol primitives (setsecret, generate + public key, compute shared secret) + - RNG (generate, seed) endmenu -- cgit From cf514b2a5902ee4f93e9636ace5228fed27f23bb Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:50 -0500 Subject: crypto: Kconfig - simplify cipher entries Shorten menu titles and make them consistent: - acronym - name - architecture features in parenthesis - no suffixes like " algorithm", "support", or "hardware acceleration", or "optimized" Simplify help text descriptions, update references, and ensure that https references are still valid. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 40 +++++-- arch/arm64/crypto/Kconfig | 109 ++++++++++++++++-- arch/mips/crypto/Kconfig | 7 +- arch/powerpc/crypto/Kconfig | 14 ++- arch/s390/crypto/Kconfig | 28 +++-- arch/sparc/crypto/Kconfig | 48 +++----- arch/x86/crypto/Kconfig | 274 +++++++++++++++++--------------------------- crypto/Kconfig | 234 +++++++++++++++++++------------------ 8 files changed, 410 insertions(+), 344 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index e64e9b8418d6..3858c4d4cb98 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -144,11 +144,13 @@ config CRYPTO_SHA512_ARM - NEON (Advanced SIMD) extensions config CRYPTO_AES_ARM - tristate "Scalar AES cipher for ARM" + tristate "Ciphers: AES" select CRYPTO_ALGAPI select CRYPTO_AES help - Use optimized AES assembler routines for ARM platforms. + Block ciphers: AES cipher algorithms (FIPS-197) + + Architecture: arm On ARM processors without the Crypto Extensions, this is the fastest AES implementation for single blocks. For multiple @@ -160,7 +162,7 @@ config CRYPTO_AES_ARM such attacks very difficult. config CRYPTO_AES_ARM_BS - tristate "Bit sliced AES using NEON instructions" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (bit-sliced NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES @@ -168,8 +170,13 @@ config CRYPTO_AES_ARM_BS select CRYPTO_CBC select CRYPTO_SIMD help - Use a faster and more secure NEON based implementation of AES in CBC, - CTR and XTS modes + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode and for XTS mode encryption, CBC and XTS mode decryption speedup is @@ -178,19 +185,34 @@ config CRYPTO_AES_ARM_BS believed to be invulnerable to cache timing attacks. config CRYPTO_AES_ARM_CE - tristate "Accelerated AES using ARMv8 Crypto Extensions" + tristate "Ciphers: AES, modes: ECB/CBC/CTS/CTR/XTS (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES select CRYPTO_SIMD help - Use an implementation of AES in CBC, CTR and XTS modes that uses - ARMv8 Crypto Extensions + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - CTS (Cipher Text Stealing) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Architecture: arm using: + - ARMv8 Crypto Extensions config CRYPTO_CHACHA20_NEON - tristate "NEON and scalar accelerated ChaCha stream cipher algorithms" + tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (NEON)" select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms + + Architecture: arm using: + - NEON (Advanced SIMD) extensions config CRYPTO_CRC32_ARM_CE tristate "CRC32C and CRC32" diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 709598f6d2e3..7ba9bcb6d409 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -118,66 +118,155 @@ config CRYPTO_POLYVAL_ARM64_CE - ARMv8 Crypto Extensions config CRYPTO_AES_ARM64 - tristate "AES core cipher using scalar instructions" + tristate "Ciphers: AES, modes: ECB, CBC, CTR, CTS, XCTR, XTS" select CRYPTO_AES + help + Block ciphers: AES cipher algorithms (FIPS-197) + Length-preserving ciphers: AES with ECB, CBC, CTR, CTS, + XCTR, and XTS modes + AEAD cipher: AES with CBC, ESSIV, and SHA-256 + for fscrypt and dm-crypt + + Architecture: arm64 config CRYPTO_AES_ARM64_CE - tristate "AES core cipher using ARMv8 Crypto Extensions" + tristate "Ciphers: AES (ARMv8 Crypto Extensions)" depends on ARM64 && KERNEL_MODE_NEON select CRYPTO_ALGAPI select CRYPTO_LIB_AES + help + Block ciphers: AES cipher algorithms (FIPS-197) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_AES_ARM64_CE_BLK - tristate "AES in ECB/CBC/CTR/XTS/XCTR modes using ARMv8 Crypto Extensions" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_CE + help + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_AES_ARM64_NEON_BLK - tristate "AES in ECB/CBC/CTR/XTS/XCTR modes using NEON instructions" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES + help + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions config CRYPTO_CHACHA20_NEON - tristate "ChaCha20, XChaCha20, and XChaCha12 stream ciphers using NEON instructions" + tristate "Ciphers: ChaCha (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions config CRYPTO_AES_ARM64_BS - tristate "AES in ECB/CBC/CTR/XTS modes using bit-sliced NEON algorithm" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XCTR/XTS modes (bit-sliced NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_NEON_BLK select CRYPTO_LIB_AES + help + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XCTR mode for HCTR2 + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Architecture: arm64 using: + - bit-sliced algorithm + - NEON (Advanced SIMD) extensions config CRYPTO_SM4_ARM64_CE - tristate "SM4 symmetric cipher (ARMv8.2 Crypto Extensions)" + tristate "Ciphers: SM4 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_ALGAPI select CRYPTO_SM4 + help + Block ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016) + + Architecture: arm64 using: + - ARMv8.2 Crypto Extensions + - NEON (Advanced SIMD) extensions config CRYPTO_SM4_ARM64_CE_BLK - tristate "SM4 in ECB/CBC/CFB/CTR modes using ARMv8 Crypto Extensions" + tristate "Ciphers: SM4, modes: ECB/CBC/CFB/CTR (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_SM4 + help + Length-preserving ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CFB (Cipher Feedback) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions + - NEON (Advanced SIMD) extensions config CRYPTO_SM4_ARM64_NEON_BLK - tristate "SM4 in ECB/CBC/CFB/CTR modes using NEON instructions" + tristate "Ciphers: SM4, modes: ECB/CBC/CFB/CTR (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_SM4 + help + Length-preserving ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CFB (Cipher Feedback) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions config CRYPTO_AES_ARM64_CE_CCM - tristate "AES in CCM mode using ARMv8 Crypto Extensions" + tristate "AEAD cipher: AES in CCM mode (ARMv8 Crypto Extensions)" depends on ARM64 && KERNEL_MODE_NEON select CRYPTO_ALGAPI select CRYPTO_AES_ARM64_CE select CRYPTO_AEAD select CRYPTO_LIB_AES + help + AEAD cipher: AES cipher algorithms (FIPS-197) with + CCM (Counter with Cipher Block Chaining-Message Authentication Code) + authenticated encryption mode (NIST SP800-38C) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions + - NEON (Advanced SIMD) extensions config CRYPTO_CRCT10DIF_ARM64_CE tristate "CRCT10DIF (PMULL)" diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index de162f69675c..9003a5c1e879 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -61,9 +61,14 @@ config CRYPTO_SHA512_OCTEON Architecture: mips OCTEON using crypto instructions, when available config CRYPTO_CHACHA_MIPS - tristate "ChaCha stream cipher algorithms (MIPS 32r2 optimized)" + tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (MIPS32r2)" depends on CPU_MIPS32_R2 select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms + + Architecture: MIPS32r2 endmenu diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 5a4770a029ef..c1b964447401 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -73,12 +73,20 @@ config CRYPTO_SHA256_PPC_SPE - SPE (Signal Processing Engine) extensions config CRYPTO_AES_PPC_SPE - tristate "AES cipher algorithms (PPC SPE)" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)" depends on PPC && SPE select CRYPTO_SKCIPHER help - AES cipher algorithms (FIPS-197). Additionally the acceleration - for popular block cipher modes ECB, CBC, CTR and XTS is supported. + Block ciphers: AES cipher algorithms (FIPS-197) + Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes + + Architecture: powerpc using: + - SPE (Signal Processing Engine) extensions + + SPE is available for: + - Processor Type: Freescale 8500 + - CPU selection: e500 (8540) + This module should only be used for low power (router) devices without hardware AES acceleration (e.g. caam crypto). It reduces the size of the AES tables from 16KB to 8KB + 256 bytes and mitigates diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 04cc3a6467ab..06ee706b0d78 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -81,44 +81,54 @@ config CRYPTO_GHASH_S390 It is available as of z196. config CRYPTO_AES_S390 - tristate "AES cipher algorithms" + tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM" depends on S390 select CRYPTO_ALGAPI select CRYPTO_SKCIPHER help - This is the s390 hardware accelerated implementation of the - AES cipher algorithms (FIPS-197). + Block cipher: AES cipher algorithms (FIPS 197) + AEAD cipher: AES with GCM + Length-preserving ciphers: AES with ECB, CBC, XTS, and CTR modes + + Architecture: s390 As of z9 the ECB and CBC modes are hardware accelerated for 128 bit keys. + As of z10 the ECB and CBC modes are hardware accelerated for all AES key sizes. + As of z196 the CTR mode is hardware accelerated for all AES key sizes and XTS mode is hardware accelerated for 256 and 512 bit keys. config CRYPTO_DES_S390 - tristate "DES and Triple DES cipher algorithms" + tristate "Ciphers: DES and Triple DES EDE, modes: ECB, CBC, CTR" depends on S390 select CRYPTO_ALGAPI select CRYPTO_SKCIPHER select CRYPTO_LIB_DES help - This is the s390 hardware accelerated implementation of the - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). + Block ciphers: DES (FIPS 46-2) cipher algorithm + Block ciphers: Triple DES EDE (FIPS 46-3) cipher algorithm + Length-preserving ciphers: DES with ECB, CBC, and CTR modes + Length-preserving ciphers: Triple DES EDED with ECB, CBC, and CTR modes + + Architecture: s390 As of z990 the ECB and CBC mode are hardware accelerated. As of z196 the CTR mode is hardware accelerated. config CRYPTO_CHACHA_S390 - tristate "ChaCha20 stream cipher" + tristate "Ciphers: ChaCha20" depends on S390 select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA help - This is the s390 SIMD implementation of the ChaCha20 stream - cipher (RFC 7539). + Length-preserving cipher: ChaCha20 stream cipher (RFC 7539) + + Architecture: s390 It is available as of z13. diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index 519348de6860..cfe5102b1c68 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -3,14 +3,18 @@ menu "Accelerated Cryptographic Algorithms for CPU (sparc64)" config CRYPTO_DES_SPARC64 - tristate "DES and Triple DES EDE cipher algorithms (SPARC64)" + tristate "Ciphers: DES and Triple DES EDE, modes: ECB/CBC" depends on SPARC64 select CRYPTO_ALGAPI select CRYPTO_LIB_DES select CRYPTO_SKCIPHER help - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3), - optimized using SPARC64 crypto opcodes. + Block cipher: DES (FIPS 46-2) cipher algorithm + Block cipher: Triple DES EDE (FIPS 46-3) cipher algorithm + Length-preserving ciphers: DES with ECB and CBC modes + Length-preserving ciphers: Tripe DES EDE with ECB and CBC modes + + Architecture: sparc64 config CRYPTO_CRC32C_SPARC64 tristate "CRC32c" @@ -63,46 +67,24 @@ config CRYPTO_SHA512_SPARC64 Architecture: sparc64 using crypto instructions, when available config CRYPTO_AES_SPARC64 - tristate "AES cipher algorithms (SPARC64)" + tristate "Ciphers: AES, modes: ECB, CBC, CTR" depends on SPARC64 select CRYPTO_SKCIPHER help - Use SPARC64 crypto opcodes for AES algorithm. - - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. - - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. - - The AES specifies three key sizes: 128, 192 and 256 bits + Block ciphers: AES cipher algorithms (FIPS-197) + Length-preseving ciphers: AES with ECB, CBC, and CTR modes - See for more information. - - In addition to AES cipher algorithm support, the acceleration - for some popular block cipher mode is supported too, including - ECB and CBC. + Architecture: sparc64 using crypto instructions config CRYPTO_CAMELLIA_SPARC64 - tristate "Camellia cipher algorithm (SPARC64)" + tristate "Ciphers: Camellia, modes: ECB, CBC" depends on SPARC64 select CRYPTO_ALGAPI select CRYPTO_SKCIPHER help - Camellia cipher algorithm module (SPARC64). - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. + Block ciphers: Camellia cipher algorithms + Length-preserving ciphers: Camellia with ECB and CBC modes - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - + Architecture: sparc64 endmenu diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index fc24f4562700..9bb0f7939c6b 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -14,7 +14,7 @@ config CRYPTO_CURVE25519_X86 - ADX (large integer arithmetic) config CRYPTO_AES_NI_INTEL - tristate "AES cipher algorithms (AES-NI)" + tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTR, XTS, GCM (AES-NI)" depends on X86 select CRYPTO_AEAD select CRYPTO_LIB_AES @@ -22,96 +22,63 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_SKCIPHER select CRYPTO_SIMD help - Use Intel AES-NI instructions for AES algorithm. + Block cipher: AES cipher algorithms + AEAD cipher: AES with GCM + Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTR, XTS - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. - - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. - - The AES specifies three key sizes: 128, 192 and 256 bits - - See for more information. - - In addition to AES cipher algorithm support, the acceleration - for some popular block cipher mode is supported too, including - ECB, CBC, LRW, XTS. The 64 bit version has additional - acceleration for CTR and XCTR. + Architecture: x86 (32-bit and 64-bit) using: + - AES-NI (AES new instructions) config CRYPTO_BLOWFISH_X86_64 - tristate "Blowfish cipher algorithm (x86_64)" + tristate "Ciphers: Blowfish, modes: ECB, CBC" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_BLOWFISH_COMMON imply CRYPTO_CTR help - Blowfish cipher algorithm (x86_64), by Bruce Schneier. - - This is a variable key length cipher which can use keys from 32 - bits to 448 bits in length. It's fast, simple and specifically - designed for use on "large microprocessors". + Block cipher: Blowfish cipher algorithm + Length-preserving ciphers: Blowfish with ECB and CBC modes - See also: - + Architecture: x86_64 config CRYPTO_CAMELLIA_X86_64 - tristate "Camellia cipher algorithm (x86_64)" + tristate "Ciphers: Camellia with modes: ECB, CBC" depends on X86 && 64BIT select CRYPTO_SKCIPHER imply CRYPTO_CTR help - Camellia cipher algorithm module (x86_64). - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. + Block cipher: Camellia cipher algorithms + Length-preserving ciphers: Camellia with ECB and CBC modes - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - + Architecture: x86_64 config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 - tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX)" + tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAMELLIA_X86_64 select CRYPTO_SIMD imply CRYPTO_XTS help - Camellia cipher algorithm module (x86_64/AES-NI/AVX). - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. + Length-preserving ciphers: Camellia with ECB and CBC modes - See also: - + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX (Advanced Vector Extensions) config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 - tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX2)" + tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX2)" depends on X86 && 64BIT select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 help - Camellia cipher algorithm module (x86_64/AES-NI/AVX2). + Length-preserving ciphers: Camellia with ECB and CBC modes - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX2 (Advanced Vector Extensions 2) config CRYPTO_CAST5_AVX_X86_64 - tristate "CAST5 (CAST-128) cipher algorithm (x86_64/AVX)" + tristate "Ciphers: CAST5 with modes: ECB, CBC (AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAST5 @@ -119,14 +86,16 @@ config CRYPTO_CAST5_AVX_X86_64 select CRYPTO_SIMD imply CRYPTO_CTR help - The CAST5 encryption algorithm (synonymous with CAST-128) is - described in RFC2144. + Length-preserving ciphers: CAST5 (CAST-128) cipher algorithm + (RFC2144) with ECB and CBC modes - This module provides the Cast5 cipher algorithm that processes - sixteen blocks parallel using the AVX instruction set. + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) + + Processes 16 blocks in parallel. config CRYPTO_CAST6_AVX_X86_64 - tristate "CAST6 (CAST-256) cipher algorithm (x86_64/AVX)" + tristate "Ciphers: CAST6 with modes: ECB, CBC (AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_CAST6 @@ -135,66 +104,62 @@ config CRYPTO_CAST6_AVX_X86_64 imply CRYPTO_XTS imply CRYPTO_CTR help - The CAST6 encryption algorithm (synonymous with CAST-256) is - described in RFC2612. + Length-preserving ciphers: CAST6 (CAST-256) cipher algorithm + (RFC2612) with ECB and CBC modes + + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) - This module provides the Cast6 cipher algorithm that processes - eight blocks parallel using the AVX instruction set. + Processes eight blocks in parallel. config CRYPTO_DES3_EDE_X86_64 - tristate "Triple DES EDE cipher algorithm (x86-64)" + tristate "Ciphers: Triple DES EDE with modes: ECB, CBC" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_LIB_DES imply CRYPTO_CTR help - Triple DES EDE (FIPS 46-3) algorithm. + Block cipher: Triple DES EDE (FIPS 46-3) cipher algorithm + Length-preserving ciphers: Triple DES EDE with ECB and CBC modes + + Architecture: x86_64 - This module provides implementation of the Triple DES EDE cipher - algorithm that is optimized for x86-64 processors. Two versions of - algorithm are provided; regular processing one input block and - one that processes three blocks parallel. + Processes one or three blocks in parallel. config CRYPTO_SERPENT_SSE2_X86_64 - tristate "Serpent cipher algorithm (x86_64/SSE2)" + tristate "Ciphers: Serpent with modes: ECB, CBC (SSE2)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT select CRYPTO_SIMD imply CRYPTO_CTR help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. - - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes - This module provides Serpent cipher algorithm that processes eight - blocks parallel using SSE2 instruction set. + Architecture: x86_64 using: + - SSE2 (Streaming SIMD Extensions 2) - See also: - + Processes eight blocks in parallel. config CRYPTO_SERPENT_SSE2_586 - tristate "Serpent cipher algorithm (i586/SSE2)" + tristate "Ciphers: Serpent with modes: ECB, CBC (32-bit with SSE2)" depends on X86 && !64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT select CRYPTO_SIMD imply CRYPTO_CTR help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides Serpent cipher algorithm that processes four - blocks parallel using SSE2 instruction set. + Architecture: x86 (32-bit) using: + - SSE2 (Streaming SIMD Extensions 2) - See also: - + Processes four blocks in parallel. config CRYPTO_SERPENT_AVX_X86_64 - tristate "Serpent cipher algorithm (x86_64/AVX)" + tristate "Ciphers: Serpent with modes: ECB, CBC (AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SERPENT @@ -202,56 +167,50 @@ config CRYPTO_SERPENT_AVX_X86_64 imply CRYPTO_XTS imply CRYPTO_CTR help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides the Serpent cipher algorithm that processes - eight blocks parallel using the AVX instruction set. + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) - See also: - + Processes eight blocks in parallel. config CRYPTO_SERPENT_AVX2_X86_64 - tristate "Serpent cipher algorithm (x86_64/AVX2)" + tristate "Ciphers: Serpent with modes: ECB, CBC (AVX2)" depends on X86 && 64BIT select CRYPTO_SERPENT_AVX_X86_64 help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides Serpent cipher algorithm that processes 16 - blocks parallel using AVX2 instruction set. + Architecture: x86_64 using: + - AVX2 (Advanced Vector Extensions 2) - See also: - + Processes 16 blocks in parallel. config CRYPTO_SM4_AESNI_AVX_X86_64 - tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX)" + tristate "Ciphers: SM4 with modes: ECB, CBC, CFB, CTR (AES-NI/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SIMD select CRYPTO_ALGAPI select CRYPTO_SM4 help - SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX). + Length-preserving ciphers: SM4 cipher algorithms + (OSCCA GB/T 32907-2016) with ECB, CBC, CFB, and CTR modes - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX (Advanced Vector Extensions) - This is SM4 optimized implementation using AES-NI/AVX/x86_64 - instruction set for block cipher. Through two affine transforms, + Through two affine transforms, we can use the AES S-Box to simulate the SM4 S-Box to achieve the effect of instruction acceleration. If unsure, say N. config CRYPTO_SM4_AESNI_AVX2_X86_64 - tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX2)" + tristate "Ciphers: SM4 with modes: ECB, CBC, CFB, CTR (AES-NI/AVX2)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SIMD @@ -259,75 +218,58 @@ config CRYPTO_SM4_AESNI_AVX2_X86_64 select CRYPTO_SM4 select CRYPTO_SM4_AESNI_AVX_X86_64 help - SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX2). + Length-preserving ciphers: SM4 cipher algorithms + (OSCCA GB/T 32907-2016) with ECB, CBC, CFB, and CTR modes - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX2 (Advanced Vector Extensions 2) - This is SM4 optimized implementation using AES-NI/AVX2/x86_64 - instruction set for block cipher. Through two affine transforms, + Through two affine transforms, we can use the AES S-Box to simulate the SM4 S-Box to achieve the effect of instruction acceleration. If unsure, say N. config CRYPTO_TWOFISH_586 - tristate "Twofish cipher algorithms (i586)" + tristate "Ciphers: Twofish (32-bit)" depends on (X86 || UML_X86) && !64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON imply CRYPTO_CTR help - Twofish cipher algorithm. - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + Block cipher: Twofish cipher algorithm - See also: - + Architecture: x86 (32-bit) config CRYPTO_TWOFISH_X86_64 - tristate "Twofish cipher algorithm (x86_64)" + tristate "Ciphers: Twofish" depends on (X86 || UML_X86) && 64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON imply CRYPTO_CTR help - Twofish cipher algorithm (x86_64). + Block cipher: Twofish cipher algorithm - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. - - See also: - + Architecture: x86_64 config CRYPTO_TWOFISH_X86_64_3WAY - tristate "Twofish cipher algorithm (x86_64, 3-way parallel)" + tristate "Ciphers: Twofish with modes: ECB, CBC (3-way parallel)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 help - Twofish cipher algorithm (x86_64, 3-way parallel). - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + Length-preserving cipher: Twofish cipher algorithm + with ECB and CBC modes - This module provides Twofish cipher algorithm that processes three - blocks parallel, utilizing resources of out-of-order CPUs better. + Architecture: x86_64 - See also: - + Processes three blocks in parallel, better utilizing resources of + out-of-order CPUs. config CRYPTO_TWOFISH_AVX_X86_64 - tristate "Twofish cipher algorithm (x86_64/AVX)" + tristate "Ciphers: Twofish with modes: ECB, CBC (AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SIMD @@ -336,28 +278,28 @@ config CRYPTO_TWOFISH_AVX_X86_64 select CRYPTO_TWOFISH_X86_64_3WAY imply CRYPTO_XTS help - Twofish cipher algorithm (x86_64/AVX). - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + Length-preserving cipher: Twofish cipher algorithm + with ECB and CBC modes - This module provides the Twofish cipher algorithm that processes - eight blocks parallel using the AVX Instruction Set. + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) - See also: - + Processes eight blocks in parallel. config CRYPTO_CHACHA20_X86_64 - tristate "ChaCha stream cipher algorithms (x86_64/SSSE3/AVX2/AVX-512VL)" + tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (SSSE3/AVX2/AVX-512VL)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA help - SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20, - XChaCha20, and XChaCha12 stream ciphers. + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX2 (Advanced Vector Extensions 2) + - AVX-512VL (Advanced Vector Extensions-512VL) config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE2)" diff --git a/crypto/Kconfig b/crypto/Kconfig index 6621122984c0..89a6cb5ee63f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -219,7 +219,8 @@ config CRYPTO_AUTHENC select CRYPTO_NULL help Authenc: Combined mode wrapper for IPsec. - This is required for IPSec. + + This is required for IPSec ESP (XFRM_ESP). config CRYPTO_TEST tristate "Testing module" @@ -336,12 +337,11 @@ endmenu menu "Block ciphers" config CRYPTO_AES - tristate "AES cipher algorithms" + tristate "AES (Advanced Encryption Standard)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES help - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. + AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) Rijndael appears to be consistently a very good performer in both hardware and software across a wide range of computing @@ -354,13 +354,13 @@ config CRYPTO_AES The AES specifies three key sizes: 128, 192 and 256 bits - See for more information. - config CRYPTO_AES_TI - tristate "Fixed time AES cipher" + tristate "AES (Advanced Encryption Standard) (fixed time)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES help + AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) + This is a generic implementation of AES that attempts to eliminate data dependent latencies as much as possible without affecting performance too much. It is intended for use by the generic CCM @@ -376,25 +376,24 @@ config CRYPTO_AES_TI are evicted when the CPU is interrupted to do something else. config CRYPTO_ANUBIS - tristate "Anubis cipher algorithm" + tristate "Anubis" depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help - Anubis cipher algorithm. + Anubis cipher algorithm Anubis is a variable key length cipher which can use keys from 128 bits to 320 bits in length. It was evaluated as a entrant in the NESSIE competition. - See also: - - + See https://web.archive.org/web/20160606112246/http://www.larc.usp.br/~pbarreto/AnubisPage.html + for further information. config CRYPTO_ARIA - tristate "ARIA cipher algorithm" + tristate "ARIA" select CRYPTO_ALGAPI help - ARIA cipher algorithm (RFC5794). + ARIA cipher algorithm (RFC5794) ARIA is a standard encryption algorithm of the Republic of Korea. The ARIA specifies three key sizes and rounds. @@ -402,22 +401,21 @@ config CRYPTO_ARIA 192-bit: 14 rounds. 256-bit: 16 rounds. - See also: - + See: + https://seed.kisa.or.kr/kisa/algorithm/EgovAriaInfo.do config CRYPTO_BLOWFISH - tristate "Blowfish cipher algorithm" + tristate "Blowfish" select CRYPTO_ALGAPI select CRYPTO_BLOWFISH_COMMON help - Blowfish cipher algorithm, by Bruce Schneier. + Blowfish cipher algorithm, by Bruce Schneier This is a variable key length cipher which can use keys from 32 bits to 448 bits in length. It's fast, simple and specifically designed for use on "large microprocessors". - See also: - + See https://www.schneier.com/blowfish.html for further information. config CRYPTO_BLOWFISH_COMMON tristate @@ -425,22 +423,18 @@ config CRYPTO_BLOWFISH_COMMON Common parts of the Blowfish cipher algorithm shared by the generic c and the assembler implementations. - See also: - - config CRYPTO_CAMELLIA - tristate "Camellia cipher algorithms" + tristate "Camellia" select CRYPTO_ALGAPI help - Camellia cipher algorithms module. + Camellia cipher algorithms (ISO/IEC 18033-3) Camellia is a symmetric key block cipher developed jointly at NTT and Mitsubishi Electric Corporation. The Camellia specifies three key sizes: 128, 192 and 256 bits. - See also: - + See https://info.isl.ntt.co.jp/crypt/eng/camellia/ for further information. config CRYPTO_CAST_COMMON tristate @@ -449,85 +443,87 @@ config CRYPTO_CAST_COMMON generic c and the assembler implementations. config CRYPTO_CAST5 - tristate "CAST5 (CAST-128) cipher algorithm" + tristate "CAST5 (CAST-128)" select CRYPTO_ALGAPI select CRYPTO_CAST_COMMON help - The CAST5 encryption algorithm (synonymous with CAST-128) is - described in RFC2144. + CAST5 (CAST-128) cipher algorithm (RFC2144, ISO/IEC 18033-3) config CRYPTO_CAST6 - tristate "CAST6 (CAST-256) cipher algorithm" + tristate "CAST6 (CAST-256)" select CRYPTO_ALGAPI select CRYPTO_CAST_COMMON help - The CAST6 encryption algorithm (synonymous with CAST-256) is - described in RFC2612. + CAST6 (CAST-256) encryption algorithm (RFC2612) config CRYPTO_DES - tristate "DES and Triple DES EDE cipher algorithms" + tristate "DES and Triple DES EDE" select CRYPTO_ALGAPI select CRYPTO_LIB_DES help - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). + DES (Data Encryption Standard)(FIPS 46-2, ISO/IEC 18033-3) and + Triple DES EDE (Encrypt/Decrypt/Encrypt) (FIPS 46-3, ISO/IEC 18033-3) + cipher algorithms config CRYPTO_FCRYPT - tristate "FCrypt cipher algorithm" + tristate "FCrypt" select CRYPTO_ALGAPI select CRYPTO_SKCIPHER help - FCrypt algorithm used by RxRPC. + FCrypt algorithm used by RxRPC + + See https://ota.polyonymo.us/fcrypt-paper.txt config CRYPTO_KHAZAD - tristate "Khazad cipher algorithm" + tristate "Khazad" depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help - Khazad cipher algorithm. + Khazad cipher algorithm Khazad was a finalist in the initial NESSIE competition. It is an algorithm optimized for 64-bit processors with good performance on 32-bit processors. Khazad uses an 128 bit key size. - See also: - + See https://web.archive.org/web/20171011071731/http://www.larc.usp.br/~pbarreto/KhazadPage.html + for further information. config CRYPTO_SEED - tristate "SEED cipher algorithm" + tristate "SEED" depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help - SEED cipher algorithm (RFC4269). + SEED cipher algorithm (RFC4269, ISO/IEC 18033-3) SEED is a 128-bit symmetric key block cipher that has been developed by KISA (Korea Information Security Agency) as a national standard encryption algorithm of the Republic of Korea. It is a 16 round block cipher with the key size of 128 bit. - See also: - + See https://seed.kisa.or.kr/kisa/algorithm/EgovSeedInfo.do + for further information. config CRYPTO_SERPENT - tristate "Serpent cipher algorithm" + tristate "Serpent" select CRYPTO_ALGAPI help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + Serpent cipher algorithm, by Anderson, Biham & Knudsen Keys are allowed to be from 0 to 256 bits in length, in steps of 8 bits. - See also: - + See https://www.cl.cam.ac.uk/~rja14/serpent.html for further information. config CRYPTO_SM4 tristate config CRYPTO_SM4_GENERIC - tristate "SM4 cipher algorithm" + tristate "SM4 (ShangMi 4)" select CRYPTO_ALGAPI select CRYPTO_SM4 help - SM4 cipher algorithms (OSCCA GB/T 32907-2016). + SM4 cipher algorithms (OSCCA GB/T 32907-2016, + ISO/IEC 18033-3:2010/Amd 1:2021) SM4 (GBT.32907-2016) is a cryptographic standard issued by the Organization of State Commercial Administration of China (OSCCA) @@ -544,16 +540,16 @@ config CRYPTO_SM4_GENERIC The input, output, and key of SMS4 are each 128 bits. - See also: + See https://eprint.iacr.org/2008/329.pdf for further information. If unsure, say N. config CRYPTO_TEA - tristate "TEA, XTEA and XETA cipher algorithms" + tristate "TEA, XTEA and XETA" depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help - TEA cipher algorithm. + TEA (Tiny Encryption Algorithm) cipher algorithms Tiny Encryption Algorithm is a simple cipher that uses many rounds for security. It is very fast and uses @@ -567,19 +563,18 @@ config CRYPTO_TEA of the XTEA algorithm for compatibility purposes. config CRYPTO_TWOFISH - tristate "Twofish cipher algorithm" + tristate "Twofish" select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON help - Twofish cipher algorithm. + Twofish cipher algorithm Twofish was submitted as an AES (Advanced Encryption Standard) candidate cipher by researchers at CounterPane Systems. It is a 16 round block cipher supporting key sizes of 128, 192, and 256 bits. - See also: - + See https://www.schneier.com/twofish.html for further information. config CRYPTO_TWOFISH_COMMON tristate @@ -592,14 +587,15 @@ endmenu menu "Length-preserving ciphers and modes" config CRYPTO_ADIANTUM - tristate "Adiantum support" + tristate "Adiantum" select CRYPTO_CHACHA20 select CRYPTO_LIB_POLY1305_GENERIC select CRYPTO_NHPOLY1305 select CRYPTO_MANAGER help - Adiantum is a tweakable, length-preserving encryption mode - designed for fast and secure disk encryption, especially on + Adiantum tweakable, length-preserving encryption mode + + Designed for fast and secure disk encryption, especially on CPUs without dedicated crypto instructions. It encrypts each sector using the XChaCha12 stream cipher, two passes of an ε-almost-∆-universal hash function, and an invocation of @@ -616,12 +612,12 @@ config CRYPTO_ADIANTUM If unsure, say N. config CRYPTO_ARC4 - tristate "ARC4 cipher algorithm" + tristate "ARC4 (Alleged Rivest Cipher 4)" depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_SKCIPHER select CRYPTO_LIB_ARC4 help - ARC4 cipher algorithm. + ARC4 cipher algorithm ARC4 is a stream cipher using keys ranging from 8 bits to 2048 bits in length. This algorithm is required for driver-based @@ -629,113 +625,118 @@ config CRYPTO_ARC4 weakness of the algorithm. config CRYPTO_CHACHA20 - tristate "ChaCha stream cipher algorithms" + tristate "ChaCha" select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_SKCIPHER help - The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms. + The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms ChaCha20 is a 256-bit high-speed stream cipher designed by Daniel J. Bernstein and further specified in RFC7539 for use in IETF protocols. - This is the portable C implementation of ChaCha20. See also: - + This is the portable C implementation of ChaCha20. See + https://cr.yp.to/chacha/chacha-20080128.pdf for further information. XChaCha20 is the application of the XSalsa20 construction to ChaCha20 rather than to Salsa20. XChaCha20 extends ChaCha20's nonce length from 64 bits (or 96 bits using the RFC7539 convention) to 192 bits, - while provably retaining ChaCha20's security. See also: - + while provably retaining ChaCha20's security. See + https://cr.yp.to/snuffle/xsalsa-20081128.pdf for further information. XChaCha12 is XChaCha20 reduced to 12 rounds, with correspondingly reduced security margin but increased performance. It can be needed in some performance-sensitive scenarios. config CRYPTO_CBC - tristate "CBC support" + tristate "CBC (Cipher Block Chaining)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - CBC: Cipher Block Chaining mode - This block cipher algorithm is required for IPSec. + CBC (Cipher Block Chaining) mode (NIST SP800-38A) + + This block cipher mode is required for IPSec ESP (XFRM_ESP). config CRYPTO_CFB - tristate "CFB support" + tristate "CFB (Cipher Feedback)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - CFB: Cipher FeedBack mode - This block cipher algorithm is required for TPM2 Cryptography. + CFB (Cipher Feedback) mode (NIST SP800-38A) + + This block cipher mode is required for TPM2 Cryptography. config CRYPTO_CTR - tristate "CTR support" + tristate "CTR (Counter)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - CTR: Counter mode - This block cipher algorithm is required for IPSec. + CTR (Counter) mode (NIST SP800-38A) config CRYPTO_CTS - tristate "CTS support" + tristate "CTS (Cipher Text Stealing)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - CTS: Cipher Text Stealing - This is the Cipher Text Stealing mode as described by - Section 8 of rfc2040 and referenced by rfc3962 - (rfc3962 includes errata information in its Appendix A) or - CBC-CS3 as defined by NIST in Sp800-38A addendum from Oct 2010. + CBC-CS3 variant of CTS (Cipher Text Stealing) (NIST + Addendum to SP800-38A (October 2010)) + This mode is required for Kerberos gss mechanism support for AES encryption. - See: https://csrc.nist.gov/publications/detail/sp/800-38a/addendum/final - config CRYPTO_ECB - tristate "ECB support" + tristate "ECB (Electronic Codebook)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - ECB: Electronic CodeBook mode - This is the simplest block cipher algorithm. It simply encrypts - the input block by block. + ECB (Electronic Codebook) mode (NIST SP800-38A) config CRYPTO_HCTR2 - tristate "HCTR2 support" + tristate "HCTR2" select CRYPTO_XCTR select CRYPTO_POLYVAL select CRYPTO_MANAGER help - HCTR2 is a length-preserving encryption mode for storage encryption that - is efficient on processors with instructions to accelerate AES and - carryless multiplication, e.g. x86 processors with AES-NI and CLMUL, and - ARM processors with the ARMv8 crypto extensions. + HCTR2 length-preserving encryption mode + + A mode for storage encryption that is efficient on processors with + instructions to accelerate AES and carryless multiplication, e.g. + x86 processors with AES-NI and CLMUL, and ARM processors with the + ARMv8 crypto extensions. + + See https://eprint.iacr.org/2021/1441 config CRYPTO_KEYWRAP - tristate "Key wrapping support" + tristate "KW (AES Key Wrap)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - Support for key wrapping (NIST SP800-38F / RFC3394) without - padding. + KW (AES Key Wrap) authenticated encryption mode (NIST SP800-38F + and RFC3394) without padding. config CRYPTO_LRW - tristate "LRW support" + tristate "LRW (Liskov Rivest Wagner)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER select CRYPTO_GF128MUL select CRYPTO_ECB help - LRW: Liskov Rivest Wagner, a tweakable, non malleable, non movable + LRW (Liskov Rivest Wagner) mode + + A tweakable, non malleable, non movable narrow block cipher mode for dm-crypt. Use it with cipher specification string aes-lrw-benbi, the key must be 256, 320 or 384. The first 128, 192 or 256 bits in the key are used for AES and the rest is used to tie each cipher block to its logical position. + See https://people.csail.mit.edu/rivest/pubs/LRW02.pdf + config CRYPTO_OFB - tristate "OFB support" + tristate "OFB (Output Feedback)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - OFB: the Output Feedback mode makes a block cipher into a synchronous + OFB (Output Feedback) mode (NIST SP800-38A) + + This mode makes a block cipher into a synchronous stream cipher. It generates keystream blocks, which are then XORed with the plaintext blocks to get the ciphertext. Flipping a bit in the ciphertext produces a flipped bit in the plaintext at the same @@ -743,31 +744,38 @@ config CRYPTO_OFB normally even when applied before encryption. config CRYPTO_PCBC - tristate "PCBC support" + tristate "PCBC (Propagating Cipher Block Chaining)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - PCBC: Propagating Cipher Block Chaining mode - This block cipher algorithm is required for RxRPC. + PCBC (Propagating Cipher Block Chaining) mode + + This block cipher mode is required for RxRPC. config CRYPTO_XCTR tristate select CRYPTO_SKCIPHER select CRYPTO_MANAGER help - XCTR: XOR Counter mode. This blockcipher mode is a variant of CTR mode - using XORs and little-endian addition rather than big-endian arithmetic. + XCTR (XOR Counter) mode for HCTR2 + + This blockcipher mode is a variant of CTR mode using XORs and little-endian + addition rather than big-endian arithmetic. + XCTR mode is used to implement HCTR2. config CRYPTO_XTS - tristate "XTS support" + tristate "XTS (XOR Encrypt XOR with ciphertext stealing)" select CRYPTO_SKCIPHER select CRYPTO_MANAGER select CRYPTO_ECB help - XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain, - key size 256, 384 or 512 bits. This implementation currently - can't handle a sectorsize which is not a multiple of 16 bytes. + XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Use with aes-xts-plain, key size 256, 384 or 512 bits. This + implementation currently can't handle a sectorsize which is not a + multiple of 16 bytes. config CRYPTO_NHPOLY1305 tristate @@ -806,7 +814,7 @@ config CRYPTO_CHACHA20POLY1305 mode (RFC8439) config CRYPTO_CCM - tristate "CCM (Counter with Cipher Block Chaining-Message Authentication Code)" + tristate "CCM (Counter with Cipher Block Chaining-MAC)" select CRYPTO_CTR select CRYPTO_HASH select CRYPTO_AEAD @@ -816,7 +824,7 @@ config CRYPTO_CCM authenticated encryption mode (NIST SP800-38C) config CRYPTO_GCM - tristate "GCM (Galois/Counter Mode) and GMAC (GCM Message Authentication Code)" + tristate "GCM (Galois/Counter Mode) and GMAC (GCM MAC)" select CRYPTO_CTR select CRYPTO_AEAD select CRYPTO_GHASH -- cgit From a9a98d49da52c5bf63cd8bbc1f33b52edec2e9c9 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Sat, 20 Aug 2022 13:41:51 -0500 Subject: crypto: Kconfig - simplify compression/RNG entries Shorten menu titles and make them consistent: - acronym - name - architecture features in parenthesis - no suffixes like " algorithm", "support", or "hardware acceleration", or "optimized" Simplify help text descriptions, update references, and ensure that https references are still valid. Signed-off-by: Robert Elliott Signed-off-by: Herbert Xu --- crypto/Kconfig | 82 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 89a6cb5ee63f..40423a14f86f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1178,81 +1178,92 @@ endmenu menu "Compression" config CRYPTO_DEFLATE - tristate "Deflate compression algorithm" + tristate "Deflate" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select ZLIB_INFLATE select ZLIB_DEFLATE help - This is the Deflate algorithm (RFC1951), specified for use in - IPSec with the IPCOMP protocol (RFC3173, RFC2394). + Deflate compression algorithm (RFC1951) - You will most probably want this if using IPSec. + Used by IPSec with the IPCOMP protocol (RFC3173, RFC2394) config CRYPTO_LZO - tristate "LZO compression algorithm" + tristate "LZO" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select LZO_COMPRESS select LZO_DECOMPRESS help - This is the LZO algorithm. + LZO compression algorithm + + See https://www.oberhumer.com/opensource/lzo/ for further information. config CRYPTO_842 - tristate "842 compression algorithm" + tristate "842" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select 842_COMPRESS select 842_DECOMPRESS help - This is the 842 algorithm. + 842 compression algorithm by IBM + + See https://github.com/plauth/lib842 for further information. config CRYPTO_LZ4 - tristate "LZ4 compression algorithm" + tristate "LZ4" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select LZ4_COMPRESS select LZ4_DECOMPRESS help - This is the LZ4 algorithm. + LZ4 compression algorithm + + See https://github.com/lz4/lz4 for further information. config CRYPTO_LZ4HC - tristate "LZ4HC compression algorithm" + tristate "LZ4HC" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select LZ4HC_COMPRESS select LZ4_DECOMPRESS help - This is the LZ4 high compression mode algorithm. + LZ4 high compression mode algorithm + + See https://github.com/lz4/lz4 for further information. config CRYPTO_ZSTD - tristate "Zstd compression algorithm" + tristate "Zstd" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select ZSTD_COMPRESS select ZSTD_DECOMPRESS help - This is the zstd algorithm. + zstd compression algorithm + + See https://github.com/facebook/zstd for further information. endmenu menu "Random number generation" config CRYPTO_ANSI_CPRNG - tristate "Pseudo Random Number Generation for Cryptographic modules" + tristate "ANSI PRNG (Pseudo Random Number Generator)" select CRYPTO_AES select CRYPTO_RNG help - This option enables the generic pseudo random number generator - for cryptographic modules. Uses the Algorithm specified in - ANSI X9.31 A.2.4. Note that this option must be enabled if - CRYPTO_FIPS is selected + Pseudo RNG (random number generator) (ANSI X9.31 Appendix A.2.4) + + This uses the AES cipher algorithm. + + Note that this option must be enabled if CRYPTO_FIPS is selected menuconfig CRYPTO_DRBG_MENU - tristate "NIST SP800-90A DRBG" + tristate "NIST SP800-90A DRBG (Deterministic Random Bit Generator)" help - NIST SP800-90A compliant DRBG. In the following submenu, one or - more of the DRBG types must be selected. + DRBG (Deterministic Random Bit Generator) (NIST SP800-90A) + + In the following submenu, one or more of the DRBG types must be selected. if CRYPTO_DRBG_MENU @@ -1263,17 +1274,21 @@ config CRYPTO_DRBG_HMAC select CRYPTO_SHA512 config CRYPTO_DRBG_HASH - bool "Enable Hash DRBG" + bool "Hash_DRBG" select CRYPTO_SHA256 help - Enable the Hash DRBG variant as defined in NIST SP800-90A. + Hash_DRBG variant as defined in NIST SP800-90A. + + This uses the SHA-1, SHA-256, SHA-384, or SHA-512 hash algorithms. config CRYPTO_DRBG_CTR - bool "Enable CTR DRBG" + bool "CTR_DRBG" select CRYPTO_AES select CRYPTO_CTR help - Enable the CTR DRBG variant as defined in NIST SP800-90A. + CTR_DRBG variant as defined in NIST SP800-90A. + + This uses the AES cipher algorithm with the counter block mode. config CRYPTO_DRBG tristate @@ -1284,14 +1299,17 @@ config CRYPTO_DRBG endif # if CRYPTO_DRBG_MENU config CRYPTO_JITTERENTROPY - tristate "Jitterentropy Non-Deterministic Random Number Generator" + tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)" select CRYPTO_RNG help - The Jitterentropy RNG is a noise that is intended - to provide seed to another RNG. The RNG does not - perform any cryptographic whitening of the generated - random numbers. This Jitterentropy RNG registers with - the kernel crypto API and can be used by any caller. + CPU Jitter RNG (Random Number Generator) from the Jitterentropy library + + A non-physical non-deterministic ("true") RNG (e.g., an entropy source + compliant with NIST SP800-90B) intended to provide a seed to a + deterministic RNG (e.g. per NIST SP800-90C). + This RNG does not perform any cryptographic whitening of the generated + + See https://www.chronox.de/jent.html config CRYPTO_KDF800108_CTR tristate -- cgit From fb1e1257b0cb93328f2909755b0fa2dc582cc508 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 Sep 2022 18:15:53 +0800 Subject: Revert "crypto: gemini - Fix error check for dma_map_sg" This reverts commit 545665ad1e84eb8f047018a2f607e78cef29c7fa. The original code was correct and arguably more robust than the patched version. Signed-off-by: Herbert Xu --- drivers/crypto/gemini/sl3516-ce-cipher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c index 34fea8aa91b6..14d0d83d388d 100644 --- a/drivers/crypto/gemini/sl3516-ce-cipher.c +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -149,7 +149,7 @@ static int sl3516_ce_cipher(struct skcipher_request *areq) if (areq->src == areq->dst) { nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL); - if (!nr_sgs || nr_sgs > MAXDESC / 2) { + if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; @@ -158,14 +158,14 @@ static int sl3516_ce_cipher(struct skcipher_request *areq) } else { nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); - if (!nr_sgs || nr_sgs > MAXDESC / 2) { + if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; } nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE); - if (!nr_sgd || nr_sgd > MAXDESC) { + if (nr_sgd <= 0 || nr_sgd > MAXDESC) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); err = -EINVAL; goto theend_sgs; -- cgit From 2ad548ebb85cc416587ba68c0e530a2f00b2273a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 2 Sep 2022 18:19:27 +0800 Subject: Revert "crypto: allwinner - Fix dma_map_sg error check" This reverts commit 2b02187bdb0bb75000850bd0309e70eb8664159e. The original code was correct and arguably more robust than the patched version. Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c | 6 +++--- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 2 +- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c | 4 ++-- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index be7f46faef7e..74b4e910a38d 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -208,7 +208,7 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req if (areq->src == areq->dst) { nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL); - if (!nr_sgs || nr_sgs > MAX_SG) { + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend_iv; @@ -216,13 +216,13 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req nr_sgd = nr_sgs; } else { nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - if (!nr_sgs || nr_sgs > MAX_SG) { + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend_iv; } nr_sgd = dma_map_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); - if (!nr_sgd || nr_sgd > MAX_SG) { + if (nr_sgd <= 0 || nr_sgd > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); err = -EINVAL; goto theend_sgs; diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 0e6843ec197f..8b5b9b9d04c3 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -389,7 +389,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) cet->t_asym_ctl = 0; nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - if (!nr_sgs || nr_sgs > MAX_SG) { + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index fdcc98cdecaa..910d6751644c 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -232,13 +232,13 @@ static int sun8i_ss_cipher(struct skcipher_request *areq) nr_sgd = nr_sgs; } else { nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE); - if (!nr_sgs || nr_sgs > 8) { + if (nr_sgs <= 0 || nr_sgs > 8) { dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend_iv; } nr_sgd = dma_map_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE); - if (!nr_sgd || nr_sgd > 8) { + if (nr_sgd <= 0 || nr_sgd > 8) { dev_err(ss->dev, "Invalid sg number %d\n", nr_sgd); err = -EINVAL; goto theend_sgs; diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index fcb8c41cc957..36a82b22953c 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -527,7 +527,7 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) rctx->method = ss->variant->alg_hash[algt->ss_algo_id]; nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); - if (!nr_sgs || nr_sgs > MAX_SG) { + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; goto theend; -- cgit From 10a2199caf437e893d9027d97700b3c6010048b7 Mon Sep 17 00:00:00 2001 From: Kshitiz Varshney Date: Mon, 22 Aug 2022 13:19:03 +0200 Subject: hwrng: imx-rngc - Moving IRQ handler registering after imx_rngc_irq_mask_clear() Issue: While servicing interrupt, if the IRQ happens to be because of a SEED_DONE due to a previous boot stage, you end up completing the completion prematurely, hence causing kernel to crash while booting. Fix: Moving IRQ handler registering after imx_rngc_irq_mask_clear() Fixes: 1d5449445bd0 (hwrng: mx-rngc - add a driver for Freescale RNGC) Signed-off-by: Kshitiz Varshney Signed-off-by: Herbert Xu --- drivers/char/hw_random/imx-rngc.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index f5d07b472d8a..a1c24148ed31 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -264,13 +264,6 @@ static int imx_rngc_probe(struct platform_device *pdev) if (rng_type != RNGC_TYPE_RNGC && rng_type != RNGC_TYPE_RNGB) return -ENODEV; - ret = devm_request_irq(&pdev->dev, - irq, imx_rngc_irq, 0, pdev->name, (void *)rngc); - if (ret) { - dev_err(rngc->dev, "Can't get interrupt working.\n"); - return ret; - } - init_completion(&rngc->rng_op_done); rngc->rng.name = pdev->name; @@ -284,6 +277,13 @@ static int imx_rngc_probe(struct platform_device *pdev) imx_rngc_irq_mask_clear(rngc); + ret = devm_request_irq(&pdev->dev, + irq, imx_rngc_irq, 0, pdev->name, (void *)rngc); + if (ret) { + dev_err(rngc->dev, "Can't get interrupt working.\n"); + return ret; + } + if (self_test) { ret = imx_rngc_self_test(rngc); if (ret) { -- cgit From 8c8e5b6ae43a55d0f6098606c311123f1a39d112 Mon Sep 17 00:00:00 2001 From: wangjianli Date: Tue, 23 Aug 2022 21:52:23 +0800 Subject: crypto: n2 - fix repeated words in comments Delete the redundant word 'to'. Signed-off-by: wangjianli Signed-off-by: Herbert Xu --- drivers/crypto/n2_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 3b0bf6fea491..31e24df18877 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -1494,7 +1494,7 @@ static void n2_unregister_algs(void) * * So we have to back-translate, going through the 'intr' and 'ino' * property tables of the n2cp MDESC node, matching it with the OF - * 'interrupts' property entries, in order to to figure out which + * 'interrupts' property entries, in order to figure out which * devino goes to which already-translated IRQ. */ static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, -- cgit From 8e971e06b00bf995dd5f2c6fa9c068bc23cb5e56 Mon Sep 17 00:00:00 2001 From: wangjianli Date: Tue, 23 Aug 2022 21:53:53 +0800 Subject: crypto: marvell/octeontx - fix repeated words in comments Delete the redundant word 'is'. Signed-off-by: wangjianli Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h b/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h index b8bdb9f134f3..205eacac4a34 100644 --- a/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h +++ b/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h @@ -403,7 +403,7 @@ union otx_cptx_pf_exe_bist_status { * big-endian format in memory. * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back. * 0 = The hardware issues NCB transient load (LDT) towards the cache, - * which if the line hits and is is dirty will cause the line to be + * which if the line hits and is dirty will cause the line to be * written back before being replaced. * 1 = The hardware issues NCB LDWB read-and-invalidate command towards * the cache when fetching the last word of instructions; as a result the -- cgit From 0e831f3d2fd973d5f6645ef3911c594af9ddd485 Mon Sep 17 00:00:00 2001 From: wangjianli Date: Tue, 23 Aug 2022 21:57:30 +0800 Subject: crypto: bcm - fix repeated words in comments Delete the redundant word 'in'. Signed-off-by: wangjianli Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 71281a3bdbdc..d6d87332140a 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -231,7 +231,7 @@ struct iproc_ctx_s { /* * shash descriptor - needed to perform incremental hashing in - * in software, when hw doesn't support it. + * software, when hw doesn't support it. */ struct shash_desc *shash; -- cgit From 442f06067f155aeb35696cf59f4f458ee7da83a8 Mon Sep 17 00:00:00 2001 From: Lucas Segarra Fernandez Date: Thu, 25 Aug 2022 12:24:51 +0200 Subject: crypto: testmgr - fix indentation for test_acomp() args Set right indentation for test_acomp(). Signed-off-by: Lucas Segarra Fernandez Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- crypto/testmgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 2ad4bcc58617..e4bb03b8b924 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3322,7 +3322,7 @@ out: } static int test_acomp(struct crypto_acomp *tfm, - const struct comp_testvec *ctemplate, + const struct comp_testvec *ctemplate, const struct comp_testvec *dtemplate, int ctcount, int dtcount) { -- cgit From cc40b04c08400d86d2d6ea0159e0617e717f729c Mon Sep 17 00:00:00 2001 From: Lucas Segarra Fernandez Date: Thu, 25 Aug 2022 12:32:16 +0200 Subject: crypto: qat - fix default value of WDT timer The QAT HW supports an hardware mechanism to detect an accelerator hang. The reporting of a hang occurs after a watchdog timer (WDT) expires. The value of the WDT set previously was too small and was causing false positives. Change the default value of the WDT to 0x7000000ULL to avoid this. Fixes: 1c4d9d5bbb5a ("crypto: qat - enable detection of accelerators hang") Reviewed-by: Giovanni Cabiddu Signed-off-by: Lucas Segarra Fernandez Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen4_hw_data.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h index 43b8f864806b..4fb4b3df5a18 100644 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h @@ -107,7 +107,7 @@ do { \ * Timeout is in cycles. Clock speed may vary across products but this * value should be a few milli-seconds. */ -#define ADF_SSM_WDT_DEFAULT_VALUE 0x200000 +#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL #define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000 #define ADF_SSMWDTL_OFFSET 0x54 #define ADF_SSMWDTH_OFFSET 0x5C -- cgit From 31b39755e32568b43c80814c5e13d7b1ab796d73 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 26 Aug 2022 19:05:31 +0800 Subject: crypto: aspeed - Enable compile testing This driver compile tests just fine. Signed-off-by: Herbert Xu Reviewed-by: Neal Liu Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/aspeed/Kconfig b/drivers/crypto/aspeed/Kconfig index 001bf2e09a72..ae3eb0eb57f6 100644 --- a/drivers/crypto/aspeed/Kconfig +++ b/drivers/crypto/aspeed/Kconfig @@ -1,6 +1,6 @@ config CRYPTO_DEV_ASPEED tristate "Support for Aspeed cryptographic engine driver" - depends on ARCH_ASPEED + depends on ARCH_ASPEED || COMPILE_TEST help Hash and Crypto Engine (HACE) is designed to accelerate the throughput of hash data digest, encryption and decryption. -- cgit From 95b66bc4e789c5698b973e92235c2e901c7546e0 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 27 Aug 2022 18:26:57 +0800 Subject: crypto: hisilicon/qm - check mailbox operation result After the mailbox operation is complete, the result may be unsuccessful. It needs to check the status bits of the mailbox register, if it fails, -EIO is returned. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index c180ad33d2f8..1c661c89308d 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -36,6 +36,7 @@ #define QM_MB_PING_ALL_VFS 0xffff #define QM_MB_CMD_DATA_SHIFT 32 #define QM_MB_CMD_DATA_MASK GENMASK(31, 0) +#define QM_MB_STATUS_MASK GENMASK(12, 9) /* sqc shift */ #define QM_SQ_HOP_NUM_SHIFT 0 @@ -728,8 +729,12 @@ static void qm_mb_write(struct hisi_qm *qm, const void *src) static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) { + int ret; + u32 val; + if (unlikely(hisi_qm_wait_mb_ready(qm))) { dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); + ret = -EBUSY; goto mb_busy; } @@ -737,6 +742,14 @@ static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) if (unlikely(hisi_qm_wait_mb_ready(qm))) { dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); + ret = -ETIMEDOUT; + goto mb_busy; + } + + val = readl(qm->io_base + QM_MB_CMD_SEND_BASE); + if (val & QM_MB_STATUS_MASK) { + dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n"); + ret = -EIO; goto mb_busy; } @@ -744,7 +757,7 @@ static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) mb_busy: atomic64_inc(&qm->debug.dfx.mb_err_cnt); - return -EBUSY; + return ret; } int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, -- cgit From 5afc904f443de2afd31c4e0686ba178beede86fe Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 27 Aug 2022 18:27:37 +0800 Subject: crypto: hisilicon/qm - fix missing put dfx access In function qm_cmd_write(), if function returns from branch 'atomic_read(&qm->status.flags) == QM_STOP', the got dfx access is forgotten to put. Fixes: 607c191b371d ("crypto: hisilicon - support runtime PM for accelerator device") Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 1c661c89308d..fd9fb159048f 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2216,8 +2216,10 @@ static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer, return ret; /* Judge if the instance is being reset. */ - if (unlikely(atomic_read(&qm->status.flags) == QM_STOP)) - return 0; + if (unlikely(atomic_read(&qm->status.flags) == QM_STOP)) { + ret = 0; + goto put_dfx_access; + } if (count > QM_DBG_WRITE_LEN) { ret = -ENOSPC; -- cgit From fa2bf6e35091e66fc83af1aebea06a78a5a2fde4 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 27 Aug 2022 18:27:56 +0800 Subject: crypto: hisilicon/qm - return failure if vfs_num exceeds total VFs The accelerator drivers supports users to enable VFs through the module parameter 'vfs_num'. If the number of VFs to be enabled exceeds the total VFs, all VFs are enabled. Change it to the same as enabling VF through the 'sriov_numvfs' file. Returns -ERANGE if the number of VFs to be enabled exceeds total VFs. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index fd9fb159048f..54bbd7fa57cc 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4766,7 +4766,13 @@ int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) goto err_put_sync; } - num_vfs = min_t(int, max_vfs, total_vfs); + if (max_vfs > total_vfs) { + pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs); + ret = -ERANGE; + goto err_put_sync; + } + + num_vfs = max_vfs; ret = qm_vf_q_assign(qm, num_vfs); if (ret) { pci_err(pdev, "Can't assign queues for VF!\n"); -- cgit From 2be570849efc4e1ceda6a8384184f493f558f188 Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Mon, 5 Sep 2022 10:54:33 +0800 Subject: crypto: aspeed - fix build module error If CONFIG_MODULES=y and CONFIG_CRYPTO_DEV_ASPEED=m, build modpost would be failed. Error messages: ERROR: modpost: "aspeed_register_hace_hash_algs" [drivers/crypto/aspeed/aspeed_crypto.ko] undefined! ERROR: modpost: "aspeed_unregister_hace_hash_algs" [drivers/crypto/aspeed/aspeed_crypto.ko] undefined! Change build sequence to fix this. Reported-by: kernel test robot Signed-off-by: Neal Liu Tested-by: Sudip Mukherjee Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/Makefile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/aspeed/Makefile b/drivers/crypto/aspeed/Makefile index 421e2ca9c53e..3be78cec0ecb 100644 --- a/drivers/crypto/aspeed/Makefile +++ b/drivers/crypto/aspeed/Makefile @@ -1,9 +1,6 @@ +hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace.o aspeed-hace-hash.o +hace-crypto-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) := aspeed-hace.o aspeed-hace-crypto.o + obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed_crypto.o -aspeed_crypto-objs := aspeed-hace.o \ - $(hace-hash-y) \ +aspeed_crypto-objs := $(hace-hash-y) \ $(hace-crypto-y) - -obj-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) += aspeed-hace-hash.o -hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace-hash.o -obj-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) += aspeed-hace-crypto.o -hace-crypto-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) := aspeed-hace-crypto.o -- cgit From aa450316c662f599189e0910444cf064c4f31602 Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Wed, 7 Sep 2022 11:34:31 +0800 Subject: crypto: aspeed: fix format unexpected build warning This fixes the following similar build warning when enabling compile test: aspeed-hace-hash.c:188:9: warning: format '%x' expects argument of type 'unsigned int', but argument 7 has type 'size_t' {aka 'long unsigned int'} [-Wformat=] Reported-by: kernel test robot Signed-off-by: Neal Liu Acked-by: Randy Dunlap Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/aspeed-hace-hash.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/aspeed/aspeed-hace-hash.c b/drivers/crypto/aspeed/aspeed-hace-hash.c index 0a44ffc0e13b..5a8a3a611dd4 100644 --- a/drivers/crypto/aspeed/aspeed-hace-hash.c +++ b/drivers/crypto/aspeed/aspeed-hace-hash.c @@ -185,7 +185,7 @@ static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) remain = (rctx->total + rctx->bufcnt) % rctx->block_size; length = rctx->total + rctx->bufcnt - remain; - AHASH_DBG(hace_dev, "%s:0x%x, %s:0x%x, %s:0x%x, %s:0x%x\n", + AHASH_DBG(hace_dev, "%s:0x%x, %s:%zu, %s:0x%x, %s:0x%x\n", "rctx total", rctx->total, "bufcnt", rctx->bufcnt, "length", length, "remain", remain); @@ -324,8 +324,8 @@ static int aspeed_hace_ahash_trigger(struct aspeed_hace_dev *hace_dev, struct ahash_request *req = hash_engine->req; struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - AHASH_DBG(hace_dev, "src_dma:0x%x, digest_dma:0x%x, length:0x%x\n", - hash_engine->src_dma, hash_engine->digest_dma, + AHASH_DBG(hace_dev, "src_dma:%pad, digest_dma:%pad, length:%zu\n", + &hash_engine->src_dma, &hash_engine->digest_dma, hash_engine->src_length); rctx->cmd |= HASH_CMD_INT_ENABLE; -- cgit From efc96d43ec38381dacbd51679c2d01438511371d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 6 Sep 2022 13:05:35 +0800 Subject: crypto: aspeed - Fix sparse warnings This patch fixes a bunch of bit endianness warnings and two missing static modifiers. Signed-off-by: Herbert Xu Reviewed-by: Neal Liu Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/aspeed-hace-crypto.c | 42 ++++++++++++++---------------- drivers/crypto/aspeed/aspeed-hace-hash.c | 38 ++++++++++++++------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c index ba6158f5cf18..ef73b0028b4d 100644 --- a/drivers/crypto/aspeed/aspeed-hace-crypto.c +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -258,21 +258,20 @@ static int aspeed_sk_start_sg(struct aspeed_hace_dev *hace_dev) total = req->cryptlen; for_each_sg(req->src, s, src_sg_len, i) { - src_list[i].phy_addr = sg_dma_address(s); + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); - if (total > sg_dma_len(s)) { - src_list[i].len = sg_dma_len(s); - total -= src_list[i].len; - - } else { + if (total > len) + total -= len; + else { /* last sg list */ - src_list[i].len = total; - src_list[i].len |= BIT(31); + len = total; + len |= BIT(31); total = 0; } - src_list[i].phy_addr = cpu_to_le32(src_list[i].phy_addr); - src_list[i].len = cpu_to_le32(src_list[i].len); + src_list[i].phy_addr = cpu_to_le32(phy_addr); + src_list[i].len = cpu_to_le32(len); } if (total != 0) { @@ -290,21 +289,20 @@ static int aspeed_sk_start_sg(struct aspeed_hace_dev *hace_dev) total = req->cryptlen; for_each_sg(req->dst, s, dst_sg_len, i) { - dst_list[i].phy_addr = sg_dma_address(s); - - if (total > sg_dma_len(s)) { - dst_list[i].len = sg_dma_len(s); - total -= dst_list[i].len; + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); - } else { + if (total > len) + total -= len; + else { /* last sg list */ - dst_list[i].len = total; - dst_list[i].len |= BIT(31); + len = total; + len |= BIT(31); total = 0; } - dst_list[i].phy_addr = cpu_to_le32(dst_list[i].phy_addr); - dst_list[i].len = cpu_to_le32(dst_list[i].len); + dst_list[i].phy_addr = cpu_to_le32(phy_addr); + dst_list[i].len = cpu_to_le32(len); } @@ -731,7 +729,7 @@ static void aspeed_crypto_cra_exit(struct crypto_skcipher *tfm) crypto_free_skcipher(ctx->fallback_tfm); } -struct aspeed_hace_alg aspeed_crypto_algs[] = { +static struct aspeed_hace_alg aspeed_crypto_algs[] = { { .alg.skcipher = { .min_keysize = AES_MIN_KEY_SIZE, @@ -1019,7 +1017,7 @@ struct aspeed_hace_alg aspeed_crypto_algs[] = { }, }; -struct aspeed_hace_alg aspeed_crypto_algs_g6[] = { +static struct aspeed_hace_alg aspeed_crypto_algs_g6[] = { { .alg.skcipher = { .ivsize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/aspeed/aspeed-hace-hash.c b/drivers/crypto/aspeed/aspeed-hace-hash.c index 5a8a3a611dd4..935135229ebd 100644 --- a/drivers/crypto/aspeed/aspeed-hace-hash.c +++ b/drivers/crypto/aspeed/aspeed-hace-hash.c @@ -208,6 +208,9 @@ static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) } if (rctx->bufcnt != 0) { + u32 phy_addr; + u32 len; + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, rctx->buffer, rctx->block_size * 2, @@ -218,36 +221,35 @@ static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) goto free_rctx_digest; } - src_list[0].phy_addr = rctx->buffer_dma_addr; - src_list[0].len = rctx->bufcnt; - length -= src_list[0].len; + phy_addr = rctx->buffer_dma_addr; + len = rctx->bufcnt; + length -= len; /* Last sg list */ if (length == 0) - src_list[0].len |= HASH_SG_LAST_LIST; + len |= HASH_SG_LAST_LIST; - src_list[0].phy_addr = cpu_to_le32(src_list[0].phy_addr); - src_list[0].len = cpu_to_le32(src_list[0].len); + src_list[0].phy_addr = cpu_to_le32(phy_addr); + src_list[0].len = cpu_to_le32(len); src_list++; } if (length != 0) { for_each_sg(rctx->src_sg, s, sg_len, i) { - src_list[i].phy_addr = sg_dma_address(s); - - if (length > sg_dma_len(s)) { - src_list[i].len = sg_dma_len(s); - length -= sg_dma_len(s); + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); - } else { + if (length > len) + length -= len; + else { /* Last sg list */ - src_list[i].len = length; - src_list[i].len |= HASH_SG_LAST_LIST; + len = length; + len |= HASH_SG_LAST_LIST; length = 0; } - src_list[i].phy_addr = cpu_to_le32(src_list[i].phy_addr); - src_list[i].len = cpu_to_le32(src_list[i].len); + src_list[i].phy_addr = cpu_to_le32(phy_addr); + src_list[i].len = cpu_to_le32(len); } } @@ -913,7 +915,7 @@ static int aspeed_sham_import(struct ahash_request *req, const void *in) return 0; } -struct aspeed_hace_alg aspeed_ahash_algs[] = { +static struct aspeed_hace_alg aspeed_ahash_algs[] = { { .alg.ahash = { .init = aspeed_sham_init, @@ -1099,7 +1101,7 @@ struct aspeed_hace_alg aspeed_ahash_algs[] = { }, }; -struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { +static struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { { .alg.ahash = { .init = aspeed_sham_init, -- cgit From dc377e013bec20dfedaec5d1f6327e4d57da359b Mon Sep 17 00:00:00 2001 From: Sun Ke Date: Tue, 30 Aug 2022 11:13:47 +0800 Subject: crypto: aspeed - fix return value check in aspeed_hace_probe() In case of error, the function devm_ioremap_resource() returns ERR_PTR() not NULL. The NULL test in the return value check must be replaced with IS_ERR(). Fixes: 108713a713c7 ("crypto: aspeed - Add HACE hash driver") Signed-off-by: Sun Ke Reviewed-by: Neal Liu Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/aspeed-hace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c index 4fefc9e69d72..3f880aafb6a2 100644 --- a/drivers/crypto/aspeed/aspeed-hace.c +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -123,9 +123,9 @@ static int aspeed_hace_probe(struct platform_device *pdev) platform_set_drvdata(pdev, hace_dev); hace_dev->regs = devm_ioremap_resource(&pdev->dev, res); - if (!hace_dev->regs) { + if (IS_ERR(hace_dev->regs)) { dev_err(&pdev->dev, "Failed to map resources\n"); - return -ENOMEM; + return PTR_ERR(hace_dev->regs); } /* Get irq number and register it */ -- cgit From bc155c6c188c2f0c5749993b1405673d25a80389 Mon Sep 17 00:00:00 2001 From: Ignat Korchagin Date: Wed, 31 Aug 2022 19:37:06 +0100 Subject: crypto: akcipher - default implementation for setting a private key Changes from v1: * removed the default implementation from set_pub_key: it is assumed that an implementation must always have this callback defined as there are no use case for an algorithm, which doesn't need a public key Many akcipher implementations (like ECDSA) support only signature verifications, so they don't have all callbacks defined. Commit 78a0324f4a53 ("crypto: akcipher - default implementations for request callbacks") introduced default callbacks for sign/verify operations, which just return an error code. However, these are not enough, because before calling sign the caller would likely call set_priv_key first on the instantiated transform (as the in-kernel testmgr does). This function does not have a default stub, so the kernel crashes, when trying to set a private key on an akcipher, which doesn't support signature generation. I've noticed this, when trying to add a KAT vector for ECDSA signature to the testmgr. With this patch the testmgr returns an error in dmesg (as it should) instead of crashing the kernel NULL ptr dereference. Fixes: 78a0324f4a53 ("crypto: akcipher - default implementations for request callbacks") Signed-off-by: Ignat Korchagin Signed-off-by: Herbert Xu --- crypto/akcipher.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crypto/akcipher.c b/crypto/akcipher.c index f866085c8a4a..ab975a420e1e 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -120,6 +120,12 @@ static int akcipher_default_op(struct akcipher_request *req) return -ENOSYS; } +static int akcipher_default_set_key(struct crypto_akcipher *tfm, + const void *key, unsigned int keylen) +{ + return -ENOSYS; +} + int crypto_register_akcipher(struct akcipher_alg *alg) { struct crypto_alg *base = &alg->base; @@ -132,6 +138,8 @@ int crypto_register_akcipher(struct akcipher_alg *alg) alg->encrypt = akcipher_default_op; if (!alg->decrypt) alg->decrypt = akcipher_default_op; + if (!alg->set_priv_key) + alg->set_priv_key = akcipher_default_set_key; akcipher_prepare_alg(alg); return crypto_register_alg(base); -- cgit From 24ddd4e1dbfcf530b89c3d994789cc2e57cbe713 Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Thu, 1 Sep 2022 07:43:48 +0000 Subject: crypto: octeontx - Remove the unneeded result variable Return the value cptvf_send_msg_to_pf_timeout() directly instead of storing it in another redundant variable. Reported-by: Zeal Robot Signed-off-by: ye xingchen Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c index 5663787c7a62..90fdafb7c468 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c @@ -159,12 +159,10 @@ static int cptvf_send_msg_to_pf_timeout(struct otx_cptvf *cptvf, int otx_cptvf_check_pf_ready(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_READY; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -174,13 +172,11 @@ int otx_cptvf_check_pf_ready(struct otx_cptvf *cptvf) int otx_cptvf_send_vq_size_msg(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_QLEN; mbx.data = cptvf->qsize; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -208,14 +204,12 @@ int otx_cptvf_send_vf_to_grp_msg(struct otx_cptvf *cptvf, int group) int otx_cptvf_send_vf_priority_msg(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_VQ_PRIORITY; /* Convey group of the VF */ mbx.data = cptvf->priority; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -224,12 +218,10 @@ int otx_cptvf_send_vf_priority_msg(struct otx_cptvf *cptvf) int otx_cptvf_send_vf_up(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_VF_UP; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -238,10 +230,8 @@ int otx_cptvf_send_vf_up(struct otx_cptvf *cptvf) int otx_cptvf_send_vf_down(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_VF_DOWN; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } -- cgit From 68dbe80f5b510c66c800b9e8055235c5b07e37d1 Mon Sep 17 00:00:00 2001 From: Koba Ko Date: Thu, 1 Sep 2022 22:47:12 +0800 Subject: crypto: ccp - Release dma channels before dmaengine unrgister A warning is shown during shutdown, __dma_async_device_channel_unregister called while 2 clients hold a reference WARNING: CPU: 15 PID: 1 at drivers/dma/dmaengine.c:1110 __dma_async_device_channel_unregister+0xb7/0xc0 Call dma_release_channel for occupied channles before dma_async_device_unregister. Fixes: 54cce8ecb925 ("crypto: ccp - ccp_dmaengine_unregister release dma channels") Reported-by: kernel test robot Signed-off-by: Koba Ko Acked-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dmaengine.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index 7d4b4ad1db1f..9f753cb4f5f1 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -641,6 +641,10 @@ static void ccp_dma_release(struct ccp_device *ccp) for (i = 0; i < ccp->cmd_q_count; i++) { chan = ccp->ccp_dma_chan + i; dma_chan = &chan->dma_chan; + + if (dma_chan->client_count) + dma_release_channel(dma_chan); + tasklet_kill(&chan->cleanup_tasklet); list_del_rcu(&dma_chan->device_node); } @@ -766,8 +770,8 @@ void ccp_dmaengine_unregister(struct ccp_device *ccp) if (!dmaengine) return; - dma_async_device_unregister(dma_dev); ccp_dma_release(ccp); + dma_async_device_unregister(dma_dev); kmem_cache_destroy(ccp->dma_desc_cache); kmem_cache_destroy(ccp->dma_cmd_cache); -- cgit From 96d3e6f05f5b7ec1cadb6a26c05093efcf062432 Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Fri, 2 Sep 2022 07:30:55 +0000 Subject: crypto: nx - Remove the unneeded result variable Return the value set_msg_len() directly instead of storing it in another redundant variable. Reported-by: Zeal Robot Signed-off-by: ye xingchen Reviewed-by: Breno Leitao Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-aes-ccm.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c index 3793885f928d..c843f4c6f684 100644 --- a/drivers/crypto/nx/nx-aes-ccm.c +++ b/drivers/crypto/nx/nx-aes-ccm.c @@ -134,7 +134,6 @@ static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize, unsigned int cryptlen, u8 *b0) { unsigned int l, lp, m = authsize; - int rc; memcpy(b0, iv, 16); @@ -148,9 +147,7 @@ static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize, if (assoclen) *b0 |= 64; - rc = set_msg_len(b0 + 16 - l, cryptlen, l); - - return rc; + return set_msg_len(b0 + 16 - l, cryptlen, l); } static int generate_pat(u8 *iv, -- cgit From 664593407e936b6438fbfaaf98876910fd31cf9a Mon Sep 17 00:00:00 2001 From: Peter Harliman Liem Date: Tue, 6 Sep 2022 10:51:28 +0800 Subject: crypto: inside-secure - Change swab to swab32 The use of swab() is causing failures in 64-bit arch, as it translates to __swab64() instead of the intended __swab32(). It eventually causes wrong results in xcbcmac & cmac algo. Fixes: 78cf1c8bfcb8 ("crypto: inside-secure - Move ipad/opad into safexcel_context") Signed-off-by: Peter Harliman Liem Acked-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index bc60b5802256..2124416742f8 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -383,7 +383,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, u32 x; x = ipad[i] ^ ipad[i + 4]; - cache[i] ^= swab(x); + cache[i] ^= swab32(x); } } cache_len = AES_BLOCK_SIZE; @@ -821,7 +821,7 @@ static int safexcel_ahash_final(struct ahash_request *areq) u32 *result = (void *)areq->result; /* K3 */ - result[i] = swab(ctx->base.ipad.word[i + 4]); + result[i] = swab32(ctx->base.ipad.word[i + 4]); } areq->result[0] ^= 0x80; // 10- padding crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result); @@ -2106,7 +2106,7 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE, "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->base.ipad.word[i] = swab(key_tmp[i]); + ctx->base.ipad.word[i] = swab32(key_tmp[i]); crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & @@ -2189,7 +2189,7 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < len / sizeof(u32); i++) - ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]); + ctx->base.ipad.word[i + 8] = swab32(aes.key_enc[i]); /* precompute the CMAC key material */ crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); -- cgit From 0413623c27a380d0da7240717f9435d24776b985 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 9 Sep 2022 06:28:11 +0000 Subject: crypto: hisilicon/sec - delete redundant blank lines Some coding style fixes in sec crypto file. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 77c9f13cf69a..ead061089e0c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -1679,7 +1679,6 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) aead_req->out_mac, authsize, a_req->cryptlen + a_req->assoclen); - if (unlikely(sz != authsize)) { dev_err(c->dev, "copy out mac err!\n"); err = -EINVAL; @@ -1966,7 +1965,6 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) return sec_aead_ctx_init(tfm, "sha512"); } - static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, struct sec_req *sreq) { -- cgit From 82f00b24f532557fb0e15a6a2747859e4b70c4bd Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 9 Sep 2022 17:46:55 +0800 Subject: crypto: hisilicon/qm - get hardware features from hardware registers Before hardware V3, hardwares do not provide the feature registers, driver resolves hardware differences based on the hardware version. As a result, the driver does not support the new hardware. Hardware V3 and later versions support to obtain hardware features, such as power-gating management and doorbell isolation, through the hardware registers. To be compatible with later hardware versions, the features of the current device is obtained by reading the hardware registers instead of the hardware version. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 4 +- drivers/crypto/hisilicon/qm.c | 196 ++++++++++++++++++++---------- drivers/crypto/hisilicon/sec2/sec_main.c | 4 +- drivers/crypto/hisilicon/zip/zip_main.c | 4 +- include/linux/hisi_acc_qm.h | 31 ++++- 5 files changed, 170 insertions(+), 69 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 8e0e87cede6b..d484105b2716 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -457,7 +457,7 @@ static void hpre_open_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; /* Enable prefetch */ @@ -478,7 +478,7 @@ static void hpre_close_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 54bbd7fa57cc..fa77b469761b 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -87,9 +87,7 @@ #define QM_DB_CMD_SHIFT_V1 16 #define QM_DB_INDEX_SHIFT_V1 32 #define QM_DB_PRIORITY_SHIFT_V1 48 -#define QM_QUE_ISO_CFG_V 0x0030 #define QM_PAGE_SIZE 0x0034 -#define QM_QUE_ISO_EN 0x100154 #define QM_CAPBILITY 0x100158 #define QM_QP_NUN_MASK GENMASK(10, 0) #define QM_QP_DB_INTERVAL 0x10000 @@ -206,6 +204,8 @@ #define MAX_WAIT_COUNTS 1000 #define QM_CACHE_WB_START 0x204 #define QM_CACHE_WB_DONE 0x208 +#define QM_FUNC_CAPS_REG 0x3100 +#define QM_CAPBILITY_VERSION GENMASK(7, 0) #define PCI_BAR_2 2 #define PCI_BAR_4 4 @@ -330,6 +330,22 @@ enum qm_mb_cmd { QM_VF_GET_QOS, }; +static const struct hisi_qm_cap_info qm_cap_info_comm[] = { + {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, + {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, + {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1}, + {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, + {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, +}; + +static const struct hisi_qm_cap_info qm_cap_info_pf[] = { + {QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1}, +}; + +static const struct hisi_qm_cap_info qm_cap_info_vf[] = { + {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0}, +}; + struct qm_cqe { __le32 rsvd0; __le16 cmd_id; @@ -427,10 +443,7 @@ struct hisi_qm_hw_ops { void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe); void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); - int (*stop_qp)(struct hisi_qp *qp); int (*set_msi)(struct hisi_qm *qm, bool set); - int (*ping_all_vfs)(struct hisi_qm *qm, u64 cmd); - int (*ping_pf)(struct hisi_qm *qm, u64 cmd); }; struct qm_dfx_item { @@ -841,6 +854,36 @@ static int qm_dev_mem_reset(struct hisi_qm *qm) POLL_TIMEOUT); } +/** + * hisi_qm_get_hw_info() - Get device information. + * @qm: The qm which want to get information. + * @info_table: Array for storing device information. + * @index: Index in info_table. + * @is_read: Whether read from reg, 0: not support read from reg. + * + * This function returns device information the caller needs. + */ +u32 hisi_qm_get_hw_info(struct hisi_qm *qm, + const struct hisi_qm_cap_info *info_table, + u32 index, bool is_read) +{ + u32 val; + + switch (qm->ver) { + case QM_HW_V1: + return info_table[index].v1_val; + case QM_HW_V2: + return info_table[index].v2_val; + default: + if (!is_read) + return info_table[index].v3_val; + + val = readl(qm->io_base + info_table[index].offset); + return (val >> info_table[index].shift) & info_table[index].mask; + } +} +EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info); + static u32 qm_get_irq_num_v1(struct hisi_qm *qm) { return QM_IRQ_NUM_V1; @@ -867,7 +910,7 @@ static int qm_pm_get_sync(struct hisi_qm *qm) struct device *dev = &qm->pdev->dev; int ret; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return 0; ret = pm_runtime_resume_and_get(dev); @@ -883,7 +926,7 @@ static void qm_pm_put_sync(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return; pm_runtime_mark_last_busy(dev); @@ -1164,7 +1207,7 @@ static void qm_init_prefetch(struct hisi_qm *qm) struct device *dev = &qm->pdev->dev; u32 page_type = 0x0; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; switch (PAGE_SIZE) { @@ -1283,7 +1326,7 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, } break; case SHAPER_VFT: - if (qm->ver >= QM_HW_V3) { + if (factor) { tmp = factor->cir_b | (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) | (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) | @@ -1301,10 +1344,13 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, u32 fun_num, u32 base, u32 number) { - struct qm_shaper_factor *factor = &qm->factor[fun_num]; + struct qm_shaper_factor *factor = NULL; unsigned int val; int ret; + if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + factor = &qm->factor[fun_num]; + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, val & BIT(0), POLL_PERIOD, POLL_TIMEOUT); @@ -1362,7 +1408,7 @@ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, } /* init default shaper qos val */ - if (qm->ver >= QM_HW_V3) { + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { ret = qm_shaper_init_vft(qm, fun_num); if (ret) goto back_sqc_cqc; @@ -2466,7 +2512,7 @@ static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) u64 val; u32 i; - if (!qm->vfs_num || qm->ver < QM_HW_V3) + if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) return 0; while (true) { @@ -2751,10 +2797,7 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .hw_error_init = qm_hw_error_init_v3, .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, - .stop_qp = qm_stop_qp, .set_msi = qm_set_msi_v3, - .ping_all_vfs = qm_ping_all_vfs, - .ping_pf = qm_ping_pf, }; static void *qm_get_avail_sqe(struct hisi_qp *qp) @@ -3051,8 +3094,8 @@ static int qm_drain_qp(struct hisi_qp *qp) return 0; /* Kunpeng930 supports drain qp by device */ - if (qm->ops->stop_qp) { - ret = qm->ops->stop_qp(qp); + if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { + ret = qm_stop_qp(qp); if (ret) dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id); return ret; @@ -3282,7 +3325,7 @@ static int hisi_qm_uacce_mmap(struct uacce_queue *q, if (qm->ver == QM_HW_V1) { if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR) return -EINVAL; - } else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) { + } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE)) return -EINVAL; @@ -3436,7 +3479,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) if (qm->ver == QM_HW_V1) mmio_page_nr = QM_DOORBELL_PAGE_NR; - else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) + else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) mmio_page_nr = QM_DOORBELL_PAGE_NR + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; else @@ -3598,7 +3641,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; qm->misc_ctl = false; - if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V2) { + if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev))) dev_info(&pdev->dev, "_PS0 and _PR0 are not defined"); } @@ -3608,7 +3651,7 @@ static void qm_cmd_uninit(struct hisi_qm *qm) { u32 val; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) return; val = readl(qm->io_base + QM_IFC_INT_MASK); @@ -3620,7 +3663,7 @@ static void qm_cmd_init(struct hisi_qm *qm) { u32 val; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) return; /* Clear communication interrupt source */ @@ -3636,7 +3679,7 @@ static void qm_put_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; - if (qm->use_db_isolation) + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) iounmap(qm->db_io_base); iounmap(qm->io_base); @@ -3686,7 +3729,9 @@ static void hisi_qm_memory_uninit(struct hisi_qm *qm) } idr_destroy(&qm->qp_idr); - kfree(qm->factor); + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + kfree(qm->factor); } /** @@ -4469,12 +4514,10 @@ static int qm_vf_read_qos(struct hisi_qm *qm) qm->mb_qos = 0; /* vf ping pf to get function qos */ - if (qm->ops->ping_pf) { - ret = qm->ops->ping_pf(qm, QM_VF_GET_QOS); - if (ret) { - pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); - return ret; - } + ret = qm_ping_pf(qm, QM_VF_GET_QOS); + if (ret) { + pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); + return ret; } while (true) { @@ -4646,14 +4689,14 @@ static const struct file_operations qm_algqos_fops = { * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. * @qm: The qm for which we want to add debugfs files. * - * Create function qos debugfs files. + * Create function qos debugfs files, VF ping PF to get function qos. */ static void hisi_qm_set_algqos_init(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_PF) debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, qm, &qm_algqos_fops); - else + else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, qm, &qm_algqos_fops); } @@ -4701,7 +4744,7 @@ void hisi_qm_debug_init(struct hisi_qm *qm) &qm_atomic64_ops); } - if (qm->ver >= QM_HW_V3) + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) hisi_qm_set_algqos_init(qm); } EXPORT_SYMBOL_GPL(hisi_qm_debug_init); @@ -4824,7 +4867,9 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) pci_disable_sriov(pdev); /* clear vf function shaper configure array */ - memset(qm->factor + 1, 0, sizeof(struct qm_shaper_factor) * total_vfs); + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + memset(qm->factor + 1, 0, sizeof(struct qm_shaper_factor) * total_vfs); + ret = qm_clear_vft_config(qm); if (ret) return ret; @@ -5048,8 +5093,8 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, return 0; /* Kunpeng930 supports to notify VFs to stop before PF reset */ - if (qm->ops->ping_all_vfs) { - ret = qm->ops->ping_all_vfs(qm, cmd); + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_all_vfs(qm, cmd); if (ret) pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); } else { @@ -5240,8 +5285,8 @@ static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) } /* Kunpeng930 supports to notify VFs to start after PF reset. */ - if (qm->ops->ping_all_vfs) { - ret = qm->ops->ping_all_vfs(qm, cmd); + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_all_vfs(qm, cmd); if (ret) pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); } else { @@ -5687,7 +5732,7 @@ err_prepare: hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); out: pci_save_state(pdev); - ret = qm->ops->ping_pf(qm, cmd); + ret = qm_ping_pf(qm, cmd); if (ret) dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n"); } @@ -5705,7 +5750,7 @@ static void qm_pf_reset_vf_done(struct hisi_qm *qm) cmd = QM_VF_START_FAIL; } - ret = qm->ops->ping_pf(qm, cmd); + ret = qm_ping_pf(qm, cmd); if (ret) dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); @@ -5910,7 +5955,7 @@ static int qm_get_qp_num(struct hisi_qm *qm) qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) & QM_QP_NUN_MASK; - if (qm->use_db_isolation) + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) qm->max_qp_num = (readl(qm->io_base + QM_CAPBILITY) >> QM_QP_MAX_NUM_SHIFT) & QM_QP_NUN_MASK; else @@ -5926,6 +5971,39 @@ static int qm_get_qp_num(struct hisi_qm *qm) return 0; } +static void qm_get_hw_caps(struct hisi_qm *qm) +{ + const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? + qm_cap_info_pf : qm_cap_info_vf; + u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) : + ARRAY_SIZE(qm_cap_info_vf); + u32 val, i; + + /* Doorbell isolate register is a independent register. */ + val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true); + if (val) + set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); + + if (qm->ver >= QM_HW_V3) { + val = readl(qm->io_base + QM_FUNC_CAPS_REG); + qm->cap_ver = val & QM_CAPBILITY_VERSION; + } + + /* Get PF/VF common capbility */ + for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) { + val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver); + if (val) + set_bit(qm_cap_info_comm[i].type, &qm->caps); + } + + /* Get PF/VF different capbility */ + for (i = 0; i < size; i++) { + val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver); + if (val) + set_bit(cap_info[i].type, &qm->caps); + } +} + static int qm_get_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5945,16 +6023,8 @@ static int qm_get_pci_res(struct hisi_qm *qm) goto err_request_mem_regions; } - if (qm->ver > QM_HW_V2) { - if (qm->fun_type == QM_HW_PF) - qm->use_db_isolation = readl(qm->io_base + - QM_QUE_ISO_EN) & BIT(0); - else - qm->use_db_isolation = readl(qm->io_base + - QM_QUE_ISO_CFG_V) & BIT(0); - } - - if (qm->use_db_isolation) { + qm_get_hw_caps(qm); + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { qm->db_interval = QM_QP_DB_INTERVAL; qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); qm->db_io_base = ioremap(qm->db_phys_base, @@ -5978,7 +6048,7 @@ static int qm_get_pci_res(struct hisi_qm *qm) return 0; err_db_ioremap: - if (qm->use_db_isolation) + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) iounmap(qm->db_io_base); err_ioremap: iounmap(qm->io_base); @@ -6095,12 +6165,15 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) int ret, total_func, i; size_t off = 0; - total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; - qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); - if (!qm->factor) - return -ENOMEM; - for (i = 0; i < total_func; i++) - qm->factor[i].func_qos = QM_QOS_MAX_VAL; + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { + total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; + qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); + if (!qm->factor) + return -ENOMEM; + + for (i = 0; i < total_func; i++) + qm->factor[i].func_qos = QM_QOS_MAX_VAL; + } #define QM_INIT_BUF(qm, type, num) do { \ (qm)->type = ((qm)->qdma.va + (off)); \ @@ -6136,7 +6209,8 @@ err_alloc_qp_array: dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); err_destroy_idr: idr_destroy(&qm->qp_idr); - kfree(qm->factor); + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + kfree(qm->factor); return ret; } @@ -6279,7 +6353,7 @@ void hisi_qm_pm_init(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return; pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY); @@ -6298,7 +6372,7 @@ void hisi_qm_pm_uninit(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return; pm_runtime_get_noresume(dev); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2c0be91c0b09..1ec3b06345fd 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -415,7 +415,7 @@ static void sec_open_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; /* Enable prefetch */ @@ -435,7 +435,7 @@ static void sec_close_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 04c8a4c65d77..36809bae6334 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -348,7 +348,7 @@ static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; /* Enable prefetch */ @@ -368,7 +368,7 @@ static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 116e8bd68c99..851c962ba473 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -168,6 +168,15 @@ enum qm_vf_state { QM_NOT_READY, }; +enum qm_cap_bits { + QM_SUPPORT_DB_ISOLATION = 0x0, + QM_SUPPORT_FUNC_QOS, + QM_SUPPORT_STOP_QP, + QM_SUPPORT_MB_COMMAND, + QM_SUPPORT_SVA_PREFETCH, + QM_SUPPORT_RPM, +}; + struct dfx_diff_registers { u32 *regs; u32 reg_offset; @@ -258,6 +267,18 @@ struct hisi_qm_err_ini { void (*err_info_init)(struct hisi_qm *qm); }; +struct hisi_qm_cap_info { + u32 type; + /* Register offset */ + u32 offset; + /* Bit offset in register */ + u32 shift; + u32 mask; + u32 v1_val; + u32 v2_val; + u32 v3_val; +}; + struct hisi_qm_list { struct mutex lock; struct list_head list; @@ -278,6 +299,9 @@ struct hisi_qm { struct pci_dev *pdev; void __iomem *io_base; void __iomem *db_io_base; + + /* Capbility version, 0: not supports */ + u32 cap_ver; u32 sqe_size; u32 qp_base; u32 qp_num; @@ -304,6 +328,8 @@ struct hisi_qm { struct hisi_qm_err_info err_info; struct hisi_qm_err_status err_status; unsigned long misc_ctl; /* driver removing and reset sched */ + /* Device capability bit */ + unsigned long caps; struct rw_semaphore qps_lock; struct idr qp_idr; @@ -326,8 +352,6 @@ struct hisi_qm { bool use_sva; bool is_frozen; - /* doorbell isolation enable */ - bool use_db_isolation; resource_size_t phys_base; resource_size_t db_phys_base; struct uacce_device *uacce; @@ -501,6 +525,9 @@ void hisi_qm_pm_init(struct hisi_qm *qm); int hisi_qm_get_dfx_access(struct hisi_qm *qm); void hisi_qm_put_dfx_access(struct hisi_qm *qm); void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset); +u32 hisi_qm_get_hw_info(struct hisi_qm *qm, + const struct hisi_qm_cap_info *info_table, + u32 index, bool is_read); /* Used by VFIO ACC live migration driver */ struct pci_driver *hisi_sec_get_pf_driver(void); -- cgit From 129a9f340172b4f3857260a7a7bb9d7b3496ba50 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 9 Sep 2022 17:46:56 +0800 Subject: crypto: hisilicon/qm - get qp num and depth from hardware registers Hardware V3 and later versions can obtain qp num and depth supported by the hardware from registers. To be compatible with later hardware versions, get qp num and depth from registers instead of fixed marcos. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 4 +- drivers/crypto/hisilicon/qm.c | 166 ++++++++++++++++------------ drivers/crypto/hisilicon/sec2/sec.h | 5 +- drivers/crypto/hisilicon/sec2/sec_crypto.c | 148 +++++++++++++++---------- drivers/crypto/hisilicon/sec2/sec_main.c | 1 - drivers/crypto/hisilicon/zip/zip_crypto.c | 6 +- drivers/crypto/hisilicon/zip/zip_main.c | 1 - include/linux/hisi_acc_qm.h | 5 +- 8 files changed, 202 insertions(+), 134 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 3ba6f15deafc..2aa91a4f77da 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -147,7 +147,7 @@ static int hpre_alloc_req_id(struct hpre_ctx *ctx) int id; spin_lock_irqsave(&ctx->req_lock, flags); - id = idr_alloc(&ctx->req_idr, NULL, 0, QM_Q_DEPTH, GFP_ATOMIC); + id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); spin_unlock_irqrestore(&ctx->req_lock, flags); return id; @@ -488,7 +488,7 @@ static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) qp->qp_ctx = ctx; qp->req_cb = hpre_alg_cb; - ret = hpre_ctx_set(ctx, qp, QM_Q_DEPTH); + ret = hpre_ctx_set(ctx, qp, qp->sq_depth); if (ret) hpre_stop_qp_and_put(qp); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index fa77b469761b..b3216ee627e5 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -78,6 +78,9 @@ #define QM_EQ_OVERFLOW 1 #define QM_CQE_ERROR 2 +#define QM_XQ_DEPTH_SHIFT 16 +#define QM_XQ_DEPTH_MASK GENMASK(15, 0) + #define QM_DOORBELL_CMD_SQ 0 #define QM_DOORBELL_CMD_CQ 1 #define QM_DOORBELL_CMD_EQ 2 @@ -88,8 +91,6 @@ #define QM_DB_INDEX_SHIFT_V1 32 #define QM_DB_PRIORITY_SHIFT_V1 48 #define QM_PAGE_SIZE 0x0034 -#define QM_CAPBILITY 0x100158 -#define QM_QP_NUN_MASK GENMASK(10, 0) #define QM_QP_DB_INTERVAL 0x10000 #define QM_MEM_START_INIT 0x100040 @@ -222,7 +223,6 @@ #define WAIT_PERIOD 20 #define REMOVE_WAIT_DELAY 10 #define QM_SQE_ADDR_MASK GENMASK(7, 0) -#define QM_EQ_DEPTH (1024 * 2) #define QM_DRIVER_REMOVING 0 #define QM_RST_SCHED 1 @@ -271,8 +271,8 @@ ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \ ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) -#define QM_MK_CQC_DW3_V2(cqe_sz) \ - ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) +#define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \ + ((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) #define QM_MK_SQC_W13(priority, orders, alg_type) \ (((priority) << QM_SQ_PRIORITY_SHIFT) | \ @@ -285,8 +285,8 @@ ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \ ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) -#define QM_MK_SQC_DW3_V2(sqe_sz) \ - ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) +#define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \ + ((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) #define INIT_QC_COMMON(qc, base, pasid) do { \ (qc)->head = 0; \ @@ -330,6 +330,13 @@ enum qm_mb_cmd { QM_VF_GET_QOS, }; +enum qm_basic_type { + QM_TOTAL_QP_NUM_CAP = 0x0, + QM_FUNC_MAX_QP_CAP, + QM_XEQ_DEPTH_CAP, + QM_QP_DEPTH_CAP, +}; + static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, @@ -346,6 +353,13 @@ static const struct hisi_qm_cap_info qm_cap_info_vf[] = { {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0}, }; +static const struct hisi_qm_cap_info qm_basic_info[] = { + {QM_TOTAL_QP_NUM_CAP, 0x100158, 0, GENMASK(10, 0), 0x1000, 0x400, 0x400}, + {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400}, + {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(15, 0), 0x800, 0x4000800, 0x4000800}, + {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400}, +}; + struct qm_cqe { __le32 rsvd0; __le16 cmd_id; @@ -884,6 +898,16 @@ u32 hisi_qm_get_hw_info(struct hisi_qm *qm, } EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info); +static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, + u16 *high_bits, enum qm_basic_type type) +{ + u32 depth; + + depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver); + *high_bits = depth & QM_XQ_DEPTH_MASK; + *low_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; +} + static u32 qm_get_irq_num_v1(struct hisi_qm *qm) { return QM_IRQ_NUM_V1; @@ -935,7 +959,7 @@ static void qm_pm_put_sync(struct hisi_qm *qm) static void qm_cq_head_update(struct hisi_qp *qp) { - if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) { + if (qp->qp_status.cq_head == qp->cq_depth - 1) { qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase; qp->qp_status.cq_head = 0; } else { @@ -967,6 +991,7 @@ static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) { struct hisi_qm *qm = poll_data->qm; struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; + u16 eq_depth = qm->eq_depth; int eqe_num = 0; u16 cqn; @@ -975,7 +1000,7 @@ static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) poll_data->qp_finish_id[eqe_num] = cqn; eqe_num++; - if (qm->status.eq_head == QM_EQ_DEPTH - 1) { + if (qm->status.eq_head == eq_depth - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; eqe = qm->eqe; qm->status.eq_head = 0; @@ -984,7 +1009,7 @@ static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) qm->status.eq_head++; } - if (eqe_num == (QM_EQ_DEPTH >> 1) - 1) + if (eqe_num == (eq_depth >> 1) - 1) break; } @@ -1124,6 +1149,7 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; + u16 aeq_depth = qm->aeq_depth; u32 type, qp_id; while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { @@ -1148,7 +1174,7 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) break; } - if (qm->status.aeq_head == QM_Q_DEPTH - 1) { + if (qm->status.aeq_head == aeq_depth - 1) { qm->status.aeqc_phase = !qm->status.aeqc_phase; aeqe = qm->aeqe; qm->status.aeq_head = 0; @@ -2050,7 +2076,7 @@ err_free_ctx: } static int q_dump_param_parse(struct hisi_qm *qm, char *s, - u32 *e_id, u32 *q_id) + u32 *e_id, u32 *q_id, u16 q_depth) { struct device *dev = &qm->pdev->dev; unsigned int qp_num = qm->qp_num; @@ -2076,8 +2102,8 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, } ret = kstrtou32(presult, 0, e_id); - if (ret || *e_id >= QM_Q_DEPTH) { - dev_err(dev, "Please input sqe num (0-%d)", QM_Q_DEPTH - 1); + if (ret || *e_id >= q_depth) { + dev_err(dev, "Please input sqe num (0-%u)", q_depth - 1); return -EINVAL; } @@ -2091,21 +2117,22 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, static int qm_sq_dump(struct hisi_qm *qm, char *s) { + u16 sq_depth = qm->qp_array->cq_depth; void *sqe, *sqe_curr; struct hisi_qp *qp; u32 qp_id, sqe_id; int ret; - ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id); + ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id, sq_depth); if (ret) return ret; - sqe = kzalloc(qm->sqe_size * QM_Q_DEPTH, GFP_KERNEL); + sqe = kzalloc(qm->sqe_size * sq_depth, GFP_KERNEL); if (!sqe) return -ENOMEM; qp = &qm->qp_array[qp_id]; - memcpy(sqe, qp->sqe, qm->sqe_size * QM_Q_DEPTH); + memcpy(sqe, qp->sqe, qm->sqe_size * sq_depth); sqe_curr = sqe + (u32)(sqe_id * qm->sqe_size); memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK, qm->debug.sqe_mask_len); @@ -2124,7 +2151,7 @@ static int qm_cq_dump(struct hisi_qm *qm, char *s) u32 qp_id, cqe_id; int ret; - ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id); + ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id, qm->qp_array->cq_depth); if (ret) return ret; @@ -2150,11 +2177,11 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, if (ret) return -EINVAL; - if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) { - dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1); + if (!strcmp(name, "EQE") && xeqe_id >= qm->eq_depth) { + dev_err(dev, "Please input eqe num (0-%u)", qm->eq_depth - 1); return -EINVAL; - } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) { - dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1); + } else if (!strcmp(name, "AEQE") && xeqe_id >= qm->aeq_depth) { + dev_err(dev, "Please input aeqe num (0-%u)", qm->eq_depth - 1); return -EINVAL; } @@ -2805,7 +2832,7 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1)) + if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1)) return NULL; return qp->sqe + sq_tail * qp->qm->sqe_size; @@ -2846,7 +2873,7 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) qp = &qm->qp_array[qp_id]; hisi_qm_unset_hw_reset(qp); - memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH); + memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth); qp->event_cb = NULL; qp->req_cb = NULL; @@ -2927,9 +2954,9 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) INIT_QC_COMMON(sqc, qp->sqe_dma, pasid); if (ver == QM_HW_V1) { sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size)); - sqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); + sqc->w8 = cpu_to_le16(qp->sq_depth - 1); } else { - sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size)); + sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth)); sqc->w8 = 0; /* rand_qc */ } sqc->cq_num = cpu_to_le16(qp_id); @@ -2970,9 +2997,9 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) if (ver == QM_HW_V1) { cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE)); - cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); + cqc->w8 = cpu_to_le16(qp->cq_depth - 1); } else { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE)); + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth)); cqc->w8 = 0; /* rand_qc */ } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); @@ -3059,13 +3086,14 @@ static void qp_stop_fail_cb(struct hisi_qp *qp) { int qp_used = atomic_read(&qp->qp_status.used); u16 cur_tail = qp->qp_status.sq_tail; - u16 cur_head = (cur_tail + QM_Q_DEPTH - qp_used) % QM_Q_DEPTH; + u16 sq_depth = qp->sq_depth; + u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth; struct hisi_qm *qm = qp->qm; u16 pos; int i; for (i = 0; i < qp_used; i++) { - pos = (i + cur_head) % QM_Q_DEPTH; + pos = (i + cur_head) % sq_depth; qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); atomic_dec(&qp->qp_status.used); } @@ -3213,7 +3241,7 @@ int hisi_qp_send(struct hisi_qp *qp, const void *msg) { struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH; + u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth; void *sqe = qm_get_avail_sqe(qp); if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || @@ -3442,6 +3470,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) struct uacce_device *uacce; unsigned long mmio_page_nr; unsigned long dus_page_nr; + u16 sq_depth, cq_depth; struct uacce_interface interface = { .flags = UACCE_DEV_SVA, .ops = &uacce_qm_ops, @@ -3485,9 +3514,11 @@ static int qm_alloc_uacce(struct hisi_qm *qm) else mmio_page_nr = qm->db_interval / PAGE_SIZE; + qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); + /* Add one more page for device or qp status */ - dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH + - sizeof(struct qm_cqe) * QM_Q_DEPTH + PAGE_SIZE) >> + dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + + sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >> PAGE_SHIFT; uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; @@ -3592,10 +3623,11 @@ static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) kfree(qm->qp_array); } -static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) +static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, + u16 sq_depth, u16 cq_depth) { struct device *dev = &qm->pdev->dev; - size_t off = qm->sqe_size * QM_Q_DEPTH; + size_t off = qm->sqe_size * sq_depth; struct hisi_qp *qp; int ret = -ENOMEM; @@ -3615,6 +3647,8 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) qp->cqe = qp->qdma.va + off; qp->cqe_dma = qp->qdma.dma + off; qp->qdma.size = dma_size; + qp->sq_depth = sq_depth; + qp->cq_depth = cq_depth; qp->qm = qm; qp->qp_id = id; @@ -3858,7 +3892,7 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); - eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc->dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), DMA_TO_DEVICE); @@ -3887,7 +3921,7 @@ static int qm_aeq_ctx_cfg(struct hisi_qm *qm) aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); - aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + aeqc->dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc), DMA_TO_DEVICE); @@ -5947,19 +5981,21 @@ EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); static int qm_get_qp_num(struct hisi_qm *qm) { - if (qm->ver == QM_HW_V1) - qm->ctrl_qp_num = QM_QNUM_V1; - else if (qm->ver == QM_HW_V2) - qm->ctrl_qp_num = QM_QNUM_V2; - else - qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) & - QM_QP_NUN_MASK; + bool is_db_isolation; - if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) - qm->max_qp_num = (readl(qm->io_base + QM_CAPBILITY) >> - QM_QP_MAX_NUM_SHIFT) & QM_QP_NUN_MASK; - else - qm->max_qp_num = qm->ctrl_qp_num; + /* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */ + if (qm->fun_type == QM_HW_VF) { + if (qm->ver != QM_HW_V1) + /* v2 starts to support get vft by mailbox */ + return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + + return 0; + } + + is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); + qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true); + qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, + QM_FUNC_MAX_QP_CAP, is_db_isolation); /* check if qp number is valid */ if (qm->qp_num > qm->max_qp_num) { @@ -6039,11 +6075,9 @@ static int qm_get_pci_res(struct hisi_qm *qm) qm->db_interval = 0; } - if (qm->fun_type == QM_HW_PF) { - ret = qm_get_qp_num(qm); - if (ret) - goto err_db_ioremap; - } + ret = qm_get_qp_num(qm); + if (ret) + goto err_db_ioremap; return 0; @@ -6126,6 +6160,7 @@ static int hisi_qm_init_work(struct hisi_qm *qm) static int hisi_qp_alloc_memory(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; + u16 sq_depth, cq_depth; size_t qp_dma_size; int i, ret; @@ -6139,13 +6174,14 @@ static int hisi_qp_alloc_memory(struct hisi_qm *qm) return -ENOMEM; } + qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); + /* one more page for device or qp statuses */ - qp_dma_size = qm->sqe_size * QM_Q_DEPTH + - sizeof(struct qm_cqe) * QM_Q_DEPTH; + qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth; qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; for (i = 0; i < qm->qp_num; i++) { qm->poll_data[i].qm = qm; - ret = hisi_qp_memory_init(qm, qp_dma_size, i); + ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth); if (ret) goto err_init_qp_mem; @@ -6182,8 +6218,9 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) } while (0) idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + - QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + + qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) + + QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, @@ -6194,8 +6231,8 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) goto err_destroy_idr; } - QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); - QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, eqe, qm->eq_depth); + QM_INIT_BUF(qm, aeqe, qm->aeq_depth); QM_INIT_BUF(qm, sqc, qm->qp_num); QM_INIT_BUF(qm, cqc, qm->qp_num); @@ -6257,13 +6294,6 @@ int hisi_qm_init(struct hisi_qm *qm) if (ret) goto err_pci_init; - if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) { - /* v2 starts to support get vft by mailbox */ - ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); - if (ret) - goto err_irq_register; - } - if (qm->fun_type == QM_HW_PF) { qm_disable_clock_gate(qm); ret = qm_dev_mem_reset(qm); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index d2a0bc93e752..04e034abc5e8 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -17,6 +17,7 @@ struct sec_alg_res { dma_addr_t a_ivin_dma; u8 *out_mac; dma_addr_t out_mac_dma; + u16 depth; }; /* Cipher request of SEC private */ @@ -115,9 +116,9 @@ struct sec_cipher_ctx { /* SEC queue context which defines queue's relatives */ struct sec_qp_ctx { struct hisi_qp *qp; - struct sec_req *req_list[QM_Q_DEPTH]; + struct sec_req **req_list; struct idr req_idr; - struct sec_alg_res res[QM_Q_DEPTH]; + struct sec_alg_res *res; struct sec_ctx *ctx; spinlock_t req_lock; struct list_head backlog; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index ead061089e0c..eb23bffdcac8 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -59,14 +59,14 @@ #define SEC_ICV_MASK 0x000E #define SEC_SQE_LEN_RATE_MASK 0x3 -#define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) +#define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) #define SEC_SGL_SGE_NR 128 #define SEC_CIPHER_AUTH 0xfe #define SEC_AUTH_CIPHER 0x1 #define SEC_MAX_MAC_LEN 64 #define SEC_MAX_AAD_LEN 65535 #define SEC_MAX_CCM_AAD_LEN 65279 -#define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) +#define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) #define SEC_PBUF_SZ 512 #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ @@ -74,11 +74,11 @@ #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ SEC_MAX_MAC_LEN * 2) #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) -#define SEC_PBUF_PAGE_NUM (QM_Q_DEPTH / SEC_PBUF_NUM) -#define SEC_PBUF_LEFT_SZ (SEC_PBUF_PKG * (QM_Q_DEPTH - \ - SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM)) -#define SEC_TOTAL_PBUF_SZ (PAGE_SIZE * SEC_PBUF_PAGE_NUM + \ - SEC_PBUF_LEFT_SZ) +#define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM) +#define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \ + SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM)) +#define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ + SEC_PBUF_LEFT_SZ(depth)) #define SEC_SQE_LEN_RATE 4 #define SEC_SQE_CFLAG 2 @@ -128,9 +128,7 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) int req_id; spin_lock_bh(&qp_ctx->req_lock); - - req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, - 0, QM_Q_DEPTH, GFP_ATOMIC); + req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); spin_unlock_bh(&qp_ctx->req_lock); if (unlikely(req_id < 0)) { dev_err(req->ctx->dev, "alloc req id fail!\n"); @@ -148,7 +146,7 @@ static void sec_free_req_id(struct sec_req *req) struct sec_qp_ctx *qp_ctx = req->qp_ctx; int req_id = req->req_id; - if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { + if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) { dev_err(req->ctx->dev, "free request id invalid!\n"); return; } @@ -300,14 +298,15 @@ static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) /* Get DMA memory resources */ static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; int i; - res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, + res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), &res->c_ivin_dma, GFP_KERNEL); if (!res->c_ivin) return -ENOMEM; - for (i = 1; i < QM_Q_DEPTH; i++) { + for (i = 1; i < q_depth; i++) { res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; } @@ -318,20 +317,21 @@ static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) { if (res->c_ivin) - dma_free_coherent(dev, SEC_TOTAL_IV_SZ, + dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), res->c_ivin, res->c_ivin_dma); } static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; int i; - res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, + res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), &res->a_ivin_dma, GFP_KERNEL); if (!res->a_ivin) return -ENOMEM; - for (i = 1; i < QM_Q_DEPTH; i++) { + for (i = 1; i < q_depth; i++) { res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; } @@ -342,20 +342,21 @@ static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) { if (res->a_ivin) - dma_free_coherent(dev, SEC_TOTAL_IV_SZ, + dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), res->a_ivin, res->a_ivin_dma); } static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; int i; - res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1, + res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1, &res->out_mac_dma, GFP_KERNEL); if (!res->out_mac) return -ENOMEM; - for (i = 1; i < QM_Q_DEPTH; i++) { + for (i = 1; i < q_depth; i++) { res[i].out_mac_dma = res->out_mac_dma + i * (SEC_MAX_MAC_LEN << 1); res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); @@ -367,14 +368,14 @@ static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) { if (res->out_mac) - dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1, + dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1, res->out_mac, res->out_mac_dma); } static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) { if (res->pbuf) - dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ, + dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth), res->pbuf, res->pbuf_dma); } @@ -384,10 +385,12 @@ static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) */ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; + int size = SEC_PBUF_PAGE_NUM(q_depth); int pbuf_page_offset; int i, j, k; - res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ, + res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth), &res->pbuf_dma, GFP_KERNEL); if (!res->pbuf) return -ENOMEM; @@ -400,11 +403,11 @@ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) * So we need SEC_PBUF_PAGE_NUM numbers of PAGE * for the SEC_TOTAL_PBUF_SZ */ - for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) { + for (i = 0; i <= size; i++) { pbuf_page_offset = PAGE_SIZE * i; for (j = 0; j < SEC_PBUF_NUM; j++) { k = i * SEC_PBUF_NUM + j; - if (k == QM_Q_DEPTH) + if (k == q_depth) break; res[k].pbuf = res->pbuf + j * SEC_PBUF_PKG + pbuf_page_offset; @@ -470,36 +473,29 @@ static void sec_alg_resource_free(struct sec_ctx *ctx, sec_free_mac_resource(dev, qp_ctx->res); } -static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, - int qp_ctx_id, int alg_type) +static int sec_alloc_qp_ctx_resource(struct hisi_qm *qm, struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) { + u16 q_depth = qp_ctx->qp->sq_depth; struct device *dev = ctx->dev; - struct sec_qp_ctx *qp_ctx; - struct hisi_qp *qp; int ret = -ENOMEM; - qp_ctx = &ctx->qp_ctx[qp_ctx_id]; - qp = ctx->qps[qp_ctx_id]; - qp->req_type = 0; - qp->qp_ctx = qp_ctx; - qp_ctx->qp = qp; - qp_ctx->ctx = ctx; - - qp->req_cb = sec_req_cb; + qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL); + if (!qp_ctx->req_list) + return ret; - spin_lock_init(&qp_ctx->req_lock); - idr_init(&qp_ctx->req_idr); - INIT_LIST_HEAD(&qp_ctx->backlog); + qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL); + if (!qp_ctx->res) + goto err_free_req_list; + qp_ctx->res->depth = q_depth; - qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, - SEC_SGL_SGE_NR); + qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); if (IS_ERR(qp_ctx->c_in_pool)) { dev_err(dev, "fail to create sgl pool for input!\n"); - goto err_destroy_idr; + goto err_free_res; } - qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, - SEC_SGL_SGE_NR); + qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); if (IS_ERR(qp_ctx->c_out_pool)) { dev_err(dev, "fail to create sgl pool for output!\n"); goto err_free_c_in_pool; @@ -509,34 +505,72 @@ static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, if (ret) goto err_free_c_out_pool; - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) - goto err_queue_free; - return 0; -err_queue_free: - sec_alg_resource_free(ctx, qp_ctx); err_free_c_out_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); err_free_c_in_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); -err_destroy_idr: - idr_destroy(&qp_ctx->req_idr); +err_free_res: + kfree(qp_ctx->res); +err_free_req_list: + kfree(qp_ctx->req_list); return ret; } -static void sec_release_qp_ctx(struct sec_ctx *ctx, - struct sec_qp_ctx *qp_ctx) +static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { struct device *dev = ctx->dev; - hisi_qm_stop_qp(qp_ctx->qp); sec_alg_resource_free(ctx, qp_ctx); - hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); + kfree(qp_ctx->res); + kfree(qp_ctx->req_list); +} + +static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, + int qp_ctx_id, int alg_type) +{ + struct sec_qp_ctx *qp_ctx; + struct hisi_qp *qp; + int ret; + qp_ctx = &ctx->qp_ctx[qp_ctx_id]; + qp = ctx->qps[qp_ctx_id]; + qp->req_type = 0; + qp->qp_ctx = qp_ctx; + qp_ctx->qp = qp; + qp_ctx->ctx = ctx; + + qp->req_cb = sec_req_cb; + + spin_lock_init(&qp_ctx->req_lock); + idr_init(&qp_ctx->req_idr); + INIT_LIST_HEAD(&qp_ctx->backlog); + + ret = sec_alloc_qp_ctx_resource(qm, ctx, qp_ctx); + if (ret) + goto err_destroy_idr; + + ret = hisi_qm_start_qp(qp, 0); + if (ret < 0) + goto err_resource_free; + + return 0; + +err_resource_free: + sec_free_qp_ctx_resource(ctx, qp_ctx); +err_destroy_idr: + idr_destroy(&qp_ctx->req_idr); + return ret; +} + +static void sec_release_qp_ctx(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + hisi_qm_stop_qp(qp_ctx->qp); + sec_free_qp_ctx_resource(ctx, qp_ctx); idr_destroy(&qp_ctx->req_idr); } @@ -559,7 +593,7 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) ctx->pbuf_supported = ctx->sec->iommu_used; /* Half of queue depth is taken as fake requests limit in the queue. */ - ctx->fake_req_limit = QM_Q_DEPTH >> 1; + ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1; ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), GFP_KERNEL); if (!ctx->qp_ctx) { diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 1ec3b06345fd..ea7f85f72b10 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -27,7 +27,6 @@ #define SEC_BD_ERR_CHK_EN3 0xffffbfff #define SEC_SQE_SIZE 128 -#define SEC_SQ_SIZE (SEC_SQE_SIZE * QM_Q_DEPTH) #define SEC_PF_DEF_Q_NUM 256 #define SEC_PF_DEF_Q_BASE 0 #define SEC_CTX_Q_NUM_DEF 2 diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index a6c914d527eb..a7f6884c3ab3 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -599,12 +599,13 @@ static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) { + u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; struct hisi_zip_req_q *req_q; int i, ret; for (i = 0; i < HZIP_CTX_Q_NUM; i++) { req_q = &ctx->qp_ctx[i].req_q; - req_q->size = QM_Q_DEPTH; + req_q->size = q_depth; req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); if (!req_q->req_bitmap) { @@ -650,6 +651,7 @@ static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) { + u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; struct hisi_zip_qp_ctx *tmp; struct device *dev; int i; @@ -657,7 +659,7 @@ static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) for (i = 0; i < HZIP_CTX_Q_NUM; i++) { tmp = &ctx->qp_ctx[i]; dev = &tmp->qp->qm->pdev->dev; - tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH << 1, + tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, q_depth << 1, sgl_sge_nr); if (IS_ERR(tmp->sgl_pool)) { if (i == 1) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 36809bae6334..24d5226e0a0a 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -82,7 +82,6 @@ #define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \ HZIP_DECOMP_CORE_NUM) #define HZIP_SQE_SIZE 128 -#define HZIP_SQ_SIZE (HZIP_SQE_SIZE * QM_Q_DEPTH) #define HZIP_PF_DEF_Q_NUM 64 #define HZIP_PF_DEF_Q_BASE 0 diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 851c962ba473..371c0e7ffd27 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -109,7 +109,6 @@ QM_MAILBOX_TIMEOUT | QM_FLR_TIMEOUT) #define QM_BASE_CE QM_ECC_1BIT -#define QM_Q_DEPTH 1024 #define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 #define QM_SHAPER_CFG 0x100164 @@ -310,6 +309,8 @@ struct hisi_qm { u32 max_qp_num; u32 vfs_num; u32 db_interval; + u16 eq_depth; + u16 aeq_depth; struct list_head list; struct hisi_qm_list *qm_list; @@ -375,6 +376,8 @@ struct hisi_qp_ops { struct hisi_qp { u32 qp_id; + u16 sq_depth; + u16 cq_depth; u8 alg_type; u8 req_type; -- cgit From c832da79cbf9448e7ece097c3a93996b4c74a83e Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 9 Sep 2022 17:46:57 +0800 Subject: crypto: hisilicon/qm - add UACCE_CMD_QM_SET_QP_INFO support To be compatible with accelerator devices of different versions, 'UACCE_CMD_QM_SET_QP_INFO' ioctl is added to obtain queue information in userspace, including queue depth and buffer description size. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 21 ++++++++++++++++++--- include/uapi/misc/uacce/hisi_qm.h | 17 ++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index b3216ee627e5..e227a3e50600 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3430,6 +3430,7 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, unsigned long arg) { struct hisi_qp *qp = q->priv; + struct hisi_qp_info qp_info; struct hisi_qp_ctx qp_ctx; if (cmd == UACCE_CMD_QM_SET_QP_CTX) { @@ -3446,11 +3447,25 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, if (copy_to_user((void __user *)arg, &qp_ctx, sizeof(struct hisi_qp_ctx))) return -EFAULT; - } else { - return -EINVAL; + + return 0; + } else if (cmd == UACCE_CMD_QM_SET_QP_INFO) { + if (copy_from_user(&qp_info, (void __user *)arg, + sizeof(struct hisi_qp_info))) + return -EFAULT; + + qp_info.sqe_size = qp->qm->sqe_size; + qp_info.sq_depth = qp->sq_depth; + qp_info.cq_depth = qp->cq_depth; + + if (copy_to_user((void __user *)arg, &qp_info, + sizeof(struct hisi_qp_info))) + return -EFAULT; + + return 0; } - return 0; + return -EINVAL; } static const struct uacce_ops uacce_qm_ops = { diff --git a/include/uapi/misc/uacce/hisi_qm.h b/include/uapi/misc/uacce/hisi_qm.h index 1faef5ff87ef..3e66dbc2f323 100644 --- a/include/uapi/misc/uacce/hisi_qm.h +++ b/include/uapi/misc/uacce/hisi_qm.h @@ -14,11 +14,26 @@ struct hisi_qp_ctx { __u16 qc_type; }; +/** + * struct hisi_qp_info - User data for hisi qp. + * @sqe_size: Submission queue element size + * @sq_depth: The number of sqe + * @cq_depth: The number of cqe + * @reserved: Reserved data + */ +struct hisi_qp_info { + __u32 sqe_size; + __u16 sq_depth; + __u16 cq_depth; + __u64 reserved; +}; + #define HISI_QM_API_VER_BASE "hisi_qm_v1" #define HISI_QM_API_VER2_BASE "hisi_qm_v2" #define HISI_QM_API_VER3_BASE "hisi_qm_v3" /* UACCE_CMD_QM_SET_QP_CTX: Set qp algorithm type */ #define UACCE_CMD_QM_SET_QP_CTX _IOWR('H', 10, struct hisi_qp_ctx) - +/* UACCE_CMD_QM_SET_QP_INFO: Set qp depth and BD size */ +#define UACCE_CMD_QM_SET_QP_INFO _IOWR('H', 11, struct hisi_qp_info) #endif -- cgit From d90fab0deb8e580aa001f6876e4436c21e944f27 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 9 Sep 2022 17:46:58 +0800 Subject: crypto: hisilicon/qm - get error type from hardware registers Hardware V3 and later versions support get error type from registers. To be compatible with later hardware versions, get error type from registers instead of fixed marco. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 68 +++++++++++++++----- drivers/crypto/hisilicon/qm.c | 101 +++++++++++++----------------- drivers/crypto/hisilicon/sec2/sec.h | 11 ++++ drivers/crypto/hisilicon/sec2/sec_main.c | 51 ++++++++++----- drivers/crypto/hisilicon/zip/zip_main.c | 74 ++++++++++++++++------ include/linux/hisi_acc_qm.h | 27 ++------ 6 files changed, 206 insertions(+), 126 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index d484105b2716..36177c437c56 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -53,9 +53,7 @@ #define HPRE_CORE_IS_SCHD_OFFSET 0x90 #define HPRE_RAS_CE_ENB 0x301410 -#define HPRE_HAC_RAS_CE_ENABLE (BIT(0) | BIT(22) | BIT(23)) #define HPRE_RAS_NFE_ENB 0x301414 -#define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe #define HPRE_RAS_FE_ENB 0x301418 #define HPRE_OOO_SHUTDOWN_SEL 0x301a3c #define HPRE_HAC_RAS_FE_ENABLE 0 @@ -147,6 +145,28 @@ static const char * const hpre_debug_file_name[] = { [HPRE_CLUSTER_CTRL] = "cluster_ctrl", }; +enum hpre_cap_type { + HPRE_QM_NFE_MASK_CAP, + HPRE_QM_RESET_MASK_CAP, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, + HPRE_QM_CE_MASK_CAP, + HPRE_NFE_MASK_CAP, + HPRE_RESET_MASK_CAP, + HPRE_OOO_SHUTDOWN_MASK_CAP, + HPRE_CE_MASK_CAP, +}; + +static const struct hisi_qm_cap_info hpre_basic_info[] = { + {HPRE_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C37, 0x7C37}, + {HPRE_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC37, 0x6C37}, + {HPRE_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C37}, + {HPRE_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {HPRE_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xFFFFFE}, + {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFFFE}, + {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFFFE}, + {HPRE_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, +}; + static const struct hpre_hw_error hpre_hw_errors[] = { { .int_msk = BIT(0), @@ -630,7 +650,8 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); if (enable) { val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE; - val2 = HPRE_HAC_RAS_NFE_ENABLE; + val2 = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); } else { val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; val2 = 0x0; @@ -644,21 +665,30 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hpre_hw_error_disable(struct hisi_qm *qm) { - /* disable hpre hw error interrupts */ - writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK); + u32 ce, nfe; + ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); + nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + + /* disable hpre hw error interrupts */ + writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_INT_MASK); /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ hpre_master_ooo_ctrl(qm, false); } static void hpre_hw_error_enable(struct hisi_qm *qm) { + u32 ce, nfe; + + ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); + nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + /* clear HPRE hw error source if having */ - writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT); + writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_HAC_SOURCE_INT); /* configure error type */ - writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB); - writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB); + writel(ce, qm->io_base + HPRE_RAS_CE_ENB); + writel(nfe, qm->io_base + HPRE_RAS_NFE_ENB); writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB); /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */ @@ -1125,7 +1155,11 @@ static u32 hpre_get_hw_err_status(struct hisi_qm *qm) static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { + u32 nfe; + writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT); + nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + writel(nfe, qm->io_base + HPRE_RAS_NFE_ENB); } static void hpre_open_axi_master_ooo(struct hisi_qm *qm) @@ -1143,14 +1177,20 @@ static void hpre_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; - err_info->ce = QM_BASE_CE; - err_info->fe = 0; - err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | - HPRE_OOO_ECC_2BIT_ERR; - err_info->dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE; + err_info->fe = HPRE_HAC_RAS_FE_ENABLE; + err_info->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver); + err_info->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver); + err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR; + err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_RESET_MASK_CAP, qm->cap_ver); + err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HPRE_WR_MSI_PORT; err_info->acpi_rst = "HRST"; - err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT; } static const struct hisi_qm_err_ini hpre_err_ini = { diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index e227a3e50600..f62e48ccef2b 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -126,7 +126,6 @@ #define QM_DFX_CNT_CLR_CE 0x100118 #define QM_ABNORMAL_INT_SOURCE 0x100000 -#define QM_ABNORMAL_INT_SOURCE_CLR GENMASK(14, 0) #define QM_ABNORMAL_INT_MASK 0x100004 #define QM_ABNORMAL_INT_MASK_VALUE 0x7fff #define QM_ABNORMAL_INT_STATUS 0x100008 @@ -144,8 +143,10 @@ #define QM_RAS_NFE_ENABLE 0x1000f4 #define QM_RAS_CE_THRESHOLD 0x1000f8 #define QM_RAS_CE_TIMES_PER_IRQ 1 -#define QM_RAS_MSI_INT_SEL 0x1040f4 #define QM_OOO_SHUTDOWN_SEL 0x1040f8 +#define QM_ECC_MBIT BIT(2) +#define QM_DB_TIMEOUT BIT(10) +#define QM_OF_FIFO_OF BIT(11) #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 @@ -454,7 +455,7 @@ struct hisi_qm_hw_ops { u8 cmd, u16 index, u8 priority); u32 (*get_irq_num)(struct hisi_qm *qm); int (*debug_init)(struct hisi_qm *qm); - void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe); + void (*hw_error_init)(struct hisi_qm *qm); void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); int (*set_msi)(struct hisi_qm *qm, bool set); @@ -651,22 +652,17 @@ static u32 qm_get_dev_err_status(struct hisi_qm *qm) } /* Check if the error causes the master ooo block */ -static int qm_check_dev_error(struct hisi_qm *qm) +static bool qm_check_dev_error(struct hisi_qm *qm) { u32 val, dev_val; if (qm->fun_type == QM_HW_VF) - return 0; - - val = qm_get_hw_error_status(qm); - dev_val = qm_get_dev_err_status(qm); + return false; - if (qm->ver < QM_HW_V3) - return (val & QM_ECC_MBIT) || - (dev_val & qm->err_info.ecc_2bits_mask); + val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask; + dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask; - return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) || - (dev_val & (~qm->err_info.dev_ce_mask)); + return val || dev_val; } static int qm_wait_reset_finish(struct hisi_qm *qm) @@ -2346,58 +2342,65 @@ static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir, file->debug = &qm->debug; } -static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_init_v1(struct hisi_qm *qm) { writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); } -static void qm_hw_error_cfg(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_cfg(struct hisi_qm *qm) { - qm->error_mask = ce | nfe | fe; + struct hisi_qm_err_info *err_info = &qm->err_info; + + qm->error_mask = err_info->nfe | err_info->ce | err_info->fe; /* clear QM hw residual error source */ - writel(QM_ABNORMAL_INT_SOURCE_CLR, - qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); /* configure error type */ - writel(ce, qm->io_base + QM_RAS_CE_ENABLE); + writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE); writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); - writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE); - writel(fe, qm->io_base + QM_RAS_FE_ENABLE); + writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE); } -static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_init_v2(struct hisi_qm *qm) { - u32 irq_enable = ce | nfe | fe; - u32 irq_unmask = ~irq_enable; + u32 irq_unmask; - qm_hw_error_cfg(qm, ce, nfe, fe); + qm_hw_error_cfg(qm); + irq_unmask = ~qm->error_mask; irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_uninit_v2(struct hisi_qm *qm) { - writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); + u32 irq_mask = qm->error_mask; + + irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); + writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); } -static void qm_hw_error_init_v3(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_init_v3(struct hisi_qm *qm) { - u32 irq_enable = ce | nfe | fe; - u32 irq_unmask = ~irq_enable; + u32 irq_unmask; - qm_hw_error_cfg(qm, ce, nfe, fe); + qm_hw_error_cfg(qm); /* enable close master ooo when hardware error happened */ - writel(nfe & (~QM_DB_RANDOM_INVALID), qm->io_base + QM_OOO_SHUTDOWN_SEL); + writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + irq_unmask = ~qm->error_mask; irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_uninit_v3(struct hisi_qm *qm) { - writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); + u32 irq_mask = qm->error_mask; + + irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); + writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); /* disable close master ooo when hardware error happened */ writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); @@ -2442,7 +2445,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { - u32 error_status, tmp, val; + u32 error_status, tmp; /* read err sts */ tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS); @@ -2453,17 +2456,11 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); - val = error_status | QM_DB_RANDOM_INVALID | QM_BASE_CE; - /* ce error does not need to be reset */ - if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) { - writel(error_status, qm->io_base + - QM_ABNORMAL_INT_SOURCE); - writel(qm->err_info.nfe, - qm->io_base + QM_RAS_NFE_ENABLE); - return ACC_ERR_RECOVERED; - } + if (error_status & qm->err_info.qm_reset_mask) + return ACC_ERR_NEED_RESET; - return ACC_ERR_NEED_RESET; + writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE); } return ACC_ERR_RECOVERED; @@ -4202,14 +4199,12 @@ DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, static void qm_hw_error_init(struct hisi_qm *qm) { - struct hisi_qm_err_info *err_info = &qm->err_info; - if (!qm->ops->hw_error_init) { dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); return; } - qm->ops->hw_error_init(qm, err_info->ce, err_info->nfe, err_info->fe); + qm->ops->hw_error_init(qm); } static void qm_hw_error_uninit(struct hisi_qm *qm) @@ -4963,17 +4958,11 @@ static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) if (qm->err_ini->log_dev_hw_err) qm->err_ini->log_dev_hw_err(qm, err_sts); - /* ce error does not need to be reset */ - if ((err_sts | qm->err_info.dev_ce_mask) == - qm->err_info.dev_ce_mask) { - if (qm->err_ini->clear_dev_hw_err_status) - qm->err_ini->clear_dev_hw_err_status(qm, - err_sts); - - return ACC_ERR_RECOVERED; - } + if (err_sts & qm->err_info.dev_reset_mask) + return ACC_ERR_NEED_RESET; - return ACC_ERR_NEED_RESET; + if (qm->err_ini->clear_dev_hw_err_status) + qm->err_ini->clear_dev_hw_err_status(qm, err_sts); } return ACC_ERR_RECOVERED; diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 04e034abc5e8..895ba9b47554 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -192,6 +192,17 @@ struct sec_dev { bool iommu_used; }; +enum sec_cap_type { + SEC_QM_NFE_MASK_CAP = 0x0, + SEC_QM_RESET_MASK_CAP, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, + SEC_QM_CE_MASK_CAP, + SEC_NFE_MASK_CAP, + SEC_RESET_MASK_CAP, + SEC_OOO_SHUTDOWN_MASK_CAP, + SEC_CE_MASK_CAP, +}; + void sec_destroy_qps(struct hisi_qp **qps, int qp_num); struct hisi_qp **sec_create_qps(void); int sec_register_to_crypto(struct hisi_qm *qm); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index ea7f85f72b10..e814f94dd0d4 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -41,16 +41,12 @@ #define SEC_ECC_NUM 16 #define SEC_ECC_MASH 0xFF #define SEC_CORE_INT_DISABLE 0x0 -#define SEC_CORE_INT_ENABLE 0x7c1ff -#define SEC_CORE_INT_CLEAR 0x7c1ff #define SEC_SAA_ENABLE 0x17f #define SEC_RAS_CE_REG 0x301050 #define SEC_RAS_FE_REG 0x301054 #define SEC_RAS_NFE_REG 0x301058 -#define SEC_RAS_CE_ENB_MSK 0x88 #define SEC_RAS_FE_ENB_MSK 0x0 -#define SEC_RAS_NFE_ENB_MSK 0x7c177 #define SEC_OOO_SHUTDOWN_SEL 0x301014 #define SEC_RAS_DISABLE 0x0 #define SEC_MEM_START_INIT_REG 0x301100 @@ -136,6 +132,17 @@ static struct hisi_qm_list sec_devices = { .unregister_from_crypto = sec_unregister_from_crypto, }; +static const struct hisi_qm_cap_info sec_basic_info[] = { + {SEC_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C77, 0x7C77}, + {SEC_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC77, 0x6C77}, + {SEC_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, + {SEC_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {SEC_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x177, 0x60177}, + {SEC_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177}, + {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177}, + {SEC_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088}, +}; + static const struct sec_hw_error sec_hw_errors[] = { { .int_msk = BIT(0), @@ -575,7 +582,8 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + SEC_CONTROL_REG); if (enable) { val1 |= SEC_AXI_SHUTDOWN_ENABLE; - val2 = SEC_RAS_NFE_ENB_MSK; + val2 = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); } else { val1 &= SEC_AXI_SHUTDOWN_DISABLE; val2 = 0x0; @@ -589,25 +597,30 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void sec_hw_error_enable(struct hisi_qm *qm) { + u32 ce, nfe; + if (qm->ver == QM_HW_V1) { writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); pci_info(qm->pdev, "V1 not support hw error handle\n"); return; } + ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); + nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + /* clear SEC hw error source if having */ - writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE); + writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_SOURCE); /* enable RAS int */ - writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG); + writel(ce, qm->io_base + SEC_RAS_CE_REG); writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); - writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG); + writel(nfe, qm->io_base + SEC_RAS_NFE_REG); /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ sec_master_ooo_ctrl(qm, true); /* enable SEC hw error interrupts */ - writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); + writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_MASK); } static void sec_hw_error_disable(struct hisi_qm *qm) @@ -938,7 +951,11 @@ static u32 sec_get_hw_err_status(struct hisi_qm *qm) static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { + u32 nfe; + writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); + nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + writel(nfe, qm->io_base + SEC_RAS_NFE_REG); } static void sec_open_axi_master_ooo(struct hisi_qm *qm) @@ -954,14 +971,20 @@ static void sec_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; - err_info->ce = QM_BASE_CE; - err_info->fe = 0; + err_info->fe = SEC_RAS_FE_ENB_MSK; + err_info->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); + err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; - err_info->dev_ce_mask = SEC_RAS_CE_ENB_MSK; + err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_RESET_MASK_CAP, qm->cap_ver); + err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = BIT(0); err_info->acpi_rst = "SRST"; - err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | - QM_ACC_WB_NOT_READY_TIMEOUT; } static const struct hisi_qm_err_ini sec_err_ini = { diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 24d5226e0a0a..6ed10ec05a73 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -69,11 +69,10 @@ #define HZIP_CORE_INT_STATUS_M_ECC BIT(1) #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 #define HZIP_CORE_INT_RAS_CE_ENB 0x301160 -#define HZIP_CORE_INT_RAS_CE_ENABLE 0x1 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168 +#define HZIP_CORE_INT_RAS_FE_ENB_MASK 0x0 #define HZIP_OOO_SHUTDOWN_SEL 0x30120C -#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x1FFE #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 #define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0) @@ -186,6 +185,28 @@ struct hisi_zip_ctrl { struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; }; +enum zip_cap_type { + ZIP_QM_NFE_MASK_CAP = 0x0, + ZIP_QM_RESET_MASK_CAP, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, + ZIP_QM_CE_MASK_CAP, + ZIP_NFE_MASK_CAP, + ZIP_RESET_MASK_CAP, + ZIP_OOO_SHUTDOWN_MASK_CAP, + ZIP_CE_MASK_CAP, +}; + +static struct hisi_qm_cap_info zip_basic_cap_info[] = { + {ZIP_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C57, 0x7C77}, + {ZIP_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC57, 0x6C77}, + {ZIP_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, + {ZIP_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {ZIP_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x1FFE}, + {ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE}, + {ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE}, + {ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, +}; + enum { HZIP_COMP_CORE0, HZIP_COMP_CORE1, @@ -457,7 +478,8 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); if (enable) { val1 |= HZIP_AXI_SHUTDOWN_ENABLE; - val2 = HZIP_CORE_INT_RAS_NFE_ENABLE; + val2 = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); } else { val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE; val2 = 0x0; @@ -471,6 +493,8 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hisi_zip_hw_error_enable(struct hisi_qm *qm) { + u32 nfe, ce; + if (qm->ver == QM_HW_V1) { writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); @@ -478,17 +502,17 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) return; } + nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); + /* clear ZIP hw error source if having */ - writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_SOURCE); + writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_SOURCE); /* configure error type */ - writel(HZIP_CORE_INT_RAS_CE_ENABLE, - qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); - writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); - writel(HZIP_CORE_INT_RAS_NFE_ENABLE, - qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); + writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); - /* enable ZIP block master OOO when nfe occurs on Kunpeng930 */ hisi_zip_master_ooo_ctrl(qm, true); /* enable ZIP hw error interrupts */ @@ -497,10 +521,13 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) static void hisi_zip_hw_error_disable(struct hisi_qm *qm) { + u32 nfe, ce; + /* disable ZIP hw error interrupts */ - writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); + nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); + writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG); - /* disable ZIP block master OOO when nfe occurs on Kunpeng930 */ hisi_zip_master_ooo_ctrl(qm, false); } @@ -900,7 +927,11 @@ static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm) static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { + u32 nfe; + writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE); + nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); } static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) @@ -934,16 +965,21 @@ static void hisi_zip_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; - err_info->ce = QM_BASE_CE; - err_info->fe = 0; + err_info->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + err_info->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver); + err_info->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_NFE_MASK_CAP, qm->cap_ver); err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; - err_info->dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE; + err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_RESET_MASK_CAP, qm->cap_ver); + err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HZIP_WR_PORT; err_info->acpi_rst = "ZRST"; - err_info->nfe = QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT; - - if (qm->ver >= QM_HW_V3) - err_info->nfe |= QM_ACC_DO_TASK_TIMEOUT; } static const struct hisi_qm_err_ini hisi_zip_err_ini = { diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 371c0e7ffd27..e230c7c46110 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -87,28 +87,6 @@ #define PEH_AXUSER_CFG 0x401001 #define PEH_AXUSER_CFG_ENABLE 0xffffffff -#define QM_AXI_RRESP BIT(0) -#define QM_AXI_BRESP BIT(1) -#define QM_ECC_MBIT BIT(2) -#define QM_ECC_1BIT BIT(3) -#define QM_ACC_GET_TASK_TIMEOUT BIT(4) -#define QM_ACC_DO_TASK_TIMEOUT BIT(5) -#define QM_ACC_WB_NOT_READY_TIMEOUT BIT(6) -#define QM_SQ_CQ_VF_INVALID BIT(7) -#define QM_CQ_VF_INVALID BIT(8) -#define QM_SQ_VF_INVALID BIT(9) -#define QM_DB_TIMEOUT BIT(10) -#define QM_OF_FIFO_OF BIT(11) -#define QM_DB_RANDOM_INVALID BIT(12) -#define QM_MAILBOX_TIMEOUT BIT(13) -#define QM_FLR_TIMEOUT BIT(14) - -#define QM_BASE_NFE (QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \ - QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \ - QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID | \ - QM_MAILBOX_TIMEOUT | QM_FLR_TIMEOUT) -#define QM_BASE_CE QM_ECC_1BIT - #define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 #define QM_SHAPER_CFG 0x100164 @@ -240,7 +218,10 @@ struct hisi_qm_err_info { char *acpi_rst; u32 msi_wr_port; u32 ecc_2bits_mask; - u32 dev_ce_mask; + u32 qm_shutdown_mask; + u32 dev_shutdown_mask; + u32 qm_reset_mask; + u32 dev_reset_mask; u32 ce; u32 nfe; u32 fe; -- cgit From 3536cc55cadaf2a03241915f9cfdaf6cd073e4fe Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 9 Sep 2022 17:46:59 +0800 Subject: crypto: hisilicon/qm - support get device irq information from hardware registers Support get device irq information from hardware registers instead of fixed macros. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 294 ++++++++++++++++++++++++++++-------------- 1 file changed, 195 insertions(+), 99 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index f62e48ccef2b..e4a506d02d23 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -22,15 +22,11 @@ #define QM_VF_AEQ_INT_MASK 0x4 #define QM_VF_EQ_INT_SOURCE 0x8 #define QM_VF_EQ_INT_MASK 0xc -#define QM_IRQ_NUM_V1 1 -#define QM_IRQ_NUM_PF_V2 4 -#define QM_IRQ_NUM_VF_V2 2 -#define QM_IRQ_NUM_VF_V3 3 -#define QM_EQ_EVENT_IRQ_VECTOR 0 -#define QM_AEQ_EVENT_IRQ_VECTOR 1 -#define QM_CMD_EVENT_IRQ_VECTOR 2 -#define QM_ABNORMAL_EVENT_IRQ_VECTOR 3 +#define QM_IRQ_VECTOR_MASK GENMASK(15, 0) +#define QM_IRQ_TYPE_MASK GENMASK(15, 0) +#define QM_IRQ_TYPE_SHIFT 16 +#define QM_ABN_IRQ_TYPE_MASK GENMASK(7, 0) /* mailbox */ #define QM_MB_PING_ALL_VFS 0xffff @@ -336,6 +332,12 @@ enum qm_basic_type { QM_FUNC_MAX_QP_CAP, QM_XEQ_DEPTH_CAP, QM_QP_DEPTH_CAP, + QM_EQ_IRQ_TYPE_CAP, + QM_AEQ_IRQ_TYPE_CAP, + QM_ABN_IRQ_TYPE_CAP, + QM_PF2VF_IRQ_TYPE_CAP, + QM_PF_IRQ_NUM_CAP, + QM_VF_IRQ_NUM_CAP, }; static const struct hisi_qm_cap_info qm_cap_info_comm[] = { @@ -359,6 +361,12 @@ static const struct hisi_qm_cap_info qm_basic_info[] = { {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400}, {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(15, 0), 0x800, 0x4000800, 0x4000800}, {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400}, + {QM_EQ_IRQ_TYPE_CAP, 0x310c, 0, GENMASK(31, 0), 0x10000, 0x10000, 0x10000}, + {QM_AEQ_IRQ_TYPE_CAP, 0x3110, 0, GENMASK(31, 0), 0x0, 0x10001, 0x10001}, + {QM_ABN_IRQ_TYPE_CAP, 0x3114, 0, GENMASK(31, 0), 0x0, 0x10003, 0x10003}, + {QM_PF2VF_IRQ_TYPE_CAP, 0x3118, 0, GENMASK(31, 0), 0x0, 0x0, 0x10002}, + {QM_PF_IRQ_NUM_CAP, 0x311c, 16, GENMASK(15, 0), 0x1, 0x4, 0x4}, + {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, }; struct qm_cqe { @@ -453,7 +461,6 @@ struct hisi_qm_hw_ops { int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number); void (*qm_db)(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority); - u32 (*get_irq_num)(struct hisi_qm *qm); int (*debug_init)(struct hisi_qm *qm); void (*hw_error_init)(struct hisi_qm *qm); void (*hw_error_uninit)(struct hisi_qm *qm); @@ -562,6 +569,8 @@ static struct qm_typical_qos_table shaper_cbs_s[] = { {50100, 100000, 19} }; +static void qm_irqs_unregister(struct hisi_qm *qm); + static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new) { enum qm_state curr = atomic_read(&qm->status.flags); @@ -904,25 +913,12 @@ static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, *low_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; } -static u32 qm_get_irq_num_v1(struct hisi_qm *qm) -{ - return QM_IRQ_NUM_V1; -} - -static u32 qm_get_irq_num_v2(struct hisi_qm *qm) -{ - if (qm->fun_type == QM_HW_PF) - return QM_IRQ_NUM_PF_V2; - else - return QM_IRQ_NUM_VF_V2; -} - -static u32 qm_get_irq_num_v3(struct hisi_qm *qm) +static u32 qm_get_irq_num(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_PF) - return QM_IRQ_NUM_PF_V2; + return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver); - return QM_IRQ_NUM_VF_V3; + return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver); } static int qm_pm_get_sync(struct hisi_qm *qm) @@ -1196,24 +1192,6 @@ static irqreturn_t qm_aeq_irq(int irq, void *data) return IRQ_WAKE_THREAD; } -static void qm_irq_unregister(struct hisi_qm *qm) -{ - struct pci_dev *pdev = qm->pdev; - - free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); - - if (qm->ver > QM_HW_V1) { - free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); - - if (qm->fun_type == QM_HW_PF) - free_irq(pci_irq_vector(pdev, - QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); - } - - if (qm->ver > QM_HW_V2) - free_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), qm); -} - static void qm_init_qp_status(struct hisi_qp *qp) { struct hisi_qp_status *qp_status = &qp->qp_status; @@ -2799,7 +2777,6 @@ static int qm_set_msi_v3(struct hisi_qm *qm, bool set) static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, - .get_irq_num = qm_get_irq_num_v1, .hw_error_init = qm_hw_error_init_v1, .set_msi = qm_set_msi, }; @@ -2807,7 +2784,6 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { .get_vft = qm_get_vft_v2, .qm_db = qm_db_v2, - .get_irq_num = qm_get_irq_num_v2, .hw_error_init = qm_hw_error_init_v2, .hw_error_uninit = qm_hw_error_uninit_v2, .hw_error_handle = qm_hw_error_handle_v2, @@ -2817,7 +2793,6 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .get_vft = qm_get_vft_v2, .qm_db = qm_db_v2, - .get_irq_num = qm_get_irq_num_v3, .hw_error_init = qm_hw_error_init_v3, .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, @@ -3803,7 +3778,7 @@ void hisi_qm_uninit(struct hisi_qm *qm) hisi_qm_set_state(qm, QM_NOT_READY); up_write(&qm->qps_lock); - qm_irq_unregister(qm); + qm_irqs_unregister(qm); hisi_qm_pci_uninit(qm); if (qm->use_sva) { uacce_remove(qm->uacce); @@ -5658,51 +5633,6 @@ static irqreturn_t qm_abnormal_irq(int irq, void *data) return IRQ_HANDLED; } -static int qm_irq_register(struct hisi_qm *qm) -{ - struct pci_dev *pdev = qm->pdev; - int ret; - - ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), - qm_irq, 0, qm->dev_name, qm); - if (ret) - return ret; - - if (qm->ver > QM_HW_V1) { - ret = request_threaded_irq(pci_irq_vector(pdev, - QM_AEQ_EVENT_IRQ_VECTOR), - qm_aeq_irq, qm_aeq_thread, - 0, qm->dev_name, qm); - if (ret) - goto err_aeq_irq; - - if (qm->fun_type == QM_HW_PF) { - ret = request_irq(pci_irq_vector(pdev, - QM_ABNORMAL_EVENT_IRQ_VECTOR), - qm_abnormal_irq, 0, qm->dev_name, qm); - if (ret) - goto err_abonormal_irq; - } - } - - if (qm->ver > QM_HW_V2) { - ret = request_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), - qm_mb_cmd_irq, 0, qm->dev_name, qm); - if (ret) - goto err_mb_cmd_irq; - } - - return 0; - -err_mb_cmd_irq: - if (qm->fun_type == QM_HW_PF) - free_irq(pci_irq_vector(pdev, QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); -err_abonormal_irq: - free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); -err_aeq_irq: - free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); - return ret; -} /** * hisi_qm_dev_shutdown() - Shutdown device. @@ -5983,6 +5913,176 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); +static void qm_unregister_abnormal_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + if (qm->fun_type == QM_HW_VF) + return; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_abnormal_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + if (qm->fun_type == QM_HW_VF) + return 0; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_mb_cmd_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_aeq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_aeq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), qm_aeq_irq, + qm_aeq_thread, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_eq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_eq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + + return ret; +} + +static void qm_irqs_unregister(struct hisi_qm *qm) +{ + qm_unregister_mb_cmd_irq(qm); + qm_unregister_abnormal_irq(qm); + qm_unregister_aeq_irq(qm); + qm_unregister_eq_irq(qm); +} + +static int qm_irqs_register(struct hisi_qm *qm) +{ + int ret; + + ret = qm_register_eq_irq(qm); + if (ret) + return ret; + + ret = qm_register_aeq_irq(qm); + if (ret) + goto free_eq_irq; + + ret = qm_register_abnormal_irq(qm); + if (ret) + goto free_aeq_irq; + + ret = qm_register_mb_cmd_irq(qm); + if (ret) + goto free_abnormal_irq; + + return 0; + +free_abnormal_irq: + qm_unregister_abnormal_irq(qm); +free_aeq_irq: + qm_unregister_aeq_irq(qm); +free_eq_irq: + qm_unregister_eq_irq(qm); + return ret; +} + static int qm_get_qp_num(struct hisi_qm *qm) { bool is_db_isolation; @@ -6117,11 +6217,7 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) goto err_get_pci_res; pci_set_master(pdev); - if (!qm->ops->get_irq_num) { - ret = -EOPNOTSUPP; - goto err_get_pci_res; - } - num_vec = qm->ops->get_irq_num(qm); + num_vec = qm_get_irq_num(qm); ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to enable MSI vectors!\n"); @@ -6294,7 +6390,7 @@ int hisi_qm_init(struct hisi_qm *qm) if (ret) return ret; - ret = qm_irq_register(qm); + ret = qm_irqs_register(qm); if (ret) goto err_pci_init; @@ -6336,7 +6432,7 @@ err_alloc_uacce: qm->uacce = NULL; } err_irq_register: - qm_irq_unregister(qm); + qm_irqs_unregister(qm); err_pci_init: hisi_qm_pci_uninit(qm); return ret; -- cgit From f214d59a0603543cfa7c6c1cf2eb130ac77480c3 Mon Sep 17 00:00:00 2001 From: Zhiqi Song Date: Fri, 9 Sep 2022 17:47:00 +0800 Subject: crypto: hisilicon/hpre - support hpre capability Read some hpre device configuration info from capability register, instead of fixed macros. Signed-off-by: Zhiqi Song Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre.h | 8 +- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 134 ++++++++++++++++++++++------ drivers/crypto/hisilicon/hpre/hpre_main.c | 53 +++++++++-- 3 files changed, 157 insertions(+), 38 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 9a0558ed82f9..9f0b94c8e03d 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -22,7 +22,8 @@ enum { HPRE_CLUSTER0, HPRE_CLUSTER1, HPRE_CLUSTER2, - HPRE_CLUSTER3 + HPRE_CLUSTER3, + HPRE_CLUSTERS_NUM_MAX }; enum hpre_ctrl_dbgfs_file { @@ -42,9 +43,6 @@ enum hpre_dfx_dbgfs_file { HPRE_DFX_FILE_NUM }; -#define HPRE_CLUSTERS_NUM_V2 (HPRE_CLUSTER3 + 1) -#define HPRE_CLUSTERS_NUM_V3 1 -#define HPRE_CLUSTERS_NUM_MAX HPRE_CLUSTERS_NUM_V2 #define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM_MAX - 1) struct hpre_debugfs_file { @@ -105,5 +103,5 @@ struct hpre_sqe { struct hisi_qp *hpre_create_qp(u8 type); int hpre_algs_register(struct hisi_qm *qm); void hpre_algs_unregister(struct hisi_qm *qm); - +bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg); #endif diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 2aa91a4f77da..ad1475feb313 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -51,6 +51,12 @@ struct hpre_ctx; #define HPRE_ECC_HW256_KSZ_B 32 #define HPRE_ECC_HW384_KSZ_B 48 +/* capability register mask */ +#define HPRE_DRV_RSA_MASK_CAP BIT(0) +#define HPRE_DRV_DH_MASK_CAP BIT(1) +#define HPRE_DRV_ECDH_MASK_CAP BIT(2) +#define HPRE_DRV_X25519_MASK_CAP BIT(5) + typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); struct hpre_rsa_ctx { @@ -2070,22 +2076,75 @@ static struct kpp_alg curve25519_alg = { }, }; +static int hpre_register_rsa(struct hisi_qm *qm) +{ + int ret; + + if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) + return 0; -static int hpre_register_ecdh(void) + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + if (ret) + dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret); + + return ret; +} + +static void hpre_unregister_rsa(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) + return; + + crypto_unregister_akcipher(&rsa); +} + +static int hpre_register_dh(struct hisi_qm *qm) { int ret; - ret = crypto_register_kpp(&ecdh_nist_p192); + if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) + return 0; + + ret = crypto_register_kpp(&dh); if (ret) + dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret); + + return ret; +} + +static void hpre_unregister_dh(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) + return; + + crypto_unregister_kpp(&dh); +} + +static int hpre_register_ecdh(struct hisi_qm *qm) +{ + int ret; + + if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) + return 0; + + ret = crypto_register_kpp(&ecdh_nist_p192); + if (ret) { + dev_err(&qm->pdev->dev, "failed to register ecdh_nist_p192 (%d)!\n", ret); return ret; + } ret = crypto_register_kpp(&ecdh_nist_p256); - if (ret) + if (ret) { + dev_err(&qm->pdev->dev, "failed to register ecdh_nist_p256 (%d)!\n", ret); goto unregister_ecdh_p192; + } ret = crypto_register_kpp(&ecdh_nist_p384); - if (ret) + if (ret) { + dev_err(&qm->pdev->dev, "failed to register ecdh_nist_p384 (%d)!\n", ret); goto unregister_ecdh_p256; + } return 0; @@ -2096,52 +2155,73 @@ unregister_ecdh_p192: return ret; } -static void hpre_unregister_ecdh(void) +static void hpre_unregister_ecdh(struct hisi_qm *qm) { + if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) + return; + crypto_unregister_kpp(&ecdh_nist_p384); crypto_unregister_kpp(&ecdh_nist_p256); crypto_unregister_kpp(&ecdh_nist_p192); } +static int hpre_register_x25519(struct hisi_qm *qm) +{ + int ret; + + if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) + return 0; + + ret = crypto_register_kpp(&curve25519_alg); + if (ret) + dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret); + + return ret; +} + +static void hpre_unregister_x25519(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) + return; + + crypto_unregister_kpp(&curve25519_alg); +} + int hpre_algs_register(struct hisi_qm *qm) { int ret; - rsa.base.cra_flags = 0; - ret = crypto_register_akcipher(&rsa); + ret = hpre_register_rsa(qm); if (ret) return ret; - ret = crypto_register_kpp(&dh); + ret = hpre_register_dh(qm); if (ret) goto unreg_rsa; - if (qm->ver >= QM_HW_V3) { - ret = hpre_register_ecdh(); - if (ret) - goto unreg_dh; - ret = crypto_register_kpp(&curve25519_alg); - if (ret) - goto unreg_ecdh; - } - return 0; + ret = hpre_register_ecdh(qm); + if (ret) + goto unreg_dh; + + ret = hpre_register_x25519(qm); + if (ret) + goto unreg_ecdh; + + return ret; unreg_ecdh: - hpre_unregister_ecdh(); + hpre_unregister_ecdh(qm); unreg_dh: - crypto_unregister_kpp(&dh); + hpre_unregister_dh(qm); unreg_rsa: - crypto_unregister_akcipher(&rsa); + hpre_unregister_rsa(qm); return ret; } void hpre_algs_unregister(struct hisi_qm *qm) { - if (qm->ver >= QM_HW_V3) { - crypto_unregister_kpp(&curve25519_alg); - hpre_unregister_ecdh(); - } - - crypto_unregister_kpp(&dh); - crypto_unregister_akcipher(&rsa); + hpre_unregister_x25519(qm); + hpre_unregister_ecdh(qm); + hpre_unregister_dh(qm); + hpre_unregister_rsa(qm); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 36177c437c56..407cdd9d8413 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -77,8 +77,6 @@ #define HPRE_QM_AXI_CFG_MASK GENMASK(15, 0) #define HPRE_QM_VFG_AX_MASK GENMASK(7, 0) #define HPRE_BD_USR_MASK GENMASK(1, 0) -#define HPRE_CLUSTER_CORE_MASK_V2 GENMASK(3, 0) -#define HPRE_CLUSTER_CORE_MASK_V3 GENMASK(7, 0) #define HPRE_PREFETCH_CFG 0x301130 #define HPRE_SVA_PREFTCH_DFX 0x30115C #define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30))) @@ -154,6 +152,23 @@ enum hpre_cap_type { HPRE_RESET_MASK_CAP, HPRE_OOO_SHUTDOWN_MASK_CAP, HPRE_CE_MASK_CAP, + HPRE_CLUSTER_NUM_CAP, + HPRE_CORE_TYPE_NUM_CAP, + HPRE_CORE_NUM_CAP, + HPRE_CLUSTER_CORE_NUM_CAP, + HPRE_CORE_ENABLE_BITMAP_CAP, + HPRE_DRV_ALG_BITMAP_CAP, + HPRE_DEV_ALG_BITMAP_CAP, + HPRE_CORE1_ALG_BITMAP_CAP, + HPRE_CORE2_ALG_BITMAP_CAP, + HPRE_CORE3_ALG_BITMAP_CAP, + HPRE_CORE4_ALG_BITMAP_CAP, + HPRE_CORE5_ALG_BITMAP_CAP, + HPRE_CORE6_ALG_BITMAP_CAP, + HPRE_CORE7_ALG_BITMAP_CAP, + HPRE_CORE8_ALG_BITMAP_CAP, + HPRE_CORE9_ALG_BITMAP_CAP, + HPRE_CORE10_ALG_BITMAP_CAP }; static const struct hisi_qm_cap_info hpre_basic_info[] = { @@ -165,6 +180,23 @@ static const struct hisi_qm_cap_info hpre_basic_info[] = { {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFFFE}, {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFFFE}, {HPRE_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, + {HPRE_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x0, 0x4, 0x1}, + {HPRE_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x0, 0x2, 0x2}, + {HPRE_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x0, 0x8, 0xA}, + {HPRE_CLUSTER_CORE_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x0, 0x2, 0xA}, + {HPRE_CORE_ENABLE_BITMAP_CAP, 0x3140, 0, GENMASK(31, 0), 0x0, 0xF, 0x3FF}, + {HPRE_DRV_ALG_BITMAP_CAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x03, 0x27}, + {HPRE_DEV_ALG_BITMAP_CAP, 0x3148, 0, GENMASK(31, 0), 0x0, 0x03, 0x7F}, + {HPRE_CORE1_ALG_BITMAP_CAP, 0x314c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE2_ALG_BITMAP_CAP, 0x3150, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE3_ALG_BITMAP_CAP, 0x3154, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE4_ALG_BITMAP_CAP, 0x3158, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE5_ALG_BITMAP_CAP, 0x315c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE6_ALG_BITMAP_CAP, 0x3160, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE7_ALG_BITMAP_CAP, 0x3164, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE8_ALG_BITMAP_CAP, 0x3168, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE9_ALG_BITMAP_CAP, 0x316c, 0, GENMASK(31, 0), 0x0, 0x10, 0x10}, + {HPRE_CORE10_ALG_BITMAP_CAP, 0x3170, 0, GENMASK(31, 0), 0x0, 0x10, 0x10} }; static const struct hpre_hw_error hpre_hw_errors[] = { @@ -282,6 +314,17 @@ static struct dfx_diff_registers hpre_diff_regs[] = { }, }; +bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) +{ + u32 cap_val; + + cap_val = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DRV_ALG_BITMAP_CAP, qm->cap_ver); + if (alg & cap_val) + return true; + + return false; +} + static int hpre_diff_regs_show(struct seq_file *s, void *unused) { struct hisi_qm *qm = s->private; @@ -350,14 +393,12 @@ MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); static inline int hpre_cluster_num(struct hisi_qm *qm) { - return (qm->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : - HPRE_CLUSTERS_NUM_V2; + return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CLUSTER_NUM_CAP, qm->cap_ver); } static inline int hpre_cluster_core_mask(struct hisi_qm *qm) { - return (qm->ver >= QM_HW_V3) ? - HPRE_CLUSTER_CORE_MASK_V3 : HPRE_CLUSTER_CORE_MASK_V2; + return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CORE_ENABLE_BITMAP_CAP, qm->cap_ver); } struct hisi_qp *hpre_create_qp(u8 type) -- cgit From b1be70a8c983a5bd12d88181b75d0f550086cb44 Mon Sep 17 00:00:00 2001 From: Zhiqi Song Date: Fri, 9 Sep 2022 17:47:01 +0800 Subject: crypto: hisilicon/hpre - optimize registration of ecdh Use table to store the different ecdh curve configuration, making the registration of ecdh clearer and expansion more convenient. Signed-off-by: Zhiqi Song Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 136 +++++++++++++--------------- 1 file changed, 63 insertions(+), 73 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index ad1475feb313..ac7fabf65865 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -2008,55 +2008,53 @@ static struct kpp_alg dh = { }, }; -static struct kpp_alg ecdh_nist_p192 = { - .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, - .max_size = hpre_ecdh_max_size, - .init = hpre_ecdh_nist_p192_init_tfm, - .exit = hpre_ecdh_exit_tfm, - .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, - .base = { - .cra_ctxsize = sizeof(struct hpre_ctx), - .cra_priority = HPRE_CRYPTO_ALG_PRI, - .cra_name = "ecdh-nist-p192", - .cra_driver_name = "hpre-ecdh-nist-p192", - .cra_module = THIS_MODULE, - }, -}; - -static struct kpp_alg ecdh_nist_p256 = { - .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, - .max_size = hpre_ecdh_max_size, - .init = hpre_ecdh_nist_p256_init_tfm, - .exit = hpre_ecdh_exit_tfm, - .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, - .base = { - .cra_ctxsize = sizeof(struct hpre_ctx), - .cra_priority = HPRE_CRYPTO_ALG_PRI, - .cra_name = "ecdh-nist-p256", - .cra_driver_name = "hpre-ecdh-nist-p256", - .cra_module = THIS_MODULE, - }, -}; - -static struct kpp_alg ecdh_nist_p384 = { - .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, - .max_size = hpre_ecdh_max_size, - .init = hpre_ecdh_nist_p384_init_tfm, - .exit = hpre_ecdh_exit_tfm, - .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, - .base = { - .cra_ctxsize = sizeof(struct hpre_ctx), - .cra_priority = HPRE_CRYPTO_ALG_PRI, - .cra_name = "ecdh-nist-p384", - .cra_driver_name = "hpre-ecdh-nist-p384", - .cra_module = THIS_MODULE, - }, +static struct kpp_alg ecdh_curves[] = { + { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p192_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p192", + .cra_driver_name = "hpre-ecdh-nist-p192", + .cra_module = THIS_MODULE, + }, + }, { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p256_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "hpre-ecdh-nist-p256", + .cra_module = THIS_MODULE, + }, + }, { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p384_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "hpre-ecdh-nist-p384", + .cra_module = THIS_MODULE, + }, + } }; static struct kpp_alg curve25519_alg = { @@ -2123,46 +2121,38 @@ static void hpre_unregister_dh(struct hisi_qm *qm) static int hpre_register_ecdh(struct hisi_qm *qm) { - int ret; + int ret, i; if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) return 0; - ret = crypto_register_kpp(&ecdh_nist_p192); - if (ret) { - dev_err(&qm->pdev->dev, "failed to register ecdh_nist_p192 (%d)!\n", ret); - return ret; - } - - ret = crypto_register_kpp(&ecdh_nist_p256); - if (ret) { - dev_err(&qm->pdev->dev, "failed to register ecdh_nist_p256 (%d)!\n", ret); - goto unregister_ecdh_p192; - } - - ret = crypto_register_kpp(&ecdh_nist_p384); - if (ret) { - dev_err(&qm->pdev->dev, "failed to register ecdh_nist_p384 (%d)!\n", ret); - goto unregister_ecdh_p256; + for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) { + ret = crypto_register_kpp(&ecdh_curves[i]); + if (ret) { + dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n", + ecdh_curves[i].base.cra_name, ret); + goto unreg_kpp; + } } return 0; -unregister_ecdh_p256: - crypto_unregister_kpp(&ecdh_nist_p256); -unregister_ecdh_p192: - crypto_unregister_kpp(&ecdh_nist_p192); +unreg_kpp: + for (--i; i >= 0; --i) + crypto_unregister_kpp(&ecdh_curves[i]); + return ret; } static void hpre_unregister_ecdh(struct hisi_qm *qm) { + int i; + if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) return; - crypto_unregister_kpp(&ecdh_nist_p384); - crypto_unregister_kpp(&ecdh_nist_p256); - crypto_unregister_kpp(&ecdh_nist_p192); + for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i) + crypto_unregister_kpp(&ecdh_curves[i]); } static int hpre_register_x25519(struct hisi_qm *qm) -- cgit From db700974b69d2c12a8fe84c45820892416a1e265 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 9 Sep 2022 17:47:02 +0800 Subject: crypto: hisilicon/zip - support zip capability Add function 'hisi_zip_alg_support' to get device configuration information from capability registers, instead of determining whether to register an algorithm based on hardware platform's version. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip.h | 1 + drivers/crypto/hisilicon/zip/zip_crypto.c | 67 ++++++++++++++++---- drivers/crypto/hisilicon/zip/zip_main.c | 102 +++++++++++++++++++++--------- 3 files changed, 128 insertions(+), 42 deletions(-) diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index f289656e9ac0..f2e6da3240ae 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -84,4 +84,5 @@ struct hisi_zip_sqe { int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); +bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index a7f6884c3ab3..6608971d10cd 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -39,6 +39,9 @@ #define HZIP_ALG_PRIORITY 300 #define HZIP_SGL_SGE_NR 10 +#define HZIP_ALG_ZLIB GENMASK(1, 0) +#define HZIP_ALG_GZIP GENMASK(3, 2) + static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 @@ -756,6 +759,28 @@ static struct acomp_alg hisi_zip_acomp_zlib = { } }; +static int hisi_zip_register_zlib(struct hisi_qm *qm) +{ + int ret; + + if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) + return 0; + + ret = crypto_register_acomp(&hisi_zip_acomp_zlib); + if (ret) + dev_err(&qm->pdev->dev, "failed to register to zlib (%d)!\n", ret); + + return ret; +} + +static void hisi_zip_unregister_zlib(struct hisi_qm *qm) +{ + if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) + return; + + crypto_unregister_acomp(&hisi_zip_acomp_zlib); +} + static struct acomp_alg hisi_zip_acomp_gzip = { .init = hisi_zip_acomp_init, .exit = hisi_zip_acomp_exit, @@ -770,27 +795,45 @@ static struct acomp_alg hisi_zip_acomp_gzip = { } }; -int hisi_zip_register_to_crypto(struct hisi_qm *qm) +static int hisi_zip_register_gzip(struct hisi_qm *qm) { int ret; - ret = crypto_register_acomp(&hisi_zip_acomp_zlib); - if (ret) { - pr_err("failed to register to zlib (%d)!\n", ret); - return ret; - } + if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) + return 0; ret = crypto_register_acomp(&hisi_zip_acomp_gzip); - if (ret) { - pr_err("failed to register to gzip (%d)!\n", ret); - crypto_unregister_acomp(&hisi_zip_acomp_zlib); - } + if (ret) + dev_err(&qm->pdev->dev, "failed to register to gzip (%d)!\n", ret); return ret; } -void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) +static void hisi_zip_unregister_gzip(struct hisi_qm *qm) { + if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) + return; + crypto_unregister_acomp(&hisi_zip_acomp_gzip); - crypto_unregister_acomp(&hisi_zip_acomp_zlib); +} + +int hisi_zip_register_to_crypto(struct hisi_qm *qm) +{ + int ret = 0; + + ret = hisi_zip_register_zlib(qm); + if (ret) + return ret; + + ret = hisi_zip_register_gzip(qm); + if (ret) + hisi_zip_unregister_zlib(qm); + + return ret; +} + +void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) +{ + hisi_zip_unregister_zlib(qm); + hisi_zip_unregister_gzip(qm); } diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 6ed10ec05a73..a8aedddac67a 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -20,18 +20,6 @@ #define HZIP_QUEUE_NUM_V1 4096 #define HZIP_CLOCK_GATE_CTRL 0x301004 -#define COMP0_ENABLE BIT(0) -#define COMP1_ENABLE BIT(1) -#define DECOMP0_ENABLE BIT(2) -#define DECOMP1_ENABLE BIT(3) -#define DECOMP2_ENABLE BIT(4) -#define DECOMP3_ENABLE BIT(5) -#define DECOMP4_ENABLE BIT(6) -#define DECOMP5_ENABLE BIT(7) -#define HZIP_ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ - DECOMP0_ENABLE | DECOMP1_ENABLE | \ - DECOMP2_ENABLE | DECOMP3_ENABLE | \ - DECOMP4_ENABLE | DECOMP5_ENABLE) #define HZIP_DECOMP_CHECK_ENABLE BIT(16) #define HZIP_FSM_MAX_CNT 0x301008 @@ -76,10 +64,6 @@ #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 #define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0) -#define HZIP_COMP_CORE_NUM 2 -#define HZIP_DECOMP_CORE_NUM 6 -#define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \ - HZIP_DECOMP_CORE_NUM) #define HZIP_SQE_SIZE 128 #define HZIP_PF_DEF_Q_NUM 64 #define HZIP_PF_DEF_Q_BASE 0 @@ -194,6 +178,21 @@ enum zip_cap_type { ZIP_RESET_MASK_CAP, ZIP_OOO_SHUTDOWN_MASK_CAP, ZIP_CE_MASK_CAP, + ZIP_CLUSTER_NUM_CAP, + ZIP_CORE_TYPE_NUM_CAP, + ZIP_CORE_NUM_CAP, + ZIP_CLUSTER_COMP_NUM_CAP, + ZIP_CLUSTER_DECOMP_NUM_CAP, + ZIP_DECOMP_ENABLE_BITMAP, + ZIP_COMP_ENABLE_BITMAP, + ZIP_DRV_ALG_BITMAP, + ZIP_DEV_ALG_BITMAP, + ZIP_CORE1_ALG_BITMAP, + ZIP_CORE2_ALG_BITMAP, + ZIP_CORE3_ALG_BITMAP, + ZIP_CORE4_ALG_BITMAP, + ZIP_CORE5_ALG_BITMAP, + ZIP_CAP_MAX }; static struct hisi_qm_cap_info zip_basic_cap_info[] = { @@ -205,6 +204,21 @@ static struct hisi_qm_cap_info zip_basic_cap_info[] = { {ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE}, {ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE}, {ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, + {ZIP_CLUSTER_NUM_CAP, 0x313C, 28, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {ZIP_CORE_TYPE_NUM_CAP, 0x313C, 24, GENMASK(3, 0), 0x2, 0x2, 0x2}, + {ZIP_CORE_NUM_CAP, 0x313C, 16, GENMASK(7, 0), 0x8, 0x8, 0x5}, + {ZIP_CLUSTER_COMP_NUM_CAP, 0x313C, 8, GENMASK(7, 0), 0x2, 0x2, 0x2}, + {ZIP_CLUSTER_DECOMP_NUM_CAP, 0x313C, 0, GENMASK(7, 0), 0x6, 0x6, 0x3}, + {ZIP_DECOMP_ENABLE_BITMAP, 0x3140, 16, GENMASK(15, 0), 0xFC, 0xFC, 0x1C}, + {ZIP_COMP_ENABLE_BITMAP, 0x3140, 0, GENMASK(15, 0), 0x3, 0x3, 0x3}, + {ZIP_DRV_ALG_BITMAP, 0x3144, 0, GENMASK(31, 0), 0xF, 0xF, 0xF}, + {ZIP_DEV_ALG_BITMAP, 0x3148, 0, GENMASK(31, 0), 0xF, 0xF, 0xFF}, + {ZIP_CORE1_ALG_BITMAP, 0x314C, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5}, + {ZIP_CORE2_ALG_BITMAP, 0x3150, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5}, + {ZIP_CORE3_ALG_BITMAP, 0x3154, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CORE4_ALG_BITMAP, 0x3158, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CORE5_ALG_BITMAP, 0x315C, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0} }; enum { @@ -363,6 +377,17 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); } +bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) +{ + u32 cap_val; + + cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DRV_ALG_BITMAP, qm->cap_ver); + if ((alg & cap_val) == alg) + return true; + + return false; +} + static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) { u32 val; @@ -421,6 +446,7 @@ static void hisi_zip_enable_clock_gate(struct hisi_qm *qm) static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) { void __iomem *base = qm->io_base; + u32 dcomp_bm, comp_bm; /* qm user domain */ writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); @@ -458,8 +484,11 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) } /* let's open all compression/decompression cores */ - writel(HZIP_DECOMP_CHECK_ENABLE | HZIP_ALL_COMP_DECOMP_EN, - base + HZIP_CLOCK_GATE_CTRL); + dcomp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_DECOMP_ENABLE_BITMAP, qm->cap_ver); + comp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_COMP_ENABLE_BITMAP, qm->cap_ver); + writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL); /* enable sqc,cqc writeback */ writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | @@ -678,18 +707,23 @@ DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs); static int hisi_zip_core_debug_init(struct hisi_qm *qm) { + u32 zip_core_num, zip_comp_core_num; struct device *dev = &qm->pdev->dev; struct debugfs_regset32 *regset; struct dentry *tmp_d; char buf[HZIP_BUF_SIZE]; int i; - for (i = 0; i < HZIP_CORE_NUM; i++) { - if (i < HZIP_COMP_CORE_NUM) + zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + zip_comp_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CLUSTER_COMP_NUM_CAP, + qm->cap_ver); + + for (i = 0; i < zip_core_num; i++) { + if (i < zip_comp_core_num) scnprintf(buf, sizeof(buf), "comp_core%d", i); else scnprintf(buf, sizeof(buf), "decomp_core%d", - i - HZIP_COMP_CORE_NUM); + i - zip_comp_core_num); regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); if (!regset) @@ -702,7 +736,7 @@ static int hisi_zip_core_debug_init(struct hisi_qm *qm) tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); debugfs_create_file("regs", 0444, tmp_d, regset, - &hisi_zip_regs_fops); + &hisi_zip_regs_fops); } return 0; @@ -822,10 +856,13 @@ static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs); struct qm_debug *debug = &qm->debug; void __iomem *io_base; + u32 zip_core_num; int i, j, idx; - debug->last_words = kcalloc(core_dfx_regs_num * HZIP_CORE_NUM + - com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); + zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + + debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num, + sizeof(unsigned int), GFP_KERNEL); if (!debug->last_words) return -ENOMEM; @@ -834,7 +871,7 @@ static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) debug->last_words[i] = readl_relaxed(io_base); } - for (i = 0; i < HZIP_CORE_NUM; i++) { + for (i = 0; i < zip_core_num; i++) { io_base = qm->io_base + core_offsets[i]; for (j = 0; j < core_dfx_regs_num; j++) { idx = com_dfx_regs_num + i * core_dfx_regs_num + j; @@ -861,6 +898,7 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) { int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs); int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs); + u32 zip_core_num, zip_comp_core_num; struct qm_debug *debug = &qm->debug; char buf[HZIP_BUF_SIZE]; void __iomem *base; @@ -874,15 +912,18 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) val = readl_relaxed(qm->io_base + hzip_com_dfx_regs[i].offset); if (debug->last_words[i] != val) pci_info(qm->pdev, "com_dfx: %s \t= 0x%08x => 0x%08x\n", - hzip_com_dfx_regs[i].name, debug->last_words[i], val); + hzip_com_dfx_regs[i].name, debug->last_words[i], val); } - for (i = 0; i < HZIP_CORE_NUM; i++) { - if (i < HZIP_COMP_CORE_NUM) + zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + zip_comp_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CLUSTER_COMP_NUM_CAP, + qm->cap_ver); + for (i = 0; i < zip_core_num; i++) { + if (i < zip_comp_core_num) scnprintf(buf, sizeof(buf), "Comp_core-%d", i); else scnprintf(buf, sizeof(buf), "Decomp_core-%d", - i - HZIP_COMP_CORE_NUM); + i - zip_comp_core_num); base = qm->io_base + core_offsets[i]; pci_info(qm->pdev, "==>%s:\n", buf); @@ -892,7 +933,8 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) val = readl_relaxed(base + hzip_dump_dfx_regs[j].offset); if (debug->last_words[idx] != val) pci_info(qm->pdev, "%s \t= 0x%08x => 0x%08x\n", - hzip_dump_dfx_regs[j].name, debug->last_words[idx], val); + hzip_dump_dfx_regs[j].name, + debug->last_words[idx], val); } } } -- cgit From 921715b6b7827157bba6e8153d7a09774b0d034f Mon Sep 17 00:00:00 2001 From: Wenkai Lin Date: Fri, 9 Sep 2022 17:47:03 +0800 Subject: crypto: hisilicon/sec - get algorithm bitmap from registers Add function 'sec_get_alg_bitmap' to get hardware algorithm bitmap before register algorithm to crypto, instead of determining whether to register an algorithm based on hardware platform's version. Signed-off-by: Wenkai Lin Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec.h | 18 ++ drivers/crypto/hisilicon/sec2/sec_crypto.c | 306 ++++++++++++++++++----------- drivers/crypto/hisilicon/sec2/sec_main.c | 33 +++- 3 files changed, 236 insertions(+), 121 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 895ba9b47554..3e57fc04b377 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -201,10 +201,28 @@ enum sec_cap_type { SEC_RESET_MASK_CAP, SEC_OOO_SHUTDOWN_MASK_CAP, SEC_CE_MASK_CAP, + SEC_CLUSTER_NUM_CAP, + SEC_CORE_TYPE_NUM_CAP, + SEC_CORE_NUM_CAP, + SEC_CORES_PER_CLUSTER_NUM_CAP, + SEC_CORE_ENABLE_BITMAP, + SEC_DRV_ALG_BITMAP_LOW, + SEC_DRV_ALG_BITMAP_HIGH, + SEC_DEV_ALG_BITMAP_LOW, + SEC_DEV_ALG_BITMAP_HIGH, + SEC_CORE1_ALG_BITMAP_LOW, + SEC_CORE1_ALG_BITMAP_HIGH, + SEC_CORE2_ALG_BITMAP_LOW, + SEC_CORE2_ALG_BITMAP_HIGH, + SEC_CORE3_ALG_BITMAP_LOW, + SEC_CORE3_ALG_BITMAP_HIGH, + SEC_CORE4_ALG_BITMAP_LOW, + SEC_CORE4_ALG_BITMAP_HIGH, }; void sec_destroy_qps(struct hisi_qp **qps, int qp_num); struct hisi_qp **sec_create_qps(void); int sec_register_to_crypto(struct hisi_qm *qm); void sec_unregister_from_crypto(struct hisi_qm *qm); +u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index eb23bffdcac8..84ae8ddd1a13 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -104,6 +104,16 @@ #define IV_CTR_INIT 0x1 #define IV_BYTE_OFFSET 0x8 +struct sec_skcipher { + u64 alg_msk; + struct skcipher_alg alg; +}; + +struct sec_aead { + u64 alg_msk; + struct aead_alg alg; +}; + /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) { @@ -2158,67 +2168,80 @@ static int sec_skcipher_decrypt(struct skcipher_request *sk_req) .min_keysize = sec_min_key_size,\ .max_keysize = sec_max_key_size,\ .ivsize = iv_size,\ -}, +} #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ max_key_size, blk_size, iv_size) \ SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) -static struct skcipher_alg sec_skciphers[] = { - SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - AES_BLOCK_SIZE, 0) - - SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, - SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, - SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, - DES3_EDE_BLOCK_SIZE, 0) - - SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, - SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, - DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, - SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) -}; - -static struct skcipher_alg sec_skciphers_v3[] = { - SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) +static struct sec_skcipher sec_skciphers[] = { + { + .alg_msk = BIT(0), + .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0), + }, + { + .alg_msk = BIT(1), + .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(2), + .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(3), + .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE, + SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(4), + .alg = SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(5), + .alg = SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(12), + .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(13), + .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(14), + .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE, + SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(15), + .alg = SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(16), + .alg = SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(23), + .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), + }, + { + .alg_msk = BIT(24), + .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, + DES3_EDE_BLOCK_SIZE), + }, }; static int aead_iv_demension_check(struct aead_request *aead_req) @@ -2412,90 +2435,135 @@ static int sec_aead_decrypt(struct aead_request *a_req) .maxauthsize = max_authsize,\ } -static struct aead_alg sec_aeads[] = { - SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", - sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, - sec_aead_ctx_exit, AES_BLOCK_SIZE, - AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), +static struct sec_aead sec_aeads[] = { + { + .alg_msk = BIT(6), + .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(7), + .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(17), + .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(18), + .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(43), + .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, + sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), + }, + { + .alg_msk = BIT(44), + .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, + sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), + }, + { + .alg_msk = BIT(45), + .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, + sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + }, +}; + +static void sec_unregister_skcipher(u64 alg_mask, int end) +{ + int i; - SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", - sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, - sec_aead_ctx_exit, AES_BLOCK_SIZE, - AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), + for (i = 0; i < end; i++) + if (sec_skciphers[i].alg_msk & alg_mask) + crypto_unregister_skcipher(&sec_skciphers[i].alg); +} - SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", - sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, - sec_aead_ctx_exit, AES_BLOCK_SIZE, - AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), +static int sec_register_skcipher(u64 alg_mask) +{ + int i, ret, count; - SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - AES_BLOCK_SIZE, AES_BLOCK_SIZE), + count = ARRAY_SIZE(sec_skciphers); - SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - SEC_AIV_SIZE, AES_BLOCK_SIZE) -}; + for (i = 0; i < count; i++) { + if (!(sec_skciphers[i].alg_msk & alg_mask)) + continue; -static struct aead_alg sec_aeads_v3[] = { - SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - AES_BLOCK_SIZE, AES_BLOCK_SIZE), + ret = crypto_register_skcipher(&sec_skciphers[i].alg); + if (ret) + goto err; + } - SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - SEC_AIV_SIZE, AES_BLOCK_SIZE) -}; + return 0; + +err: + sec_unregister_skcipher(alg_mask, i); + + return ret; +} + +static void sec_unregister_aead(u64 alg_mask, int end) +{ + int i; + + for (i = 0; i < end; i++) + if (sec_aeads[i].alg_msk & alg_mask) + crypto_unregister_aead(&sec_aeads[i].alg); +} + +static int sec_register_aead(u64 alg_mask) +{ + int i, ret, count; + + count = ARRAY_SIZE(sec_aeads); + + for (i = 0; i < count; i++) { + if (!(sec_aeads[i].alg_msk & alg_mask)) + continue; + + ret = crypto_register_aead(&sec_aeads[i].alg); + if (ret) + goto err; + } + + return 0; + +err: + sec_unregister_aead(alg_mask, i); + + return ret; +} int sec_register_to_crypto(struct hisi_qm *qm) { + u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); int ret; - /* To avoid repeat register */ - ret = crypto_register_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); + ret = sec_register_skcipher(alg_mask); if (ret) return ret; - if (qm->ver > QM_HW_V2) { - ret = crypto_register_skciphers(sec_skciphers_v3, - ARRAY_SIZE(sec_skciphers_v3)); - if (ret) - goto reg_skcipher_fail; - } - - ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + ret = sec_register_aead(alg_mask); if (ret) - goto reg_aead_fail; - if (qm->ver > QM_HW_V2) { - ret = crypto_register_aeads(sec_aeads_v3, ARRAY_SIZE(sec_aeads_v3)); - if (ret) - goto reg_aead_v3_fail; - } - return ret; + sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); -reg_aead_v3_fail: - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); -reg_aead_fail: - if (qm->ver > QM_HW_V2) - crypto_unregister_skciphers(sec_skciphers_v3, - ARRAY_SIZE(sec_skciphers_v3)); -reg_skcipher_fail: - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); return ret; } void sec_unregister_from_crypto(struct hisi_qm *qm) { - if (qm->ver > QM_HW_V2) - crypto_unregister_aeads(sec_aeads_v3, - ARRAY_SIZE(sec_aeads_v3)); - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); - if (qm->ver > QM_HW_V2) - crypto_unregister_skciphers(sec_skciphers_v3, - ARRAY_SIZE(sec_skciphers_v3)); - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); + sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads)); + sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index e814f94dd0d4..84233a489c74 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -41,7 +41,6 @@ #define SEC_ECC_NUM 16 #define SEC_ECC_MASH 0xFF #define SEC_CORE_INT_DISABLE 0x0 -#define SEC_SAA_ENABLE 0x17f #define SEC_RAS_CE_REG 0x301050 #define SEC_RAS_FE_REG 0x301054 @@ -114,6 +113,8 @@ #define SEC_DFX_COMMON1_LEN 0x45 #define SEC_DFX_COMMON2_LEN 0xBA +#define SEC_ALG_BITMAP_SHIFT 32 + struct sec_hw_error { u32 int_msk; const char *msg; @@ -141,6 +142,23 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177}, {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177}, {SEC_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088}, + {SEC_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4}, + {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4}, + {SEC_CORE_ENABLE_BITMAP, 0x3140, 32, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, + {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x187F0FF}, + {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C}, + {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE1_ALG_BITMAP_LOW, 0x3154, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE1_ALG_BITMAP_HIGH, 0x3158, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE2_ALG_BITMAP_LOW, 0x315c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE2_ALG_BITMAP_HIGH, 0x3160, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE3_ALG_BITMAP_LOW, 0x3164, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE3_ALG_BITMAP_HIGH, 0x3168, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE4_ALG_BITMAP_LOW, 0x316c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, }; static const struct sec_hw_error sec_hw_errors[] = { @@ -345,6 +363,16 @@ struct hisi_qp **sec_create_qps(void) return NULL; } +u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) +{ + u32 cap_val_h, cap_val_l; + + cap_val_h = hisi_qm_get_hw_info(qm, sec_basic_info, high, qm->cap_ver); + cap_val_l = hisi_qm_get_hw_info(qm, sec_basic_info, low, qm->cap_ver); + + return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l; +} + static const struct kernel_param_ops sec_uacce_mode_ops = { .set = uacce_mode_set, .get = param_get_int, @@ -512,7 +540,8 @@ static int sec_engine_init(struct hisi_qm *qm) writel(SEC_SINGLE_PORT_MAX_TRANS, qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS); - writel(SEC_SAA_ENABLE, qm->io_base + SEC_SAA_EN_REG); + reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver); + writel(reg, qm->io_base + SEC_SAA_EN_REG); if (qm->ver < QM_HW_V3) { /* HW V2 enable sm4 extra mode, as ctr/ecb */ -- cgit From d310dc2554a5296a338f974d2b4e4f9af2687558 Mon Sep 17 00:00:00 2001 From: Zhiqi Song Date: Fri, 9 Sep 2022 17:47:04 +0800 Subject: crypto: hisilicon - support get algs by the capability register The value of qm algorithm can change dynamically according to the value of the capability register. Add xxx_set_qm_algs() function to obtain the algs that the hardware device supported from the capability register and set them into usr mode attribute files. Signed-off-by: Zhiqi Song Signed-off-by: Wenkai Lin Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 10 ++-- drivers/crypto/hisilicon/hpre/hpre_main.c | 83 +++++++++++++++++++++++++++-- drivers/crypto/hisilicon/qm.c | 1 - drivers/crypto/hisilicon/sec2/sec_main.c | 71 +++++++++++++++++++++++- drivers/crypto/hisilicon/zip/zip_main.c | 75 ++++++++++++++++++++++++-- 5 files changed, 222 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index ac7fabf65865..ef02dadd6217 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -51,11 +51,11 @@ struct hpre_ctx; #define HPRE_ECC_HW256_KSZ_B 32 #define HPRE_ECC_HW384_KSZ_B 48 -/* capability register mask */ -#define HPRE_DRV_RSA_MASK_CAP BIT(0) -#define HPRE_DRV_DH_MASK_CAP BIT(1) -#define HPRE_DRV_ECDH_MASK_CAP BIT(2) -#define HPRE_DRV_X25519_MASK_CAP BIT(5) +/* capability register mask of driver */ +#define HPRE_DRV_RSA_MASK_CAP BIT(0) +#define HPRE_DRV_DH_MASK_CAP BIT(1) +#define HPRE_DRV_ECDH_MASK_CAP BIT(2) +#define HPRE_DRV_X25519_MASK_CAP BIT(5) typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 407cdd9d8413..471e5ca720f5 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -118,6 +118,8 @@ #define HPRE_DFX_COMMON2_LEN 0xE #define HPRE_DFX_CORE_LEN 0x43 +#define HPRE_DEV_ALG_MAX_LEN 256 + static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -133,6 +135,38 @@ struct hpre_hw_error { const char *msg; }; +struct hpre_dev_alg { + u32 alg_msk; + const char *alg; +}; + +static const struct hpre_dev_alg hpre_dev_algs[] = { + { + .alg_msk = BIT(0), + .alg = "rsa\n" + }, { + .alg_msk = BIT(1), + .alg = "dh\n" + }, { + .alg_msk = BIT(2), + .alg = "ecdh\n" + }, { + .alg_msk = BIT(3), + .alg = "ecdsa\n" + }, { + .alg_msk = BIT(4), + .alg = "sm2\n" + }, { + .alg_msk = BIT(5), + .alg = "x25519\n" + }, { + .alg_msk = BIT(6), + .alg = "x448\n" + }, { + /* sentinel */ + } +}; + static struct hisi_qm_list hpre_devices = { .register_to_crypto = hpre_algs_register, .unregister_from_crypto = hpre_algs_unregister, @@ -325,6 +359,35 @@ bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) return false; } +static int hpre_set_qm_algs(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + u32 alg_msk; + int i; + + if (!qm->use_sva) + return 0; + + algs = devm_kzalloc(dev, HPRE_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + alg_msk = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DEV_ALG_BITMAP_CAP, qm->cap_ver); + + for (i = 0; i < ARRAY_SIZE(hpre_dev_algs); i++) + if (alg_msk & hpre_dev_algs[i].alg_msk) + strcat(algs, hpre_dev_algs[i].alg); + + ptr = strrchr(algs, '\n'); + if (ptr) + *ptr = '\0'; + + qm->uacce->algs = algs; + + return 0; +} + static int hpre_diff_regs_show(struct seq_file *s, void *unused) { struct hisi_qm *qm = s->private; @@ -1073,15 +1136,13 @@ static void hpre_debugfs_exit(struct hisi_qm *qm) static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + if (pdev->revision == QM_HW_V1) { pci_warn(pdev, "HPRE version 1 is not supported!\n"); return -EINVAL; } - if (pdev->revision >= QM_HW_V3) - qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2"; - else - qm->algs = "rsa\ndh"; qm->mode = uacce_mode; qm->pdev = pdev; qm->ver = pdev->revision; @@ -1097,7 +1158,19 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qm_list = &hpre_devices; } - return hisi_qm_init(qm); + ret = hisi_qm_init(qm); + if (ret) { + pci_err(pdev, "Failed to init hpre qm configures!\n"); + return ret; + } + + ret = hpre_set_qm_algs(qm); + if (ret) { + pci_err(pdev, "Failed to set hpre algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; } static int hpre_show_last_regs_init(struct hisi_qm *qm) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index e4a506d02d23..30fdf0673f00 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3484,7 +3484,6 @@ static int qm_alloc_uacce(struct hisi_qm *qm) uacce->is_vf = pdev->is_virtfn; uacce->priv = qm; - uacce->algs = qm->algs; if (qm->ver == QM_HW_V1) uacce->api_ver = HISI_QM_API_VER_BASE; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 84233a489c74..3705412bac5f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -115,6 +115,14 @@ #define SEC_ALG_BITMAP_SHIFT 32 +#define SEC_CIPHER_BITMAP (GENMASK_ULL(5, 0) | GENMASK_ULL(16, 12) | \ + GENMASK(24, 21)) +#define SEC_DIGEST_BITMAP (GENMASK_ULL(11, 8) | GENMASK_ULL(20, 19) | \ + GENMASK_ULL(42, 25)) +#define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \ + GENMASK_ULL(45, 43)) +#define SEC_DEV_ALG_MAX_LEN 256 + struct sec_hw_error { u32 int_msk; const char *msg; @@ -125,6 +133,11 @@ struct sec_dfx_item { u32 offset; }; +struct sec_dev_alg { + u64 alg_msk; + const char *algs; +}; + static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; @@ -161,6 +174,18 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, }; +static const struct sec_dev_alg sec_dev_algs[] = { { + .alg_msk = SEC_CIPHER_BITMAP, + .algs = "cipher\n", + }, { + .alg_msk = SEC_DIGEST_BITMAP, + .algs = "digest\n", + }, { + .alg_msk = SEC_AEAD_BITMAP, + .algs = "aead\n", + }, +}; + static const struct sec_hw_error sec_hw_errors[] = { { .int_msk = BIT(0), @@ -1052,11 +1077,41 @@ static int sec_pf_probe_init(struct sec_dev *sec) return ret; } +static int sec_set_qm_algs(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + u64 alg_mask; + int i; + + if (!qm->use_sva) + return 0; + + algs = devm_kzalloc(dev, SEC_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + alg_mask = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH, SEC_DEV_ALG_BITMAP_LOW); + + for (i = 0; i < ARRAY_SIZE(sec_dev_algs); i++) + if (alg_mask & sec_dev_algs[i].alg_msk) + strcat(algs, sec_dev_algs[i].algs); + + ptr = strrchr(algs, '\n'); + if (ptr) + *ptr = '\0'; + + qm->uacce->algs = algs; + + return 0; +} + static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + qm->pdev = pdev; qm->ver = pdev->revision; - qm->algs = "cipher\ndigest\naead"; qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; @@ -1079,7 +1134,19 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; } - return hisi_qm_init(qm); + ret = hisi_qm_init(qm); + if (ret) { + pci_err(qm->pdev, "Failed to init sec qm configures!\n"); + return ret; + } + + ret = sec_set_qm_algs(qm); + if (ret) { + pci_err(qm->pdev, "Failed to set sec algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; } static void sec_qm_uninit(struct hisi_qm *qm) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index a8aedddac67a..c863435e8c75 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -74,6 +74,12 @@ #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) #define HZIP_WR_PORT BIT(11) +#define HZIP_DEV_ALG_MAX_LEN 256 +#define HZIP_ALG_ZLIB_BIT GENMASK(1, 0) +#define HZIP_ALG_GZIP_BIT GENMASK(3, 2) +#define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4) +#define HZIP_ALG_LZ77_BIT GENMASK(7, 6) + #define HZIP_BUF_SIZE 22 #define HZIP_SQE_MASK_OFFSET 64 #define HZIP_SQE_MASK_LEN 48 @@ -114,6 +120,26 @@ struct zip_dfx_item { u32 offset; }; +struct zip_dev_alg { + u32 alg_msk; + const char *algs; +}; + +static const struct zip_dev_alg zip_dev_algs[] = { { + .alg_msk = HZIP_ALG_ZLIB_BIT, + .algs = "zlib\n", + }, { + .alg_msk = HZIP_ALG_GZIP_BIT, + .algs = "gzip\n", + }, { + .alg_msk = HZIP_ALG_DEFLATE_BIT, + .algs = "deflate\n", + }, { + .alg_msk = HZIP_ALG_LZ77_BIT, + .algs = "lz77_zstd\n", + }, +}; + static struct hisi_qm_list zip_devices = { .register_to_crypto = hisi_zip_register_to_crypto, .unregister_from_crypto = hisi_zip_unregister_from_crypto, @@ -388,6 +414,35 @@ bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) return false; } +static int hisi_zip_set_qm_algs(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + u32 alg_mask; + int i; + + if (!qm->use_sva) + return 0; + + algs = devm_kzalloc(dev, HZIP_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + alg_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DEV_ALG_BITMAP, qm->cap_ver); + + for (i = 0; i < ARRAY_SIZE(zip_dev_algs); i++) + if (alg_mask & zip_dev_algs[i].alg_msk) + strcat(algs, zip_dev_algs[i].algs); + + ptr = strrchr(algs, '\n'); + if (ptr) + *ptr = '\0'; + + qm->uacce->algs = algs; + + return 0; +} + static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) { u32 val; @@ -1071,12 +1126,10 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + qm->pdev = pdev; qm->ver = pdev->revision; - if (pdev->revision >= QM_HW_V3) - qm->algs = "zlib\ngzip\ndeflate\nlz77_zstd"; - else - qm->algs = "zlib\ngzip"; qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; @@ -1100,7 +1153,19 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; } - return hisi_qm_init(qm); + ret = hisi_qm_init(qm); + if (ret) { + pci_err(qm->pdev, "Failed to init zip qm configures!\n"); + return ret; + } + + ret = hisi_zip_set_qm_algs(qm); + if (ret) { + pci_err(qm->pdev, "Failed to set zip algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; } static void hisi_zip_qm_uninit(struct hisi_qm *qm) -- cgit From cf5bb835b7c8a5fee7f26455099cca7feb57f5e9 Mon Sep 17 00:00:00 2001 From: Damian Muszynski Date: Fri, 9 Sep 2022 11:49:12 +0100 Subject: crypto: qat - fix DMA transfer direction When CONFIG_DMA_API_DEBUG is selected, while running the crypto self test on the QAT crypto algorithms, the function add_dma_entry() reports a warning similar to the one below, saying that overlapping mappings are not supported. This occurs in tests where the input and the output scatter list point to the same buffers (i.e. two different scatter lists which point to the same chunks of memory). The logic that implements the mapping uses the flag DMA_BIDIRECTIONAL for both the input and the output scatter lists which leads to overlapped write mappings. These are not supported by the DMA layer. Fix by specifying the correct DMA transfer directions when mapping buffers. For in-place operations where the input scatter list matches the output scatter list, buffers are mapped once with DMA_BIDIRECTIONAL, otherwise input buffers are mapped using the flag DMA_TO_DEVICE and output buffers are mapped with DMA_FROM_DEVICE. Overlapping a read mapping with a write mapping is a valid case in dma-coherent devices like QAT. The function that frees and unmaps the buffers, qat_alg_free_bufl() has been changed accordingly to the changes to the mapping function. DMA-API: 4xxx 0000:06:00.0: cacheline tracking EEXIST, overlapping mappings aren't supported WARNING: CPU: 53 PID: 4362 at kernel/dma/debug.c:570 add_dma_entry+0x1e9/0x270 ... Call Trace: dma_map_page_attrs+0x82/0x2d0 ? preempt_count_add+0x6a/0xa0 qat_alg_sgl_to_bufl+0x45b/0x990 [intel_qat] qat_alg_aead_dec+0x71/0x250 [intel_qat] crypto_aead_decrypt+0x3d/0x70 test_aead_vec_cfg+0x649/0x810 ? number+0x310/0x3a0 ? vsnprintf+0x2a3/0x550 ? scnprintf+0x42/0x70 ? valid_sg_divisions.constprop.0+0x86/0xa0 ? test_aead_vec+0xdf/0x120 test_aead_vec+0xdf/0x120 alg_test_aead+0x185/0x400 alg_test+0x3d8/0x500 ? crypto_acomp_scomp_free_ctx+0x30/0x30 ? __schedule+0x32a/0x12a0 ? ttwu_queue_wakelist+0xbf/0x110 ? _raw_spin_unlock_irqrestore+0x23/0x40 ? try_to_wake_up+0x83/0x570 ? _raw_spin_unlock_irqrestore+0x23/0x40 ? __set_cpus_allowed_ptr_locked+0xea/0x1b0 ? crypto_acomp_scomp_free_ctx+0x30/0x30 cryptomgr_test+0x27/0x50 kthread+0xe6/0x110 ? kthread_complete_and_exit+0x20/0x20 ret_from_fork+0x1f/0x30 Fixes: d370cec ("crypto: qat - Intel(R) QAT crypto interface") Link: https://lore.kernel.org/linux-crypto/20220223080400.139367-1-gilad@benyossef.com/ Signed-off-by: Damian Muszynski Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index fb45fa83841c..cad9c58caab1 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -673,11 +673,14 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, dma_addr_t blpout = qat_req->buf.bloutp; size_t sz = qat_req->buf.sz; size_t sz_out = qat_req->buf.sz_out; + int bl_dma_dir; int i; + bl_dma_dir = blp != blpout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + for (i = 0; i < bl->num_bufs; i++) dma_unmap_single(dev, bl->bufers[i].addr, - bl->bufers[i].len, DMA_BIDIRECTIONAL); + bl->bufers[i].len, bl_dma_dir); dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); @@ -691,7 +694,7 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, for (i = bufless; i < blout->num_bufs; i++) { dma_unmap_single(dev, blout->bufers[i].addr, blout->bufers[i].len, - DMA_BIDIRECTIONAL); + DMA_FROM_DEVICE); } dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); @@ -716,6 +719,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, struct scatterlist *sg; size_t sz_out, sz = struct_size(bufl, bufers, n); int node = dev_to_node(&GET_DEV(inst->accel_dev)); + int bufl_dma_dir; if (unlikely(!n)) return -EINVAL; @@ -733,6 +737,8 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, qat_req->buf.sgl_src_valid = true; } + bufl_dma_dir = sgl != sglout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + for_each_sg(sgl, sg, n, i) bufl->bufers[i].addr = DMA_MAPPING_ERROR; @@ -744,7 +750,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, - DMA_BIDIRECTIONAL); + bufl_dma_dir); bufl->bufers[y].len = sg->length; if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr))) goto err_in; @@ -787,7 +793,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, - DMA_BIDIRECTIONAL); + DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(dev, bufers[y].addr))) goto err_out; bufers[y].len = sg->length; @@ -817,7 +823,7 @@ err_out: if (!dma_mapping_error(dev, buflout->bufers[i].addr)) dma_unmap_single(dev, buflout->bufers[i].addr, buflout->bufers[i].len, - DMA_BIDIRECTIONAL); + DMA_FROM_DEVICE); if (!qat_req->buf.sgl_dst_valid) kfree(buflout); @@ -831,7 +837,7 @@ err_in: if (!dma_mapping_error(dev, bufl->bufers[i].addr)) dma_unmap_single(dev, bufl->bufers[i].addr, bufl->bufers[i].len, - DMA_BIDIRECTIONAL); + bufl_dma_dir); if (!qat_req->buf.sgl_src_valid) kfree(bufl); -- cgit From 9c5f21b198d259bfe1191b1fedf08e2eab15b33b Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 9 Sep 2022 11:49:13 +0100 Subject: Revert "crypto: qat - reduce size of mapped region" This reverts commit e48767c17718067ba21fb2ef461779ec2506f845. In an attempt to resolve a set of warnings reported by the static analyzer Smatch, the reverted commit improperly reduced the sizes of the DMA mappings used for the input and output parameters for both RSA and DH creating a mismatch (map size=8 bytes, unmap size=64 bytes). This issue is reported when CONFIG_DMA_API_DEBUG is selected, when the crypto self test is run. The function dma_unmap_single() reports a warning similar to the one below, saying that the `device driver frees DMA memory with different size`. DMA-API: 4xxx 0000:06:00.0: device driver frees DMA memory with different size [device address=0x0000000123206c80] [map size=8 bytes] [unmap size=64 bytes] WARNING: CPU: 0 PID: 0 at kernel/dma/debug.c:973 check_unmap+0x3d0/0x8c0\ ... Call Trace: debug_dma_unmap_page+0x5c/0x60 qat_dh_cb+0xd7/0x110 [intel_qat] qat_alg_asym_callback+0x1a/0x30 [intel_qat] adf_response_handler+0xbd/0x1a0 [intel_qat] tasklet_action_common.constprop.0+0xcd/0xe0 __do_softirq+0xf8/0x30c __irq_exit_rcu+0xbf/0x140 common_interrupt+0xb9/0xd0 The original commit was correct. Cc: Reported-by: Herbert Xu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 095ed2a404d2..85b0f30712e1 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -333,13 +333,13 @@ static int qat_dh_compute_value(struct kpp_request *req) qat_req->out.dh.out_tab[1] = 0; /* Mapping in.in.b or in.in_g2.xa is the same */ qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh.in.b, - sizeof(qat_req->in.dh.in.b), + sizeof(struct qat_dh_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh.r, - sizeof(qat_req->out.dh.r), + sizeof(struct qat_dh_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -730,13 +730,13 @@ static int qat_rsa_enc(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m, - sizeof(qat_req->in.rsa.enc.m), + sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.enc.c, - sizeof(qat_req->out.rsa.enc.c), + sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -876,13 +876,13 @@ static int qat_rsa_dec(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.dec.c, - sizeof(qat_req->in.rsa.dec.c), + sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.dec.m, - sizeof(qat_req->out.rsa.dec.m), + sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; -- cgit From 072d36eefd6fde17928d214df64fdac32f60b4f4 Mon Sep 17 00:00:00 2001 From: Damian Muszynski Date: Fri, 9 Sep 2022 11:49:14 +0100 Subject: crypto: qat - use reference to structure in dma_map_single() When mapping the input and output parameters, the implementations of RSA and DH pass to the function dma_map_single() a pointer to the first member of the structure they want to map instead of a pointer to the actual structure. This results in set of warnings reported by the static analyser Smatch: drivers/crypto/qat/qat_common/qat_asym_algs.c:335 qat_dh_compute_value() error: dma_map_single_attrs() '&qat_req->in.dh.in.b' too small (8 vs 64) drivers/crypto/qat/qat_common/qat_asym_algs.c:341 qat_dh_compute_value() error: dma_map_single_attrs() '&qat_req->out.dh.r' too small (8 vs 64) drivers/crypto/qat/qat_common/qat_asym_algs.c:732 qat_rsa_enc() error: dma_map_single_attrs() '&qat_req->in.rsa.enc.m' too small (8 vs 64) drivers/crypto/qat/qat_common/qat_asym_algs.c:738 qat_rsa_enc() error: dma_map_single_attrs() '&qat_req->out.rsa.enc.c' too small (8 vs 64) drivers/crypto/qat/qat_common/qat_asym_algs.c:878 qat_rsa_dec() error: dma_map_single_attrs() '&qat_req->in.rsa.dec.c' too small (8 vs 64) drivers/crypto/qat/qat_common/qat_asym_algs.c:884 qat_rsa_dec() error: dma_map_single_attrs() '&qat_req->out.rsa.dec.m' too small (8 vs 64) Where the address of the first element of a structure is used as an input for the function dma_map_single(), replace it with the address of the structure. This fix does not introduce any functional change as the addresses are the same. Signed-off-by: Damian Muszynski Signed-off-by: Giovanni Cabiddu Reviewed-by: Adam Guerin Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 85b0f30712e1..94a26702aeae 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -332,13 +332,13 @@ static int qat_dh_compute_value(struct kpp_request *req) qat_req->in.dh.in_tab[n_input_params] = 0; qat_req->out.dh.out_tab[1] = 0; /* Mapping in.in.b or in.in_g2.xa is the same */ - qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh.in.b, + qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh, sizeof(struct qat_dh_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh.r, + qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh, sizeof(struct qat_dh_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) @@ -729,13 +729,13 @@ static int qat_rsa_enc(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m, + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.enc.c, + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) @@ -875,13 +875,13 @@ static int qat_rsa_dec(struct akcipher_request *req) else qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.dec.c, + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.dec.m, + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) -- cgit From 49186a7d9e46ff132a0ed9b721ad6b6a58dba6c1 Mon Sep 17 00:00:00 2001 From: Peter Harliman Liem Date: Tue, 13 Sep 2022 16:03:47 +0800 Subject: crypto: inside_secure - Avoid dma map if size is zero From commit d03c54419274 ("dma-mapping: disallow .map_sg operations from returning zero on error"), dma_map_sg() produces warning if size is 0. This results in visible warnings if crypto length is zero. To avoid that, we avoid calling dma_map_sg if size is zero. Signed-off-by: Peter Harliman Liem Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 44 ++++++++++++++++++-------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index d68ef16650d4..5a222c228c3b 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -642,10 +642,16 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin safexcel_complete(priv, ring); if (src == dst) { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_TO_DEVICE); + if (sreq->nr_dst > 0) + dma_unmap_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); } /* @@ -737,23 +743,29 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, max(totlen_src, totlen_dst)); return -EINVAL; } - dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); + if (sreq->nr_src > 0) + dma_map_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); } else { if (unlikely(totlen_src && (sreq->nr_src <= 0))) { dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", totlen_src); return -EINVAL; } - dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); + + if (sreq->nr_src > 0) + dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", totlen_dst); - dma_unmap_sg(priv->dev, src, sreq->nr_src, - DMA_TO_DEVICE); - return -EINVAL; + ret = -EINVAL; + goto unmap; } - dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); + + if (sreq->nr_dst > 0) + dma_map_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); } memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); @@ -883,12 +895,18 @@ rdesc_rollback: cdesc_rollback: for (i = 0; i < n_cdesc; i++) safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); - +unmap: if (src == dst) { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_TO_DEVICE); + if (sreq->nr_dst > 0) + dma_unmap_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); } return ret; -- cgit From 320406cb60b6408d87f6a5bc729285fc44107352 Mon Sep 17 00:00:00 2001 From: Peter Harliman Liem Date: Tue, 13 Sep 2022 16:03:48 +0800 Subject: crypto: inside-secure - Replace generic aes with libaes Commit 363a90c2d517 ("crypto: safexcel/aes - switch to library version of key expansion routine") removed CRYPTO_AES in the config. However, some portions of codes still rely on generic AES cipher (e.g. refer to safexcel_aead_gcm_cra_init(), safexcel_xcbcmac_cra_init()). This causes transform allocation failure for those algos, if CRYPTO_AES is not manually enabled. To resolve that, we replace all existing AES cipher dependent codes with their AES library counterpart. Fixes: 363a90c2d517 ("crypto: safexcel/aes - switch to library version of key expansion routine") Signed-off-by: Peter Harliman Liem Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 16 +------ drivers/crypto/inside-secure/safexcel_hash.c | 59 +++++++++----------------- 2 files changed, 21 insertions(+), 54 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 5a222c228c3b..32a37e3850c5 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -63,7 +63,6 @@ struct safexcel_cipher_ctx { u32 hash_alg; u32 state_sz; - struct crypto_cipher *hkaes; struct crypto_aead *fback; }; @@ -2607,15 +2606,8 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, ctx->key_len = len; /* Compute hash key by encrypting zeroes with cipher key */ - crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->hkaes, key, len); - if (ret) - return ret; - memset(hashkey, 0, AES_BLOCK_SIZE); - crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey); + aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey); if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { @@ -2644,15 +2636,11 @@ static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) ctx->xcm = EIP197_XCM_MODE_GCM; ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ - ctx->hkaes = crypto_alloc_cipher("aes", 0, 0); - return PTR_ERR_OR_ZERO(ctx->hkaes); + return 0; } static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) { - struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_cipher(ctx->hkaes); safexcel_aead_cra_exit(tfm); } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 2124416742f8..103fc551d2af 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -30,7 +30,7 @@ struct safexcel_ahash_ctx { bool fb_init_done; bool fb_do_setkey; - struct crypto_cipher *kaes; + struct crypto_aes_ctx *aes; struct crypto_ahash *fback; struct crypto_shash *shpre; struct shash_desc *shdesc; @@ -824,7 +824,7 @@ static int safexcel_ahash_final(struct ahash_request *areq) result[i] = swab32(ctx->base.ipad.word[i + 4]); } areq->result[0] ^= 0x80; // 10- padding - crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result); + aes_encrypt(ctx->aes, areq->result, areq->result); return 0; } else if (unlikely(req->hmac && (req->len == req->block_sz) && @@ -2083,37 +2083,26 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int len) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct crypto_aes_ctx aes; u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)]; int ret, i; - ret = aes_expandkey(&aes, key, len); + ret = aes_expandkey(ctx->aes, key, len); if (ret) return ret; /* precompute the XCBC key material */ - crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->kaes, key, len); - if (ret) - return ret; - - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, - "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp, - "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); + aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); + aes_encrypt(ctx->aes, (u8 *)key_tmp, + "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); + aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) ctx->base.ipad.word[i] = swab32(key_tmp[i]); - crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->kaes, - (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, - AES_MIN_KEY_SIZE); + ret = aes_expandkey(ctx->aes, + (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + AES_MIN_KEY_SIZE); if (ret) return ret; @@ -2121,7 +2110,6 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; ctx->cbcmac = false; - memzero_explicit(&aes, sizeof(aes)); return 0; } @@ -2130,15 +2118,15 @@ static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); safexcel_ahash_cra_init(tfm); - ctx->kaes = crypto_alloc_cipher("aes", 0, 0); - return PTR_ERR_OR_ZERO(ctx->kaes); + ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL); + return PTR_ERR_OR_ZERO(ctx->aes); } static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_cipher(ctx->kaes); + kfree(ctx->aes); safexcel_ahash_cra_exit(tfm); } @@ -2178,31 +2166,23 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int len) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct crypto_aes_ctx aes; __be64 consts[4]; u64 _const[2]; u8 msb_mask, gfmask; int ret, i; - ret = aes_expandkey(&aes, key, len); + /* precompute the CMAC key material */ + ret = aes_expandkey(ctx->aes, key, len); if (ret) return ret; for (i = 0; i < len / sizeof(u32); i++) - ctx->base.ipad.word[i + 8] = swab32(aes.key_enc[i]); - - /* precompute the CMAC key material */ - crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->kaes, key, len); - if (ret) - return ret; + ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]); /* code below borrowed from crypto/cmac.c */ /* encrypt the zero block */ memset(consts, 0, AES_BLOCK_SIZE); - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts); + aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts); gfmask = 0x87; _const[0] = be64_to_cpu(consts[1]); @@ -2234,7 +2214,6 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, } ctx->cbcmac = false; - memzero_explicit(&aes, sizeof(aes)); return 0; } -- cgit From 611d451e4041b4be1c59e0888b64caa4ff1204ad Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 14 Sep 2022 10:36:26 +0200 Subject: crypto: arm64 - revert unintended config name change for CRYPTO_SHA1_ARM64_CE Commit 3f342a23257d ("crypto: Kconfig - simplify hash entries") makes various changes to the config descriptions as part of some consolidation and clean-up, but among all those changes, it also accidently renames CRYPTO_SHA1_ARM64_CE to CRYPTO_SHA1_ARM64. Revert this unintended config name change. See Link for the author's confirmation of this happening accidently. Fixes: 3f342a23257d ("crypto: Kconfig - simplify hash entries") Link: https://lore.kernel.org/all/MW5PR84MB18424AB8C095BFC041AE33FDAB479@MW5PR84MB1842.NAMPRD84.PROD.OUTLOOK.COM/ Signed-off-by: Lukas Bulwahn Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 7ba9bcb6d409..8bd80508a710 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -36,7 +36,7 @@ config CRYPTO_POLY1305_NEON Architecture: arm64 using: - NEON (Advanced SIMD) extensions -config CRYPTO_SHA1_ARM64 +config CRYPTO_SHA1_ARM64_CE tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH -- cgit From 1b79573de717cfabe28221a98afaa6a3ff0e7458 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 14 Sep 2022 10:38:27 +0200 Subject: crypto: blake2s - revert unintended config addition of CRYPTO_BLAKE2S Commit 2d16803c562e ("crypto: blake2s - remove shash module") removes the config CRYPTO_BLAKE2S. Commit 3f342a23257d ("crypto: Kconfig - simplify hash entries") makes various changes to the config descriptions as part of some consolidation and clean-up, but among all those changes, it also accidently adds back CRYPTO_BLAKE2S after its removal due to the original patch being based on a state before the CRYPTO_BLAKE2S removal. See Link for the author's confirmation of this happening accidently. Fixes: 3f342a23257d ("crypto: Kconfig - simplify hash entries") Link: https://lore.kernel.org/all/MW5PR84MB18424AB8C095BFC041AE33FDAB479@MW5PR84MB1842.NAMPRD84.PROD.OUTLOOK.COM/ Signed-off-by: Lukas Bulwahn Signed-off-by: Herbert Xu --- crypto/Kconfig | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 40423a14f86f..2589ad5357df 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -916,27 +916,6 @@ config CRYPTO_BLAKE2B See https://blake2.net for further information. - config CRYPTO_BLAKE2S - tristate "BLAKE2s" - select CRYPTO_LIB_BLAKE2S_GENERIC - select CRYPTO_HASH - help - BLAKE2s cryptographic hash function (RFC 7693) - - BLAKE2s is optimized for 8 to 32-bit platforms and can produce - digests of any size between 1 and 32 bytes. The keyed hash is - also implemented. - - This module provides the following algorithms: - - blake2s-128 - - blake2s-160 - - blake2s-224 - - blake2s-256 - - Used by Wireguard. - - See https://blake2.net for further information. - config CRYPTO_CMAC tristate "CMAC (Cipher-based MAC)" select CRYPTO_HASH -- cgit From 33837be33367172d66d1f2bd6964cc41448e6e7c Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Thu, 15 Sep 2022 11:36:15 +0800 Subject: crypto: add __init/__exit annotations to init/exit funcs Add missing __init/__exit annotations to init/exit funcs. Signed-off-by: Xiu Jianfeng Signed-off-by: Herbert Xu --- crypto/async_tx/raid6test.c | 4 ++-- crypto/curve25519-generic.c | 4 ++-- crypto/dh.c | 4 ++-- crypto/ecdh.c | 4 ++-- crypto/ecdsa.c | 4 ++-- crypto/rsa.c | 4 ++-- crypto/sm2.c | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c index c9d218e53bcb..9719c7520661 100644 --- a/crypto/async_tx/raid6test.c +++ b/crypto/async_tx/raid6test.c @@ -189,7 +189,7 @@ static int test(int disks, int *tests) } -static int raid6_test(void) +static int __init raid6_test(void) { int err = 0; int tests = 0; @@ -236,7 +236,7 @@ static int raid6_test(void) return 0; } -static void raid6_test_exit(void) +static void __exit raid6_test_exit(void) { } diff --git a/crypto/curve25519-generic.c b/crypto/curve25519-generic.c index bd88fd571393..d055b0784c77 100644 --- a/crypto/curve25519-generic.c +++ b/crypto/curve25519-generic.c @@ -72,12 +72,12 @@ static struct kpp_alg curve25519_alg = { .max_size = curve25519_max_size, }; -static int curve25519_init(void) +static int __init curve25519_init(void) { return crypto_register_kpp(&curve25519_alg); } -static void curve25519_exit(void) +static void __exit curve25519_exit(void) { crypto_unregister_kpp(&curve25519_alg); } diff --git a/crypto/dh.c b/crypto/dh.c index 4406aeb1ff61..99c3b2ef7adc 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -893,7 +893,7 @@ static struct crypto_template crypto_ffdhe_templates[] = {}; #endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */ -static int dh_init(void) +static int __init dh_init(void) { int err; @@ -911,7 +911,7 @@ static int dh_init(void) return 0; } -static void dh_exit(void) +static void __exit dh_exit(void) { crypto_unregister_templates(crypto_ffdhe_templates, ARRAY_SIZE(crypto_ffdhe_templates)); diff --git a/crypto/ecdh.c b/crypto/ecdh.c index e4857d534344..80afee3234fb 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -200,7 +200,7 @@ static struct kpp_alg ecdh_nist_p384 = { static bool ecdh_nist_p192_registered; -static int ecdh_init(void) +static int __init ecdh_init(void) { int ret; @@ -227,7 +227,7 @@ nist_p256_error: return ret; } -static void ecdh_exit(void) +static void __exit ecdh_exit(void) { if (ecdh_nist_p192_registered) crypto_unregister_kpp(&ecdh_nist_p192); diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index b3a8a6b572ba..fbd76498aba8 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -332,7 +332,7 @@ static struct akcipher_alg ecdsa_nist_p192 = { }; static bool ecdsa_nist_p192_registered; -static int ecdsa_init(void) +static int __init ecdsa_init(void) { int ret; @@ -359,7 +359,7 @@ nist_p256_error: return ret; } -static void ecdsa_exit(void) +static void __exit ecdsa_exit(void) { if (ecdsa_nist_p192_registered) crypto_unregister_akcipher(&ecdsa_nist_p192); diff --git a/crypto/rsa.c b/crypto/rsa.c index 0e555ee4addb..c50f2d2a4d06 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -327,7 +327,7 @@ static struct akcipher_alg rsa = { }, }; -static int rsa_init(void) +static int __init rsa_init(void) { int err; @@ -344,7 +344,7 @@ static int rsa_init(void) return 0; } -static void rsa_exit(void) +static void __exit rsa_exit(void) { crypto_unregister_template(&rsa_pkcs1pad_tmpl); crypto_unregister_akcipher(&rsa); diff --git a/crypto/sm2.c b/crypto/sm2.c index f3e1592965c0..ed9307dac3d1 100644 --- a/crypto/sm2.c +++ b/crypto/sm2.c @@ -441,12 +441,12 @@ static struct akcipher_alg sm2 = { }, }; -static int sm2_init(void) +static int __init sm2_init(void) { return crypto_register_akcipher(&sm2); } -static void sm2_exit(void) +static void __exit sm2_exit(void) { crypto_unregister_akcipher(&sm2); } -- cgit From 4532f1cf9caaf2588d48d1de7770a6d5d1f95e89 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 16 Sep 2022 18:35:50 +0800 Subject: crypto: artpec6 - Fix printk warning on size_t/%d Switch to %zu instead of %d for printing size_t. Signed-off-by: Herbert Xu Acked-by: Jesper Nilsson Signed-off-by: Herbert Xu --- drivers/crypto/axis/artpec6_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index b4820594ab80..51c66afbe677 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -1712,7 +1712,7 @@ static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) cipher_len = regk_crypto_key_256; break; default: - pr_err("%s: Invalid key length %d!\n", + pr_err("%s: Invalid key length %zu!\n", MODULE_NAME, ctx->key_length); return -EINVAL; } -- cgit From a9b0838dd82534c49dd4e5e2172ddea3fb2b5d39 Mon Sep 17 00:00:00 2001 From: Taehee Yoo Date: Fri, 16 Sep 2022 12:57:34 +0000 Subject: crypto: aria - prepare generic module for optimized implementations It renames aria to aria_generic and exports some functions such as aria_set_key(), aria_encrypt(), and aria_decrypt() to be able to be used by aria-avx implementation. Signed-off-by: Taehee Yoo Signed-off-by: Herbert Xu --- crypto/Makefile | 2 +- crypto/aria.c | 288 ---------------------------------------------- crypto/aria_generic.c | 313 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/crypto/aria.h | 17 ++- 4 files changed, 321 insertions(+), 299 deletions(-) delete mode 100644 crypto/aria.c create mode 100644 crypto/aria_generic.c diff --git a/crypto/Makefile b/crypto/Makefile index a6f94e04e1da..303b21c43df0 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -149,7 +149,7 @@ obj-$(CONFIG_CRYPTO_TEA) += tea.o obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o -obj-$(CONFIG_CRYPTO_ARIA) += aria.o +obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o diff --git a/crypto/aria.c b/crypto/aria.c deleted file mode 100644 index ac3dffac34bb..000000000000 --- a/crypto/aria.c +++ /dev/null @@ -1,288 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * ARIA Cipher Algorithm. - * - * Documentation of ARIA can be found in RFC 5794. - * Copyright (c) 2022 Taehee Yoo - * - * Information for ARIA - * http://210.104.33.10/ARIA/index-e.html (English) - * http://seed.kisa.or.kr/ (Korean) - * - * Public domain version is distributed above. - */ - -#include - -static void aria_set_encrypt_key(struct aria_ctx *ctx, const u8 *in_key, - unsigned int key_len) -{ - const __be32 *key = (const __be32 *)in_key; - u32 w0[4], w1[4], w2[4], w3[4]; - u32 reg0, reg1, reg2, reg3; - const u32 *ck; - int rkidx = 0; - - ck = &key_rc[(key_len - 16) / 8][0]; - - w0[0] = be32_to_cpu(key[0]); - w0[1] = be32_to_cpu(key[1]); - w0[2] = be32_to_cpu(key[2]); - w0[3] = be32_to_cpu(key[3]); - - reg0 = w0[0] ^ ck[0]; - reg1 = w0[1] ^ ck[1]; - reg2 = w0[2] ^ ck[2]; - reg3 = w0[3] ^ ck[3]; - - aria_subst_diff_odd(®0, ®1, ®2, ®3); - - if (key_len > 16) { - w1[0] = be32_to_cpu(key[4]); - w1[1] = be32_to_cpu(key[5]); - if (key_len > 24) { - w1[2] = be32_to_cpu(key[6]); - w1[3] = be32_to_cpu(key[7]); - } else { - w1[2] = 0; - w1[3] = 0; - } - } else { - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - } - - w1[0] ^= reg0; - w1[1] ^= reg1; - w1[2] ^= reg2; - w1[3] ^= reg3; - - reg0 = w1[0]; - reg1 = w1[1]; - reg2 = w1[2]; - reg3 = w1[3]; - - reg0 ^= ck[4]; - reg1 ^= ck[5]; - reg2 ^= ck[6]; - reg3 ^= ck[7]; - - aria_subst_diff_even(®0, ®1, ®2, ®3); - - reg0 ^= w0[0]; - reg1 ^= w0[1]; - reg2 ^= w0[2]; - reg3 ^= w0[3]; - - w2[0] = reg0; - w2[1] = reg1; - w2[2] = reg2; - w2[3] = reg3; - - reg0 ^= ck[8]; - reg1 ^= ck[9]; - reg2 ^= ck[10]; - reg3 ^= ck[11]; - - aria_subst_diff_odd(®0, ®1, ®2, ®3); - - w3[0] = reg0 ^ w1[0]; - w3[1] = reg1 ^ w1[1]; - w3[2] = reg2 ^ w1[2]; - w3[3] = reg3 ^ w1[3]; - - aria_gsrk(ctx->enc_key[rkidx], w0, w1, 19); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w1, w2, 19); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w2, w3, 19); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w3, w0, 19); - - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w0, w1, 31); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w1, w2, 31); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w2, w3, 31); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w3, w0, 31); - - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w0, w1, 67); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w1, w2, 67); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w2, w3, 67); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w3, w0, 67); - - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w0, w1, 97); - if (key_len > 16) { - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w1, w2, 97); - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w2, w3, 97); - - if (key_len > 24) { - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w3, w0, 97); - - rkidx++; - aria_gsrk(ctx->enc_key[rkidx], w0, w1, 109); - } - } -} - -static void aria_set_decrypt_key(struct aria_ctx *ctx) -{ - int i; - - for (i = 0; i < 4; i++) { - ctx->dec_key[0][i] = ctx->enc_key[ctx->rounds][i]; - ctx->dec_key[ctx->rounds][i] = ctx->enc_key[0][i]; - } - - for (i = 1; i < ctx->rounds; i++) { - ctx->dec_key[i][0] = aria_m(ctx->enc_key[ctx->rounds - i][0]); - ctx->dec_key[i][1] = aria_m(ctx->enc_key[ctx->rounds - i][1]); - ctx->dec_key[i][2] = aria_m(ctx->enc_key[ctx->rounds - i][2]); - ctx->dec_key[i][3] = aria_m(ctx->enc_key[ctx->rounds - i][3]); - - aria_diff_word(&ctx->dec_key[i][0], &ctx->dec_key[i][1], - &ctx->dec_key[i][2], &ctx->dec_key[i][3]); - aria_diff_byte(&ctx->dec_key[i][1], - &ctx->dec_key[i][2], &ctx->dec_key[i][3]); - aria_diff_word(&ctx->dec_key[i][0], &ctx->dec_key[i][1], - &ctx->dec_key[i][2], &ctx->dec_key[i][3]); - } -} - -static int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct aria_ctx *ctx = crypto_tfm_ctx(tfm); - - if (key_len != 16 && key_len != 24 && key_len != 32) - return -EINVAL; - - ctx->key_length = key_len; - ctx->rounds = (key_len + 32) / 4; - - aria_set_encrypt_key(ctx, in_key, key_len); - aria_set_decrypt_key(ctx); - - return 0; -} - -static void __aria_crypt(struct aria_ctx *ctx, u8 *out, const u8 *in, - u32 key[][ARIA_RD_KEY_WORDS]) -{ - const __be32 *src = (const __be32 *)in; - __be32 *dst = (__be32 *)out; - u32 reg0, reg1, reg2, reg3; - int rounds, rkidx = 0; - - rounds = ctx->rounds; - - reg0 = be32_to_cpu(src[0]); - reg1 = be32_to_cpu(src[1]); - reg2 = be32_to_cpu(src[2]); - reg3 = be32_to_cpu(src[3]); - - aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); - rkidx++; - - aria_subst_diff_odd(®0, ®1, ®2, ®3); - aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); - rkidx++; - - while ((rounds -= 2) > 0) { - aria_subst_diff_even(®0, ®1, ®2, ®3); - aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); - rkidx++; - - aria_subst_diff_odd(®0, ®1, ®2, ®3); - aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); - rkidx++; - } - - reg0 = key[rkidx][0] ^ make_u32((u8)(x1[get_u8(reg0, 0)]), - (u8)(x2[get_u8(reg0, 1)] >> 8), - (u8)(s1[get_u8(reg0, 2)]), - (u8)(s2[get_u8(reg0, 3)])); - reg1 = key[rkidx][1] ^ make_u32((u8)(x1[get_u8(reg1, 0)]), - (u8)(x2[get_u8(reg1, 1)] >> 8), - (u8)(s1[get_u8(reg1, 2)]), - (u8)(s2[get_u8(reg1, 3)])); - reg2 = key[rkidx][2] ^ make_u32((u8)(x1[get_u8(reg2, 0)]), - (u8)(x2[get_u8(reg2, 1)] >> 8), - (u8)(s1[get_u8(reg2, 2)]), - (u8)(s2[get_u8(reg2, 3)])); - reg3 = key[rkidx][3] ^ make_u32((u8)(x1[get_u8(reg3, 0)]), - (u8)(x2[get_u8(reg3, 1)] >> 8), - (u8)(s1[get_u8(reg3, 2)]), - (u8)(s2[get_u8(reg3, 3)])); - - dst[0] = cpu_to_be32(reg0); - dst[1] = cpu_to_be32(reg1); - dst[2] = cpu_to_be32(reg2); - dst[3] = cpu_to_be32(reg3); -} - -static void aria_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct aria_ctx *ctx = crypto_tfm_ctx(tfm); - - __aria_crypt(ctx, out, in, ctx->enc_key); -} - -static void aria_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct aria_ctx *ctx = crypto_tfm_ctx(tfm); - - __aria_crypt(ctx, out, in, ctx->dec_key); -} - -static struct crypto_alg aria_alg = { - .cra_name = "aria", - .cra_driver_name = "aria-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = ARIA_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aria_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = ARIA_MIN_KEY_SIZE, - .cia_max_keysize = ARIA_MAX_KEY_SIZE, - .cia_setkey = aria_set_key, - .cia_encrypt = aria_encrypt, - .cia_decrypt = aria_decrypt - } - } -}; - -static int __init aria_init(void) -{ - return crypto_register_alg(&aria_alg); -} - -static void __exit aria_fini(void) -{ - crypto_unregister_alg(&aria_alg); -} - -subsys_initcall(aria_init); -module_exit(aria_fini); - -MODULE_DESCRIPTION("ARIA Cipher Algorithm"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Taehee Yoo "); -MODULE_ALIAS_CRYPTO("aria"); diff --git a/crypto/aria_generic.c b/crypto/aria_generic.c new file mode 100644 index 000000000000..4cc29b82b99d --- /dev/null +++ b/crypto/aria_generic.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API. + * + * ARIA Cipher Algorithm. + * + * Documentation of ARIA can be found in RFC 5794. + * Copyright (c) 2022 Taehee Yoo + * + * Information for ARIA + * http://210.104.33.10/ARIA/index-e.html (English) + * http://seed.kisa.or.kr/ (Korean) + * + * Public domain version is distributed above. + */ + +#include + +static const u32 key_rc[20] = { + 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0, + 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0, + 0xdb92371d, 0x2126e970, 0x03249775, 0x04e8c90e, + 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0, + 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 +}; + +static void aria_set_encrypt_key(struct aria_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + const __be32 *key = (const __be32 *)in_key; + u32 w0[4], w1[4], w2[4], w3[4]; + u32 reg0, reg1, reg2, reg3; + const u32 *ck; + int rkidx = 0; + + ck = &key_rc[(key_len - 16) / 2]; + + w0[0] = be32_to_cpu(key[0]); + w0[1] = be32_to_cpu(key[1]); + w0[2] = be32_to_cpu(key[2]); + w0[3] = be32_to_cpu(key[3]); + + reg0 = w0[0] ^ ck[0]; + reg1 = w0[1] ^ ck[1]; + reg2 = w0[2] ^ ck[2]; + reg3 = w0[3] ^ ck[3]; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + + if (key_len > 16) { + w1[0] = be32_to_cpu(key[4]); + w1[1] = be32_to_cpu(key[5]); + if (key_len > 24) { + w1[2] = be32_to_cpu(key[6]); + w1[3] = be32_to_cpu(key[7]); + } else { + w1[2] = 0; + w1[3] = 0; + } + } else { + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + } + + w1[0] ^= reg0; + w1[1] ^= reg1; + w1[2] ^= reg2; + w1[3] ^= reg3; + + reg0 = w1[0]; + reg1 = w1[1]; + reg2 = w1[2]; + reg3 = w1[3]; + + reg0 ^= ck[4]; + reg1 ^= ck[5]; + reg2 ^= ck[6]; + reg3 ^= ck[7]; + + aria_subst_diff_even(®0, ®1, ®2, ®3); + + reg0 ^= w0[0]; + reg1 ^= w0[1]; + reg2 ^= w0[2]; + reg3 ^= w0[3]; + + w2[0] = reg0; + w2[1] = reg1; + w2[2] = reg2; + w2[3] = reg3; + + reg0 ^= ck[8]; + reg1 ^= ck[9]; + reg2 ^= ck[10]; + reg3 ^= ck[11]; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + + w3[0] = reg0 ^ w1[0]; + w3[1] = reg1 ^ w1[1]; + w3[2] = reg2 ^ w1[2]; + w3[3] = reg3 ^ w1[3]; + + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 19); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 19); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 19); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 19); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 31); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 31); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 31); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 31); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 67); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 67); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 67); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 67); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 97); + if (key_len > 16) { + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 97); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 97); + + if (key_len > 24) { + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 97); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 109); + } + } +} + +static void aria_set_decrypt_key(struct aria_ctx *ctx) +{ + int i; + + for (i = 0; i < 4; i++) { + ctx->dec_key[0][i] = ctx->enc_key[ctx->rounds][i]; + ctx->dec_key[ctx->rounds][i] = ctx->enc_key[0][i]; + } + + for (i = 1; i < ctx->rounds; i++) { + ctx->dec_key[i][0] = aria_m(ctx->enc_key[ctx->rounds - i][0]); + ctx->dec_key[i][1] = aria_m(ctx->enc_key[ctx->rounds - i][1]); + ctx->dec_key[i][2] = aria_m(ctx->enc_key[ctx->rounds - i][2]); + ctx->dec_key[i][3] = aria_m(ctx->enc_key[ctx->rounds - i][3]); + + aria_diff_word(&ctx->dec_key[i][0], &ctx->dec_key[i][1], + &ctx->dec_key[i][2], &ctx->dec_key[i][3]); + aria_diff_byte(&ctx->dec_key[i][1], + &ctx->dec_key[i][2], &ctx->dec_key[i][3]); + aria_diff_word(&ctx->dec_key[i][0], &ctx->dec_key[i][1], + &ctx->dec_key[i][2], &ctx->dec_key[i][3]); + } +} + +int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) +{ + struct aria_ctx *ctx = crypto_tfm_ctx(tfm); + + if (key_len != 16 && key_len != 24 && key_len != 32) + return -EINVAL; + + ctx->key_length = key_len; + ctx->rounds = (key_len + 32) / 4; + + aria_set_encrypt_key(ctx, in_key, key_len); + aria_set_decrypt_key(ctx); + + return 0; +} +EXPORT_SYMBOL_GPL(aria_set_key); + +static void __aria_crypt(struct aria_ctx *ctx, u8 *out, const u8 *in, + u32 key[][ARIA_RD_KEY_WORDS]) +{ + const __be32 *src = (const __be32 *)in; + __be32 *dst = (__be32 *)out; + u32 reg0, reg1, reg2, reg3; + int rounds, rkidx = 0; + + rounds = ctx->rounds; + + reg0 = be32_to_cpu(src[0]); + reg1 = be32_to_cpu(src[1]); + reg2 = be32_to_cpu(src[2]); + reg3 = be32_to_cpu(src[3]); + + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + + while ((rounds -= 2) > 0) { + aria_subst_diff_even(®0, ®1, ®2, ®3); + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + } + + reg0 = key[rkidx][0] ^ make_u32((u8)(x1[get_u8(reg0, 0)]), + (u8)(x2[get_u8(reg0, 1)] >> 8), + (u8)(s1[get_u8(reg0, 2)]), + (u8)(s2[get_u8(reg0, 3)])); + reg1 = key[rkidx][1] ^ make_u32((u8)(x1[get_u8(reg1, 0)]), + (u8)(x2[get_u8(reg1, 1)] >> 8), + (u8)(s1[get_u8(reg1, 2)]), + (u8)(s2[get_u8(reg1, 3)])); + reg2 = key[rkidx][2] ^ make_u32((u8)(x1[get_u8(reg2, 0)]), + (u8)(x2[get_u8(reg2, 1)] >> 8), + (u8)(s1[get_u8(reg2, 2)]), + (u8)(s2[get_u8(reg2, 3)])); + reg3 = key[rkidx][3] ^ make_u32((u8)(x1[get_u8(reg3, 0)]), + (u8)(x2[get_u8(reg3, 1)] >> 8), + (u8)(s1[get_u8(reg3, 2)]), + (u8)(s2[get_u8(reg3, 3)])); + + dst[0] = cpu_to_be32(reg0); + dst[1] = cpu_to_be32(reg1); + dst[2] = cpu_to_be32(reg2); + dst[3] = cpu_to_be32(reg3); +} + +void aria_encrypt(void *_ctx, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = (struct aria_ctx *)_ctx; + + __aria_crypt(ctx, out, in, ctx->enc_key); +} +EXPORT_SYMBOL_GPL(aria_encrypt); + +void aria_decrypt(void *_ctx, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = (struct aria_ctx *)_ctx; + + __aria_crypt(ctx, out, in, ctx->dec_key); +} +EXPORT_SYMBOL_GPL(aria_decrypt); + +static void __aria_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = crypto_tfm_ctx(tfm); + + __aria_crypt(ctx, out, in, ctx->enc_key); +} + +static void __aria_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = crypto_tfm_ctx(tfm); + + __aria_crypt(ctx, out, in, ctx->dec_key); +} + +static struct crypto_alg aria_alg = { + .cra_name = "aria", + .cra_driver_name = "aria-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = ARIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aria_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = ARIA_MIN_KEY_SIZE, + .cia_max_keysize = ARIA_MAX_KEY_SIZE, + .cia_setkey = aria_set_key, + .cia_encrypt = __aria_encrypt, + .cia_decrypt = __aria_decrypt + } + } +}; + +static int __init aria_init(void) +{ + return crypto_register_alg(&aria_alg); +} + +static void __exit aria_fini(void) +{ + crypto_unregister_alg(&aria_alg); +} + +subsys_initcall(aria_init); +module_exit(aria_fini); + +MODULE_DESCRIPTION("ARIA Cipher Algorithm"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Taehee Yoo "); +MODULE_ALIAS_CRYPTO("aria"); +MODULE_ALIAS_CRYPTO("aria-generic"); diff --git a/include/crypto/aria.h b/include/crypto/aria.h index 4a86661788e8..254da46cc385 100644 --- a/include/crypto/aria.h +++ b/include/crypto/aria.h @@ -32,18 +32,10 @@ #define ARIA_RD_KEY_WORDS (ARIA_BLOCK_SIZE / sizeof(u32)) struct aria_ctx { - int key_length; - int rounds; u32 enc_key[ARIA_MAX_RD_KEYS][ARIA_RD_KEY_WORDS]; u32 dec_key[ARIA_MAX_RD_KEYS][ARIA_RD_KEY_WORDS]; -}; - -static const u32 key_rc[5][4] = { - { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, - { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 }, - { 0xdb92371d, 0x2126e970, 0x03249775, 0x04e8c90e }, - { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, - { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 } + int rounds; + int key_length; }; static const u32 s1[256] = { @@ -458,4 +450,9 @@ static inline void aria_gsrk(u32 *rk, u32 *x, u32 *y, u32 n) ((y[(q + 2) % 4]) << (32 - r)); } +void aria_encrypt(void *ctx, u8 *out, const u8 *in); +void aria_decrypt(void *ctx, u8 *out, const u8 *in); +int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len); + #endif -- cgit From ba3579e6e45c693495a50c516278749c5e3d9977 Mon Sep 17 00:00:00 2001 From: Taehee Yoo Date: Fri, 16 Sep 2022 12:57:35 +0000 Subject: crypto: aria-avx - add AES-NI/AVX/x86_64/GFNI assembler implementation of aria cipher The implementation is based on the 32-bit implementation of the aria. Also, aria-avx process steps are the similar to the camellia-avx. 1. Byteslice(16way) 2. Add-round-key. 3. Sbox 4. Diffusion layer. Except for s-box, all steps are the same as the aria-generic implementation. s-box step is very similar to camellia and sm4 implementation. There are 2 implementations for s-box step. One is to use AES-NI and affine transformation, which is the same as Camellia, sm4, and others. Another is to use GFNI. GFNI implementation is faster than AES-NI implementation. So, it uses GFNI implementation if the running CPU supports GFNI. There are 4 s-boxes in the ARIA and the 2 s-boxes are the same as AES's s-boxes. To calculate the first sbox, it just uses the aesenclast and then inverts shift_row. No more process is needed for this job because the first s-box is the same as the AES encryption s-box. To calculate the second sbox(invert of s1), it just uses the aesdeclast and then inverts shift_row. No more process is needed for this job because the second s-box is the same as the AES decryption s-box. To calculate the third s-box, it uses the aesenclast, then affine transformation, which is combined AES inverse affine and ARIA S2. To calculate the last s-box, it uses the aesdeclast, then affine transformation, which is combined X2 and AES forward affine. The optimized third and last s-box logic and GFNI s-box logic are implemented by Jussi Kivilinna. The aria-generic implementation is based on a 32-bit implementation, not an 8-bit implementation. the aria-avx Diffusion Layer implementation is based on aria-generic implementation because 8-bit implementation is not fit for parallel implementation but 32-bit is enough to fit for this. Signed-off-by: Taehee Yoo Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 18 + arch/x86/crypto/Makefile | 3 + arch/x86/crypto/aria-aesni-avx-asm_64.S | 1303 +++++++++++++++++++++++++++++++ arch/x86/crypto/aria-avx.h | 16 + arch/x86/crypto/aria_aesni_avx_glue.c | 213 +++++ 5 files changed, 1553 insertions(+) create mode 100644 arch/x86/crypto/aria-aesni-avx-asm_64.S create mode 100644 arch/x86/crypto/aria-avx.h create mode 100644 arch/x86/crypto/aria_aesni_avx_glue.c diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 9bb0f7939c6b..71c4c473d34b 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -286,6 +286,24 @@ config CRYPTO_TWOFISH_AVX_X86_64 Processes eight blocks in parallel. +config CRYPTO_ARIA_AESNI_AVX_X86_64 + tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX/GFNI)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_ALGAPI + select CRYPTO_ARIA + help + Length-preserving cipher: ARIA cipher algorithms + (RFC 5794) with ECB and CTR modes + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX (Advanced Vector Extensions) + - GFNI (Galois Field New Instructions) + + Processes 16 blocks in parallel. + config CRYPTO_CHACHA20_X86_64 tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (SSSE3/AVX2/AVX-512VL)" depends on X86 && 64BIT diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 04d07ab744b2..3b1d701a4f6c 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -100,6 +100,9 @@ sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX2_X86_64) += sm4-aesni-avx2-x86_64.o sm4-aesni-avx2-x86_64-y := sm4-aesni-avx2-asm_64.o sm4_aesni_avx2_glue.o +obj-$(CONFIG_CRYPTO_ARIA_AESNI_AVX_X86_64) += aria-aesni-avx-x86_64.o +aria-aesni-avx-x86_64-y := aria-aesni-avx-asm_64.o aria_aesni_avx_glue.o + quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $< > $@ $(obj)/%.S: $(src)/%.pl FORCE diff --git a/arch/x86/crypto/aria-aesni-avx-asm_64.S b/arch/x86/crypto/aria-aesni-avx-asm_64.S new file mode 100644 index 000000000000..c75fd7d015ed --- /dev/null +++ b/arch/x86/crypto/aria-aesni-avx-asm_64.S @@ -0,0 +1,1303 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ARIA Cipher 16-way parallel algorithm (AVX) + * + * Copyright (c) 2022 Taehee Yoo + * + */ + +#include +#include + +/* struct aria_ctx: */ +#define enc_key 0 +#define dec_key 272 +#define rounds 544 + +/* register macros */ +#define CTX %rdi + + +#define BV8(a0, a1, a2, a3, a4, a5, a6, a7) \ + ( (((a0) & 1) << 0) | \ + (((a1) & 1) << 1) | \ + (((a2) & 1) << 2) | \ + (((a3) & 1) << 3) | \ + (((a4) & 1) << 4) | \ + (((a5) & 1) << 5) | \ + (((a6) & 1) << 6) | \ + (((a7) & 1) << 7) ) + +#define BM8X8(l0, l1, l2, l3, l4, l5, l6, l7) \ + ( ((l7) << (0 * 8)) | \ + ((l6) << (1 * 8)) | \ + ((l5) << (2 * 8)) | \ + ((l4) << (3 * 8)) | \ + ((l3) << (4 * 8)) | \ + ((l2) << (5 * 8)) | \ + ((l1) << (6 * 8)) | \ + ((l0) << (7 * 8)) ) + +#define inc_le128(x, minus_one, tmp) \ + vpcmpeqq minus_one, x, tmp; \ + vpsubq minus_one, x, x; \ + vpslldq $8, tmp, tmp; \ + vpsubq tmp, x, x; + +#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ + vpand x, mask4bit, tmp0; \ + vpandn x, mask4bit, x; \ + vpsrld $4, x, x; \ + \ + vpshufb tmp0, lo_t, tmp0; \ + vpshufb x, hi_t, x; \ + vpxor tmp0, x, x; + +#define transpose_4x4(x0, x1, x2, x3, t1, t2) \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x1, x0, x0; \ + \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x2; \ + \ + vpunpckhqdq t1, x0, x1; \ + vpunpcklqdq t1, x0, x0; \ + \ + vpunpckhqdq x2, t2, x3; \ + vpunpcklqdq x2, t2, x2; + +#define byteslice_16x16b(a0, b0, c0, d0, \ + a1, b1, c1, d1, \ + a2, b2, c2, d2, \ + a3, b3, c3, d3, \ + st0, st1) \ + vmovdqu d2, st0; \ + vmovdqu d3, st1; \ + transpose_4x4(a0, a1, a2, a3, d2, d3); \ + transpose_4x4(b0, b1, b2, b3, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu a0, st0; \ + vmovdqu a1, st1; \ + transpose_4x4(c0, c1, c2, c3, a0, a1); \ + transpose_4x4(d0, d1, d2, d3, a0, a1); \ + \ + vmovdqu .Lshufb_16x16b, a0; \ + vmovdqu st1, a1; \ + vpshufb a0, a2, a2; \ + vpshufb a0, a3, a3; \ + vpshufb a0, b0, b0; \ + vpshufb a0, b1, b1; \ + vpshufb a0, b2, b2; \ + vpshufb a0, b3, b3; \ + vpshufb a0, a1, a1; \ + vpshufb a0, c0, c0; \ + vpshufb a0, c1, c1; \ + vpshufb a0, c2, c2; \ + vpshufb a0, c3, c3; \ + vpshufb a0, d0, d0; \ + vpshufb a0, d1, d1; \ + vpshufb a0, d2, d2; \ + vpshufb a0, d3, d3; \ + vmovdqu d3, st1; \ + vmovdqu st0, d3; \ + vpshufb a0, d3, a0; \ + vmovdqu d2, st0; \ + \ + transpose_4x4(a0, b0, c0, d0, d2, d3); \ + transpose_4x4(a1, b1, c1, d1, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu b0, st0; \ + vmovdqu b1, st1; \ + transpose_4x4(a2, b2, c2, d2, b0, b1); \ + transpose_4x4(a3, b3, c3, d3, b0, b1); \ + vmovdqu st0, b0; \ + vmovdqu st1, b1; \ + /* does not adjust output bytes inside vectors */ + +#define debyteslice_16x16b(a0, b0, c0, d0, \ + a1, b1, c1, d1, \ + a2, b2, c2, d2, \ + a3, b3, c3, d3, \ + st0, st1) \ + vmovdqu d2, st0; \ + vmovdqu d3, st1; \ + transpose_4x4(a0, a1, a2, a3, d2, d3); \ + transpose_4x4(b0, b1, b2, b3, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu a0, st0; \ + vmovdqu a1, st1; \ + transpose_4x4(c0, c1, c2, c3, a0, a1); \ + transpose_4x4(d0, d1, d2, d3, a0, a1); \ + \ + vmovdqu .Lshufb_16x16b, a0; \ + vmovdqu st1, a1; \ + vpshufb a0, a2, a2; \ + vpshufb a0, a3, a3; \ + vpshufb a0, b0, b0; \ + vpshufb a0, b1, b1; \ + vpshufb a0, b2, b2; \ + vpshufb a0, b3, b3; \ + vpshufb a0, a1, a1; \ + vpshufb a0, c0, c0; \ + vpshufb a0, c1, c1; \ + vpshufb a0, c2, c2; \ + vpshufb a0, c3, c3; \ + vpshufb a0, d0, d0; \ + vpshufb a0, d1, d1; \ + vpshufb a0, d2, d2; \ + vpshufb a0, d3, d3; \ + vmovdqu d3, st1; \ + vmovdqu st0, d3; \ + vpshufb a0, d3, a0; \ + vmovdqu d2, st0; \ + \ + transpose_4x4(c0, d0, a0, b0, d2, d3); \ + transpose_4x4(c1, d1, a1, b1, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu b0, st0; \ + vmovdqu b1, st1; \ + transpose_4x4(c2, d2, a2, b2, b0, b1); \ + transpose_4x4(c3, d3, a3, b3, b0, b1); \ + vmovdqu st0, b0; \ + vmovdqu st1, b1; \ + /* does not adjust output bytes inside vectors */ + +/* load blocks to registers and apply pre-whitening */ +#define inpack16_pre(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + rio) \ + vmovdqu (0 * 16)(rio), x0; \ + vmovdqu (1 * 16)(rio), x1; \ + vmovdqu (2 * 16)(rio), x2; \ + vmovdqu (3 * 16)(rio), x3; \ + vmovdqu (4 * 16)(rio), x4; \ + vmovdqu (5 * 16)(rio), x5; \ + vmovdqu (6 * 16)(rio), x6; \ + vmovdqu (7 * 16)(rio), x7; \ + vmovdqu (8 * 16)(rio), y0; \ + vmovdqu (9 * 16)(rio), y1; \ + vmovdqu (10 * 16)(rio), y2; \ + vmovdqu (11 * 16)(rio), y3; \ + vmovdqu (12 * 16)(rio), y4; \ + vmovdqu (13 * 16)(rio), y5; \ + vmovdqu (14 * 16)(rio), y6; \ + vmovdqu (15 * 16)(rio), y7; + +/* byteslice pre-whitened blocks and store to temporary memory */ +#define inpack16_post(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_ab, mem_cd) \ + byteslice_16x16b(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + (mem_ab), (mem_cd)); \ + \ + vmovdqu x0, 0 * 16(mem_ab); \ + vmovdqu x1, 1 * 16(mem_ab); \ + vmovdqu x2, 2 * 16(mem_ab); \ + vmovdqu x3, 3 * 16(mem_ab); \ + vmovdqu x4, 4 * 16(mem_ab); \ + vmovdqu x5, 5 * 16(mem_ab); \ + vmovdqu x6, 6 * 16(mem_ab); \ + vmovdqu x7, 7 * 16(mem_ab); \ + vmovdqu y0, 0 * 16(mem_cd); \ + vmovdqu y1, 1 * 16(mem_cd); \ + vmovdqu y2, 2 * 16(mem_cd); \ + vmovdqu y3, 3 * 16(mem_cd); \ + vmovdqu y4, 4 * 16(mem_cd); \ + vmovdqu y5, 5 * 16(mem_cd); \ + vmovdqu y6, 6 * 16(mem_cd); \ + vmovdqu y7, 7 * 16(mem_cd); + +#define write_output(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem) \ + vmovdqu x0, 0 * 16(mem); \ + vmovdqu x1, 1 * 16(mem); \ + vmovdqu x2, 2 * 16(mem); \ + vmovdqu x3, 3 * 16(mem); \ + vmovdqu x4, 4 * 16(mem); \ + vmovdqu x5, 5 * 16(mem); \ + vmovdqu x6, 6 * 16(mem); \ + vmovdqu x7, 7 * 16(mem); \ + vmovdqu y0, 8 * 16(mem); \ + vmovdqu y1, 9 * 16(mem); \ + vmovdqu y2, 10 * 16(mem); \ + vmovdqu y3, 11 * 16(mem); \ + vmovdqu y4, 12 * 16(mem); \ + vmovdqu y5, 13 * 16(mem); \ + vmovdqu y6, 14 * 16(mem); \ + vmovdqu y7, 15 * 16(mem); \ + +#define aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, idx) \ + vmovdqu x0, ((idx + 0) * 16)(mem_tmp); \ + vmovdqu x1, ((idx + 1) * 16)(mem_tmp); \ + vmovdqu x2, ((idx + 2) * 16)(mem_tmp); \ + vmovdqu x3, ((idx + 3) * 16)(mem_tmp); \ + vmovdqu x4, ((idx + 4) * 16)(mem_tmp); \ + vmovdqu x5, ((idx + 5) * 16)(mem_tmp); \ + vmovdqu x6, ((idx + 6) * 16)(mem_tmp); \ + vmovdqu x7, ((idx + 7) * 16)(mem_tmp); + +#define aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, idx) \ + vmovdqu ((idx + 0) * 16)(mem_tmp), x0; \ + vmovdqu ((idx + 1) * 16)(mem_tmp), x1; \ + vmovdqu ((idx + 2) * 16)(mem_tmp), x2; \ + vmovdqu ((idx + 3) * 16)(mem_tmp), x3; \ + vmovdqu ((idx + 4) * 16)(mem_tmp), x4; \ + vmovdqu ((idx + 5) * 16)(mem_tmp), x5; \ + vmovdqu ((idx + 6) * 16)(mem_tmp), x6; \ + vmovdqu ((idx + 7) * 16)(mem_tmp), x7; + +#define aria_ark_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + t0, rk, idx, round) \ + /* AddRoundKey */ \ + vpbroadcastb ((round * 16) + idx + 3)(rk), t0; \ + vpxor t0, x0, x0; \ + vpbroadcastb ((round * 16) + idx + 2)(rk), t0; \ + vpxor t0, x1, x1; \ + vpbroadcastb ((round * 16) + idx + 1)(rk), t0; \ + vpxor t0, x2, x2; \ + vpbroadcastb ((round * 16) + idx + 0)(rk), t0; \ + vpxor t0, x3, x3; \ + vpbroadcastb ((round * 16) + idx + 7)(rk), t0; \ + vpxor t0, x4, x4; \ + vpbroadcastb ((round * 16) + idx + 6)(rk), t0; \ + vpxor t0, x5, x5; \ + vpbroadcastb ((round * 16) + idx + 5)(rk), t0; \ + vpxor t0, x6, x6; \ + vpbroadcastb ((round * 16) + idx + 4)(rk), t0; \ + vpxor t0, x7, x7; + +#define aria_sbox_8way_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + t0, t1, t2, t3, \ + t4, t5, t6, t7) \ + vpbroadcastq .Ltf_s2_bitmatrix, t0; \ + vpbroadcastq .Ltf_inv_bitmatrix, t1; \ + vpbroadcastq .Ltf_id_bitmatrix, t2; \ + vpbroadcastq .Ltf_aff_bitmatrix, t3; \ + vpbroadcastq .Ltf_x2_bitmatrix, t4; \ + vgf2p8affineinvqb $(tf_s2_const), t0, x1, x1; \ + vgf2p8affineinvqb $(tf_s2_const), t0, x5, x5; \ + vgf2p8affineqb $(tf_inv_const), t1, x2, x2; \ + vgf2p8affineqb $(tf_inv_const), t1, x6, x6; \ + vgf2p8affineinvqb $0, t2, x2, x2; \ + vgf2p8affineinvqb $0, t2, x6, x6; \ + vgf2p8affineinvqb $(tf_aff_const), t3, x0, x0; \ + vgf2p8affineinvqb $(tf_aff_const), t3, x4, x4; \ + vgf2p8affineqb $(tf_x2_const), t4, x3, x3; \ + vgf2p8affineqb $(tf_x2_const), t4, x7, x7; \ + vgf2p8affineinvqb $0, t2, x3, x3; \ + vgf2p8affineinvqb $0, t2, x7, x7 + +#define aria_sbox_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + t0, t1, t2, t3, \ + t4, t5, t6, t7) \ + vpxor t7, t7, t7; \ + vmovdqa .Linv_shift_row, t0; \ + vmovdqa .Lshift_row, t1; \ + vpbroadcastd .L0f0f0f0f, t6; \ + vmovdqa .Ltf_lo__inv_aff__and__s2, t2; \ + vmovdqa .Ltf_hi__inv_aff__and__s2, t3; \ + vmovdqa .Ltf_lo__x2__and__fwd_aff, t4; \ + vmovdqa .Ltf_hi__x2__and__fwd_aff, t5; \ + \ + vaesenclast t7, x0, x0; \ + vaesenclast t7, x4, x4; \ + vaesenclast t7, x1, x1; \ + vaesenclast t7, x5, x5; \ + vaesdeclast t7, x2, x2; \ + vaesdeclast t7, x6, x6; \ + \ + /* AES inverse shift rows */ \ + vpshufb t0, x0, x0; \ + vpshufb t0, x4, x4; \ + vpshufb t0, x1, x1; \ + vpshufb t0, x5, x5; \ + vpshufb t1, x3, x3; \ + vpshufb t1, x7, x7; \ + vpshufb t1, x2, x2; \ + vpshufb t1, x6, x6; \ + \ + /* affine transformation for S2 */ \ + filter_8bit(x1, t2, t3, t6, t0); \ + /* affine transformation for S2 */ \ + filter_8bit(x5, t2, t3, t6, t0); \ + \ + /* affine transformation for X2 */ \ + filter_8bit(x3, t4, t5, t6, t0); \ + /* affine transformation for X2 */ \ + filter_8bit(x7, t4, t5, t6, t0); \ + vaesdeclast t7, x3, x3; \ + vaesdeclast t7, x7, x7; + +#define aria_diff_m(x0, x1, x2, x3, \ + t0, t1, t2, t3) \ + /* T = rotr32(X, 8); */ \ + /* X ^= T */ \ + vpxor x0, x3, t0; \ + vpxor x1, x0, t1; \ + vpxor x2, x1, t2; \ + vpxor x3, x2, t3; \ + /* X = T ^ rotr(X, 16); */ \ + vpxor t2, x0, x0; \ + vpxor x1, t3, t3; \ + vpxor t0, x2, x2; \ + vpxor t1, x3, x1; \ + vmovdqu t3, x3; + +#define aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7) \ + /* t1 ^= t2; */ \ + vpxor y0, x4, x4; \ + vpxor y1, x5, x5; \ + vpxor y2, x6, x6; \ + vpxor y3, x7, x7; \ + \ + /* t2 ^= t3; */ \ + vpxor y4, y0, y0; \ + vpxor y5, y1, y1; \ + vpxor y6, y2, y2; \ + vpxor y7, y3, y3; \ + \ + /* t0 ^= t1; */ \ + vpxor x4, x0, x0; \ + vpxor x5, x1, x1; \ + vpxor x6, x2, x2; \ + vpxor x7, x3, x3; \ + \ + /* t3 ^= t1; */ \ + vpxor x4, y4, y4; \ + vpxor x5, y5, y5; \ + vpxor x6, y6, y6; \ + vpxor x7, y7, y7; \ + \ + /* t2 ^= t0; */ \ + vpxor x0, y0, y0; \ + vpxor x1, y1, y1; \ + vpxor x2, y2, y2; \ + vpxor x3, y3, y3; \ + \ + /* t1 ^= t2; */ \ + vpxor y0, x4, x4; \ + vpxor y1, x5, x5; \ + vpxor y2, x6, x6; \ + vpxor y3, x7, x7; + +#define aria_fe(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T3 = ABCD -> BADC \ + * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \ + * T0 = ABCD -> CDAB \ + * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \ + * T1 = ABCD -> DCBA \ + * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \ + */ \ + aria_diff_word(x2, x3, x0, x1, \ + x7, x6, x5, x4, \ + y0, y1, y2, y3, \ + y5, y4, y7, y6); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_fo(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T1 = ABCD -> BADC \ + * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \ + * T2 = ABCD -> CDAB \ + * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \ + * T3 = ABCD -> DCBA \ + * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \ + */ \ + aria_diff_word(x0, x1, x2, x3, \ + x5, x4, x7, x6, \ + y2, y3, y0, y1, \ + y7, y6, y5, y4); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_ff(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round, last_round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, last_round); \ + \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, last_round); \ + \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); + +#define aria_fe_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T3 = ABCD -> BADC \ + * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \ + * T0 = ABCD -> CDAB \ + * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \ + * T1 = ABCD -> DCBA \ + * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \ + */ \ + aria_diff_word(x2, x3, x0, x1, \ + x7, x6, x5, x4, \ + y0, y1, y2, y3, \ + y5, y4, y7, y6); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_fo_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T1 = ABCD -> BADC \ + * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \ + * T2 = ABCD -> CDAB \ + * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \ + * T3 = ABCD -> DCBA \ + * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \ + */ \ + aria_diff_word(x0, x1, x2, x3, \ + x5, x4, x7, x6, \ + y2, y3, y0, y1, \ + y7, y6, y5, y4); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_ff_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round, last_round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, last_round); \ + \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, last_round); \ + \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); + +/* NB: section is mergeable, all elements must be aligned 16-byte blocks */ +.section .rodata.cst16, "aM", @progbits, 16 +.align 16 + +#define SHUFB_BYTES(idx) \ + 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) + +.Lshufb_16x16b: + .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3); +/* For isolating SubBytes from AESENCLAST, inverse shift row */ +.Linv_shift_row: + .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b + .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 +.Lshift_row: + .byte 0x00, 0x05, 0x0a, 0x0f, 0x04, 0x09, 0x0e, 0x03 + .byte 0x08, 0x0d, 0x02, 0x07, 0x0c, 0x01, 0x06, 0x0b +/* For CTR-mode IV byteswap */ +.Lbswap128_mask: + .byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08 + .byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 + +/* AES inverse affine and S2 combined: + * 1 1 0 0 0 0 0 1 x0 0 + * 0 1 0 0 1 0 0 0 x1 0 + * 1 1 0 0 1 1 1 1 x2 0 + * 0 1 1 0 1 0 0 1 x3 1 + * 0 1 0 0 1 1 0 0 * x4 + 0 + * 0 1 0 1 1 0 0 0 x5 0 + * 0 0 0 0 0 1 0 1 x6 0 + * 1 1 1 0 0 1 1 1 x7 1 + */ +.Ltf_lo__inv_aff__and__s2: + .octa 0x92172DA81A9FA520B2370D883ABF8500 +.Ltf_hi__inv_aff__and__s2: + .octa 0x2B15FFC1AF917B45E6D8320C625CB688 + +/* X2 and AES forward affine combined: + * 1 0 1 1 0 0 0 1 x0 0 + * 0 1 1 1 1 0 1 1 x1 0 + * 0 0 0 1 1 0 1 0 x2 1 + * 0 1 0 0 0 1 0 0 x3 0 + * 0 0 1 1 1 0 1 1 * x4 + 0 + * 0 1 0 0 1 0 0 0 x5 0 + * 1 1 0 1 0 0 1 1 x6 0 + * 0 1 0 0 1 0 1 0 x7 0 + */ +.Ltf_lo__x2__and__fwd_aff: + .octa 0xEFAE0544FCBD1657B8F95213ABEA4100 +.Ltf_hi__x2__and__fwd_aff: + .octa 0x3F893781E95FE1576CDA64D2BA0CB204 + +.section .rodata.cst8, "aM", @progbits, 8 +.align 8 +/* AES affine: */ +#define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0) +.Ltf_aff_bitmatrix: + .quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1), + BV8(1, 1, 0, 0, 0, 1, 1, 1), + BV8(1, 1, 1, 0, 0, 0, 1, 1), + BV8(1, 1, 1, 1, 0, 0, 0, 1), + BV8(1, 1, 1, 1, 1, 0, 0, 0), + BV8(0, 1, 1, 1, 1, 1, 0, 0), + BV8(0, 0, 1, 1, 1, 1, 1, 0), + BV8(0, 0, 0, 1, 1, 1, 1, 1)) + +/* AES inverse affine: */ +#define tf_inv_const BV8(1, 0, 1, 0, 0, 0, 0, 0) +.Ltf_inv_bitmatrix: + .quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1), + BV8(1, 0, 0, 1, 0, 0, 1, 0), + BV8(0, 1, 0, 0, 1, 0, 0, 1), + BV8(1, 0, 1, 0, 0, 1, 0, 0), + BV8(0, 1, 0, 1, 0, 0, 1, 0), + BV8(0, 0, 1, 0, 1, 0, 0, 1), + BV8(1, 0, 0, 1, 0, 1, 0, 0), + BV8(0, 1, 0, 0, 1, 0, 1, 0)) + +/* S2: */ +#define tf_s2_const BV8(0, 1, 0, 0, 0, 1, 1, 1) +.Ltf_s2_bitmatrix: + .quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1), + BV8(0, 0, 1, 1, 1, 1, 1, 1), + BV8(1, 1, 1, 0, 1, 1, 0, 1), + BV8(1, 1, 0, 0, 0, 0, 1, 1), + BV8(0, 1, 0, 0, 0, 0, 1, 1), + BV8(1, 1, 0, 0, 1, 1, 1, 0), + BV8(0, 1, 1, 0, 0, 0, 1, 1), + BV8(1, 1, 1, 1, 0, 1, 1, 0)) + +/* X2: */ +#define tf_x2_const BV8(0, 0, 1, 1, 0, 1, 0, 0) +.Ltf_x2_bitmatrix: + .quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0), + BV8(0, 0, 1, 0, 0, 1, 1, 0), + BV8(0, 0, 0, 0, 1, 0, 1, 0), + BV8(1, 1, 1, 0, 0, 0, 1, 1), + BV8(1, 1, 1, 0, 1, 1, 0, 0), + BV8(0, 1, 1, 0, 1, 0, 1, 1), + BV8(1, 0, 1, 1, 1, 1, 0, 1), + BV8(1, 0, 0, 1, 0, 0, 1, 1)) + +/* Identity matrix: */ +.Ltf_id_bitmatrix: + .quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0), + BV8(0, 1, 0, 0, 0, 0, 0, 0), + BV8(0, 0, 1, 0, 0, 0, 0, 0), + BV8(0, 0, 0, 1, 0, 0, 0, 0), + BV8(0, 0, 0, 0, 1, 0, 0, 0), + BV8(0, 0, 0, 0, 0, 1, 0, 0), + BV8(0, 0, 0, 0, 0, 0, 1, 0), + BV8(0, 0, 0, 0, 0, 0, 0, 1)) + +/* 4-bit mask */ +.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4 +.align 4 +.L0f0f0f0f: + .long 0x0f0f0f0f + +.text + +SYM_FUNC_START_LOCAL(__aria_aesni_avx_crypt_16way) + /* input: + * %r9: rk + * %rsi: dst + * %rdx: src + * %xmm0..%xmm15: 16 byte-sliced blocks + */ + + FRAME_BEGIN + + movq %rsi, %rax; + leaq 8 * 16(%rax), %r8; + + inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r8); + aria_fo(%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 0); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 1); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 2); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 3); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 4); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 5); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 6); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 7); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 8); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 9); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 10); + cmpl $12, rounds(CTX); + jne .Laria_192; + aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11, 12); + jmp .Laria_end; +.Laria_192: + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 12); + cmpl $14, rounds(CTX); + jne .Laria_256; + aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13, 14); + jmp .Laria_end; +.Laria_256: + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 14); + aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 15, 16); +.Laria_end: + debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4, + %xmm9, %xmm13, %xmm0, %xmm5, + %xmm10, %xmm14, %xmm3, %xmm6, + %xmm11, %xmm15, %xmm2, %xmm7, + (%rax), (%r8)); + + FRAME_END + RET; +SYM_FUNC_END(__aria_aesni_avx_crypt_16way) + +SYM_FUNC_START(aria_aesni_avx_encrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq enc_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_encrypt_16way) + +SYM_FUNC_START(aria_aesni_avx_decrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq dec_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_decrypt_16way) + +SYM_FUNC_START_LOCAL(__aria_aesni_avx_ctr_gen_keystream_16way) + /* input: + * %rdi: ctx + * %rsi: dst + * %rdx: src + * %rcx: keystream + * %r8: iv (big endian, 128bit) + */ + + FRAME_BEGIN + /* load IV and byteswap */ + vmovdqu (%r8), %xmm8; + + vmovdqa .Lbswap128_mask (%rip), %xmm1; + vpshufb %xmm1, %xmm8, %xmm3; /* be => le */ + + vpcmpeqd %xmm0, %xmm0, %xmm0; + vpsrldq $8, %xmm0, %xmm0; /* low: -1, high: 0 */ + + /* construct IVs */ + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm9; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm10; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm11; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm12; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm13; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm14; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm15; + vmovdqu %xmm8, (0 * 16)(%rcx); + vmovdqu %xmm9, (1 * 16)(%rcx); + vmovdqu %xmm10, (2 * 16)(%rcx); + vmovdqu %xmm11, (3 * 16)(%rcx); + vmovdqu %xmm12, (4 * 16)(%rcx); + vmovdqu %xmm13, (5 * 16)(%rcx); + vmovdqu %xmm14, (6 * 16)(%rcx); + vmovdqu %xmm15, (7 * 16)(%rcx); + + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm8; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm9; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm10; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm11; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm12; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm13; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm14; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm15; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm4; + vmovdqu %xmm4, (%r8); + + vmovdqu (0 * 16)(%rcx), %xmm0; + vmovdqu (1 * 16)(%rcx), %xmm1; + vmovdqu (2 * 16)(%rcx), %xmm2; + vmovdqu (3 * 16)(%rcx), %xmm3; + vmovdqu (4 * 16)(%rcx), %xmm4; + vmovdqu (5 * 16)(%rcx), %xmm5; + vmovdqu (6 * 16)(%rcx), %xmm6; + vmovdqu (7 * 16)(%rcx), %xmm7; + + FRAME_END + RET; +SYM_FUNC_END(__aria_aesni_avx_ctr_gen_keystream_16way) + +SYM_FUNC_START(aria_aesni_avx_ctr_crypt_16way) + /* input: + * %rdi: ctx + * %rsi: dst + * %rdx: src + * %rcx: keystream + * %r8: iv (big endian, 128bit) + */ + FRAME_BEGIN + + call __aria_aesni_avx_ctr_gen_keystream_16way; + + leaq (%rsi), %r10; + leaq (%rdx), %r11; + leaq (%rcx), %rsi; + leaq (%rcx), %rdx; + leaq enc_key(CTX), %r9; + + call __aria_aesni_avx_crypt_16way; + + vpxor (0 * 16)(%r11), %xmm1, %xmm1; + vpxor (1 * 16)(%r11), %xmm0, %xmm0; + vpxor (2 * 16)(%r11), %xmm3, %xmm3; + vpxor (3 * 16)(%r11), %xmm2, %xmm2; + vpxor (4 * 16)(%r11), %xmm4, %xmm4; + vpxor (5 * 16)(%r11), %xmm5, %xmm5; + vpxor (6 * 16)(%r11), %xmm6, %xmm6; + vpxor (7 * 16)(%r11), %xmm7, %xmm7; + vpxor (8 * 16)(%r11), %xmm8, %xmm8; + vpxor (9 * 16)(%r11), %xmm9, %xmm9; + vpxor (10 * 16)(%r11), %xmm10, %xmm10; + vpxor (11 * 16)(%r11), %xmm11, %xmm11; + vpxor (12 * 16)(%r11), %xmm12, %xmm12; + vpxor (13 * 16)(%r11), %xmm13, %xmm13; + vpxor (14 * 16)(%r11), %xmm14, %xmm14; + vpxor (15 * 16)(%r11), %xmm15, %xmm15; + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %r10); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_ctr_crypt_16way) + +SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way) + /* input: + * %r9: rk + * %rsi: dst + * %rdx: src + * %xmm0..%xmm15: 16 byte-sliced blocks + */ + + FRAME_BEGIN + + movq %rsi, %rax; + leaq 8 * 16(%rax), %r8; + + inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r8); + aria_fo_gfni(%xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 0); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 1); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 2); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 3); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 4); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 5); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 6); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 7); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 8); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 9); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 10); + cmpl $12, rounds(CTX); + jne .Laria_gfni_192; + aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11, 12); + jmp .Laria_gfni_end; +.Laria_gfni_192: + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 12); + cmpl $14, rounds(CTX); + jne .Laria_gfni_256; + aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13, 14); + jmp .Laria_gfni_end; +.Laria_gfni_256: + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 14); + aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 15, 16); +.Laria_gfni_end: + debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4, + %xmm9, %xmm13, %xmm0, %xmm5, + %xmm10, %xmm14, %xmm3, %xmm6, + %xmm11, %xmm15, %xmm2, %xmm7, + (%rax), (%r8)); + + FRAME_END + RET; +SYM_FUNC_END(__aria_aesni_avx_gfni_crypt_16way) + +SYM_FUNC_START(aria_aesni_avx_gfni_encrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq enc_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_gfni_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_gfni_encrypt_16way) + +SYM_FUNC_START(aria_aesni_avx_gfni_decrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq dec_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_gfni_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_gfni_decrypt_16way) + +SYM_FUNC_START(aria_aesni_avx_gfni_ctr_crypt_16way) + /* input: + * %rdi: ctx + * %rsi: dst + * %rdx: src + * %rcx: keystream + * %r8: iv (big endian, 128bit) + */ + FRAME_BEGIN + + call __aria_aesni_avx_ctr_gen_keystream_16way + + leaq (%rsi), %r10; + leaq (%rdx), %r11; + leaq (%rcx), %rsi; + leaq (%rcx), %rdx; + leaq enc_key(CTX), %r9; + + call __aria_aesni_avx_gfni_crypt_16way; + + vpxor (0 * 16)(%r11), %xmm1, %xmm1; + vpxor (1 * 16)(%r11), %xmm0, %xmm0; + vpxor (2 * 16)(%r11), %xmm3, %xmm3; + vpxor (3 * 16)(%r11), %xmm2, %xmm2; + vpxor (4 * 16)(%r11), %xmm4, %xmm4; + vpxor (5 * 16)(%r11), %xmm5, %xmm5; + vpxor (6 * 16)(%r11), %xmm6, %xmm6; + vpxor (7 * 16)(%r11), %xmm7, %xmm7; + vpxor (8 * 16)(%r11), %xmm8, %xmm8; + vpxor (9 * 16)(%r11), %xmm9, %xmm9; + vpxor (10 * 16)(%r11), %xmm10, %xmm10; + vpxor (11 * 16)(%r11), %xmm11, %xmm11; + vpxor (12 * 16)(%r11), %xmm12, %xmm12; + vpxor (13 * 16)(%r11), %xmm13, %xmm13; + vpxor (14 * 16)(%r11), %xmm14, %xmm14; + vpxor (15 * 16)(%r11), %xmm15, %xmm15; + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %r10); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_gfni_ctr_crypt_16way) diff --git a/arch/x86/crypto/aria-avx.h b/arch/x86/crypto/aria-avx.h new file mode 100644 index 000000000000..01e9a01dc157 --- /dev/null +++ b/arch/x86/crypto/aria-avx.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef ASM_X86_ARIA_AVX_H +#define ASM_X86_ARIA_AVX_H + +#include + +#define ARIA_AESNI_PARALLEL_BLOCKS 16 +#define ARIA_AESNI_PARALLEL_BLOCK_SIZE (ARIA_BLOCK_SIZE * 16) + +struct aria_avx_ops { + void (*aria_encrypt_16way)(const void *ctx, u8 *dst, const u8 *src); + void (*aria_decrypt_16way)(const void *ctx, u8 *dst, const u8 *src); + void (*aria_ctr_crypt_16way)(const void *ctx, u8 *dst, const u8 *src, + u8 *keystream, u8 *iv); +}; +#endif diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c new file mode 100644 index 000000000000..c561ea4fefa5 --- /dev/null +++ b/arch/x86/crypto/aria_aesni_avx_glue.c @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Glue Code for the AVX/AES-NI/GFNI assembler implementation of the ARIA Cipher + * + * Copyright (c) 2022 Taehee Yoo + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ecb_cbc_helpers.h" +#include "aria-avx.h" + +asmlinkage void aria_aesni_avx_encrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_decrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_ctr_crypt_16way(const void *ctx, u8 *dst, + const u8 *src, + u8 *keystream, u8 *iv); +asmlinkage void aria_aesni_avx_gfni_encrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_gfni_decrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_gfni_ctr_crypt_16way(const void *ctx, u8 *dst, + const u8 *src, + u8 *keystream, u8 *iv); + +static struct aria_avx_ops aria_ops; + +static int ecb_do_encrypt(struct skcipher_request *req, const u32 *rkey) +{ + ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_encrypt_16way); + ECB_BLOCK(1, aria_encrypt); + ECB_WALK_END(); +} + +static int ecb_do_decrypt(struct skcipher_request *req, const u32 *rkey) +{ + ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_decrypt_16way); + ECB_BLOCK(1, aria_decrypt); + ECB_WALK_END(); +} + +static int aria_avx_ecb_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); + + return ecb_do_encrypt(req, ctx->enc_key[0]); +} + +static int aria_avx_ecb_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); + + return ecb_do_decrypt(req, ctx->dec_key[0]); +} + +static int aria_avx_set_key(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + return aria_set_key(&tfm->base, key, keylen); +} + +static int aria_avx_ctr_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) > 0) { + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + + while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) { + u8 keystream[ARIA_AESNI_PARALLEL_BLOCK_SIZE]; + + kernel_fpu_begin(); + aria_ops.aria_ctr_crypt_16way(ctx, dst, src, keystream, + walk.iv); + kernel_fpu_end(); + dst += ARIA_AESNI_PARALLEL_BLOCK_SIZE; + src += ARIA_AESNI_PARALLEL_BLOCK_SIZE; + nbytes -= ARIA_AESNI_PARALLEL_BLOCK_SIZE; + } + + while (nbytes >= ARIA_BLOCK_SIZE) { + u8 keystream[ARIA_BLOCK_SIZE]; + + memcpy(keystream, walk.iv, ARIA_BLOCK_SIZE); + crypto_inc(walk.iv, ARIA_BLOCK_SIZE); + + aria_encrypt(ctx, keystream, keystream); + + crypto_xor_cpy(dst, src, keystream, ARIA_BLOCK_SIZE); + dst += ARIA_BLOCK_SIZE; + src += ARIA_BLOCK_SIZE; + nbytes -= ARIA_BLOCK_SIZE; + } + + if (walk.nbytes == walk.total && nbytes > 0) { + u8 keystream[ARIA_BLOCK_SIZE]; + + memcpy(keystream, walk.iv, ARIA_BLOCK_SIZE); + crypto_inc(walk.iv, ARIA_BLOCK_SIZE); + + aria_encrypt(ctx, keystream, keystream); + + crypto_xor_cpy(dst, src, keystream, nbytes); + dst += nbytes; + src += nbytes; + nbytes = 0; + } + err = skcipher_walk_done(&walk, nbytes); + } + + return err; +} + +static struct skcipher_alg aria_algs[] = { + { + .base.cra_name = "__ecb(aria)", + .base.cra_driver_name = "__ecb-aria-avx", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = ARIA_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aria_ctx), + .base.cra_module = THIS_MODULE, + .min_keysize = ARIA_MIN_KEY_SIZE, + .max_keysize = ARIA_MAX_KEY_SIZE, + .setkey = aria_avx_set_key, + .encrypt = aria_avx_ecb_encrypt, + .decrypt = aria_avx_ecb_decrypt, + }, { + .base.cra_name = "__ctr(aria)", + .base.cra_driver_name = "__ctr-aria-avx", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aria_ctx), + .base.cra_module = THIS_MODULE, + .min_keysize = ARIA_MIN_KEY_SIZE, + .max_keysize = ARIA_MAX_KEY_SIZE, + .ivsize = ARIA_BLOCK_SIZE, + .chunksize = ARIA_BLOCK_SIZE, + .walksize = 16 * ARIA_BLOCK_SIZE, + .setkey = aria_avx_set_key, + .encrypt = aria_avx_ctr_encrypt, + .decrypt = aria_avx_ctr_encrypt, + } +}; + +static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; + +static int __init aria_avx_init(void) +{ + const char *feature_name; + + if (!boot_cpu_has(X86_FEATURE_AVX) || + !boot_cpu_has(X86_FEATURE_AES) || + !boot_cpu_has(X86_FEATURE_OSXSAVE)) { + pr_info("AVX or AES-NI instructions are not detected.\n"); + return -ENODEV; + } + + if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, + &feature_name)) { + pr_info("CPU feature '%s' is not supported.\n", feature_name); + return -ENODEV; + } + + if (boot_cpu_has(X86_FEATURE_GFNI)) { + aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; + aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; + aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; + } else { + aria_ops.aria_encrypt_16way = aria_aesni_avx_encrypt_16way; + aria_ops.aria_decrypt_16way = aria_aesni_avx_decrypt_16way; + aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way; + } + + return simd_register_skciphers_compat(aria_algs, + ARRAY_SIZE(aria_algs), + aria_simd_algs); +} + +static void __exit aria_avx_exit(void) +{ + simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), + aria_simd_algs); +} + +module_init(aria_avx_init); +module_exit(aria_avx_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Taehee Yoo "); +MODULE_DESCRIPTION("ARIA Cipher Algorithm, AVX/AES-NI/GFNI optimized"); +MODULE_ALIAS_CRYPTO("aria"); +MODULE_ALIAS_CRYPTO("aria-aesni-avx"); -- cgit From c4b1ce72b5c9f7d5772b2f2d4efa25ef0e6fb576 Mon Sep 17 00:00:00 2001 From: Taehee Yoo Date: Fri, 16 Sep 2022 12:57:36 +0000 Subject: crypto: tcrypt - add async speed test for aria cipher In order to test for the performance of aria-avx implementation, it needs an async speed test. So, it adds async speed tests to the tcrypt. Signed-off-by: Taehee Yoo Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index e85f623c3c54..a82679b576bb 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -2205,6 +2205,13 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) NULL, 0, 16, 8, speed_template_16_24_32); break; + case 229: + test_mb_aead_speed("gcm(aria)", ENCRYPT, sec, NULL, 0, 16, 8, + speed_template_16, num_mb); + test_mb_aead_speed("gcm(aria)", DECRYPT, sec, NULL, 0, 16, 8, + speed_template_16, num_mb); + break; + case 300: if (alg) { test_hash_speed(alg, sec, generic_hash_speed_template); @@ -2625,6 +2632,17 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16); break; + case 519: + test_acipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_acipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_acipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_acipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + break; + case 600: test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32, num_mb); @@ -2836,6 +2854,18 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0, speed_template_8_32, num_mb); break; + + case 610: + test_mb_skcipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + break; + } return ret; -- cgit From b21dc631222bbe61c372dfb19373fb9d83718314 Mon Sep 17 00:00:00 2001 From: Liu Shixin Date: Fri, 16 Sep 2022 22:13:40 +0800 Subject: crypto: sun4i-ss - use DEFINE_SHOW_ATTRIBUTE to simplify sun4i_ss_debugfs Use DEFINE_SHOW_ATTRIBUTE helper macro to simplify the code. Signed-off-by: Liu Shixin Acked-by: Corentin Labbe Tested-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index 44b8fc4b786d..006e40133c28 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -235,7 +235,7 @@ static struct sun4i_ss_alg_template ss_algs[] = { #endif }; -static int sun4i_ss_dbgfs_read(struct seq_file *seq, void *v) +static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v) { unsigned int i; @@ -266,19 +266,7 @@ static int sun4i_ss_dbgfs_read(struct seq_file *seq, void *v) } return 0; } - -static int sun4i_ss_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, sun4i_ss_dbgfs_read, inode->i_private); -} - -static const struct file_operations sun4i_ss_debugfs_fops = { - .owner = THIS_MODULE, - .open = sun4i_ss_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(sun4i_ss_debugfs); /* * Power management strategy: The device is suspended unless a TFM exists for -- cgit From f5b657e5dbf830cfcb19b588b784b8190a5164a0 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 17 Sep 2022 10:03:45 +0000 Subject: crypto: hisilicon/qm - fix the qos value initialization The default qos value is not initialized when sriov is repeatedly enabled and disabled. So add the vf qos value initialized in the sriov enable process. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 30fdf0673f00..8b387de69d22 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4801,6 +4801,14 @@ void hisi_qm_debug_regs_clear(struct hisi_qm *qm) } EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); +static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func) +{ + int i; + + for (i = 1; i <= total_func; i++) + qm->factor[i].func_qos = QM_QOS_MAX_VAL; +} + /** * hisi_qm_sriov_enable() - enable virtual functions * @pdev: the PCIe device @@ -4834,6 +4842,10 @@ int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) } num_vfs = max_vfs; + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + hisi_qm_init_vf_qos(qm, num_vfs); + ret = qm_vf_q_assign(qm, num_vfs); if (ret) { pci_err(pdev, "Can't assign queues for VF!\n"); @@ -4869,7 +4881,6 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); - int total_vfs = pci_sriov_get_totalvfs(qm->pdev); int ret; if (pci_vfs_assigned(pdev)) { @@ -4884,9 +4895,6 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) } pci_disable_sriov(pdev); - /* clear vf function shaper configure array */ - if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) - memset(qm->factor + 1, 0, sizeof(struct qm_shaper_factor) * total_vfs); ret = qm_clear_vft_config(qm); if (ret) @@ -6297,7 +6305,7 @@ err_init_qp_mem: static int hisi_qm_memory_init(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - int ret, total_func, i; + int ret, total_func; size_t off = 0; if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { @@ -6306,8 +6314,8 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) if (!qm->factor) return -ENOMEM; - for (i = 0; i < total_func; i++) - qm->factor[i].func_qos = QM_QOS_MAX_VAL; + /* Only the PF value needs to be initialized */ + qm->factor[0].func_qos = QM_QOS_MAX_VAL; } #define QM_INIT_BUF(qm, type, num) do { \ -- cgit From f78f6f0bf34fd85c17ebcb31d645536112aa25d3 Mon Sep 17 00:00:00 2001 From: Neal Liu Date: Mon, 19 Sep 2022 14:37:05 +0800 Subject: crypto: aspeed - fix build error when only CRYPTO_DEV_ASPEED is enabled Fix build error within the following configs setting: - CONFIG_CRYPTO_DEV_ASPEED=y - CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH is not set - CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO is not set Error messages: make[4]: *** No rule to make target 'drivers/crypto/aspeed/aspeed_crypto.o' , needed by 'drivers/crypto/aspeed/built-in.a'. make[4]: Target '__build' not remade because of errors. Reported-by: kernel test robot Signed-off-by: Neal Liu Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/Kconfig | 3 +-- drivers/crypto/aspeed/Makefile | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/aspeed/Kconfig b/drivers/crypto/aspeed/Kconfig index ae3eb0eb57f6..ae2710ae8d8f 100644 --- a/drivers/crypto/aspeed/Kconfig +++ b/drivers/crypto/aspeed/Kconfig @@ -1,6 +1,7 @@ config CRYPTO_DEV_ASPEED tristate "Support for Aspeed cryptographic engine driver" depends on ARCH_ASPEED || COMPILE_TEST + select CRYPTO_ENGINE help Hash and Crypto Engine (HACE) is designed to accelerate the throughput of hash data digest, encryption and decryption. @@ -20,7 +21,6 @@ config CRYPTO_DEV_ASPEED_DEBUG config CRYPTO_DEV_ASPEED_HACE_HASH bool "Enable Aspeed Hash & Crypto Engine (HACE) hash" depends on CRYPTO_DEV_ASPEED - select CRYPTO_ENGINE select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 @@ -34,7 +34,6 @@ config CRYPTO_DEV_ASPEED_HACE_HASH config CRYPTO_DEV_ASPEED_HACE_CRYPTO bool "Enable Aspeed Hash & Crypto Engine (HACE) crypto" depends on CRYPTO_DEV_ASPEED - select CRYPTO_ENGINE select CRYPTO_AES select CRYPTO_DES select CRYPTO_ECB diff --git a/drivers/crypto/aspeed/Makefile b/drivers/crypto/aspeed/Makefile index 3be78cec0ecb..a0ed40ddaad1 100644 --- a/drivers/crypto/aspeed/Makefile +++ b/drivers/crypto/aspeed/Makefile @@ -1,6 +1,7 @@ -hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace.o aspeed-hace-hash.o -hace-crypto-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) := aspeed-hace.o aspeed-hace-crypto.o +hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace-hash.o +hace-crypto-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) := aspeed-hace-crypto.o obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed_crypto.o -aspeed_crypto-objs := $(hace-hash-y) \ +aspeed_crypto-objs := aspeed-hace.o \ + $(hace-hash-y) \ $(hace-crypto-y) -- cgit From caca37cf6c749ff0303f68418cfe7b757a4e0697 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 19 Sep 2022 09:43:19 +0300 Subject: crypto: marvell/octeontx - prevent integer overflows The "code_length" value comes from the firmware file. If your firmware is untrusted realistically there is probably very little you can do to protect yourself. Still we try to limit the damage as much as possible. Also Smatch marks any data read from the filesystem as untrusted and prints warnings if it not capped correctly. The "code_length * 2" can overflow. The round_up(ucode_size, 16) + sizeof() expression can overflow too. Prevent these overflows. Fixes: d9110b0b01ff ("crypto: marvell - add support for OCTEON TX CPT engine") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index 23c6edc70914..df9c2b8747e6 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -286,6 +286,7 @@ static int process_tar_file(struct device *dev, struct tar_ucode_info_t *tar_info; struct otx_cpt_ucode_hdr *ucode_hdr; int ucode_type, ucode_size; + unsigned int code_length; /* * If size is less than microcode header size then don't report @@ -303,7 +304,13 @@ static int process_tar_file(struct device *dev, if (get_ucode_type(ucode_hdr, &ucode_type)) return 0; - ucode_size = ntohl(ucode_hdr->code_length) * 2; + code_length = ntohl(ucode_hdr->code_length); + if (code_length >= INT_MAX / 2) { + dev_err(dev, "Invalid code_length %u\n", code_length); + return -EINVAL; + } + + ucode_size = code_length * 2; if (!ucode_size || (size < round_up(ucode_size, 16) + sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) { dev_err(dev, "Ucode %s invalid size\n", filename); @@ -886,6 +893,7 @@ static int ucode_load(struct device *dev, struct otx_cpt_ucode *ucode, { struct otx_cpt_ucode_hdr *ucode_hdr; const struct firmware *fw; + unsigned int code_length; int ret; set_ucode_filename(ucode, ucode_filename); @@ -896,7 +904,13 @@ static int ucode_load(struct device *dev, struct otx_cpt_ucode *ucode, ucode_hdr = (struct otx_cpt_ucode_hdr *) fw->data; memcpy(ucode->ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ); ucode->ver_num = ucode_hdr->ver_num; - ucode->size = ntohl(ucode_hdr->code_length) * 2; + code_length = ntohl(ucode_hdr->code_length); + if (code_length >= INT_MAX / 2) { + dev_err(dev, "Ucode invalid code_length %u\n", code_length); + ret = -EINVAL; + goto release_fw; + } + ucode->size = code_length * 2; if (!ucode->size || (fw->size < round_up(ucode->size, 16) + sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) { dev_err(dev, "Ucode %s invalid size\n", ucode_filename); -- cgit From 2526d6bf27d15054bb0778b2f7bc6625fd934905 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 19 Sep 2022 09:43:27 +0300 Subject: crypto: cavium - prevent integer overflow loading firmware The "code_length" value comes from the firmware file. If your firmware is untrusted realistically there is probably very little you can do to protect yourself. Still we try to limit the damage as much as possible. Also Smatch marks any data read from the filesystem as untrusted and prints warnings if it not capped correctly. The "ntohl(ucode->code_length) * 2" multiplication can have an integer overflow. Fixes: 9e2c7d99941d ("crypto: cavium - Add Support for Octeon-tx CPT Engine") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptpf_main.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 8c32d0eb8fcf..6872ac344001 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -253,6 +253,7 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) const struct firmware *fw_entry; struct device *dev = &cpt->pdev->dev; struct ucode_header *ucode; + unsigned int code_length; struct microcode *mcode; int j, ret = 0; @@ -263,11 +264,12 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) ucode = (struct ucode_header *)fw_entry->data; mcode = &cpt->mcode[cpt->next_mc_idx]; memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ); - mcode->code_size = ntohl(ucode->code_length) * 2; - if (!mcode->code_size) { + code_length = ntohl(ucode->code_length); + if (code_length == 0 || code_length >= INT_MAX / 2) { ret = -EINVAL; goto fw_release; } + mcode->code_size = code_length * 2; mcode->is_ae = is_ae; mcode->core_mask = 0ULL; -- cgit From 4a209078656c3ada49c81d69c4b556be2dda1310 Mon Sep 17 00:00:00 2001 From: lei he Date: Mon, 19 Sep 2022 15:51:58 +0800 Subject: crypto: virtio - fix memory-leak Fix memory-leak for virtio-crypto akcipher request, this problem is introduced by 59ca6c93387d3(virtio-crypto: implement RSA algorithm). The leak can be reproduced and tested with the following script inside virtual machine: #!/bin/bash LOOP_TIMES=10000 # required module: pkcs8_key_parser, virtio_crypto modprobe pkcs8_key_parser # if CONFIG_PKCS8_PRIVATE_KEY_PARSER=m modprobe virtio_crypto # if CONFIG_CRYPTO_DEV_VIRTIO=m rm -rf /tmp/data dd if=/dev/random of=/tmp/data count=1 bs=230 # generate private key and self-signed cert openssl req -nodes -x509 -newkey rsa:2048 -keyout key.pem \ -outform der -out cert.der \ -subj "/C=CN/ST=GD/L=SZ/O=vihoo/OU=dev/CN=always.com/emailAddress=yy@always.com" # convert private key from pem to der openssl pkcs8 -in key.pem -topk8 -nocrypt -outform DER -out key.der # add key PRIV_KEY_ID=`cat key.der | keyctl padd asymmetric test_priv_key @s` echo "priv key id = "$PRIV_KEY_ID PUB_KEY_ID=`cat cert.der | keyctl padd asymmetric test_pub_key @s` echo "pub key id = "$PUB_KEY_ID # query key keyctl pkey_query $PRIV_KEY_ID 0 keyctl pkey_query $PUB_KEY_ID 0 # here we only run pkey_encrypt becasuse it is the fastest interface function bench_pub() { keyctl pkey_encrypt $PUB_KEY_ID 0 /tmp/data enc=pkcs1 >/tmp/enc.pub } # do bench_pub in loop to obtain the memory leak for (( i = 0; i < ${LOOP_TIMES}; ++i )); do bench_pub done Signed-off-by: lei he Acked-by: Michael S. Tsirkin Reviewed-by: Gonglei Signed-off-by: Herbert Xu --- drivers/crypto/virtio/virtio_crypto_akcipher_algs.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 2a60d0525cde..168195672e2e 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -56,6 +56,10 @@ static void virtio_crypto_akcipher_finalize_req( struct virtio_crypto_akcipher_request *vc_akcipher_req, struct akcipher_request *req, int err) { + kfree(vc_akcipher_req->src_buf); + kfree(vc_akcipher_req->dst_buf); + vc_akcipher_req->src_buf = NULL; + vc_akcipher_req->dst_buf = NULL; virtcrypto_clear_request(&vc_akcipher_req->base); crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err); -- cgit From 70513e1d65599f39aba4fa6594546f7c81fa59f4 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Tue, 20 Sep 2022 11:21:18 +0800 Subject: crypto: aspeed - Fix check for platform_get_irq() errors The platform_get_irq() function returns negative on error and positive non-zero values on success. It never returns zero, but if it did then treat that as a success. Also remove redundant dev_err() print as platform_get_irq() already prints an error. Fixes: 108713a713c7 ("crypto: aspeed - Add HACE hash driver") Signed-off-by: YueHaibing Reviewed-by: Neal Liu Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/aspeed-hace.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c index 3f880aafb6a2..f7f1d33defb1 100644 --- a/drivers/crypto/aspeed/aspeed-hace.c +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -130,10 +130,8 @@ static int aspeed_hace_probe(struct platform_device *pdev) /* Get irq number and register it */ hace_dev->irq = platform_get_irq(pdev, 0); - if (!hace_dev->irq) { - dev_err(&pdev->dev, "Failed to get interrupt\n"); + if (hace_dev->irq < 0) return -ENXIO; - } rc = devm_request_irq(&pdev->dev, hace_dev->irq, aspeed_hace_irq, 0, dev_name(&pdev->dev), hace_dev); -- cgit From 6a40fb0d9db15f95b9ea884fbaedf8f82c51399f Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Tue, 20 Sep 2022 06:32:52 +0000 Subject: crypto: ccp - Remove the unneeded result variable Return the value ccp_crypto_enqueue_request() directly instead of storing it in another redundant variable. Reported-by: Zeal Robot Signed-off-by: ye xingchen Acked-by: John Allen Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-des3.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index ec97daf0fcb7..278636ed251a 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -64,7 +64,6 @@ static int ccp_des3_crypt(struct skcipher_request *req, bool encrypt) struct ccp_des3_req_ctx *rctx = skcipher_request_ctx(req); struct scatterlist *iv_sg = NULL; unsigned int iv_len = 0; - int ret; if (!ctx->u.des3.key_len) return -EINVAL; @@ -100,9 +99,7 @@ static int ccp_des3_crypt(struct skcipher_request *req, bool encrypt) rctx->cmd.u.des3.src_len = req->cryptlen; rctx->cmd.u.des3.dst = req->dst; - ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); - - return ret; + return ccp_crypto_enqueue_request(&req->base, &rctx->cmd); } static int ccp_des3_encrypt(struct skcipher_request *req) -- cgit From 0cb3c9cdf7fcc2ef75a6008223d2e3ee58ea00e1 Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Tue, 20 Sep 2022 06:47:54 +0000 Subject: crypto: octeontx2 - Remove the unneeded result variable Return the value otx2_cpt_send_mbox_msg() directly instead of storing it in another redundant variable. Reported-by: Zeal Robot Signed-off-by: ye xingchen Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c index 02cb9e44afd8..75c403f2b1d9 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -191,7 +191,6 @@ int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) struct otx2_mbox *mbox = &cptvf->pfvf_mbox; struct pci_dev *pdev = cptvf->pdev; struct mbox_msghdr *req; - int ret; req = (struct mbox_msghdr *) otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), @@ -204,7 +203,5 @@ int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) req->sig = OTX2_MBOX_REQ_SIG; req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); - ret = otx2_cpt_send_mbox_msg(mbox, pdev); - - return ret; + return otx2_cpt_send_mbox_msg(mbox, pdev); } -- cgit From 72f6e0ea2b0ecea8585f3cd4298286c85c5121e6 Mon Sep 17 00:00:00 2001 From: Adam Guerin Date: Wed, 21 Sep 2022 10:38:30 +0100 Subject: crypto: qat - add limit to linked list parsing adf_copy_key_value_data() copies data from userland to kernel, based on a linked link provided by userland. If userland provides a circular list (or just a very long one) then it would drive a long loop where allocation occurs in every loop. This could lead to low memory conditions. Adding a limit to stop endless loop. Signed-off-by: Adam Guerin Co-developed-by: Ciunas Bennett Signed-off-by: Ciunas Bennett Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_ctl_drv.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 508c18edd692..82b69e1f725b 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -16,6 +16,9 @@ #include "adf_cfg_common.h" #include "adf_cfg_user.h" +#define ADF_CFG_MAX_SECTION 512 +#define ADF_CFG_MAX_KEY_VAL 256 + #define DEVICE_NAME "qat_adf_ctl" static DEFINE_MUTEX(adf_ctl_lock); @@ -137,10 +140,11 @@ static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, struct adf_user_cfg_key_val key_val; struct adf_user_cfg_key_val *params_head; struct adf_user_cfg_section section, *section_head; + int i, j; section_head = ctl_data->config_section; - while (section_head) { + for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) { if (copy_from_user(§ion, (void __user *)section_head, sizeof(*section_head))) { dev_err(&GET_DEV(accel_dev), @@ -156,7 +160,7 @@ static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, params_head = section.params; - while (params_head) { + for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) { if (copy_from_user(&key_val, (void __user *)params_head, sizeof(key_val))) { dev_err(&GET_DEV(accel_dev), -- cgit From 4edff849f7a0abca962374512907b3e2151091f4 Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Thu, 22 Sep 2022 11:27:53 +0000 Subject: crypto: zip - remove the unneeded result variable Return the value directly instead of storing it in another redundant variable. Reported-by: Zeal Robot Signed-off-by: ye xingchen Signed-off-by: Herbert Xu --- drivers/crypto/cavium/zip/zip_crypto.c | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c index 7df71fcebe8f..1046a746d36f 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ b/drivers/crypto/cavium/zip/zip_crypto.c @@ -198,22 +198,16 @@ static int zip_decompress(const u8 *src, unsigned int slen, /* Legacy Compress framework start */ int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_ctx_init(zip_ctx, 0); - - return ret; + return zip_ctx_init(zip_ctx, 0); } int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_ctx_init(zip_ctx, 1); - - return ret; + return zip_ctx_init(zip_ctx, 1); } void zip_free_comp_ctx(struct crypto_tfm *tfm) @@ -227,24 +221,18 @@ int zip_comp_compress(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_compress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_compress(src, slen, dst, dlen, zip_ctx); } int zip_comp_decompress(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_decompress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_decompress(src, slen, dst, dlen, zip_ctx); } /* Legacy compress framework end */ /* SCOMP framework start */ @@ -298,22 +286,16 @@ int zip_scomp_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { - int ret; struct zip_kernel_ctx *zip_ctx = ctx; - ret = zip_compress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_compress(src, slen, dst, dlen, zip_ctx); } int zip_scomp_decompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { - int ret; struct zip_kernel_ctx *zip_ctx = ctx; - ret = zip_decompress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_decompress(src, slen, dst, dlen, zip_ctx); } /* SCOMP framework end */ -- cgit From b006c439d58db625318bf2207feabf847510a8a6 Mon Sep 17 00:00:00 2001 From: Dominik Brodowski Date: Thu, 22 Sep 2022 15:59:31 +0200 Subject: hwrng: core - start hwrng kthread also for untrusted sources Start the hwrng kthread even if the hwrng source has a quality setting of zero. Then, every crng reseed interval, one batch of data from this zero-quality hwrng source will be mixed into the CRNG pool. This patch is based on the assumption that data from a hwrng source will not actively harm the CRNG state. Instead, many hwrng sources (such as TPM devices), even though they are assigend a quality level of zero, actually provide some entropy, which is good enough to mix into the CRNG pool every once in a while. Cc: Herbert Xu Cc: Jason A. Donenfeld Signed-off-by: Dominik Brodowski Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index d7045dfaf16c..cc002b0c2f0c 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -52,7 +52,7 @@ MODULE_PARM_DESC(default_quality, static void drop_current_rng(void); static int hwrng_init(struct hwrng *rng); -static void hwrng_manage_rngd(struct hwrng *rng); +static int hwrng_fillfn(void *unused); static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, int wait); @@ -96,6 +96,15 @@ static int set_current_rng(struct hwrng *rng) drop_current_rng(); current_rng = rng; + /* if necessary, start hwrng thread */ + if (!hwrng_fill) { + hwrng_fill = kthread_run(hwrng_fillfn, NULL, "hwrng"); + if (IS_ERR(hwrng_fill)) { + pr_err("hwrng_fill thread creation failed\n"); + hwrng_fill = NULL; + } + } + return 0; } @@ -167,8 +176,6 @@ skip_init: rng->quality = 1024; current_quality = rng->quality; /* obsolete */ - hwrng_manage_rngd(rng); - return 0; } @@ -454,10 +461,6 @@ static ssize_t rng_quality_store(struct device *dev, /* the best available RNG may have changed */ ret = enable_best_rng(); - /* start/stop rngd if necessary */ - if (current_rng) - hwrng_manage_rngd(current_rng); - out: mutex_unlock(&rng_mutex); return ret ? ret : len; @@ -513,9 +516,6 @@ static int hwrng_fillfn(void *unused) put_rng(rng); - if (!quality) - break; - if (rc <= 0) continue; @@ -534,22 +534,6 @@ static int hwrng_fillfn(void *unused) return 0; } -static void hwrng_manage_rngd(struct hwrng *rng) -{ - if (WARN_ON(!mutex_is_locked(&rng_mutex))) - return; - - if (rng->quality == 0 && hwrng_fill) - kthread_stop(hwrng_fill); - if (rng->quality > 0 && !hwrng_fill) { - hwrng_fill = kthread_run(hwrng_fillfn, NULL, "hwrng"); - if (IS_ERR(hwrng_fill)) { - pr_err("hwrng_fill thread creation failed\n"); - hwrng_fill = NULL; - } - } -} - int hwrng_register(struct hwrng *rng) { int err = -EINVAL; -- cgit From edfc7e76d2252eebb98328b23e09336d47810569 Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Fri, 23 Sep 2022 01:29:52 +0000 Subject: crypto: marvell/octeontx - use sysfs_emit() to instead of scnprintf() Replace the open-code with sysfs_emit() to simplify the code. Signed-off-by: ye xingchen Reviewed-by: Kees Cook Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptvf_main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 36d72e35ebeb..88a41d1ca5f6 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -661,7 +661,7 @@ static ssize_t vf_type_show(struct device *dev, msg = "Invalid"; } - return scnprintf(buf, PAGE_SIZE, "%s\n", msg); + return sysfs_emit(buf, "%s\n", msg); } static ssize_t vf_engine_group_show(struct device *dev, @@ -670,7 +670,7 @@ static ssize_t vf_engine_group_show(struct device *dev, { struct otx_cptvf *cptvf = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", cptvf->vfgrp); + return sysfs_emit(buf, "%d\n", cptvf->vfgrp); } static ssize_t vf_engine_group_store(struct device *dev, @@ -706,7 +706,7 @@ static ssize_t vf_coalesc_time_wait_show(struct device *dev, { struct otx_cptvf *cptvf = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", + return sysfs_emit(buf, "%d\n", cptvf_read_vq_done_timewait(cptvf)); } @@ -716,7 +716,7 @@ static ssize_t vf_coalesc_num_wait_show(struct device *dev, { struct otx_cptvf *cptvf = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", + return sysfs_emit(buf, "%d\n", cptvf_read_vq_done_numwait(cptvf)); } -- cgit From 5e9578b29aff681ec30a3866c049305e26629a41 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Fri, 23 Sep 2022 17:08:21 +0800 Subject: crypto: bcm - Simplify obtain the name for cipher The crypto_ahash_alg_name(tfm) can obtain the name for cipher in include/crypto/hash.h, but now the function is not in use, so we use it to simplify the code, and optimize the code structure. Signed-off-by: Gaosheng Cui Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 053315e260c2..c8c799428fe0 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -1928,7 +1928,7 @@ static int ahash_enqueue(struct ahash_request *req) /* SPU2 hardware does not compute hash of zero length data */ if ((rctx->is_final == 1) && (rctx->total_todo == 0) && (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) { - alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm)); + alg_name = crypto_ahash_alg_name(tfm); flow_log("Doing %sfinal %s zero-len hash request in software\n", rctx->is_final ? "" : "non-", alg_name); err = do_shash((unsigned char *)alg_name, req->result, @@ -2029,7 +2029,7 @@ static int ahash_init(struct ahash_request *req) * supported by the hardware, we need to handle it in software * by calling synchronous hash functions. */ - alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm)); + alg_name = crypto_ahash_alg_name(tfm); hash = crypto_alloc_shash(alg_name, 0, 0); if (IS_ERR(hash)) { ret = PTR_ERR(hash); -- cgit From d126edd77148e0eacf27039a14b32d1c5ac51c6e Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Fri, 23 Sep 2022 17:08:22 +0800 Subject: crypto: aead - Remove unused inline functions from aead The aead_enqueue_request, aead_dequeue_request and aead_get_backlog are no longer used since commit 04a4616e6a21 ("crypto: omap-aes-gcm - convert to use crypto engine"), their functinoality has been replaced by crypto engine, so remove them. Signed-off-by: Gaosheng Cui Signed-off-by: Herbert Xu --- include/crypto/internal/aead.h | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/include/crypto/internal/aead.h b/include/crypto/internal/aead.h index 27b7b0224ea6..d482017f3e20 100644 --- a/include/crypto/internal/aead.h +++ b/include/crypto/internal/aead.h @@ -114,31 +114,6 @@ static inline void aead_init_queue(struct aead_queue *queue, crypto_init_queue(&queue->base, max_qlen); } -static inline int aead_enqueue_request(struct aead_queue *queue, - struct aead_request *request) -{ - return crypto_enqueue_request(&queue->base, &request->base); -} - -static inline struct aead_request *aead_dequeue_request( - struct aead_queue *queue) -{ - struct crypto_async_request *req; - - req = crypto_dequeue_request(&queue->base); - - return req ? container_of(req, struct aead_request, base) : NULL; -} - -static inline struct aead_request *aead_get_backlog(struct aead_queue *queue) -{ - struct crypto_async_request *req; - - req = crypto_get_backlog(&queue->base); - - return req ? container_of(req, struct aead_request, base) : NULL; -} - static inline unsigned int crypto_aead_alg_chunksize(struct aead_alg *alg) { return alg->chunksize; -- cgit From d438d94d6483c379935d94a7dcc2d1bf9cdf0803 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Fri, 23 Sep 2022 17:08:23 +0800 Subject: crypto: scatterwalk - Remove unused inline function scatterwalk_aligned() The scatterwalk_aligned() are no longer used since removing blkcipher and ablkcipher support, all use of it has been removed since commit d63007eb954e ("crypto: ablkcipher - remove deprecated and unused ablkcipher support"), so remove it. Signed-off-by: Gaosheng Cui Signed-off-by: Herbert Xu --- include/crypto/scatterwalk.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 6407b4b61350..ccdb05f68a75 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -46,12 +46,6 @@ static inline void scatterwalk_advance(struct scatter_walk *walk, walk->offset += nbytes; } -static inline unsigned int scatterwalk_aligned(struct scatter_walk *walk, - unsigned int alignmask) -{ - return !(walk->offset & alignmask); -} - static inline struct page *scatterwalk_page(struct scatter_walk *walk) { return sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); -- cgit From b411b1a0c8bddd470fc8c3457629ac25a168cba0 Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Fri, 23 Sep 2022 18:01:59 +0800 Subject: crypto: aspeed - Remove redundant dev_err call devm_ioremap_resource() prints error message in itself. Remove the dev_err call to avoid redundant error message. Signed-off-by: Shang XiaoJing Signed-off-by: Herbert Xu --- drivers/crypto/aspeed/aspeed-hace.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c index f7f1d33defb1..656cb92c8bb6 100644 --- a/drivers/crypto/aspeed/aspeed-hace.c +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -123,10 +123,8 @@ static int aspeed_hace_probe(struct platform_device *pdev) platform_set_drvdata(pdev, hace_dev); hace_dev->regs = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(hace_dev->regs)) { - dev_err(&pdev->dev, "Failed to map resources\n"); + if (IS_ERR(hace_dev->regs)) return PTR_ERR(hace_dev->regs); - } /* Get irq number and register it */ hace_dev->irq = platform_get_irq(pdev, 0); -- cgit