From 8123455a648581ed08b1740e4fc33863eb353687 Mon Sep 17 00:00:00 2001 From: Meng Yu Date: Thu, 4 Mar 2021 14:35:44 +0800 Subject: crypto: hisilicon/hpre - add version adapt to new algorithms A new generation of accelerator Kunpeng930 has appeared, and the corresponding driver needs to be updated to support some new algorithms of Kunpeng930. To be compatible with Kunpeng920, we add parameter 'struct hisi_qm *qm' to sec_algs_(un)register to identify the chip's version. Signed-off-by: Meng Yu Reviewed-by: Zaibo Xu Reviewed-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre.h | 5 +++-- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 4 ++-- drivers/crypto/hisilicon/qm.c | 4 ++-- drivers/crypto/hisilicon/qm.h | 4 ++-- drivers/crypto/hisilicon/sec2/sec.h | 4 ++-- drivers/crypto/hisilicon/sec2/sec_crypto.c | 4 ++-- drivers/crypto/hisilicon/sec2/sec_crypto.h | 4 ++-- drivers/crypto/hisilicon/zip/zip.h | 4 ++-- drivers/crypto/hisilicon/zip/zip_crypto.c | 4 ++-- 9 files changed, 19 insertions(+), 18 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 181c109b19f7..cc50f23947cf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -93,7 +93,8 @@ struct hpre_sqe { }; struct hisi_qp *hpre_create_qp(void); -int hpre_algs_register(void); -void hpre_algs_unregister(void); +int hpre_algs_register(struct hisi_qm *qm); +void hpre_algs_unregister(struct hisi_qm *qm); + #endif diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index a87f9904087a..d89b2f57c5bf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1154,7 +1154,7 @@ static struct kpp_alg dh = { }; #endif -int hpre_algs_register(void) +int hpre_algs_register(struct hisi_qm *qm) { int ret; @@ -1171,7 +1171,7 @@ int hpre_algs_register(void) return ret; } -void hpre_algs_unregister(void) +void hpre_algs_unregister(struct hisi_qm *qm) { crypto_unregister_akcipher(&rsa); #ifdef CONFIG_CRYPTO_DH diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 13cb4216561a..bc231742ad36 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4084,7 +4084,7 @@ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) mutex_unlock(&qm_list->lock); if (flag) { - ret = qm_list->register_to_crypto(); + ret = qm_list->register_to_crypto(qm); if (ret) { mutex_lock(&qm_list->lock); list_del(&qm->list); @@ -4115,7 +4115,7 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) mutex_unlock(&qm_list->lock); if (list_empty(&qm_list->list)) - qm_list->unregister_from_crypto(); + qm_list->unregister_from_crypto(qm); } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 54967c6b9c78..f91110fcf6a4 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -199,8 +199,8 @@ struct hisi_qm_err_ini { struct hisi_qm_list { struct mutex lock; struct list_head list; - int (*register_to_crypto)(void); - void (*unregister_from_crypto)(void); + int (*register_to_crypto)(struct hisi_qm *qm); + void (*unregister_from_crypto)(struct hisi_qm *qm); }; struct hisi_qm { diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 08491912afd5..17ddb20ad7a1 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -183,6 +183,6 @@ struct sec_dev { void sec_destroy_qps(struct hisi_qp **qps, int qp_num); struct hisi_qp **sec_create_qps(void); -int sec_register_to_crypto(void); -void sec_unregister_from_crypto(void); +int sec_register_to_crypto(struct hisi_qm *qm); +void sec_unregister_from_crypto(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 2eaa516b3231..f8355140cd46 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -1634,7 +1634,7 @@ static struct aead_alg sec_aeads[] = { AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), }; -int sec_register_to_crypto(void) +int sec_register_to_crypto(struct hisi_qm *qm) { int ret; @@ -1651,7 +1651,7 @@ int sec_register_to_crypto(void) return ret; } -void sec_unregister_from_crypto(void) +void sec_unregister_from_crypto(struct hisi_qm *qm) { crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers)); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h index b2786e17d8fe..0e933e7858e7 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.h +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -211,6 +211,6 @@ struct sec_sqe { struct sec_sqe_type2 type2; }; -int sec_register_to_crypto(void); -void sec_unregister_from_crypto(void); +int sec_register_to_crypto(struct hisi_qm *qm); +void sec_unregister_from_crypto(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 92397f993e23..9ed74611f722 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -62,6 +62,6 @@ struct hisi_zip_sqe { }; int zip_create_qps(struct hisi_qp **qps, int ctx_num, int node); -int hisi_zip_register_to_crypto(void); -void hisi_zip_unregister_from_crypto(void); +int hisi_zip_register_to_crypto(struct hisi_qm *qm); +void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 08b4660b014c..41f69662024a 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -665,7 +665,7 @@ static struct acomp_alg hisi_zip_acomp_gzip = { } }; -int hisi_zip_register_to_crypto(void) +int hisi_zip_register_to_crypto(struct hisi_qm *qm) { int ret; @@ -684,7 +684,7 @@ int hisi_zip_register_to_crypto(void) return ret; } -void hisi_zip_unregister_from_crypto(void) +void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) { crypto_unregister_acomp(&hisi_zip_acomp_gzip); crypto_unregister_acomp(&hisi_zip_acomp_zlib); -- cgit From 9b94ae729068f6608ec7364dea891ee86dd3dad2 Mon Sep 17 00:00:00 2001 From: Meng Yu Date: Thu, 4 Mar 2021 14:35:45 +0800 Subject: crypto: hisilicon/hpre - add algorithm type Algorithm type is brought in to get hardware HPRE queue to support different algorithms. Signed-off-by: Meng Yu Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre.h | 10 +++++++++- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 12 ++++++------ drivers/crypto/hisilicon/hpre/hpre_main.c | 11 +++++++++-- 3 files changed, 24 insertions(+), 9 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index cc50f23947cf..02193e1ac694 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -10,6 +10,14 @@ #define HPRE_PF_DEF_Q_NUM 64 #define HPRE_PF_DEF_Q_BASE 0 +/* + * type used in qm sqc DW6. + * 0 - Algorithm which has been supported in V2, like RSA, DH and so on; + * 1 - ECC algorithm in V3. + */ +#define HPRE_V2_ALG_TYPE 0 +#define HPRE_V3_ECC_ALG_TYPE 1 + enum { HPRE_CLUSTER0, HPRE_CLUSTER1, @@ -92,7 +100,7 @@ struct hpre_sqe { __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; }; -struct hisi_qp *hpre_create_qp(void); +struct hisi_qp *hpre_create_qp(u8 type); int hpre_algs_register(struct hisi_qm *qm); void hpre_algs_unregister(struct hisi_qm *qm); diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index d89b2f57c5bf..712bea9f61d5 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -152,12 +152,12 @@ static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) } } -static struct hisi_qp *hpre_get_qp_and_start(void) +static struct hisi_qp *hpre_get_qp_and_start(u8 type) { struct hisi_qp *qp; int ret; - qp = hpre_create_qp(); + qp = hpre_create_qp(type); if (!qp) { pr_err("Can not create hpre qp!\n"); return ERR_PTR(-ENODEV); @@ -422,11 +422,11 @@ static void hpre_alg_cb(struct hisi_qp *qp, void *resp) req->cb(ctx, resp); } -static int hpre_ctx_init(struct hpre_ctx *ctx) +static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) { struct hisi_qp *qp; - qp = hpre_get_qp_and_start(); + qp = hpre_get_qp_and_start(type); if (IS_ERR(qp)) return PTR_ERR(qp); @@ -674,7 +674,7 @@ static int hpre_dh_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - return hpre_ctx_init(ctx); + return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); } static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) @@ -1100,7 +1100,7 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) return PTR_ERR(ctx->rsa.soft_tfm); } - ret = hpre_ctx_init(ctx); + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); if (ret) crypto_free_akcipher(ctx->rsa.soft_tfm); diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index e7a2c70eb9cf..76f0a87c584d 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -226,13 +226,20 @@ static u32 vfs_num; module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); -struct hisi_qp *hpre_create_qp(void) +struct hisi_qp *hpre_create_qp(u8 type) { int node = cpu_to_node(smp_processor_id()); struct hisi_qp *qp = NULL; int ret; - ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, 0, node, &qp); + if (type != HPRE_V2_ALG_TYPE && type != HPRE_V3_ECC_ALG_TYPE) + return NULL; + + /* + * type: 0 - RSA/DH. algorithm supported in V2, + * 1 - ECC algorithm in V3. + */ + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp); if (!ret) return qp; -- cgit From 05e7b906aa7c8690906135dc86ab0fc12ee37481 Mon Sep 17 00:00:00 2001 From: Meng Yu Date: Thu, 4 Mar 2021 14:35:48 +0800 Subject: crypto: hisilicon/hpre - add 'ECDH' algorithm 1. Enable 'ECDH' algorithm in Kunpeng 930; 2. HPRE ECDH Support: ecdh-nist-p192, ecdh-nist-p256. Signed-off-by: Meng Yu Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre.h | 2 +- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 515 +++++++++++++++++++++++++++- drivers/crypto/hisilicon/hpre/hpre_main.c | 1 + 3 files changed, 513 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 02193e1ac694..50e6b2e0e517 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -83,6 +83,7 @@ enum hpre_alg_type { HPRE_ALG_KG_CRT = 0x3, HPRE_ALG_DH_G2 = 0x4, HPRE_ALG_DH = 0x5, + HPRE_ALG_ECC_MUL = 0xD, }; struct hpre_sqe { @@ -104,5 +105,4 @@ struct hisi_qp *hpre_create_qp(u8 type); int hpre_algs_register(struct hisi_qm *qm); void hpre_algs_unregister(struct hisi_qm *qm); - #endif diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 712bea9f61d5..a6010b104b18 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -2,6 +2,8 @@ /* Copyright (c) 2019 HiSilicon Limited. */ #include #include +#include +#include #include #include #include @@ -36,6 +38,13 @@ struct hpre_ctx; #define HPRE_DFX_SEC_TO_US 1000000 #define HPRE_DFX_US_TO_NS 1000 +/* size in bytes of the n prime */ +#define HPRE_ECC_NIST_P192_N_SIZE 24 +#define HPRE_ECC_NIST_P256_N_SIZE 32 + +/* size in bytes */ +#define HPRE_ECC_HW256_KSZ_B 32 + typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); struct hpre_rsa_ctx { @@ -61,14 +70,25 @@ struct hpre_dh_ctx { * else if base if the counterpart public key we * compute the shared secret * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] + * low address: d--->n, please refer to Hisilicon HPRE UM */ - char *xa_p; /* low address: d--->n, please refer to Hisilicon HPRE UM */ + char *xa_p; dma_addr_t dma_xa_p; char *g; /* m */ dma_addr_t dma_g; }; +struct hpre_ecdh_ctx { + /* low address: p->a->k->b */ + unsigned char *p; + dma_addr_t dma_p; + + /* low address: x->y */ + unsigned char *g; + dma_addr_t dma_g; +}; + struct hpre_ctx { struct hisi_qp *qp; struct hpre_asym_request **req_list; @@ -80,7 +100,10 @@ struct hpre_ctx { union { struct hpre_rsa_ctx rsa; struct hpre_dh_ctx dh; + struct hpre_ecdh_ctx ecdh; }; + /* for ecc algorithms */ + unsigned int curve_id; }; struct hpre_asym_request { @@ -91,6 +114,7 @@ struct hpre_asym_request { union { struct akcipher_request *rsa; struct kpp_request *dh; + struct kpp_request *ecdh; } areq; int err; int req_id; @@ -1115,6 +1139,416 @@ static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) crypto_free_akcipher(ctx->rsa.soft_tfm); } +static void hpre_key_to_big_end(u8 *data, int len) +{ + int i, j; + u8 tmp; + + for (i = 0; i < len / 2; i++) { + j = len - i - 1; + tmp = data[j]; + data[j] = data[i]; + data[i] = tmp; + } +} + +static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, + bool is_ecdh) +{ + struct device *dev = HPRE_DEV(ctx); + unsigned int sz = ctx->key_sz; + unsigned int shift = sz << 1; + + if (is_clear_all) + hisi_qm_stop_qp(ctx->qp); + + if (is_ecdh && ctx->ecdh.p) { + /* ecdh: p->a->k->b */ + memzero_explicit(ctx->ecdh.p + shift, sz); + dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); + ctx->ecdh.p = NULL; + } + + hpre_ctx_clear(ctx, is_clear_all); +} + +static unsigned int hpre_ecdh_supported_curve(unsigned short id) +{ + switch (id) { + case ECC_CURVE_NIST_P192: + case ECC_CURVE_NIST_P256: + return HPRE_ECC_HW256_KSZ_B; + default: + break; + } + + return 0; +} + +static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits) +{ + unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64); + u8 i = 0; + + while (i < ndigits - 1) { + memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64)); + i++; + } + + memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz); + hpre_key_to_big_end((u8 *)addr, cur_sz); +} + +static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params, + unsigned int cur_sz) +{ + unsigned int shifta = ctx->key_sz << 1; + unsigned int shiftb = ctx->key_sz << 2; + void *p = ctx->ecdh.p + ctx->key_sz - cur_sz; + void *a = ctx->ecdh.p + shifta - cur_sz; + void *b = ctx->ecdh.p + shiftb - cur_sz; + void *x = ctx->ecdh.g + ctx->key_sz - cur_sz; + void *y = ctx->ecdh.g + shifta - cur_sz; + const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id); + char *n; + + if (unlikely(!curve)) + return -EINVAL; + + n = kzalloc(ctx->key_sz, GFP_KERNEL); + if (!n) + return -ENOMEM; + + fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits); + fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits); + fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits); + fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits); + fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits); + fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits); + + if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) { + kfree(n); + return -EINVAL; + } + + kfree(n); + return 0; +} + +static unsigned int hpre_ecdh_get_curvesz(unsigned short id) +{ + switch (id) { + case ECC_CURVE_NIST_P192: + return HPRE_ECC_NIST_P192_N_SIZE; + case ECC_CURVE_NIST_P256: + return HPRE_ECC_NIST_P256_N_SIZE; + default: + break; + } + + return 0; +} + +static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) +{ + struct device *dev = HPRE_DEV(ctx); + unsigned int sz, shift, curve_sz; + int ret; + + ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id); + if (!ctx->key_sz) + return -EINVAL; + + curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); + if (!curve_sz || params->key_size > curve_sz) + return -EINVAL; + + sz = ctx->key_sz; + + if (!ctx->ecdh.p) { + ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p, + GFP_KERNEL); + if (!ctx->ecdh.p) + return -ENOMEM; + } + + shift = sz << 2; + ctx->ecdh.g = ctx->ecdh.p + shift; + ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift; + + ret = hpre_ecdh_fill_curve(ctx, params, curve_sz); + if (ret) { + dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret); + dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); + ctx->ecdh.p = NULL; + return ret; + } + + return 0; +} + +static bool hpre_key_is_zero(char *key, unsigned short key_sz) +{ + int i; + + for (i = 0; i < key_sz; i++) + if (key[i]) + return false; + + return true; +} + +static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = HPRE_DEV(ctx); + unsigned int sz, sz_shift; + struct ecdh params; + int ret; + + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { + dev_err(dev, "failed to decode ecdh key!\n"); + return -EINVAL; + } + + if (hpre_key_is_zero(params.key, params.key_size)) { + dev_err(dev, "Invalid hpre key!\n"); + return -EINVAL; + } + + hpre_ecc_clear_ctx(ctx, false, true); + + ret = hpre_ecdh_set_param(ctx, ¶ms); + if (ret < 0) { + dev_err(dev, "failed to set hpre param, ret = %d!\n", ret); + return ret; + } + + sz = ctx->key_sz; + sz_shift = (sz << 1) + sz - params.key_size; + memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size); + + return 0; +} + +static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, + struct hpre_asym_request *req, + struct scatterlist *dst, + struct scatterlist *src) +{ + struct device *dev = HPRE_DEV(ctx); + struct hpre_sqe *sqe = &req->req; + dma_addr_t dma; + + dma = le64_to_cpu(sqe->in); + if (unlikely(!dma)) + return; + + if (src && req->src) + dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); + + dma = le64_to_cpu(sqe->out); + if (unlikely(!dma)) + return; + + if (req->dst) + dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); + if (dst) + dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE); +} + +static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) +{ + unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *req = NULL; + struct kpp_request *areq; + u64 overtime_thrhld; + char *p; + int ret; + + ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); + areq = req->areq.ecdh; + areq->dst_len = ctx->key_sz << 1; + + overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); + if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) + atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + + p = sg_virt(areq->dst); + memmove(p, p + ctx->key_sz - curve_sz, curve_sz); + memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); + + hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); + kpp_request_complete(areq, ret); + + atomic64_inc(&dfx[HPRE_RECV_CNT].value); +} + +static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, + struct kpp_request *req) +{ + struct hpre_asym_request *h_req; + struct hpre_sqe *msg; + int req_id; + void *tmp; + + if (req->dst_len < ctx->key_sz << 1) { + req->dst_len = ctx->key_sz << 1; + return -EINVAL; + } + + tmp = kpp_request_ctx(req); + h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); + h_req->cb = hpre_ecdh_cb; + h_req->areq.ecdh = req; + msg = &h_req->req; + memset(msg, 0, sizeof(*msg)); + msg->key = cpu_to_le64(ctx->ecdh.dma_p); + + msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); + msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; + h_req->ctx = ctx; + + req_id = hpre_add_req_to_ctx(h_req); + if (req_id < 0) + return -EBUSY; + + msg->tag = cpu_to_le16((u16)req_id); + return 0; +} + +static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len) +{ + struct hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = HPRE_DEV(ctx); + unsigned int tmpshift; + dma_addr_t dma = 0; + void *ptr; + int shift; + + /* Src_data include gx and gy. */ + shift = ctx->key_sz - (len >> 1); + if (unlikely(shift < 0)) + return -EINVAL; + + ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL); + if (unlikely(!ptr)) + return -ENOMEM; + + tmpshift = ctx->key_sz << 1; + scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0); + memcpy(ptr + shift, ptr + tmpshift, len >> 1); + memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1); + + hpre_req->src = ptr; + msg->in = cpu_to_le64(dma); + return 0; +} + +static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len) +{ + struct hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = HPRE_DEV(ctx); + dma_addr_t dma = 0; + + if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { + dev_err(dev, "data or data length is illegal!\n"); + return -EINVAL; + } + + hpre_req->dst = NULL; + dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, dma))) { + dev_err(dev, "dma map data err!\n"); + return -ENOMEM; + } + + msg->out = cpu_to_le64(dma); + return 0; +} + +static int hpre_ecdh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = HPRE_DEV(ctx); + void *tmp = kpp_request_ctx(req); + struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); + struct hpre_sqe *msg = &hpre_req->req; + int ret; + + ret = hpre_ecdh_msg_request_set(ctx, req); + if (unlikely(ret)) { + dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret); + return ret; + } + + if (req->src) { + ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len); + if (unlikely(ret)) { + dev_err(dev, "failed to init src data, ret = %d!\n", ret); + goto clear_all; + } + } else { + msg->in = cpu_to_le64(ctx->ecdh.dma_g); + } + + ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len); + if (unlikely(ret)) { + dev_err(dev, "failed to init dst data, ret = %d!\n", ret); + goto clear_all; + } + + msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); + ret = hpre_send(ctx, msg); + if (likely(!ret)) + return -EINPROGRESS; + +clear_all: + hpre_rm_req_from_ctx(hpre_req); + hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); + return ret; +} + +static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + /* max size is the pub_key_size, include x and y */ + return ctx->key_sz << 1; +} + +static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P192; + + return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); +} + +static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P256; + + return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); +} + +static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + hpre_ecc_clear_ctx(ctx, true, true); +} + static struct akcipher_alg rsa = { .sign = hpre_rsa_dec, .verify = hpre_rsa_enc, @@ -1154,6 +1588,63 @@ static struct kpp_alg dh = { }; #endif +static struct kpp_alg ecdh_nist_p192 = { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p192_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p192", + .cra_driver_name = "hpre-ecdh", + .cra_module = THIS_MODULE, + }, +}; + +static struct kpp_alg ecdh_nist_p256 = { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p256_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "hpre-ecdh", + .cra_module = THIS_MODULE, + }, +}; + +static int hpre_register_ecdh(void) +{ + int ret; + + ret = crypto_register_kpp(&ecdh_nist_p192); + if (ret) + return ret; + + ret = crypto_register_kpp(&ecdh_nist_p256); + if (ret) { + crypto_unregister_kpp(&ecdh_nist_p192); + return ret; + } + + return 0; +} + +static void hpre_unregister_ecdh(void) +{ + crypto_unregister_kpp(&ecdh_nist_p256); + crypto_unregister_kpp(&ecdh_nist_p192); +} + int hpre_algs_register(struct hisi_qm *qm) { int ret; @@ -1164,17 +1655,33 @@ int hpre_algs_register(struct hisi_qm *qm) return ret; #ifdef CONFIG_CRYPTO_DH ret = crypto_register_kpp(&dh); - if (ret) + if (ret) { crypto_unregister_akcipher(&rsa); + return ret; + } #endif - return ret; + if (qm->ver >= QM_HW_V3) { + ret = hpre_register_ecdh(); + if (ret) { +#ifdef CONFIG_CRYPTO_DH + crypto_unregister_kpp(&dh); +#endif + crypto_unregister_akcipher(&rsa); + return ret; + } + } + + return 0; } void hpre_algs_unregister(struct hisi_qm *qm) { - crypto_unregister_akcipher(&rsa); + if (qm->ver >= QM_HW_V3) + hpre_unregister_ecdh(); + #ifdef CONFIG_CRYPTO_DH crypto_unregister_kpp(&dh); #endif + crypto_unregister_akcipher(&rsa); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 76f0a87c584d..87e8f4d60474 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -1082,4 +1082,5 @@ module_exit(hpre_exit); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Zaibo Xu "); +MODULE_AUTHOR("Meng Yu "); MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator"); -- cgit From 90274769cf7926a7a14f1cd3f47b66de76e0adb4 Mon Sep 17 00:00:00 2001 From: Meng Yu Date: Thu, 4 Mar 2021 14:35:50 +0800 Subject: crypto: hisilicon/hpre - add 'CURVE25519' algorithm Enable 'CURVE25519' algorithm in Kunpeng 930. Signed-off-by: Meng Yu Reviewed-by: Zaibo Xu Reported-by: kernel test robot Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/Kconfig | 1 + drivers/crypto/hisilicon/hpre/hpre.h | 2 + drivers/crypto/hisilicon/hpre/hpre_crypto.c | 366 +++++++++++++++++++++++++++- 3 files changed, 361 insertions(+), 8 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 843192666dc3..c45adb15ce8d 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -65,6 +65,7 @@ config CRYPTO_DEV_HISI_HPRE depends on UACCE || UACCE=n depends on ARM64 || (COMPILE_TEST && 64BIT) depends on ACPI + select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_DEV_HISI_QM select CRYPTO_DH select CRYPTO_RSA diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 50e6b2e0e517..92892e373cd0 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -84,6 +84,8 @@ enum hpre_alg_type { HPRE_ALG_DH_G2 = 0x4, HPRE_ALG_DH = 0x5, HPRE_ALG_ECC_MUL = 0xD, + /* shared by x25519 and x448, but x448 is not supported now */ + HPRE_ALG_CURVE25519_MUL = 0x10, }; struct hpre_sqe { diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index a6010b104b18..53068d2a19cf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2019 HiSilicon Limited. */ #include +#include #include #include #include @@ -89,6 +90,16 @@ struct hpre_ecdh_ctx { dma_addr_t dma_g; }; +struct hpre_curve25519_ctx { + /* low address: p->a->k */ + unsigned char *p; + dma_addr_t dma_p; + + /* gx coordinate */ + unsigned char *g; + dma_addr_t dma_g; +}; + struct hpre_ctx { struct hisi_qp *qp; struct hpre_asym_request **req_list; @@ -101,6 +112,7 @@ struct hpre_ctx { struct hpre_rsa_ctx rsa; struct hpre_dh_ctx dh; struct hpre_ecdh_ctx ecdh; + struct hpre_curve25519_ctx curve25519; }; /* for ecc algorithms */ unsigned int curve_id; @@ -115,6 +127,7 @@ struct hpre_asym_request { struct akcipher_request *rsa; struct kpp_request *dh; struct kpp_request *ecdh; + struct kpp_request *curve25519; } areq; int err; int req_id; @@ -437,7 +450,6 @@ static void hpre_alg_cb(struct hisi_qp *qp, void *resp) struct hpre_sqe *sqe = resp; struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; - if (unlikely(!req)) { atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); return; @@ -1167,6 +1179,12 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, memzero_explicit(ctx->ecdh.p + shift, sz); dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); ctx->ecdh.p = NULL; + } else if (!is_ecdh && ctx->curve25519.p) { + /* curve25519: p->a->k */ + memzero_explicit(ctx->curve25519.p + shift, sz); + dma_free_coherent(dev, sz << 2, ctx->curve25519.p, + ctx->curve25519.dma_p); + ctx->curve25519.p = NULL; } hpre_ctx_clear(ctx, is_clear_all); @@ -1549,6 +1567,312 @@ static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) hpre_ecc_clear_ctx(ctx, true, true); } +static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf, + unsigned int len) +{ + u8 secret[CURVE25519_KEY_SIZE] = { 0 }; + unsigned int sz = ctx->key_sz; + const struct ecc_curve *curve; + unsigned int shift = sz << 1; + void *p; + + /* + * The key from 'buf' is in little-endian, we should preprocess it as + * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64", + * then convert it to big endian. Only in this way, the result can be + * the same as the software curve-25519 that exists in crypto. + */ + memcpy(secret, buf, len); + curve25519_clamp_secret(secret); + hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE); + + p = ctx->curve25519.p + sz - len; + + curve = ecc_get_curve25519(); + + /* fill curve parameters */ + fill_curve_param(p, curve->p, len, curve->g.ndigits); + fill_curve_param(p + sz, curve->a, len, curve->g.ndigits); + memcpy(p + shift, secret, len); + fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits); + memzero_explicit(secret, CURVE25519_KEY_SIZE); +} + +static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf, + unsigned int len) +{ + struct device *dev = HPRE_DEV(ctx); + unsigned int sz = ctx->key_sz; + unsigned int shift = sz << 1; + + /* p->a->k->gx */ + if (!ctx->curve25519.p) { + ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2, + &ctx->curve25519.dma_p, + GFP_KERNEL); + if (!ctx->curve25519.p) + return -ENOMEM; + } + + ctx->curve25519.g = ctx->curve25519.p + shift + sz; + ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz; + + hpre_curve25519_fill_curve(ctx, buf, len); + + return 0; +} + +static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = HPRE_DEV(ctx); + int ret = -EINVAL; + + if (len != CURVE25519_KEY_SIZE || + !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) { + dev_err(dev, "key is null or key len is not 32bytes!\n"); + return ret; + } + + /* Free old secret if any */ + hpre_ecc_clear_ctx(ctx, false, false); + + ctx->key_sz = CURVE25519_KEY_SIZE; + ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE); + if (ret) { + dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret); + hpre_ecc_clear_ctx(ctx, false, false); + return ret; + } + + return 0; +} + +static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, + struct hpre_asym_request *req, + struct scatterlist *dst, + struct scatterlist *src) +{ + struct device *dev = HPRE_DEV(ctx); + struct hpre_sqe *sqe = &req->req; + dma_addr_t dma; + + dma = le64_to_cpu(sqe->in); + if (unlikely(!dma)) + return; + + if (src && req->src) + dma_free_coherent(dev, ctx->key_sz, req->src, dma); + + dma = le64_to_cpu(sqe->out); + if (unlikely(!dma)) + return; + + if (req->dst) + dma_free_coherent(dev, ctx->key_sz, req->dst, dma); + if (dst) + dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE); +} + +static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp) +{ + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *req = NULL; + struct kpp_request *areq; + u64 overtime_thrhld; + int ret; + + ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); + areq = req->areq.curve25519; + areq->dst_len = ctx->key_sz; + + overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); + if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) + atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + + hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE); + + hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); + kpp_request_complete(areq, ret); + + atomic64_inc(&dfx[HPRE_RECV_CNT].value); +} + +static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, + struct kpp_request *req) +{ + struct hpre_asym_request *h_req; + struct hpre_sqe *msg; + int req_id; + void *tmp; + + if (unlikely(req->dst_len < ctx->key_sz)) { + req->dst_len = ctx->key_sz; + return -EINVAL; + } + + tmp = kpp_request_ctx(req); + h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); + h_req->cb = hpre_curve25519_cb; + h_req->areq.curve25519 = req; + msg = &h_req->req; + memset(msg, 0, sizeof(*msg)); + msg->key = cpu_to_le64(ctx->curve25519.dma_p); + + msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); + msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; + h_req->ctx = ctx; + + req_id = hpre_add_req_to_ctx(h_req); + if (req_id < 0) + return -EBUSY; + + msg->tag = cpu_to_le16((u16)req_id); + return 0; +} + +static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len) +{ + struct hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = HPRE_DEV(ctx); + u8 p[CURVE25519_KEY_SIZE] = { 0 }; + const struct ecc_curve *curve; + dma_addr_t dma = 0; + u8 *ptr; + + if (len != CURVE25519_KEY_SIZE) { + dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len); + return -EINVAL; + } + + ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL); + if (unlikely(!ptr)) + return -ENOMEM; + + scatterwalk_map_and_copy(ptr, data, 0, len, 0); + + if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) { + dev_err(dev, "gx is null!\n"); + goto err; + } + + /* + * Src_data(gx) is in little-endian order, MSB in the final byte should + * be masked as discribed in RFC7748, then transform it to big-endian + * form, then hisi_hpre can use the data. + */ + ptr[31] &= 0x7f; + hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE); + + curve = ecc_get_curve25519(); + + fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits); + if (memcmp(ptr, p, ctx->key_sz) >= 0) { + dev_err(dev, "gx is out of p!\n"); + goto err; + } + + hpre_req->src = ptr; + msg->in = cpu_to_le64(dma); + return 0; + +err: + dma_free_coherent(dev, ctx->key_sz, ptr, dma); + return -EINVAL; +} + +static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len) +{ + struct hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = HPRE_DEV(ctx); + dma_addr_t dma = 0; + + if (!data || !sg_is_last(data) || len != ctx->key_sz) { + dev_err(dev, "data or data length is illegal!\n"); + return -EINVAL; + } + + hpre_req->dst = NULL; + dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, dma))) { + dev_err(dev, "dma map data err!\n"); + return -ENOMEM; + } + + msg->out = cpu_to_le64(dma); + return 0; +} + +static int hpre_curve25519_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = HPRE_DEV(ctx); + void *tmp = kpp_request_ctx(req); + struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); + struct hpre_sqe *msg = &hpre_req->req; + int ret; + + ret = hpre_curve25519_msg_request_set(ctx, req); + if (unlikely(ret)) { + dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret); + return ret; + } + + if (req->src) { + ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len); + if (unlikely(ret)) { + dev_err(dev, "failed to init src data, ret = %d!\n", + ret); + goto clear_all; + } + } else { + msg->in = cpu_to_le64(ctx->curve25519.dma_g); + } + + ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len); + if (unlikely(ret)) { + dev_err(dev, "failed to init dst data, ret = %d!\n", ret); + goto clear_all; + } + + msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL); + ret = hpre_send(ctx, msg); + if (likely(!ret)) + return -EINPROGRESS; + +clear_all: + hpre_rm_req_from_ctx(hpre_req); + hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); + return ret; +} + +static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + return ctx->key_sz; +} + +static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); +} + +static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + hpre_ecc_clear_ctx(ctx, true, false); +} + static struct akcipher_alg rsa = { .sign = hpre_rsa_dec, .verify = hpre_rsa_enc, @@ -1622,6 +1946,24 @@ static struct kpp_alg ecdh_nist_p256 = { }, }; +static struct kpp_alg curve25519_alg = { + .set_secret = hpre_curve25519_set_secret, + .generate_public_key = hpre_curve25519_compute_value, + .compute_shared_secret = hpre_curve25519_compute_value, + .max_size = hpre_curve25519_max_size, + .init = hpre_curve25519_init_tfm, + .exit = hpre_curve25519_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "curve25519", + .cra_driver_name = "hpre-curve25519", + .cra_module = THIS_MODULE, + }, +}; + + static int hpre_register_ecdh(void) { int ret; @@ -1663,22 +2005,30 @@ int hpre_algs_register(struct hisi_qm *qm) if (qm->ver >= QM_HW_V3) { ret = hpre_register_ecdh(); + if (ret) + goto reg_err; + ret = crypto_register_kpp(&curve25519_alg); if (ret) { -#ifdef CONFIG_CRYPTO_DH - crypto_unregister_kpp(&dh); -#endif - crypto_unregister_akcipher(&rsa); - return ret; + hpre_unregister_ecdh(); + goto reg_err; } } - return 0; + +reg_err: +#ifdef CONFIG_CRYPTO_DH + crypto_unregister_kpp(&dh); +#endif + crypto_unregister_akcipher(&rsa); + return ret; } void hpre_algs_unregister(struct hisi_qm *qm) { - if (qm->ver >= QM_HW_V3) + if (qm->ver >= QM_HW_V3) { + crypto_unregister_kpp(&curve25519_alg); hpre_unregister_ecdh(); + } #ifdef CONFIG_CRYPTO_DH crypto_unregister_kpp(&dh); -- cgit From 4b7aef0230418345be1fb77abbb1592801869901 Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Sat, 13 Mar 2021 15:28:23 +0800 Subject: crypto: hisilicon/sec - fixes a printing error When the log is output here, the device has not been initialized yet. Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index f8355140cd46..2176e4da5b3a 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -546,7 +546,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { - dev_err(SEC_CTX_DEV(ctx), "get error skcipher iv size!\n"); + pr_err("get error skcipher iv size!\n"); return -EINVAL; } -- cgit From a44dce504bce620daff97a3e77650b7b579e8753 Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Sat, 13 Mar 2021 15:28:24 +0800 Subject: crypto: hisilicon/sec - fixes some coding style 1.delete the original complex method of obtaining the current device and replace it with the initialized device pointer. 2.fixes some coding style Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec.h | 5 +- drivers/crypto/hisilicon/sec2/sec_crypto.c | 75 +++++++++++++++--------------- drivers/crypto/hisilicon/sec2/sec_crypto.h | 2 - 3 files changed, 39 insertions(+), 43 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 17ddb20ad7a1..f7eba9bd312b 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -4,8 +4,6 @@ #ifndef __HISI_SEC_V2_H #define __HISI_SEC_V2_H -#include - #include "../qm.h" #include "sec_crypto.h" @@ -50,7 +48,7 @@ struct sec_req { int err_type; int req_id; - int flag; + u32 flag; /* Status of the SEC request */ bool fake_busy; @@ -139,6 +137,7 @@ struct sec_ctx { bool pbuf_supported; struct sec_cipher_ctx c_ctx; struct sec_auth_ctx a_ctx; + struct device *dev; }; enum sec_endian { diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 2176e4da5b3a..c0efa611f2be 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -43,7 +43,6 @@ #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) #define SEC_SGL_SGE_NR 128 -#define SEC_CTX_DEV(ctx) (&(ctx)->sec->qm.pdev->dev) #define SEC_CIPHER_AUTH 0xfe #define SEC_AUTH_CIPHER 0x1 #define SEC_MAX_MAC_LEN 64 @@ -96,7 +95,7 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 0, QM_Q_DEPTH, GFP_ATOMIC); mutex_unlock(&qp_ctx->req_lock); if (unlikely(req_id < 0)) { - dev_err(SEC_CTX_DEV(req->ctx), "alloc req id fail!\n"); + dev_err(req->ctx->dev, "alloc req id fail!\n"); return req_id; } @@ -112,7 +111,7 @@ static void sec_free_req_id(struct sec_req *req) int req_id = req->req_id; if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { - dev_err(SEC_CTX_DEV(req->ctx), "free request id invalid!\n"); + dev_err(req->ctx->dev, "free request id invalid!\n"); return; } @@ -138,7 +137,7 @@ static int sec_aead_verify(struct sec_req *req) aead_req->cryptlen + aead_req->assoclen - authsize); if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { - dev_err(SEC_CTX_DEV(req->ctx), "aead verify failure!\n"); + dev_err(req->ctx->dev, "aead verify failure!\n"); return -EBADMSG; } @@ -177,7 +176,7 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) if (unlikely(req->err_type || done != SEC_SQE_DONE || (ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) || (ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) { - dev_err(SEC_CTX_DEV(ctx), + dev_err_ratelimited(ctx->dev, "err_type[%d],done[%d],flag[%d]\n", req->err_type, done, flag); err = -EIO; @@ -326,8 +325,8 @@ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) static int sec_alg_resource_alloc(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { - struct device *dev = SEC_CTX_DEV(ctx); struct sec_alg_res *res = qp_ctx->res; + struct device *dev = ctx->dev; int ret; ret = sec_alloc_civ_resource(dev, res); @@ -360,7 +359,7 @@ alloc_fail: static void sec_alg_resource_free(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; sec_free_civ_resource(dev, qp_ctx->res); @@ -373,7 +372,7 @@ static void sec_alg_resource_free(struct sec_ctx *ctx, static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, int qp_ctx_id, int alg_type) { - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; struct sec_qp_ctx *qp_ctx; struct hisi_qp *qp; int ret = -ENOMEM; @@ -428,7 +427,7 @@ err_destroy_idr: static void sec_release_qp_ctx(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; hisi_qm_stop_qp(qp_ctx->qp); sec_alg_resource_free(ctx, qp_ctx); @@ -452,6 +451,7 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); ctx->sec = sec; + ctx->dev = &sec->qm.pdev->dev; ctx->hlf_q_num = sec->ctx_q_num >> 1; ctx->pbuf_supported = ctx->sec->iommu_used; @@ -476,11 +476,9 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) err_sec_release_qp_ctx: for (i = i - 1; i >= 0; i--) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); - kfree(ctx->qp_ctx); err_destroy_qps: sec_destroy_qps(ctx->qps, sec->ctx_q_num); - return ret; } @@ -499,7 +497,7 @@ static int sec_cipher_init(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; - c_ctx->c_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, + c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, &c_ctx->c_key_dma, GFP_KERNEL); if (!c_ctx->c_key) return -ENOMEM; @@ -512,7 +510,7 @@ static void sec_cipher_uninit(struct sec_ctx *ctx) struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); - dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, + dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, c_ctx->c_key, c_ctx->c_key_dma); } @@ -520,7 +518,7 @@ static int sec_auth_init(struct sec_ctx *ctx) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; - a_ctx->a_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, + a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, &a_ctx->a_key_dma, GFP_KERNEL); if (!a_ctx->a_key) return -ENOMEM; @@ -533,7 +531,7 @@ static void sec_auth_uninit(struct sec_ctx *ctx) struct sec_auth_ctx *a_ctx = &ctx->a_ctx; memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE); - dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, + dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, a_ctx->a_key, a_ctx->a_key_dma); } @@ -633,12 +631,13 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, { struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct device *dev = ctx->dev; int ret; if (c_mode == SEC_CMODE_XTS) { ret = xts_verify_key(tfm, key, keylen); if (ret) { - dev_err(SEC_CTX_DEV(ctx), "xts mode key err!\n"); + dev_err(dev, "xts mode key err!\n"); return ret; } } @@ -659,7 +658,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } if (ret) { - dev_err(SEC_CTX_DEV(ctx), "set sec key err!\n"); + dev_err(dev, "set sec key err!\n"); return ret; } @@ -691,7 +690,7 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, struct aead_request *aead_req = req->aead_req.aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; int copy_size, pbuf_length; int req_id = req->req_id; @@ -701,9 +700,8 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, copy_size = c_req->c_len; pbuf_length = sg_copy_to_buffer(src, sg_nents(src), - qp_ctx->res[req_id].pbuf, - copy_size); - + qp_ctx->res[req_id].pbuf, + copy_size); if (unlikely(pbuf_length != copy_size)) { dev_err(dev, "copy src data to pbuf error!\n"); return -EINVAL; @@ -727,7 +725,7 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, struct aead_request *aead_req = req->aead_req.aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; int copy_size, pbuf_length; int req_id = req->req_id; @@ -739,7 +737,6 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf, copy_size); - if (unlikely(pbuf_length != copy_size)) dev_err(dev, "copy pbuf data to dst error!\n"); } @@ -751,7 +748,7 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, struct sec_aead_req *a_req = &req->aead_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; struct sec_alg_res *res = &qp_ctx->res[req->req_id]; - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; int ret; if (req->use_pbuf) { @@ -806,7 +803,7 @@ static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src, struct scatterlist *dst) { struct sec_cipher_req *c_req = &req->c_req; - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; if (req->use_pbuf) { sec_cipher_pbuf_unmap(ctx, req, dst); @@ -891,6 +888,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, { struct sec_ctx *ctx = crypto_aead_ctx(tfm); struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct device *dev = ctx->dev; struct crypto_authenc_keys keys; int ret; @@ -904,13 +902,13 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, ret = sec_aead_aes_set_key(c_ctx, &keys); if (ret) { - dev_err(SEC_CTX_DEV(ctx), "set sec cipher key err!\n"); + dev_err(dev, "set sec cipher key err!\n"); goto bad_key; } ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); if (ret) { - dev_err(SEC_CTX_DEV(ctx), "set sec auth key err!\n"); + dev_err(dev, "set sec auth key err!\n"); goto bad_key; } @@ -1062,7 +1060,7 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, cryptlen - iv_size); if (unlikely(sz != iv_size)) - dev_err(SEC_CTX_DEV(req->ctx), "copy output iv error!\n"); + dev_err(req->ctx->dev, "copy output iv error!\n"); } static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, @@ -1160,7 +1158,7 @@ static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) ret = sec_skcipher_bd_fill(ctx, req); if (unlikely(ret)) { - dev_err(SEC_CTX_DEV(ctx), "skcipher bd fill is error!\n"); + dev_err(ctx->dev, "skcipher bd fill is error!\n"); return ret; } @@ -1194,7 +1192,7 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) a_req->assoclen); if (unlikely(sz != authsize)) { - dev_err(SEC_CTX_DEV(req->ctx), "copy out mac err!\n"); + dev_err(c->dev, "copy out mac err!\n"); err = -EINVAL; } } @@ -1259,7 +1257,7 @@ static int sec_process(struct sec_ctx *ctx, struct sec_req *req) ret = ctx->req_op->bd_send(ctx, req); if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { - dev_err_ratelimited(SEC_CTX_DEV(ctx), "send sec request failed!\n"); + dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); goto err_send_req; } @@ -1325,7 +1323,7 @@ static int sec_aead_init(struct crypto_aead *tfm) ctx->alg_type = SEC_AEAD; ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { - dev_err(SEC_CTX_DEV(ctx), "get error aead iv size!\n"); + dev_err(ctx->dev, "get error aead iv size!\n"); return -EINVAL; } @@ -1374,7 +1372,7 @@ static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); if (IS_ERR(auth_ctx->hash_tfm)) { - dev_err(SEC_CTX_DEV(ctx), "aead alloc shash error!\n"); + dev_err(ctx->dev, "aead alloc shash error!\n"); sec_aead_exit(tfm); return PTR_ERR(auth_ctx->hash_tfm); } @@ -1408,7 +1406,7 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) { struct skcipher_request *sk_req = sreq->c_req.sk_req; - struct device *dev = SEC_CTX_DEV(ctx); + struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; if (unlikely(!sk_req->src || !sk_req->dst)) { @@ -1531,14 +1529,15 @@ static struct skcipher_alg sec_skciphers[] = { static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) { - u8 c_alg = ctx->c_ctx.c_alg; struct aead_request *req = sreq->aead_req.aead_req; struct crypto_aead *tfm = crypto_aead_reqtfm(req); size_t authsize = crypto_aead_authsize(tfm); + struct device *dev = ctx->dev; + u8 c_alg = ctx->c_ctx.c_alg; if (unlikely(!req->src || !req->dst || !req->cryptlen || req->assoclen > SEC_MAX_AAD_LEN)) { - dev_err(SEC_CTX_DEV(ctx), "aead input param error!\n"); + dev_err(dev, "aead input param error!\n"); return -EINVAL; } @@ -1550,7 +1549,7 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) /* Support AES only */ if (unlikely(c_alg != SEC_CALG_AES)) { - dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n"); + dev_err(dev, "aead crypto alg error!\n"); return -EINVAL; } if (sreq->c_req.encrypt) @@ -1559,7 +1558,7 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) sreq->c_req.c_len = req->cryptlen - authsize; if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { - dev_err(SEC_CTX_DEV(ctx), "aead crypto length error!\n"); + dev_err(dev, "aead crypto length error!\n"); return -EINVAL; } diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h index 0e933e7858e7..9c78edac56a4 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.h +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -64,7 +64,6 @@ enum sec_addr_type { }; struct sec_sqe_type2 { - /* * mac_len: 0~4 bits * a_key_len: 5~10 bits @@ -120,7 +119,6 @@ struct sec_sqe_type2 { /* c_pad_len_field: 0~1 bits */ __le16 c_pad_len_field; - __le64 long_a_data_len; __le64 a_ivin_addr; __le64 a_key_addr; -- cgit From 8d759bec84b1028860287e45cd625bc294a7e16b Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Sat, 13 Mar 2021 15:28:25 +0800 Subject: crypto: hisilicon/sec - fixes some driver coding style cleanup static check errors for SEC Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_main.c | 131 ++++++++++++++++++------------- 1 file changed, 76 insertions(+), 55 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index dc68ba76f65e..78a604394140 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -35,15 +35,13 @@ #define SEC_CTX_Q_NUM_MAX 32 #define SEC_CTRL_CNT_CLR_CE 0x301120 -#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0) -#define SEC_ENGINE_PF_CFG_OFF 0x300000 -#define SEC_ACC_COMMON_REG_OFF 0x1000 +#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0) #define SEC_CORE_INT_SOURCE 0x301010 #define SEC_CORE_INT_MASK 0x301000 #define SEC_CORE_INT_STATUS 0x301008 #define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C14 -#define SEC_ECC_NUM(err) (((err) >> 16) & 0xFF) -#define SEC_ECC_ADDR(err) ((err) >> 0) +#define SEC_ECC_NUM 16 +#define SEC_ECC_MASH 0xFF #define SEC_CORE_INT_DISABLE 0x0 #define SEC_CORE_INT_ENABLE 0x1ff #define SEC_CORE_INT_CLEAR 0x1ff @@ -55,23 +53,23 @@ #define SEC_RAS_CE_ENB_MSK 0x88 #define SEC_RAS_FE_ENB_MSK 0x0 #define SEC_RAS_NFE_ENB_MSK 0x177 -#define SEC_RAS_DISABLE 0x0 -#define SEC_MEM_START_INIT_REG 0x0100 -#define SEC_MEM_INIT_DONE_REG 0x0104 +#define SEC_RAS_DISABLE 0x0 +#define SEC_MEM_START_INIT_REG 0x301100 +#define SEC_MEM_INIT_DONE_REG 0x301104 -#define SEC_CONTROL_REG 0x0200 +#define SEC_CONTROL_REG 0x301200 #define SEC_TRNG_EN_SHIFT 8 #define SEC_CLK_GATE_ENABLE BIT(3) #define SEC_CLK_GATE_DISABLE (~BIT(3)) #define SEC_AXI_SHUTDOWN_ENABLE BIT(12) #define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF -#define SEC_INTERFACE_USER_CTRL0_REG 0x0220 -#define SEC_INTERFACE_USER_CTRL1_REG 0x0224 -#define SEC_SAA_EN_REG 0x0270 -#define SEC_BD_ERR_CHK_EN_REG0 0x0380 -#define SEC_BD_ERR_CHK_EN_REG1 0x0384 -#define SEC_BD_ERR_CHK_EN_REG3 0x038c +#define SEC_INTERFACE_USER_CTRL0_REG 0x301220 +#define SEC_INTERFACE_USER_CTRL1_REG 0x301224 +#define SEC_SAA_EN_REG 0x301270 +#define SEC_BD_ERR_CHK_EN_REG0 0x301380 +#define SEC_BD_ERR_CHK_EN_REG1 0x301384 +#define SEC_BD_ERR_CHK_EN_REG3 0x30138c #define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) #define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) @@ -95,9 +93,6 @@ #define SEC_SQE_MASK_OFFSET 64 #define SEC_SQE_MASK_LEN 48 -#define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \ - SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF) - struct sec_hw_error { u32 int_msk; const char *msg; @@ -117,16 +112,43 @@ static struct hisi_qm_list sec_devices = { }; static const struct sec_hw_error sec_hw_errors[] = { - {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"}, - {.int_msk = BIT(1), .msg = "sec_axi_bresp_err_rint"}, - {.int_msk = BIT(2), .msg = "sec_ecc_2bit_err_rint"}, - {.int_msk = BIT(3), .msg = "sec_ecc_1bit_err_rint"}, - {.int_msk = BIT(4), .msg = "sec_req_trng_timeout_rint"}, - {.int_msk = BIT(5), .msg = "sec_fsm_hbeat_rint"}, - {.int_msk = BIT(6), .msg = "sec_channel_req_rng_timeout_rint"}, - {.int_msk = BIT(7), .msg = "sec_bd_err_rint"}, - {.int_msk = BIT(8), .msg = "sec_chain_buff_err_rint"}, - { /* sentinel */ } + { + .int_msk = BIT(0), + .msg = "sec_axi_rresp_err_rint" + }, + { + .int_msk = BIT(1), + .msg = "sec_axi_bresp_err_rint" + }, + { + .int_msk = BIT(2), + .msg = "sec_ecc_2bit_err_rint" + }, + { + .int_msk = BIT(3), + .msg = "sec_ecc_1bit_err_rint" + }, + { + .int_msk = BIT(4), + .msg = "sec_req_trng_timeout_rint" + }, + { + .int_msk = BIT(5), + .msg = "sec_fsm_hbeat_rint" + }, + { + .int_msk = BIT(6), + .msg = "sec_channel_req_rng_timeout_rint" + }, + { + .int_msk = BIT(7), + .msg = "sec_bd_err_rint" + }, + { + .int_msk = BIT(8), + .msg = "sec_chain_buff_err_rint" + }, + {} }; static const char * const sec_dbg_file_name[] = { @@ -277,9 +299,7 @@ static u8 sec_get_endian(struct hisi_qm *qm) "cannot access a register in VF!\n"); return SEC_LE; } - reg = readl_relaxed(qm->io_base + SEC_ENGINE_PF_CFG_OFF + - SEC_ACC_COMMON_REG_OFF + SEC_CONTROL_REG); - + reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); /* BD little endian mode */ if (!(reg & BIT(0))) return SEC_LE; @@ -299,13 +319,13 @@ static int sec_engine_init(struct hisi_qm *qm) u32 reg; /* disable clock gate control */ - reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); + reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); reg &= SEC_CLK_GATE_DISABLE; - writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); + writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); - writel_relaxed(0x1, SEC_ADDR(qm, SEC_MEM_START_INIT_REG)); + writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG); - ret = readl_relaxed_poll_timeout(SEC_ADDR(qm, SEC_MEM_INIT_DONE_REG), + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG, reg, reg & 0x1, SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); if (ret) { @@ -313,40 +333,40 @@ static int sec_engine_init(struct hisi_qm *qm) return ret; } - reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); + reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); reg |= (0x1 << SEC_TRNG_EN_SHIFT); - writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); + writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); - reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); + reg = readl_relaxed(qm->io_base + SEC_INTERFACE_USER_CTRL0_REG); reg |= SEC_USER0_SMMU_NORMAL; - writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); + writel_relaxed(reg, qm->io_base + SEC_INTERFACE_USER_CTRL0_REG); - reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); + reg = readl_relaxed(qm->io_base + SEC_INTERFACE_USER_CTRL1_REG); reg &= SEC_USER1_SMMU_MASK; if (qm->use_sva && qm->ver == QM_HW_V2) reg |= SEC_USER1_SMMU_SVA; else reg |= SEC_USER1_SMMU_NORMAL; - writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); + writel_relaxed(reg, qm->io_base + SEC_INTERFACE_USER_CTRL1_REG); writel(SEC_SINGLE_PORT_MAX_TRANS, qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS); - writel(SEC_SAA_ENABLE, SEC_ADDR(qm, SEC_SAA_EN_REG)); + writel(SEC_SAA_ENABLE, qm->io_base + SEC_SAA_EN_REG); /* Enable sm4 extra mode, as ctr/ecb */ writel_relaxed(SEC_BD_ERR_CHK_EN0, - SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG0)); + qm->io_base + SEC_BD_ERR_CHK_EN_REG0); /* Enable sm4 xts mode multiple iv */ writel_relaxed(SEC_BD_ERR_CHK_EN1, - SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG1)); + qm->io_base + SEC_BD_ERR_CHK_EN_REG1); writel_relaxed(SEC_BD_ERR_CHK_EN3, - SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG3)); + qm->io_base + SEC_BD_ERR_CHK_EN_REG3); /* config endian */ - reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); + reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); reg |= sec_get_endian(qm); - writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); + writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); return 0; } @@ -406,7 +426,7 @@ static void sec_hw_error_enable(struct hisi_qm *qm) return; } - val = readl(SEC_ADDR(qm, SEC_CONTROL_REG)); + val = readl(qm->io_base + SEC_CONTROL_REG); /* clear SEC hw error source if having */ writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE); @@ -422,14 +442,14 @@ static void sec_hw_error_enable(struct hisi_qm *qm) /* enable SEC block master OOO when m-bit error occur */ val = val | SEC_AXI_SHUTDOWN_ENABLE; - writel(val, SEC_ADDR(qm, SEC_CONTROL_REG)); + writel(val, qm->io_base + SEC_CONTROL_REG); } static void sec_hw_error_disable(struct hisi_qm *qm) { u32 val; - val = readl(SEC_ADDR(qm, SEC_CONTROL_REG)); + val = readl(qm->io_base + SEC_CONTROL_REG); /* disable RAS int */ writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); @@ -442,7 +462,7 @@ static void sec_hw_error_disable(struct hisi_qm *qm) /* disable SEC block master OOO when m-bit error occur */ val = val & SEC_AXI_SHUTDOWN_DISABLE; - writel(val, SEC_ADDR(qm, SEC_CONTROL_REG)); + writel(val, qm->io_base + SEC_CONTROL_REG); } static u32 sec_current_qm_read(struct sec_debug_file *file) @@ -712,7 +732,8 @@ static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) err_val = readl(qm->io_base + SEC_CORE_SRAM_ECC_ERR_INFO); dev_err(dev, "multi ecc sram num=0x%x\n", - SEC_ECC_NUM(err_val)); + ((err_val) >> SEC_ECC_NUM) & + SEC_ECC_MASH); } } errs++; @@ -733,9 +754,9 @@ static void sec_open_axi_master_ooo(struct hisi_qm *qm) { u32 val; - val = readl(SEC_ADDR(qm, SEC_CONTROL_REG)); - writel(val & SEC_AXI_SHUTDOWN_DISABLE, SEC_ADDR(qm, SEC_CONTROL_REG)); - writel(val | SEC_AXI_SHUTDOWN_ENABLE, SEC_ADDR(qm, SEC_CONTROL_REG)); + val = readl(qm->io_base + SEC_CONTROL_REG); + writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG); + writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG); } static const struct hisi_qm_err_ini sec_err_ini = { -- cgit From 45394566b0428e518b59b99745593625d924116d Mon Sep 17 00:00:00 2001 From: Lee Jones Date: Thu, 18 Mar 2021 12:44:13 +0000 Subject: crypto: hisilicon/sec - Supply missing description for 'sec_queue_empty()'s 'queue' param Fixes the following W=1 kernel build warning(s): drivers/crypto/hisilicon/sec/sec_drv.c:843: warning: Function parameter or member 'queue' not described in 'sec_queue_empty' Cc: Herbert Xu Cc: "David S. Miller" Cc: Zaibo Xu Cc: Jonathan Cameron Cc: linux-crypto@vger.kernel.org Signed-off-by: Lee Jones Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec/sec_drv.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c index 91ee2bb575df..3c26871db12f 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.c +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -834,6 +834,7 @@ int sec_queue_stop_release(struct sec_queue *queue) /** * sec_queue_empty() - Is this hardware queue currently empty. + * @queue: The queue to test * * We need to know if we have an empty queue for some of the chaining modes * as if it is not empty we may need to hold the message in a software queue -- cgit From bbe6c4ba518d82974aab3361a445d60c0785d0cc Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Fri, 19 Mar 2021 18:44:18 +0800 Subject: crypto: hisilicon/hpre - delete wrap of 'CONFIG_CRYPTO_DH' 'CRYPTO_DH' has selected in 'Kconfig', so delete 'CONFIG_CRYPTO_DH'. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 53068d2a19cf..50ccd1034aae 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -546,7 +546,6 @@ static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) return ret; } -#ifdef CONFIG_CRYPTO_DH static int hpre_dh_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); @@ -719,7 +718,6 @@ static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) hpre_dh_clear_ctx(ctx, true); } -#endif static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) { @@ -1893,7 +1891,6 @@ static struct akcipher_alg rsa = { }, }; -#ifdef CONFIG_CRYPTO_DH static struct kpp_alg dh = { .set_secret = hpre_dh_set_secret, .generate_public_key = hpre_dh_compute_value, @@ -1910,7 +1907,6 @@ static struct kpp_alg dh = { .cra_module = THIS_MODULE, }, }; -#endif static struct kpp_alg ecdh_nist_p192 = { .set_secret = hpre_ecdh_set_secret, @@ -1995,13 +1991,12 @@ int hpre_algs_register(struct hisi_qm *qm) ret = crypto_register_akcipher(&rsa); if (ret) return ret; -#ifdef CONFIG_CRYPTO_DH + ret = crypto_register_kpp(&dh); if (ret) { crypto_unregister_akcipher(&rsa); return ret; } -#endif if (qm->ver >= QM_HW_V3) { ret = hpre_register_ecdh(); @@ -2016,9 +2011,7 @@ int hpre_algs_register(struct hisi_qm *qm) return 0; reg_err: -#ifdef CONFIG_CRYPTO_DH crypto_unregister_kpp(&dh); -#endif crypto_unregister_akcipher(&rsa); return ret; } @@ -2030,8 +2023,6 @@ void hpre_algs_unregister(struct hisi_qm *qm) hpre_unregister_ecdh(); } -#ifdef CONFIG_CRYPTO_DH crypto_unregister_kpp(&dh); -#endif crypto_unregister_akcipher(&rsa); } -- cgit From ed48466d3ff94fac09ca5c521a24501eb5908277 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Fri, 19 Mar 2021 18:44:19 +0800 Subject: crypto: hisilicon/hpre - optimise 'hpre_algs_register' error path There is redundant code especially when registing new algorithms in the future. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 50ccd1034aae..b77473be5131 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1993,25 +1993,24 @@ int hpre_algs_register(struct hisi_qm *qm) return ret; ret = crypto_register_kpp(&dh); - if (ret) { - crypto_unregister_akcipher(&rsa); - return ret; - } + if (ret) + goto unreg_rsa; if (qm->ver >= QM_HW_V3) { ret = hpre_register_ecdh(); if (ret) - goto reg_err; + goto unreg_dh; ret = crypto_register_kpp(&curve25519_alg); - if (ret) { - hpre_unregister_ecdh(); - goto reg_err; - } + if (ret) + goto unreg_ecdh; } return 0; -reg_err: +unreg_ecdh: + hpre_unregister_ecdh(); +unreg_dh: crypto_unregister_kpp(&dh); +unreg_rsa: crypto_unregister_akcipher(&rsa); return ret; } -- cgit From a9214b0b6ed245b0c5239e0576b7de509815c2a0 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Fri, 19 Mar 2021 18:45:05 +0800 Subject: crypto: hisilicon - fix the check on dma address System may be able to get physical address of zero if not reserved by firmware. The dma address obtained by 'dma_alloc_coherent' is valid, since already checking cpu va before, so do not check again. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 12 ------------ drivers/crypto/hisilicon/qm.c | 2 +- drivers/crypto/hisilicon/sec2/sec_crypto.c | 6 ------ 3 files changed, 1 insertion(+), 19 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index b77473be5131..c90d0b82c3fa 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -298,8 +298,6 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, dma_addr_t tmp; tmp = le64_to_cpu(sqe->in); - if (unlikely(!tmp)) - return; if (src) { if (req->src) @@ -309,8 +307,6 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, } tmp = le64_to_cpu(sqe->out); - if (unlikely(!tmp)) - return; if (req->dst) { if (dst) @@ -1358,15 +1354,11 @@ static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, dma_addr_t dma; dma = le64_to_cpu(sqe->in); - if (unlikely(!dma)) - return; if (src && req->src) dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); dma = le64_to_cpu(sqe->out); - if (unlikely(!dma)) - return; if (req->dst) dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); @@ -1657,15 +1649,11 @@ static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, dma_addr_t dma; dma = le64_to_cpu(sqe->in); - if (unlikely(!dma)) - return; if (src && req->src) dma_free_coherent(dev, ctx->key_sz, req->src, dma); dma = le64_to_cpu(sqe->out); - if (unlikely(!dma)) - return; if (req->dst) dma_free_coherent(dev, ctx->key_sz, req->dst, dma); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index bc231742ad36..e3688bf74604 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2681,7 +2681,7 @@ static int __hisi_qm_start(struct hisi_qm *qm) { int ret; - WARN_ON(!qm->qdma.dma); + WARN_ON(!qm->qdma.va); if (qm->fun_type == QM_HW_PF) { ret = qm_dev_mem_reset(qm); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index c0efa611f2be..c74082ced4a0 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -708,12 +708,6 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, } c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; - - if (!c_req->c_in_dma) { - dev_err(dev, "fail to set pbuffer address!\n"); - return -ENOMEM; - } - c_req->c_out_dma = c_req->c_in_dma; return 0; -- cgit From 670fefb9240950fe9e3e4740039a7e13e5910470 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Fri, 19 Mar 2021 18:45:27 +0800 Subject: crypto: hisilicon/hpre - fix "hpre_ctx_init" resource leak When calling "hpre_ctx_set" fails, stop and put qp, otherwise will leak qp resource. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index c90d0b82c3fa..d743c540d602 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -454,9 +454,16 @@ static void hpre_alg_cb(struct hisi_qp *qp, void *resp) req->cb(ctx, resp); } +static void hpre_stop_qp_and_put(struct hisi_qp *qp) +{ + hisi_qm_stop_qp(qp); + hisi_qm_free_qps(&qp, 1); +} + static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) { struct hisi_qp *qp; + int ret; qp = hpre_get_qp_and_start(type); if (IS_ERR(qp)) @@ -465,7 +472,11 @@ static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) qp->qp_ctx = ctx; qp->req_cb = hpre_alg_cb; - return hpre_ctx_set(ctx, qp, QM_Q_DEPTH); + ret = hpre_ctx_set(ctx, qp, QM_Q_DEPTH); + if (ret) + hpre_stop_qp_and_put(qp); + + return ret; } static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) -- cgit From 7d15697983c720c4405ce47b30520d0c7dd47487 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Fri, 19 Mar 2021 18:45:39 +0800 Subject: crypto: hisilicon/hpre - fix Kconfig hpre select 'CRYPTO_ECDH' and 'CRYPTO_CURVE25519'. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index c45adb15ce8d..e572f9982d4e 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -65,10 +65,11 @@ config CRYPTO_DEV_HISI_HPRE depends on UACCE || UACCE=n depends on ARM64 || (COMPILE_TEST && 64BIT) depends on ACPI - select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_DEV_HISI_QM select CRYPTO_DH select CRYPTO_RSA + select CRYPTO_CURVE25519 + select CRYPTO_ECDH help Support for HiSilicon HPRE(High Performance RSA Engine) accelerator, which can accelerate RSA and DH algorithms. -- cgit From 45bb26d946cd89c08e6b8410a76b9bf3614c9d78 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 20 Mar 2021 19:27:43 +0800 Subject: crypto: hisilicon/qm - set the total number of queues Move the configuration of the total number of queues 'ctrl_qp_num' from sec2/hpre/zip to qm.c. And get the total number of queues from the hardware register for Kunpeng930. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 3 --- drivers/crypto/hisilicon/qm.c | 16 ++++++++++++++++ drivers/crypto/hisilicon/sec2/sec_main.c | 6 ------ drivers/crypto/hisilicon/zip/zip_main.c | 7 ------- 4 files changed, 16 insertions(+), 16 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 87e8f4d60474..818e7f3dc825 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -13,7 +13,6 @@ #include #include "hpre.h" -#define HPRE_QUEUE_NUM_V2 1024 #define HPRE_QM_ABNML_INT_MASK 0x100004 #define HPRE_CTRL_CNT_CLR_CE_BIT BIT(0) #define HPRE_COMM_CNT_CLR_CE 0x0 @@ -899,8 +898,6 @@ static int hpre_pf_probe_init(struct hpre *hpre) struct hisi_qm *qm = &hpre->qm; int ret; - qm->ctrl_qp_num = HPRE_QUEUE_NUM_V2; - ret = hpre_set_user_domain_and_cache(qm); if (ret) return ret; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index e3688bf74604..d1d51378bd58 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -93,6 +93,8 @@ #define QM_DB_PRIORITY_SHIFT_V1 48 #define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000 #define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000 +#define QM_CAPBILITY 0x100158 +#define QM_QP_NUN_MASK GENMASK(10, 0) #define QM_DB_CMD_SHIFT_V2 12 #define QM_DB_RAND_SHIFT_V2 16 #define QM_DB_INDEX_SHIFT_V2 32 @@ -4119,6 +4121,17 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); +static void qm_get_qp_num(struct hisi_qm *qm) +{ + if (qm->ver == QM_HW_V1) + qm->ctrl_qp_num = QM_QNUM_V1; + else if (qm->ver == QM_HW_V2) + qm->ctrl_qp_num = QM_QNUM_V2; + else + qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) & + QM_QP_NUN_MASK; +} + static int hisi_qm_pci_init(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -4146,6 +4159,9 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) goto err_release_mem_regions; } + if (qm->fun_type == QM_HW_PF) + qm_get_qp_num(qm); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (ret < 0) goto err_iounmap; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 78a604394140..2360d31417aa 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -19,7 +19,6 @@ #define SEC_VF_NUM 63 #define SEC_QUEUE_NUM_V1 4096 -#define SEC_QUEUE_NUM_V2 1024 #define SEC_PF_PCI_DEVICE_ID 0xa255 #define SEC_VF_PCI_DEVICE_ID 0xa256 @@ -784,11 +783,6 @@ static int sec_pf_probe_init(struct sec_dev *sec) struct hisi_qm *qm = &sec->qm; int ret; - if (qm->ver == QM_HW_V1) - qm->ctrl_qp_num = SEC_QUEUE_NUM_V1; - else - qm->ctrl_qp_num = SEC_QUEUE_NUM_V2; - qm->err_ini = &sec_err_ini; ret = sec_set_user_domain_and_cache(qm); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 02c445722445..8f67d701ea3d 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -18,7 +18,6 @@ #define PCI_DEVICE_ID_ZIP_VF 0xa251 #define HZIP_QUEUE_NUM_V1 4096 -#define HZIP_QUEUE_NUM_V2 1024 #define HZIP_CLOCK_GATE_CTRL 0x301004 #define COMP0_ENABLE BIT(0) @@ -746,12 +745,6 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) hisi_zip->ctrl = ctrl; ctrl->hisi_zip = hisi_zip; - - if (qm->ver == QM_HW_V1) - qm->ctrl_qp_num = HZIP_QUEUE_NUM_V1; - else - qm->ctrl_qp_num = HZIP_QUEUE_NUM_V2; - qm->err_ini = &hisi_zip_err_ini; hisi_zip_set_user_domain_and_cache(qm); -- cgit From c4392b46ee95be9815e682a1c8cb0aa2f92f07e2 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 20 Mar 2021 19:27:44 +0800 Subject: crypto: hisilicon/qm - move 'CURRENT_QM' code to qm.c Since the code related to 'CURRENT_QM' debugfs is exactly same in sec/hpre/zip driver, move 'CURRENT_QM' to qm.c to reduce duplicate code. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre.h | 1 - drivers/crypto/hisilicon/hpre/hpre_main.c | 61 ---------------------- drivers/crypto/hisilicon/qm.c | 87 ++++++++++++++++++++++++++----- drivers/crypto/hisilicon/qm.h | 9 +--- drivers/crypto/hisilicon/sec2/sec.h | 1 - drivers/crypto/hisilicon/sec2/sec_main.c | 59 +-------------------- drivers/crypto/hisilicon/zip/zip_main.c | 58 +-------------------- 7 files changed, 76 insertions(+), 200 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 92892e373cd0..e0b4a1982ee9 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -26,7 +26,6 @@ enum { }; enum hpre_ctrl_dbgfs_file { - HPRE_CURRENT_QM, HPRE_CLEAR_ENABLE, HPRE_CLUSTER_CTRL, HPRE_DEBUG_FILE_NUM, diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 818e7f3dc825..a8b9c4cf42ff 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -118,7 +118,6 @@ static struct hisi_qm_list hpre_devices = { }; static const char * const hpre_debug_file_name[] = { - [HPRE_CURRENT_QM] = "current_qm", [HPRE_CLEAR_ENABLE] = "rdclr_en", [HPRE_CLUSTER_CTRL] = "cluster_ctrl", }; @@ -407,10 +406,6 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm) unsigned long offset; int i; - /* clear current_qm */ - writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); - writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); - /* clear clusterX/cluster_ctrl */ for (i = 0; i < clusters_num; i++) { offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; @@ -462,49 +457,6 @@ static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) return &hpre->qm; } -static u32 hpre_current_qm_read(struct hpre_debugfs_file *file) -{ - struct hisi_qm *qm = hpre_file_to_qm(file); - - return readl(qm->io_base + QM_DFX_MB_CNT_VF); -} - -static int hpre_current_qm_write(struct hpre_debugfs_file *file, u32 val) -{ - struct hisi_qm *qm = hpre_file_to_qm(file); - u32 num_vfs = qm->vfs_num; - u32 vfq_num, tmp; - - if (val > num_vfs) - return -EINVAL; - - /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ - if (val == 0) { - qm->debug.curr_qm_qp_num = qm->qp_num; - } else { - vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs; - if (val == num_vfs) { - qm->debug.curr_qm_qp_num = - qm->ctrl_qp_num - qm->qp_num - (num_vfs - 1) * vfq_num; - } else { - qm->debug.curr_qm_qp_num = vfq_num; - } - } - - writel(val, qm->io_base + QM_DFX_MB_CNT_VF); - writel(val, qm->io_base + QM_DFX_DB_CNT_VF); - - tmp = val | - (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); - writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); - - tmp = val | - (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); - writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); - - return 0; -} - static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file) { struct hisi_qm *qm = hpre_file_to_qm(file); @@ -560,9 +512,6 @@ static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf, spin_lock_irq(&file->lock); switch (file->type) { - case HPRE_CURRENT_QM: - val = hpre_current_qm_read(file); - break; case HPRE_CLEAR_ENABLE: val = hpre_clear_enable_read(file); break; @@ -603,11 +552,6 @@ static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf, spin_lock_irq(&file->lock); switch (file->type) { - case HPRE_CURRENT_QM: - ret = hpre_current_qm_write(file, val); - if (ret) - goto err_input; - break; case HPRE_CLEAR_ENABLE: ret = hpre_clear_enable_write(file, val); if (ret) @@ -746,11 +690,6 @@ static int hpre_ctrl_debug_init(struct hisi_qm *qm) { int ret; - ret = hpre_create_debugfs_file(qm, NULL, HPRE_CURRENT_QM, - HPRE_CURRENT_QM); - if (ret) - return ret; - ret = hpre_create_debugfs_file(qm, NULL, HPRE_CLEAR_ENABLE, HPRE_CLEAR_ENABLE); if (ret) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d1d51378bd58..8b9fb809cc6d 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -166,6 +166,14 @@ #define ACC_AM_ROB_ECC_INT_STS 0x300104 #define ACC_ROB_ECC_ERR_MULTPL BIT(1) +#define QM_DFX_MB_CNT_VF 0x104010 +#define QM_DFX_DB_CNT_VF 0x104020 +#define QM_DFX_SQE_CNT_VF_SQN 0x104030 +#define QM_DFX_CQE_CNT_VF_CQN 0x104040 +#define QM_DFX_QN_SHIFT 16 +#define CURRENT_FUN_MASK GENMASK(5, 0) +#define CURRENT_Q_MASK GENMASK(31, 16) + #define POLL_PERIOD 10 #define POLL_TIMEOUT 1000 #define WAIT_PERIOD_US_MAX 200 @@ -352,6 +360,7 @@ static struct qm_dfx_item qm_dfx_files[] = { }; static const char * const qm_debug_file_name[] = { + [CURRENT_QM] = "current_qm", [CURRENT_Q] = "current_q", [CLEAR_ENABLE] = "clear_enable", }; @@ -920,6 +929,50 @@ static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl) return 0; } +static u32 current_qm_read(struct debugfs_file *file) +{ + struct hisi_qm *qm = file_to_qm(file); + + return readl(qm->io_base + QM_DFX_MB_CNT_VF); +} + +static int current_qm_write(struct debugfs_file *file, u32 val) +{ + struct hisi_qm *qm = file_to_qm(file); + u32 vfq_num; + u32 tmp; + + if (val > qm->vfs_num) + return -EINVAL; + + /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ + if (!val) { + qm->debug.curr_qm_qp_num = qm->qp_num; + } else { + vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num; + + if (val == qm->vfs_num) + qm->debug.curr_qm_qp_num = + qm->ctrl_qp_num - qm->qp_num - + (qm->vfs_num - 1) * vfq_num; + else + qm->debug.curr_qm_qp_num = vfq_num; + } + + writel(val, qm->io_base + QM_DFX_MB_CNT_VF); + writel(val, qm->io_base + QM_DFX_DB_CNT_VF); + + tmp = val | + (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); + writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); + + tmp = val | + (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); + writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); + + return 0; +} + static ssize_t qm_debug_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { @@ -931,6 +984,9 @@ static ssize_t qm_debug_read(struct file *filp, char __user *buf, mutex_lock(&file->lock); switch (index) { + case CURRENT_QM: + val = current_qm_read(file); + break; case CURRENT_Q: val = current_q_read(file); break; @@ -973,27 +1029,24 @@ static ssize_t qm_debug_write(struct file *filp, const char __user *buf, mutex_lock(&file->lock); switch (index) { + case CURRENT_QM: + ret = current_qm_write(file, val); + break; case CURRENT_Q: ret = current_q_write(file, val); - if (ret) - goto err_input; break; case CLEAR_ENABLE: ret = clear_enable_write(file, val); - if (ret) - goto err_input; break; default: ret = -EINVAL; - goto err_input; } mutex_unlock(&file->lock); - return count; + if (ret) + return ret; -err_input: - mutex_unlock(&file->lock); - return ret; + return count; } static const struct file_operations qm_debug_fops = { @@ -1531,12 +1584,12 @@ static const struct file_operations qm_cmd_fops = { .write = qm_cmd_write, }; -static void qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) +static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir, + enum qm_debug_file index) { - struct dentry *qm_d = qm->debug.qm_d; struct debugfs_file *file = qm->debug.files + index; - debugfs_create_file(qm_debug_file_name[index], 0600, qm_d, file, + debugfs_create_file(qm_debug_file_name[index], 0600, dir, file, &qm_debug_fops); file->index = index; @@ -2932,9 +2985,11 @@ void hisi_qm_debug_init(struct hisi_qm *qm) qm->debug.qm_d = qm_d; /* only show this in PF */ - if (qm->fun_type == QM_HW_PF) + if (qm->fun_type == QM_HW_PF) { + qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM); for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) - qm_create_debugfs_file(qm, i); + qm_create_debugfs_file(qm, qm_d, i); + } debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); @@ -2962,6 +3017,10 @@ void hisi_qm_debug_regs_clear(struct hisi_qm *qm) struct qm_dfx_registers *regs; int i; + /* clear current_qm */ + writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); + writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + /* clear current_q */ writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index f91110fcf6a4..5c4797b005bf 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -51,14 +51,6 @@ #define PEH_AXUSER_CFG 0x401001 #define PEH_AXUSER_CFG_ENABLE 0xffffffff -#define QM_DFX_MB_CNT_VF 0x104010 -#define QM_DFX_DB_CNT_VF 0x104020 -#define QM_DFX_SQE_CNT_VF_SQN 0x104030 -#define QM_DFX_CQE_CNT_VF_CQN 0x104040 -#define QM_DFX_QN_SHIFT 16 -#define CURRENT_FUN_MASK GENMASK(5, 0) -#define CURRENT_Q_MASK GENMASK(31, 16) - #define QM_AXI_RRESP BIT(0) #define QM_AXI_BRESP BIT(1) #define QM_ECC_MBIT BIT(2) @@ -123,6 +115,7 @@ enum qm_fun_type { }; enum qm_debug_file { + CURRENT_QM, CURRENT_Q, CLEAR_ENABLE, DEBUG_FILE_NUM, diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index f7eba9bd312b..dfdce2f21e65 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -147,7 +147,6 @@ enum sec_endian { }; enum sec_debug_file_index { - SEC_CURRENT_QM, SEC_CLEAR_ENABLE, SEC_DEBUG_FILE_NUM, }; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2360d31417aa..b1818f793520 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -151,7 +151,6 @@ static const struct sec_hw_error sec_hw_errors[] = { }; static const char * const sec_dbg_file_name[] = { - [SEC_CURRENT_QM] = "current_qm", [SEC_CLEAR_ENABLE] = "clear_enable", }; @@ -400,10 +399,6 @@ static void sec_debug_regs_clear(struct hisi_qm *qm) { int i; - /* clear current_qm */ - writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); - writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); - /* clear sec dfx regs */ writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE); for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) @@ -464,50 +459,6 @@ static void sec_hw_error_disable(struct hisi_qm *qm) writel(val, qm->io_base + SEC_CONTROL_REG); } -static u32 sec_current_qm_read(struct sec_debug_file *file) -{ - struct hisi_qm *qm = file->qm; - - return readl(qm->io_base + QM_DFX_MB_CNT_VF); -} - -static int sec_current_qm_write(struct sec_debug_file *file, u32 val) -{ - struct hisi_qm *qm = file->qm; - u32 vfq_num; - u32 tmp; - - if (val > qm->vfs_num) - return -EINVAL; - - /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ - if (!val) { - qm->debug.curr_qm_qp_num = qm->qp_num; - } else { - vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num; - - if (val == qm->vfs_num) - qm->debug.curr_qm_qp_num = - qm->ctrl_qp_num - qm->qp_num - - (qm->vfs_num - 1) * vfq_num; - else - qm->debug.curr_qm_qp_num = vfq_num; - } - - writel(val, qm->io_base + QM_DFX_MB_CNT_VF); - writel(val, qm->io_base + QM_DFX_DB_CNT_VF); - - tmp = val | - (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); - writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); - - tmp = val | - (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); - writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); - - return 0; -} - static u32 sec_clear_enable_read(struct sec_debug_file *file) { struct hisi_qm *qm = file->qm; @@ -542,9 +493,6 @@ static ssize_t sec_debug_read(struct file *filp, char __user *buf, spin_lock_irq(&file->lock); switch (file->index) { - case SEC_CURRENT_QM: - val = sec_current_qm_read(file); - break; case SEC_CLEAR_ENABLE: val = sec_clear_enable_read(file); break; @@ -585,11 +533,6 @@ static ssize_t sec_debug_write(struct file *filp, const char __user *buf, spin_lock_irq(&file->lock); switch (file->index) { - case SEC_CURRENT_QM: - ret = sec_current_qm_write(file, val); - if (ret) - goto err_input; - break; case SEC_CLEAR_ENABLE: ret = sec_clear_enable_write(file, val); if (ret) @@ -674,7 +617,7 @@ static int sec_debug_init(struct hisi_qm *qm) int i; if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) { - for (i = SEC_CURRENT_QM; i < SEC_DEBUG_FILE_NUM; i++) { + for (i = SEC_CLEAR_ENABLE; i < SEC_DEBUG_FILE_NUM; i++) { spin_lock_init(&sec->debug.files[i].lock); sec->debug.files[i].index = i; sec->debug.files[i].qm = qm; diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 8f67d701ea3d..4d2e9c85cc1a 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -137,13 +137,11 @@ static const struct hisi_zip_hw_error zip_hw_error[] = { }; enum ctrl_debug_file_index { - HZIP_CURRENT_QM, HZIP_CLEAR_ENABLE, HZIP_DEBUG_FILE_NUM, }; static const char * const ctrl_debug_file_name[] = { - [HZIP_CURRENT_QM] = "current_qm", [HZIP_CLEAR_ENABLE] = "clear_enable", }; @@ -362,48 +360,6 @@ static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) return &hisi_zip->qm; } -static u32 current_qm_read(struct ctrl_debug_file *file) -{ - struct hisi_qm *qm = file_to_qm(file); - - return readl(qm->io_base + QM_DFX_MB_CNT_VF); -} - -static int current_qm_write(struct ctrl_debug_file *file, u32 val) -{ - struct hisi_qm *qm = file_to_qm(file); - u32 vfq_num; - u32 tmp; - - if (val > qm->vfs_num) - return -EINVAL; - - /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ - if (val == 0) { - qm->debug.curr_qm_qp_num = qm->qp_num; - } else { - vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num; - if (val == qm->vfs_num) - qm->debug.curr_qm_qp_num = qm->ctrl_qp_num - - qm->qp_num - (qm->vfs_num - 1) * vfq_num; - else - qm->debug.curr_qm_qp_num = vfq_num; - } - - writel(val, qm->io_base + QM_DFX_MB_CNT_VF); - writel(val, qm->io_base + QM_DFX_DB_CNT_VF); - - tmp = val | - (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); - writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); - - tmp = val | - (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); - writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); - - return 0; -} - static u32 clear_enable_read(struct ctrl_debug_file *file) { struct hisi_qm *qm = file_to_qm(file); @@ -437,9 +393,6 @@ static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf, spin_lock_irq(&file->lock); switch (file->index) { - case HZIP_CURRENT_QM: - val = current_qm_read(file); - break; case HZIP_CLEAR_ENABLE: val = clear_enable_read(file); break; @@ -477,11 +430,6 @@ static ssize_t hisi_zip_ctrl_debug_write(struct file *filp, spin_lock_irq(&file->lock); switch (file->index) { - case HZIP_CURRENT_QM: - ret = current_qm_write(file, val); - if (ret) - goto err_input; - break; case HZIP_CLEAR_ENABLE: ret = clear_enable_write(file, val); if (ret) @@ -579,7 +527,7 @@ static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm) struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm); int i; - for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) { + for (i = HZIP_CLEAR_ENABLE; i < HZIP_DEBUG_FILE_NUM; i++) { spin_lock_init(&zip->ctrl->files[i].lock); zip->ctrl->files[i].ctrl = zip->ctrl; zip->ctrl->files[i].index = i; @@ -626,10 +574,6 @@ static void hisi_zip_debug_regs_clear(struct hisi_qm *qm) { int i, j; - /* clear current_qm */ - writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); - writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); - /* enable register read_clear bit */ writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); for (i = 0; i < ARRAY_SIZE(core_offsets); i++) -- cgit From 6250383a2083e8f66635d441977f74e0ee4e52f7 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 20 Mar 2021 19:27:45 +0800 Subject: crypto: hisilicon/qm - set the number of queues for function Kunpeng930 supports queue doorbell isolation. When doorbell isolation is enabled, it supports to obtain the maximum number of queues of one function from hardware register. Otherwise, the 'max_qp_num' is the total number of queues. When assigning queues to VF, it is necessary to ensure that the number of VF queues does not exceed 'max_qp_num'. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 100 +++++++++++++++++++++++++++++++----------- drivers/crypto/hisilicon/qm.h | 4 ++ 2 files changed, 78 insertions(+), 26 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 8b9fb809cc6d..912ae117f130 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -93,8 +93,11 @@ #define QM_DB_PRIORITY_SHIFT_V1 48 #define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000 #define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000 +#define QM_QUE_ISO_CFG_V 0x0030 +#define QM_QUE_ISO_EN 0x100154 #define QM_CAPBILITY 0x100158 #define QM_QP_NUN_MASK GENMASK(10, 0) +#define QM_QP_MAX_NUM_SHIFT 11 #define QM_DB_CMD_SHIFT_V2 12 #define QM_DB_RAND_SHIFT_V2 16 #define QM_DB_INDEX_SHIFT_V2 32 @@ -876,6 +879,26 @@ static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) return 0; } +static int qm_get_vf_qp_num(struct hisi_qm *qm, u32 fun_num) +{ + u32 remain_q_num, vfq_num; + u32 num_vfs = qm->vfs_num; + + vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs; + if (vfq_num >= qm->max_qp_num) + return qm->max_qp_num; + + remain_q_num = (qm->ctrl_qp_num - qm->qp_num) % num_vfs; + if (vfq_num + remain_q_num <= qm->max_qp_num) + return fun_num == num_vfs ? vfq_num + remain_q_num : vfq_num; + + /* + * if vfq_num + remain_q_num > max_qp_num, the last VFs, + * each with one more queue. + */ + return fun_num + remain_q_num > num_vfs ? vfq_num + 1 : vfq_num; +} + static struct hisi_qm *file_to_qm(struct debugfs_file *file) { struct qm_debug *debug = file->debug; @@ -939,25 +962,16 @@ static u32 current_qm_read(struct debugfs_file *file) static int current_qm_write(struct debugfs_file *file, u32 val) { struct hisi_qm *qm = file_to_qm(file); - u32 vfq_num; u32 tmp; if (val > qm->vfs_num) return -EINVAL; /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ - if (!val) { + if (!val) qm->debug.curr_qm_qp_num = qm->qp_num; - } else { - vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num; - - if (val == qm->vfs_num) - qm->debug.curr_qm_qp_num = - qm->ctrl_qp_num - qm->qp_num - - (qm->vfs_num - 1) * vfq_num; - else - qm->debug.curr_qm_qp_num = vfq_num; - } + else + qm->debug.curr_qm_qp_num = qm_get_vf_qp_num(qm, val); writel(val, qm->io_base + QM_DFX_MB_CNT_VF); writel(val, qm->io_base + QM_DFX_DB_CNT_VF); @@ -3236,30 +3250,46 @@ EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node); static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) { - u32 remain_q_num, q_num, i, j; + u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j; + u32 max_qp_num = qm->max_qp_num; u32 q_base = qm->qp_num; int ret; if (!num_vfs) return -EINVAL; - remain_q_num = qm->ctrl_qp_num - qm->qp_num; + vfs_q_num = qm->ctrl_qp_num - qm->qp_num; - /* If remain queues not enough, return error. */ - if (qm->ctrl_qp_num < qm->qp_num || remain_q_num < num_vfs) + /* If vfs_q_num is less than num_vfs, return error. */ + if (vfs_q_num < num_vfs) return -EINVAL; - q_num = remain_q_num / num_vfs; - for (i = 1; i <= num_vfs; i++) { - if (i == num_vfs) - q_num += remain_q_num % num_vfs; - ret = hisi_qm_set_vft(qm, i, q_base, q_num); + q_num = vfs_q_num / num_vfs; + remain_q_num = vfs_q_num % num_vfs; + + for (i = num_vfs; i > 0; i--) { + /* + * if q_num + remain_q_num > max_qp_num in last vf, divide the + * remaining queues equally. + */ + if (i == num_vfs && q_num + remain_q_num <= max_qp_num) { + act_q_num = q_num + remain_q_num; + remain_q_num = 0; + } else if (remain_q_num > 0) { + act_q_num = q_num + 1; + remain_q_num--; + } else { + act_q_num = q_num; + } + + act_q_num = min_t(int, act_q_num, max_qp_num); + ret = hisi_qm_set_vft(qm, i, q_base, act_q_num); if (ret) { - for (j = i; j > 0; j--) + for (j = num_vfs; j > i; j--) hisi_qm_set_vft(qm, j, 0, 0); return ret; } - q_base += q_num; + q_base += act_q_num; } return 0; @@ -4180,7 +4210,7 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); -static void qm_get_qp_num(struct hisi_qm *qm) +static int qm_get_qp_num(struct hisi_qm *qm) { if (qm->ver == QM_HW_V1) qm->ctrl_qp_num = QM_QNUM_V1; @@ -4189,6 +4219,21 @@ static void qm_get_qp_num(struct hisi_qm *qm) else qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) & QM_QP_NUN_MASK; + + if (qm->use_db_isolation) + qm->max_qp_num = (readl(qm->io_base + QM_CAPBILITY) >> + QM_QP_MAX_NUM_SHIFT) & QM_QP_NUN_MASK; + else + qm->max_qp_num = qm->ctrl_qp_num; + + /* check if qp number is valid */ + if (qm->qp_num > qm->max_qp_num) { + dev_err(&qm->pdev->dev, "qp num(%u) is more than max qp num(%u)!\n", + qm->qp_num, qm->max_qp_num); + return -EINVAL; + } + + return 0; } static int hisi_qm_pci_init(struct hisi_qm *qm) @@ -4218,8 +4263,11 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) goto err_release_mem_regions; } - if (qm->fun_type == QM_HW_PF) - qm_get_qp_num(qm); + if (qm->fun_type == QM_HW_PF) { + ret = qm_get_qp_num(qm); + if (ret) + goto err_iounmap; + } ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (ret < 0) diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 5c4797b005bf..cf0290f35680 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -207,6 +207,7 @@ struct hisi_qm { u32 qp_num; u32 qp_in_used; u32 ctrl_qp_num; + u32 max_qp_num; u32 vfs_num; struct list_head list; struct hisi_qm_list *qm_list; @@ -245,6 +246,9 @@ struct hisi_qm { const char *algs; bool use_sva; bool is_frozen; + + /* doorbell isolation enable */ + bool use_db_isolation; resource_size_t phys_base; resource_size_t phys_size; struct uacce_device *uacce; -- cgit From 8bbecfb402f76b6977a6c5661ad3cfb0051a9776 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 20 Mar 2021 19:27:46 +0800 Subject: crypto: hisilicon/qm - add queue isolation support for Kunpeng930 Kunpeng930 supports doorbell isolation to ensure that each queue has an independent doorbell address space. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 161 ++++++++++++++++++++++++++++++------------ drivers/crypto/hisilicon/qm.h | 4 +- 2 files changed, 117 insertions(+), 48 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 912ae117f130..c91e57a62cdb 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -97,6 +97,7 @@ #define QM_QUE_ISO_EN 0x100154 #define QM_CAPBILITY 0x100158 #define QM_QP_NUN_MASK GENMASK(10, 0) +#define QM_QP_DB_INTERVAL 0x10000 #define QM_QP_MAX_NUM_SHIFT 11 #define QM_DB_CMD_SHIFT_V2 12 #define QM_DB_RAND_SHIFT_V2 16 @@ -186,6 +187,7 @@ #define QM_CACHE_WB_DONE 0x208 #define PCI_BAR_2 2 +#define PCI_BAR_4 4 #define QM_SQE_DATA_ALIGN_MASK GENMASK(6, 0) #define QMC_ALIGN(sz) ALIGN(sz, 32) @@ -571,21 +573,22 @@ static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) { - u64 doorbell; - u64 dbase; + void __iomem *io_base = qm->io_base; u16 randata = 0; + u64 doorbell; if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) - dbase = QM_DOORBELL_SQ_CQ_BASE_V2; + io_base = qm->db_io_base + (u64)qn * qm->db_interval + + QM_DOORBELL_SQ_CQ_BASE_V2; else - dbase = QM_DOORBELL_EQ_AEQ_BASE_V2; + io_base += QM_DOORBELL_EQ_AEQ_BASE_V2; doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | ((u64)randata << QM_DB_RAND_SHIFT_V2) | ((u64)index << QM_DB_INDEX_SHIFT_V2) | ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); - writeq(doorbell, qm->io_base + dbase); + writeq(doorbell, io_base); } static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) @@ -2201,6 +2204,8 @@ static int hisi_qm_uacce_mmap(struct uacce_queue *q, { struct hisi_qp *qp = q->priv; struct hisi_qm *qm = qp->qm; + resource_size_t phys_base = qm->db_phys_base + + qp->qp_id * qm->db_interval; size_t sz = vma->vm_end - vma->vm_start; struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; @@ -2212,16 +2217,19 @@ static int hisi_qm_uacce_mmap(struct uacce_queue *q, if (qm->ver == QM_HW_V1) { if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR) return -EINVAL; - } else { + } else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) { if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE)) return -EINVAL; + } else { + if (sz > qm->db_interval) + return -EINVAL; } vma->vm_flags |= VM_IO; return remap_pfn_range(vma, vma->vm_start, - qm->phys_base >> PAGE_SHIFT, + phys_base >> PAGE_SHIFT, sz, pgprot_noncached(vma->vm_page_prot)); case UACCE_QFRT_DUS: if (sz != qp->qdma.size) @@ -2336,14 +2344,20 @@ static int qm_alloc_uacce(struct hisi_qm *qm) uacce->priv = qm; uacce->algs = qm->algs; - if (qm->ver == QM_HW_V1) { - mmio_page_nr = QM_DOORBELL_PAGE_NR; + if (qm->ver == QM_HW_V1) uacce->api_ver = HISI_QM_API_VER_BASE; - } else { + else if (qm->ver == QM_HW_V2) + uacce->api_ver = HISI_QM_API_VER2_BASE; + else + uacce->api_ver = HISI_QM_API_VER3_BASE; + + if (qm->ver == QM_HW_V1) + mmio_page_nr = QM_DOORBELL_PAGE_NR; + else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) mmio_page_nr = QM_DOORBELL_PAGE_NR + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; - uacce->api_ver = HISI_QM_API_VER2_BASE; - } + else + mmio_page_nr = qm->db_interval / PAGE_SIZE; dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH + sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT; @@ -2561,13 +2575,23 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->misc_ctl = false; } -static void hisi_qm_pci_uninit(struct hisi_qm *qm) +static void qm_put_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; - pci_free_irq_vectors(pdev); + if (qm->use_db_isolation) + iounmap(qm->db_io_base); + iounmap(qm->io_base); pci_release_mem_regions(pdev); +} + +static void hisi_qm_pci_uninit(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + pci_free_irq_vectors(pdev); + qm_put_pci_res(qm); pci_disable_device(pdev); } @@ -4236,61 +4260,104 @@ static int qm_get_qp_num(struct hisi_qm *qm) return 0; } -static int hisi_qm_pci_init(struct hisi_qm *qm) +static int qm_get_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; - unsigned int num_vec; int ret; - ret = pci_enable_device_mem(pdev); - if (ret < 0) { - dev_err(dev, "Failed to enable device mem!\n"); - return ret; - } - ret = pci_request_mem_regions(pdev, qm->dev_name); if (ret < 0) { dev_err(dev, "Failed to request mem regions!\n"); - goto err_disable_pcidev; + return ret; } qm->phys_base = pci_resource_start(pdev, PCI_BAR_2); - qm->phys_size = pci_resource_len(qm->pdev, PCI_BAR_2); - qm->io_base = ioremap(qm->phys_base, qm->phys_size); + qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2)); if (!qm->io_base) { ret = -EIO; - goto err_release_mem_regions; + goto err_request_mem_regions; + } + + if (qm->ver > QM_HW_V2) { + if (qm->fun_type == QM_HW_PF) + qm->use_db_isolation = readl(qm->io_base + + QM_QUE_ISO_EN) & BIT(0); + else + qm->use_db_isolation = readl(qm->io_base + + QM_QUE_ISO_CFG_V) & BIT(0); + } + + if (qm->use_db_isolation) { + qm->db_interval = QM_QP_DB_INTERVAL; + qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); + qm->db_io_base = ioremap(qm->db_phys_base, + pci_resource_len(pdev, PCI_BAR_4)); + if (!qm->db_io_base) { + ret = -EIO; + goto err_ioremap; + } + } else { + qm->db_phys_base = qm->phys_base; + qm->db_io_base = qm->io_base; + qm->db_interval = 0; } if (qm->fun_type == QM_HW_PF) { ret = qm_get_qp_num(qm); if (ret) - goto err_iounmap; + goto err_db_ioremap; } + return 0; + +err_db_ioremap: + if (qm->use_db_isolation) + iounmap(qm->db_io_base); +err_ioremap: + iounmap(qm->io_base); +err_request_mem_regions: + pci_release_mem_regions(pdev); + return ret; +} + +static int hisi_qm_pci_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + unsigned int num_vec; + int ret; + + ret = pci_enable_device_mem(pdev); + if (ret < 0) { + dev_err(dev, "Failed to enable device mem!\n"); + return ret; + } + + ret = qm_get_pci_res(qm); + if (ret) + goto err_disable_pcidev; + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (ret < 0) - goto err_iounmap; + goto err_get_pci_res; pci_set_master(pdev); if (!qm->ops->get_irq_num) { ret = -EOPNOTSUPP; - goto err_iounmap; + goto err_get_pci_res; } num_vec = qm->ops->get_irq_num(qm); ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to enable MSI vectors!\n"); - goto err_iounmap; + goto err_get_pci_res; } return 0; -err_iounmap: - iounmap(qm->io_base); -err_release_mem_regions: - pci_release_mem_regions(pdev); +err_get_pci_res: + qm_put_pci_res(qm); err_disable_pcidev: pci_disable_device(pdev); return ret; @@ -4310,28 +4377,28 @@ int hisi_qm_init(struct hisi_qm *qm) hisi_qm_pre_init(qm); - ret = qm_alloc_uacce(qm); - if (ret < 0) - dev_warn(dev, "fail to alloc uacce (%d)\n", ret); - ret = hisi_qm_pci_init(qm); if (ret) - goto err_remove_uacce; + return ret; ret = qm_irq_register(qm); if (ret) - goto err_pci_uninit; + goto err_pci_init; if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) { /* v2 starts to support get vft by mailbox */ ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); if (ret) - goto err_irq_unregister; + goto err_irq_register; } + ret = qm_alloc_uacce(qm); + if (ret < 0) + dev_warn(dev, "fail to alloc uacce (%d)\n", ret); + ret = hisi_qm_memory_init(qm); if (ret) - goto err_irq_unregister; + goto err_alloc_uacce; INIT_WORK(&qm->work, qm_work_process); if (qm->fun_type == QM_HW_PF) @@ -4341,13 +4408,13 @@ int hisi_qm_init(struct hisi_qm *qm) return 0; -err_irq_unregister: - qm_irq_unregister(qm); -err_pci_uninit: - hisi_qm_pci_uninit(qm); -err_remove_uacce: +err_alloc_uacce: uacce_remove(qm->uacce); qm->uacce = NULL; +err_irq_register: + qm_irq_unregister(qm); +err_pci_init: + hisi_qm_pci_uninit(qm); return ret; } EXPORT_SYMBOL_GPL(hisi_qm_init); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index cf0290f35680..34e69fe3b671 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -202,6 +202,7 @@ struct hisi_qm { const char *dev_name; struct pci_dev *pdev; void __iomem *io_base; + void __iomem *db_io_base; u32 sqe_size; u32 qp_base; u32 qp_num; @@ -209,6 +210,7 @@ struct hisi_qm { u32 ctrl_qp_num; u32 max_qp_num; u32 vfs_num; + u32 db_interval; struct list_head list; struct hisi_qm_list *qm_list; @@ -250,7 +252,7 @@ struct hisi_qm { /* doorbell isolation enable */ bool use_db_isolation; resource_size_t phys_base; - resource_size_t phys_size; + resource_size_t db_phys_base; struct uacce_device *uacce; int mode; }; -- cgit From 5377265f54dedc01db1eb4325f65c7a6a4035ec0 Mon Sep 17 00:00:00 2001 From: Hao Fang Date: Fri, 26 Mar 2021 16:42:39 +0800 Subject: crypto: hisilicon - use the correct HiSilicon copyright s/Hisilicon/HiSilicon/g, according to https://www.hisilicon.com/en/terms-of-use. Signed-off-by: Hao Fang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec/sec_algs.c | 2 +- drivers/crypto/hisilicon/sec/sec_drv.c | 6 +++--- drivers/crypto/hisilicon/sec/sec_drv.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c index 8ca945ac297e..0a3c8f019b02 100644 --- a/drivers/crypto/hisilicon/sec/sec_algs.c +++ b/drivers/crypto/hisilicon/sec/sec_algs.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2016-2017 Hisilicon Limited. */ +/* Copyright (c) 2016-2017 HiSilicon Limited. */ #include #include #include diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c index 3c26871db12f..cf44f076f976 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.c +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Driver for the Hisilicon SEC units found on Hip06 Hip07 + * Driver for the HiSilicon SEC units found on Hip06 Hip07 * - * Copyright (c) 2016-2017 Hisilicon Limited. + * Copyright (c) 2016-2017 HiSilicon Limited. */ #include #include @@ -1316,6 +1316,6 @@ static struct platform_driver sec_driver = { module_platform_driver(sec_driver); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Hisilicon Security Accelerators"); +MODULE_DESCRIPTION("HiSilicon Security Accelerators"); MODULE_AUTHOR("Zaibo Xu "); diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h index 4d9063a8b10b..179a8250d691 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.h +++ b/drivers/crypto/hisilicon/sec/sec_drv.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (c) 2016-2017 Hisilicon Limited. */ +/* Copyright (c) 2016-2017 HiSilicon Limited. */ #ifndef _SEC_DRV_H_ #define _SEC_DRV_H_ -- cgit From 30a4240e06ea7eecbc5645916501112ef8532533 Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 27 Mar 2021 15:28:45 +0800 Subject: crypto: hisilicon/zip - adjust functions location This patch changes nothing about functions except location in order to make code logic clearly. This adjustment follows three principles: 1.The called functions are listed in order above the calling functions. 2.The paired functions are next to each other. 3.Logically similar functions are placed in the same area. Here, we use the callback of 'acomp_alg' as the basis for dividing areas. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_crypto.c | 564 +++++++++++++++--------------- 1 file changed, 282 insertions(+), 282 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 41f69662024a..989b273a3aa7 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -119,6 +119,129 @@ static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); +static u16 get_extra_field_size(const u8 *start) +{ + return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; +} + +static u32 get_name_field_size(const u8 *start) +{ + return strlen(start) + 1; +} + +static u32 get_comment_field_size(const u8 *start) +{ + return strlen(start) + 1; +} + +static u32 __get_gzip_head_size(const u8 *src) +{ + u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); + u32 size = GZIP_HEAD_FEXTRA_SHIFT; + + if (head_flg & GZIP_HEAD_FEXTRA_BIT) + size += get_extra_field_size(src + size); + if (head_flg & GZIP_HEAD_FNAME_BIT) + size += get_name_field_size(src + size); + if (head_flg & GZIP_HEAD_FCOMMENT_BIT) + size += get_comment_field_size(src + size); + if (head_flg & GZIP_HEAD_FHCRC_BIT) + size += GZIP_HEAD_FHCRC_SIZE; + + return size; +} + +static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) +{ + char buf[HZIP_GZIP_HEAD_BUF]; + + sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); + + return __get_gzip_head_size(buf); +} + +static int add_comp_head(struct scatterlist *dst, u8 req_type) +{ + int head_size = TO_HEAD_SIZE(req_type); + const u8 *head = TO_HEAD(req_type); + int ret; + + ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); + if (ret != head_size) { + pr_err("the head size of buffer is wrong (%d)!\n", ret); + return -ENOMEM; + } + + return head_size; +} + +static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) +{ + if (!acomp_req->src || !acomp_req->slen) + return -EINVAL; + + if (req_type == HZIP_ALG_TYPE_GZIP && + acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT) + return -EINVAL; + + switch (req_type) { + case HZIP_ALG_TYPE_ZLIB: + return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); + case HZIP_ALG_TYPE_GZIP: + return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); + default: + pr_err("request type does not support!\n"); + return -EINVAL; + } +} + +static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, + struct hisi_zip_qp_ctx *qp_ctx, + size_t head_size, bool is_comp) +{ + struct hisi_zip_req_q *req_q = &qp_ctx->req_q; + struct hisi_zip_req *q = req_q->q; + struct hisi_zip_req *req_cache; + int req_id; + + write_lock(&req_q->req_lock); + + req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); + if (req_id >= req_q->size) { + write_unlock(&req_q->req_lock); + dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); + return ERR_PTR(-EAGAIN); + } + set_bit(req_id, req_q->req_bitmap); + + req_cache = q + req_id; + req_cache->req_id = req_id; + req_cache->req = req; + + if (is_comp) { + req_cache->sskip = 0; + req_cache->dskip = head_size; + } else { + req_cache->sskip = head_size; + req_cache->dskip = 0; + } + + write_unlock(&req_q->req_lock); + + return req_cache; +} + +static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, + struct hisi_zip_req *req) +{ + struct hisi_zip_req_q *req_q = &qp_ctx->req_q; + + write_lock(&req_q->req_lock); + clear_bit(req->req_id, req_q->req_bitmap); + memset(req, 0, sizeof(struct hisi_zip_req)); + write_unlock(&req_q->req_lock); +} + static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) { u32 val; @@ -150,6 +273,159 @@ static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, sqe->dest_addr_h = upper_32_bits(d_addr); } +static int hisi_zip_do_work(struct hisi_zip_req *req, + struct hisi_zip_qp_ctx *qp_ctx) +{ + struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; + struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; + struct acomp_req *a_req = req->req; + struct hisi_qp *qp = qp_ctx->qp; + struct device *dev = &qp->qm->pdev->dev; + struct hisi_zip_sqe zip_sqe; + dma_addr_t input, output; + int ret; + + if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) + return -EINVAL; + + req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, + req->req_id << 1, &input); + if (IS_ERR(req->hw_src)) { + dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", + PTR_ERR(req->hw_src)); + return PTR_ERR(req->hw_src); + } + req->dma_src = input; + + req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, + (req->req_id << 1) + 1, + &output); + if (IS_ERR(req->hw_dst)) { + ret = PTR_ERR(req->hw_dst); + dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + ret); + goto err_unmap_input; + } + req->dma_dst = output; + + hisi_zip_fill_sqe(&zip_sqe, qp->req_type, input, output, a_req->slen, + a_req->dlen, req->sskip, req->dskip); + hisi_zip_config_buf_type(&zip_sqe, HZIP_SGL); + hisi_zip_config_tag(&zip_sqe, req->req_id); + + /* send command to start a task */ + atomic64_inc(&dfx->send_cnt); + ret = hisi_qp_send(qp, &zip_sqe); + if (ret < 0) { + atomic64_inc(&dfx->send_busy_cnt); + ret = -EAGAIN; + dev_dbg_ratelimited(dev, "failed to send request!\n"); + goto err_unmap_output; + } + + return -EINPROGRESS; + +err_unmap_output: + hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); +err_unmap_input: + hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); + return ret; +} + +static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) +{ + struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; + struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; + struct hisi_zip_req_q *req_q = &qp_ctx->req_q; + struct device *dev = &qp->qm->pdev->dev; + struct hisi_zip_sqe *sqe = data; + struct hisi_zip_req *req = req_q->q + sqe->tag; + struct acomp_req *acomp_req = req->req; + u32 status, dlen, head_size; + int err = 0; + + atomic64_inc(&dfx->recv_cnt); + status = sqe->dw3 & HZIP_BD_STATUS_M; + if (status != 0 && status != HZIP_NC_ERR) { + dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", + (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, + sqe->produced); + atomic64_inc(&dfx->err_bd_cnt); + err = -EIO; + } + dlen = sqe->produced; + + hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); + hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); + + head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; + acomp_req->dlen = dlen + head_size; + + if (acomp_req->base.complete) + acomp_request_complete(acomp_req, err); + + hisi_zip_remove_req(qp_ctx, req); +} + +static int hisi_zip_acompress(struct acomp_req *acomp_req) +{ + struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; + struct hisi_zip_req *req; + int head_size; + int ret; + + /* let's output compression head now */ + head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", + head_size); + return head_size; + } + + req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); + if (IS_ERR(req)) + return PTR_ERR(req); + + ret = hisi_zip_do_work(req, qp_ctx); + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); + hisi_zip_remove_req(qp_ctx, req); + } + + return ret; +} + +static int hisi_zip_adecompress(struct acomp_req *acomp_req) +{ + struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; + struct hisi_zip_req *req; + int head_size, ret; + + head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", + head_size); + return head_size; + } + + req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); + if (IS_ERR(req)) + return PTR_ERR(req); + + ret = hisi_zip_do_work(req, qp_ctx); + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", + ret); + hisi_zip_remove_req(qp_ctx, req); + } + + return ret; +} + static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, int alg_type, int req_type) { @@ -207,46 +483,14 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int } return 0; -} - -static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) -{ - int i; - - for (i = 1; i >= 0; i--) - hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); -} - -static u16 get_extra_field_size(const u8 *start) -{ - return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; -} - -static u32 get_name_field_size(const u8 *start) -{ - return strlen(start) + 1; -} - -static u32 get_comment_field_size(const u8 *start) -{ - return strlen(start) + 1; -} - -static u32 __get_gzip_head_size(const u8 *src) -{ - u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); - u32 size = GZIP_HEAD_FEXTRA_SHIFT; +} - if (head_flg & GZIP_HEAD_FEXTRA_BIT) - size += get_extra_field_size(src + size); - if (head_flg & GZIP_HEAD_FNAME_BIT) - size += get_name_field_size(src + size); - if (head_flg & GZIP_HEAD_FCOMMENT_BIT) - size += get_comment_field_size(src + size); - if (head_flg & GZIP_HEAD_FHCRC_BIT) - size += GZIP_HEAD_FHCRC_SIZE; +static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) +{ + int i; - return size; + for (i = 1; i >= 0; i--) + hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); } static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) @@ -336,52 +580,6 @@ static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) ctx->qp_ctx[i].sgl_pool); } -static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, - struct hisi_zip_req *req) -{ - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; - - write_lock(&req_q->req_lock); - clear_bit(req->req_id, req_q->req_bitmap); - memset(req, 0, sizeof(struct hisi_zip_req)); - write_unlock(&req_q->req_lock); -} - -static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) -{ - struct hisi_zip_sqe *sqe = data; - struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; - struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; - struct hisi_zip_req *req = req_q->q + sqe->tag; - struct acomp_req *acomp_req = req->req; - struct device *dev = &qp->qm->pdev->dev; - u32 status, dlen, head_size; - int err = 0; - - atomic64_inc(&dfx->recv_cnt); - status = sqe->dw3 & HZIP_BD_STATUS_M; - if (status != 0 && status != HZIP_NC_ERR) { - dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", - (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, - sqe->produced); - atomic64_inc(&dfx->err_bd_cnt); - err = -EIO; - } - dlen = sqe->produced; - - hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); - hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); - - head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; - acomp_req->dlen = dlen + head_size; - - if (acomp_req->base.complete) - acomp_request_complete(acomp_req, err); - - hisi_zip_remove_req(qp_ctx, req); -} - static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, void (*fn)(struct hisi_qp *, void *)) { @@ -439,204 +637,6 @@ static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) hisi_zip_ctx_exit(ctx); } -static int add_comp_head(struct scatterlist *dst, u8 req_type) -{ - int head_size = TO_HEAD_SIZE(req_type); - const u8 *head = TO_HEAD(req_type); - int ret; - - ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); - if (ret != head_size) { - pr_err("the head size of buffer is wrong (%d)!\n", ret); - return -ENOMEM; - } - - return head_size; -} - -static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) -{ - char buf[HZIP_GZIP_HEAD_BUF]; - - sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); - - return __get_gzip_head_size(buf); -} - -static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) -{ - if (!acomp_req->src || !acomp_req->slen) - return -EINVAL; - - if ((req_type == HZIP_ALG_TYPE_GZIP) && - (acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) - return -EINVAL; - - switch (req_type) { - case HZIP_ALG_TYPE_ZLIB: - return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); - case HZIP_ALG_TYPE_GZIP: - return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); - default: - pr_err("request type does not support!\n"); - return -EINVAL; - } -} - -static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, - struct hisi_zip_qp_ctx *qp_ctx, - size_t head_size, bool is_comp) -{ - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; - struct hisi_zip_req *q = req_q->q; - struct hisi_zip_req *req_cache; - int req_id; - - write_lock(&req_q->req_lock); - - req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); - if (req_id >= req_q->size) { - write_unlock(&req_q->req_lock); - dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); - return ERR_PTR(-EAGAIN); - } - set_bit(req_id, req_q->req_bitmap); - - req_cache = q + req_id; - req_cache->req_id = req_id; - req_cache->req = req; - - if (is_comp) { - req_cache->sskip = 0; - req_cache->dskip = head_size; - } else { - req_cache->sskip = head_size; - req_cache->dskip = 0; - } - - write_unlock(&req_q->req_lock); - - return req_cache; -} - -static int hisi_zip_do_work(struct hisi_zip_req *req, - struct hisi_zip_qp_ctx *qp_ctx) -{ - struct acomp_req *a_req = req->req; - struct hisi_qp *qp = qp_ctx->qp; - struct device *dev = &qp->qm->pdev->dev; - struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; - struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_sqe zip_sqe; - dma_addr_t input, output; - int ret; - - if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) - return -EINVAL; - - req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, - req->req_id << 1, &input); - if (IS_ERR(req->hw_src)) { - dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", - PTR_ERR(req->hw_src)); - return PTR_ERR(req->hw_src); - } - req->dma_src = input; - - req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, - (req->req_id << 1) + 1, - &output); - if (IS_ERR(req->hw_dst)) { - ret = PTR_ERR(req->hw_dst); - dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", - ret); - goto err_unmap_input; - } - req->dma_dst = output; - - hisi_zip_fill_sqe(&zip_sqe, qp->req_type, input, output, a_req->slen, - a_req->dlen, req->sskip, req->dskip); - hisi_zip_config_buf_type(&zip_sqe, HZIP_SGL); - hisi_zip_config_tag(&zip_sqe, req->req_id); - - /* send command to start a task */ - atomic64_inc(&dfx->send_cnt); - ret = hisi_qp_send(qp, &zip_sqe); - if (ret < 0) { - atomic64_inc(&dfx->send_busy_cnt); - ret = -EAGAIN; - dev_dbg_ratelimited(dev, "failed to send request!\n"); - goto err_unmap_output; - } - - return -EINPROGRESS; - -err_unmap_output: - hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); -err_unmap_input: - hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); - return ret; -} - -static int hisi_zip_acompress(struct acomp_req *acomp_req) -{ - struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; - struct hisi_zip_req *req; - int head_size; - int ret; - - /* let's output compression head now */ - head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); - if (head_size < 0) { - dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", - head_size); - return head_size; - } - - req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); - if (IS_ERR(req)) - return PTR_ERR(req); - - ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) { - dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); - hisi_zip_remove_req(qp_ctx, req); - } - - return ret; -} - -static int hisi_zip_adecompress(struct acomp_req *acomp_req) -{ - struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; - struct hisi_zip_req *req; - int head_size, ret; - - head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); - if (head_size < 0) { - dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", - head_size); - return head_size; - } - - req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); - if (IS_ERR(req)) - return PTR_ERR(req); - - ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) { - dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", - ret); - hisi_zip_remove_req(qp_ctx, req); - } - - return ret; -} - static struct acomp_alg hisi_zip_acomp_zlib = { .init = hisi_zip_acomp_init, .exit = hisi_zip_acomp_exit, -- cgit From d746881855ba167597d835de512150b6e6ea19ae Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 27 Mar 2021 15:28:46 +0800 Subject: crypto: hisilicon/zip - add comments for 'hisi_zip_sqe' Some fields of 'hisi_zip_sqe' are unused, and some fields have misc utilities. So add comments for used fields and make others unnamed. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip.h | 45 ++++++++++++++++++++++--------- drivers/crypto/hisilicon/zip/zip_crypto.c | 4 +-- 2 files changed, 34 insertions(+), 15 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 9ed74611f722..b4d3e03f5609 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -33,31 +33,50 @@ struct hisi_zip_sqe { u32 consumed; u32 produced; u32 comp_data_length; + /* + * status: 0~7 bits + * rsvd: 8~31 bits + */ u32 dw3; u32 input_data_length; - u32 lba_l; - u32 lba_h; + u32 dw5; + u32 dw6; + /* + * in_sge_data_offset: 0~23 bits + * rsvd: 24~27 bits + * sqe_type: 29~31 bits + */ u32 dw7; + /* + * out_sge_data_offset: 0~23 bits + * rsvd: 24~31 bits + */ u32 dw8; + /* + * request_type: 0~7 bits + * buffer_type: 8~11 bits + * rsvd: 13~31 bits + */ u32 dw9; u32 dw10; - u32 priv_info; + u32 dw11; u32 dw12; - u32 tag; + /* tag: in sqe type 0 */ + u32 dw13; u32 dest_avail_out; - u32 rsvd0; - u32 comp_head_addr_l; - u32 comp_head_addr_h; + u32 dw15; + u32 dw16; + u32 dw17; u32 source_addr_l; u32 source_addr_h; u32 dest_addr_l; u32 dest_addr_h; - u32 stream_ctx_addr_l; - u32 stream_ctx_addr_h; - u32 cipher_key1_addr_l; - u32 cipher_key1_addr_h; - u32 cipher_key2_addr_l; - u32 cipher_key2_addr_h; + u32 dw22; + u32 dw23; + u32 dw24; + u32 dw25; + u32 dw26; + u32 dw27; u32 rsvd1[4]; }; diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 989b273a3aa7..3bc2148535f7 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -253,7 +253,7 @@ static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag) { - sqe->tag = tag; + sqe->dw13 = tag; } static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, @@ -339,7 +339,7 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct device *dev = &qp->qm->pdev->dev; struct hisi_zip_sqe *sqe = data; - struct hisi_zip_req *req = req_q->q + sqe->tag; + struct hisi_zip_req *req = req_q->q + sqe->dw13; struct acomp_req *acomp_req = req->req; u32 status, dlen, head_size; int err = 0; -- cgit From 2bcf36348ce50e650fe8e1db046029afc89ef098 Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 27 Mar 2021 15:28:47 +0800 Subject: crypto: hisilicon/zip - initialize operations about 'sqe' in 'acomp_alg.init' The operations about 'sqe' are different on some hardwares. Add a struct 'hisi_zip_sqe_ops' to describe the operations in a hardware. And choose the 'ops' in 'hisi_zip_acomp_init' according to the hardware. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_crypto.c | 141 +++++++++++++++++++++++------- 1 file changed, 110 insertions(+), 31 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 3bc2148535f7..466ebf1529c3 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -10,6 +10,7 @@ #define HZIP_BD_STATUS_M GENMASK(7, 0) /* hisi_zip_sqe dw7 */ #define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) +#define HZIP_SQE_TYPE_M GENMASK(31, 28) /* hisi_zip_sqe dw8 */ #define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) /* hisi_zip_sqe dw9 */ @@ -91,8 +92,22 @@ struct hisi_zip_qp_ctx { struct hisi_zip_ctx *ctx; }; +struct hisi_zip_sqe_ops { + u8 sqe_type; + void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); + void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); + void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type); + void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); + void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); + void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); + u32 (*get_tag)(struct hisi_zip_sqe *sqe); + u32 (*get_status)(struct hisi_zip_sqe *sqe); + u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); +}; + struct hisi_zip_ctx { struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; + const struct hisi_zip_sqe_ops *ops; }; static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) @@ -242,35 +257,69 @@ static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, write_unlock(&req_q->req_lock); } -static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) +static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) +{ + sqe->source_addr_l = lower_32_bits(req->dma_src); + sqe->source_addr_h = upper_32_bits(req->dma_src); + sqe->dest_addr_l = lower_32_bits(req->dma_dst); + sqe->dest_addr_h = upper_32_bits(req->dma_dst); +} + +static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) +{ + struct acomp_req *a_req = req->req; + + sqe->input_data_length = a_req->slen - req->sskip; + sqe->dest_avail_out = a_req->dlen - req->dskip; + sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, req->sskip); + sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, req->dskip); +} + +static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) { u32 val; - val = (sqe->dw9) & ~HZIP_BUF_TYPE_M; + val = sqe->dw9 & ~HZIP_BUF_TYPE_M; val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); sqe->dw9 = val; } -static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag) +static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) { - sqe->dw13 = tag; + u32 val; + + val = sqe->dw9 & ~HZIP_REQ_TYPE_M; + val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type); + sqe->dw9 = val; } -static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, - dma_addr_t s_addr, dma_addr_t d_addr, u32 slen, - u32 dlen, u32 sskip, u32 dskip) +static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) { + sqe->dw13 = req->req_id; +} + +static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) +{ + u32 val; + + val = sqe->dw7 & ~HZIP_SQE_TYPE_M; + val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type); + sqe->dw7 = val; +} + +static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe, + u8 req_type, struct hisi_zip_req *req) +{ + const struct hisi_zip_sqe_ops *ops = ctx->ops; + memset(sqe, 0, sizeof(struct hisi_zip_sqe)); - sqe->input_data_length = slen - sskip; - sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, sskip); - sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, dskip); - sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type); - sqe->dest_avail_out = dlen - dskip; - sqe->source_addr_l = lower_32_bits(s_addr); - sqe->source_addr_h = upper_32_bits(s_addr); - sqe->dest_addr_l = lower_32_bits(d_addr); - sqe->dest_addr_h = upper_32_bits(d_addr); + ops->fill_addr(sqe, req); + ops->fill_buf_size(sqe, req); + ops->fill_buf_type(sqe, HZIP_SGL); + ops->fill_req_type(sqe, req_type); + ops->fill_tag(sqe, req); + ops->fill_sqe_type(sqe, ops->sqe_type); } static int hisi_zip_do_work(struct hisi_zip_req *req, @@ -282,36 +331,30 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, struct hisi_qp *qp = qp_ctx->qp; struct device *dev = &qp->qm->pdev->dev; struct hisi_zip_sqe zip_sqe; - dma_addr_t input, output; int ret; if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) return -EINVAL; req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, - req->req_id << 1, &input); + req->req_id << 1, &req->dma_src); if (IS_ERR(req->hw_src)) { dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", PTR_ERR(req->hw_src)); return PTR_ERR(req->hw_src); } - req->dma_src = input; req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, (req->req_id << 1) + 1, - &output); + &req->dma_dst); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", ret); goto err_unmap_input; } - req->dma_dst = output; - hisi_zip_fill_sqe(&zip_sqe, qp->req_type, input, output, a_req->slen, - a_req->dlen, req->sskip, req->dskip); - hisi_zip_config_buf_type(&zip_sqe, HZIP_SGL); - hisi_zip_config_tag(&zip_sqe, req->req_id); + hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); /* send command to start a task */ atomic64_inc(&dfx->send_cnt); @@ -332,20 +375,37 @@ err_unmap_input: return ret; } +static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe) +{ + return sqe->dw13; +} + +static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) +{ + return sqe->dw3 & HZIP_BD_STATUS_M; +} + +static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) +{ + return sqe->produced; +} + static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) { struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; + const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct device *dev = &qp->qm->pdev->dev; struct hisi_zip_sqe *sqe = data; - struct hisi_zip_req *req = req_q->q + sqe->dw13; + u32 tag = ops->get_tag(sqe); + struct hisi_zip_req *req = req_q->q + tag; struct acomp_req *acomp_req = req->req; u32 status, dlen, head_size; int err = 0; atomic64_inc(&dfx->recv_cnt); - status = sqe->dw3 & HZIP_BD_STATUS_M; + status = ops->get_status(sqe); if (status != 0 && status != HZIP_NC_ERR) { dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, @@ -353,7 +413,8 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) atomic64_inc(&dfx->err_bd_cnt); err = -EIO; } - dlen = sqe->produced; + + dlen = ops->get_dstlen(sqe); hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); @@ -453,9 +514,23 @@ static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) hisi_qm_release_qp(ctx->qp); } +static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { + .sqe_type = 0, + .fill_addr = hisi_zip_fill_addr, + .fill_buf_size = hisi_zip_fill_buf_size, + .fill_buf_type = hisi_zip_fill_buf_type, + .fill_req_type = hisi_zip_fill_req_type, + .fill_tag = hisi_zip_fill_tag_v1, + .fill_sqe_type = hisi_zip_fill_sqe_type, + .get_tag = hisi_zip_get_tag_v1, + .get_status = hisi_zip_get_status, + .get_dstlen = hisi_zip_get_dstlen, +}; + static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node) { struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; + struct hisi_zip_qp_ctx *qp_ctx; struct hisi_zip *hisi_zip; int ret, i, j; @@ -469,8 +544,9 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int for (i = 0; i < HZIP_CTX_Q_NUM; i++) { /* alg_type = 0 for compress, 1 for decompress in hw sqe */ - ret = hisi_zip_start_qp(qps[i], &hisi_zip_ctx->qp_ctx[i], i, - req_type); + qp_ctx = &hisi_zip_ctx->qp_ctx[i]; + qp_ctx->ctx = hisi_zip_ctx; + ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); if (ret) { for (j = i - 1; j >= 0; j--) hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); @@ -479,9 +555,12 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int return ret; } - hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip; + qp_ctx->zip_dev = hisi_zip; } + if (hisi_zip->qm.ver < QM_HW_V3) + hisi_zip_ctx->ops = &hisi_zip_ops_v1; + return 0; } -- cgit From 95c612b30898ca5fe7c76ef31d910e37ace6f34e Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 27 Mar 2021 15:28:48 +0800 Subject: crypto: hisilicon/zip - support new 'sqe' type in Kunpeng930 The Kunpeng930 changes some field meanings in 'sqe'. So add a new 'hisi_zip_sqe_ops' to describe the 'sqe' operations. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip.h | 1 + drivers/crypto/hisilicon/zip/zip_crypto.c | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index b4d3e03f5609..517fdbdff3ea 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -75,6 +75,7 @@ struct hisi_zip_sqe { u32 dw23; u32 dw24; u32 dw25; + /* tag: in sqe type 3 */ u32 dw26; u32 dw27; u32 rsvd1[4]; diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 466ebf1529c3..9520a4113c81 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -298,6 +298,11 @@ static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req * sqe->dw13 = req->req_id; } +static void hisi_zip_fill_tag_v2(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) +{ + sqe->dw26 = req->req_id; +} + static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) { u32 val; @@ -380,6 +385,11 @@ static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe) return sqe->dw13; } +static u32 hisi_zip_get_tag_v2(struct hisi_zip_sqe *sqe) +{ + return sqe->dw26; +} + static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) { return sqe->dw3 & HZIP_BD_STATUS_M; @@ -527,6 +537,19 @@ static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { .get_dstlen = hisi_zip_get_dstlen, }; +static const struct hisi_zip_sqe_ops hisi_zip_ops_v2 = { + .sqe_type = 0x3, + .fill_addr = hisi_zip_fill_addr, + .fill_buf_size = hisi_zip_fill_buf_size, + .fill_buf_type = hisi_zip_fill_buf_type, + .fill_req_type = hisi_zip_fill_req_type, + .fill_tag = hisi_zip_fill_tag_v2, + .fill_sqe_type = hisi_zip_fill_sqe_type, + .get_tag = hisi_zip_get_tag_v2, + .get_status = hisi_zip_get_status, + .get_dstlen = hisi_zip_get_dstlen, +}; + static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node) { struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; @@ -560,6 +583,8 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int if (hisi_zip->qm.ver < QM_HW_V3) hisi_zip_ctx->ops = &hisi_zip_ops_v1; + else + hisi_zip_ctx->ops = &hisi_zip_ops_v2; return 0; } -- cgit From 64e80c8f48a01cc5c727472f1e37324539bb6ff7 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Sat, 27 Mar 2021 16:29:44 +0800 Subject: crypto: hisilicon/hpre - fix PASID setting on kunpeng 920 We must confirm the PASID is disabled before using no-sva mode. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 43 +++++++++++++------------------ 1 file changed, 18 insertions(+), 25 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index a8b9c4cf42ff..dd541a04c5aa 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -244,28 +244,24 @@ struct hisi_qp *hpre_create_qp(u8 type) return NULL; } -static void hpre_pasid_enable(struct hisi_qm *qm) +static void hpre_config_pasid(struct hisi_qm *qm) { - u32 val; - - val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); - val |= BIT(HPRE_PASID_EN_BIT); - writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG); - val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); - val |= BIT(HPRE_PASID_EN_BIT); - writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG); -} + u32 val1, val2; -static void hpre_pasid_disable(struct hisi_qm *qm) -{ - u32 val; + if (qm->ver >= QM_HW_V3) + return; - val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); - val &= ~BIT(HPRE_PASID_EN_BIT); - writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG); - val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); - val &= ~BIT(HPRE_PASID_EN_BIT); - writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG); + val1 = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); + val2 = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); + if (qm->use_sva) { + val1 |= BIT(HPRE_PASID_EN_BIT); + val2 |= BIT(HPRE_PASID_EN_BIT); + } else { + val1 &= ~BIT(HPRE_PASID_EN_BIT); + val2 &= ~BIT(HPRE_PASID_EN_BIT); + } + writel_relaxed(val1, qm->io_base + HPRE_DATA_RUSER_CFG); + writel_relaxed(val2, qm->io_base + HPRE_DATA_WUSER_CFG); } static int hpre_cfg_by_dsm(struct hisi_qm *qm) @@ -391,12 +387,11 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) dev_err(dev, "acpi_evaluate_dsm err.\n"); disable_flr_of_bme(qm); - - /* Enable data buffer pasid */ - if (qm->use_sva) - hpre_pasid_enable(qm); } + /* Config data buffer pasid needed by Kunpeng 920 */ + hpre_config_pasid(qm); + return ret; } @@ -949,8 +944,6 @@ static void hpre_remove(struct pci_dev *pdev) hisi_qm_stop(qm, QM_NORMAL); if (qm->fun_type == QM_HW_PF) { - if (qm->use_sva && qm->ver == QM_HW_V2) - hpre_pasid_disable(qm); hpre_cnt_regs_clear(qm); qm->debug.curr_qm_qp_num = 0; hisi_qm_dev_err_uninit(qm); -- cgit From 09fd266f174144daf1be5033f32ccd096c765452 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Sat, 27 Mar 2021 16:32:08 +0800 Subject: crypto: hisilicon/hpre - fix a typo and delete redundant blank line s/shoul/should/ Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index dd541a04c5aa..6696ea540f03 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -321,7 +321,7 @@ static int hpre_set_cluster(struct hisi_qm *qm) } /* - * For Kunpeng 920, we shoul disable FLR triggered by hardware (BME/PM/SRIOV). + * For Kunpeng 920, we should disable FLR triggered by hardware (BME/PM/SRIOV). * Or it may stay in D3 state when we bind and unbind hpre quickly, * as it does FLR triggered by hardware. */ @@ -952,7 +952,6 @@ static void hpre_remove(struct pci_dev *pdev) hisi_qm_uninit(qm); } - static const struct pci_error_handlers hpre_err_handler = { .error_detected = hisi_qm_dev_err_detected, .slot_reset = hisi_qm_dev_slot_reset, -- cgit From 9bb3fbbff3710dfebeed2f8cf9ee61c415722543 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Sat, 27 Mar 2021 16:32:45 +0800 Subject: crypto: hisilicon/hpre - delete redundant '\n' It has newline already by sysfs, so delete redundant '\n' Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 4 ++-- drivers/crypto/hisilicon/sec2/sec_main.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 6696ea540f03..a8c79212057d 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -752,9 +752,9 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) } if (pdev->revision >= QM_HW_V3) - qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2\n"; + qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2"; else - qm->algs = "rsa\ndh\n"; + qm->algs = "rsa\ndh"; qm->mode = uacce_mode; qm->pdev = pdev; qm->ver = pdev->revision; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index b1818f793520..d38b893f21b7 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -744,7 +744,7 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->pdev = pdev; qm->ver = pdev->revision; - qm->algs = "cipher\ndigest\naead\n"; + qm->algs = "cipher\ndigest\naead"; qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; -- cgit From 0ae869926fab00b95709518d72cca2d50a29eccd Mon Sep 17 00:00:00 2001 From: Meng Yu Date: Sat, 27 Mar 2021 17:56:16 +0800 Subject: crypto: hisilicon/hpre - Add processing of src_data in 'CURVE25519' HPRE can only deal with src_data smaller than 'p' in 'CURVE25519' algorithm, but 'rfc7748' says: 'Implementations MUST accept non-canonical values and process them as if they had been reduced modulo the field prime' So we get its modulus to p, and then deal it with HPRE. Signed-off-by: Meng Yu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index d743c540d602..f363653b32e9 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1729,6 +1729,17 @@ static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, return 0; } +static void hpre_curve25519_src_modulo_p(u8 *ptr) +{ + int i; + + for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++) + ptr[i] = 0; + + /* The modulus is ptr's last byte minus '0xed'(last byte of p) */ + ptr[i] -= 0xed; +} + static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, struct scatterlist *data, unsigned int len) { @@ -1767,10 +1778,13 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, curve = ecc_get_curve25519(); fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits); - if (memcmp(ptr, p, ctx->key_sz) >= 0) { - dev_err(dev, "gx is out of p!\n"); - goto err; - } + + /* + * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p, + * we get its modulus to p, and then use it. + */ + if (memcmp(ptr, p, ctx->key_sz) >= 0) + hpre_curve25519_src_modulo_p(ptr); hpre_req->src = ptr; msg->in = cpu_to_le64(dma); -- cgit From 059c5342812c448f21bed1881a75134320e9c8a2 Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Sat, 27 Mar 2021 18:28:30 +0800 Subject: crypto: hisilicon/sec - Fixes AES algorithm mode parameter problem The input data of the ECB (AES) algorithm needs to be aligned with 16 bytes, and the input data of the XTS (AES) algorithm is at least 16 bytes. Otherwise the SEC hardware will go wrong. Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 37 ++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index c74082ced4a0..71c01256ef0c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -1397,6 +1397,36 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) return sec_aead_ctx_init(tfm, "sha512"); } + +static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, + struct sec_req *sreq) +{ + u32 cryptlen = sreq->c_req.sk_req->cryptlen; + struct device *dev = ctx->dev; + u8 c_mode = ctx->c_ctx.c_mode; + int ret = 0; + + switch (c_mode) { + case SEC_CMODE_XTS: + if (unlikely(cryptlen < AES_BLOCK_SIZE)) { + dev_err(dev, "skcipher XTS mode input length error!\n"); + ret = -EINVAL; + } + break; + case SEC_CMODE_ECB: + case SEC_CMODE_CBC: + if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "skcipher AES input length error!\n"); + ret = -EINVAL; + } + break; + default: + ret = -EINVAL; + } + + return ret; +} + static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) { struct skcipher_request *sk_req = sreq->c_req.sk_req; @@ -1421,12 +1451,9 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) } return 0; } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { - if (unlikely(sk_req->cryptlen & (AES_BLOCK_SIZE - 1))) { - dev_err(dev, "skcipher aes input length error!\n"); - return -EINVAL; - } - return 0; + return sec_skcipher_cryptlen_ckeck(ctx, sreq); } + dev_err(dev, "skcipher algorithm error!\n"); return -EINVAL; -- cgit From da6503f52bf85a7da29fbd7126320658f906dd9a Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Tue, 30 Mar 2021 16:02:32 +0800 Subject: crypto: hisilicon/sec - Fix a module parameter error ctx_q_num is a module parameter set by the user to specify the number of qp queues required to create a ctx. When the number of qp queues allocated by PF or VF is less than the ctx_q_num, an error will be reported when ctx is initialized in kernel mode, which leads to the problem that the registered algorithms cannot be used. Therefore, when PF or VF is initialized, if the number of qp queues is not enough to create a ctx, the kernel mode cannot be used, and there is no need to register the kernel mode algorithms. Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_main.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index d38b893f21b7..1affa1ba27e1 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -867,10 +867,15 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) pci_warn(pdev, "Failed to init debugfs!\n"); - ret = hisi_qm_alg_register(qm, &sec_devices); - if (ret < 0) { - pr_err("Failed to register driver to crypto.\n"); - goto err_qm_stop; + if (qm->qp_num >= ctx_q_num) { + ret = hisi_qm_alg_register(qm, &sec_devices); + if (ret < 0) { + pr_err("Failed to register driver to crypto.\n"); + goto err_qm_stop; + } + } else { + pci_warn(qm->pdev, + "Failed to use kernel mode, qp not enough!\n"); } if (qm->uacce) { @@ -906,7 +911,9 @@ static void sec_remove(struct pci_dev *pdev) struct hisi_qm *qm = pci_get_drvdata(pdev); hisi_qm_wait_task_finish(qm, &sec_devices); - hisi_qm_alg_unregister(qm, &sec_devices); + if (qm->qp_num >= ctx_q_num) + hisi_qm_alg_unregister(qm, &sec_devices); + if (qm->fun_type == QM_HW_PF && qm->vfs_num) hisi_qm_sriov_disable(pdev, true); -- cgit From fd4317b7b2c96e51b5ff3bed2af4ccfd66e9288a Mon Sep 17 00:00:00 2001 From: Ruiqi Gong Date: Tue, 30 Mar 2021 07:51:24 -0400 Subject: crypto: hisilicon/hpre - fix a typo in hpre_crypto.c Do a trivial typo fix. s/discribed/described Reported-by: Hulk Robot Signed-off-by: Ruiqi Gong Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index f363653b32e9..1b7f15125275 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1769,7 +1769,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, /* * Src_data(gx) is in little-endian order, MSB in the final byte should - * be masked as discribed in RFC7748, then transform it to big-endian + * be masked as described in RFC7748, then transform it to big-endian * form, then hisi_hpre can use the data. */ ptr[31] &= 0x7f; -- cgit From ae6ce7b17e508d335387ee4f280bda0134758aaf Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 2 Apr 2021 10:47:57 +0800 Subject: crypto: hisilicon/sec - fixup checking the 3DES weak key skcipher: Add a verifying to check whether the triple DES key is weak. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 71c01256ef0c..133aede8bf07 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -571,10 +572,18 @@ static void sec_skcipher_uninit(struct crypto_skcipher *tfm) sec_ctx_base_uninit(ctx); } -static int sec_skcipher_3des_setkey(struct sec_cipher_ctx *c_ctx, +static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen, const enum sec_cmode c_mode) { + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + int ret; + + ret = verify_skcipher_des3_key(tfm, key); + if (ret) + return ret; + switch (keylen) { case SEC_DES3_2KEY_SIZE: c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; @@ -647,7 +656,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, switch (c_alg) { case SEC_CALG_3DES: - ret = sec_skcipher_3des_setkey(c_ctx, keylen, c_mode); + ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode); break; case SEC_CALG_AES: case SEC_CALG_SM4: -- cgit From 556b64b9f39e678e64bd69fa180098e5174f3104 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 2 Apr 2021 19:22:30 +0800 Subject: crypto: hisilicon/qm - delete redundant code The "qdma" is a structure variable instead of actual data. This structure doesn't need to be zerod, The memset is useless and redundant. So delete it. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index c91e57a62cdb..35e6b5817e06 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2620,7 +2620,6 @@ void hisi_qm_uninit(struct hisi_qm *qm) hisi_qm_cache_wb(qm); dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); - memset(&qm->qdma, 0, sizeof(qm->qdma)); } qm_irq_unregister(qm); -- cgit From 7ed83901326f781524af2d969185440efe23f964 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Tue, 6 Apr 2021 20:56:02 +0800 Subject: crypto: hisilicon/qm - add stop queue by hardware Kunpeng930 could be able to stop queue by writing hardware registers, which will trigger tasks in device to be flushed out. In order to be compatible with the kunpeng920 driver, add 'qm_hw_ops_v3' to adapt Kunpeng930. And 'stop_qp' callback is added in 'qm_hw_ops_v3' to write hardware registers. Call 'stop_qp' to drain the tasks in device before releasing queue. Signed-off-by: Weili Qian Reviewed-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 35e6b5817e06..956185130e95 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -38,6 +38,7 @@ #define QM_MB_CMD_SQC_BT 0x4 #define QM_MB_CMD_CQC_BT 0x5 #define QM_MB_CMD_SQC_VFT_V2 0x6 +#define QM_MB_CMD_STOP_QP 0x8 #define QM_MB_CMD_SEND_BASE 0x300 #define QM_MB_EVENT_SHIFT 8 @@ -349,6 +350,7 @@ struct hisi_qm_hw_ops { void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe); void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); + int (*stop_qp)(struct hisi_qp *qp); }; struct qm_dfx_item { @@ -1711,6 +1713,11 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } +static int qm_stop_qp(struct hisi_qp *qp) +{ + return qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); +} + static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, .get_irq_num = qm_get_irq_num_v1, @@ -1726,6 +1733,16 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { .hw_error_handle = qm_hw_error_handle_v2, }; +static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { + .get_vft = qm_get_vft_v2, + .qm_db = qm_db_v2, + .get_irq_num = qm_get_irq_num_v2, + .hw_error_init = qm_hw_error_init_v2, + .hw_error_uninit = qm_hw_error_uninit_v2, + .hw_error_handle = qm_hw_error_handle_v2, + .stop_qp = qm_stop_qp, +}; + static void *qm_get_avail_sqe(struct hisi_qp *qp) { struct hisi_qp_status *qp_status = &qp->qp_status; @@ -2005,6 +2022,14 @@ static int qm_drain_qp(struct hisi_qp *qp) if (qm->err_status.is_qm_ecc_mbit || qm->err_status.is_dev_ecc_mbit) return 0; + /* Kunpeng930 supports drain qp by device */ + if (qm->ops->stop_qp) { + ret = qm->ops->stop_qp(qp); + if (ret) + dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id); + return ret; + } + addr = qm_ctx_alloc(qm, size, &dma_addr); if (IS_ERR(addr)) { dev_err(dev, "Failed to alloc ctx for sqc and cqc!\n"); @@ -2565,8 +2590,10 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) if (qm->ver == QM_HW_V1) qm->ops = &qm_hw_ops_v1; - else + else if (qm->ver == QM_HW_V2) qm->ops = &qm_hw_ops_v2; + else + qm->ops = &qm_hw_ops_v3; pci_set_drvdata(pdev, qm); mutex_init(&qm->mailbox_lock); -- cgit From 6e57871c3b756029f44caf08815a9cdee871eb59 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Wed, 7 Apr 2021 17:44:33 +0800 Subject: crypto: hisilicon/trng - add version to adapt new algorithm Kunpeng930 supports trng and prng, but Kunpeng920 only supports trng. Therefore, version information is added to ensure that prng is not registered to Crypto subsystem on Kunpeng920. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/trng/trng.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index 29712685498a..829f2caf0f67 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -18,6 +18,8 @@ #define HISI_TRNG_REG 0x00F0 #define HISI_TRNG_BYTES 4 #define HISI_TRNG_QUALITY 512 +#define HISI_TRNG_VERSION 0x01B8 +#define HISI_TRNG_VER_V1 GENMASK(31, 0) #define SLEEP_US 10 #define TIMEOUT_US 10000 #define SW_DRBG_NUM_SHIFT 2 @@ -50,6 +52,7 @@ struct hisi_trng { struct hisi_trng_list *trng_list; struct list_head list; struct hwrng rng; + u32 ver; bool is_used; struct mutex mutex; }; @@ -260,6 +263,7 @@ static int hisi_trng_probe(struct platform_device *pdev) return PTR_ERR(trng->base); trng->is_used = false; + trng->ver = readl(trng->base + HISI_TRNG_VERSION); if (!trng_devices.is_init) { INIT_LIST_HEAD(&trng_devices.list); mutex_init(&trng_devices.lock); @@ -267,7 +271,8 @@ static int hisi_trng_probe(struct platform_device *pdev) } hisi_trng_add_to_list(trng); - if (atomic_inc_return(&trng_active_devs) == 1) { + if (trng->ver != HISI_TRNG_VER_V1 && + atomic_inc_return(&trng_active_devs) == 1) { ret = crypto_register_rng(&hisi_trng_alg); if (ret) { dev_err(&pdev->dev, @@ -289,7 +294,8 @@ static int hisi_trng_probe(struct platform_device *pdev) return ret; err_crypto_unregister: - if (atomic_dec_return(&trng_active_devs) == 0) + if (trng->ver != HISI_TRNG_VER_V1 && + atomic_dec_return(&trng_active_devs) == 0) crypto_unregister_rng(&hisi_trng_alg); err_remove_from_list: @@ -305,7 +311,8 @@ static int hisi_trng_remove(struct platform_device *pdev) while (hisi_trng_del_from_list(trng)) ; - if (atomic_dec_return(&trng_active_devs) == 0) + if (trng->ver != HISI_TRNG_VER_V1 && + atomic_dec_return(&trng_active_devs) == 0) crypto_unregister_rng(&hisi_trng_alg); return 0; -- cgit From 07a4356bafa932eb64159866059ce17847520ecb Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Thu, 8 Apr 2021 18:23:51 +0800 Subject: crypto: hisilicon/sec - use the correct print format According to Documentation/core-api/printk-formats.rst, Use the correct print format. Printing an unsigned int value should use %u instead of %d. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec/sec_drv.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c index cf44f076f976..c8de1b51c843 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.c +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -233,7 +233,7 @@ static int sec_queue_map_io(struct sec_queue *queue) IORESOURCE_MEM, 2 + queue->queue_id); if (!res) { - dev_err(dev, "Failed to get queue %d memory resource\n", + dev_err(dev, "Failed to get queue %u memory resource\n", queue->queue_id); return -ENOMEM; } @@ -653,12 +653,12 @@ static int sec_queue_free(struct sec_queue *queue) struct sec_dev_info *info = queue->dev_info; if (queue->queue_id >= SEC_Q_NUM) { - dev_err(info->dev, "No queue %d\n", queue->queue_id); + dev_err(info->dev, "No queue %u\n", queue->queue_id); return -ENODEV; } if (!queue->in_use) { - dev_err(info->dev, "Queue %d is idle\n", queue->queue_id); + dev_err(info->dev, "Queue %u is idle\n", queue->queue_id); return -ENODEV; } -- cgit From 51028c6efa90ba4ab2311b0977287aac8c154f5b Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 9 Apr 2021 17:04:00 +0800 Subject: crypto: hisilicon/sgl - add a comment for block size initialization This seems "32" and "31" is obfuscating, It might be better to add a comment, which explain it. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sgl.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 3bff6394acaf..cf1629c367b3 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -66,6 +66,11 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, sgl_size = sizeof(struct acc_hw_sge) * sge_nr + sizeof(struct hisi_acc_hw_sgl); + + /* + * the pool may allocate a block of memory of size PAGE_SIZE * 2^(MAX_ORDER - 1), + * block size may exceed 2^31 on ia64, so the max of block size is 2^31 + */ block_size = 1 << (PAGE_SHIFT + MAX_ORDER <= 32 ? PAGE_SHIFT + MAX_ORDER - 1 : 31); sgl_num_per_block = block_size / sgl_size; -- cgit From c5f735bbf35a67c5f3c6db0d7621159f1cb536d1 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 9 Apr 2021 17:04:01 +0800 Subject: crypto: hisilicon/sgl - delete unneeded variable initialization Delete unneeded variable initialization Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sgl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index cf1629c367b3..b8a811fb3346 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -56,7 +56,7 @@ struct hisi_acc_sgl_pool { struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, u32 count, u32 sge_nr) { - u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl = 0; + u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl; struct hisi_acc_sgl_pool *pool; struct mem_block *block; u32 i, j; -- cgit From 197272b8f0bb8de37248a30469262a79edb8e512 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 9 Apr 2021 17:04:02 +0800 Subject: crypto: hisilicon/sgl - add some dfx logs Add some dfx logs in some abnormal exit situations. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sgl.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index b8a811fb3346..d04e551d5538 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -90,8 +90,10 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, block[i].sgl = dma_alloc_coherent(dev, block_size, &block[i].sgl_dma, GFP_KERNEL); - if (!block[i].sgl) + if (!block[i].sgl) { + dev_err(dev, "Fail to allocate hw SG buffer!\n"); goto err_free_mem; + } block[i].size = block_size; } @@ -100,8 +102,10 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size, &block[i].sgl_dma, GFP_KERNEL); - if (!block[i].sgl) + if (!block[i].sgl) { + dev_err(dev, "Fail to allocate remained hw SG buffer!\n"); goto err_free_mem; + } block[i].size = remain_sgl * sgl_size; } @@ -216,16 +220,19 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, sg_n = sg_nents(sgl); sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); - if (!sg_n_mapped) + if (!sg_n_mapped) { + dev_err(dev, "DMA mapping for SG error!\n"); return ERR_PTR(-EINVAL); + } if (sg_n_mapped > pool->sge_nr) { - dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + dev_err(dev, "the number of entries in input scatterlist is bigger than SGL pool setting.\n"); return ERR_PTR(-EINVAL); } curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma); if (IS_ERR(curr_hw_sgl)) { + dev_err(dev, "Get SGL error!\n"); dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); return ERR_PTR(-ENOMEM); -- cgit From 4b95e17b8fcce5a137403f508fc77cad7d014c68 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 9 Apr 2021 17:04:03 +0800 Subject: crypto: hisilicon/sgl - fix the soft sg map to hardware sg The buffer of the hardware sge needs to be initialized by soft sgl. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sgl.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index d04e551d5538..7f11e41385ae 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -176,6 +176,7 @@ static void sg_map_to_hw_sg(struct scatterlist *sgl, { hw_sge->buf = sg_dma_address(sgl); hw_sge->len = cpu_to_le32(sg_dma_len(sgl)); + hw_sge->page_ctrl = sg_virt(sgl); } static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) -- cgit From 7e958d301c0db051c358001d818c8b8637131190 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Fri, 9 Apr 2021 17:04:04 +0800 Subject: crypto: hisilicon/sgl - fix the sg buf unmap Add data cleared operation for sge data. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sgl.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 7f11e41385ae..057273769f26 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -192,6 +192,18 @@ static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum) hw_sgl->entry_sum_in_chain = cpu_to_le16(sum); } +static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) +{ + struct acc_hw_sge *hw_sge = hw_sgl->sge_entries; + int i; + + for (i = 0; i < le16_to_cpu(hw_sgl->entry_sum_in_sgl); i++) { + hw_sge[i].page_ctrl = NULL; + hw_sge[i].buf = 0; + hw_sge[i].len = 0; + } +} + /** * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl. * @dev: The device which hw sgl belongs to. @@ -269,7 +281,7 @@ void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, return; dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL); - + clear_hw_sgl_sge(hw_sgl); hw_sgl->entry_sum_in_chain = 0; hw_sgl->entry_sum_in_sgl = 0; hw_sgl->entry_length_in_sgl = 0; -- cgit From a2035904f012090e6d5362bd9e0fe35eb3a32eb1 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Sat, 10 Apr 2021 17:46:59 +0800 Subject: crypto: hisilicon/hpre - delete the rudundant space after return There are two spaces after return, just keep one. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 2 +- drivers/crypto/hisilicon/hpre/hpre_main.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 1b7f15125275..e523ef70472d 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -340,7 +340,7 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, HREE_SQE_DONE_MASK; if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) - return 0; + return 0; return -EINVAL; } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index a8c79212057d..2962c33ca75c 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -472,7 +472,7 @@ static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val) ~HPRE_CTRL_CNT_CLR_CE_BIT) | val; writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE); - return 0; + return 0; } static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file) @@ -494,7 +494,7 @@ static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY); - return 0; + return 0; } static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf, -- cgit From c4433247d91f5e1fe2c8db6b09288d7a0808037c Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Sat, 10 Apr 2021 17:47:00 +0800 Subject: crypto: hisilicon/hpre - use the correct variable type The return value of 'le32_to_cpu' is unsigned, so change the variable type from 'int' to 'unsigned int'. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index e523ef70472d..e45e6a10d7a8 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -322,7 +322,8 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { struct hpre_asym_request *req; - int err, id, done; + unsigned int err, done; + int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 -- cgit From 5bc3962c53c7062a711ed84cf97a15bef97ecf17 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Sat, 10 Apr 2021 17:47:01 +0800 Subject: crypto: hisilicon/hpre - add debug log When the received sqe is abnormal, the error message in the sqe written back by the hardware is printed to help to analyze the abnormal causes. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index e45e6a10d7a8..a380087c83f7 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -321,14 +321,16 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { + struct device *dev = HPRE_DEV(ctx); struct hpre_asym_request *req; - unsigned int err, done; + unsigned int err, done, alg; int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 #define HREE_HW_ERR_MASK 0x7ff #define HREE_SQE_DONE_MASK 0x3 +#define HREE_ALG_TYPE_MASK 0x1f id = (int)le16_to_cpu(sqe->tag); req = ctx->req_list[id]; hpre_rm_req_from_ctx(req); @@ -343,6 +345,10 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) return 0; + alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; + dev_err_ratelimited(dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", + alg, done, err); + return -EINVAL; } -- cgit From 705f6e134ad17b171884fb7a64458184e3ecc808 Mon Sep 17 00:00:00 2001 From: Hui Tang Date: Sat, 10 Apr 2021 17:50:32 +0800 Subject: crypto: hisilicon/hpre - delete redundant log and return in advance 'hpre_cfg_by_dsm' has checked and printed error path internally. It is not necessary to do it here, so remove it. It should return error immediately when return value of 'hpre_cfg_by_dsm' is non-zero, and no need to execute the remaining sentences. Signed-off-by: Hui Tang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 2962c33ca75c..916af3f32ed5 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -384,7 +384,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) if (qm->ver == QM_HW_V2) { ret = hpre_cfg_by_dsm(qm); if (ret) - dev_err(dev, "acpi_evaluate_dsm err.\n"); + return ret; disable_flr_of_bme(qm); } -- cgit From d9e21600dc0930d49a09060dd20ec8544983f202 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Mon, 12 Apr 2021 20:31:32 +0800 Subject: crypto: hisilicon - dynamic configuration 'err_info' 'err_info' does not support dynamic configuration since it is const type. Therefore, in order to support new error type later, 'err_info' is changed to dynamic configuration. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 26 ++++++++++++++++---------- drivers/crypto/hisilicon/qm.c | 22 ++++++++++------------ drivers/crypto/hisilicon/qm.h | 3 ++- drivers/crypto/hisilicon/sec2/sec_main.c | 26 ++++++++++++++++---------- drivers/crypto/hisilicon/zip/zip_main.c | 25 +++++++++++++++---------- 5 files changed, 59 insertions(+), 43 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 916af3f32ed5..046bc962c8b2 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -807,6 +807,20 @@ static void hpre_open_axi_master_ooo(struct hisi_qm *qm) HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB)); } +static void hpre_err_info_init(struct hisi_qm *qm) +{ + struct hisi_qm_err_info *err_info = &qm->err_info; + + err_info->ce = QM_BASE_CE; + err_info->fe = 0; + err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | + HPRE_OOO_ECC_2BIT_ERR; + err_info->dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE; + err_info->msi_wr_port = HPRE_WR_MSI_PORT; + err_info->acpi_rst = "HRST"; + err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT; +} + static const struct hisi_qm_err_ini hpre_err_ini = { .hw_init = hpre_set_user_domain_and_cache, .hw_err_enable = hpre_hw_error_enable, @@ -815,16 +829,7 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .clear_dev_hw_err_status = hpre_clear_hw_err_status, .log_dev_hw_err = hpre_log_hw_error, .open_axi_master_ooo = hpre_open_axi_master_ooo, - .err_info = { - .ce = QM_BASE_CE, - .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT, - .fe = 0, - .ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | - HPRE_OOO_ECC_2BIT_ERR, - .dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE, - .msi_wr_port = HPRE_WR_MSI_PORT, - .acpi_rst = "HRST", - } + .err_info_init = hpre_err_info_init, }; static int hpre_pf_probe_init(struct hpre *hpre) @@ -837,6 +842,7 @@ static int hpre_pf_probe_init(struct hpre *hpre) return ret; qm->err_ini = &hpre_err_ini; + qm->err_ini->err_info_init(qm); hisi_qm_dev_err_init(qm); return 0; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 956185130e95..d261025386d7 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1702,7 +1702,7 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) { writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); - writel(qm->err_ini->err_info.nfe, + writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE); return ACC_ERR_RECOVERED; } @@ -3107,7 +3107,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); static void qm_hw_error_init(struct hisi_qm *qm) { - const struct hisi_qm_err_info *err_info = &qm->err_ini->err_info; + struct hisi_qm_err_info *err_info = &qm->err_info; if (!qm->ops->hw_error_init) { dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); @@ -3459,15 +3459,15 @@ static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) /* get device hardware error status */ err_sts = qm->err_ini->get_dev_hw_err_status(qm); if (err_sts) { - if (err_sts & qm->err_ini->err_info.ecc_2bits_mask) + if (err_sts & qm->err_info.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; if (qm->err_ini->log_dev_hw_err) qm->err_ini->log_dev_hw_err(qm, err_sts); /* ce error does not need to be reset */ - if ((err_sts | qm->err_ini->err_info.dev_ce_mask) == - qm->err_ini->err_info.dev_ce_mask) { + if ((err_sts | qm->err_info.dev_ce_mask) == + qm->err_info.dev_ce_mask) { if (qm->err_ini->clear_dev_hw_err_status) qm->err_ini->clear_dev_hw_err_status(qm, err_sts); @@ -3780,7 +3780,7 @@ static int qm_soft_reset(struct hisi_qm *qm) acpi_status s; s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev), - qm->err_ini->err_info.acpi_rst, + qm->err_info.acpi_rst, NULL, &value); if (ACPI_FAILURE(s)) { pci_err(pdev, "NO controller reset method!\n"); @@ -3848,12 +3848,11 @@ static void qm_restart_prepare(struct hisi_qm *qm) /* temporarily close the OOO port used for PEH to write out MSI */ value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); - writel(value & ~qm->err_ini->err_info.msi_wr_port, + writel(value & ~qm->err_info.msi_wr_port, qm->io_base + ACC_AM_CFG_PORT_WR_EN); /* clear dev ecc 2bit error source if having */ - value = qm_get_dev_err_status(qm) & - qm->err_ini->err_info.ecc_2bits_mask; + value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask; if (value && qm->err_ini->clear_dev_hw_err_status) qm->err_ini->clear_dev_hw_err_status(qm, value); @@ -3877,7 +3876,7 @@ static void qm_restart_done(struct hisi_qm *qm) /* open the OOO port for PEH to write out MSI */ value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); - value |= qm->err_ini->err_info.msi_wr_port; + value |= qm->err_info.msi_wr_port; writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN); qm->err_status.is_qm_ecc_mbit = false; @@ -4016,8 +4015,7 @@ static int qm_check_dev_error(struct hisi_qm *qm) if (ret) return ret; - return (qm_get_dev_err_status(qm) & - qm->err_ini->err_info.ecc_2bits_mask); + return (qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask); } void hisi_qm_reset_prepare(struct pci_dev *pdev) diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 34e69fe3b671..8c67712e3972 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -186,7 +186,7 @@ struct hisi_qm_err_ini { void (*open_axi_master_ooo)(struct hisi_qm *qm); void (*close_axi_master_ooo)(struct hisi_qm *qm); void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts); - struct hisi_qm_err_info err_info; + void (*err_info_init)(struct hisi_qm *qm); }; struct hisi_qm_list { @@ -226,6 +226,7 @@ struct hisi_qm { struct hisi_qm_status status; const struct hisi_qm_err_ini *err_ini; + struct hisi_qm_err_info err_info; struct hisi_qm_err_status err_status; unsigned long misc_ctl; /* driver removing and reset sched */ diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 1affa1ba27e1..26b9904a6617 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -701,6 +701,20 @@ static void sec_open_axi_master_ooo(struct hisi_qm *qm) writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG); } +static void sec_err_info_init(struct hisi_qm *qm) +{ + struct hisi_qm_err_info *err_info = &qm->err_info; + + err_info->ce = QM_BASE_CE; + err_info->fe = 0; + err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; + err_info->dev_ce_mask = SEC_RAS_CE_ENB_MSK; + err_info->msi_wr_port = BIT(0); + err_info->acpi_rst = "SRST"; + err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | + QM_ACC_WB_NOT_READY_TIMEOUT; +} + static const struct hisi_qm_err_ini sec_err_ini = { .hw_init = sec_set_user_domain_and_cache, .hw_err_enable = sec_hw_error_enable, @@ -709,16 +723,7 @@ static const struct hisi_qm_err_ini sec_err_ini = { .clear_dev_hw_err_status = sec_clear_hw_err_status, .log_dev_hw_err = sec_log_hw_error, .open_axi_master_ooo = sec_open_axi_master_ooo, - .err_info = { - .ce = QM_BASE_CE, - .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | - QM_ACC_WB_NOT_READY_TIMEOUT, - .fe = 0, - .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, - .dev_ce_mask = SEC_RAS_CE_ENB_MSK, - .msi_wr_port = BIT(0), - .acpi_rst = "SRST", - } + .err_info_init = sec_err_info_init, }; static int sec_pf_probe_init(struct sec_dev *sec) @@ -727,6 +732,7 @@ static int sec_pf_probe_init(struct sec_dev *sec) int ret; qm->err_ini = &sec_err_ini; + qm->err_ini->err_info_init(qm); ret = sec_set_user_domain_and_cache(qm); if (ret) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4d2e9c85cc1a..50407bbfa089 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -657,6 +657,19 @@ static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) qm->io_base + HZIP_CORE_INT_SET); } +static void hisi_zip_err_info_init(struct hisi_qm *qm) +{ + struct hisi_qm_err_info *err_info = &qm->err_info; + + err_info->ce = QM_BASE_CE; + err_info->fe = 0; + err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; + err_info->dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE; + err_info->msi_wr_port = HZIP_WR_PORT; + err_info->acpi_rst = "ZRST"; + err_info->nfe = QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT; +} + static const struct hisi_qm_err_ini hisi_zip_err_ini = { .hw_init = hisi_zip_set_user_domain_and_cache, .hw_err_enable = hisi_zip_hw_error_enable, @@ -666,16 +679,7 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .log_dev_hw_err = hisi_zip_log_hw_error, .open_axi_master_ooo = hisi_zip_open_axi_master_ooo, .close_axi_master_ooo = hisi_zip_close_axi_master_ooo, - .err_info = { - .ce = QM_BASE_CE, - .nfe = QM_BASE_NFE | - QM_ACC_WB_NOT_READY_TIMEOUT, - .fe = 0, - .ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC, - .dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE, - .msi_wr_port = HZIP_WR_PORT, - .acpi_rst = "ZRST", - } + .err_info_init = hisi_zip_err_info_init, }; static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) @@ -690,6 +694,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) hisi_zip->ctrl = ctrl; ctrl->hisi_zip = hisi_zip; qm->err_ini = &hisi_zip_err_ini; + qm->err_ini->err_info_init(qm); hisi_zip_set_user_domain_and_cache(qm); hisi_qm_dev_err_init(qm); -- cgit From b7220a7439fa5e2506428252e9046963ab51d48d Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Mon, 12 Apr 2021 20:31:33 +0800 Subject: crypto: hisilicon - support new error types for ZIP Kunpeng930 ZIP adds 'zip_axi_poison_err' 'zip_sva_err' and 'QM_ACC_DO_TASK_TIMEOUT' hardware error types. This patch enables the error types and configures the error types as NFE. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_main.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 50407bbfa089..2178b40e9f82 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -68,10 +68,10 @@ #define HZIP_CORE_INT_RAS_CE_ENABLE 0x1 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168 -#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x7FE +#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x1FFE #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 -#define HZIP_CORE_INT_MASK_ALL GENMASK(10, 0) +#define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0) #define HZIP_COMP_CORE_NUM 2 #define HZIP_DECOMP_CORE_NUM 6 #define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \ @@ -133,6 +133,8 @@ static const struct hisi_zip_hw_error zip_hw_error[] = { { .int_msk = BIT(8), .msg = "zip_com_inf_err" }, { .int_msk = BIT(9), .msg = "zip_enc_inf_err" }, { .int_msk = BIT(10), .msg = "zip_pre_out_err" }, + { .int_msk = BIT(11), .msg = "zip_axi_poison_err" }, + { .int_msk = BIT(12), .msg = "zip_sva_err" }, { /* sentinel */ } }; @@ -668,6 +670,9 @@ static void hisi_zip_err_info_init(struct hisi_qm *qm) err_info->msi_wr_port = HZIP_WR_PORT; err_info->acpi_rst = "ZRST"; err_info->nfe = QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT; + + if (qm->ver >= QM_HW_V3) + err_info->nfe |= QM_ACC_DO_TASK_TIMEOUT; } static const struct hisi_qm_err_ini hisi_zip_err_ini = { -- cgit From 10594d1e5ff79f0bcc96d2abde364b58adc06ffb Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Mon, 12 Apr 2021 20:31:34 +0800 Subject: crypto: hisilicon - add new error type for SEC Kunpeng930 SEC adds several new hardware error types. This patch enables the new error types and configures the error types as NFE. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_main.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 26b9904a6617..6f0062d4408c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -42,8 +42,8 @@ #define SEC_ECC_NUM 16 #define SEC_ECC_MASH 0xFF #define SEC_CORE_INT_DISABLE 0x0 -#define SEC_CORE_INT_ENABLE 0x1ff -#define SEC_CORE_INT_CLEAR 0x1ff +#define SEC_CORE_INT_ENABLE 0x7c1ff +#define SEC_CORE_INT_CLEAR 0x7c1ff #define SEC_SAA_ENABLE 0x17f #define SEC_RAS_CE_REG 0x301050 @@ -51,7 +51,7 @@ #define SEC_RAS_NFE_REG 0x301058 #define SEC_RAS_CE_ENB_MSK 0x88 #define SEC_RAS_FE_ENB_MSK 0x0 -#define SEC_RAS_NFE_ENB_MSK 0x177 +#define SEC_RAS_NFE_ENB_MSK 0x7c177 #define SEC_RAS_DISABLE 0x0 #define SEC_MEM_START_INIT_REG 0x301100 #define SEC_MEM_INIT_DONE_REG 0x301104 @@ -147,6 +147,26 @@ static const struct sec_hw_error sec_hw_errors[] = { .int_msk = BIT(8), .msg = "sec_chain_buff_err_rint" }, + { + .int_msk = BIT(14), + .msg = "sec_no_secure_access" + }, + { + .int_msk = BIT(15), + .msg = "sec_wrapping_key_auth_err" + }, + { + .int_msk = BIT(16), + .msg = "sec_km_key_crc_fail" + }, + { + .int_msk = BIT(17), + .msg = "sec_axi_poison_err" + }, + { + .int_msk = BIT(18), + .msg = "sec_sva_err" + }, {} }; -- cgit From c4aab24448a3948bcc318a318bd963e25e4c5b02 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Mon, 12 Apr 2021 20:31:35 +0800 Subject: crypto: hisilicon - enable new error types for QM QM adds 'qm_mailbox_timeout' and 'qm_flr_timeout' hardware error types on Kunpeng930. This patch enables the new error types and configures the error types as NFE. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 6 ++++-- drivers/crypto/hisilicon/qm.h | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'drivers/crypto/hisilicon') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d261025386d7..ce439a0c66c9 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -136,9 +136,9 @@ #define QM_DFX_CNT_CLR_CE 0x100118 #define QM_ABNORMAL_INT_SOURCE 0x100000 -#define QM_ABNORMAL_INT_SOURCE_CLR GENMASK(12, 0) +#define QM_ABNORMAL_INT_SOURCE_CLR GENMASK(14, 0) #define QM_ABNORMAL_INT_MASK 0x100004 -#define QM_ABNORMAL_INT_MASK_VALUE 0x1fff +#define QM_ABNORMAL_INT_MASK_VALUE 0x7fff #define QM_ABNORMAL_INT_STATUS 0x100008 #define QM_ABNORMAL_INT_SET 0x10000c #define QM_ABNORMAL_INF00 0x100010 @@ -391,6 +391,8 @@ static const struct hisi_qm_hw_error qm_hw_error[] = { { .int_msk = BIT(10), .msg = "qm_db_timeout" }, { .int_msk = BIT(11), .msg = "qm_of_fifo_of" }, { .int_msk = BIT(12), .msg = "qm_db_random_invalid" }, + { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" }, + { .int_msk = BIT(14), .msg = "qm_flr_timeout" }, { /* sentinel */ } }; diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 8c67712e3972..acefdf8b3a50 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -64,10 +64,13 @@ #define QM_DB_TIMEOUT BIT(10) #define QM_OF_FIFO_OF BIT(11) #define QM_DB_RANDOM_INVALID BIT(12) +#define QM_MAILBOX_TIMEOUT BIT(13) +#define QM_FLR_TIMEOUT BIT(14) #define QM_BASE_NFE (QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \ QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \ - QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID) + QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID | \ + QM_MAILBOX_TIMEOUT | QM_FLR_TIMEOUT) #define QM_BASE_CE QM_ECC_1BIT #define QM_Q_DEPTH 1024 -- cgit