diff options
Diffstat (limited to 'drivers/crypto')
317 files changed, 18489 insertions, 13296 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 0a9cdd31cbd9..8d3b5d2890f8 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -180,6 +180,7 @@ config CRYPTO_PAES_S390 depends on PKEY select CRYPTO_ALGAPI select CRYPTO_SKCIPHER + select CRYPTO_ENGINE help This is the s390 hardware accelerated implementation of the AES cipher algorithms for use with protected key. @@ -187,6 +188,19 @@ config CRYPTO_PAES_S390 Select this option if you want to use the paes cipher for example to use protected key encrypted devices. +config CRYPTO_PHMAC_S390 + tristate "PHMAC cipher algorithms" + depends on S390 + depends on PKEY + select CRYPTO_HASH + select CRYPTO_ENGINE + help + This is the s390 hardware accelerated implementation of the + protected key HMAC support for SHA224, SHA256, SHA384 and SHA512. + + Select this option if you want to use the phmac digests + for example to use dm-integrity with secure/protected keys. + config S390_PRNG tristate "Pseudo random number generator device driver" depends on S390 @@ -200,23 +214,6 @@ config S390_PRNG It is available as of z9. -config CRYPTO_DEV_NIAGARA2 - tristate "Niagara2 Stream Processing Unit driver" - select CRYPTO_LIB_DES - select CRYPTO_SKCIPHER - select CRYPTO_HASH - select CRYPTO_MD5 - select CRYPTO_SHA1 - select CRYPTO_SHA256 - depends on SPARC64 - help - Each core of a Niagara2 processor contains a Stream - Processing Unit, which itself contains several cryptographic - sub-units. One set provides the Modular Arithmetic Unit, - used for SSL offload. The other set provides the Cipher - Group, which can perform encryption, decryption, hashing, - checksumming, and raw copies. - config CRYPTO_DEV_SL3516 tristate "Storlink SL3516 crypto offloader" depends on ARCH_GEMINI || COMPILE_TEST @@ -442,7 +439,7 @@ config CRYPTO_DEV_ATMEL_AUTHENC config CRYPTO_DEV_ATMEL_AES tristate "Support for Atmel AES hw accelerator" - depends on ARCH_AT91 || COMPILE_TEST + depends on ARCH_MICROCHIP || COMPILE_TEST select CRYPTO_AES select CRYPTO_AEAD select CRYPTO_SKCIPHER @@ -547,13 +544,6 @@ source "drivers/crypto/cavium/nitrox/Kconfig" source "drivers/crypto/marvell/Kconfig" source "drivers/crypto/intel/Kconfig" -config CRYPTO_DEV_CAVIUM_ZIP - tristate "Cavium ZIP driver" - depends on PCI && 64BIT && (ARM64 || COMPILE_TEST) - help - Select this option if you want to enable compression/decompression - acceleration on Cavium's ARM based SoCs - config CRYPTO_DEV_QCE tristate "Qualcomm crypto engine accelerator" depends on ARCH_QCOM || COMPILE_TEST @@ -735,6 +725,19 @@ config CRYPTO_DEV_TEGRA Select this to enable Tegra Security Engine which accelerates various AES encryption/decryption and HASH algorithms. +config CRYPTO_DEV_XILINX_TRNG + tristate "Support for Xilinx True Random Generator" + depends on ZYNQMP_FIRMWARE || COMPILE_TEST + select CRYPTO_DF80090A + select CRYPTO_RNG + select HW_RANDOM + help + Xilinx Versal SoC driver provides kernel-side support for True Random Number + Generator and Pseudo random Number in CTR_DRBG mode as defined in NIST SP800-90A. + + To compile this driver as a module, choose M here: the module + will be called xilinx-trng. + config CRYPTO_DEV_ZYNQMP_AES tristate "Support for Xilinx ZynqMP AES hw accelerator" depends on ZYNQMP_FIRMWARE || COMPILE_TEST @@ -850,6 +853,7 @@ config CRYPTO_DEV_CCREE If unsure say Y. source "drivers/crypto/hisilicon/Kconfig" +source "drivers/crypto/loongson/Kconfig" source "drivers/crypto/amlogic/Kconfig" @@ -872,5 +876,7 @@ config CRYPTO_DEV_SA2UL source "drivers/crypto/aspeed/Kconfig" source "drivers/crypto/starfive/Kconfig" +source "drivers/crypto/inside-secure/eip93/Kconfig" +source "drivers/crypto/ti/Kconfig" endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index ad4ccef67d12..322ae8854e3e 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -8,12 +8,9 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ -obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/ -obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/ obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o @@ -21,8 +18,6 @@ obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o obj-$(CONFIG_CRYPTO_DEV_MARVELL) += marvell/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o -obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o -n2_crypto-y := n2_core.o n2_asm.o obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ obj-$(CONFIG_CRYPTO_DEV_OMAP) += omap-crypto.o obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes-driver.o @@ -45,10 +40,13 @@ obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra/ obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/ #obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/ -obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/ +obj-y += inside-secure/ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-y += xilinx/ obj-y += hisilicon/ +obj-y += loongson/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ obj-y += intel/ obj-y += starfive/ +obj-y += cavium/ +obj-y += ti/ diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index 19b7fb4a93e8..021614b65e39 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -33,22 +33,30 @@ static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) if (sg_nents_for_len(areq->src, areq->cryptlen) > MAX_SG || sg_nents_for_len(areq->dst, areq->cryptlen) > MAX_SG) { - algt->stat_fb_maxsg++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_maxsg++; + return true; } if (areq->cryptlen < crypto_skcipher_ivsize(tfm)) { - algt->stat_fb_leniv++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_leniv++; + return true; } if (areq->cryptlen == 0) { - algt->stat_fb_len0++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_len0++; + return true; } if (areq->cryptlen % 16) { - algt->stat_fb_mod16++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_mod16++; + return true; } @@ -56,12 +64,16 @@ static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) sg = areq->src; while (sg) { if (!IS_ALIGNED(sg->offset, sizeof(u32))) { - algt->stat_fb_srcali++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srcali++; + return true; } todo = min(len, sg->length); if (todo % 4) { - algt->stat_fb_srclen++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srclen++; + return true; } len -= todo; @@ -72,12 +84,16 @@ static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) sg = areq->dst; while (sg) { if (!IS_ALIGNED(sg->offset, sizeof(u32))) { - algt->stat_fb_dstali++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_dstali++; + return true; } todo = min(len, sg->length); if (todo % 4) { - algt->stat_fb_dstlen++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_dstlen++; + return true; } len -= todo; @@ -95,14 +111,12 @@ static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct sun8i_ce_alg_template *algt __maybe_unused; + struct sun8i_ce_alg_template *algt; algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG algt->stat_fb++; -#endif } skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); @@ -117,21 +131,19 @@ static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) return err; } -static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req) +static int sun8i_ce_cipher_prepare(struct skcipher_request *areq, + struct ce_task *cet) { - struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun8i_ce_dev *ce = op->ce; struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); struct skcipher_alg *alg = crypto_skcipher_alg(tfm); struct sun8i_ce_alg_template *algt; - struct sun8i_ce_flow *chan; - struct ce_task *cet; struct scatterlist *sg; unsigned int todo, len, offset, ivsize; u32 common, sym; - int flow, i; + int i; int nr_sgs = 0; int nr_sgd = 0; int err = 0; @@ -146,18 +158,12 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), op->keylen); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_req++; -#endif - - flow = rctx->flow; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_req++; - chan = &ce->chanlist[flow]; - - cet = chan->tl; memset(cet, 0, sizeof(struct ce_task)); - cet->t_id = cpu_to_le32(flow); + cet->t_id = cpu_to_le32(rctx->flow); common = ce->variant->alg_cipher[algt->ce_algo_id]; common |= rctx->op_dir | CE_COMM_INT; cet->t_common_ctl = cpu_to_le32(common); @@ -193,15 +199,14 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req cet->t_key = desc_addr_val_le32(ce, rctx->addr_key); ivsize = crypto_skcipher_ivsize(tfm); - if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { - rctx->ivlen = ivsize; + if (areq->iv && ivsize > 0) { if (rctx->op_dir & CE_DECRYPTION) { offset = areq->cryptlen - ivsize; - scatterwalk_map_and_copy(chan->backup_iv, areq->src, + scatterwalk_map_and_copy(rctx->backup_iv, areq->src, offset, ivsize, 0); } - memcpy(chan->bounce_iv, areq->iv, ivsize); - rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, rctx->ivlen, + memcpy(rctx->bounce_iv, areq->iv, ivsize); + rctx->addr_iv = dma_map_single(ce->dev, rctx->bounce_iv, ivsize, DMA_TO_DEVICE); if (dma_mapping_error(ce->dev, rctx->addr_iv)) { dev_err(ce->dev, "Cannot DMA MAP IV\n"); @@ -264,9 +269,8 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req goto theend_sgs; } - chan->timeout = areq->cryptlen; - rctx->nr_sgs = nr_sgs; - rctx->nr_sgd = nr_sgd; + rctx->nr_sgs = ns; + rctx->nr_sgd = nd; return 0; theend_sgs: @@ -275,22 +279,26 @@ theend_sgs: } else { if (nr_sgs > 0) dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); + + if (nr_sgd > 0) + dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); } theend_iv: if (areq->iv && ivsize > 0) { - if (rctx->addr_iv) - dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); + if (!dma_mapping_error(ce->dev, rctx->addr_iv)) + dma_unmap_single(ce->dev, rctx->addr_iv, ivsize, + DMA_TO_DEVICE); + offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { - memcpy(areq->iv, chan->backup_iv, ivsize); - memzero_explicit(chan->backup_iv, ivsize); + memcpy(areq->iv, rctx->backup_iv, ivsize); + memzero_explicit(rctx->backup_iv, ivsize); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, ivsize, 0); } - memzero_explicit(chan->bounce_iv, ivsize); + memzero_explicit(rctx->bounce_iv, ivsize); } dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); @@ -299,24 +307,17 @@ theend: return err; } -static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine, - void *async_req) +static void sun8i_ce_cipher_unprepare(struct skcipher_request *areq, + struct ce_task *cet) { - struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun8i_ce_dev *ce = op->ce; struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); - struct sun8i_ce_flow *chan; - struct ce_task *cet; unsigned int ivsize, offset; int nr_sgs = rctx->nr_sgs; int nr_sgd = rctx->nr_sgd; - int flow; - flow = rctx->flow; - chan = &ce->chanlist[flow]; - cet = chan->tl; ivsize = crypto_skcipher_ivsize(tfm); if (areq->src == areq->dst) { @@ -329,46 +330,47 @@ static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine, if (areq->iv && ivsize > 0) { if (cet->t_iv) - dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, rctx->addr_iv, ivsize, + DMA_TO_DEVICE); offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { - memcpy(areq->iv, chan->backup_iv, ivsize); - memzero_explicit(chan->backup_iv, ivsize); + memcpy(areq->iv, rctx->backup_iv, ivsize); + memzero_explicit(rctx->backup_iv, ivsize); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, ivsize, 0); } - memzero_explicit(chan->bounce_iv, ivsize); + memzero_explicit(rctx->bounce_iv, ivsize); } dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); } -static void sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq) -{ - struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq); - struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); - struct sun8i_ce_dev *ce = op->ce; - struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq); - int flow, err; - - flow = rctx->flow; - err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm)); - sun8i_ce_cipher_unprepare(engine, areq); - local_bh_disable(); - crypto_finalize_skcipher_request(engine, breq, err); - local_bh_enable(); -} - int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq) { - int err = sun8i_ce_cipher_prepare(engine, areq); + struct skcipher_request *req = skcipher_request_cast(areq); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct sun8i_cipher_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = ctx->ce; + struct sun8i_ce_flow *chan; + int err; + + chan = &ce->chanlist[rctx->flow]; + err = sun8i_ce_cipher_prepare(req, chan->tl); if (err) return err; - sun8i_ce_cipher_run(engine, areq); + err = sun8i_ce_run_task(ce, rctx->flow, + crypto_tfm_alg_name(req->base.tfm)); + + sun8i_ce_cipher_unprepare(req, chan->tl); + + local_bh_disable(); + crypto_finalize_skcipher_request(engine, req, err); + local_bh_enable(); + return 0; } @@ -434,17 +436,17 @@ int sun8i_ce_cipher_init(struct crypto_tfm *tfm) crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) + crypto_skcipher_reqsize(op->fallback_tfm)); - memcpy(algt->fbname, - crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)), - CRYPTO_MAX_ALG_NAME); + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + memcpy(algt->fbname, + crypto_skcipher_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); - err = pm_runtime_get_sync(op->ce->dev); + err = pm_runtime_resume_and_get(op->ce->dev); if (err < 0) goto error_pm; return 0; error_pm: - pm_runtime_put_noidle(op->ce->dev); crypto_free_skcipher(op->fallback_tfm); return err; } diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index ec1ffda9ea32..c16bb6ce6ee3 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -169,6 +169,12 @@ static const struct ce_variant ce_r40_variant = { .trng = CE_ID_NOTSUPP, }; +static void sun8i_ce_dump_task_descriptors(struct sun8i_ce_flow *chan) +{ + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + chan->tl, sizeof(struct ce_task), false); +} + /* * sun8i_ce_get_engine_number() get the next channel slot * This is a simple round-robin way of getting the next channel @@ -183,7 +189,6 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) { u32 v; int err = 0; - struct ce_task *cet = ce->chanlist[flow].tl; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG ce->chanlist[flow].stat_req++; @@ -210,11 +215,10 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) mutex_unlock(&ce->mlock); wait_for_completion_interruptible_timeout(&ce->chanlist[flow].complete, - msecs_to_jiffies(ce->chanlist[flow].timeout)); + msecs_to_jiffies(CE_DMA_TIMEOUT_MS)); if (ce->chanlist[flow].status == 0) { - dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name, - ce->chanlist[flow].timeout, flow); + dev_err(ce->dev, "DMA timeout for %s on flow %d\n", name, flow); err = -EFAULT; } /* No need to lock for this read, the channel is locked so @@ -226,9 +230,8 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) /* Sadly, the error bit is not per flow */ if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); err = -EFAULT; - print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, - cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -245,9 +248,8 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) v &= 0xF; if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); err = -EFAULT; - print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, - cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -261,9 +263,8 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) v &= 0xFF; if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); err = -EFAULT; - print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, - cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -758,18 +759,6 @@ static int sun8i_ce_allocate_chanlist(struct sun8i_ce_dev *ce) err = -ENOMEM; goto error_engine; } - ce->chanlist[i].bounce_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, - GFP_KERNEL | GFP_DMA); - if (!ce->chanlist[i].bounce_iv) { - err = -ENOMEM; - goto error_engine; - } - ce->chanlist[i].backup_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, - GFP_KERNEL); - if (!ce->chanlist[i].backup_iv) { - err = -ENOMEM; - goto error_engine; - } } return 0; error_engine: @@ -832,13 +821,12 @@ static int sun8i_ce_pm_init(struct sun8i_ce_dev *ce) err = pm_runtime_set_suspended(ce->dev); if (err) return err; - pm_runtime_enable(ce->dev); - return err; -} -static void sun8i_ce_pm_exit(struct sun8i_ce_dev *ce) -{ - pm_runtime_disable(ce->dev); + err = devm_pm_runtime_enable(ce->dev); + if (err) + return err; + + return 0; } static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce) @@ -1041,7 +1029,7 @@ static int sun8i_ce_probe(struct platform_device *pdev) "sun8i-ce-ns", ce); if (err) { dev_err(ce->dev, "Cannot request CryptoEngine Non-secure IRQ (err=%d)\n", err); - goto error_irq; + goto error_pm; } err = sun8i_ce_register_algs(ce); @@ -1064,7 +1052,7 @@ static int sun8i_ce_probe(struct platform_device *pdev) pm_runtime_put_sync(ce->dev); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct dentry *dbgfs_dir __maybe_unused; + struct dentry *dbgfs_dir; struct dentry *dbgfs_stats __maybe_unused; /* Ignore error of debugfs */ @@ -1082,8 +1070,6 @@ static int sun8i_ce_probe(struct platform_device *pdev) return 0; error_alg: sun8i_ce_unregister_algs(ce); -error_irq: - sun8i_ce_pm_exit(ce); error_pm: sun8i_ce_free_chanlist(ce, MAXFLOW - 1); return err; @@ -1104,8 +1090,6 @@ static void sun8i_ce_remove(struct platform_device *pdev) #endif sun8i_ce_free_chanlist(ce, MAXFLOW - 1); - - sun8i_ce_pm_exit(ce); } static const struct of_device_id sun8i_ce_crypto_of_match_table[] = { diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 6072dd9f390b..d01594353d9a 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -23,6 +23,18 @@ #include <linux/string.h> #include "sun8i-ce.h" +static void sun8i_ce_hash_stat_fb_inc(struct crypto_ahash *tfm) +{ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { + struct sun8i_ce_alg_template *algt; + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + algt = container_of(alg, struct sun8i_ce_alg_template, + alg.hash.base); + algt->stat_fb++; + } +} + int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) { struct sun8i_ce_hash_tfm_ctx *op = crypto_ahash_ctx(tfm); @@ -46,17 +58,19 @@ int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) crypto_ahash_set_reqsize(tfm, sizeof(struct sun8i_ce_hash_reqctx) + - crypto_ahash_reqsize(op->fallback_tfm)); + crypto_ahash_reqsize(op->fallback_tfm) + + CRYPTO_DMA_PADDING); - memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm), - CRYPTO_MAX_ALG_NAME); + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + memcpy(algt->fbname, + crypto_ahash_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); - err = pm_runtime_get_sync(op->ce->dev); + err = pm_runtime_resume_and_get(op->ce->dev); if (err < 0) goto error_pm; return 0; error_pm: - pm_runtime_put_noidle(op->ce->dev); crypto_free_ahash(op->fallback_tfm); return err; } @@ -71,134 +85,112 @@ void sun8i_ce_hash_exit_tfm(struct crypto_ahash *tfm) int sun8i_ce_hash_init(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx)); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_init(&rctx->fallback_req); } int sun8i_ce_hash_export(struct ahash_request *areq, void *out) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } int sun8i_ce_hash_final(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = areq->result; + sun8i_ce_hash_stat_fb_inc(tfm); - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); - - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); return crypto_ahash_final(&rctx->fallback_req); } int sun8i_ce_hash_update(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); return crypto_ahash_update(&rctx->fallback_req); } int sun8i_ce_hash_finup(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + sun8i_ce_hash_stat_fb_inc(tfm); - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); - - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) { - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + sun8i_ce_hash_stat_fb_inc(tfm); - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { - struct sun8i_ce_alg_template *algt __maybe_unused; - struct ahash_alg *alg = crypto_ahash_alg(tfm); - - algt = container_of(alg, struct sun8i_ce_alg_template, - alg.hash.base); -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_fb++; -#endif - } + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -213,22 +205,30 @@ static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); if (areq->nbytes == 0) { - algt->stat_fb_len0++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_len0++; + return true; } /* we need to reserve one SG for padding one */ if (sg_nents_for_len(areq->src, areq->nbytes) > MAX_SG - 1) { - algt->stat_fb_maxsg++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_maxsg++; + return true; } sg = areq->src; while (sg) { if (sg->length % 4) { - algt->stat_fb_srclen++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srclen++; + return true; } if (!IS_ALIGNED(sg->offset, sizeof(u32))) { - algt->stat_fb_srcali++; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srcali++; + return true; } sg = sg_next(sg); @@ -239,29 +239,15 @@ static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) int sun8i_ce_hash_digest(struct ahash_request *areq) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); - struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); - struct sun8i_ce_alg_template *algt; - struct sun8i_ce_dev *ce; + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct sun8i_ce_dev *ce = ctx->ce; struct crypto_engine *engine; - struct scatterlist *sg; - int nr_sgs, e, i; + int e; if (sun8i_ce_hash_need_fallback(areq)) return sun8i_ce_hash_digest_fb(areq); - nr_sgs = sg_nents_for_len(areq->src, areq->nbytes); - if (nr_sgs > MAX_SG - 1) - return sun8i_ce_hash_digest_fb(areq); - - for_each_sg(areq->src, sg, nr_sgs, i) { - if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) - return sun8i_ce_hash_digest_fb(areq); - } - - algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); - ce = algt->ce; - e = sun8i_ce_get_engine_number(ce); rctx->flow = e; engine = ce->chanlist[e].engine; @@ -327,66 +313,43 @@ static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, return j; } -int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) +static int sun8i_ce_hash_prepare(struct ahash_request *areq, struct ce_task *cet) { - struct ahash_request *areq = container_of(breq, struct ahash_request, base); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); - struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); struct sun8i_ce_alg_template *algt; struct sun8i_ce_dev *ce; - struct sun8i_ce_flow *chan; - struct ce_task *cet; struct scatterlist *sg; - int nr_sgs, flow, err; + int nr_sgs, err; unsigned int len; u32 common; u64 byte_count; __le32 *bf; - void *buf = NULL; int j, i, todo; - void *result = NULL; u64 bs; int digestsize; - dma_addr_t addr_res, addr_pad; - int ns = sg_nents_for_len(areq->src, areq->nbytes); algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); ce = algt->ce; - bs = algt->alg.hash.base.halg.base.cra_blocksize; - digestsize = algt->alg.hash.base.halg.digestsize; + bs = crypto_ahash_blocksize(tfm); + digestsize = crypto_ahash_digestsize(tfm); if (digestsize == SHA224_DIGEST_SIZE) digestsize = SHA256_DIGEST_SIZE; if (digestsize == SHA384_DIGEST_SIZE) digestsize = SHA512_DIGEST_SIZE; - /* the padding could be up to two block. */ - buf = kcalloc(2, bs, GFP_KERNEL | GFP_DMA); - if (!buf) { - err = -ENOMEM; - goto theend; - } - bf = (__le32 *)buf; + bf = (__le32 *)rctx->pad; - result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); - if (!result) { - err = -ENOMEM; - goto theend; - } + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_req++; - flow = rctx->flow; - chan = &ce->chanlist[flow]; - -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_req++; -#endif dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes); - cet = chan->tl; memset(cet, 0, sizeof(struct ce_task)); - cet->t_id = cpu_to_le32(flow); + cet->t_id = cpu_to_le32(rctx->flow); common = ce->variant->alg_hash[algt->ce_algo_id]; common |= CE_COMM_INT; cet->t_common_ctl = cpu_to_le32(common); @@ -394,11 +357,12 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) cet->t_sym_ctl = 0; cet->t_asym_ctl = 0; - nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + rctx->nr_sgs = sg_nents_for_len(areq->src, areq->nbytes); + nr_sgs = dma_map_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; - goto theend; + goto err_out; } len = areq->nbytes; @@ -411,15 +375,18 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (len > 0) { dev_err(ce->dev, "remaining len %d\n", len); err = -EINVAL; - goto theend; + goto err_unmap_src; } - addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE); - cet->t_dst[0].addr = desc_addr_val_le32(ce, addr_res); - cet->t_dst[0].len = cpu_to_le32(digestsize / 4); - if (dma_mapping_error(ce->dev, addr_res)) { + + rctx->result_len = digestsize; + rctx->addr_res = dma_map_single(ce->dev, rctx->result, rctx->result_len, + DMA_FROM_DEVICE); + cet->t_dst[0].addr = desc_addr_val_le32(ce, rctx->addr_res); + cet->t_dst[0].len = cpu_to_le32(rctx->result_len / 4); + if (dma_mapping_error(ce->dev, rctx->addr_res)) { dev_err(ce->dev, "DMA map dest\n"); err = -EINVAL; - goto theend; + goto err_unmap_src; } byte_count = areq->nbytes; @@ -441,16 +408,18 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) } if (!j) { err = -EINVAL; - goto theend; + goto err_unmap_result; } - addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE); - cet->t_src[i].addr = desc_addr_val_le32(ce, addr_pad); + rctx->pad_len = j * 4; + rctx->addr_pad = dma_map_single(ce->dev, rctx->pad, rctx->pad_len, + DMA_TO_DEVICE); + cet->t_src[i].addr = desc_addr_val_le32(ce, rctx->addr_pad); cet->t_src[i].len = cpu_to_le32(j); - if (dma_mapping_error(ce->dev, addr_pad)) { + if (dma_mapping_error(ce->dev, rctx->addr_pad)) { dev_err(ce->dev, "DMA error on padding SG\n"); err = -EINVAL; - goto theend; + goto err_unmap_result; } if (ce->variant->hash_t_dlen_in_bits) @@ -458,21 +427,60 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) else cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j); - chan->timeout = areq->nbytes; + return 0; + +err_unmap_result: + dma_unmap_single(ce->dev, rctx->addr_res, rctx->result_len, + DMA_FROM_DEVICE); + +err_unmap_src: + dma_unmap_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); - err = sun8i_ce_run_task(ce, flow, crypto_ahash_alg_name(tfm)); +err_out: + return err; +} - dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE); - dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); +static void sun8i_ce_hash_unprepare(struct ahash_request *areq, + struct ce_task *cet) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_dev *ce = ctx->ce; + dma_unmap_single(ce->dev, rctx->addr_pad, rctx->pad_len, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, rctx->addr_res, rctx->result_len, + DMA_FROM_DEVICE); + dma_unmap_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); +} + +int sun8i_ce_hash_run(struct crypto_engine *engine, void *async_req) +{ + struct ahash_request *areq = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct sun8i_ce_dev *ce = ctx->ce; + struct sun8i_ce_flow *chan; + int err; + + chan = &ce->chanlist[rctx->flow]; + + err = sun8i_ce_hash_prepare(areq, chan->tl); + if (err) + return err; + + err = sun8i_ce_run_task(ce, rctx->flow, crypto_ahash_alg_name(tfm)); + + sun8i_ce_hash_unprepare(areq, chan->tl); + + if (!err) + memcpy(areq->result, rctx->result, + crypto_ahash_digestsize(tfm)); - memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize); -theend: - kfree(buf); - kfree(result); local_bh_disable(); - crypto_finalize_hash_request(engine, breq, err); + crypto_finalize_hash_request(engine, async_req, err); local_bh_enable(); + return 0; } diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c index 762459867b6c..d0a1ac66738b 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c @@ -137,7 +137,6 @@ int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst); cet->t_dst[0].len = cpu_to_le32(todo / 4); - ce->chanlist[flow].timeout = 2000; err = sun8i_ce_run_task(ce, 3, "PRNG"); mutex_unlock(&ce->rnglock); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c index e1e8bc15202e..244529bf0616 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -79,7 +79,6 @@ static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wa cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst); cet->t_dst[0].len = cpu_to_le32(todo / 4); - ce->chanlist[flow].timeout = todo; err = sun8i_ce_run_task(ce, 3, "TRNG"); mutex_unlock(&ce->rnglock); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 3b5c2af013d0..71f5a0cd3d45 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -106,9 +106,13 @@ #define MAX_SG 8 #define CE_MAX_CLOCKS 4 +#define CE_DMA_TIMEOUT_MS 3000 #define MAXFLOW 4 +#define CE_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE +#define CE_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE + /* * struct ce_clock - Describe clocks used by sun8i-ce * @name: Name of clock needed by this variant @@ -187,8 +191,6 @@ struct ce_task { * @status: set to 1 by interrupt if task is done * @t_phy: Physical address of task * @tl: pointer to the current ce_task for this flow - * @backup_iv: buffer which contain the next IV to store - * @bounce_iv: buffer which contain the IV * @stat_req: number of request done by this flow */ struct sun8i_ce_flow { @@ -196,10 +198,7 @@ struct sun8i_ce_flow { struct completion complete; int status; dma_addr_t t_phy; - int timeout; struct ce_task *tl; - void *backup_iv; - void *bounce_iv; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG unsigned long stat_req; #endif @@ -260,21 +259,23 @@ static inline __le32 desc_addr_val_le32(struct sun8i_ce_dev *dev, * struct sun8i_cipher_req_ctx - context for a skcipher request * @op_dir: direction (encrypt vs decrypt) for this request * @flow: the flow to use for this request - * @ivlen: size of bounce_iv * @nr_sgs: The number of source SG (as given by dma_map_sg()) * @nr_sgd: The number of destination SG (as given by dma_map_sg()) * @addr_iv: The IV addr returned by dma_map_single, need to unmap later * @addr_key: The key addr returned by dma_map_single, need to unmap later + * @bounce_iv: Current IV buffer + * @backup_iv: Next IV buffer * @fallback_req: request struct for invoking the fallback skcipher TFM */ struct sun8i_cipher_req_ctx { u32 op_dir; int flow; - unsigned int ivlen; int nr_sgs; int nr_sgd; dma_addr_t addr_iv; dma_addr_t addr_key; + u8 bounce_iv[AES_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 backup_iv[AES_BLOCK_SIZE]; struct skcipher_request fallback_req; // keep at the end }; @@ -306,10 +307,24 @@ struct sun8i_ce_hash_tfm_ctx { * struct sun8i_ce_hash_reqctx - context for an ahash request * @fallback_req: pre-allocated fallback request * @flow: the flow to use for this request + * @nr_sgs: number of entries in the source scatterlist + * @result_len: result length in bytes + * @pad_len: padding length in bytes + * @addr_res: DMA address of the result buffer, returned by dma_map_single() + * @addr_pad: DMA address of the padding buffer, returned by dma_map_single() + * @result: per-request result buffer + * @pad: per-request padding buffer (up to 2 blocks) */ struct sun8i_ce_hash_reqctx { - struct ahash_request fallback_req; int flow; + int nr_sgs; + size_t result_len; + size_t pad_len; + dma_addr_t addr_res; + dma_addr_t addr_pad; + u8 result[CE_MAX_HASH_DIGEST_SIZE] __aligned(CRYPTO_DMA_ALIGN); + u8 pad[2 * CE_MAX_HASH_BLOCK_SIZE]; + struct ahash_request fallback_req; // keep at the end }; /* diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 9b9605ce8ee6..8831bcb230c2 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -141,7 +141,7 @@ static int sun8i_ss_setup_ivs(struct skcipher_request *areq) /* we need to copy all IVs from source in case DMA is bi-directionnal */ while (sg && len) { - if (sg_dma_len(sg) == 0) { + if (sg->length == 0) { sg = sg_next(sg); continue; } diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 753f67a36dc5..36a1ebca2e70 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -150,7 +150,9 @@ int sun8i_ss_hash_init(struct ahash_request *areq) memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx)); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -162,7 +164,9 @@ int sun8i_ss_hash_export(struct ahash_request *areq, void *out) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } @@ -174,7 +178,9 @@ int sun8i_ss_hash_import(struct ahash_request *areq, const void *in) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -186,9 +192,10 @@ int sun8i_ss_hash_final(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); @@ -212,10 +219,10 @@ int sun8i_ss_hash_update(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -227,12 +234,11 @@ int sun8i_ss_hash_finup(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); @@ -256,12 +262,11 @@ static int sun8i_ss_hash_digest_fb(struct ahash_request *areq) struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { struct ahash_alg *alg = crypto_ahash_alg(tfm); @@ -497,6 +502,7 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); ss = algt->ss; + j = 0; digestsize = crypto_ahash_digestsize(tfm); if (digestsize == SHA224_DIGEST_SIZE) @@ -531,7 +537,6 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) goto err_dma_result; } - j = 0; len = areq->nbytes; sg = areq->src; i = 0; diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index e0af611a95d8..38e8a61e9166 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -12,9 +12,6 @@ #include <linux/interrupt.h> #include <linux/spinlock_types.h> #include <linux/scatterlist.h> -#include <linux/crypto.h> -#include <linux/hash.h> -#include <crypto/internal/hash.h> #include <linux/dma-mapping.h> #include <crypto/algapi.h> #include <crypto/aead.h> @@ -72,7 +69,7 @@ static inline int crypto4xx_crypt(struct skcipher_request *req, { struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); - __le32 iv[AES_IV_SIZE]; + __le32 iv[AES_IV_SIZE / 4]; if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) return -EINVAL; @@ -429,7 +426,7 @@ static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt) struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); - __le32 iv[16]; + __le32 iv[4]; u32 tmp_sa[SA_AES128_CCM_LEN + 4]; struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa; unsigned int len = req->cryptlen; @@ -602,106 +599,3 @@ int crypto4xx_decrypt_aes_gcm(struct aead_request *req) { return crypto4xx_crypt_aes_gcm(req, true); } - -/* - * HASH SHA1 Functions - */ -static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, - unsigned int sa_len, - unsigned char ha, - unsigned char hm) -{ - struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *my_alg; - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - struct dynamic_sa_hash160 *sa; - int rc; - - my_alg = container_of(__crypto_ahash_alg(alg), struct crypto4xx_alg, - alg.u.hash); - ctx->dev = my_alg->dev; - - /* Create SA */ - if (ctx->sa_in || ctx->sa_out) - crypto4xx_free_sa(ctx); - - rc = crypto4xx_alloc_sa(ctx, sa_len); - if (rc) - return rc; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct crypto4xx_ctx)); - sa = (struct dynamic_sa_hash160 *)ctx->sa_in; - set_dynamic_sa_command_0(&sa->ctrl, SA_SAVE_HASH, SA_NOT_SAVE_IV, - SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA, - SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL, - SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, - SA_OPCODE_HASH, DIR_INBOUND); - set_dynamic_sa_command_1(&sa->ctrl, 0, SA_HASH_MODE_HASH, - CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, - SA_SEQ_MASK_OFF, SA_MC_ENABLE, - SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, - SA_NOT_COPY_HDR); - /* Need to zero hash digest in SA */ - memset(sa->inner_digest, 0, sizeof(sa->inner_digest)); - memset(sa->outer_digest, 0, sizeof(sa->outer_digest)); - - return 0; -} - -int crypto4xx_hash_init(struct ahash_request *req) -{ - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - int ds; - struct dynamic_sa_ctl *sa; - - sa = ctx->sa_in; - ds = crypto_ahash_digestsize( - __crypto_ahash_cast(req->base.tfm)); - sa->sa_command_0.bf.digest_len = ds >> 2; - sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA; - - return 0; -} - -int crypto4xx_hash_update(struct ahash_request *req) -{ - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct scatterlist dst; - unsigned int ds = crypto_ahash_digestsize(ahash); - - sg_init_one(&dst, req->result, ds); - - return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, - req->nbytes, NULL, 0, ctx->sa_in, - ctx->sa_len, 0, NULL); -} - -int crypto4xx_hash_final(struct ahash_request *req) -{ - return 0; -} - -int crypto4xx_hash_digest(struct ahash_request *req) -{ - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct scatterlist dst; - unsigned int ds = crypto_ahash_digestsize(ahash); - - sg_init_one(&dst, req->result, ds); - - return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, - req->nbytes, NULL, 0, ctx->sa_in, - ctx->sa_len, 0, NULL); -} - -/* - * SHA1 Algorithm - */ -int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm) -{ - return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1, - SA_HASH_MODE_HASH); -} diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index ec3ccfa60445..8cdc66d520c9 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -485,18 +485,6 @@ static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, } } -static void crypto4xx_copy_digest_to_dst(void *dst, - struct pd_uinfo *pd_uinfo, - struct crypto4xx_ctx *ctx) -{ - struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; - - if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { - memcpy(dst, pd_uinfo->sr_va->save_digest, - SA_HASH_ALG_SHA1_DIGEST_SIZE); - } -} - static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo) { @@ -549,23 +537,6 @@ static void crypto4xx_cipher_done(struct crypto4xx_device *dev, skcipher_request_complete(req, 0); } -static void crypto4xx_ahash_done(struct crypto4xx_device *dev, - struct pd_uinfo *pd_uinfo) -{ - struct crypto4xx_ctx *ctx; - struct ahash_request *ahash_req; - - ahash_req = ahash_request_cast(pd_uinfo->async_req); - ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(ahash_req)); - - crypto4xx_copy_digest_to_dst(ahash_req->result, pd_uinfo, ctx); - crypto4xx_ret_sg_desc(dev, pd_uinfo); - - if (pd_uinfo->state & PD_ENTRY_BUSY) - ahash_request_complete(ahash_req, -EINPROGRESS); - ahash_request_complete(ahash_req, 0); -} - static void crypto4xx_aead_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo, struct ce_pd *pd) @@ -642,9 +613,6 @@ static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) case CRYPTO_ALG_TYPE_AEAD: crypto4xx_aead_done(dev, pd_uinfo, pd); break; - case CRYPTO_ALG_TYPE_AHASH: - crypto4xx_ahash_done(dev, pd_uinfo); - break; } } @@ -676,7 +644,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct scatterlist *src, struct scatterlist *dst, const unsigned int datalen, - const __le32 *iv, const u32 iv_len, + const void *iv, const u32 iv_len, const struct dynamic_sa_ctl *req_sa, const unsigned int sa_len, const unsigned int assoclen, @@ -912,8 +880,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, } pd->pd_ctl.w = PD_CTL_HOST_READY | - ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) || - (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? PD_CTL_HASH_FINAL : 0); pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); @@ -1019,10 +986,6 @@ static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, rc = crypto_register_aead(&alg->alg.u.aead); break; - case CRYPTO_ALG_TYPE_AHASH: - rc = crypto_register_ahash(&alg->alg.u.hash); - break; - case CRYPTO_ALG_TYPE_RNG: rc = crypto_register_rng(&alg->alg.u.rng); break; @@ -1048,10 +1011,6 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) list_for_each_entry_safe(alg, tmp, &sec_dev->alg_list, entry) { list_del(&alg->entry); switch (alg->alg.type) { - case CRYPTO_ALG_TYPE_AHASH: - crypto_unregister_ahash(&alg->alg.u.hash); - break; - case CRYPTO_ALG_TYPE_AEAD: crypto_unregister_aead(&alg->alg.u.aead); break; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 3adcc5e65694..ee36630c670f 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -16,7 +16,6 @@ #include <linux/ratelimit.h> #include <linux/mutex.h> #include <linux/scatterlist.h> -#include <crypto/internal/hash.h> #include <crypto/internal/aead.h> #include <crypto/internal/rng.h> #include <crypto/internal/skcipher.h> @@ -135,7 +134,6 @@ struct crypto4xx_alg_common { u32 type; union { struct skcipher_alg cipher; - struct ahash_alg hash; struct aead_alg aead; struct rng_alg rng; } u; @@ -147,6 +145,12 @@ struct crypto4xx_alg { struct crypto4xx_device *dev; }; +#if IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION >= 120000 +#define BUILD_PD_ACCESS __attribute__((access(read_only, 6, 7))) +#else +#define BUILD_PD_ACCESS +#endif + int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); int crypto4xx_build_pd(struct crypto_async_request *req, @@ -154,11 +158,11 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct scatterlist *src, struct scatterlist *dst, const unsigned int datalen, - const __le32 *iv, const u32 iv_len, + const void *iv, const u32 iv_len, const struct dynamic_sa_ctl *sa, const unsigned int sa_len, const unsigned int assoclen, - struct scatterlist *dst_tmp); + struct scatterlist *dst_tmp) BUILD_PD_ACCESS; int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher, @@ -177,11 +181,6 @@ int crypto4xx_encrypt_noiv_block(struct skcipher_request *req); int crypto4xx_decrypt_noiv_block(struct skcipher_request *req); int crypto4xx_rfc3686_encrypt(struct skcipher_request *req); int crypto4xx_rfc3686_decrypt(struct skcipher_request *req); -int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); -int crypto4xx_hash_digest(struct ahash_request *req); -int crypto4xx_hash_final(struct ahash_request *req); -int crypto4xx_hash_update(struct ahash_request *req); -int crypto4xx_hash_init(struct ahash_request *req); /* * Note: Only use this function to copy items that is word aligned. diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c index 8d1c79aaca07..5993bcba9716 100644 --- a/drivers/crypto/aspeed/aspeed-acry.c +++ b/drivers/crypto/aspeed/aspeed-acry.c @@ -787,7 +787,6 @@ static int aspeed_acry_probe(struct platform_device *pdev) err_engine_rsa_start: crypto_engine_exit(acry_dev->crypt_engine_rsa); clk_exit: - clk_disable_unprepare(acry_dev->clk); return rc; } @@ -799,7 +798,6 @@ static void aspeed_acry_remove(struct platform_device *pdev) aspeed_acry_unregister(acry_dev); crypto_engine_exit(acry_dev->crypt_engine_rsa); tasklet_kill(&acry_dev->done_task); - clk_disable_unprepare(acry_dev->clk); } MODULE_DEVICE_TABLE(of, aspeed_acry_of_matches); diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c index a72dfebc53ff..fa201dae1f81 100644 --- a/drivers/crypto/aspeed/aspeed-hace-crypto.c +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -346,7 +346,7 @@ free_req: } else { dma_unmap_sg(hace_dev->dev, req->dst, rctx->dst_nents, - DMA_TO_DEVICE); + DMA_FROM_DEVICE); dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); } diff --git a/drivers/crypto/aspeed/aspeed-hace-hash.c b/drivers/crypto/aspeed/aspeed-hace-hash.c index 0b6e49c06eff..f8f37c9d5f3c 100644 --- a/drivers/crypto/aspeed/aspeed-hace-hash.c +++ b/drivers/crypto/aspeed/aspeed-hace-hash.c @@ -5,7 +5,6 @@ #include "aspeed-hace.h" #include <crypto/engine.h> -#include <crypto/hmac.h> #include <crypto/internal/hash.h> #include <crypto/scatterwalk.h> #include <crypto/sha1.h> @@ -14,6 +13,7 @@ #include <linux/err.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/scatterlist.h> #include <linux/string.h> #ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG @@ -59,6 +59,46 @@ static const __be64 sha512_iv[8] = { cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7) }; +static int aspeed_sham_init(struct ahash_request *req); +static int aspeed_ahash_req_update(struct aspeed_hace_dev *hace_dev); + +static int aspeed_sham_export(struct ahash_request *req, void *out) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + union { + u8 *u8; + u64 *u64; + } p = { .u8 = out }; + + memcpy(out, rctx->digest, rctx->ivsize); + p.u8 += rctx->ivsize; + put_unaligned(rctx->digcnt[0], p.u64++); + if (rctx->ivsize == 64) + put_unaligned(rctx->digcnt[1], p.u64); + return 0; +} + +static int aspeed_sham_import(struct ahash_request *req, const void *in) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + union { + const u8 *u8; + const u64 *u64; + } p = { .u8 = in }; + int err; + + err = aspeed_sham_init(req); + if (err) + return err; + + memcpy(rctx->digest, in, rctx->ivsize); + p.u8 += rctx->ivsize; + rctx->digcnt[0] = get_unaligned(p.u64++); + if (rctx->ivsize == 64) + rctx->digcnt[1] = get_unaligned(p.u64); + return 0; +} + /* The purpose of this padding is to ensure that the padded message is a * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). * The bit "1" is appended at the end of the message followed by @@ -74,10 +114,10 @@ static const __be64 sha512_iv[8] = { * - if message length < 112 bytes then padlen = 112 - message length * - else padlen = 128 + 112 - message length */ -static void aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev, - struct aspeed_sham_reqctx *rctx) +static int aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev, + struct aspeed_sham_reqctx *rctx, u8 *buf) { - unsigned int index, padlen; + unsigned int index, padlen, bitslen; __be64 bits[2]; AHASH_DBG(hace_dev, "rctx flags:0x%x\n", (u32)rctx->flags); @@ -87,25 +127,32 @@ static void aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev, case SHA_FLAGS_SHA224: case SHA_FLAGS_SHA256: bits[0] = cpu_to_be64(rctx->digcnt[0] << 3); - index = rctx->bufcnt & 0x3f; + index = rctx->digcnt[0] & 0x3f; padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); - *(rctx->buffer + rctx->bufcnt) = 0x80; - memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1); - memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 8); - rctx->bufcnt += padlen + 8; + bitslen = 8; break; default: bits[1] = cpu_to_be64(rctx->digcnt[0] << 3); bits[0] = cpu_to_be64(rctx->digcnt[1] << 3 | rctx->digcnt[0] >> 61); - index = rctx->bufcnt & 0x7f; + index = rctx->digcnt[0] & 0x7f; padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); - *(rctx->buffer + rctx->bufcnt) = 0x80; - memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1); - memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 16); - rctx->bufcnt += padlen + 16; + bitslen = 16; break; } + buf[0] = 0x80; + memset(buf + 1, 0, padlen - 1); + memcpy(buf + padlen, bits, bitslen); + return padlen + bitslen; +} + +static void aspeed_ahash_update_counter(struct aspeed_sham_reqctx *rctx, + unsigned int len) +{ + rctx->offset += len; + rctx->digcnt[0] += len; + if (rctx->digcnt[0] < len) + rctx->digcnt[1]++; } /* @@ -117,31 +164,31 @@ static int aspeed_ahash_dma_prepare(struct aspeed_hace_dev *hace_dev) struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; struct ahash_request *req = hash_engine->req; struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - int length, remain; + unsigned int length, remain; + bool final = false; - length = rctx->total + rctx->bufcnt; - remain = length % rctx->block_size; + length = rctx->total - rctx->offset; + remain = length - round_down(length, rctx->block_size); AHASH_DBG(hace_dev, "length:0x%x, remain:0x%x\n", length, remain); - if (rctx->bufcnt) - memcpy(hash_engine->ahash_src_addr, rctx->buffer, rctx->bufcnt); - - if (rctx->total + rctx->bufcnt < ASPEED_CRYPTO_SRC_DMA_BUF_LEN) { - scatterwalk_map_and_copy(hash_engine->ahash_src_addr + - rctx->bufcnt, rctx->src_sg, - rctx->offset, rctx->total - remain, 0); - rctx->offset += rctx->total - remain; - - } else { - dev_warn(hace_dev->dev, "Hash data length is too large\n"); - return -EINVAL; - } - - scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg, - rctx->offset, remain, 0); + if (length > ASPEED_HASH_SRC_DMA_BUF_LEN) + length = ASPEED_HASH_SRC_DMA_BUF_LEN; + else if (rctx->flags & SHA_FLAGS_FINUP) { + if (round_up(length, rctx->block_size) + rctx->block_size > + ASPEED_CRYPTO_SRC_DMA_BUF_LEN) + length = round_down(length - 1, rctx->block_size); + else + final = true; + } else + length -= remain; + scatterwalk_map_and_copy(hash_engine->ahash_src_addr, rctx->src_sg, + rctx->offset, length, 0); + aspeed_ahash_update_counter(rctx, length); + if (final) + length += aspeed_ahash_fill_padding( + hace_dev, rctx, hash_engine->ahash_src_addr + length); - rctx->bufcnt = remain; rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); @@ -150,7 +197,7 @@ static int aspeed_ahash_dma_prepare(struct aspeed_hace_dev *hace_dev) return -ENOMEM; } - hash_engine->src_length = length - remain; + hash_engine->src_length = length; hash_engine->src_dma = hash_engine->ahash_src_dma_addr; hash_engine->digest_dma = rctx->digest_dma_addr; @@ -166,16 +213,20 @@ static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; struct ahash_request *req = hash_engine->req; struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + bool final = rctx->flags & SHA_FLAGS_FINUP; + int remain, sg_len, i, max_sg_nents; + unsigned int length, offset, total; struct aspeed_sg_list *src_list; struct scatterlist *s; - int length, remain, sg_len, i; int rc = 0; - remain = (rctx->total + rctx->bufcnt) % rctx->block_size; - length = rctx->total + rctx->bufcnt - remain; + offset = rctx->offset; + length = rctx->total - offset; + remain = final ? 0 : length - round_down(length, rctx->block_size); + length -= remain; - AHASH_DBG(hace_dev, "%s:0x%x, %s:%zu, %s:0x%x, %s:0x%x\n", - "rctx total", rctx->total, "bufcnt", rctx->bufcnt, + AHASH_DBG(hace_dev, "%s:0x%x, %s:0x%x, %s:0x%x\n", + "rctx total", rctx->total, "length", length, "remain", remain); sg_len = dma_map_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, @@ -186,6 +237,8 @@ static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) goto end; } + max_sg_nents = ASPEED_HASH_SRC_DMA_BUF_LEN / sizeof(*src_list) - final; + sg_len = min(sg_len, max_sg_nents); src_list = (struct aspeed_sg_list *)hash_engine->ahash_src_addr; rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, SHA512_DIGEST_SIZE, @@ -196,68 +249,66 @@ static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) goto free_src_sg; } - if (rctx->bufcnt != 0) { - u32 phy_addr; - u32 len; + total = 0; + for_each_sg(rctx->src_sg, s, sg_len, i) { + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); - rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, - rctx->buffer, - rctx->block_size * 2, - DMA_TO_DEVICE); - if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { - dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); - rc = -ENOMEM; - goto free_rctx_digest; + if (len <= offset) { + offset -= len; + continue; } - phy_addr = rctx->buffer_dma_addr; - len = rctx->bufcnt; - length -= len; + len -= offset; + phy_addr += offset; + offset = 0; - /* Last sg list */ - if (length == 0) - len |= HASH_SG_LAST_LIST; + if (length > len) + length -= len; + else { + /* Last sg list */ + len = length; + length = 0; + } - src_list[0].phy_addr = cpu_to_le32(phy_addr); - src_list[0].len = cpu_to_le32(len); - src_list++; + total += len; + src_list[i].phy_addr = cpu_to_le32(phy_addr); + src_list[i].len = cpu_to_le32(len); } if (length != 0) { - for_each_sg(rctx->src_sg, s, sg_len, i) { - u32 phy_addr = sg_dma_address(s); - u32 len = sg_dma_len(s); - - if (length > len) - length -= len; - else { - /* Last sg list */ - len = length; - len |= HASH_SG_LAST_LIST; - length = 0; - } + total = round_down(total, rctx->block_size); + final = false; + } + + aspeed_ahash_update_counter(rctx, total); + if (final) { + int len = aspeed_ahash_fill_padding(hace_dev, rctx, + rctx->buffer); - src_list[i].phy_addr = cpu_to_le32(phy_addr); - src_list[i].len = cpu_to_le32(len); + total += len; + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, + rctx->buffer, + sizeof(rctx->buffer), + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; } - } - if (length != 0) { - rc = -EINVAL; - goto free_rctx_buffer; + src_list[i].phy_addr = cpu_to_le32(rctx->buffer_dma_addr); + src_list[i].len = cpu_to_le32(len); + i++; } + src_list[i - 1].len |= cpu_to_le32(HASH_SG_LAST_LIST); - rctx->offset = rctx->total - remain; - hash_engine->src_length = rctx->total + rctx->bufcnt - remain; + hash_engine->src_length = total; hash_engine->src_dma = hash_engine->ahash_src_dma_addr; hash_engine->digest_dma = rctx->digest_dma_addr; return 0; -free_rctx_buffer: - if (rctx->bufcnt != 0) - dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, - rctx->block_size * 2, DMA_TO_DEVICE); free_rctx_digest: dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); @@ -272,24 +323,6 @@ static int aspeed_ahash_complete(struct aspeed_hace_dev *hace_dev) { struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; struct ahash_request *req = hash_engine->req; - - AHASH_DBG(hace_dev, "\n"); - - hash_engine->flags &= ~CRYPTO_FLAGS_BUSY; - - crypto_finalize_hash_request(hace_dev->crypt_engine_hash, req, 0); - - return 0; -} - -/* - * Copy digest to the corresponding request result. - * This function will be called at final() stage. - */ -static int aspeed_ahash_transfer(struct aspeed_hace_dev *hace_dev) -{ - struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; - struct ahash_request *req = hash_engine->req; struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); AHASH_DBG(hace_dev, "\n"); @@ -297,12 +330,19 @@ static int aspeed_ahash_transfer(struct aspeed_hace_dev *hace_dev) dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); - dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, - rctx->block_size * 2, DMA_TO_DEVICE); + if (rctx->total - rctx->offset >= rctx->block_size || + (rctx->total != rctx->offset && rctx->flags & SHA_FLAGS_FINUP)) + return aspeed_ahash_req_update(hace_dev); - memcpy(req->result, rctx->digest, rctx->digsize); + hash_engine->flags &= ~CRYPTO_FLAGS_BUSY; - return aspeed_ahash_complete(hace_dev); + if (rctx->flags & SHA_FLAGS_FINUP) + memcpy(req->result, rctx->digest, rctx->digsize); + + crypto_finalize_hash_request(hace_dev->crypt_engine_hash, req, + rctx->total - rctx->offset); + + return 0; } /* @@ -338,118 +378,6 @@ static int aspeed_hace_ahash_trigger(struct aspeed_hace_dev *hace_dev, return -EINPROGRESS; } -/* - * HMAC resume aims to do the second pass produces - * the final HMAC code derived from the inner hash - * result and the outer key. - */ -static int aspeed_ahash_hmac_resume(struct aspeed_hace_dev *hace_dev) -{ - struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; - struct ahash_request *req = hash_engine->req; - struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); - struct aspeed_sha_hmac_ctx *bctx = tctx->base; - int rc = 0; - - AHASH_DBG(hace_dev, "\n"); - - dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, - SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); - - dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, - rctx->block_size * 2, DMA_TO_DEVICE); - - /* o key pad + hash sum 1 */ - memcpy(rctx->buffer, bctx->opad, rctx->block_size); - memcpy(rctx->buffer + rctx->block_size, rctx->digest, rctx->digsize); - - rctx->bufcnt = rctx->block_size + rctx->digsize; - rctx->digcnt[0] = rctx->block_size + rctx->digsize; - - aspeed_ahash_fill_padding(hace_dev, rctx); - memcpy(rctx->digest, rctx->sha_iv, rctx->ivsize); - - rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, - SHA512_DIGEST_SIZE, - DMA_BIDIRECTIONAL); - if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { - dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); - rc = -ENOMEM; - goto end; - } - - rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, rctx->buffer, - rctx->block_size * 2, - DMA_TO_DEVICE); - if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { - dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); - rc = -ENOMEM; - goto free_rctx_digest; - } - - hash_engine->src_dma = rctx->buffer_dma_addr; - hash_engine->src_length = rctx->bufcnt; - hash_engine->digest_dma = rctx->digest_dma_addr; - - return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer); - -free_rctx_digest: - dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, - SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); -end: - return rc; -} - -static int aspeed_ahash_req_final(struct aspeed_hace_dev *hace_dev) -{ - struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; - struct ahash_request *req = hash_engine->req; - struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - int rc = 0; - - AHASH_DBG(hace_dev, "\n"); - - aspeed_ahash_fill_padding(hace_dev, rctx); - - rctx->digest_dma_addr = dma_map_single(hace_dev->dev, - rctx->digest, - SHA512_DIGEST_SIZE, - DMA_BIDIRECTIONAL); - if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { - dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); - rc = -ENOMEM; - goto end; - } - - rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, - rctx->buffer, - rctx->block_size * 2, - DMA_TO_DEVICE); - if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { - dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); - rc = -ENOMEM; - goto free_rctx_digest; - } - - hash_engine->src_dma = rctx->buffer_dma_addr; - hash_engine->src_length = rctx->bufcnt; - hash_engine->digest_dma = rctx->digest_dma_addr; - - if (rctx->flags & SHA_FLAGS_HMAC) - return aspeed_hace_ahash_trigger(hace_dev, - aspeed_ahash_hmac_resume); - - return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer); - -free_rctx_digest: - dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, - SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); -end: - return rc; -} - static int aspeed_ahash_update_resume_sg(struct aspeed_hace_dev *hace_dev) { struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; @@ -461,40 +389,12 @@ static int aspeed_ahash_update_resume_sg(struct aspeed_hace_dev *hace_dev) dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, DMA_TO_DEVICE); - if (rctx->bufcnt != 0) + if (rctx->flags & SHA_FLAGS_FINUP && rctx->total == rctx->offset) dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, - rctx->block_size * 2, - DMA_TO_DEVICE); + sizeof(rctx->buffer), DMA_TO_DEVICE); - dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, - SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); - - scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg, rctx->offset, - rctx->total - rctx->offset, 0); - - rctx->bufcnt = rctx->total - rctx->offset; rctx->cmd &= ~HASH_CMD_HASH_SRC_SG_CTRL; - if (rctx->flags & SHA_FLAGS_FINUP) - return aspeed_ahash_req_final(hace_dev); - - return aspeed_ahash_complete(hace_dev); -} - -static int aspeed_ahash_update_resume(struct aspeed_hace_dev *hace_dev) -{ - struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; - struct ahash_request *req = hash_engine->req; - struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - - AHASH_DBG(hace_dev, "\n"); - - dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, - SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); - - if (rctx->flags & SHA_FLAGS_FINUP) - return aspeed_ahash_req_final(hace_dev); - return aspeed_ahash_complete(hace_dev); } @@ -513,7 +413,7 @@ static int aspeed_ahash_req_update(struct aspeed_hace_dev *hace_dev) resume = aspeed_ahash_update_resume_sg; } else { - resume = aspeed_ahash_update_resume; + resume = aspeed_ahash_complete; } ret = hash_engine->dma_prepare(hace_dev); @@ -530,26 +430,47 @@ static int aspeed_hace_hash_handle_queue(struct aspeed_hace_dev *hace_dev, hace_dev->crypt_engine_hash, req); } +static noinline int aspeed_ahash_fallback(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + HASH_FBREQ_ON_STACK(fbreq, req); + u8 *state = rctx->buffer; + struct scatterlist sg[2]; + struct scatterlist *ssg; + int ret; + + ssg = scatterwalk_ffwd(sg, req->src, rctx->offset); + ahash_request_set_crypt(fbreq, ssg, req->result, + rctx->total - rctx->offset); + + ret = aspeed_sham_export(req, state) ?: + crypto_ahash_import_core(fbreq, state); + + if (rctx->flags & SHA_FLAGS_FINUP) + ret = ret ?: crypto_ahash_finup(fbreq); + else + ret = ret ?: crypto_ahash_update(fbreq) ?: + crypto_ahash_export_core(fbreq, state) ?: + aspeed_sham_import(req, state); + HASH_REQUEST_ZERO(fbreq); + return ret; +} + static int aspeed_ahash_do_request(struct crypto_engine *engine, void *areq) { struct ahash_request *req = ahash_request_cast(areq); - struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); struct aspeed_hace_dev *hace_dev = tctx->hace_dev; struct aspeed_engine_hash *hash_engine; - int ret = 0; + int ret; hash_engine = &hace_dev->hash_engine; hash_engine->flags |= CRYPTO_FLAGS_BUSY; - if (rctx->op == SHA_OP_UPDATE) - ret = aspeed_ahash_req_update(hace_dev); - else if (rctx->op == SHA_OP_FINAL) - ret = aspeed_ahash_req_final(hace_dev); - + ret = aspeed_ahash_req_update(hace_dev); if (ret != -EINPROGRESS) - return ret; + return aspeed_ahash_fallback(req); return 0; } @@ -590,45 +511,7 @@ static int aspeed_sham_update(struct ahash_request *req) rctx->total = req->nbytes; rctx->src_sg = req->src; rctx->offset = 0; - rctx->src_nents = sg_nents(req->src); - rctx->op = SHA_OP_UPDATE; - - rctx->digcnt[0] += rctx->total; - if (rctx->digcnt[0] < rctx->total) - rctx->digcnt[1]++; - - if (rctx->bufcnt + rctx->total < rctx->block_size) { - scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, - rctx->src_sg, rctx->offset, - rctx->total, 0); - rctx->bufcnt += rctx->total; - - return 0; - } - - return aspeed_hace_hash_handle_queue(hace_dev, req); -} - -static int aspeed_sham_shash_digest(struct crypto_shash *tfm, u32 flags, - const u8 *data, unsigned int len, u8 *out) -{ - SHASH_DESC_ON_STACK(shash, tfm); - - shash->tfm = tfm; - - return crypto_shash_digest(shash, data, len, out); -} - -static int aspeed_sham_final(struct ahash_request *req) -{ - struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); - struct aspeed_hace_dev *hace_dev = tctx->hace_dev; - - AHASH_DBG(hace_dev, "req->nbytes:%d, rctx->total:%d\n", - req->nbytes, rctx->total); - rctx->op = SHA_OP_FINAL; + rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); return aspeed_hace_hash_handle_queue(hace_dev, req); } @@ -639,23 +522,12 @@ static int aspeed_sham_finup(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); struct aspeed_hace_dev *hace_dev = tctx->hace_dev; - int rc1, rc2; AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes); rctx->flags |= SHA_FLAGS_FINUP; - rc1 = aspeed_sham_update(req); - if (rc1 == -EINPROGRESS || rc1 == -EBUSY) - return rc1; - - /* - * final() has to be always called to cleanup resources - * even if update() failed, except EINPROGRESS - */ - rc2 = aspeed_sham_final(req); - - return rc1 ? : rc2; + return aspeed_sham_update(req); } static int aspeed_sham_init(struct ahash_request *req) @@ -664,7 +536,6 @@ static int aspeed_sham_init(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); struct aspeed_hace_dev *hace_dev = tctx->hace_dev; - struct aspeed_sha_hmac_ctx *bctx = tctx->base; AHASH_DBG(hace_dev, "%s: digest size:%d\n", crypto_tfm_alg_name(&tfm->base), @@ -679,7 +550,6 @@ static int aspeed_sham_init(struct ahash_request *req) rctx->flags |= SHA_FLAGS_SHA1; rctx->digsize = SHA1_DIGEST_SIZE; rctx->block_size = SHA1_BLOCK_SIZE; - rctx->sha_iv = sha1_iv; rctx->ivsize = 32; memcpy(rctx->digest, sha1_iv, rctx->ivsize); break; @@ -688,7 +558,6 @@ static int aspeed_sham_init(struct ahash_request *req) rctx->flags |= SHA_FLAGS_SHA224; rctx->digsize = SHA224_DIGEST_SIZE; rctx->block_size = SHA224_BLOCK_SIZE; - rctx->sha_iv = sha224_iv; rctx->ivsize = 32; memcpy(rctx->digest, sha224_iv, rctx->ivsize); break; @@ -697,7 +566,6 @@ static int aspeed_sham_init(struct ahash_request *req) rctx->flags |= SHA_FLAGS_SHA256; rctx->digsize = SHA256_DIGEST_SIZE; rctx->block_size = SHA256_BLOCK_SIZE; - rctx->sha_iv = sha256_iv; rctx->ivsize = 32; memcpy(rctx->digest, sha256_iv, rctx->ivsize); break; @@ -707,7 +575,6 @@ static int aspeed_sham_init(struct ahash_request *req) rctx->flags |= SHA_FLAGS_SHA384; rctx->digsize = SHA384_DIGEST_SIZE; rctx->block_size = SHA384_BLOCK_SIZE; - rctx->sha_iv = (const __be32 *)sha384_iv; rctx->ivsize = 64; memcpy(rctx->digest, sha384_iv, rctx->ivsize); break; @@ -717,7 +584,6 @@ static int aspeed_sham_init(struct ahash_request *req) rctx->flags |= SHA_FLAGS_SHA512; rctx->digsize = SHA512_DIGEST_SIZE; rctx->block_size = SHA512_BLOCK_SIZE; - rctx->sha_iv = (const __be32 *)sha512_iv; rctx->ivsize = 64; memcpy(rctx->digest, sha512_iv, rctx->ivsize); break; @@ -727,19 +593,10 @@ static int aspeed_sham_init(struct ahash_request *req) return -EINVAL; } - rctx->bufcnt = 0; rctx->total = 0; rctx->digcnt[0] = 0; rctx->digcnt[1] = 0; - /* HMAC init */ - if (tctx->flags & SHA_FLAGS_HMAC) { - rctx->digcnt[0] = rctx->block_size; - rctx->bufcnt = rctx->block_size; - memcpy(rctx->buffer, bctx->ipad, rctx->block_size); - rctx->flags |= SHA_FLAGS_HMAC; - } - return 0; } @@ -748,102 +605,14 @@ static int aspeed_sham_digest(struct ahash_request *req) return aspeed_sham_init(req) ? : aspeed_sham_finup(req); } -static int aspeed_sham_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) +static int aspeed_sham_cra_init(struct crypto_ahash *tfm) { + struct ahash_alg *alg = crypto_ahash_alg(tfm); struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); - struct aspeed_hace_dev *hace_dev = tctx->hace_dev; - struct aspeed_sha_hmac_ctx *bctx = tctx->base; - int ds = crypto_shash_digestsize(bctx->shash); - int bs = crypto_shash_blocksize(bctx->shash); - int err = 0; - int i; - - AHASH_DBG(hace_dev, "%s: keylen:%d\n", crypto_tfm_alg_name(&tfm->base), - keylen); - - if (keylen > bs) { - err = aspeed_sham_shash_digest(bctx->shash, - crypto_shash_get_flags(bctx->shash), - key, keylen, bctx->ipad); - if (err) - return err; - keylen = ds; - - } else { - memcpy(bctx->ipad, key, keylen); - } - - memset(bctx->ipad + keylen, 0, bs - keylen); - memcpy(bctx->opad, bctx->ipad, bs); - - for (i = 0; i < bs; i++) { - bctx->ipad[i] ^= HMAC_IPAD_VALUE; - bctx->opad[i] ^= HMAC_OPAD_VALUE; - } - - return err; -} - -static int aspeed_sham_cra_init(struct crypto_tfm *tfm) -{ - struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); - struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm); struct aspeed_hace_alg *ast_alg; ast_alg = container_of(alg, struct aspeed_hace_alg, alg.ahash.base); tctx->hace_dev = ast_alg->hace_dev; - tctx->flags = 0; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct aspeed_sham_reqctx)); - - if (ast_alg->alg_base) { - /* hmac related */ - struct aspeed_sha_hmac_ctx *bctx = tctx->base; - - tctx->flags |= SHA_FLAGS_HMAC; - bctx->shash = crypto_alloc_shash(ast_alg->alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(bctx->shash)) { - dev_warn(ast_alg->hace_dev->dev, - "base driver '%s' could not be loaded.\n", - ast_alg->alg_base); - return PTR_ERR(bctx->shash); - } - } - - return 0; -} - -static void aspeed_sham_cra_exit(struct crypto_tfm *tfm) -{ - struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm); - struct aspeed_hace_dev *hace_dev = tctx->hace_dev; - - AHASH_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(tfm)); - - if (tctx->flags & SHA_FLAGS_HMAC) { - struct aspeed_sha_hmac_ctx *bctx = tctx->base; - - crypto_free_shash(bctx->shash); - } -} - -static int aspeed_sham_export(struct ahash_request *req, void *out) -{ - struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - - memcpy(out, rctx, sizeof(*rctx)); - - return 0; -} - -static int aspeed_sham_import(struct ahash_request *req, const void *in) -{ - struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); - - memcpy(rctx, in, sizeof(*rctx)); return 0; } @@ -853,11 +622,11 @@ static struct aspeed_hace_alg aspeed_ahash_algs[] = { .alg.ahash.base = { .init = aspeed_sham_init, .update = aspeed_sham_update, - .final = aspeed_sham_final, .finup = aspeed_sham_finup, .digest = aspeed_sham_digest, .export = aspeed_sham_export, .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, .halg = { .digestsize = SHA1_DIGEST_SIZE, .statesize = sizeof(struct aspeed_sham_reqctx), @@ -867,13 +636,13 @@ static struct aspeed_hace_alg aspeed_ahash_algs[] = { .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, } } }, @@ -885,11 +654,11 @@ static struct aspeed_hace_alg aspeed_ahash_algs[] = { .alg.ahash.base = { .init = aspeed_sham_init, .update = aspeed_sham_update, - .final = aspeed_sham_final, .finup = aspeed_sham_finup, .digest = aspeed_sham_digest, .export = aspeed_sham_export, .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, .halg = { .digestsize = SHA256_DIGEST_SIZE, .statesize = sizeof(struct aspeed_sham_reqctx), @@ -899,13 +668,13 @@ static struct aspeed_hace_alg aspeed_ahash_algs[] = { .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, } } }, @@ -917,11 +686,11 @@ static struct aspeed_hace_alg aspeed_ahash_algs[] = { .alg.ahash.base = { .init = aspeed_sham_init, .update = aspeed_sham_update, - .final = aspeed_sham_final, .finup = aspeed_sham_finup, .digest = aspeed_sham_digest, .export = aspeed_sham_export, .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, .halg = { .digestsize = SHA224_DIGEST_SIZE, .statesize = sizeof(struct aspeed_sham_reqctx), @@ -931,118 +700,13 @@ static struct aspeed_hace_alg aspeed_ahash_algs[] = { .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, - } - } - }, - .alg.ahash.op = { - .do_one_request = aspeed_ahash_do_one, - }, - }, - { - .alg_base = "sha1", - .alg.ahash.base = { - .init = aspeed_sham_init, - .update = aspeed_sham_update, - .final = aspeed_sham_final, - .finup = aspeed_sham_finup, - .digest = aspeed_sham_digest, - .setkey = aspeed_sham_setkey, - .export = aspeed_sham_export, - .import = aspeed_sham_import, - .halg = { - .digestsize = SHA1_DIGEST_SIZE, - .statesize = sizeof(struct aspeed_sham_reqctx), - .base = { - .cra_name = "hmac(sha1)", - .cra_driver_name = "aspeed-hmac-sha1", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + - sizeof(struct aspeed_sha_hmac_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, - } - } - }, - .alg.ahash.op = { - .do_one_request = aspeed_ahash_do_one, - }, - }, - { - .alg_base = "sha224", - .alg.ahash.base = { - .init = aspeed_sham_init, - .update = aspeed_sham_update, - .final = aspeed_sham_final, - .finup = aspeed_sham_finup, - .digest = aspeed_sham_digest, - .setkey = aspeed_sham_setkey, - .export = aspeed_sham_export, - .import = aspeed_sham_import, - .halg = { - .digestsize = SHA224_DIGEST_SIZE, - .statesize = sizeof(struct aspeed_sham_reqctx), - .base = { - .cra_name = "hmac(sha224)", - .cra_driver_name = "aspeed-hmac-sha224", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + - sizeof(struct aspeed_sha_hmac_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, - } - } - }, - .alg.ahash.op = { - .do_one_request = aspeed_ahash_do_one, - }, - }, - { - .alg_base = "sha256", - .alg.ahash.base = { - .init = aspeed_sham_init, - .update = aspeed_sham_update, - .final = aspeed_sham_final, - .finup = aspeed_sham_finup, - .digest = aspeed_sham_digest, - .setkey = aspeed_sham_setkey, - .export = aspeed_sham_export, - .import = aspeed_sham_import, - .halg = { - .digestsize = SHA256_DIGEST_SIZE, - .statesize = sizeof(struct aspeed_sham_reqctx), - .base = { - .cra_name = "hmac(sha256)", - .cra_driver_name = "aspeed-hmac-sha256", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + - sizeof(struct aspeed_sha_hmac_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, } } }, @@ -1057,11 +721,11 @@ static struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { .alg.ahash.base = { .init = aspeed_sham_init, .update = aspeed_sham_update, - .final = aspeed_sham_final, .finup = aspeed_sham_finup, .digest = aspeed_sham_digest, .export = aspeed_sham_export, .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, .halg = { .digestsize = SHA384_DIGEST_SIZE, .statesize = sizeof(struct aspeed_sham_reqctx), @@ -1071,13 +735,13 @@ static struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, } } }, @@ -1089,11 +753,11 @@ static struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { .alg.ahash.base = { .init = aspeed_sham_init, .update = aspeed_sham_update, - .final = aspeed_sham_final, .finup = aspeed_sham_finup, .digest = aspeed_sham_digest, .export = aspeed_sham_export, .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, .halg = { .digestsize = SHA512_DIGEST_SIZE, .statesize = sizeof(struct aspeed_sham_reqctx), @@ -1103,83 +767,13 @@ static struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, - } - } - }, - .alg.ahash.op = { - .do_one_request = aspeed_ahash_do_one, - }, - }, - { - .alg_base = "sha384", - .alg.ahash.base = { - .init = aspeed_sham_init, - .update = aspeed_sham_update, - .final = aspeed_sham_final, - .finup = aspeed_sham_finup, - .digest = aspeed_sham_digest, - .setkey = aspeed_sham_setkey, - .export = aspeed_sham_export, - .import = aspeed_sham_import, - .halg = { - .digestsize = SHA384_DIGEST_SIZE, - .statesize = sizeof(struct aspeed_sham_reqctx), - .base = { - .cra_name = "hmac(sha384)", - .cra_driver_name = "aspeed-hmac-sha384", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + - sizeof(struct aspeed_sha_hmac_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, - } - } - }, - .alg.ahash.op = { - .do_one_request = aspeed_ahash_do_one, - }, - }, - { - .alg_base = "sha512", - .alg.ahash.base = { - .init = aspeed_sham_init, - .update = aspeed_sham_update, - .final = aspeed_sham_final, - .finup = aspeed_sham_finup, - .digest = aspeed_sham_digest, - .setkey = aspeed_sham_setkey, - .export = aspeed_sham_export, - .import = aspeed_sham_import, - .halg = { - .digestsize = SHA512_DIGEST_SIZE, - .statesize = sizeof(struct aspeed_sham_reqctx), - .base = { - .cra_name = "hmac(sha512)", - .cra_driver_name = "aspeed-hmac-sha512", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + - sizeof(struct aspeed_sha_hmac_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = aspeed_sham_cra_init, - .cra_exit = aspeed_sham_cra_exit, } } }, diff --git a/drivers/crypto/aspeed/aspeed-hace.h b/drivers/crypto/aspeed/aspeed-hace.h index 68f70e01fccb..b1d07730d543 100644 --- a/drivers/crypto/aspeed/aspeed-hace.h +++ b/drivers/crypto/aspeed/aspeed-hace.h @@ -119,7 +119,6 @@ #define SHA_FLAGS_SHA512 BIT(4) #define SHA_FLAGS_SHA512_224 BIT(5) #define SHA_FLAGS_SHA512_256 BIT(6) -#define SHA_FLAGS_HMAC BIT(8) #define SHA_FLAGS_FINUP BIT(9) #define SHA_FLAGS_MASK (0xff) @@ -161,22 +160,18 @@ struct aspeed_engine_hash { aspeed_hace_fn_t dma_prepare; }; -struct aspeed_sha_hmac_ctx { - struct crypto_shash *shash; - u8 ipad[SHA512_BLOCK_SIZE]; - u8 opad[SHA512_BLOCK_SIZE]; -}; - struct aspeed_sham_ctx { struct aspeed_hace_dev *hace_dev; - unsigned long flags; /* hmac flag */ - - struct aspeed_sha_hmac_ctx base[]; }; struct aspeed_sham_reqctx { + /* DMA buffer written by hardware */ + u8 digest[SHA512_DIGEST_SIZE] __aligned(64); + + /* Software state sorted by size. */ + u64 digcnt[2]; + unsigned long flags; /* final update flag should no use*/ - unsigned long op; /* final or update */ u32 cmd; /* trigger cmd */ /* walk state */ @@ -188,17 +183,12 @@ struct aspeed_sham_reqctx { size_t digsize; size_t block_size; size_t ivsize; - const __be32 *sha_iv; - /* remain data buffer */ - u8 buffer[SHA512_BLOCK_SIZE * 2]; dma_addr_t buffer_dma_addr; - size_t bufcnt; /* buffer counter */ - - /* output buffer */ - u8 digest[SHA512_DIGEST_SIZE] __aligned(64); dma_addr_t digest_dma_addr; - u64 digcnt[2]; + + /* This is DMA too but read-only for hardware. */ + u8 buffer[SHA512_BLOCK_SIZE + 16]; }; struct aspeed_engine_crypto { diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 14bf86957d31..3a2684208dda 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -1743,7 +1743,8 @@ static struct skcipher_alg aes_xts_alg = { .base.cra_driver_name = "atmel-xts-aes", .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), - .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -2220,7 +2221,7 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) { - alg->cra_flags |= CRYPTO_ALG_ASYNC; + alg->cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; alg->cra_alignmask = 0xf; alg->cra_priority = ATMEL_AES_PRIORITY; alg->cra_module = THIS_MODULE; @@ -2296,6 +2297,7 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) /* keep only major version number */ switch (dd->hw_version & 0xff0) { + case 0x800: case 0x700: case 0x600: case 0x500: diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index a895e4289efa..9688d116d07e 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -402,7 +402,7 @@ EXPORT_SYMBOL(atmel_i2c_probe); static int __init atmel_i2c_init(void) { - atmel_wq = alloc_workqueue("atmel_wq", 0, 0); + atmel_wq = alloc_workqueue("atmel_wq", WQ_PERCPU, 0); return atmel_wq ? 0 : -ENOMEM; } diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 67a170608566..3d7573c7bd1c 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -1254,7 +1254,8 @@ static int atmel_sha_cra_init(struct crypto_tfm *tfm) static void atmel_sha_alg_init(struct ahash_alg *alg) { alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; - alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_ctx); alg->halg.base.cra_module = THIS_MODULE; alg->halg.base.cra_init = atmel_sha_cra_init; @@ -2041,7 +2042,8 @@ static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm) static void atmel_sha_hmac_alg_init(struct ahash_alg *alg) { alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; - alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx); alg->halg.base.cra_module = THIS_MODULE; alg->halg.base.cra_init = atmel_sha_hmac_cra_init; @@ -2532,6 +2534,7 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd) /* keep only major version number */ switch (dd->hw_version & 0xff0) { + case 0x800: case 0x700: case 0x600: case 0x510: diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index 75bebec2c757..0fcf4a39de27 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -163,6 +163,12 @@ static int atmel_sha204a_probe(struct i2c_client *client) i2c_priv->hwrng.name = dev_name(&client->dev); i2c_priv->hwrng.read = atmel_sha204a_rng_read; + /* + * According to review by Bill Cox [1], this HWRNG has very low entropy. + * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html + */ + i2c_priv->hwrng.quality = 1; + ret = devm_hwrng_register(&client->dev, &i2c_priv->hwrng); if (ret) dev_warn(&client->dev, "failed to register RNG (%d)\n", ret); diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index de9717e221e4..3b2a92029b16 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -512,7 +512,7 @@ static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) if (err && (dd->flags & TDES_FLAGS_FAST)) { dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); } return err; @@ -785,7 +785,7 @@ static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) { alg->base.cra_priority = ATMEL_TDES_PRIORITY; - alg->base.cra_flags = CRYPTO_ALG_ASYNC; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); alg->base.cra_module = THIS_MODULE; diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 1c1f57baef0e..b04d6379244a 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -252,7 +252,7 @@ struct artpec6_crypto_dma_descriptors { }; enum artpec6_crypto_variant { - ARTPEC6_CRYPTO, + ARTPEC6_CRYPTO = 1, ARTPEC7_CRYPTO, }; @@ -2067,12 +2067,12 @@ static void artpec6_crypto_process_queue(struct artpec6_crypto *ac, if (ac->pending_count) mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); else - del_timer(&ac->timer); + timer_delete(&ac->timer); } static void artpec6_crypto_timeout(struct timer_list *t) { - struct artpec6_crypto *ac = from_timer(ac, t, timer); + struct artpec6_crypto *ac = timer_container_of(ac, t, timer); dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); @@ -2842,7 +2842,6 @@ MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); static int artpec6_crypto_probe(struct platform_device *pdev) { - const struct of_device_id *match; enum artpec6_crypto_variant variant; struct artpec6_crypto *ac; struct device *dev = &pdev->dev; @@ -2853,12 +2852,10 @@ static int artpec6_crypto_probe(struct platform_device *pdev) if (artpec6_crypto_dev) return -ENODEV; - match = of_match_node(artpec6_crypto_of_match, dev->of_node); - if (!match) + variant = (enum artpec6_crypto_variant)of_device_get_match_data(dev); + if (!variant) return -EINVAL; - variant = (enum artpec6_crypto_variant)match->data; - base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); @@ -2897,13 +2894,13 @@ static int artpec6_crypto_probe(struct platform_device *pdev) tasklet_init(&ac->task, artpec6_crypto_task, (unsigned long)ac); - ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + ac->pad_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, GFP_KERNEL); if (!ac->pad_buffer) return -ENOMEM; ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); - ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + ac->zero_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, GFP_KERNEL); if (!ac->zero_buffer) return -ENOMEM; @@ -2963,7 +2960,7 @@ static void artpec6_crypto_remove(struct platform_device *pdev) tasklet_disable(&ac->task); devm_free_irq(&pdev->dev, irq, ac); tasklet_kill(&ac->task); - del_timer_sync(&ac->timer); + timer_delete_sync(&ac->timer); artpec6_crypto_disable_hw(ac); diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 9e6798efbfb7..6b80d033648e 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -15,6 +15,7 @@ #include <linux/kthread.h> #include <linux/rtnetlink.h> #include <linux/sched.h> +#include <linux/string_choices.h> #include <linux/of.h> #include <linux/io.h> #include <linux/bitops.h> @@ -140,8 +141,8 @@ spu_skcipher_rx_sg_create(struct brcm_message *mssg, struct iproc_ctx_s *ctx = rctx->ctx; u32 datalen; /* Number of bytes of response data expected */ - mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist), - rctx->gfp); + mssg->spu.dst = kmalloc_array(rx_frag_num, sizeof(struct scatterlist), + rctx->gfp); if (!mssg->spu.dst) return -ENOMEM; @@ -204,8 +205,8 @@ spu_skcipher_tx_sg_create(struct brcm_message *mssg, u32 datalen; /* Number of bytes of response data expected */ u32 stat_len; - mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist), - rctx->gfp); + mssg->spu.src = kmalloc_array(tx_frag_num, sizeof(struct scatterlist), + rctx->gfp); if (unlikely(!mssg->spu.src)) return -ENOMEM; @@ -531,8 +532,8 @@ spu_ahash_rx_sg_create(struct brcm_message *mssg, struct scatterlist *sg; /* used to build sgs in mbox message */ struct iproc_ctx_s *ctx = rctx->ctx; - mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist), - rctx->gfp); + mssg->spu.dst = kmalloc_array(rx_frag_num, sizeof(struct scatterlist), + rctx->gfp); if (!mssg->spu.dst) return -ENOMEM; @@ -586,8 +587,8 @@ spu_ahash_tx_sg_create(struct brcm_message *mssg, u32 datalen; /* Number of bytes of response data expected */ u32 stat_len; - mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist), - rctx->gfp); + mssg->spu.src = kmalloc_array(tx_frag_num, sizeof(struct scatterlist), + rctx->gfp); if (!mssg->spu.src) return -ENOMEM; @@ -1076,8 +1077,8 @@ static int spu_aead_rx_sg_create(struct brcm_message *mssg, /* have to catch gcm pad in separate buffer */ rx_frag_num++; - mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist), - rctx->gfp); + mssg->spu.dst = kmalloc_array(rx_frag_num, sizeof(struct scatterlist), + rctx->gfp); if (!mssg->spu.dst) return -ENOMEM; @@ -1178,8 +1179,8 @@ static int spu_aead_tx_sg_create(struct brcm_message *mssg, u32 assoc_offset = 0; u32 stat_len; - mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist), - rctx->gfp); + mssg->spu.src = kmalloc_array(tx_frag_num, sizeof(struct scatterlist), + rctx->gfp); if (!mssg->spu.src) return -ENOMEM; @@ -2687,7 +2688,7 @@ static int aead_enqueue(struct aead_request *req, bool is_encrypt) flow_log(" iv_ctr_len:%u\n", rctx->iv_ctr_len); flow_dump(" iv: ", req->iv, rctx->iv_ctr_len); flow_log(" authkeylen:%u\n", ctx->authkeylen); - flow_log(" is_esp: %s\n", ctx->is_esp ? "yes" : "no"); + flow_log(" is_esp: %s\n", str_yes_no(ctx->is_esp)); if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) flow_log(" max_payload infinite"); diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c index 6283e8c6d51d..86c227caa722 100644 --- a/drivers/crypto/bcm/spu.c +++ b/drivers/crypto/bcm/spu.c @@ -836,7 +836,6 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) u32 cipher_bits = 0; u32 ecf_bits = 0; u8 sctx_words = 0; - u8 *ptr = spu_hdr; flow_log("%s()\n", __func__); flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg, @@ -847,7 +846,6 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) /* starting out: zero the header (plus some) */ memset(spu_hdr, 0, sizeof(struct SPUHEADER)); - ptr += sizeof(struct SPUHEADER); /* format master header word */ /* Do not set the next bit even though the datasheet says to */ @@ -861,10 +859,8 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) /* copy the encryption keys in the SAD entry */ if (cipher_parms->alg) { - if (cipher_parms->key_len) { - ptr += cipher_parms->key_len; + if (cipher_parms->key_len) sctx_words += cipher_parms->key_len / 4; - } /* * if encrypting then set IV size, use SCTX IV unless no IV @@ -873,7 +869,6 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) if (cipher_parms->iv_len) { /* Use SCTX IV */ ecf_bits |= SCTX_IV; - ptr += cipher_parms->iv_len; sctx_words += cipher_parms->iv_len / 4; } } diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index 3fdc64b5a65e..ce322cf1baa5 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -11,6 +11,7 @@ #include <linux/kernel.h> #include <linux/string.h> +#include <linux/string_choices.h> #include "util.h" #include "spu.h" @@ -999,7 +1000,7 @@ u32 spu2_create_request(u8 *spu_hdr, req_opts->is_inbound, req_opts->auth_first); flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg, cipher_parms->mode, cipher_parms->type); - flow_log(" is_esp: %s\n", req_opts->is_esp ? "yes" : "no"); + flow_log(" is_esp: %s\n", str_yes_no(req_opts->is_esp)); flow_log(" key: %d\n", cipher_parms->key_len); flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len); flow_log(" iv: %d\n", cipher_parms->iv_len); diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index acf1b197eb84..d2eaf5205b1c 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -25,10 +25,6 @@ caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_BLOB_GEN) += blob_gen.o caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o -ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) - ccflags-y += -DCONFIG_CAAM_QI -endif - caam-$(CONFIG_DEBUG_FS) += debugfs.o obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o diff --git a/drivers/crypto/caam/blob_gen.c b/drivers/crypto/caam/blob_gen.c index 87781c1534ee..c18dbac56493 100644 --- a/drivers/crypto/caam/blob_gen.c +++ b/drivers/crypto/caam/blob_gen.c @@ -2,12 +2,14 @@ /* * Copyright (C) 2015 Pengutronix, Steffen Trumtrar <kernel@pengutronix.de> * Copyright (C) 2021 Pengutronix, Ahmad Fatoum <kernel@pengutronix.de> + * Copyright 2024-2025 NXP */ #define pr_fmt(fmt) "caam blob_gen: " fmt #include <linux/bitfield.h> #include <linux/device.h> +#include <keys/trusted-type.h> #include <soc/fsl/caam-blob.h> #include "compat.h" @@ -59,18 +61,27 @@ static void caam_blob_job_done(struct device *dev, u32 *desc, u32 err, void *con complete(&res->completion); } +static u32 check_caam_state(struct device *jrdev) +{ + const struct caam_drv_private *ctrlpriv; + + ctrlpriv = dev_get_drvdata(jrdev->parent); + return FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->jr[0]->perfmon.status)); +} + int caam_process_blob(struct caam_blob_priv *priv, struct caam_blob_info *info, bool encap) { - const struct caam_drv_private *ctrlpriv; struct caam_blob_job_result testres; struct device *jrdev = &priv->jrdev; dma_addr_t dma_in, dma_out; int op = OP_PCLID_BLOB; + int hwbk_caam_ovhd = 0; size_t output_len; u32 *desc; u32 moo; int ret; + int len; if (info->key_mod_len > CAAM_BLOB_KEYMOD_LENGTH) return -EINVAL; @@ -81,14 +92,29 @@ int caam_process_blob(struct caam_blob_priv *priv, } else { op |= OP_TYPE_DECAP_PROTOCOL; output_len = info->input_len - CAAM_BLOB_OVERHEAD; + info->output_len = output_len; + } + + if (encap && info->pkey_info.is_pkey) { + op |= OP_PCL_BLOB_BLACK; + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + op |= OP_PCL_BLOB_EKT; + hwbk_caam_ovhd = CAAM_CCM_OVERHEAD; + } + if ((info->input_len + hwbk_caam_ovhd) > MAX_KEY_SIZE) + return -EINVAL; + + len = info->input_len + hwbk_caam_ovhd; + } else { + len = info->input_len; } desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL); if (!desc) return -ENOMEM; - dma_in = dma_map_single(jrdev, info->input, info->input_len, - DMA_TO_DEVICE); + dma_in = dma_map_single(jrdev, info->input, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (dma_mapping_error(jrdev, dma_in)) { dev_err(jrdev, "unable to map input DMA buffer\n"); ret = -ENOMEM; @@ -103,8 +129,7 @@ int caam_process_blob(struct caam_blob_priv *priv, goto out_unmap_in; } - ctrlpriv = dev_get_drvdata(jrdev->parent); - moo = FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->ctrl->perfmon.status)); + moo = check_caam_state(jrdev); if (moo != CSTA_MOO_SECURE && moo != CSTA_MOO_TRUSTED) dev_warn(jrdev, "using insecure test key, enable HAB to use unique device key!\n"); @@ -116,18 +141,48 @@ int caam_process_blob(struct caam_blob_priv *priv, * Class 1 Context DWords 0+1+2+3. The random BK is stored in the * Class 1 Key Register. Operation Mode is set to AES-CCM. */ - init_job_desc(desc, 0); + + if (encap && info->pkey_info.is_pkey) { + /*!1. key command used to load class 1 key register + * from input plain key. + */ + append_key(desc, dma_in, info->input_len, + CLASS_1 | KEY_DEST_CLASS_REG); + /*!2. Fifostore to store protected key from class 1 key register. */ + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_CCM_JKEK); + } else { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_KEK); + } + /* + * JUMP_OFFSET specifies the offset of the JUMP target from + * the JUMP command's address in the descriptor buffer. + */ + append_jump(desc, JUMP_COND_NOP | BIT(0) << JUMP_OFFSET_SHIFT); + } + + /*!3. Load class 2 key with key modifier. */ append_key_as_imm(desc, info->key_mod, info->key_mod_len, - info->key_mod_len, CLASS_2 | KEY_DEST_CLASS_REG); - append_seq_in_ptr_intlen(desc, dma_in, info->input_len, 0); - append_seq_out_ptr_intlen(desc, dma_out, output_len, 0); + info->key_mod_len, CLASS_2 | KEY_DEST_CLASS_REG); + + /*!4. SEQ IN PTR Command. */ + append_seq_in_ptr(desc, dma_in, info->input_len, 0); + + /*!5. SEQ OUT PTR Command. */ + append_seq_out_ptr(desc, dma_out, output_len, 0); + + /*!6. Blob encapsulation/decapsulation PROTOCOL Command. */ append_operation(desc, op); - print_hex_dump_debug("data@"__stringify(__LINE__)": ", + print_hex_dump_debug("data@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, info->input, - info->input_len, false); - print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + len, false); + print_hex_dump_debug("jobdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, desc, desc_bytes(desc), false); @@ -138,7 +193,7 @@ int caam_process_blob(struct caam_blob_priv *priv, if (ret == -EINPROGRESS) { wait_for_completion(&testres.completion); ret = testres.err; - print_hex_dump_debug("output@"__stringify(__LINE__)": ", + print_hex_dump_debug("output@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, info->output, output_len, false); } @@ -148,10 +203,10 @@ int caam_process_blob(struct caam_blob_priv *priv, dma_unmap_single(jrdev, dma_out, output_len, DMA_FROM_DEVICE); out_unmap_in: - dma_unmap_single(jrdev, dma_in, info->input_len, DMA_TO_DEVICE); + dma_unmap_single(jrdev, dma_in, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); out_free: kfree(desc); - return ret; } EXPORT_SYMBOL(caam_process_blob); diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 2cfb1b8d8c7c..32a6e6e15ee2 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for crypto API * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * Copyright 2016-2019, 2023 NXP + * Copyright 2016-2019, 2023, 2025 NXP * * Based on talitos crypto API driver. * @@ -61,13 +61,16 @@ #include <crypto/internal/engine.h> #include <crypto/internal/skcipher.h> #include <crypto/xts.h> +#include <keys/trusted-type.h> #include <linux/dma-mapping.h> #include <linux/device.h> #include <linux/err.h> #include <linux/module.h> #include <linux/kernel.h> +#include <linux/key-type.h> #include <linux/slab.h> #include <linux/string.h> +#include <soc/fsl/caam-blob.h> /* * crypto alg @@ -119,12 +122,15 @@ struct caam_ctx { dma_addr_t sh_desc_enc_dma; dma_addr_t sh_desc_dec_dma; dma_addr_t key_dma; + u8 protected_key[CAAM_MAX_KEY_SIZE]; + dma_addr_t protected_key_dma; enum dma_data_direction dir; struct device *jrdev; struct alginfo adata; struct alginfo cdata; unsigned int authsize; bool xts_key_fallback; + bool is_blob; struct crypto_skcipher *fallback; }; @@ -751,9 +757,14 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, print_hex_dump_debug("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + /* Here keylen is actual key length */ ctx->cdata.keylen = keylen; ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; + /* Here protected key len is plain key length */ + ctx->cdata.plain_keylen = keylen; + ctx->cdata.key_cmd_opt = 0; + /* skcipher_encrypt shared descriptor */ desc = ctx->sh_desc_enc; @@ -772,6 +783,62 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, return 0; } +static int paes_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, + unsigned int keylen) +{ + struct caam_pkey_info *pkey_info = (struct caam_pkey_info *)key; + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + int err; + + ctx->cdata.key_inline = false; + + keylen = keylen - CAAM_PKEY_HEADER; + + /* Retrieve the length of key */ + ctx->cdata.plain_keylen = pkey_info->plain_key_sz; + + /* Retrieve the length of blob*/ + ctx->cdata.keylen = keylen; + + /* Retrieve the address of the blob */ + ctx->cdata.key_virt = pkey_info->key_buf; + + /* Validate key length for AES algorithms */ + err = aes_check_keylen(ctx->cdata.plain_keylen); + if (err) { + dev_err(jrdev, "bad key length\n"); + return err; + } + + /* set command option */ + ctx->cdata.key_cmd_opt |= KEY_ENC; + + /* check if the Protected-Key is CCM key */ + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->cdata.key_cmd_opt |= KEY_EKT; + + memcpy(ctx->key, ctx->cdata.key_virt, keylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.key_dma = ctx->key_dma; + + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen + + CAAM_CCM_OVERHEAD, + DMA_FROM_DEVICE); + else + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen, + DMA_FROM_DEVICE); + + ctx->cdata.protected_key_dma = ctx->protected_key_dma; + ctx->is_blob = true; + + return 0; +} + static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, unsigned int keylen) { @@ -1254,7 +1321,9 @@ static void init_skcipher_job(struct skcipher_request *req, struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); struct device *jrdev = ctx->jrdev; int ivsize = crypto_skcipher_ivsize(skcipher); - u32 *desc = edesc->hw_desc; + u32 *desc = !ctx->is_blob ? edesc->hw_desc : + (u32 *)((u8 *)edesc->hw_desc + CAAM_DESC_BYTES_MAX); + dma_addr_t desc_dma; u32 *sh_desc; u32 in_options = 0, out_options = 0; dma_addr_t src_dma, dst_dma, ptr; @@ -1269,11 +1338,6 @@ static void init_skcipher_job(struct skcipher_request *req, DUMP_PREFIX_ADDRESS, 16, 4, req->src, edesc->src_nents > 1 ? 100 : req->cryptlen, 1); - sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; - ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; - - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); if (ivsize || edesc->mapped_src_nents > 1) { src_dma = edesc->sec4_sg_dma; @@ -1283,8 +1347,6 @@ static void init_skcipher_job(struct skcipher_request *req, src_dma = sg_dma_address(req->src); } - append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); - if (likely(req->src == req->dst)) { dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); out_options = in_options; @@ -1296,7 +1358,25 @@ static void init_skcipher_job(struct skcipher_request *req, out_options = LDST_SGF; } - append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); + if (ctx->is_blob) { + cnstr_desc_skcipher_enc_dec(desc, &ctx->cdata, + src_dma, dst_dma, req->cryptlen + ivsize, + in_options, out_options, + ivsize, encrypt); + + desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); + + cnstr_desc_protected_blob_decap(edesc->hw_desc, &ctx->cdata, desc_dma); + } else { + sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; + ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; + + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); + + append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); + } } /* @@ -1817,6 +1897,7 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int ret = 0; + int len; /* * XTS is expected to return an error even for input length = 0 @@ -1842,8 +1923,12 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) crypto_skcipher_decrypt(&rctx->fallback_req); } + len = DESC_JOB_IO_LEN * CAAM_CMD_SZ; + if (ctx->is_blob) + len += CAAM_DESC_BYTES_MAX; + /* allocate extended descriptor */ - edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); + edesc = skcipher_edesc_alloc(req, len); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1888,6 +1973,27 @@ static struct caam_skcipher_alg driver_algs[] = { { .skcipher.base = { .base = { + .cra_name = "cbc(paes)", + .cra_driver_name = "cbc-paes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = paes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .max_keysize = AES_MAX_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .ivsize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .skcipher.base = { + .base = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-caam", .cra_blocksize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 7571e1ac913b..04c1105eb1f5 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -2,12 +2,13 @@ /* * Shared descriptors for aead, skcipher algorithms * - * Copyright 2016-2019 NXP + * Copyright 2016-2019, 2025 NXP */ #include "compat.h" #include "desc_constr.h" #include "caamalg_desc.h" +#include <soc/fsl/caam-blob.h> /* * For aead functions, read payload and write payload, @@ -1364,6 +1365,84 @@ static inline void skcipher_append_src_dst(u32 *desc) append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); } +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt) +{ + u32 options = cdata->algtype | OP_ALG_AS_INIT; + + if (encrypt) + options |= OP_ALG_ENCRYPT; + else + options |= OP_ALG_DECRYPT; + + init_job_desc(desc, 0); + + append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL | + JUMP_COND_NOP | JUMP_TEST_ALL | 1); + + append_key(desc, cdata->protected_key_dma, cdata->plain_keylen, + CLASS_1 | KEY_DEST_CLASS_REG | cdata->key_cmd_opt); + + append_seq_in_ptr(desc, src, data_sz, in_options); + + append_seq_out_ptr(desc, dst, data_sz, out_options); + + /* Load IV, if there is one */ + if (ivsize) + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + append_operation(desc, options); + + skcipher_append_src_dst(desc); + + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + print_hex_dump_debug("skcipher_enc_dec job desc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_desc_skcipher_enc_dec); + +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc_addr) +{ + u32 protected_store; + + init_job_desc(desc, 0); + + /* Load key modifier */ + append_load_as_imm(desc, KEYMOD, sizeof(KEYMOD) - 1, + LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY); + + append_seq_in_ptr_intlen(desc, cdata->key_dma, + cdata->plain_keylen + CAAM_BLOB_OVERHEAD, 0); + + append_seq_out_ptr_intlen(desc, cdata->protected_key_dma, + cdata->plain_keylen, 0); + + protected_store = OP_PCLID_BLOB | OP_PCL_BLOB_BLACK; + if ((cdata->key_cmd_opt >> KEY_EKT_OFFSET) & 1) + protected_store |= OP_PCL_BLOB_EKT; + + append_operation(desc, OP_TYPE_DECAP_PROTOCOL | protected_store); + + if (next_desc_addr) { + append_jump(desc, JUMP_TYPE_NONLOCAL | JUMP_TEST_ALL); + append_ptr(desc, next_desc_addr); + } + + print_hex_dump_debug("protected blob decap job desc@" __stringify(__LINE__) ":", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +} +EXPORT_SYMBOL(cnstr_desc_protected_blob_decap); + /** * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor * @desc: pointer to buffer used for descriptor construction @@ -1391,7 +1470,8 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, - cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { @@ -1466,7 +1546,8 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, - cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index f2893393ba5e..323490a4a756 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -2,7 +2,7 @@ /* * Shared descriptors for aead, skcipher algorithms * - * Copyright 2016 NXP + * Copyright 2016, 2025 NXP */ #ifndef _CAAMALG_DESC_H_ @@ -48,6 +48,9 @@ #define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \ 16 * CAAM_CMD_SZ) +/* Key modifier for CAAM Protected blobs */ +#define KEYMOD "SECURE_KEY" + void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, unsigned int icvsize, int era); @@ -113,4 +116,12 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata); void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata); +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc); + +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt); + #endif /* _CAAMALG_DESC_H_ */ diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index e809d030ab11..107ccb2ade42 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -19,6 +19,7 @@ #include <linux/dma-mapping.h> #include <linux/fsl/mc.h> #include <linux/kernel.h> +#include <linux/string_choices.h> #include <soc/fsl/dpaa2-io.h> #include <soc/fsl/dpaa2-fd.h> #include <crypto/xts.h> @@ -5175,7 +5176,7 @@ static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) return err; } - dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); + dev_dbg(dev, "disable: %s\n", str_false_true(enabled)); for (i = 0; i < priv->num_pairs; i++) { ppriv = per_cpu_ptr(priv->ppriv, i); diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index b3d14a7f4dd1..0eb43c862516 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -181,7 +181,9 @@ static inline void test_len(struct hwrng *rng, size_t len, bool wait) struct device *dev = ctx->ctrldev; buf = kcalloc(CAAM_RNG_MAX_FIFO_STORE_SIZE, sizeof(u8), GFP_KERNEL); - + if (!buf) { + return; + } while (len > 0) { read_len = rng->read(rng, buf, len, wait); diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index d4b39184dbdb..320be5d77737 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -24,7 +24,7 @@ bool caam_dpaa2; EXPORT_SYMBOL(caam_dpaa2); -#ifdef CONFIG_CAAM_QI +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI #include "qi.h" #endif @@ -573,6 +573,7 @@ static const struct soc_device_attribute caam_imx_soc_table[] = { { .soc_id = "i.MX7*", .data = &caam_imx7_data }, { .soc_id = "i.MX8M*", .data = &caam_imx7_data }, { .soc_id = "i.MX8ULP", .data = &caam_imx8ulp_data }, + { .soc_id = "i.MX8Q*", .data = &caam_imx8ulp_data }, { .soc_id = "VF*", .data = &caam_vf610_data }, { .family = "Freescale i.MX" }, { /* sentinel */ } @@ -591,9 +592,9 @@ static int init_clocks(struct device *dev, const struct caam_imx_data *data) int ret; ctrlpriv->num_clks = data->num_clks; - ctrlpriv->clks = devm_kmemdup(dev, data->clks, - data->num_clks * sizeof(data->clks[0]), - GFP_KERNEL); + ctrlpriv->clks = devm_kmemdup_array(dev, data->clks, + data->num_clks, sizeof(*data->clks), + GFP_KERNEL); if (!ctrlpriv->clks) return -ENOMEM; @@ -702,12 +703,12 @@ static int caam_ctrl_rng_init(struct device *dev) */ if (needs_entropy_delay_adjustment()) ent_delay = 12000; - if (!(ctrlpriv->rng4_sh_init || inst_handles)) { + if (!inst_handles) { dev_info(dev, "Entropy delay = %u\n", ent_delay); kick_trng(dev, ent_delay); - ent_delay += 400; + ent_delay = ent_delay * 2; } /* * if instantiate_rng(...) fails, the loop will rerun @@ -830,7 +831,7 @@ static int caam_ctrl_suspend(struct device *dev) { const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); - if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en) + if (ctrlpriv->caam_off_during_pm && !ctrlpriv->no_page0) caam_state_save(dev); return 0; @@ -841,7 +842,7 @@ static int caam_ctrl_resume(struct device *dev) struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); int ret = 0; - if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en) { + if (ctrlpriv->caam_off_during_pm && !ctrlpriv->no_page0) { caam_state_restore(dev); /* HW and rng will be reset so deinstantiation can be removed */ @@ -907,6 +908,7 @@ static int caam_probe(struct platform_device *pdev) imx_soc_data = imx_soc_match->data; reg_access = reg_access && imx_soc_data->page0_access; + ctrlpriv->no_page0 = !reg_access; /* * CAAM clocks cannot be controlled from kernel. */ @@ -966,7 +968,7 @@ iomap_ctrl: caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2); ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); -#ifdef CONFIG_CAAM_QI +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI /* If (DPAA 1.x) QI present, check whether dependencies are available */ if (ctrlpriv->qi_present && !caam_dpaa2) { ret = qman_is_probed(); @@ -1097,7 +1099,7 @@ set_dma_mask: wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN); /* If QMAN driver is present, init CAAM-QI backend */ -#ifdef CONFIG_CAAM_QI +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI ret = caam_qi_init(pdev); if (ret) dev_err(dev, "caam qi i/f init failed: %d\n", ret); diff --git a/drivers/crypto/caam/debugfs.c b/drivers/crypto/caam/debugfs.c index 6358d3cabf57..718352b7afb5 100644 --- a/drivers/crypto/caam/debugfs.c +++ b/drivers/crypto/caam/debugfs.c @@ -22,7 +22,7 @@ static int caam_debugfs_u32_get(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); -#ifdef CONFIG_CAAM_QI +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI /* * This is a counter for the number of times the congestion group (where all * the request and response queueus are) reached congestion. Incremented diff --git a/drivers/crypto/caam/debugfs.h b/drivers/crypto/caam/debugfs.h index 8b5d1acd21a7..ef238c71f92a 100644 --- a/drivers/crypto/caam/debugfs.h +++ b/drivers/crypto/caam/debugfs.h @@ -18,7 +18,7 @@ static inline void caam_debugfs_init(struct caam_drv_private *ctrlpriv, {} #endif -#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CAAM_QI) +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) void caam_debugfs_qi_congested(void); void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv); #else diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index e13470901586..c28e94fcb8c7 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -4,7 +4,7 @@ * Definitions to support CAAM descriptor instruction generation * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018, 2025 NXP */ #ifndef DESC_H @@ -162,6 +162,7 @@ * Enhanced Encryption of Key */ #define KEY_EKT 0x00100000 +#define KEY_EKT_OFFSET 20 /* * Encrypted with Trusted Key @@ -403,6 +404,7 @@ #define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_CCM_JKEK (0x14 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT) @@ -1001,6 +1003,11 @@ #define OP_PCL_TLS12_AES_256_CBC_SHA384 0xff63 #define OP_PCL_TLS12_AES_256_CBC_SHA512 0xff65 +/* Blob protocol protinfo bits */ + +#define OP_PCL_BLOB_BLACK 0x0004 +#define OP_PCL_BLOB_EKT 0x0100 + /* For DTLS - OP_PCLID_DTLS */ #define OP_PCL_DTLS_AES_128_CBC_SHA 0x002f diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 824c94d44f94..2a29dd2c9c8a 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -3,7 +3,7 @@ * caam descriptor construction helper functions * * Copyright 2008-2012 Freescale Semiconductor, Inc. - * Copyright 2019 NXP + * Copyright 2019, 2025 NXP */ #ifndef DESC_CONSTR_H @@ -498,17 +498,23 @@ do { \ * @keylen: length of the provided algorithm key, in bytes * @keylen_pad: padded length of the provided algorithm key, in bytes * @key_dma: dma (bus) address where algorithm key resides + * @protected_key_dma: dma (bus) address where protected key resides * @key_virt: virtual address where algorithm key resides * @key_inline: true - key can be inlined in the descriptor; false - key is * referenced by the descriptor + * @plain_keylen: size of the key to be loaded by the CAAM + * @key_cmd_opt: optional parameters for KEY command */ struct alginfo { u32 algtype; unsigned int keylen; unsigned int keylen_pad; dma_addr_t key_dma; + dma_addr_t protected_key_dma; const void *key_virt; bool key_inline; + u32 plain_keylen; + u32 key_cmd_opt; }; /** diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index e51320150872..a88da0d31b23 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -115,6 +115,7 @@ struct caam_drv_private { u8 blob_present; /* Nonzero if BLOB support present in device */ u8 mc_en; /* Nonzero if MC f/w is active */ u8 optee_en; /* Nonzero if OP-TEE f/w is active */ + u8 no_page0; /* Nonzero if register page 0 is not controlled by Linux */ bool pr_support; /* RNG prediction resistance available */ int secvio_irq; /* Security violation interrupt number */ int virt_en; /* Virtualization enabled in CAAM */ @@ -226,7 +227,7 @@ static inline int caam_prng_register(struct device *dev) static inline void caam_prng_unregister(void *data) {} #endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PRNG_API */ -#ifdef CONFIG_CAAM_QI +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI int caam_qi_algapi_init(struct device *dev); void caam_qi_algapi_exit(void); @@ -242,7 +243,7 @@ static inline void caam_qi_algapi_exit(void) { } -#endif /* CONFIG_CAAM_QI */ +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI */ static inline u64 caam_get_dma_mask(struct device *dev) { diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 9fcdb64084ac..0ef00df9730e 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -629,8 +629,7 @@ static int caam_jr_probe(struct platform_device *pdev) } /* Initialize crypto engine */ - jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, NULL, - false, + jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, false, CRYPTO_ENGINE_MAX_QLEN); if (!jrpriv->engine) { dev_err(jrdev, "Could not init crypto-engine\n"); diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 7701d00bcb3a..1e731ed8702b 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -122,12 +122,12 @@ int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) qm_fd_addr_set64(&fd, addr); do { + refcount_inc(&req->drv_ctx->refcnt); ret = qman_enqueue(req->drv_ctx->req_fq, &fd); - if (likely(!ret)) { - refcount_inc(&req->drv_ctx->refcnt); + if (likely(!ret)) return 0; - } + refcount_dec(&req->drv_ctx->refcnt); if (ret != -EBUSY) break; num_retries++; @@ -442,11 +442,8 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, if (!cpumask_test_cpu(*cpu, cpus)) { int *pcpu = &get_cpu_var(last_cpu); - *pcpu = cpumask_next(*pcpu, cpus); - if (*pcpu >= nr_cpu_ids) - *pcpu = cpumask_first(cpus); + *pcpu = cpumask_next_wrap(*pcpu, cpus); *cpu = *pcpu; - put_cpu_var(last_cpu); } drv_ctx->cpu = *cpu; diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile index 4679c06b611f..75227c587ed0 100644 --- a/drivers/crypto/cavium/Makefile +++ b/drivers/crypto/cavium/Makefile @@ -2,4 +2,5 @@ # # Makefile for Cavium crypto device drivers # -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += zip/ +obj-$(CONFIG_CRYPTO_DEV_CPT) += cpt/ +obj-$(CONFIG_CRYPTO_DEV_NITROX) += nitrox/ diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index d4e06999af9b..a6a76e50ba84 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -192,7 +192,7 @@ int nitrox_mbox_init(struct nitrox_device *ndev) } /* allocate pf2vf response workqueue */ - ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", 0, 0); + ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", WQ_PERCPU, 0); if (!ndev->iov.pf2vf_wq) { kfree(ndev->iov.vfdev); ndev->iov.vfdev = NULL; diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile deleted file mode 100644 index 020d189d793d..000000000000 --- a/drivers/crypto/cavium/zip/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for Cavium's ZIP Driver. -# - -obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += thunderx_zip.o -thunderx_zip-y := zip_main.o \ - zip_device.o \ - zip_crypto.o \ - zip_mem.o \ - zip_deflate.o \ - zip_inflate.o diff --git a/drivers/crypto/cavium/zip/common.h b/drivers/crypto/cavium/zip/common.h deleted file mode 100644 index 54f6fb054119..000000000000 --- a/drivers/crypto/cavium/zip/common.h +++ /dev/null @@ -1,222 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __COMMON_H__ -#define __COMMON_H__ - -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/pci.h> -#include <linux/seq_file.h> -#include <linux/string.h> -#include <linux/types.h> - -/* Device specific zlib function definitions */ -#include "zip_device.h" - -/* ZIP device definitions */ -#include "zip_main.h" - -/* ZIP memory allocation/deallocation related definitions */ -#include "zip_mem.h" - -/* Device specific structure definitions */ -#include "zip_regs.h" - -#define ZIP_ERROR -1 - -#define ZIP_FLUSH_FINISH 4 - -#define RAW_FORMAT 0 /* for rawpipe */ -#define ZLIB_FORMAT 1 /* for zpipe */ -#define GZIP_FORMAT 2 /* for gzpipe */ -#define LZS_FORMAT 3 /* for lzspipe */ - -/* Max number of ZIP devices supported */ -#define MAX_ZIP_DEVICES 2 - -/* Configures the number of zip queues to be used */ -#define ZIP_NUM_QUEUES 2 - -#define DYNAMIC_STOP_EXCESS 1024 - -/* Maximum buffer sizes in direct mode */ -#define MAX_INPUT_BUFFER_SIZE (64 * 1024) -#define MAX_OUTPUT_BUFFER_SIZE (64 * 1024) - -/** - * struct zip_operation - common data structure for comp and decomp operations - * @input: Next input byte is read from here - * @output: Next output byte written here - * @ctx_addr: Inflate context buffer address - * @history: Pointer to the history buffer - * @input_len: Number of bytes available at next_in - * @input_total_len: Total number of input bytes read - * @output_len: Remaining free space at next_out - * @output_total_len: Total number of bytes output so far - * @csum: Checksum value of the uncompressed data - * @flush: Flush flag - * @format: Format (depends on stream's wrap) - * @speed: Speed depends on stream's level - * @ccode: Compression code ( stream's strategy) - * @lzs_flag: Flag for LZS support - * @begin_file: Beginning of file indication for inflate - * @history_len: Size of the history data - * @end_file: Ending of the file indication for inflate - * @compcode: Completion status of the ZIP invocation - * @bytes_read: Input bytes read in current instruction - * @bits_processed: Total bits processed for entire file - * @sizeofptr: To distinguish between ILP32 and LP64 - * @sizeofzops: Optional just for padding - * - * This structure is used to maintain the required meta data for the - * comp and decomp operations. - */ -struct zip_operation { - u8 *input; - u8 *output; - u64 ctx_addr; - u64 history; - - u32 input_len; - u32 input_total_len; - - u32 output_len; - u32 output_total_len; - - u32 csum; - u32 flush; - - u32 format; - u32 speed; - u32 ccode; - u32 lzs_flag; - - u32 begin_file; - u32 history_len; - - u32 end_file; - u32 compcode; - u32 bytes_read; - u32 bits_processed; - - u32 sizeofptr; - u32 sizeofzops; -}; - -static inline int zip_poll_result(union zip_zres_s *result) -{ - int retries = 1000; - - while (!result->s.compcode) { - if (!--retries) { - pr_err("ZIP ERR: request timed out"); - return -ETIMEDOUT; - } - udelay(10); - /* - * Force re-reading of compcode which is updated - * by the ZIP coprocessor. - */ - rmb(); - } - return 0; -} - -/* error messages */ -#define zip_err(fmt, args...) pr_err("ZIP ERR:%s():%d: " \ - fmt "\n", __func__, __LINE__, ## args) - -#ifdef MSG_ENABLE -/* Enable all messages */ -#define zip_msg(fmt, args...) pr_info("ZIP_MSG:" fmt "\n", ## args) -#else -#define zip_msg(fmt, args...) -#endif - -#if defined(ZIP_DEBUG_ENABLE) && defined(MSG_ENABLE) - -#ifdef DEBUG_LEVEL - -#define FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : \ - strrchr(__FILE__, '\\') ? strrchr(__FILE__, '\\') + 1 : __FILE__) - -#if DEBUG_LEVEL >= 4 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ - fmt "\n", FILE_NAME, __func__, __LINE__, ## args) - -#elif DEBUG_LEVEL >= 3 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ - fmt "\n", FILE_NAME, __func__, __LINE__, ## args) - -#elif DEBUG_LEVEL >= 2 - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s() : %d: " \ - fmt "\n", __func__, __LINE__, ## args) - -#else - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) - -#endif /* DEBUG LEVEL >=4 */ - -#else - -#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) - -#endif /* DEBUG_LEVEL */ -#else - -#define zip_dbg(fmt, args...) - -#endif /* ZIP_DEBUG_ENABLE && MSG_ENABLE*/ - -#endif diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c deleted file mode 100644 index 1046a746d36f..000000000000 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ /dev/null @@ -1,301 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "zip_crypto.h" - -static void zip_static_init_zip_ops(struct zip_operation *zip_ops, - int lzs_flag) -{ - zip_ops->flush = ZIP_FLUSH_FINISH; - - /* equivalent to level 6 of opensource zlib */ - zip_ops->speed = 1; - - if (!lzs_flag) { - zip_ops->ccode = 0; /* Auto Huffman */ - zip_ops->lzs_flag = 0; - zip_ops->format = ZLIB_FORMAT; - } else { - zip_ops->ccode = 3; /* LZS Encoding */ - zip_ops->lzs_flag = 1; - zip_ops->format = LZS_FORMAT; - } - zip_ops->begin_file = 1; - zip_ops->history_len = 0; - zip_ops->end_file = 1; - zip_ops->compcode = 0; - zip_ops->csum = 1; /* Adler checksum desired */ -} - -static int zip_ctx_init(struct zip_kernel_ctx *zip_ctx, int lzs_flag) -{ - struct zip_operation *comp_ctx = &zip_ctx->zip_comp; - struct zip_operation *decomp_ctx = &zip_ctx->zip_decomp; - - zip_static_init_zip_ops(comp_ctx, lzs_flag); - zip_static_init_zip_ops(decomp_ctx, lzs_flag); - - comp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); - if (!comp_ctx->input) - return -ENOMEM; - - comp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); - if (!comp_ctx->output) - goto err_comp_input; - - decomp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); - if (!decomp_ctx->input) - goto err_comp_output; - - decomp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); - if (!decomp_ctx->output) - goto err_decomp_input; - - return 0; - -err_decomp_input: - zip_data_buf_free(decomp_ctx->input, MAX_INPUT_BUFFER_SIZE); - -err_comp_output: - zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); - -err_comp_input: - zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); - - return -ENOMEM; -} - -static void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *comp_ctx = &zip_ctx->zip_comp; - struct zip_operation *dec_ctx = &zip_ctx->zip_decomp; - - zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); - zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); - - zip_data_buf_free(dec_ctx->input, MAX_INPUT_BUFFER_SIZE); - zip_data_buf_free(dec_ctx->output, MAX_OUTPUT_BUFFER_SIZE); -} - -static int zip_compress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, - struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *zip_ops = NULL; - struct zip_state *zip_state; - struct zip_device *zip = NULL; - int ret; - - if (!zip_ctx || !src || !dst || !dlen) - return -ENOMEM; - - zip = zip_get_device(zip_get_node_id()); - if (!zip) - return -ENODEV; - - zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); - if (!zip_state) - return -ENOMEM; - - zip_ops = &zip_ctx->zip_comp; - - zip_ops->input_len = slen; - zip_ops->output_len = *dlen; - memcpy(zip_ops->input, src, slen); - - ret = zip_deflate(zip_ops, zip_state, zip); - - if (!ret) { - *dlen = zip_ops->output_len; - memcpy(dst, zip_ops->output, *dlen); - } - kfree(zip_state); - return ret; -} - -static int zip_decompress(const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, - struct zip_kernel_ctx *zip_ctx) -{ - struct zip_operation *zip_ops = NULL; - struct zip_state *zip_state; - struct zip_device *zip = NULL; - int ret; - - if (!zip_ctx || !src || !dst || !dlen) - return -ENOMEM; - - zip = zip_get_device(zip_get_node_id()); - if (!zip) - return -ENODEV; - - zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC); - if (!zip_state) - return -ENOMEM; - - zip_ops = &zip_ctx->zip_decomp; - memcpy(zip_ops->input, src, slen); - - /* Work around for a bug in zlib which needs an extra bytes sometimes */ - if (zip_ops->ccode != 3) /* Not LZS Encoding */ - zip_ops->input[slen++] = 0; - - zip_ops->input_len = slen; - zip_ops->output_len = *dlen; - - ret = zip_inflate(zip_ops, zip_state, zip); - - if (!ret) { - *dlen = zip_ops->output_len; - memcpy(dst, zip_ops->output, *dlen); - } - kfree(zip_state); - return ret; -} - -/* Legacy Compress framework start */ -int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm) -{ - struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - - return zip_ctx_init(zip_ctx, 0); -} - -int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm) -{ - struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - - return zip_ctx_init(zip_ctx, 1); -} - -void zip_free_comp_ctx(struct crypto_tfm *tfm) -{ - struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - - zip_ctx_exit(zip_ctx); -} - -int zip_comp_compress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - - return zip_compress(src, slen, dst, dlen, zip_ctx); -} - -int zip_comp_decompress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - - return zip_decompress(src, slen, dst, dlen, zip_ctx); -} /* Legacy compress framework end */ - -/* SCOMP framework start */ -void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) -{ - int ret; - struct zip_kernel_ctx *zip_ctx; - - zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); - if (!zip_ctx) - return ERR_PTR(-ENOMEM); - - ret = zip_ctx_init(zip_ctx, 0); - - if (ret) { - kfree_sensitive(zip_ctx); - return ERR_PTR(ret); - } - - return zip_ctx; -} - -void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) -{ - int ret; - struct zip_kernel_ctx *zip_ctx; - - zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); - if (!zip_ctx) - return ERR_PTR(-ENOMEM); - - ret = zip_ctx_init(zip_ctx, 1); - - if (ret) { - kfree_sensitive(zip_ctx); - return ERR_PTR(ret); - } - - return zip_ctx; -} - -void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - zip_ctx_exit(zip_ctx); - kfree_sensitive(zip_ctx); -} - -int zip_scomp_compress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - return zip_compress(src, slen, dst, dlen, zip_ctx); -} - -int zip_scomp_decompress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - struct zip_kernel_ctx *zip_ctx = ctx; - - return zip_decompress(src, slen, dst, dlen, zip_ctx); -} /* SCOMP framework end */ diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h deleted file mode 100644 index b59ddfcacd34..000000000000 --- a/drivers/crypto/cavium/zip/zip_crypto.h +++ /dev/null @@ -1,79 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_CRYPTO_H__ -#define __ZIP_CRYPTO_H__ - -#include <linux/crypto.h> -#include <crypto/internal/scompress.h> -#include "common.h" -#include "zip_deflate.h" -#include "zip_inflate.h" - -struct zip_kernel_ctx { - struct zip_operation zip_comp; - struct zip_operation zip_decomp; -}; - -int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm); -int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm); -void zip_free_comp_ctx(struct crypto_tfm *tfm); -int zip_comp_compress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen); -int zip_comp_decompress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen); - -void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm); -void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm); -void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx); -int zip_scomp_compress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx); -int zip_scomp_decompress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx); -#endif diff --git a/drivers/crypto/cavium/zip/zip_deflate.c b/drivers/crypto/cavium/zip/zip_deflate.c deleted file mode 100644 index d7133f857d67..000000000000 --- a/drivers/crypto/cavium/zip/zip_deflate.c +++ /dev/null @@ -1,200 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include <linux/delay.h> -#include <linux/sched.h> - -#include "common.h" -#include "zip_deflate.h" - -/* Prepares the deflate zip command */ -static int prepare_zip_command(struct zip_operation *zip_ops, - struct zip_state *s, union zip_inst_s *zip_cmd) -{ - union zip_zres_s *result_ptr = &s->result; - - memset(zip_cmd, 0, sizeof(s->zip_cmd)); - memset(result_ptr, 0, sizeof(s->result)); - - /* IWORD #0 */ - /* History gather */ - zip_cmd->s.hg = 0; - /* compression enable = 1 for deflate */ - zip_cmd->s.ce = 1; - /* sf (sync flush) */ - zip_cmd->s.sf = 1; - /* ef (end of file) */ - if (zip_ops->flush == ZIP_FLUSH_FINISH) { - zip_cmd->s.ef = 1; - zip_cmd->s.sf = 0; - } - - zip_cmd->s.cc = zip_ops->ccode; - /* ss (compression speed/storage) */ - zip_cmd->s.ss = zip_ops->speed; - - /* IWORD #1 */ - /* adler checksum */ - zip_cmd->s.adlercrc32 = zip_ops->csum; - zip_cmd->s.historylength = zip_ops->history_len; - zip_cmd->s.dg = 0; - - /* IWORD # 6 and 7 - compression input/history pointer */ - zip_cmd->s.inp_ptr_addr.s.addr = __pa(zip_ops->input); - zip_cmd->s.inp_ptr_ctl.s.length = (zip_ops->input_len + - zip_ops->history_len); - zip_cmd->s.ds = 0; - - /* IWORD # 8 and 9 - Output pointer */ - zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); - zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; - /* maximum number of output-stream bytes that can be written */ - zip_cmd->s.totaloutputlength = zip_ops->output_len; - - /* IWORD # 10 and 11 - Result pointer */ - zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); - /* Clearing completion code */ - result_ptr->s.compcode = 0; - - return 0; -} - -/** - * zip_deflate - API to offload deflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to zip device structure - * - * This function prepares the zip deflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev) -{ - union zip_inst_s *zip_cmd = &s->zip_cmd; - union zip_zres_s *result_ptr = &s->result; - u32 queue; - - /* Prepares zip command based on the input parameters */ - prepare_zip_command(zip_ops, s, zip_cmd); - - atomic64_add(zip_ops->input_len, &zip_dev->stats.comp_in_bytes); - /* Loads zip command into command queues and rings door bell */ - queue = zip_load_instr(zip_cmd, zip_dev); - - /* Stats update for compression requests submitted */ - atomic64_inc(&zip_dev->stats.comp_req_submit); - - /* Wait for completion or error */ - zip_poll_result(result_ptr); - - /* Stats update for compression requests completed */ - atomic64_inc(&zip_dev->stats.comp_req_complete); - - zip_ops->compcode = result_ptr->s.compcode; - switch (zip_ops->compcode) { - case ZIP_CMD_NOTDONE: - zip_dbg("Zip instruction not yet completed"); - return ZIP_ERROR; - - case ZIP_CMD_SUCCESS: - zip_dbg("Zip instruction completed successfully"); - zip_update_cmd_bufs(zip_dev, queue); - break; - - case ZIP_CMD_DTRUNC: - zip_dbg("Output Truncate error"); - /* Returning ZIP_ERROR to avoid copy to user */ - return ZIP_ERROR; - - default: - zip_err("Zip instruction failed. Code:%d", zip_ops->compcode); - return ZIP_ERROR; - } - - /* Update the CRC depending on the format */ - switch (zip_ops->format) { - case RAW_FORMAT: - zip_dbg("RAW Format: %d ", zip_ops->format); - /* Get checksum from engine, need to feed it again */ - zip_ops->csum = result_ptr->s.adler32; - break; - - case ZLIB_FORMAT: - zip_dbg("ZLIB Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.adler32; - break; - - case GZIP_FORMAT: - zip_dbg("GZIP Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.crc32; - break; - - case LZS_FORMAT: - zip_dbg("LZS Format: %d ", zip_ops->format); - break; - - default: - zip_err("Unknown Format:%d\n", zip_ops->format); - } - - atomic64_add(result_ptr->s.totalbyteswritten, - &zip_dev->stats.comp_out_bytes); - - /* Update output_len */ - if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { - /* Dynamic stop && strm->output_len < zipconstants[onfsize] */ - zip_err("output_len (%d) < total bytes written(%d)\n", - zip_ops->output_len, result_ptr->s.totalbyteswritten); - zip_ops->output_len = 0; - - } else { - zip_ops->output_len = result_ptr->s.totalbyteswritten; - } - - return 0; -} diff --git a/drivers/crypto/cavium/zip/zip_deflate.h b/drivers/crypto/cavium/zip/zip_deflate.h deleted file mode 100644 index 1d32e76edc4d..000000000000 --- a/drivers/crypto/cavium/zip/zip_deflate.h +++ /dev/null @@ -1,62 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_DEFLATE_H__ -#define __ZIP_DEFLATE_H__ - -/** - * zip_deflate - API to offload deflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to the structure representing zip device - * - * This function prepares the zip deflate command and submits it to the zip - * engine by ringing the doorbell. - * - * Return: 0 if successful or error code - */ -int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev); -#endif diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c deleted file mode 100644 index f174ec29ed69..000000000000 --- a/drivers/crypto/cavium/zip/zip_device.c +++ /dev/null @@ -1,202 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "common.h" -#include "zip_deflate.h" - -/** - * zip_cmd_queue_consumed - Calculates the space consumed in the command queue. - * - * @zip_dev: Pointer to zip device structure - * @queue: Queue number - * - * Return: Bytes consumed in the command queue buffer. - */ -static inline u32 zip_cmd_queue_consumed(struct zip_device *zip_dev, int queue) -{ - return ((zip_dev->iq[queue].sw_head - zip_dev->iq[queue].sw_tail) * - sizeof(u64 *)); -} - -/** - * zip_load_instr - Submits the instruction into the ZIP command queue - * @instr: Pointer to the instruction to be submitted - * @zip_dev: Pointer to ZIP device structure to which the instruction is to - * be submitted - * - * This function copies the ZIP instruction to the command queue and rings the - * doorbell to notify the engine of the instruction submission. The command - * queue is maintained in a circular fashion. When there is space for exactly - * one instruction in the queue, next chunk pointer of the queue is made to - * point to the head of the queue, thus maintaining a circular queue. - * - * Return: Queue number to which the instruction was submitted - */ -u32 zip_load_instr(union zip_inst_s *instr, - struct zip_device *zip_dev) -{ - union zip_quex_doorbell dbell; - u32 queue = 0; - u32 consumed = 0; - u64 *ncb_ptr = NULL; - union zip_nptr_s ncp; - - /* - * Distribute the instructions between the enabled queues based on - * the CPU id. - */ - if (raw_smp_processor_id() % 2 == 0) - queue = 0; - else - queue = 1; - - zip_dbg("CPU Core: %d Queue number:%d", raw_smp_processor_id(), queue); - - /* Take cmd buffer lock */ - spin_lock(&zip_dev->iq[queue].lock); - - /* - * Command Queue implementation - * 1. If there is place for new instructions, push the cmd at sw_head. - * 2. If there is place for exactly one instruction, push the new cmd - * at the sw_head. Make sw_head point to the sw_tail to make it - * circular. Write sw_head's physical address to the "Next-Chunk - * Buffer Ptr" to make it cmd_hw_tail. - * 3. Ring the door bell. - */ - zip_dbg("sw_head : %lx", zip_dev->iq[queue].sw_head); - zip_dbg("sw_tail : %lx", zip_dev->iq[queue].sw_tail); - - consumed = zip_cmd_queue_consumed(zip_dev, queue); - /* Check if there is space to push just one cmd */ - if ((consumed + 128) == (ZIP_CMD_QBUF_SIZE - 8)) { - zip_dbg("Cmd queue space available for single command"); - /* Space for one cmd, pust it and make it circular queue */ - memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, - sizeof(union zip_inst_s)); - zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ - - /* Now, point the "Next-Chunk Buffer Ptr" to sw_head */ - ncb_ptr = zip_dev->iq[queue].sw_head; - - zip_dbg("ncb addr :0x%lx sw_head addr :0x%lx", - ncb_ptr, zip_dev->iq[queue].sw_head - 16); - - /* Using Circular command queue */ - zip_dev->iq[queue].sw_head = zip_dev->iq[queue].sw_tail; - /* Mark this buffer for free */ - zip_dev->iq[queue].free_flag = 1; - - /* Write new chunk buffer address at "Next-Chunk Buffer Ptr" */ - ncp.u_reg64 = 0ull; - ncp.s.addr = __pa(zip_dev->iq[queue].sw_head); - *ncb_ptr = ncp.u_reg64; - zip_dbg("*ncb_ptr :0x%lx sw_head[phys] :0x%lx", - *ncb_ptr, __pa(zip_dev->iq[queue].sw_head)); - - zip_dev->iq[queue].pend_cnt++; - - } else { - zip_dbg("Enough space is available for commands"); - /* Push this cmd to cmd queue buffer */ - memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, - sizeof(union zip_inst_s)); - zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ - - zip_dev->iq[queue].pend_cnt++; - } - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, - zip_dev->iq[queue].hw_tail); - - zip_dbg(" Pushed the new cmd : pend_cnt : %d", - zip_dev->iq[queue].pend_cnt); - - /* Ring the doorbell */ - dbell.u_reg64 = 0ull; - dbell.s.dbell_cnt = 1; - zip_reg_write(dbell.u_reg64, - (zip_dev->reg_base + ZIP_QUEX_DOORBELL(queue))); - - /* Unlock cmd buffer lock */ - spin_unlock(&zip_dev->iq[queue].lock); - - return queue; -} - -/** - * zip_update_cmd_bufs - Updates the queue statistics after posting the - * instruction - * @zip_dev: Pointer to zip device structure - * @queue: Queue number - */ -void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue) -{ - /* Take cmd buffer lock */ - spin_lock(&zip_dev->iq[queue].lock); - - /* Check if the previous buffer can be freed */ - if (zip_dev->iq[queue].free_flag == 1) { - zip_dbg("Free flag. Free cmd buffer, adjust sw head and tail"); - /* Reset the free flag */ - zip_dev->iq[queue].free_flag = 0; - - /* Point the hw_tail to start of the new chunk buffer */ - zip_dev->iq[queue].hw_tail = zip_dev->iq[queue].sw_head; - } else { - zip_dbg("Free flag not set. increment hw tail"); - zip_dev->iq[queue].hw_tail += 16; /* 16 64_bit words = 128B */ - } - - zip_dev->iq[queue].done_cnt++; - zip_dev->iq[queue].pend_cnt--; - - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, - zip_dev->iq[queue].hw_tail); - zip_dbg(" Got CC : pend_cnt : %d\n", zip_dev->iq[queue].pend_cnt); - - spin_unlock(&zip_dev->iq[queue].lock); -} diff --git a/drivers/crypto/cavium/zip/zip_device.h b/drivers/crypto/cavium/zip/zip_device.h deleted file mode 100644 index 9e18b3b93d38..000000000000 --- a/drivers/crypto/cavium/zip/zip_device.h +++ /dev/null @@ -1,108 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_DEVICE_H__ -#define __ZIP_DEVICE_H__ - -#include <linux/types.h> -#include "zip_main.h" - -struct sg_info { - /* - * Pointer to the input data when scatter_gather == 0 and - * pointer to the input gather list buffer when scatter_gather == 1 - */ - union zip_zptr_s *gather; - - /* - * Pointer to the output data when scatter_gather == 0 and - * pointer to the output scatter list buffer when scatter_gather == 1 - */ - union zip_zptr_s *scatter; - - /* - * Holds size of the output buffer pointed by scatter list - * when scatter_gather == 1 - */ - u64 scatter_buf_size; - - /* for gather data */ - u64 gather_enable; - - /* for scatter data */ - u64 scatter_enable; - - /* Number of gather list pointers for gather data */ - u32 gbuf_cnt; - - /* Number of scatter list pointers for scatter data */ - u32 sbuf_cnt; - - /* Buffers allocation state */ - u8 alloc_state; -}; - -/** - * struct zip_state - Structure representing the required information related - * to a command - * @zip_cmd: Pointer to zip instruction structure - * @result: Pointer to zip result structure - * @ctx: Context pointer for inflate - * @history: Decompression history pointer - * @sginfo: Scatter-gather info structure - */ -struct zip_state { - union zip_inst_s zip_cmd; - union zip_zres_s result; - union zip_zptr_s *ctx; - union zip_zptr_s *history; - struct sg_info sginfo; -}; - -#define ZIP_CONTEXT_SIZE 2048 -#define ZIP_INFLATE_HISTORY_SIZE 32768 -#define ZIP_DEFLATE_HISTORY_SIZE 32768 - -#endif diff --git a/drivers/crypto/cavium/zip/zip_inflate.c b/drivers/crypto/cavium/zip/zip_inflate.c deleted file mode 100644 index 7e0d73e2f89e..000000000000 --- a/drivers/crypto/cavium/zip/zip_inflate.c +++ /dev/null @@ -1,223 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include <linux/delay.h> -#include <linux/sched.h> - -#include "common.h" -#include "zip_inflate.h" - -static int prepare_inflate_zcmd(struct zip_operation *zip_ops, - struct zip_state *s, union zip_inst_s *zip_cmd) -{ - union zip_zres_s *result_ptr = &s->result; - - memset(zip_cmd, 0, sizeof(s->zip_cmd)); - memset(result_ptr, 0, sizeof(s->result)); - - /* IWORD#0 */ - - /* Decompression History Gather list - no gather list */ - zip_cmd->s.hg = 0; - /* For decompression, CE must be 0x0. */ - zip_cmd->s.ce = 0; - /* For decompression, SS must be 0x0. */ - zip_cmd->s.ss = 0; - /* For decompression, SF should always be set. */ - zip_cmd->s.sf = 1; - - /* Begin File */ - if (zip_ops->begin_file == 0) - zip_cmd->s.bf = 0; - else - zip_cmd->s.bf = 1; - - zip_cmd->s.ef = 1; - /* 0: for Deflate decompression, 3: for LZS decompression */ - zip_cmd->s.cc = zip_ops->ccode; - - /* IWORD #1*/ - - /* adler checksum */ - zip_cmd->s.adlercrc32 = zip_ops->csum; - - /* - * HISTORYLENGTH must be 0x0 for any ZIP decompress operation. - * History data is added to a decompression operation via IWORD3. - */ - zip_cmd->s.historylength = 0; - zip_cmd->s.ds = 0; - - /* IWORD # 8 and 9 - Output pointer */ - zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); - zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; - - /* Maximum number of output-stream bytes that can be written */ - zip_cmd->s.totaloutputlength = zip_ops->output_len; - - zip_dbg("Data Direct Input case "); - - /* IWORD # 6 and 7 - input pointer */ - zip_cmd->s.dg = 0; - zip_cmd->s.inp_ptr_addr.s.addr = __pa((u8 *)zip_ops->input); - zip_cmd->s.inp_ptr_ctl.s.length = zip_ops->input_len; - - /* IWORD # 10 and 11 - Result pointer */ - zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); - - /* Clearing completion code */ - result_ptr->s.compcode = 0; - - /* Returning 0 for time being.*/ - return 0; -} - -/** - * zip_inflate - API to offload inflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to zip device structure - * - * This function prepares the zip inflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev) -{ - union zip_inst_s *zip_cmd = &s->zip_cmd; - union zip_zres_s *result_ptr = &s->result; - u32 queue; - - /* Prepare inflate zip command */ - prepare_inflate_zcmd(zip_ops, s, zip_cmd); - - atomic64_add(zip_ops->input_len, &zip_dev->stats.decomp_in_bytes); - - /* Load inflate command to zip queue and ring the doorbell */ - queue = zip_load_instr(zip_cmd, zip_dev); - - /* Decompression requests submitted stats update */ - atomic64_inc(&zip_dev->stats.decomp_req_submit); - - /* Wait for completion or error */ - zip_poll_result(result_ptr); - - /* Decompression requests completed stats update */ - atomic64_inc(&zip_dev->stats.decomp_req_complete); - - zip_ops->compcode = result_ptr->s.compcode; - switch (zip_ops->compcode) { - case ZIP_CMD_NOTDONE: - zip_dbg("Zip Instruction not yet completed\n"); - return ZIP_ERROR; - - case ZIP_CMD_SUCCESS: - zip_dbg("Zip Instruction completed successfully\n"); - break; - - case ZIP_CMD_DYNAMIC_STOP: - zip_dbg(" Dynamic stop Initiated\n"); - break; - - default: - zip_dbg("Instruction failed. Code = %d\n", zip_ops->compcode); - atomic64_inc(&zip_dev->stats.decomp_bad_reqs); - zip_update_cmd_bufs(zip_dev, queue); - return ZIP_ERROR; - } - - zip_update_cmd_bufs(zip_dev, queue); - - if ((zip_ops->ccode == 3) && (zip_ops->flush == 4) && - (zip_ops->compcode != ZIP_CMD_DYNAMIC_STOP)) - result_ptr->s.ef = 1; - - zip_ops->csum = result_ptr->s.adler32; - - atomic64_add(result_ptr->s.totalbyteswritten, - &zip_dev->stats.decomp_out_bytes); - - if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { - zip_err("output_len (%d) < total bytes written (%d)\n", - zip_ops->output_len, result_ptr->s.totalbyteswritten); - zip_ops->output_len = 0; - } else { - zip_ops->output_len = result_ptr->s.totalbyteswritten; - } - - zip_ops->bytes_read = result_ptr->s.totalbytesread; - zip_ops->bits_processed = result_ptr->s.totalbitsprocessed; - zip_ops->end_file = result_ptr->s.ef; - if (zip_ops->end_file) { - switch (zip_ops->format) { - case RAW_FORMAT: - zip_dbg("RAW Format: %d ", zip_ops->format); - /* Get checksum from engine */ - zip_ops->csum = result_ptr->s.adler32; - break; - - case ZLIB_FORMAT: - zip_dbg("ZLIB Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.adler32; - break; - - case GZIP_FORMAT: - zip_dbg("GZIP Format: %d ", zip_ops->format); - zip_ops->csum = result_ptr->s.crc32; - break; - - case LZS_FORMAT: - zip_dbg("LZS Format: %d ", zip_ops->format); - break; - - default: - zip_err("Format error:%d\n", zip_ops->format); - } - } - - return 0; -} diff --git a/drivers/crypto/cavium/zip/zip_inflate.h b/drivers/crypto/cavium/zip/zip_inflate.h deleted file mode 100644 index 6b20f179978e..000000000000 --- a/drivers/crypto/cavium/zip/zip_inflate.h +++ /dev/null @@ -1,62 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_INFLATE_H__ -#define __ZIP_INFLATE_H__ - -/** - * zip_inflate - API to offload inflate operation to hardware - * @zip_ops: Pointer to zip operation structure - * @s: Pointer to the structure representing zip state - * @zip_dev: Pointer to the structure representing zip device - * - * This function prepares the zip inflate command and submits it to the zip - * engine for processing. - * - * Return: 0 if successful or error code - */ -int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, - struct zip_device *zip_dev); -#endif diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c deleted file mode 100644 index dc5b7bf7e1fd..000000000000 --- a/drivers/crypto/cavium/zip/zip_main.c +++ /dev/null @@ -1,651 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include "common.h" -#include "zip_crypto.h" - -#define DRV_NAME "ThunderX-ZIP" - -static struct zip_device *zip_dev[MAX_ZIP_DEVICES]; - -static const struct pci_device_id zip_id_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) }, - { 0, } -}; - -static void zip_debugfs_init(void); -static void zip_debugfs_exit(void); -static int zip_register_compression_device(void); -static void zip_unregister_compression_device(void); - -void zip_reg_write(u64 val, u64 __iomem *addr) -{ - writeq(val, addr); -} - -u64 zip_reg_read(u64 __iomem *addr) -{ - return readq(addr); -} - -/* - * Allocates new ZIP device structure - * Returns zip_device pointer or NULL if cannot allocate memory for zip_device - */ -static struct zip_device *zip_alloc_device(struct pci_dev *pdev) -{ - struct zip_device *zip = NULL; - int idx; - - for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) { - if (!zip_dev[idx]) - break; - } - - /* To ensure that the index is within the limit */ - if (idx < MAX_ZIP_DEVICES) - zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL); - - if (!zip) - return NULL; - - zip_dev[idx] = zip; - zip->index = idx; - return zip; -} - -/** - * zip_get_device - Get ZIP device based on node id of cpu - * - * @node: Node id of the current cpu - * Return: Pointer to Zip device structure - */ -struct zip_device *zip_get_device(int node) -{ - if ((node < MAX_ZIP_DEVICES) && (node >= 0)) - return zip_dev[node]; - - zip_err("ZIP device not found for node id %d\n", node); - return NULL; -} - -/** - * zip_get_node_id - Get the node id of the current cpu - * - * Return: Node id of the current cpu - */ -int zip_get_node_id(void) -{ - return cpu_to_node(raw_smp_processor_id()); -} - -/* Initializes the ZIP h/w sub-system */ -static int zip_init_hw(struct zip_device *zip) -{ - union zip_cmd_ctl cmd_ctl; - union zip_constants constants; - union zip_que_ena que_ena; - union zip_quex_map que_map; - union zip_que_pri que_pri; - - union zip_quex_sbuf_addr que_sbuf_addr; - union zip_quex_sbuf_ctl que_sbuf_ctl; - - int q = 0; - - /* Enable the ZIP Engine(Core) Clock */ - cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL); - cmd_ctl.s.forceclk = 1; - zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL)); - - zip_msg("ZIP_CMD_CTL : 0x%016llx", - zip_reg_read(zip->reg_base + ZIP_CMD_CTL)); - - constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS); - zip->depth = constants.s.depth; - zip->onfsize = constants.s.onfsize; - zip->ctxsize = constants.s.ctxsize; - - zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx", - zip->depth, zip->onfsize, zip->ctxsize); - - /* - * Program ZIP_QUE(0..7)_SBUF_ADDR and ZIP_QUE(0..7)_SBUF_CTL to - * have the correct buffer pointer and size configured for each - * instruction queue. - */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - que_sbuf_ctl.u_reg64 = 0ull; - que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64)); - que_sbuf_ctl.s.inst_be = 0; - que_sbuf_ctl.s.stream_id = 0; - zip_reg_write(que_sbuf_ctl.u_reg64, - (zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); - - zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); - } - - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - memset(&zip->iq[q], 0x0, sizeof(struct zip_iq)); - - spin_lock_init(&zip->iq[q].lock); - - if (zip_cmd_qbuf_alloc(zip, q)) { - while (q != 0) { - q--; - zip_cmd_qbuf_free(zip, q); - } - return -ENOMEM; - } - - /* Initialize tail ptr to head */ - zip->iq[q].sw_tail = zip->iq[q].sw_head; - zip->iq[q].hw_tail = zip->iq[q].sw_head; - - /* Write the physical addr to register */ - que_sbuf_addr.u_reg64 = 0ull; - que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >> - ZIP_128B_ALIGN); - - zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q, - (u64)que_sbuf_addr.s.ptr); - - zip_reg_write(que_sbuf_addr.u_reg64, - (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); - - zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); - - zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", - zip->iq[q].sw_head, zip->iq[q].sw_tail, - zip->iq[q].hw_tail); - zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr); - } - - /* - * Queue-to-ZIP core mapping - * If a queue is not mapped to a particular core, it is equivalent to - * the ZIP core being disabled. - */ - que_ena.u_reg64 = 0x0ull; - /* Enabling queues based on ZIP_NUM_QUEUES */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) - que_ena.s.ena |= (0x1 << q); - zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA)); - - zip_msg("QUE_ENA : 0x%016llx", - zip_reg_read(zip->reg_base + ZIP_QUE_ENA)); - - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - que_map.u_reg64 = 0ull; - /* Mapping each queue to two ZIP cores */ - que_map.s.zce = 0x3; - zip_reg_write(que_map.u_reg64, - (zip->reg_base + ZIP_QUEX_MAP(q))); - - zip_msg("QUE_MAP(%d) : 0x%016llx", q, - zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q))); - } - - que_pri.u_reg64 = 0ull; - for (q = 0; q < ZIP_NUM_QUEUES; q++) - que_pri.s.pri |= (0x1 << q); /* Higher Priority RR */ - zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI)); - - zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI)); - - return 0; -} - -static void zip_reset(struct zip_device *zip) -{ - union zip_cmd_ctl cmd_ctl; - - cmd_ctl.u_reg64 = 0x0ull; - cmd_ctl.s.reset = 1; /* Forces ZIP cores to do reset */ - zip_reg_write(cmd_ctl.u_reg64, (zip->reg_base + ZIP_CMD_CTL)); -} - -static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct device *dev = &pdev->dev; - struct zip_device *zip = NULL; - int err; - - zip = zip_alloc_device(pdev); - if (!zip) - return -ENOMEM; - - dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index, - pdev->vendor, pdev->device, dev_to_node(dev)); - - pci_set_drvdata(pdev, zip); - zip->pdev = pdev; - - err = pci_enable_device(pdev); - if (err) { - dev_err(dev, "Failed to enable PCI device"); - goto err_free_device; - } - - err = pci_request_regions(pdev, DRV_NAME); - if (err) { - dev_err(dev, "PCI request regions failed 0x%x", err); - goto err_disable_device; - } - - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); - goto err_release_regions; - } - - /* MAP configuration registers */ - zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0); - if (!zip->reg_base) { - dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting"); - err = -ENOMEM; - goto err_release_regions; - } - - /* Initialize ZIP Hardware */ - err = zip_init_hw(zip); - if (err) - goto err_release_regions; - - /* Register with the Kernel Crypto Interface */ - err = zip_register_compression_device(); - if (err < 0) { - zip_err("ZIP: Kernel Crypto Registration failed\n"); - goto err_register; - } - - /* comp-decomp statistics are handled with debugfs interface */ - zip_debugfs_init(); - - return 0; - -err_register: - zip_reset(zip); - -err_release_regions: - if (zip->reg_base) - iounmap(zip->reg_base); - pci_release_regions(pdev); - -err_disable_device: - pci_disable_device(pdev); - -err_free_device: - pci_set_drvdata(pdev, NULL); - - /* Remove zip_dev from zip_device list, free the zip_device memory */ - zip_dev[zip->index] = NULL; - devm_kfree(dev, zip); - - return err; -} - -static void zip_remove(struct pci_dev *pdev) -{ - struct zip_device *zip = pci_get_drvdata(pdev); - int q = 0; - - if (!zip) - return; - - zip_debugfs_exit(); - - zip_unregister_compression_device(); - - if (zip->reg_base) { - zip_reset(zip); - iounmap(zip->reg_base); - } - - pci_release_regions(pdev); - pci_disable_device(pdev); - - /* - * Free Command Queue buffers. This free should be called for all - * the enabled Queues. - */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) - zip_cmd_qbuf_free(zip, q); - - pci_set_drvdata(pdev, NULL); - /* remove zip device from zip device list */ - zip_dev[zip->index] = NULL; -} - -/* PCI Sub-System Interface */ -static struct pci_driver zip_driver = { - .name = DRV_NAME, - .id_table = zip_id_table, - .probe = zip_probe, - .remove = zip_remove, -}; - -/* Kernel Crypto Subsystem Interface */ - -static struct crypto_alg zip_comp_deflate = { - .cra_name = "deflate", - .cra_driver_name = "deflate-cavium", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct zip_kernel_ctx), - .cra_priority = 300, - .cra_module = THIS_MODULE, - .cra_init = zip_alloc_comp_ctx_deflate, - .cra_exit = zip_free_comp_ctx, - .cra_u = { .compress = { - .coa_compress = zip_comp_compress, - .coa_decompress = zip_comp_decompress - } } -}; - -static struct crypto_alg zip_comp_lzs = { - .cra_name = "lzs", - .cra_driver_name = "lzs-cavium", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct zip_kernel_ctx), - .cra_priority = 300, - .cra_module = THIS_MODULE, - .cra_init = zip_alloc_comp_ctx_lzs, - .cra_exit = zip_free_comp_ctx, - .cra_u = { .compress = { - .coa_compress = zip_comp_compress, - .coa_decompress = zip_comp_decompress - } } -}; - -static struct scomp_alg zip_scomp_deflate = { - .alloc_ctx = zip_alloc_scomp_ctx_deflate, - .free_ctx = zip_free_scomp_ctx, - .compress = zip_scomp_compress, - .decompress = zip_scomp_decompress, - .base = { - .cra_name = "deflate", - .cra_driver_name = "deflate-scomp-cavium", - .cra_module = THIS_MODULE, - .cra_priority = 300, - } -}; - -static struct scomp_alg zip_scomp_lzs = { - .alloc_ctx = zip_alloc_scomp_ctx_lzs, - .free_ctx = zip_free_scomp_ctx, - .compress = zip_scomp_compress, - .decompress = zip_scomp_decompress, - .base = { - .cra_name = "lzs", - .cra_driver_name = "lzs-scomp-cavium", - .cra_module = THIS_MODULE, - .cra_priority = 300, - } -}; - -static int zip_register_compression_device(void) -{ - int ret; - - ret = crypto_register_alg(&zip_comp_deflate); - if (ret < 0) { - zip_err("Deflate algorithm registration failed\n"); - return ret; - } - - ret = crypto_register_alg(&zip_comp_lzs); - if (ret < 0) { - zip_err("LZS algorithm registration failed\n"); - goto err_unregister_alg_deflate; - } - - ret = crypto_register_scomp(&zip_scomp_deflate); - if (ret < 0) { - zip_err("Deflate scomp algorithm registration failed\n"); - goto err_unregister_alg_lzs; - } - - ret = crypto_register_scomp(&zip_scomp_lzs); - if (ret < 0) { - zip_err("LZS scomp algorithm registration failed\n"); - goto err_unregister_scomp_deflate; - } - - return ret; - -err_unregister_scomp_deflate: - crypto_unregister_scomp(&zip_scomp_deflate); -err_unregister_alg_lzs: - crypto_unregister_alg(&zip_comp_lzs); -err_unregister_alg_deflate: - crypto_unregister_alg(&zip_comp_deflate); - - return ret; -} - -static void zip_unregister_compression_device(void) -{ - crypto_unregister_alg(&zip_comp_deflate); - crypto_unregister_alg(&zip_comp_lzs); - crypto_unregister_scomp(&zip_scomp_deflate); - crypto_unregister_scomp(&zip_scomp_lzs); -} - -/* - * debugfs functions - */ -#ifdef CONFIG_DEBUG_FS -#include <linux/debugfs.h> - -/* Displays ZIP device statistics */ -static int zip_stats_show(struct seq_file *s, void *unused) -{ - u64 val = 0ull; - u64 avg_chunk = 0ull, avg_cr = 0ull; - u32 q = 0; - - int index = 0; - struct zip_device *zip; - struct zip_stats *st; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - u64 pending = 0; - - if (zip_dev[index]) { - zip = zip_dev[index]; - st = &zip->stats; - - /* Get all the pending requests */ - for (q = 0; q < ZIP_NUM_QUEUES; q++) { - val = zip_reg_read((zip->reg_base + - ZIP_DBG_QUEX_STA(q))); - pending += val >> 32 & 0xffffff; - } - - val = atomic64_read(&st->comp_req_complete); - avg_chunk = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; - - val = atomic64_read(&st->comp_out_bytes); - avg_cr = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0; - seq_printf(s, " ZIP Device %d Stats\n" - "-----------------------------------\n" - "Comp Req Submitted : \t%lld\n" - "Comp Req Completed : \t%lld\n" - "Compress In Bytes : \t%lld\n" - "Compressed Out Bytes : \t%lld\n" - "Average Chunk size : \t%llu\n" - "Average Compression ratio : \t%llu\n" - "Decomp Req Submitted : \t%lld\n" - "Decomp Req Completed : \t%lld\n" - "Decompress In Bytes : \t%lld\n" - "Decompressed Out Bytes : \t%lld\n" - "Decompress Bad requests : \t%lld\n" - "Pending Req : \t%lld\n" - "---------------------------------\n", - index, - (u64)atomic64_read(&st->comp_req_submit), - (u64)atomic64_read(&st->comp_req_complete), - (u64)atomic64_read(&st->comp_in_bytes), - (u64)atomic64_read(&st->comp_out_bytes), - avg_chunk, - avg_cr, - (u64)atomic64_read(&st->decomp_req_submit), - (u64)atomic64_read(&st->decomp_req_complete), - (u64)atomic64_read(&st->decomp_in_bytes), - (u64)atomic64_read(&st->decomp_out_bytes), - (u64)atomic64_read(&st->decomp_bad_reqs), - pending); - } - } - return 0; -} - -/* Clears stats data */ -static int zip_clear_show(struct seq_file *s, void *unused) -{ - int index = 0; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - if (zip_dev[index]) { - memset(&zip_dev[index]->stats, 0, - sizeof(struct zip_stats)); - seq_printf(s, "Cleared stats for zip %d\n", index); - } - } - - return 0; -} - -static struct zip_registers zipregs[64] = { - {"ZIP_CMD_CTL ", 0x0000ull}, - {"ZIP_THROTTLE ", 0x0010ull}, - {"ZIP_CONSTANTS ", 0x00A0ull}, - {"ZIP_QUE0_MAP ", 0x1400ull}, - {"ZIP_QUE1_MAP ", 0x1408ull}, - {"ZIP_QUE_ENA ", 0x0500ull}, - {"ZIP_QUE_PRI ", 0x0508ull}, - {"ZIP_QUE0_DONE ", 0x2000ull}, - {"ZIP_QUE1_DONE ", 0x2008ull}, - {"ZIP_QUE0_DOORBELL ", 0x4000ull}, - {"ZIP_QUE1_DOORBELL ", 0x4008ull}, - {"ZIP_QUE0_SBUF_ADDR ", 0x1000ull}, - {"ZIP_QUE1_SBUF_ADDR ", 0x1008ull}, - {"ZIP_QUE0_SBUF_CTL ", 0x1200ull}, - {"ZIP_QUE1_SBUF_CTL ", 0x1208ull}, - { NULL, 0} -}; - -/* Prints registers' contents */ -static int zip_regs_show(struct seq_file *s, void *unused) -{ - u64 val = 0; - int i = 0, index = 0; - - for (index = 0; index < MAX_ZIP_DEVICES; index++) { - if (zip_dev[index]) { - seq_printf(s, "--------------------------------\n" - " ZIP Device %d Registers\n" - "--------------------------------\n", - index); - - i = 0; - - while (zipregs[i].reg_name) { - val = zip_reg_read((zip_dev[index]->reg_base + - zipregs[i].reg_offset)); - seq_printf(s, "%s: 0x%016llx\n", - zipregs[i].reg_name, val); - i++; - } - } - } - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(zip_stats); -DEFINE_SHOW_ATTRIBUTE(zip_clear); -DEFINE_SHOW_ATTRIBUTE(zip_regs); - -/* Root directory for thunderx_zip debugfs entry */ -static struct dentry *zip_debugfs_root; - -static void zip_debugfs_init(void) -{ - if (!debugfs_initialized()) - return; - - zip_debugfs_root = debugfs_create_dir("thunderx_zip", NULL); - - /* Creating files for entries inside thunderx_zip directory */ - debugfs_create_file("zip_stats", 0444, zip_debugfs_root, NULL, - &zip_stats_fops); - - debugfs_create_file("zip_clear", 0444, zip_debugfs_root, NULL, - &zip_clear_fops); - - debugfs_create_file("zip_regs", 0444, zip_debugfs_root, NULL, - &zip_regs_fops); - -} - -static void zip_debugfs_exit(void) -{ - debugfs_remove_recursive(zip_debugfs_root); -} - -#else -static void __init zip_debugfs_init(void) { } -static void __exit zip_debugfs_exit(void) { } -#endif -/* debugfs - end */ - -module_pci_driver(zip_driver); - -MODULE_AUTHOR("Cavium Inc"); -MODULE_DESCRIPTION("Cavium Inc ThunderX ZIP Driver"); -MODULE_LICENSE("GPL v2"); -MODULE_DEVICE_TABLE(pci, zip_id_table); diff --git a/drivers/crypto/cavium/zip/zip_main.h b/drivers/crypto/cavium/zip/zip_main.h deleted file mode 100644 index e1e4fa92ce80..000000000000 --- a/drivers/crypto/cavium/zip/zip_main.h +++ /dev/null @@ -1,120 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_MAIN_H__ -#define __ZIP_MAIN_H__ - -#include "zip_device.h" -#include "zip_regs.h" - -/* PCI device IDs */ -#define PCI_DEVICE_ID_THUNDERX_ZIP 0xA01A - -/* ZIP device BARs */ -#define PCI_CFG_ZIP_PF_BAR0 0 /* Base addr for normal regs */ - -/* Maximum available zip queues */ -#define ZIP_MAX_NUM_QUEUES 8 - -#define ZIP_128B_ALIGN 7 - -/* Command queue buffer size */ -#define ZIP_CMD_QBUF_SIZE (8064 + 8) - -struct zip_registers { - char *reg_name; - u64 reg_offset; -}; - -/* ZIP Compression - Decompression stats */ -struct zip_stats { - atomic64_t comp_req_submit; - atomic64_t comp_req_complete; - atomic64_t decomp_req_submit; - atomic64_t decomp_req_complete; - atomic64_t comp_in_bytes; - atomic64_t comp_out_bytes; - atomic64_t decomp_in_bytes; - atomic64_t decomp_out_bytes; - atomic64_t decomp_bad_reqs; -}; - -/* ZIP Instruction Queue */ -struct zip_iq { - u64 *sw_head; - u64 *sw_tail; - u64 *hw_tail; - u64 done_cnt; - u64 pend_cnt; - u64 free_flag; - - /* ZIP IQ lock */ - spinlock_t lock; -}; - -/* ZIP Device */ -struct zip_device { - u32 index; - void __iomem *reg_base; - struct pci_dev *pdev; - - /* Different ZIP Constants */ - u64 depth; - u64 onfsize; - u64 ctxsize; - - struct zip_iq iq[ZIP_MAX_NUM_QUEUES]; - struct zip_stats stats; -}; - -/* Prototypes */ -struct zip_device *zip_get_device(int node_id); -int zip_get_node_id(void); -void zip_reg_write(u64 val, u64 __iomem *addr); -u64 zip_reg_read(u64 __iomem *addr); -void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue); -u32 zip_load_instr(union zip_inst_s *instr, struct zip_device *zip_dev); - -#endif /* ZIP_MAIN_H */ diff --git a/drivers/crypto/cavium/zip/zip_mem.c b/drivers/crypto/cavium/zip/zip_mem.c deleted file mode 100644 index b3e0843a9169..000000000000 --- a/drivers/crypto/cavium/zip/zip_mem.c +++ /dev/null @@ -1,114 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#include <linux/types.h> -#include <linux/vmalloc.h> - -#include "common.h" - -/** - * zip_cmd_qbuf_alloc - Allocates a cmd buffer for ZIP Instruction Queue - * @zip: Pointer to zip device structure - * @q: Queue number to allocate bufffer to - * Return: 0 if successful, -ENOMEM otherwise - */ -int zip_cmd_qbuf_alloc(struct zip_device *zip, int q) -{ - zip->iq[q].sw_head = (u64 *)__get_free_pages((GFP_KERNEL | GFP_DMA), - get_order(ZIP_CMD_QBUF_SIZE)); - - if (!zip->iq[q].sw_head) - return -ENOMEM; - - memset(zip->iq[q].sw_head, 0, ZIP_CMD_QBUF_SIZE); - - zip_dbg("cmd_qbuf_alloc[%d] Success : %p\n", q, zip->iq[q].sw_head); - return 0; -} - -/** - * zip_cmd_qbuf_free - Frees the cmd Queue buffer - * @zip: Pointer to zip device structure - * @q: Queue number to free buffer of - */ -void zip_cmd_qbuf_free(struct zip_device *zip, int q) -{ - zip_dbg("Freeing cmd_qbuf 0x%lx\n", zip->iq[q].sw_tail); - - free_pages((u64)zip->iq[q].sw_tail, get_order(ZIP_CMD_QBUF_SIZE)); -} - -/** - * zip_data_buf_alloc - Allocates memory for a data bufffer - * @size: Size of the buffer to allocate - * Returns: Pointer to the buffer allocated - */ -u8 *zip_data_buf_alloc(u64 size) -{ - u8 *ptr; - - ptr = (u8 *)__get_free_pages((GFP_KERNEL | GFP_DMA), - get_order(size)); - - if (!ptr) - return NULL; - - memset(ptr, 0, size); - - zip_dbg("Data buffer allocation success\n"); - return ptr; -} - -/** - * zip_data_buf_free - Frees the memory of a data buffer - * @ptr: Pointer to the buffer - * @size: Buffer size - */ -void zip_data_buf_free(u8 *ptr, u64 size) -{ - zip_dbg("Freeing data buffer 0x%lx\n", ptr); - - free_pages((u64)ptr, get_order(size)); -} diff --git a/drivers/crypto/cavium/zip/zip_mem.h b/drivers/crypto/cavium/zip/zip_mem.h deleted file mode 100644 index f8f2f08c4a5c..000000000000 --- a/drivers/crypto/cavium/zip/zip_mem.h +++ /dev/null @@ -1,78 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_MEM_H__ -#define __ZIP_MEM_H__ - -/** - * zip_cmd_qbuf_free - Frees the cmd Queue buffer - * @zip: Pointer to zip device structure - * @q: Queue nmber to free buffer of - */ -void zip_cmd_qbuf_free(struct zip_device *zip, int q); - -/** - * zip_cmd_qbuf_alloc - Allocates a Chunk/cmd buffer for ZIP Inst(cmd) Queue - * @zip: Pointer to zip device structure - * @q: Queue number to allocate bufffer to - * Return: 0 if successful, 1 otherwise - */ -int zip_cmd_qbuf_alloc(struct zip_device *zip, int q); - -/** - * zip_data_buf_alloc - Allocates memory for a data bufffer - * @size: Size of the buffer to allocate - * Returns: Pointer to the buffer allocated - */ -u8 *zip_data_buf_alloc(u64 size); - -/** - * zip_data_buf_free - Frees the memory of a data buffer - * @ptr: Pointer to the buffer - * @size: Buffer size - */ -void zip_data_buf_free(u8 *ptr, u64 size); - -#endif diff --git a/drivers/crypto/cavium/zip/zip_regs.h b/drivers/crypto/cavium/zip/zip_regs.h deleted file mode 100644 index 874e0236c87e..000000000000 --- a/drivers/crypto/cavium/zip/zip_regs.h +++ /dev/null @@ -1,1347 +0,0 @@ -/***********************license start************************************ - * Copyright (c) 2003-2017 Cavium, Inc. - * All rights reserved. - * - * License: one of 'Cavium License' or 'GNU General Public License Version 2' - * - * This file is provided under the terms of the Cavium License (see below) - * or under the terms of GNU General Public License, Version 2, as - * published by the Free Software Foundation. When using or redistributing - * this file, you may do so under either license. - * - * Cavium License: Redistribution and use in source and binary forms, with - * or without modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * * Neither the name of Cavium Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * This Software, including technical data, may be subject to U.S. export - * control laws, including the U.S. Export Administration Act and its - * associated regulations, and may be subject to export or import - * regulations in other countries. - * - * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" - * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS - * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH - * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY - * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT - * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) - * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A - * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET - * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE - * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES - * WITH YOU. - ***********************license end**************************************/ - -#ifndef __ZIP_REGS_H__ -#define __ZIP_REGS_H__ - -/* - * Configuration and status register (CSR) address and type definitions for - * Cavium ZIP. - */ - -#include <linux/kern_levels.h> - -/* ZIP invocation result completion status codes */ -#define ZIP_CMD_NOTDONE 0x0 - -/* Successful completion. */ -#define ZIP_CMD_SUCCESS 0x1 - -/* Output truncated */ -#define ZIP_CMD_DTRUNC 0x2 - -/* Dynamic Stop */ -#define ZIP_CMD_DYNAMIC_STOP 0x3 - -/* Uncompress ran out of input data when IWORD0[EF] was set */ -#define ZIP_CMD_ITRUNC 0x4 - -/* Uncompress found the reserved block type 3 */ -#define ZIP_CMD_RBLOCK 0x5 - -/* - * Uncompress found LEN != ZIP_CMD_NLEN in an uncompressed block in the input. - */ -#define ZIP_CMD_NLEN 0x6 - -/* Uncompress found a bad code in the main Huffman codes. */ -#define ZIP_CMD_BADCODE 0x7 - -/* Uncompress found a bad code in the 19 Huffman codes encoding lengths. */ -#define ZIP_CMD_BADCODE2 0x8 - -/* Compress found a zero-length input. */ -#define ZIP_CMD_ZERO_LEN 0x9 - -/* The compress or decompress encountered an internal parity error. */ -#define ZIP_CMD_PARITY 0xA - -/* - * Uncompress found a string identifier that precedes the uncompressed data and - * decompression history. - */ -#define ZIP_CMD_FATAL 0xB - -/** - * enum zip_int_vec_e - ZIP MSI-X Vector Enumeration, enumerates the MSI-X - * interrupt vectors. - */ -enum zip_int_vec_e { - ZIP_INT_VEC_E_ECCE = 0x10, - ZIP_INT_VEC_E_FIFE = 0x11, - ZIP_INT_VEC_E_QUE0_DONE = 0x0, - ZIP_INT_VEC_E_QUE0_ERR = 0x8, - ZIP_INT_VEC_E_QUE1_DONE = 0x1, - ZIP_INT_VEC_E_QUE1_ERR = 0x9, - ZIP_INT_VEC_E_QUE2_DONE = 0x2, - ZIP_INT_VEC_E_QUE2_ERR = 0xa, - ZIP_INT_VEC_E_QUE3_DONE = 0x3, - ZIP_INT_VEC_E_QUE3_ERR = 0xb, - ZIP_INT_VEC_E_QUE4_DONE = 0x4, - ZIP_INT_VEC_E_QUE4_ERR = 0xc, - ZIP_INT_VEC_E_QUE5_DONE = 0x5, - ZIP_INT_VEC_E_QUE5_ERR = 0xd, - ZIP_INT_VEC_E_QUE6_DONE = 0x6, - ZIP_INT_VEC_E_QUE6_ERR = 0xe, - ZIP_INT_VEC_E_QUE7_DONE = 0x7, - ZIP_INT_VEC_E_QUE7_ERR = 0xf, - ZIP_INT_VEC_E_ENUM_LAST = 0x12, -}; - -/** - * union zip_zptr_addr_s - ZIP Generic Pointer Structure for ADDR. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_addr_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif - } s; - -}; - -/** - * union zip_zptr_ctl_s - ZIP Generic Pointer Structure for CTL. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_ctl_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_112_127 : 16; - u64 length : 16; - u64 reserved_67_95 : 29; - u64 fw : 1; - u64 nc : 1; - u64 data_be : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data_be : 1; - u64 nc : 1; - u64 fw : 1; - u64 reserved_67_95 : 29; - u64 length : 16; - u64 reserved_112_127 : 16; -#endif - } s; -}; - -/** - * union zip_inst_s - ZIP Instruction Structure. - * Each ZIP instruction has 16 words (they are called IWORD0 to IWORD15 within - * the structure). - */ -union zip_inst_s { - u64 u_reg64[16]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 doneint : 1; - u64 reserved_56_62 : 7; - u64 totaloutputlength : 24; - u64 reserved_27_31 : 5; - u64 exn : 3; - u64 reserved_23_23 : 1; - u64 exbits : 7; - u64 reserved_12_15 : 4; - u64 sf : 1; - u64 ss : 2; - u64 cc : 2; - u64 ef : 1; - u64 bf : 1; - u64 ce : 1; - u64 reserved_3_3 : 1; - u64 ds : 1; - u64 dg : 1; - u64 hg : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 hg : 1; - u64 dg : 1; - u64 ds : 1; - u64 reserved_3_3 : 1; - u64 ce : 1; - u64 bf : 1; - u64 ef : 1; - u64 cc : 2; - u64 ss : 2; - u64 sf : 1; - u64 reserved_12_15 : 4; - u64 exbits : 7; - u64 reserved_23_23 : 1; - u64 exn : 3; - u64 reserved_27_31 : 5; - u64 totaloutputlength : 24; - u64 reserved_56_62 : 7; - u64 doneint : 1; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 historylength : 16; - u64 reserved_96_111 : 16; - u64 adlercrc32 : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 adlercrc32 : 32; - u64 reserved_96_111 : 16; - u64 historylength : 16; -#endif - union zip_zptr_addr_s ctx_ptr_addr; - union zip_zptr_ctl_s ctx_ptr_ctl; - union zip_zptr_addr_s his_ptr_addr; - union zip_zptr_ctl_s his_ptr_ctl; - union zip_zptr_addr_s inp_ptr_addr; - union zip_zptr_ctl_s inp_ptr_ctl; - union zip_zptr_addr_s out_ptr_addr; - union zip_zptr_ctl_s out_ptr_ctl; - union zip_zptr_addr_s res_ptr_addr; - union zip_zptr_ctl_s res_ptr_ctl; -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_817_831 : 15; - u64 wq_ptr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 wq_ptr : 49; - u64 reserved_817_831 : 15; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_882_895 : 14; - u64 tt : 2; - u64 reserved_874_879 : 6; - u64 grp : 10; - u64 tag : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 tag : 32; - u64 grp : 10; - u64 reserved_874_879 : 6; - u64 tt : 2; - u64 reserved_882_895 : 14; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_896_959 : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_896_959 : 64; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_960_1023 : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_960_1023 : 64; -#endif - } s; -}; - -/** - * union zip_nptr_s - ZIP Instruction Next-Chunk-Buffer Pointer (NPTR) - * Structure - * - * ZIP_NPTR structure is used to chain all the zip instruction buffers - * together. ZIP instruction buffers are managed (allocated and released) by - * the software. - */ -union zip_nptr_s { - u64 u_reg64; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -/** - * union zip_zptr_s - ZIP Generic Pointer Structure. - * - * It is the generic format of pointers in ZIP_INST_S. - */ -union zip_zptr_s { - u64 u_reg64[2]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 49; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 addr : 49; - u64 reserved_49_63 : 15; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_112_127 : 16; - u64 length : 16; - u64 reserved_67_95 : 29; - u64 fw : 1; - u64 nc : 1; - u64 data_be : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data_be : 1; - u64 nc : 1; - u64 fw : 1; - u64 reserved_67_95 : 29; - u64 length : 16; - u64 reserved_112_127 : 16; -#endif - } s; -}; - -/** - * union zip_zres_s - ZIP Result Structure - * - * The ZIP coprocessor writes the result structure after it completes the - * invocation. The result structure is exactly 24 bytes, and each invocation of - * the ZIP coprocessor produces exactly one result structure. - */ -union zip_zres_s { - u64 u_reg64[3]; - struct { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 crc32 : 32; - u64 adler32 : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 adler32 : 32; - u64 crc32 : 32; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 totalbyteswritten : 32; - u64 totalbytesread : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 totalbytesread : 32; - u64 totalbyteswritten : 32; -#endif -#if defined(__BIG_ENDIAN_BITFIELD) - u64 totalbitsprocessed : 32; - u64 doneint : 1; - u64 reserved_155_158 : 4; - u64 exn : 3; - u64 reserved_151_151 : 1; - u64 exbits : 7; - u64 reserved_137_143 : 7; - u64 ef : 1; - - volatile u64 compcode : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - - volatile u64 compcode : 8; - u64 ef : 1; - u64 reserved_137_143 : 7; - u64 exbits : 7; - u64 reserved_151_151 : 1; - u64 exn : 3; - u64 reserved_155_158 : 4; - u64 doneint : 1; - u64 totalbitsprocessed : 32; -#endif - } s; -}; - -/** - * union zip_cmd_ctl - Structure representing the register that controls - * clock and reset. - */ -union zip_cmd_ctl { - u64 u_reg64; - struct zip_cmd_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_2_63 : 62; - u64 forceclk : 1; - u64 reset : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reset : 1; - u64 forceclk : 1; - u64 reserved_2_63 : 62; -#endif - } s; -}; - -#define ZIP_CMD_CTL 0x0ull - -/** - * union zip_constants - Data structure representing the register that contains - * all of the current implementation-related parameters of the zip core in this - * chip. - */ -union zip_constants { - u64 u_reg64; - struct zip_constants_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 nexec : 8; - u64 reserved_49_55 : 7; - u64 syncflush_capable : 1; - u64 depth : 16; - u64 onfsize : 12; - u64 ctxsize : 12; - u64 reserved_1_7 : 7; - u64 disabled : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 disabled : 1; - u64 reserved_1_7 : 7; - u64 ctxsize : 12; - u64 onfsize : 12; - u64 depth : 16; - u64 syncflush_capable : 1; - u64 reserved_49_55 : 7; - u64 nexec : 8; -#endif - } s; -}; - -#define ZIP_CONSTANTS 0x00A0ull - -/** - * union zip_corex_bist_status - Represents registers which have the BIST - * status of memories in zip cores. - * - * Each bit is the BIST result of an individual memory - * (per bit, 0 = pass and 1 = fail). - */ -union zip_corex_bist_status { - u64 u_reg64; - struct zip_corex_bist_status_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_53_63 : 11; - u64 bstatus : 53; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 bstatus : 53; - u64 reserved_53_63 : 11; -#endif - } s; -}; - -static inline u64 ZIP_COREX_BIST_STATUS(u64 param1) -{ - if (param1 <= 1) - return 0x0520ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_COREX_BIST_STATUS: %llu\n", param1); - return 0; -} - -/** - * union zip_ctl_bist_status - Represents register that has the BIST status of - * memories in ZIP_CTL (instruction buffer, G/S pointer FIFO, input data - * buffer, output data buffers). - * - * Each bit is the BIST result of an individual memory - * (per bit, 0 = pass and 1 = fail). - */ -union zip_ctl_bist_status { - u64 u_reg64; - struct zip_ctl_bist_status_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_9_63 : 55; - u64 bstatus : 9; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 bstatus : 9; - u64 reserved_9_63 : 55; -#endif - } s; -}; - -#define ZIP_CTL_BIST_STATUS 0x0510ull - -/** - * union zip_ctl_cfg - Represents the register that controls the behavior of - * the ZIP DMA engines. - * - * It is recommended to keep default values for normal operation. Changing the - * values of the fields may be useful for diagnostics. - */ -union zip_ctl_cfg { - u64 u_reg64; - struct zip_ctl_cfg_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_52_63 : 12; - u64 ildf : 4; - u64 reserved_36_47 : 12; - u64 drtf : 4; - u64 reserved_27_31 : 5; - u64 stcf : 3; - u64 reserved_19_23 : 5; - u64 ldf : 3; - u64 reserved_2_15 : 14; - u64 busy : 1; - u64 reserved_0_0 : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 reserved_0_0 : 1; - u64 busy : 1; - u64 reserved_2_15 : 14; - u64 ldf : 3; - u64 reserved_19_23 : 5; - u64 stcf : 3; - u64 reserved_27_31 : 5; - u64 drtf : 4; - u64 reserved_36_47 : 12; - u64 ildf : 4; - u64 reserved_52_63 : 12; -#endif - } s; -}; - -#define ZIP_CTL_CFG 0x0560ull - -/** - * union zip_dbg_corex_inst - Represents the registers that reflect the status - * of the current instruction that the ZIP core is executing or has executed. - * - * These registers are only for debug use. - */ -union zip_dbg_corex_inst { - u64 u_reg64; - struct zip_dbg_corex_inst_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_35_62 : 28; - u64 qid : 3; - u64 iid : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 iid : 32; - u64 qid : 3; - u64 reserved_35_62 : 28; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_COREX_INST(u64 param1) -{ - if (param1 <= 1) - return 0x0640ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_DBG_COREX_INST: %llu\n", param1); - return 0; -} - -/** - * union zip_dbg_corex_sta - Represents registers that reflect the status of - * the zip cores. - * - * They are for debug use only. - */ -union zip_dbg_corex_sta { - u64 u_reg64; - struct zip_dbg_corex_sta_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_37_62 : 26; - u64 ist : 5; - u64 nie : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 nie : 32; - u64 ist : 5; - u64 reserved_37_62 : 26; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_COREX_STA(u64 param1) -{ - if (param1 <= 1) - return 0x0680ull + (param1 & 1) * 0x8ull; - pr_err("ZIP_DBG_COREX_STA: %llu\n", param1); - return 0; -} - -/** - * union zip_dbg_quex_sta - Represets registers that reflect status of the zip - * instruction queues. - * - * They are for debug use only. - */ -union zip_dbg_quex_sta { - u64 u_reg64; - struct zip_dbg_quex_sta_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 busy : 1; - u64 reserved_56_62 : 7; - u64 rqwc : 24; - u64 nii : 32; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 nii : 32; - u64 rqwc : 24; - u64 reserved_56_62 : 7; - u64 busy : 1; -#endif - } s; -}; - -static inline u64 ZIP_DBG_QUEX_STA(u64 param1) -{ - if (param1 <= 7) - return 0x1800ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_DBG_QUEX_STA: %llu\n", param1); - return 0; -} - -/** - * union zip_ecc_ctl - Represents the register that enables ECC for each - * individual internal memory that requires ECC. - * - * For debug purpose, it can also flip one or two bits in the ECC data. - */ -union zip_ecc_ctl { - u64 u_reg64; - struct zip_ecc_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_19_63 : 45; - u64 vmem_cdis : 1; - u64 vmem_fs : 2; - u64 reserved_15_15 : 1; - u64 idf1_cdis : 1; - u64 idf1_fs : 2; - u64 reserved_11_11 : 1; - u64 idf0_cdis : 1; - u64 idf0_fs : 2; - u64 reserved_7_7 : 1; - u64 gspf_cdis : 1; - u64 gspf_fs : 2; - u64 reserved_3_3 : 1; - u64 iqf_cdis : 1; - u64 iqf_fs : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 iqf_fs : 2; - u64 iqf_cdis : 1; - u64 reserved_3_3 : 1; - u64 gspf_fs : 2; - u64 gspf_cdis : 1; - u64 reserved_7_7 : 1; - u64 idf0_fs : 2; - u64 idf0_cdis : 1; - u64 reserved_11_11 : 1; - u64 idf1_fs : 2; - u64 idf1_cdis : 1; - u64 reserved_15_15 : 1; - u64 vmem_fs : 2; - u64 vmem_cdis : 1; - u64 reserved_19_63 : 45; -#endif - } s; -}; - -#define ZIP_ECC_CTL 0x0568ull - -/* NCB - zip_ecce_ena_w1c */ -union zip_ecce_ena_w1c { - u64 u_reg64; - struct zip_ecce_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_ENA_W1C 0x0598ull - -/* NCB - zip_ecce_ena_w1s */ -union zip_ecce_ena_w1s { - u64 u_reg64; - struct zip_ecce_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_ENA_W1S 0x0590ull - -/** - * union zip_ecce_int - Represents the register that contains the status of the - * ECC interrupt sources. - */ -union zip_ecce_int { - u64 u_reg64; - struct zip_ecce_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_INT 0x0580ull - -/* NCB - zip_ecce_int_w1s */ -union zip_ecce_int_w1s { - u64 u_reg64; - struct zip_ecce_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_37_63 : 27; - u64 dbe : 5; - u64 reserved_5_31 : 27; - u64 sbe : 5; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 sbe : 5; - u64 reserved_5_31 : 27; - u64 dbe : 5; - u64 reserved_37_63 : 27; -#endif - } s; -}; - -#define ZIP_ECCE_INT_W1S 0x0588ull - -/* NCB - zip_fife_ena_w1c */ -union zip_fife_ena_w1c { - u64 u_reg64; - struct zip_fife_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_ENA_W1C 0x0090ull - -/* NCB - zip_fife_ena_w1s */ -union zip_fife_ena_w1s { - u64 u_reg64; - struct zip_fife_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_ENA_W1S 0x0088ull - -/* NCB - zip_fife_int */ -union zip_fife_int { - u64 u_reg64; - struct zip_fife_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_INT 0x0078ull - -/* NCB - zip_fife_int_w1s */ -union zip_fife_int_w1s { - u64 u_reg64; - struct zip_fife_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_42_63 : 22; - u64 asserts : 42; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 asserts : 42; - u64 reserved_42_63 : 22; -#endif - } s; -}; - -#define ZIP_FIFE_INT_W1S 0x0080ull - -/** - * union zip_msix_pbax - Represents the register that is the MSI-X PBA table - * - * The bit number is indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_pbax { - u64 u_reg64; - struct zip_msix_pbax_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 pend : 64; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 pend : 64; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_PBAX(u64 param1) -{ - if (param1 == 0) - return 0x0000838000FF0000ull; - pr_err("ZIP_MSIX_PBAX: %llu\n", param1); - return 0; -} - -/** - * union zip_msix_vecx_addr - Represents the register that is the MSI-X vector - * table, indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_vecx_addr { - u64 u_reg64; - struct zip_msix_vecx_addr_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 addr : 47; - u64 reserved_1_1 : 1; - u64 secvec : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 secvec : 1; - u64 reserved_1_1 : 1; - u64 addr : 47; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_VECX_ADDR(u64 param1) -{ - if (param1 <= 17) - return 0x0000838000F00000ull + (param1 & 31) * 0x10ull; - pr_err("ZIP_MSIX_VECX_ADDR: %llu\n", param1); - return 0; -} - -/** - * union zip_msix_vecx_ctl - Represents the register that is the MSI-X vector - * table, indexed by the ZIP_INT_VEC_E enumeration. - */ -union zip_msix_vecx_ctl { - u64 u_reg64; - struct zip_msix_vecx_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_33_63 : 31; - u64 mask : 1; - u64 reserved_20_31 : 12; - u64 data : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 data : 20; - u64 reserved_20_31 : 12; - u64 mask : 1; - u64 reserved_33_63 : 31; -#endif - } s; -}; - -static inline u64 ZIP_MSIX_VECX_CTL(u64 param1) -{ - if (param1 <= 17) - return 0x0000838000F00008ull + (param1 & 31) * 0x10ull; - pr_err("ZIP_MSIX_VECX_CTL: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done - Represents the registers that contain the per-queue - * instruction done count. - */ -union zip_quex_done { - u64 u_reg64; - struct zip_quex_done_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 done : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE(u64 param1) -{ - if (param1 <= 7) - return 0x2000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ack - Represents the registers on write to which will - * decrement the per-queue instructiona done count. - */ -union zip_quex_done_ack { - u64 u_reg64; - struct zip_quex_done_ack_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 done_ack : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ack : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ACK(u64 param1) -{ - if (param1 <= 7) - return 0x2200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ACK: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ena_w1c - Represents the register which when written - * 1 to will disable the DONEINT interrupt for the queue. - */ -union zip_quex_done_ena_w1c { - u64 u_reg64; - struct zip_quex_done_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_1_63 : 63; - u64 done_ena : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ena : 1; - u64 reserved_1_63 : 63; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ENA_W1C(u64 param1) -{ - if (param1 <= 7) - return 0x2600ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ENA_W1C: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_ena_w1s - Represents the register that when written 1 to - * will enable the DONEINT interrupt for the queue. - */ -union zip_quex_done_ena_w1s { - u64 u_reg64; - struct zip_quex_done_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_1_63 : 63; - u64 done_ena : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 done_ena : 1; - u64 reserved_1_63 : 63; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_ENA_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x2400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_ENA_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_done_wait - Represents the register that specifies the per - * queue interrupt coalescing settings. - */ -union zip_quex_done_wait { - u64 u_reg64; - struct zip_quex_done_wait_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_48_63 : 16; - u64 time_wait : 16; - u64 reserved_20_31 : 12; - u64 num_wait : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 num_wait : 20; - u64 reserved_20_31 : 12; - u64 time_wait : 16; - u64 reserved_48_63 : 16; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DONE_WAIT(u64 param1) -{ - if (param1 <= 7) - return 0x2800ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DONE_WAIT: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_doorbell - Represents doorbell registers for the ZIP - * instruction queues. - */ -union zip_quex_doorbell { - u64 u_reg64; - struct zip_quex_doorbell_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_20_63 : 44; - u64 dbell_cnt : 20; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dbell_cnt : 20; - u64 reserved_20_63 : 44; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_DOORBELL(u64 param1) -{ - if (param1 <= 7) - return 0x4000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_DOORBELL: %llu\n", param1); - return 0; -} - -union zip_quex_err_ena_w1c { - u64 u_reg64; - struct zip_quex_err_ena_w1c_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_ENA_W1C(u64 param1) -{ - if (param1 <= 7) - return 0x3600ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_ENA_W1C: %llu\n", param1); - return 0; -} - -union zip_quex_err_ena_w1s { - u64 u_reg64; - struct zip_quex_err_ena_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_ENA_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x3400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_ENA_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_err_int - Represents registers that contain the per-queue - * error interrupts. - */ -union zip_quex_err_int { - u64 u_reg64; - struct zip_quex_err_int_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_INT(u64 param1) -{ - if (param1 <= 7) - return 0x3000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_INT: %llu\n", param1); - return 0; -} - -/* NCB - zip_que#_err_int_w1s */ -union zip_quex_err_int_w1s { - u64 u_reg64; - struct zip_quex_err_int_w1s_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_5_63 : 59; - u64 mdbe : 1; - u64 nwrp : 1; - u64 nrrp : 1; - u64 irde : 1; - u64 dovf : 1; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 dovf : 1; - u64 irde : 1; - u64 nrrp : 1; - u64 nwrp : 1; - u64 mdbe : 1; - u64 reserved_5_63 : 59; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_ERR_INT_W1S(u64 param1) -{ - if (param1 <= 7) - return 0x3200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_ERR_INT_W1S: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_gcfg - Represents the registers that reflect status of the - * zip instruction queues,debug use only. - */ -union zip_quex_gcfg { - u64 u_reg64; - struct zip_quex_gcfg_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_4_63 : 60; - u64 iqb_ldwb : 1; - u64 cbw_sty : 1; - u64 l2ld_cmd : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 l2ld_cmd : 2; - u64 cbw_sty : 1; - u64 iqb_ldwb : 1; - u64 reserved_4_63 : 60; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_GCFG(u64 param1) -{ - if (param1 <= 7) - return 0x1A00ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_GCFG: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_map - Represents the registers that control how each - * instruction queue maps to zip cores. - */ -union zip_quex_map { - u64 u_reg64; - struct zip_quex_map_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_2_63 : 62; - u64 zce : 2; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 zce : 2; - u64 reserved_2_63 : 62; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_MAP(u64 param1) -{ - if (param1 <= 7) - return 0x1400ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_MAP: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_sbuf_addr - Represents the registers that set the buffer - * parameters for the instruction queues. - * - * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite - * this register to effectively reset the command buffer state machine. - * These registers must be programmed after SW programs the corresponding - * ZIP_QUE(0..7)_SBUF_CTL. - */ -union zip_quex_sbuf_addr { - u64 u_reg64; - struct zip_quex_sbuf_addr_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_49_63 : 15; - u64 ptr : 42; - u64 off : 7; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 off : 7; - u64 ptr : 42; - u64 reserved_49_63 : 15; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_SBUF_ADDR(u64 param1) -{ - if (param1 <= 7) - return 0x1000ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_SBUF_ADDR: %llu\n", param1); - return 0; -} - -/** - * union zip_quex_sbuf_ctl - Represents the registers that set the buffer - * parameters for the instruction queues. - * - * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite - * this register to effectively reset the command buffer state machine. - * These registers must be programmed before SW programs the corresponding - * ZIP_QUE(0..7)_SBUF_ADDR. - */ -union zip_quex_sbuf_ctl { - u64 u_reg64; - struct zip_quex_sbuf_ctl_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_45_63 : 19; - u64 size : 13; - u64 inst_be : 1; - u64 reserved_24_30 : 7; - u64 stream_id : 8; - u64 reserved_12_15 : 4; - u64 aura : 12; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 aura : 12; - u64 reserved_12_15 : 4; - u64 stream_id : 8; - u64 reserved_24_30 : 7; - u64 inst_be : 1; - u64 size : 13; - u64 reserved_45_63 : 19; -#endif - } s; -}; - -static inline u64 ZIP_QUEX_SBUF_CTL(u64 param1) -{ - if (param1 <= 7) - return 0x1200ull + (param1 & 7) * 0x8ull; - pr_err("ZIP_QUEX_SBUF_CTL: %llu\n", param1); - return 0; -} - -/** - * union zip_que_ena - Represents queue enable register - * - * If a queue is disabled, ZIP_CTL stops fetching instructions from the queue. - */ -union zip_que_ena { - u64 u_reg64; - struct zip_que_ena_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_8_63 : 56; - u64 ena : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 ena : 8; - u64 reserved_8_63 : 56; -#endif - } s; -}; - -#define ZIP_QUE_ENA 0x0500ull - -/** - * union zip_que_pri - Represents the register that defines the priority - * between instruction queues. - */ -union zip_que_pri { - u64 u_reg64; - struct zip_que_pri_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_8_63 : 56; - u64 pri : 8; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 pri : 8; - u64 reserved_8_63 : 56; -#endif - } s; -}; - -#define ZIP_QUE_PRI 0x0508ull - -/** - * union zip_throttle - Represents the register that controls the maximum - * number of in-flight X2I data fetch transactions. - * - * Writing 0 to this register causes the ZIP module to temporarily suspend NCB - * accesses; it is not recommended for normal operation, but may be useful for - * diagnostics. - */ -union zip_throttle { - u64 u_reg64; - struct zip_throttle_s { -#if defined(__BIG_ENDIAN_BITFIELD) - u64 reserved_6_63 : 58; - u64 ld_infl : 6; -#elif defined(__LITTLE_ENDIAN_BITFIELD) - u64 ld_infl : 6; - u64 reserved_6_63 : 58; -#endif - } s; -}; - -#define ZIP_THROTTLE 0x0010ull - -#endif /* _CSRS_ZIP__ */ diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig index f394e45e11ab..f16a0f611317 100644 --- a/drivers/crypto/ccp/Kconfig +++ b/drivers/crypto/ccp/Kconfig @@ -39,6 +39,7 @@ config CRYPTO_DEV_SP_PSP bool "Platform Security Processor (PSP) device" default y depends on CRYPTO_DEV_CCP_DD && X86_64 && AMD_IOMMU + select PCI_TSM if PCI help Provide support for the AMD Platform Security Processor (PSP). The PSP is a dedicated processor that provides support for key diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index 394484929dae..0424e08561ef 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -13,7 +13,12 @@ ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += psp-dev.o \ tee-dev.o \ platform-access.o \ dbc.o \ - hsti.o + hsti.o \ + sfs.o + +ifeq ($(CONFIG_PCI_TSM),y) +ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += sev-dev-tsm.o sev-dev-tio.o +endif obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o ccp-crypto-objs := ccp-crypto-main.o \ diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index d11daaf47f06..685d42ec7ade 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -7,15 +7,16 @@ * Author: Tom Lendacky <thomas.lendacky@amd.com> */ -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/delay.h> -#include <linux/scatterlist.h> -#include <linux/crypto.h> -#include <crypto/algapi.h> #include <crypto/aes.h> #include <crypto/ctr.h> -#include <crypto/scatterwalk.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/string.h> #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index afae30adb703..91b1189c47de 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -7,14 +7,15 @@ * Author: Gary R Hook <ghook@amd.com> */ +#include <crypto/internal/des.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/list.h> #include <linux/module.h> -#include <linux/sched.h> -#include <linux/delay.h> #include <linux/scatterlist.h> -#include <linux/crypto.h> -#include <crypto/algapi.h> -#include <crypto/scatterwalk.h> -#include <crypto/internal/des.h> +#include <linux/slab.h> +#include <linux/string.h> #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index ecd58b38c46e..bc90aba5162a 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -7,14 +7,17 @@ * Author: Tom Lendacky <thomas.lendacky@amd.com> */ -#include <linux/module.h> -#include <linux/moduleparam.h> +#include <crypto/internal/akcipher.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <linux/ccp.h> +#include <linux/err.h> #include <linux/kernel.h> #include <linux/list.h> -#include <linux/ccp.h> +#include <linux/module.h> #include <linux/scatterlist.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/akcipher.h> +#include <linux/slab.h> +#include <linux/spinlock.h> #include "ccp-crypto.h" diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c index a1055554b47a..dc26bc22c91d 100644 --- a/drivers/crypto/ccp/ccp-debugfs.c +++ b/drivers/crypto/ccp/ccp-debugfs.c @@ -319,5 +319,8 @@ void ccp5_debugfs_setup(struct ccp_device *ccp) void ccp5_debugfs_destroy(void) { + mutex_lock(&ccp_debugfs_lock); debugfs_remove_recursive(ccp_debugfs_dir); + ccp_debugfs_dir = NULL; + mutex_unlock(&ccp_debugfs_lock); } diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index c531d13d971f..246801912e1a 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -507,7 +507,7 @@ int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) { struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); u32 trng_value; - int len = min_t(int, sizeof(trng_value), max); + int len = min(sizeof(trng_value), max); /* Locking is provided by the caller so we can update device * hwrng-related fields safely diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index cb8e99936abb..d78865d9d5f0 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -8,13 +8,14 @@ * Author: Gary R Hook <gary.hook@amd.com> */ -#include <linux/dma-mapping.h> -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/interrupt.h> -#include <crypto/scatterwalk.h> #include <crypto/des.h> +#include <crypto/scatterwalk.h> +#include <crypto/utils.h> #include <linux/ccp.h> +#include <linux/dma-mapping.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "ccp-dev.h" @@ -632,10 +633,16 @@ static noinline_for_stack int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) { struct ccp_aes_engine *aes = &cmd->u.aes; - struct ccp_dm_workarea key, ctx, final_wa, tag; - struct ccp_data src, dst; - struct ccp_data aad; - struct ccp_op op; + struct { + struct ccp_dm_workarea key; + struct ccp_dm_workarea ctx; + struct ccp_dm_workarea final; + struct ccp_dm_workarea tag; + struct ccp_data src; + struct ccp_data dst; + struct ccp_data aad; + struct ccp_op op; + } *wa __cleanup(kfree) = kzalloc(sizeof *wa, GFP_KERNEL); unsigned int dm_offset; unsigned int authsize; unsigned int jobid; @@ -649,6 +656,9 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) struct scatterlist *p_outp, sg_outp[2]; struct scatterlist *p_aad; + if (!wa) + return -ENOMEM; + if (!aes->iv) return -EINVAL; @@ -695,26 +705,26 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) jobid = CCP_NEW_JOBID(cmd_q->ccp); - memset(&op, 0, sizeof(op)); - op.cmd_q = cmd_q; - op.jobid = jobid; - op.sb_key = cmd_q->sb_key; /* Pre-allocated */ - op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ - op.init = 1; - op.u.aes.type = aes->type; + memset(&wa->op, 0, sizeof(wa->op)); + wa->op.cmd_q = cmd_q; + wa->op.jobid = jobid; + wa->op.sb_key = cmd_q->sb_key; /* Pre-allocated */ + wa->op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ + wa->op.init = 1; + wa->op.u.aes.type = aes->type; /* Copy the key to the LSB */ - ret = ccp_init_dm_workarea(&key, cmd_q, + ret = ccp_init_dm_workarea(&wa->key, cmd_q, CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, DMA_TO_DEVICE); if (ret) return ret; dm_offset = CCP_SB_BYTES - aes->key_len; - ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + ret = ccp_set_dm_area(&wa->key, dm_offset, aes->key, 0, aes->key_len); if (ret) goto e_key; - ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, + ret = ccp_copy_to_sb(cmd_q, &wa->key, wa->op.jobid, wa->op.sb_key, CCP_PASSTHRU_BYTESWAP_256BIT); if (ret) { cmd->engine_error = cmd_q->cmd_error; @@ -725,58 +735,58 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) * There is an assumption here that the IV is 96 bits in length, plus * a nonce of 32 bits. If no IV is present, use a zeroed buffer. */ - ret = ccp_init_dm_workarea(&ctx, cmd_q, + ret = ccp_init_dm_workarea(&wa->ctx, cmd_q, CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, DMA_BIDIRECTIONAL); if (ret) goto e_key; dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; - ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + ret = ccp_set_dm_area(&wa->ctx, dm_offset, aes->iv, 0, aes->iv_len); if (ret) goto e_ctx; - ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + ret = ccp_copy_to_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx, CCP_PASSTHRU_BYTESWAP_256BIT); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_ctx; } - op.init = 1; + wa->op.init = 1; if (aes->aad_len > 0) { /* Step 1: Run a GHASH over the Additional Authenticated Data */ - ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, + ret = ccp_init_data(&wa->aad, cmd_q, p_aad, aes->aad_len, AES_BLOCK_SIZE, DMA_TO_DEVICE); if (ret) goto e_ctx; - op.u.aes.mode = CCP_AES_MODE_GHASH; - op.u.aes.action = CCP_AES_GHASHAAD; + wa->op.u.aes.mode = CCP_AES_MODE_GHASH; + wa->op.u.aes.action = CCP_AES_GHASHAAD; - while (aad.sg_wa.bytes_left) { - ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); + while (wa->aad.sg_wa.bytes_left) { + ccp_prepare_data(&wa->aad, NULL, &wa->op, AES_BLOCK_SIZE, true); - ret = cmd_q->ccp->vdata->perform->aes(&op); + ret = cmd_q->ccp->vdata->perform->aes(&wa->op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_aad; } - ccp_process_data(&aad, NULL, &op); - op.init = 0; + ccp_process_data(&wa->aad, NULL, &wa->op); + wa->op.init = 0; } } - op.u.aes.mode = CCP_AES_MODE_GCTR; - op.u.aes.action = aes->action; + wa->op.u.aes.mode = CCP_AES_MODE_GCTR; + wa->op.u.aes.action = aes->action; if (ilen > 0) { /* Step 2: Run a GCTR over the plaintext */ in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; - ret = ccp_init_data(&src, cmd_q, p_inp, ilen, + ret = ccp_init_data(&wa->src, cmd_q, p_inp, ilen, AES_BLOCK_SIZE, in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); @@ -784,52 +794,52 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) goto e_aad; if (in_place) { - dst = src; + wa->dst = wa->src; } else { - ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, + ret = ccp_init_data(&wa->dst, cmd_q, p_outp, ilen, AES_BLOCK_SIZE, DMA_FROM_DEVICE); if (ret) goto e_src; } - op.soc = 0; - op.eom = 0; - op.init = 1; - while (src.sg_wa.bytes_left) { - ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); - if (!src.sg_wa.bytes_left) { + wa->op.soc = 0; + wa->op.eom = 0; + wa->op.init = 1; + while (wa->src.sg_wa.bytes_left) { + ccp_prepare_data(&wa->src, &wa->dst, &wa->op, AES_BLOCK_SIZE, true); + if (!wa->src.sg_wa.bytes_left) { unsigned int nbytes = ilen % AES_BLOCK_SIZE; if (nbytes) { - op.eom = 1; - op.u.aes.size = (nbytes * 8) - 1; + wa->op.eom = 1; + wa->op.u.aes.size = (nbytes * 8) - 1; } } - ret = cmd_q->ccp->vdata->perform->aes(&op); + ret = cmd_q->ccp->vdata->perform->aes(&wa->op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; } - ccp_process_data(&src, &dst, &op); - op.init = 0; + ccp_process_data(&wa->src, &wa->dst, &wa->op); + wa->op.init = 0; } } /* Step 3: Update the IV portion of the context with the original IV */ - ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + ret = ccp_copy_from_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx, CCP_PASSTHRU_BYTESWAP_256BIT); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; } - ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + ret = ccp_set_dm_area(&wa->ctx, dm_offset, aes->iv, 0, aes->iv_len); if (ret) goto e_dst; - ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + ret = ccp_copy_to_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx, CCP_PASSTHRU_BYTESWAP_256BIT); if (ret) { cmd->engine_error = cmd_q->cmd_error; @@ -839,75 +849,75 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) /* Step 4: Concatenate the lengths of the AAD and source, and * hash that 16 byte buffer. */ - ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, + ret = ccp_init_dm_workarea(&wa->final, cmd_q, AES_BLOCK_SIZE, DMA_BIDIRECTIONAL); if (ret) goto e_dst; - final = (__be64 *)final_wa.address; + final = (__be64 *)wa->final.address; final[0] = cpu_to_be64(aes->aad_len * 8); final[1] = cpu_to_be64(ilen * 8); - memset(&op, 0, sizeof(op)); - op.cmd_q = cmd_q; - op.jobid = jobid; - op.sb_key = cmd_q->sb_key; /* Pre-allocated */ - op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ - op.init = 1; - op.u.aes.type = aes->type; - op.u.aes.mode = CCP_AES_MODE_GHASH; - op.u.aes.action = CCP_AES_GHASHFINAL; - op.src.type = CCP_MEMTYPE_SYSTEM; - op.src.u.dma.address = final_wa.dma.address; - op.src.u.dma.length = AES_BLOCK_SIZE; - op.dst.type = CCP_MEMTYPE_SYSTEM; - op.dst.u.dma.address = final_wa.dma.address; - op.dst.u.dma.length = AES_BLOCK_SIZE; - op.eom = 1; - op.u.aes.size = 0; - ret = cmd_q->ccp->vdata->perform->aes(&op); + memset(&wa->op, 0, sizeof(wa->op)); + wa->op.cmd_q = cmd_q; + wa->op.jobid = jobid; + wa->op.sb_key = cmd_q->sb_key; /* Pre-allocated */ + wa->op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ + wa->op.init = 1; + wa->op.u.aes.type = aes->type; + wa->op.u.aes.mode = CCP_AES_MODE_GHASH; + wa->op.u.aes.action = CCP_AES_GHASHFINAL; + wa->op.src.type = CCP_MEMTYPE_SYSTEM; + wa->op.src.u.dma.address = wa->final.dma.address; + wa->op.src.u.dma.length = AES_BLOCK_SIZE; + wa->op.dst.type = CCP_MEMTYPE_SYSTEM; + wa->op.dst.u.dma.address = wa->final.dma.address; + wa->op.dst.u.dma.length = AES_BLOCK_SIZE; + wa->op.eom = 1; + wa->op.u.aes.size = 0; + ret = cmd_q->ccp->vdata->perform->aes(&wa->op); if (ret) goto e_final_wa; if (aes->action == CCP_AES_ACTION_ENCRYPT) { /* Put the ciphered tag after the ciphertext. */ - ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize); + ccp_get_dm_area(&wa->final, 0, p_tag, 0, authsize); } else { /* Does this ciphered tag match the input? */ - ret = ccp_init_dm_workarea(&tag, cmd_q, authsize, + ret = ccp_init_dm_workarea(&wa->tag, cmd_q, authsize, DMA_BIDIRECTIONAL); if (ret) goto e_final_wa; - ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize); + ret = ccp_set_dm_area(&wa->tag, 0, p_tag, 0, authsize); if (ret) { - ccp_dm_free(&tag); + ccp_dm_free(&wa->tag); goto e_final_wa; } - ret = crypto_memneq(tag.address, final_wa.address, + ret = crypto_memneq(wa->tag.address, wa->final.address, authsize) ? -EBADMSG : 0; - ccp_dm_free(&tag); + ccp_dm_free(&wa->tag); } e_final_wa: - ccp_dm_free(&final_wa); + ccp_dm_free(&wa->final); e_dst: if (ilen > 0 && !in_place) - ccp_free_data(&dst, cmd_q); + ccp_free_data(&wa->dst, cmd_q); e_src: if (ilen > 0) - ccp_free_data(&src, cmd_q); + ccp_free_data(&wa->src, cmd_q); e_aad: if (aes->aad_len) - ccp_free_data(&aad, cmd_q); + ccp_free_data(&wa->aad, cmd_q); e_ctx: - ccp_dm_free(&ctx); + ccp_dm_free(&wa->ctx); e_key: - ccp_dm_free(&key); + ccp_dm_free(&wa->key); return ret; } diff --git a/drivers/crypto/ccp/dbc.c b/drivers/crypto/ccp/dbc.c index 5b105a23f699..410084a9039c 100644 --- a/drivers/crypto/ccp/dbc.c +++ b/drivers/crypto/ccp/dbc.c @@ -7,6 +7,8 @@ * Author: Mario Limonciello <mario.limonciello@amd.com> */ +#include <linux/mutex.h> + #include "dbc.h" #define DBC_DEFAULT_TIMEOUT (10 * MSEC_PER_SEC) @@ -137,64 +139,49 @@ static long dbc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return -ENODEV; dbc_dev = psp_master->dbc_data; - mutex_lock(&dbc_dev->ioctl_mutex); + guard(mutex)(&dbc_dev->ioctl_mutex); switch (cmd) { case DBCIOCNONCE: - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_nonce))) { - ret = -EFAULT; - goto unlock; - } + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_nonce))) + return -EFAULT; ret = send_dbc_nonce(dbc_dev); if (ret) - goto unlock; + return ret; - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_nonce))) { - ret = -EFAULT; - goto unlock; - } + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_nonce))) + return -EFAULT; break; case DBCIOCUID: - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_setuid))) { - ret = -EFAULT; - goto unlock; - } + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_setuid))) + return -EFAULT; *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_setuid); ret = send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_SET_UID); if (ret) - goto unlock; + return ret; - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_setuid))) { - ret = -EFAULT; - goto unlock; - } + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_setuid))) + return -EFAULT; break; case DBCIOCPARAM: - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_param))) { - ret = -EFAULT; - goto unlock; - } + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_param))) + return -EFAULT; *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_param); ret = send_dbc_parameter(dbc_dev); if (ret) - goto unlock; + return ret; - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_param))) { - ret = -EFAULT; - goto unlock; - } + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_param))) + return -EFAULT; break; default: - ret = -EINVAL; - + return -EINVAL; } -unlock: - mutex_unlock(&dbc_dev->ioctl_mutex); - return ret; + return 0; } static const struct file_operations dbc_fops = { diff --git a/drivers/crypto/ccp/hsti.c b/drivers/crypto/ccp/hsti.c index 1b39a4fb55c0..c29c6a9c0f3f 100644 --- a/drivers/crypto/ccp/hsti.c +++ b/drivers/crypto/ccp/hsti.c @@ -74,7 +74,7 @@ struct attribute_group psp_security_attr_group = { .is_visible = psp_security_is_visible, }; -static int psp_poulate_hsti(struct psp_device *psp) +static int psp_populate_hsti(struct psp_device *psp) { struct hsti_request *req; int ret; @@ -84,11 +84,11 @@ static int psp_poulate_hsti(struct psp_device *psp) return 0; /* Allocate command-response buffer */ - req = kzalloc(sizeof(*req), GFP_KERNEL | __GFP_ZERO); + req = kzalloc(sizeof(*req), GFP_KERNEL); if (!req) return -ENOMEM; - req->header.payload_size = sizeof(req); + req->header.payload_size = sizeof(*req); ret = psp_send_platform_access_msg(PSP_CMD_HSTI_QUERY, (struct psp_request *)req); if (ret) @@ -114,7 +114,7 @@ int psp_init_hsti(struct psp_device *psp) int ret; if (PSP_FEATURE(psp, HSTI)) { - ret = psp_poulate_hsti(psp); + ret = psp_populate_hsti(psp); if (ret) return ret; } diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 1c5a7189631e..9e21da0e298a 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -17,6 +17,7 @@ #include "psp-dev.h" #include "sev-dev.h" #include "tee-dev.h" +#include "sfs.h" #include "platform-access.h" #include "dbc.h" #include "hsti.h" @@ -182,6 +183,17 @@ static int psp_check_tee_support(struct psp_device *psp) return 0; } +static int psp_check_sfs_support(struct psp_device *psp) +{ + /* Check if device supports SFS feature */ + if (!psp->capability.sfs) { + dev_dbg(psp->dev, "psp does not support SFS\n"); + return -ENODEV; + } + + return 0; +} + static int psp_init(struct psp_device *psp) { int ret; @@ -198,6 +210,12 @@ static int psp_init(struct psp_device *psp) return ret; } + if (!psp_check_sfs_support(psp)) { + ret = sfs_dev_init(psp); + if (ret) + return ret; + } + if (psp->vdata->platform_access) { ret = platform_access_dev_init(psp); if (ret) @@ -302,6 +320,8 @@ void psp_dev_destroy(struct sp_device *sp) tee_dev_destroy(psp); + sfs_dev_destroy(psp); + dbc_dev_destroy(psp); platform_access_dev_destroy(psp); diff --git a/drivers/crypto/ccp/psp-dev.h b/drivers/crypto/ccp/psp-dev.h index e43ce87ede76..268c83f298cb 100644 --- a/drivers/crypto/ccp/psp-dev.h +++ b/drivers/crypto/ccp/psp-dev.h @@ -32,7 +32,8 @@ union psp_cap_register { unsigned int sev :1, tee :1, dbc_thru_ext :1, - rsvd1 :4, + sfs :1, + rsvd1 :3, security_reporting :1, fused_part :1, rsvd2 :1, @@ -68,6 +69,7 @@ struct psp_device { void *tee_data; void *platform_access_data; void *dbc_data; + void *sfs_data; union psp_cap_register capability; }; @@ -118,12 +120,16 @@ struct psp_ext_request { * @PSP_SUB_CMD_DBC_SET_UID: Set UID for DBC * @PSP_SUB_CMD_DBC_GET_PARAMETER: Get parameter from DBC * @PSP_SUB_CMD_DBC_SET_PARAMETER: Set parameter for DBC + * @PSP_SUB_CMD_SFS_GET_FW_VERS: Get firmware versions for ASP and other MP + * @PSP_SUB_CMD_SFS_UPDATE: Command to load, verify and execute SFS package */ enum psp_sub_cmd { PSP_SUB_CMD_DBC_GET_NONCE = PSP_DYNAMIC_BOOST_GET_NONCE, PSP_SUB_CMD_DBC_SET_UID = PSP_DYNAMIC_BOOST_SET_UID, PSP_SUB_CMD_DBC_GET_PARAMETER = PSP_DYNAMIC_BOOST_GET_PARAMETER, PSP_SUB_CMD_DBC_SET_PARAMETER = PSP_DYNAMIC_BOOST_SET_PARAMETER, + PSP_SUB_CMD_SFS_GET_FW_VERS = PSP_SFS_GET_FW_VERSIONS, + PSP_SUB_CMD_SFS_UPDATE = PSP_SFS_UPDATE, }; int psp_extended_mailbox_cmd(struct psp_device *psp, unsigned int timeout_msecs, diff --git a/drivers/crypto/ccp/sev-dev-tio.c b/drivers/crypto/ccp/sev-dev-tio.c new file mode 100644 index 000000000000..9a98f98c20a7 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tio.c @@ -0,0 +1,864 @@ +// SPDX-License-Identifier: GPL-2.0-only + +// Interface to PSP for CCP/SEV-TIO/SNP-VM + +#include <linux/pci.h> +#include <linux/tsm.h> +#include <linux/psp.h> +#include <linux/vmalloc.h> +#include <linux/bitfield.h> +#include <linux/pci-doe.h> +#include <asm/sev-common.h> +#include <asm/sev.h> +#include <asm/page.h> +#include "sev-dev.h" +#include "sev-dev-tio.h" + +#define to_tio_status(dev_data) \ + (container_of((dev_data), struct tio_dsm, data)->sev->tio_status) + +#define SLA_PAGE_TYPE_DATA 0 +#define SLA_PAGE_TYPE_SCATTER 1 +#define SLA_PAGE_SIZE_4K 0 +#define SLA_PAGE_SIZE_2M 1 +#define SLA_SZ(s) ((s).page_size == SLA_PAGE_SIZE_2M ? SZ_2M : SZ_4K) +#define SLA_SCATTER_LEN(s) (SLA_SZ(s) / sizeof(struct sla_addr_t)) +#define SLA_EOL ((struct sla_addr_t) { .pfn = ((1UL << 40) - 1) }) +#define SLA_NULL ((struct sla_addr_t) { 0 }) +#define IS_SLA_NULL(s) ((s).sla == SLA_NULL.sla) +#define IS_SLA_EOL(s) ((s).sla == SLA_EOL.sla) + +static phys_addr_t sla_to_pa(struct sla_addr_t sla) +{ + u64 pfn = sla.pfn; + u64 pa = pfn << PAGE_SHIFT; + + return pa; +} + +static void *sla_to_va(struct sla_addr_t sla) +{ + void *va = __va(__sme_clr(sla_to_pa(sla))); + + return va; +} + +#define sla_to_pfn(sla) (__pa(sla_to_va(sla)) >> PAGE_SHIFT) +#define sla_to_page(sla) virt_to_page(sla_to_va(sla)) + +static struct sla_addr_t make_sla(struct page *pg, bool stp) +{ + u64 pa = __sme_set(page_to_phys(pg)); + struct sla_addr_t ret = { + .pfn = pa >> PAGE_SHIFT, + .page_size = SLA_PAGE_SIZE_4K, /* Do not do SLA_PAGE_SIZE_2M ATM */ + .page_type = stp ? SLA_PAGE_TYPE_SCATTER : SLA_PAGE_TYPE_DATA + }; + + return ret; +} + +/* the BUFFER Structure */ +#define SLA_BUFFER_FLAG_ENCRYPTION BIT(0) + +/* + * struct sla_buffer_hdr - Scatter list address buffer header + * + * @capacity_sz: Total capacity of the buffer in bytes + * @payload_sz: Size of buffer payload in bytes, must be multiple of 32B + * @flags: Buffer flags (SLA_BUFFER_FLAG_ENCRYPTION: buffer is encrypted) + * @iv: Initialization vector used for encryption + * @authtag: Authentication tag for encrypted buffer + */ +struct sla_buffer_hdr { + u32 capacity_sz; + u32 payload_sz; /* The size of BUFFER_PAYLOAD in bytes. Must be multiple of 32B */ + u32 flags; + u8 reserved1[4]; + u8 iv[16]; /* IV used for the encryption of this buffer */ + u8 authtag[16]; /* Authentication tag for this buffer */ + u8 reserved2[16]; +} __packed; + +enum spdm_data_type_t { + DOBJ_DATA_TYPE_SPDM = 0x1, + DOBJ_DATA_TYPE_SECURE_SPDM = 0x2, +}; + +struct spdm_dobj_hdr_req { + struct spdm_dobj_hdr hdr; /* hdr.id == SPDM_DOBJ_ID_REQ */ + u8 data_type; /* spdm_data_type_t */ + u8 reserved2[5]; +} __packed; + +struct spdm_dobj_hdr_resp { + struct spdm_dobj_hdr hdr; /* hdr.id == SPDM_DOBJ_ID_RESP */ + u8 data_type; /* spdm_data_type_t */ + u8 reserved2[5]; +} __packed; + +/* Defined in sev-dev-tio.h so sev-dev-tsm.c can read types of blobs */ +struct spdm_dobj_hdr_cert; +struct spdm_dobj_hdr_meas; +struct spdm_dobj_hdr_report; + +/* Used in all SPDM-aware TIO commands */ +struct spdm_ctrl { + struct sla_addr_t req; + struct sla_addr_t resp; + struct sla_addr_t scratch; + struct sla_addr_t output; +} __packed; + +static size_t sla_dobj_id_to_size(u8 id) +{ + size_t n; + + BUILD_BUG_ON(sizeof(struct spdm_dobj_hdr_resp) != 0x10); + switch (id) { + case SPDM_DOBJ_ID_REQ: + n = sizeof(struct spdm_dobj_hdr_req); + break; + case SPDM_DOBJ_ID_RESP: + n = sizeof(struct spdm_dobj_hdr_resp); + break; + default: + WARN_ON(1); + n = 0; + break; + } + + return n; +} + +#define SPDM_DOBJ_HDR_SIZE(hdr) sla_dobj_id_to_size((hdr)->id) +#define SPDM_DOBJ_DATA(hdr) ((u8 *)(hdr) + SPDM_DOBJ_HDR_SIZE(hdr)) +#define SPDM_DOBJ_LEN(hdr) ((hdr)->length - SPDM_DOBJ_HDR_SIZE(hdr)) + +#define sla_to_dobj_resp_hdr(buf) ((struct spdm_dobj_hdr_resp *) \ + sla_to_dobj_hdr_check((buf), SPDM_DOBJ_ID_RESP)) +#define sla_to_dobj_req_hdr(buf) ((struct spdm_dobj_hdr_req *) \ + sla_to_dobj_hdr_check((buf), SPDM_DOBJ_ID_REQ)) + +static struct spdm_dobj_hdr *sla_to_dobj_hdr(struct sla_buffer_hdr *buf) +{ + if (!buf) + return NULL; + + return (struct spdm_dobj_hdr *) &buf[1]; +} + +static struct spdm_dobj_hdr *sla_to_dobj_hdr_check(struct sla_buffer_hdr *buf, u32 check_dobjid) +{ + struct spdm_dobj_hdr *hdr = sla_to_dobj_hdr(buf); + + if (WARN_ON_ONCE(!hdr)) + return NULL; + + if (hdr->id != check_dobjid) { + pr_err("! ERROR: expected %d, found %d\n", check_dobjid, hdr->id); + return NULL; + } + + return hdr; +} + +static void *sla_to_data(struct sla_buffer_hdr *buf, u32 dobjid) +{ + struct spdm_dobj_hdr *hdr = sla_to_dobj_hdr(buf); + + if (WARN_ON_ONCE(dobjid != SPDM_DOBJ_ID_REQ && dobjid != SPDM_DOBJ_ID_RESP)) + return NULL; + + if (!hdr) + return NULL; + + return (u8 *) hdr + sla_dobj_id_to_size(dobjid); +} + +/* + * struct sev_data_tio_status - SEV_CMD_TIO_STATUS command + * + * @length: Length of this command buffer in bytes + * @status_paddr: System physical address of the TIO_STATUS structure + */ +struct sev_data_tio_status { + u32 length; + u8 reserved[4]; + u64 status_paddr; +} __packed; + +/* TIO_INIT */ +struct sev_data_tio_init { + u32 length; + u8 reserved[12]; +} __packed; + +/* + * struct sev_data_tio_dev_create - TIO_DEV_CREATE command + * + * @length: Length in bytes of this command buffer + * @dev_ctx_sla: Scatter list address pointing to a buffer to be used as a device context buffer + * @device_id: PCIe Routing Identifier of the device to connect to + * @root_port_id: PCIe Routing Identifier of the root port of the device + * @segment_id: PCIe Segment Identifier of the device to connect to + */ +struct sev_data_tio_dev_create { + u32 length; + u8 reserved1[4]; + struct sla_addr_t dev_ctx_sla; + u16 device_id; + u16 root_port_id; + u8 segment_id; + u8 reserved2[11]; +} __packed; + +/* + * struct sev_data_tio_dev_connect - TIO_DEV_CONNECT command + * + * @length: Length in bytes of this command buffer + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + * @tc_mask: Bitmask of the traffic classes to initialize for SEV-TIO usage. + * Setting the kth bit of the TC_MASK to 1 indicates that the traffic + * class k will be initialized + * @cert_slot: Slot number of the certificate requested for constructing the SPDM session + * @ide_stream_id: IDE stream IDs to be associated with this device. + * Valid only if corresponding bit in TC_MASK is set + */ +struct sev_data_tio_dev_connect { + u32 length; + u8 reserved1[4]; + struct spdm_ctrl spdm_ctrl; + u8 reserved2[8]; + struct sla_addr_t dev_ctx_sla; + u8 tc_mask; + u8 cert_slot; + u8 reserved3[6]; + u8 ide_stream_id[8]; + u8 reserved4[8]; +} __packed; + +/* + * struct sev_data_tio_dev_disconnect - TIO_DEV_DISCONNECT command + * + * @length: Length in bytes of this command buffer + * @flags: Command flags (TIO_DEV_DISCONNECT_FLAG_FORCE: force disconnect) + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + */ +#define TIO_DEV_DISCONNECT_FLAG_FORCE BIT(0) + +struct sev_data_tio_dev_disconnect { + u32 length; + u32 flags; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; +} __packed; + +/* + * struct sev_data_tio_dev_meas - TIO_DEV_MEASUREMENTS command + * + * @length: Length in bytes of this command buffer + * @flags: Command flags (TIO_DEV_MEAS_FLAG_RAW_BITSTREAM: request raw measurements) + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + * @meas_nonce: Nonce for measurement freshness verification + */ +#define TIO_DEV_MEAS_FLAG_RAW_BITSTREAM BIT(0) + +struct sev_data_tio_dev_meas { + u32 length; + u32 flags; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; + u8 meas_nonce[32]; +} __packed; + +/* + * struct sev_data_tio_dev_certs - TIO_DEV_CERTIFICATES command + * + * @length: Length in bytes of this command buffer + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + */ +struct sev_data_tio_dev_certs { + u32 length; + u8 reserved[4]; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; +} __packed; + +/* + * struct sev_data_tio_dev_reclaim - TIO_DEV_RECLAIM command + * + * @length: Length in bytes of this command buffer + * @dev_ctx_sla: Scatter list address of the device context buffer + * + * This command reclaims resources associated with a device context. + */ +struct sev_data_tio_dev_reclaim { + u32 length; + u8 reserved[4]; + struct sla_addr_t dev_ctx_sla; +} __packed; + +static struct sla_buffer_hdr *sla_buffer_map(struct sla_addr_t sla) +{ + struct sla_buffer_hdr *buf; + + BUILD_BUG_ON(sizeof(struct sla_buffer_hdr) != 0x40); + if (IS_SLA_NULL(sla)) + return NULL; + + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + struct sla_addr_t *scatter = sla_to_va(sla); + unsigned int i, npages = 0; + + for (i = 0; i < SLA_SCATTER_LEN(sla); ++i) { + if (WARN_ON_ONCE(SLA_SZ(scatter[i]) > SZ_4K)) + return NULL; + + if (WARN_ON_ONCE(scatter[i].page_type == SLA_PAGE_TYPE_SCATTER)) + return NULL; + + if (IS_SLA_EOL(scatter[i])) { + npages = i; + break; + } + } + if (WARN_ON_ONCE(!npages)) + return NULL; + + struct page **pp = kmalloc_array(npages, sizeof(pp[0]), GFP_KERNEL); + + if (!pp) + return NULL; + + for (i = 0; i < npages; ++i) + pp[i] = sla_to_page(scatter[i]); + + buf = vm_map_ram(pp, npages, 0); + kfree(pp); + } else { + struct page *pg = sla_to_page(sla); + + buf = vm_map_ram(&pg, 1, 0); + } + + return buf; +} + +static void sla_buffer_unmap(struct sla_addr_t sla, struct sla_buffer_hdr *buf) +{ + if (!buf) + return; + + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + struct sla_addr_t *scatter = sla_to_va(sla); + unsigned int i, npages = 0; + + for (i = 0; i < SLA_SCATTER_LEN(sla); ++i) { + if (IS_SLA_EOL(scatter[i])) { + npages = i; + break; + } + } + if (!npages) + return; + + vm_unmap_ram(buf, npages); + } else { + vm_unmap_ram(buf, 1); + } +} + +static void dobj_response_init(struct sla_buffer_hdr *buf) +{ + struct spdm_dobj_hdr *dobj = sla_to_dobj_hdr(buf); + + dobj->id = SPDM_DOBJ_ID_RESP; + dobj->version.major = 0x1; + dobj->version.minor = 0; + dobj->length = 0; + buf->payload_sz = sla_dobj_id_to_size(dobj->id) + dobj->length; +} + +static void sla_free(struct sla_addr_t sla, size_t len, bool firmware_state) +{ + unsigned int npages = PAGE_ALIGN(len) >> PAGE_SHIFT; + struct sla_addr_t *scatter = NULL; + int ret = 0, i; + + if (IS_SLA_NULL(sla)) + return; + + if (firmware_state) { + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + scatter = sla_to_va(sla); + + for (i = 0; i < npages; ++i) { + if (IS_SLA_EOL(scatter[i])) + break; + + ret = snp_reclaim_pages(sla_to_pa(scatter[i]), 1, false); + if (ret) + break; + } + } else { + ret = snp_reclaim_pages(sla_to_pa(sla), 1, false); + } + } + + if (WARN_ON(ret)) + return; + + if (scatter) { + for (i = 0; i < npages; ++i) { + if (IS_SLA_EOL(scatter[i])) + break; + free_page((unsigned long)sla_to_va(scatter[i])); + } + } + + free_page((unsigned long)sla_to_va(sla)); +} + +static struct sla_addr_t sla_alloc(size_t len, bool firmware_state) +{ + unsigned long i, npages = PAGE_ALIGN(len) >> PAGE_SHIFT; + struct sla_addr_t *scatter = NULL; + struct sla_addr_t ret = SLA_NULL; + struct sla_buffer_hdr *buf; + struct page *pg; + + if (npages == 0) + return ret; + + if (WARN_ON_ONCE(npages > ((PAGE_SIZE / sizeof(struct sla_addr_t)) + 1))) + return ret; + + BUILD_BUG_ON(PAGE_SIZE < SZ_4K); + + if (npages > 1) { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + return SLA_NULL; + + ret = make_sla(pg, true); + scatter = page_to_virt(pg); + for (i = 0; i < npages; ++i) { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + goto no_reclaim_exit; + + scatter[i] = make_sla(pg, false); + } + scatter[i] = SLA_EOL; + } else { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + return SLA_NULL; + + ret = make_sla(pg, false); + } + + buf = sla_buffer_map(ret); + if (!buf) + goto no_reclaim_exit; + + buf->capacity_sz = (npages << PAGE_SHIFT); + sla_buffer_unmap(ret, buf); + + if (firmware_state) { + if (scatter) { + for (i = 0; i < npages; ++i) { + if (rmp_make_private(sla_to_pfn(scatter[i]), 0, + PG_LEVEL_4K, 0, true)) + goto free_exit; + } + } else { + if (rmp_make_private(sla_to_pfn(ret), 0, PG_LEVEL_4K, 0, true)) + goto no_reclaim_exit; + } + } + + return ret; + +no_reclaim_exit: + firmware_state = false; +free_exit: + sla_free(ret, len, firmware_state); + return SLA_NULL; +} + +/* Expands a buffer, only firmware owned buffers allowed for now */ +static int sla_expand(struct sla_addr_t *sla, size_t *len) +{ + struct sla_buffer_hdr *oldbuf = sla_buffer_map(*sla), *newbuf; + struct sla_addr_t oldsla = *sla, newsla; + size_t oldlen = *len, newlen; + + if (!oldbuf) + return -EFAULT; + + newlen = oldbuf->capacity_sz; + if (oldbuf->capacity_sz == oldlen) { + /* This buffer does not require expansion, must be another buffer */ + sla_buffer_unmap(oldsla, oldbuf); + return 1; + } + + pr_notice("Expanding BUFFER from %ld to %ld bytes\n", oldlen, newlen); + + newsla = sla_alloc(newlen, true); + if (IS_SLA_NULL(newsla)) + return -ENOMEM; + + newbuf = sla_buffer_map(newsla); + if (!newbuf) { + sla_free(newsla, newlen, true); + return -EFAULT; + } + + memcpy(newbuf, oldbuf, oldlen); + + sla_buffer_unmap(newsla, newbuf); + sla_free(oldsla, oldlen, true); + *sla = newsla; + *len = newlen; + + return 0; +} + +static int sev_tio_do_cmd(int cmd, void *data, size_t data_len, int *psp_ret, + struct tsm_dsm_tio *dev_data) +{ + int rc; + + *psp_ret = 0; + rc = sev_do_cmd(cmd, data, psp_ret); + + if (WARN_ON(!rc && *psp_ret == SEV_RET_SPDM_REQUEST)) + return -EIO; + + if (rc == 0 && *psp_ret == SEV_RET_EXPAND_BUFFER_LENGTH_REQUEST) { + int rc1, rc2; + + rc1 = sla_expand(&dev_data->output, &dev_data->output_len); + if (rc1 < 0) + return rc1; + + rc2 = sla_expand(&dev_data->scratch, &dev_data->scratch_len); + if (rc2 < 0) + return rc2; + + if (!rc1 && !rc2) + /* Neither buffer requires expansion, this is wrong */ + return -EFAULT; + + *psp_ret = 0; + rc = sev_do_cmd(cmd, data, psp_ret); + } + + if ((rc == 0 || rc == -EIO) && *psp_ret == SEV_RET_SPDM_REQUEST) { + struct spdm_dobj_hdr_resp *resp_hdr; + struct spdm_dobj_hdr_req *req_hdr; + struct sev_tio_status *tio_status = to_tio_status(dev_data); + size_t resp_len = tio_status->spdm_req_size_max - + (sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + sizeof(struct sla_buffer_hdr)); + + if (!dev_data->cmd) { + if (WARN_ON_ONCE(!data_len || (data_len != *(u32 *) data))) + return -EINVAL; + if (WARN_ON(data_len > sizeof(dev_data->cmd_data))) + return -EFAULT; + memcpy(dev_data->cmd_data, data, data_len); + memset(&dev_data->cmd_data[data_len], 0xFF, + sizeof(dev_data->cmd_data) - data_len); + dev_data->cmd = cmd; + } + + req_hdr = sla_to_dobj_req_hdr(dev_data->reqbuf); + resp_hdr = sla_to_dobj_resp_hdr(dev_data->respbuf); + switch (req_hdr->data_type) { + case DOBJ_DATA_TYPE_SPDM: + rc = PCI_DOE_FEATURE_CMA; + break; + case DOBJ_DATA_TYPE_SECURE_SPDM: + rc = PCI_DOE_FEATURE_SSESSION; + break; + default: + return -EINVAL; + } + resp_hdr->data_type = req_hdr->data_type; + dev_data->spdm.req_len = req_hdr->hdr.length - + sla_dobj_id_to_size(SPDM_DOBJ_ID_REQ); + dev_data->spdm.rsp_len = resp_len; + } else if (dev_data && dev_data->cmd) { + /* For either error or success just stop the bouncing */ + memset(dev_data->cmd_data, 0, sizeof(dev_data->cmd_data)); + dev_data->cmd = 0; + } + + return rc; +} + +int sev_tio_continue(struct tsm_dsm_tio *dev_data) +{ + struct spdm_dobj_hdr_resp *resp_hdr; + int ret; + + if (!dev_data || !dev_data->cmd) + return -EINVAL; + + resp_hdr = sla_to_dobj_resp_hdr(dev_data->respbuf); + resp_hdr->hdr.length = ALIGN(sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + + dev_data->spdm.rsp_len, 32); + dev_data->respbuf->payload_sz = resp_hdr->hdr.length; + + ret = sev_tio_do_cmd(dev_data->cmd, dev_data->cmd_data, 0, + &dev_data->psp_ret, dev_data); + if (ret) + return ret; + + if (dev_data->psp_ret != SEV_RET_SUCCESS) + return -EINVAL; + + return 0; +} + +static void spdm_ctrl_init(struct spdm_ctrl *ctrl, struct tsm_dsm_tio *dev_data) +{ + ctrl->req = dev_data->req; + ctrl->resp = dev_data->resp; + ctrl->scratch = dev_data->scratch; + ctrl->output = dev_data->output; +} + +static void spdm_ctrl_free(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + size_t len = tio_status->spdm_req_size_max - + (sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + + sizeof(struct sla_buffer_hdr)); + struct tsm_spdm *spdm = &dev_data->spdm; + + sla_buffer_unmap(dev_data->resp, dev_data->respbuf); + sla_buffer_unmap(dev_data->req, dev_data->reqbuf); + spdm->rsp = NULL; + spdm->req = NULL; + sla_free(dev_data->req, len, true); + sla_free(dev_data->resp, len, false); + sla_free(dev_data->scratch, tio_status->spdm_scratch_size_max, true); + + dev_data->req.sla = 0; + dev_data->resp.sla = 0; + dev_data->scratch.sla = 0; + dev_data->respbuf = NULL; + dev_data->reqbuf = NULL; + sla_free(dev_data->output, tio_status->spdm_out_size_max, true); +} + +static int spdm_ctrl_alloc(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct tsm_spdm *spdm = &dev_data->spdm; + int ret; + + dev_data->req = sla_alloc(tio_status->spdm_req_size_max, true); + dev_data->resp = sla_alloc(tio_status->spdm_req_size_max, false); + dev_data->scratch_len = tio_status->spdm_scratch_size_max; + dev_data->scratch = sla_alloc(dev_data->scratch_len, true); + dev_data->output_len = tio_status->spdm_out_size_max; + dev_data->output = sla_alloc(dev_data->output_len, true); + + if (IS_SLA_NULL(dev_data->req) || IS_SLA_NULL(dev_data->resp) || + IS_SLA_NULL(dev_data->scratch) || IS_SLA_NULL(dev_data->dev_ctx)) { + ret = -ENOMEM; + goto free_spdm_exit; + } + + dev_data->reqbuf = sla_buffer_map(dev_data->req); + dev_data->respbuf = sla_buffer_map(dev_data->resp); + if (!dev_data->reqbuf || !dev_data->respbuf) { + ret = -EFAULT; + goto free_spdm_exit; + } + + spdm->req = sla_to_data(dev_data->reqbuf, SPDM_DOBJ_ID_REQ); + spdm->rsp = sla_to_data(dev_data->respbuf, SPDM_DOBJ_ID_RESP); + if (!spdm->req || !spdm->rsp) { + ret = -EFAULT; + goto free_spdm_exit; + } + + dobj_response_init(dev_data->respbuf); + + return 0; + +free_spdm_exit: + spdm_ctrl_free(dev_data); + return ret; +} + +int sev_tio_init_locked(void *tio_status_page) +{ + struct sev_tio_status *tio_status = tio_status_page; + struct sev_data_tio_status data_status = { + .length = sizeof(data_status), + }; + int ret, psp_ret; + + data_status.status_paddr = __psp_pa(tio_status_page); + ret = __sev_do_cmd_locked(SEV_CMD_TIO_STATUS, &data_status, &psp_ret); + if (ret) + return ret; + + if (tio_status->length < offsetofend(struct sev_tio_status, tdictx_size) || + tio_status->reserved) + return -EFAULT; + + if (!tio_status->tio_en && !tio_status->tio_init_done) + return -ENOENT; + + if (tio_status->tio_init_done) + return -EBUSY; + + struct sev_data_tio_init ti = { .length = sizeof(ti) }; + + ret = __sev_do_cmd_locked(SEV_CMD_TIO_INIT, &ti, &psp_ret); + if (ret) + return ret; + + ret = __sev_do_cmd_locked(SEV_CMD_TIO_STATUS, &data_status, &psp_ret); + if (ret) + return ret; + + return 0; +} + +int sev_tio_dev_create(struct tsm_dsm_tio *dev_data, u16 device_id, + u16 root_port_id, u8 segment_id) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct sev_data_tio_dev_create create = { + .length = sizeof(create), + .device_id = device_id, + .root_port_id = root_port_id, + .segment_id = segment_id, + }; + void *data_pg; + int ret; + + dev_data->dev_ctx = sla_alloc(tio_status->devctx_size, true); + if (IS_SLA_NULL(dev_data->dev_ctx)) + return -ENOMEM; + + data_pg = snp_alloc_firmware_page(GFP_KERNEL_ACCOUNT); + if (!data_pg) { + ret = -ENOMEM; + goto free_ctx_exit; + } + + create.dev_ctx_sla = dev_data->dev_ctx; + ret = sev_do_cmd(SEV_CMD_TIO_DEV_CREATE, &create, &dev_data->psp_ret); + if (ret) + goto free_data_pg_exit; + + dev_data->data_pg = data_pg; + + return 0; + +free_data_pg_exit: + snp_free_firmware_page(data_pg); +free_ctx_exit: + sla_free(create.dev_ctx_sla, tio_status->devctx_size, true); + return ret; +} + +int sev_tio_dev_reclaim(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct sev_data_tio_dev_reclaim r = { + .length = sizeof(r), + .dev_ctx_sla = dev_data->dev_ctx, + }; + int ret; + + if (dev_data->data_pg) { + snp_free_firmware_page(dev_data->data_pg); + dev_data->data_pg = NULL; + } + + if (IS_SLA_NULL(dev_data->dev_ctx)) + return 0; + + ret = sev_do_cmd(SEV_CMD_TIO_DEV_RECLAIM, &r, &dev_data->psp_ret); + + sla_free(dev_data->dev_ctx, tio_status->devctx_size, true); + dev_data->dev_ctx = SLA_NULL; + + spdm_ctrl_free(dev_data); + + return ret; +} + +int sev_tio_dev_connect(struct tsm_dsm_tio *dev_data, u8 tc_mask, u8 ids[8], u8 cert_slot) +{ + struct sev_data_tio_dev_connect connect = { + .length = sizeof(connect), + .tc_mask = tc_mask, + .cert_slot = cert_slot, + .dev_ctx_sla = dev_data->dev_ctx, + .ide_stream_id = { + ids[0], ids[1], ids[2], ids[3], + ids[4], ids[5], ids[6], ids[7] + }, + }; + int ret; + + if (WARN_ON(IS_SLA_NULL(dev_data->dev_ctx))) + return -EFAULT; + if (!(tc_mask & 1)) + return -EINVAL; + + ret = spdm_ctrl_alloc(dev_data); + if (ret) + return ret; + + spdm_ctrl_init(&connect.spdm_ctrl, dev_data); + + return sev_tio_do_cmd(SEV_CMD_TIO_DEV_CONNECT, &connect, sizeof(connect), + &dev_data->psp_ret, dev_data); +} + +int sev_tio_dev_disconnect(struct tsm_dsm_tio *dev_data, bool force) +{ + struct sev_data_tio_dev_disconnect dc = { + .length = sizeof(dc), + .dev_ctx_sla = dev_data->dev_ctx, + .flags = force ? TIO_DEV_DISCONNECT_FLAG_FORCE : 0, + }; + + if (WARN_ON_ONCE(IS_SLA_NULL(dev_data->dev_ctx))) + return -EFAULT; + + spdm_ctrl_init(&dc.spdm_ctrl, dev_data); + + return sev_tio_do_cmd(SEV_CMD_TIO_DEV_DISCONNECT, &dc, sizeof(dc), + &dev_data->psp_ret, dev_data); +} + +int sev_tio_cmd_buffer_len(int cmd) +{ + switch (cmd) { + case SEV_CMD_TIO_STATUS: return sizeof(struct sev_data_tio_status); + case SEV_CMD_TIO_INIT: return sizeof(struct sev_data_tio_init); + case SEV_CMD_TIO_DEV_CREATE: return sizeof(struct sev_data_tio_dev_create); + case SEV_CMD_TIO_DEV_RECLAIM: return sizeof(struct sev_data_tio_dev_reclaim); + case SEV_CMD_TIO_DEV_CONNECT: return sizeof(struct sev_data_tio_dev_connect); + case SEV_CMD_TIO_DEV_DISCONNECT: return sizeof(struct sev_data_tio_dev_disconnect); + default: return 0; + } +} diff --git a/drivers/crypto/ccp/sev-dev-tio.h b/drivers/crypto/ccp/sev-dev-tio.h new file mode 100644 index 000000000000..67512b3dbc53 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tio.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __PSP_SEV_TIO_H__ +#define __PSP_SEV_TIO_H__ + +#include <linux/pci-tsm.h> +#include <linux/pci-ide.h> +#include <linux/tsm.h> +#include <uapi/linux/psp-sev.h> + +struct sla_addr_t { + union { + u64 sla; + struct { + u64 page_type :1, + page_size :1, + reserved1 :10, + pfn :40, + reserved2 :12; + }; + }; +} __packed; + +#define SEV_TIO_MAX_COMMAND_LENGTH 128 + +/* SPDM control structure for DOE */ +struct tsm_spdm { + unsigned long req_len; + void *req; + unsigned long rsp_len; + void *rsp; +}; + +/* Describes TIO device */ +struct tsm_dsm_tio { + u8 cert_slot; + struct sla_addr_t dev_ctx; + struct sla_addr_t req; + struct sla_addr_t resp; + struct sla_addr_t scratch; + struct sla_addr_t output; + size_t output_len; + size_t scratch_len; + struct tsm_spdm spdm; + struct sla_buffer_hdr *reqbuf; /* vmap'ed @req for DOE */ + struct sla_buffer_hdr *respbuf; /* vmap'ed @resp for DOE */ + + int cmd; + int psp_ret; + u8 cmd_data[SEV_TIO_MAX_COMMAND_LENGTH]; + void *data_pg; /* Data page for DEV_STATUS/TDI_STATUS/TDI_INFO/ASID_FENCE */ + +#define TIO_IDE_MAX_TC 8 + struct pci_ide *ide[TIO_IDE_MAX_TC]; +}; + +/* Describes TSM structure for PF0 pointed by pci_dev->tsm */ +struct tio_dsm { + struct pci_tsm_pf0 tsm; + struct tsm_dsm_tio data; + struct sev_device *sev; +}; + +/* Data object IDs */ +#define SPDM_DOBJ_ID_NONE 0 +#define SPDM_DOBJ_ID_REQ 1 +#define SPDM_DOBJ_ID_RESP 2 + +struct spdm_dobj_hdr { + u32 id; /* Data object type identifier */ + u32 length; /* Length of the data object, INCLUDING THIS HEADER */ + struct { /* Version of the data object structure */ + u8 minor; + u8 major; + } version; +} __packed; + +/** + * struct sev_tio_status - TIO_STATUS command's info_paddr buffer + * + * @length: Length of this structure in bytes + * @tio_en: Indicates that SNP_INIT_EX initialized the RMP for SEV-TIO + * @tio_init_done: Indicates TIO_INIT has been invoked + * @spdm_req_size_min: Minimum SPDM request buffer size in bytes + * @spdm_req_size_max: Maximum SPDM request buffer size in bytes + * @spdm_scratch_size_min: Minimum SPDM scratch buffer size in bytes + * @spdm_scratch_size_max: Maximum SPDM scratch buffer size in bytes + * @spdm_out_size_min: Minimum SPDM output buffer size in bytes + * @spdm_out_size_max: Maximum for the SPDM output buffer size in bytes + * @spdm_rsp_size_min: Minimum SPDM response buffer size in bytes + * @spdm_rsp_size_max: Maximum SPDM response buffer size in bytes + * @devctx_size: Size of a device context buffer in bytes + * @tdictx_size: Size of a TDI context buffer in bytes + * @tio_crypto_alg: TIO crypto algorithms supported + */ +struct sev_tio_status { + u32 length; + u32 tio_en :1, + tio_init_done :1, + reserved :30; + u32 spdm_req_size_min; + u32 spdm_req_size_max; + u32 spdm_scratch_size_min; + u32 spdm_scratch_size_max; + u32 spdm_out_size_min; + u32 spdm_out_size_max; + u32 spdm_rsp_size_min; + u32 spdm_rsp_size_max; + u32 devctx_size; + u32 tdictx_size; + u32 tio_crypto_alg; + u8 reserved2[12]; +} __packed; + +int sev_tio_init_locked(void *tio_status_page); +int sev_tio_continue(struct tsm_dsm_tio *dev_data); + +int sev_tio_dev_create(struct tsm_dsm_tio *dev_data, u16 device_id, u16 root_port_id, + u8 segment_id); +int sev_tio_dev_connect(struct tsm_dsm_tio *dev_data, u8 tc_mask, u8 ids[8], u8 cert_slot); +int sev_tio_dev_disconnect(struct tsm_dsm_tio *dev_data, bool force); +int sev_tio_dev_reclaim(struct tsm_dsm_tio *dev_data); + +#endif /* __PSP_SEV_TIO_H__ */ diff --git a/drivers/crypto/ccp/sev-dev-tsm.c b/drivers/crypto/ccp/sev-dev-tsm.c new file mode 100644 index 000000000000..ea29cd5d0ff9 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tsm.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0-only + +// Interface to CCP/SEV-TIO for generic PCIe TDISP module + +#include <linux/pci.h> +#include <linux/device.h> +#include <linux/tsm.h> +#include <linux/iommu.h> +#include <linux/pci-doe.h> +#include <linux/bitfield.h> +#include <linux/module.h> + +#include <asm/sev-common.h> +#include <asm/sev.h> + +#include "psp-dev.h" +#include "sev-dev.h" +#include "sev-dev-tio.h" + +MODULE_IMPORT_NS("PCI_IDE"); + +#define TIO_DEFAULT_NR_IDE_STREAMS 1 + +static uint nr_ide_streams = TIO_DEFAULT_NR_IDE_STREAMS; +module_param_named(ide_nr, nr_ide_streams, uint, 0644); +MODULE_PARM_DESC(ide_nr, "Set the maximum number of IDE streams per PHB"); + +#define dev_to_sp(dev) ((struct sp_device *)dev_get_drvdata(dev)) +#define dev_to_psp(dev) ((struct psp_device *)(dev_to_sp(dev)->psp_data)) +#define dev_to_sev(dev) ((struct sev_device *)(dev_to_psp(dev)->sev_data)) +#define tsm_dev_to_sev(tsmdev) dev_to_sev((tsmdev)->dev.parent) + +#define pdev_to_tio_dsm(pdev) (container_of((pdev)->tsm, struct tio_dsm, tsm.base_tsm)) + +static int sev_tio_spdm_cmd(struct tio_dsm *dsm, int ret) +{ + struct tsm_dsm_tio *dev_data = &dsm->data; + struct tsm_spdm *spdm = &dev_data->spdm; + + /* Check the main command handler response before entering the loop */ + if (ret == 0 && dev_data->psp_ret != SEV_RET_SUCCESS) + return -EINVAL; + + if (ret <= 0) + return ret; + + /* ret > 0 means "SPDM requested" */ + while (ret == PCI_DOE_FEATURE_CMA || ret == PCI_DOE_FEATURE_SSESSION) { + ret = pci_doe(dsm->tsm.doe_mb, PCI_VENDOR_ID_PCI_SIG, ret, + spdm->req, spdm->req_len, spdm->rsp, spdm->rsp_len); + if (ret < 0) + break; + + WARN_ON_ONCE(ret == 0); /* The response should never be empty */ + spdm->rsp_len = ret; + ret = sev_tio_continue(dev_data); + } + + return ret; +} + +static int stream_enable(struct pci_ide *ide) +{ + struct pci_dev *rp = pcie_find_root_port(ide->pdev); + int ret; + + ret = pci_ide_stream_enable(rp, ide); + if (ret) + return ret; + + ret = pci_ide_stream_enable(ide->pdev, ide); + if (ret) + pci_ide_stream_disable(rp, ide); + + return ret; +} + +static int streams_enable(struct pci_ide **ide) +{ + int ret = 0; + + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + ret = stream_enable(ide[i]); + if (ret) + break; + } + } + + return ret; +} + +static void stream_disable(struct pci_ide *ide) +{ + pci_ide_stream_disable(ide->pdev, ide); + pci_ide_stream_disable(pcie_find_root_port(ide->pdev), ide); +} + +static void streams_disable(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) + if (ide[i]) + stream_disable(ide[i]); +} + +static void stream_setup(struct pci_ide *ide) +{ + struct pci_dev *rp = pcie_find_root_port(ide->pdev); + + ide->partner[PCI_IDE_EP].rid_start = 0; + ide->partner[PCI_IDE_EP].rid_end = 0xffff; + ide->partner[PCI_IDE_RP].rid_start = 0; + ide->partner[PCI_IDE_RP].rid_end = 0xffff; + + ide->pdev->ide_cfg = 0; + ide->pdev->ide_tee_limit = 1; + rp->ide_cfg = 1; + rp->ide_tee_limit = 0; + + pci_warn(ide->pdev, "Forcing CFG/TEE for %s", pci_name(rp)); + pci_ide_stream_setup(ide->pdev, ide); + pci_ide_stream_setup(rp, ide); +} + +static u8 streams_setup(struct pci_ide **ide, u8 *ids) +{ + bool def = false; + u8 tc_mask = 0; + int i; + + for (i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (!ide[i]) { + ids[i] = 0xFF; + continue; + } + + tc_mask |= BIT(i); + ids[i] = ide[i]->stream_id; + + if (!def) { + struct pci_ide_partner *settings; + + settings = pci_ide_to_settings(ide[i]->pdev, ide[i]); + settings->default_stream = 1; + def = true; + } + + stream_setup(ide[i]); + } + + return tc_mask; +} + +static int streams_register(struct pci_ide **ide) +{ + int ret = 0, i; + + for (i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + ret = pci_ide_stream_register(ide[i]); + if (ret) + break; + } + } + + return ret; +} + +static void streams_unregister(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) + if (ide[i]) + pci_ide_stream_unregister(ide[i]); +} + +static void stream_teardown(struct pci_ide *ide) +{ + pci_ide_stream_teardown(ide->pdev, ide); + pci_ide_stream_teardown(pcie_find_root_port(ide->pdev), ide); +} + +static void streams_teardown(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + stream_teardown(ide[i]); + pci_ide_stream_free(ide[i]); + ide[i] = NULL; + } + } +} + +static int stream_alloc(struct pci_dev *pdev, struct pci_ide **ide, + unsigned int tc) +{ + struct pci_dev *rp = pcie_find_root_port(pdev); + struct pci_ide *ide1; + + if (ide[tc]) { + pci_err(pdev, "Stream for class=%d already registered", tc); + return -EBUSY; + } + + /* FIXME: find a better way */ + if (nr_ide_streams != TIO_DEFAULT_NR_IDE_STREAMS) + pci_notice(pdev, "Enable non-default %d streams", nr_ide_streams); + pci_ide_set_nr_streams(to_pci_host_bridge(rp->bus->bridge), nr_ide_streams); + + ide1 = pci_ide_stream_alloc(pdev); + if (!ide1) + return -EFAULT; + + /* Blindly assign streamid=0 to TC=0, and so on */ + ide1->stream_id = tc; + + ide[tc] = ide1; + + return 0; +} + +static struct pci_tsm *tio_pf0_probe(struct pci_dev *pdev, struct sev_device *sev) +{ + struct tio_dsm *dsm __free(kfree) = kzalloc(sizeof(*dsm), GFP_KERNEL); + int rc; + + if (!dsm) + return NULL; + + rc = pci_tsm_pf0_constructor(pdev, &dsm->tsm, sev->tsmdev); + if (rc) + return NULL; + + pci_dbg(pdev, "TSM enabled\n"); + dsm->sev = sev; + return &no_free_ptr(dsm)->tsm.base_tsm; +} + +static struct pci_tsm *dsm_probe(struct tsm_dev *tsmdev, struct pci_dev *pdev) +{ + struct sev_device *sev = tsm_dev_to_sev(tsmdev); + + if (is_pci_tsm_pf0(pdev)) + return tio_pf0_probe(pdev, sev); + return 0; +} + +static void dsm_remove(struct pci_tsm *tsm) +{ + struct pci_dev *pdev = tsm->pdev; + + pci_dbg(pdev, "TSM disabled\n"); + + if (is_pci_tsm_pf0(pdev)) { + struct tio_dsm *dsm = container_of(tsm, struct tio_dsm, tsm.base_tsm); + + pci_tsm_pf0_destructor(&dsm->tsm); + kfree(dsm); + } +} + +static int dsm_create(struct tio_dsm *dsm) +{ + struct pci_dev *pdev = dsm->tsm.base_tsm.pdev; + u8 segment_id = pdev->bus ? pci_domain_nr(pdev->bus) : 0; + struct pci_dev *rootport = pcie_find_root_port(pdev); + u16 device_id = pci_dev_id(pdev); + u16 root_port_id; + u32 lnkcap = 0; + + if (pci_read_config_dword(rootport, pci_pcie_cap(rootport) + PCI_EXP_LNKCAP, + &lnkcap)) + return -ENODEV; + + root_port_id = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); + + return sev_tio_dev_create(&dsm->data, device_id, root_port_id, segment_id); +} + +static int dsm_connect(struct pci_dev *pdev) +{ + struct tio_dsm *dsm = pdev_to_tio_dsm(pdev); + struct tsm_dsm_tio *dev_data = &dsm->data; + u8 ids[TIO_IDE_MAX_TC]; + u8 tc_mask; + int ret; + + if (pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_PCI_SIG, + PCI_DOE_FEATURE_SSESSION) != dsm->tsm.doe_mb) { + pci_err(pdev, "CMA DOE MB must support SSESSION\n"); + return -EFAULT; + } + + ret = stream_alloc(pdev, dev_data->ide, 0); + if (ret) + return ret; + + ret = dsm_create(dsm); + if (ret) + goto ide_free_exit; + + tc_mask = streams_setup(dev_data->ide, ids); + + ret = sev_tio_dev_connect(dev_data, tc_mask, ids, dev_data->cert_slot); + ret = sev_tio_spdm_cmd(dsm, ret); + if (ret) + goto free_exit; + + streams_enable(dev_data->ide); + + ret = streams_register(dev_data->ide); + if (ret) + goto free_exit; + + return 0; + +free_exit: + sev_tio_dev_reclaim(dev_data); + + streams_disable(dev_data->ide); +ide_free_exit: + + streams_teardown(dev_data->ide); + + return ret; +} + +static void dsm_disconnect(struct pci_dev *pdev) +{ + bool force = SYSTEM_HALT <= system_state && system_state <= SYSTEM_RESTART; + struct tio_dsm *dsm = pdev_to_tio_dsm(pdev); + struct tsm_dsm_tio *dev_data = &dsm->data; + int ret; + + ret = sev_tio_dev_disconnect(dev_data, force); + ret = sev_tio_spdm_cmd(dsm, ret); + if (ret && !force) { + ret = sev_tio_dev_disconnect(dev_data, true); + sev_tio_spdm_cmd(dsm, ret); + } + + sev_tio_dev_reclaim(dev_data); + + streams_disable(dev_data->ide); + streams_unregister(dev_data->ide); + streams_teardown(dev_data->ide); +} + +static struct pci_tsm_ops sev_tsm_ops = { + .probe = dsm_probe, + .remove = dsm_remove, + .connect = dsm_connect, + .disconnect = dsm_disconnect, +}; + +void sev_tsm_init_locked(struct sev_device *sev, void *tio_status_page) +{ + struct sev_tio_status *t = kzalloc(sizeof(*t), GFP_KERNEL); + struct tsm_dev *tsmdev; + int ret; + + WARN_ON(sev->tio_status); + + if (!t) + return; + + ret = sev_tio_init_locked(tio_status_page); + if (ret) { + pr_warn("SEV-TIO STATUS failed with %d\n", ret); + goto error_exit; + } + + tsmdev = tsm_register(sev->dev, &sev_tsm_ops); + if (IS_ERR(tsmdev)) + goto error_exit; + + memcpy(t, tio_status_page, sizeof(*t)); + + pr_notice("SEV-TIO status: EN=%d INIT_DONE=%d rq=%d..%d rs=%d..%d " + "scr=%d..%d out=%d..%d dev=%d tdi=%d algos=%x\n", + t->tio_en, t->tio_init_done, + t->spdm_req_size_min, t->spdm_req_size_max, + t->spdm_rsp_size_min, t->spdm_rsp_size_max, + t->spdm_scratch_size_min, t->spdm_scratch_size_max, + t->spdm_out_size_min, t->spdm_out_size_max, + t->devctx_size, t->tdictx_size, + t->tio_crypto_alg); + + sev->tsmdev = tsmdev; + sev->tio_status = t; + + return; + +error_exit: + kfree(t); + pr_err("Failed to enable SEV-TIO: ret=%d en=%d initdone=%d SEV=%d\n", + ret, t->tio_en, t->tio_init_done, boot_cpu_has(X86_FEATURE_SEV)); +} + +void sev_tsm_uninit(struct sev_device *sev) +{ + if (sev->tsmdev) + tsm_unregister(sev->tsmdev); + + sev->tsmdev = NULL; +} diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index af018afd9cd7..956ea609d0cc 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -28,11 +28,13 @@ #include <linux/fs_struct.h> #include <linux/psp.h> #include <linux/amd-iommu.h> +#include <linux/crash_dump.h> #include <asm/smp.h> #include <asm/cacheflush.h> #include <asm/e820/types.h> #include <asm/sev.h> +#include <asm/msr.h> #include "psp-dev.h" #include "sev-dev.h" @@ -73,6 +75,14 @@ static bool psp_init_on_probe = true; module_param(psp_init_on_probe, bool, 0444); MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it"); +#if IS_ENABLED(CONFIG_PCI_TSM) +static bool sev_tio_enabled = true; +module_param_named(tio, sev_tio_enabled, bool, 0444); +MODULE_PARM_DESC(tio, "Enables TIO in SNP_INIT_EX"); +#else +static const bool sev_tio_enabled = false; +#endif + MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */ MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */ MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */ @@ -81,6 +91,21 @@ MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */ static bool psp_dead; static int psp_timeout; +enum snp_hv_fixed_pages_state { + ALLOCATED, + HV_FIXED, +}; + +struct snp_hv_fixed_pages_entry { + struct list_head list; + struct page *page; + unsigned int order; + bool free; + enum snp_hv_fixed_pages_state page_state; +}; + +static LIST_HEAD(snp_hv_fixed_pages); + /* Trusted Memory Region (TMR): * The TMR is a 1MB area that must be 1MB aligned. Use the page allocator * to allocate the memory, which will return aligned memory for the specified @@ -109,6 +134,15 @@ static void *sev_init_ex_buffer; */ static struct sev_data_range_list *snp_range_list; +static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); + +static int snp_shutdown_on_panic(struct notifier_block *nb, + unsigned long reason, void *arg); + +static struct notifier_block snp_panic_notifier = { + .notifier_call = snp_shutdown_on_panic, +}; + static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { struct sev_device *sev = psp_master->sev_data; @@ -223,7 +257,9 @@ static int sev_cmd_buffer_len(int cmd) case SEV_CMD_SNP_GUEST_REQUEST: return sizeof(struct sev_data_snp_guest_request); case SEV_CMD_SNP_CONFIG: return sizeof(struct sev_user_data_snp_config); case SEV_CMD_SNP_COMMIT: return sizeof(struct sev_data_snp_commit); - default: return 0; + case SEV_CMD_SNP_FEATURE_INFO: return sizeof(struct sev_data_snp_feature_info); + case SEV_CMD_SNP_VLEK_LOAD: return sizeof(struct sev_user_data_snp_vlek_load); + default: return sev_tio_cmd_buffer_len(cmd); } return 0; @@ -231,27 +267,20 @@ static int sev_cmd_buffer_len(int cmd) static struct file *open_file_as_root(const char *filename, int flags, umode_t mode) { - struct file *fp; - struct path root; - struct cred *cred; - const struct cred *old_cred; + struct path root __free(path_put) = {}; task_lock(&init_task); get_fs_root(init_task.fs, &root); task_unlock(&init_task); - cred = prepare_creds(); + CLASS(prepare_creds, cred)(); if (!cred) return ERR_PTR(-ENOMEM); - cred->fsuid = GLOBAL_ROOT_UID; - old_cred = override_creds(cred); - fp = file_open_root(&root, filename, flags, mode); - path_put(&root); - - revert_creds(old_cred); + cred->fsuid = GLOBAL_ROOT_UID; - return fp; + scoped_with_creds(cred) + return file_open_root(&root, filename, flags, mode); } static int sev_read_init_ex_file(void) @@ -359,13 +388,7 @@ static int sev_write_init_ex_file_if_required(int cmd_id) return sev_write_init_ex_file(); } -/* - * snp_reclaim_pages() needs __sev_do_cmd_locked(), and __sev_do_cmd_locked() - * needs snp_reclaim_pages(), so a forward declaration is needed. - */ -static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); - -static int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked) +int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked) { int ret, err, i; @@ -399,6 +422,7 @@ cleanup: snp_leak_pages(__phys_to_pfn(paddr), npages - i); return ret; } +EXPORT_SYMBOL_GPL(snp_reclaim_pages); static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked) { @@ -424,7 +448,7 @@ cleanup: return rc; } -static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order) +static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order, bool locked) { unsigned long npages = 1ul << order, paddr; struct sev_device *sev; @@ -443,7 +467,7 @@ static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order) return page; paddr = __pa((unsigned long)page_address(page)); - if (rmp_mark_pages_firmware(paddr, npages, false)) + if (rmp_mark_pages_firmware(paddr, npages, locked)) return NULL; return page; @@ -453,7 +477,7 @@ void *snp_alloc_firmware_page(gfp_t gfp_mask) { struct page *page; - page = __snp_alloc_firmware_pages(gfp_mask, 0); + page = __snp_alloc_firmware_pages(gfp_mask, 0, false); return page ? page_address(page) : NULL; } @@ -488,7 +512,7 @@ static void *sev_fw_alloc(unsigned long len) { struct page *page; - page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len)); + page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len), true); if (!page) return NULL; @@ -829,16 +853,17 @@ static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf) return 0; } -static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) +int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) { struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0}; struct psp_device *psp = psp_master; struct sev_device *sev; unsigned int cmdbuff_hi, cmdbuff_lo; unsigned int phys_lsb, phys_msb; - unsigned int reg, ret = 0; + unsigned int reg; void *cmd_buf; int buf_len; + int ret = 0; if (!psp || !psp->sev_data) return -ENODEV; @@ -1060,7 +1085,248 @@ static inline int __sev_do_init_locked(int *psp_ret) static void snp_set_hsave_pa(void *arg) { - wrmsrl(MSR_VM_HSAVE_PA, 0); + wrmsrq(MSR_VM_HSAVE_PA, 0); +} + +/* Hypervisor Fixed pages API interface */ +static void snp_hv_fixed_pages_state_update(struct sev_device *sev, + enum snp_hv_fixed_pages_state page_state) +{ + struct snp_hv_fixed_pages_entry *entry; + + /* List is protected by sev_cmd_mutex */ + lockdep_assert_held(&sev_cmd_mutex); + + if (list_empty(&snp_hv_fixed_pages)) + return; + + list_for_each_entry(entry, &snp_hv_fixed_pages, list) + entry->page_state = page_state; +} + +/* + * Allocate HV_FIXED pages in 2MB aligned sizes to ensure the whole + * 2MB pages are marked as HV_FIXED. + */ +struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages) +{ + struct psp_device *psp_master = psp_get_master_device(); + struct snp_hv_fixed_pages_entry *entry; + struct sev_device *sev; + unsigned int order; + struct page *page; + + if (!psp_master || !psp_master->sev_data) + return NULL; + + sev = psp_master->sev_data; + + order = get_order(PMD_SIZE * num_2mb_pages); + + /* + * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list + * also needs to be protected using the same mutex. + */ + guard(mutex)(&sev_cmd_mutex); + + /* + * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed + * page state, fail if SNP is already initialized. + */ + if (sev->snp_initialized) + return NULL; + + /* Re-use freed pages that match the request */ + list_for_each_entry(entry, &snp_hv_fixed_pages, list) { + /* Hypervisor fixed page allocator implements exact fit policy */ + if (entry->order == order && entry->free) { + entry->free = false; + memset(page_address(entry->page), 0, + (1 << entry->order) * PAGE_SIZE); + return entry->page; + } + } + + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!page) + return NULL; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + __free_pages(page, order); + return NULL; + } + + entry->page = page; + entry->order = order; + list_add_tail(&entry->list, &snp_hv_fixed_pages); + + return page; +} + +void snp_free_hv_fixed_pages(struct page *page) +{ + struct psp_device *psp_master = psp_get_master_device(); + struct snp_hv_fixed_pages_entry *entry, *nentry; + + if (!psp_master || !psp_master->sev_data) + return; + + /* + * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list + * also needs to be protected using the same mutex. + */ + guard(mutex)(&sev_cmd_mutex); + + list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) { + if (entry->page != page) + continue; + + /* + * HV_FIXED page state cannot be changed until reboot + * and they cannot be used by an SNP guest, so they cannot + * be returned back to the page allocator. + * Mark the pages as free internally to allow possible re-use. + */ + if (entry->page_state == HV_FIXED) { + entry->free = true; + } else { + __free_pages(page, entry->order); + list_del(&entry->list); + kfree(entry); + } + return; + } +} + +static void snp_add_hv_fixed_pages(struct sev_device *sev, struct sev_data_range_list *range_list) +{ + struct snp_hv_fixed_pages_entry *entry; + struct sev_data_range *range; + int num_elements; + + lockdep_assert_held(&sev_cmd_mutex); + + if (list_empty(&snp_hv_fixed_pages)) + return; + + num_elements = list_count_nodes(&snp_hv_fixed_pages) + + range_list->num_elements; + + /* + * Ensure the list of HV_FIXED pages that will be passed to firmware + * do not exceed the page-sized argument buffer. + */ + if (num_elements * sizeof(*range) + sizeof(*range_list) > PAGE_SIZE) { + dev_warn(sev->dev, "Additional HV_Fixed pages cannot be accommodated, omitting\n"); + return; + } + + range = &range_list->ranges[range_list->num_elements]; + list_for_each_entry(entry, &snp_hv_fixed_pages, list) { + range->base = page_to_pfn(entry->page) << PAGE_SHIFT; + range->page_count = 1 << entry->order; + range++; + } + range_list->num_elements = num_elements; +} + +static void snp_leak_hv_fixed_pages(void) +{ + struct snp_hv_fixed_pages_entry *entry; + + /* List is protected by sev_cmd_mutex */ + lockdep_assert_held(&sev_cmd_mutex); + + if (list_empty(&snp_hv_fixed_pages)) + return; + + list_for_each_entry(entry, &snp_hv_fixed_pages, list) + if (entry->page_state == HV_FIXED) + __snp_leak_pages(page_to_pfn(entry->page), + 1 << entry->order, false); +} + +bool sev_is_snp_ciphertext_hiding_supported(void) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + + if (!psp || !psp->sev_data) + return false; + + sev = psp->sev_data; + + /* + * Feature information indicates if CipherTextHiding feature is + * supported by the SEV firmware and additionally platform status + * indicates if CipherTextHiding feature is enabled in the + * Platform BIOS. + */ + return ((sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) && + sev->snp_plat_status.ciphertext_hiding_cap); +} +EXPORT_SYMBOL_GPL(sev_is_snp_ciphertext_hiding_supported); + +static int snp_get_platform_data(struct sev_device *sev, int *error) +{ + struct sev_data_snp_feature_info snp_feat_info; + struct snp_feature_info *feat_info; + struct sev_data_snp_addr buf; + struct page *page; + int rc; + + /* + * This function is expected to be called before SNP is + * initialized. + */ + if (sev->snp_initialized) + return -EINVAL; + + buf.address = __psp_pa(&sev->snp_plat_status); + rc = sev_do_cmd(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error); + if (rc) { + dev_err(sev->dev, "SNP PLATFORM_STATUS command failed, ret = %d, error = %#x\n", + rc, *error); + return rc; + } + + sev->api_major = sev->snp_plat_status.api_major; + sev->api_minor = sev->snp_plat_status.api_minor; + sev->build = sev->snp_plat_status.build_id; + + /* + * Do feature discovery of the currently loaded firmware, + * and cache feature information from CPUID 0x8000_0024, + * sub-function 0. + */ + if (!sev->snp_plat_status.feature_info) + return 0; + + /* + * Use dynamically allocated structure for the SNP_FEATURE_INFO + * command to ensure structure is 8-byte aligned, and does not + * cross a page boundary. + */ + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + + feat_info = page_address(page); + snp_feat_info.length = sizeof(snp_feat_info); + snp_feat_info.ecx_in = 0; + snp_feat_info.feature_info_paddr = __psp_pa(feat_info); + + rc = sev_do_cmd(SEV_CMD_SNP_FEATURE_INFO, &snp_feat_info, error); + if (!rc) + sev->snp_feat_info_0 = *feat_info; + else + dev_err(sev->dev, "SNP FEATURE_INFO command failed, ret = %d, error = %#x\n", + rc, *error); + + __free_page(page); + + return rc; } static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) @@ -1093,7 +1359,7 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) return 0; } -static int __sev_snp_init_locked(int *error) +static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { struct psp_device *psp = psp_master; struct sev_data_snp_init_ex data; @@ -1112,7 +1378,7 @@ static int __sev_snp_init_locked(int *error) if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) { dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n", SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR); - return 0; + return -EOPNOTSUPP; } /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ @@ -1129,6 +1395,8 @@ static int __sev_snp_init_locked(int *error) * */ if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) { + bool tio_supp = !!(sev->snp_feat_info_0.ebx & SNP_SEV_TIO_SUPPORTED); + /* * Firmware checks that the pages containing the ranges enumerated * in the RANGES structure are either in the default page state or in the @@ -1153,10 +1421,33 @@ static int __sev_snp_init_locked(int *error) return rc; } + /* + * Add HV_Fixed pages from other PSP sub-devices, such as SFS to the + * HV_Fixed page list. + */ + snp_add_hv_fixed_pages(sev, snp_range_list); + memset(&data, 0, sizeof(data)); + + if (max_snp_asid) { + data.ciphertext_hiding_en = 1; + data.max_snp_asid = max_snp_asid; + } + data.init_rmp = 1; data.list_paddr_en = 1; data.list_paddr = __psp_pa(snp_range_list); + + data.tio_en = tio_supp && sev_tio_enabled && amd_iommu_sev_tio_supported(); + + /* + * When psp_init_on_probe is disabled, the userspace calling + * SEV ioctl can inadvertently shut down SNP and SEV-TIO causing + * unexpected state loss. + */ + if (data.tio_en && !psp_init_on_probe) + dev_warn(sev->dev, "SEV-TIO as incompatible with psp_init_on_probe=0\n"); + cmd = SEV_CMD_SNP_INIT_EX; } else { cmd = SEV_CMD_SNP_INIT; @@ -1176,21 +1467,53 @@ static int __sev_snp_init_locked(int *error) wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(cmd, arg, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n", + cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT", + rc, *error); return rc; + } /* Prepare for first SNP guest launch after INIT. */ wbinvd_on_all_cpus(); rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error); - if (rc) + if (rc) { + dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n", + rc, *error); return rc; + } + snp_hv_fixed_pages_state_update(sev, HV_FIXED); sev->snp_initialized = true; - dev_dbg(sev->dev, "SEV-SNP firmware initialized\n"); + dev_dbg(sev->dev, "SEV-SNP firmware initialized, SEV-TIO is %s\n", + data.tio_en ? "enabled" : "disabled"); + + dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major, + sev->api_minor, sev->build); + + atomic_notifier_chain_register(&panic_notifier_list, + &snp_panic_notifier); + + if (data.tio_en) { + /* + * This executes with the sev_cmd_mutex held so down the stack + * snp_reclaim_pages(locked=false) might be needed (which is extremely + * unlikely) but will cause a deadlock. + * Instead of exporting __snp_alloc_firmware_pages(), allocate a page + * for this one call here. + */ + void *tio_status = page_address(__snp_alloc_firmware_pages( + GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0, true)); + + if (tio_status) { + sev_tsm_init_locked(sev, tio_status); + __snp_free_firmware_pages(virt_to_page(tio_status), 0, true); + } + } sev_es_tmr_size = SNP_TMR_SIZE; - return rc; + return 0; } static void __sev_platform_init_handle_tmr(struct sev_device *sev) @@ -1253,15 +1576,17 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) static int __sev_platform_init_locked(int *error) { - int rc, psp_ret = SEV_RET_NO_FW_CALL; + int rc, psp_ret, dfflush_error; struct sev_device *sev; + psp_ret = dfflush_error = SEV_RET_NO_FW_CALL; + if (!psp_master || !psp_master->sev_data) return -ENODEV; sev = psp_master->sev_data; - if (sev->state == SEV_STATE_INIT) + if (sev->sev_plat_status.state == SEV_STATE_INIT) return 0; __sev_platform_init_handle_tmr(sev); @@ -1287,16 +1612,22 @@ static int __sev_platform_init_locked(int *error) if (error) *error = psp_ret; - if (rc) + if (rc) { + dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n", + sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc); return rc; + } - sev->state = SEV_STATE_INIT; + sev->sev_plat_status.state = SEV_STATE_INIT; /* Prepare for first SEV guest launch after INIT */ wbinvd_on_all_cpus(); - rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, error); - if (rc) + rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, &dfflush_error); + if (rc) { + dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n", + dfflush_error, rc); return rc; + } dev_dbg(sev->dev, "SEV firmware initialized\n"); @@ -1314,24 +1645,23 @@ static int _sev_platform_init_locked(struct sev_platform_init_args *args) if (!psp_master || !psp_master->sev_data) return -ENODEV; + /* + * Skip SNP/SEV initialization under a kdump kernel as SEV/SNP + * may already be initialized in the previous kernel. Since no + * SNP/SEV guests are run under a kdump kernel, there is no + * need to initialize SNP or SEV during kdump boot. + */ + if (is_kdump_kernel()) + return 0; + sev = psp_master->sev_data; - if (sev->state == SEV_STATE_INIT) + if (sev->sev_plat_status.state == SEV_STATE_INIT) return 0; - /* - * Legacy guests cannot be running while SNP_INIT(_EX) is executing, - * so perform SEV-SNP initialization at probe time. - */ - rc = __sev_snp_init_locked(&args->error); - if (rc && rc != -ENODEV) { - /* - * Don't abort the probe if SNP INIT failed, - * continue to initialize the legacy SEV firmware. - */ - dev_err(sev->dev, "SEV-SNP: failed to INIT rc %d, error %#x\n", - rc, args->error); - } + rc = __sev_snp_init_locked(&args->error, args->max_snp_asid); + if (rc && rc != -ENODEV) + return rc; /* Defer legacy SEV/SEV-ES support if allowed by caller/module. */ if (args->probe && !psp_init_on_probe) @@ -1363,14 +1693,17 @@ static int __sev_platform_shutdown_locked(int *error) sev = psp->sev_data; - if (sev->state == SEV_STATE_UNINIT) + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) return 0; ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error); - if (ret) + if (ret) { + dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n", + *error, ret); return ret; + } - sev->state = SEV_STATE_UNINIT; + sev->sev_plat_status.state = SEV_STATE_UNINIT; dev_dbg(sev->dev, "SEV firmware shutdown\n"); return ret; @@ -1389,6 +1722,37 @@ static int sev_get_platform_state(int *state, int *error) return rc; } +static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) +{ + struct sev_platform_init_args init_args = {0}; + int rc; + + rc = _sev_platform_init_locked(&init_args); + if (rc) { + argp->error = SEV_RET_INVALID_PLATFORM_STATE; + return rc; + } + + *shutdown_required = true; + + return 0; +} + +static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) +{ + int error, rc; + + rc = __sev_snp_init_locked(&error, 0); + if (rc) { + argp->error = SEV_RET_INVALID_PLATFORM_STATE; + return rc; + } + + *shutdown_required = true; + + return 0; +} + static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable) { int state, rc; @@ -1441,24 +1805,31 @@ static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp) static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; + bool shutdown_required = false; int rc; if (!writable) return -EPERM; - if (sev->state == SEV_STATE_UNINIT) { - rc = __sev_platform_init_locked(&argp->error); + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) { + rc = sev_move_to_init_state(argp, &shutdown_required); if (rc) return rc; } - return __sev_do_cmd_locked(cmd, NULL, &argp->error); + rc = __sev_do_cmd_locked(cmd, NULL, &argp->error); + + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + + return rc; } static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_pek_csr input; + bool shutdown_required = false; struct sev_data_pek_csr data; void __user *input_address; void *blob = NULL; @@ -1489,8 +1860,8 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) data.len = input.length; cmd: - if (sev->state == SEV_STATE_UNINIT) { - ret = __sev_platform_init_locked(&argp->error); + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) { + ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_blob; } @@ -1511,6 +1882,9 @@ cmd: } e_free_blob: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(blob); return ret; } @@ -1534,6 +1908,16 @@ static int sev_get_api_version(void) struct sev_user_data_status status; int error = 0, ret; + /* + * Cache SNP platform status and SNP feature information + * if SNP is available. + */ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) { + ret = snp_get_platform_data(sev, &error); + if (ret) + return 1; + } + ret = sev_platform_status(&status, &error); if (ret) { dev_err(sev->dev, @@ -1541,10 +1925,12 @@ static int sev_get_api_version(void) return 1; } + /* Cache SEV platform status */ + sev->sev_plat_status = status; + sev->api_major = status.api_major; sev->api_minor = status.api_minor; sev->build = status.build; - sev->state = status.state; return 0; } @@ -1682,9 +2068,12 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error); /* SHUTDOWN may require DF_FLUSH */ if (*error == SEV_RET_DFFLUSH_REQUIRED) { - ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, NULL); + int dfflush_error = SEV_RET_NO_FW_CALL; + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error); if (ret) { - dev_err(sev->dev, "SEV-SNP DF_FLUSH failed\n"); + dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n", + ret, dfflush_error); return ret; } /* reissue the shutdown command */ @@ -1692,7 +2081,8 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) error); } if (ret) { - dev_err(sev->dev, "SEV-SNP firmware shutdown failed\n"); + dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n", + ret, *error); return ret; } @@ -1715,9 +2105,22 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) return ret; } + snp_leak_hv_fixed_pages(); sev->snp_initialized = false; dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n"); + /* + * __sev_snp_shutdown_locked() deadlocks when it tries to unregister + * itself during panic as the panic notifier is called with RCU read + * lock held and notifier unregistration does RCU synchronization. + */ + if (!panic) + atomic_notifier_chain_unregister(&panic_notifier_list, + &snp_panic_notifier); + + /* Reset TMR size back to default */ + sev_es_tmr_size = SEV_TMR_SIZE; + return ret; } @@ -1726,6 +2129,7 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) struct sev_device *sev = psp_master->sev_data; struct sev_user_data_pek_cert_import input; struct sev_data_pek_cert_import data; + bool shutdown_required = false; void *pek_blob, *oca_blob; int ret; @@ -1755,8 +2159,8 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) data.oca_cert_len = input.oca_cert_len; /* If platform is not in INIT state then transition it to INIT */ - if (sev->state != SEV_STATE_INIT) { - ret = __sev_platform_init_locked(&argp->error); + if (sev->sev_plat_status.state != SEV_STATE_INIT) { + ret = sev_move_to_init_state(argp, &shutdown_required); if (ret) goto e_free_oca; } @@ -1764,6 +2168,9 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error); e_free_oca: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(oca_blob); e_free_pek: kfree(pek_blob); @@ -1880,32 +2287,23 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) struct sev_data_pdh_cert_export data; void __user *input_cert_chain_address; void __user *input_pdh_cert_address; + bool shutdown_required = false; int ret; - /* If platform is not in INIT state then transition it to INIT. */ - if (sev->state != SEV_STATE_INIT) { - if (!writable) - return -EPERM; - - ret = __sev_platform_init_locked(&argp->error); - if (ret) - return ret; - } - if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) return -EFAULT; memset(&data, 0, sizeof(data)); + input_pdh_cert_address = (void __user *)input.pdh_cert_address; + input_cert_chain_address = (void __user *)input.cert_chain_address; + /* Userspace wants to query the certificate length. */ if (!input.pdh_cert_address || !input.pdh_cert_len || !input.cert_chain_address) goto cmd; - input_pdh_cert_address = (void __user *)input.pdh_cert_address; - input_cert_chain_address = (void __user *)input.cert_chain_address; - /* Allocate a physically contiguous buffer to store the PDH blob. */ if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) return -EFAULT; @@ -1931,6 +2329,17 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) data.cert_chain_len = input.cert_chain_len; cmd: + /* If platform is not in INIT state then transition it to INIT. */ + if (sev->sev_plat_status.state != SEV_STATE_INIT) { + if (!writable) { + ret = -EPERM; + goto e_free_cert; + } + ret = sev_move_to_init_state(argp, &shutdown_required); + if (ret) + goto e_free_cert; + } + ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error); /* If we query the length, FW responded with expected data. */ @@ -1957,6 +2366,9 @@ cmd: } e_free_cert: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + kfree(cert_blob); e_free_pdh: kfree(pdh_blob); @@ -1966,12 +2378,13 @@ e_free_pdh: static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; + bool shutdown_required = false; struct sev_data_snp_addr buf; struct page *status_page; + int ret, error; void *data; - int ret; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; status_page = alloc_page(GFP_KERNEL_ACCOUNT); @@ -1980,6 +2393,12 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) data = page_address(status_page); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + goto cleanup; + } + /* * Firmware expects status page to be in firmware-owned state, otherwise * it will report firmware error code INVALID_PAGE_STATE (0x1A). @@ -2008,6 +2427,9 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) ret = -EFAULT; cleanup: + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + __free_pages(status_page, 0); return ret; } @@ -2016,21 +2438,33 @@ static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; struct sev_data_snp_commit buf; + bool shutdown_required = false; + int ret, error; - if (!sev->snp_initialized) - return -EINVAL; + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + return ret; + } buf.len = sizeof(buf); - return __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); + ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); + + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + + return ret; } static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_config config; + bool shutdown_required = false; + int ret, error; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; if (!writable) @@ -2039,17 +2473,29 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable if (copy_from_user(&config, (void __user *)argp->data, sizeof(config))) return -EFAULT; - return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + return ret; + } + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); + + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + + return ret; } static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_vlek_load input; + bool shutdown_required = false; + int ret, error; void *blob; - int ret; - if (!sev->snp_initialized || !argp->data) + if (!argp->data) return -EINVAL; if (!writable) @@ -2068,8 +2514,18 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) input.vlek_wrapped_address = __psp_pa(blob); + if (!sev->snp_initialized) { + ret = snp_move_to_init_state(argp, &shutdown_required); + if (ret) + goto cleanup; + } + ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error); + if (shutdown_required) + __sev_snp_shutdown_locked(&error, false); + +cleanup: kfree(blob); return ret; @@ -2296,7 +2752,7 @@ static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) { int error; - __sev_platform_shutdown_locked(NULL); + __sev_platform_shutdown_locked(&error); if (sev_es_tmr) { /* @@ -2334,11 +2790,69 @@ static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) static void sev_firmware_shutdown(struct sev_device *sev) { + /* + * Calling without sev_cmd_mutex held as TSM will likely try disconnecting + * IDE and this ends up calling sev_do_cmd() which locks sev_cmd_mutex. + */ + if (sev->tio_status) + sev_tsm_uninit(sev); + mutex_lock(&sev_cmd_mutex); + __sev_firmware_shutdown(sev, false); + + kfree(sev->tio_status); + sev->tio_status = NULL; + mutex_unlock(&sev_cmd_mutex); } +void sev_platform_shutdown(void) +{ + if (!psp_master || !psp_master->sev_data) + return; + + sev_firmware_shutdown(psp_master->sev_data); +} +EXPORT_SYMBOL_GPL(sev_platform_shutdown); + +u64 sev_get_snp_policy_bits(void) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + u64 policy_bits; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return 0; + + if (!psp || !psp->sev_data) + return 0; + + sev = psp->sev_data; + + policy_bits = SNP_POLICY_MASK_BASE; + + if (sev->snp_plat_status.feature_info) { + if (sev->snp_feat_info_0.ecx & SNP_RAPL_DISABLE_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_RAPL_DIS; + + if (sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM; + + if (sev->snp_feat_info_0.ecx & SNP_AES_256_XTS_POLICY_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_MEM_AES_256_XTS; + + if (sev->snp_feat_info_0.ecx & SNP_CXL_ALLOW_POLICY_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_CXL_ALLOW; + + if (sev_version_greater_or_equal(1, 58)) + policy_bits |= SNP_POLICY_MASK_PAGE_SWAP_DISABLE; + } + + return policy_bits; +} +EXPORT_SYMBOL_GPL(sev_get_snp_policy_bits); + void sev_dev_destroy(struct psp_device *psp) { struct sev_device *sev = psp->sev_data; @@ -2373,10 +2887,6 @@ static int snp_shutdown_on_panic(struct notifier_block *nb, return NOTIFY_DONE; } -static struct notifier_block snp_panic_notifier = { - .notifier_call = snp_shutdown_on_panic, -}; - int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd, void *data, int *error) { @@ -2390,9 +2900,7 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user); void sev_pci_init(void) { struct sev_device *sev = psp_master->sev_data; - struct sev_platform_init_args args = {0}; u8 api_major, api_minor, build; - int rc; if (!sev) return; @@ -2415,18 +2923,6 @@ void sev_pci_init(void) api_major, api_minor, build, sev->api_major, sev->api_minor, sev->build); - /* Initialize the platform */ - args.probe = true; - rc = sev_platform_init(&args); - if (rc) - dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", - args.error, rc); - - dev_info(sev->dev, "SEV%s API:%d.%d build:%d\n", sev->snp_initialized ? - "-SNP" : "", sev->api_major, sev->api_minor, sev->build); - - atomic_notifier_chain_register(&panic_notifier_list, - &snp_panic_notifier); return; err: @@ -2443,7 +2939,4 @@ void sev_pci_exit(void) return; sev_firmware_shutdown(sev); - - atomic_notifier_chain_unregister(&panic_notifier_list, - &snp_panic_notifier); } diff --git a/drivers/crypto/ccp/sev-dev.h b/drivers/crypto/ccp/sev-dev.h index 3e4e5574e88a..b1cd556bbbf6 100644 --- a/drivers/crypto/ccp/sev-dev.h +++ b/drivers/crypto/ccp/sev-dev.h @@ -34,6 +34,8 @@ struct sev_misc_dev { struct miscdevice misc; }; +struct sev_tio_status; + struct sev_device { struct device *dev; struct psp_device *psp; @@ -42,7 +44,6 @@ struct sev_device { struct sev_vdata *vdata; - int state; unsigned int int_rcvd; wait_queue_head_t int_queue; struct sev_misc_dev *misc; @@ -57,12 +58,29 @@ struct sev_device { bool cmd_buf_backup_active; bool snp_initialized; + + struct sev_user_data_status sev_plat_status; + + struct sev_user_data_snp_status snp_plat_status; + struct snp_feature_info snp_feat_info_0; + + struct tsm_dev *tsmdev; + struct sev_tio_status *tio_status; }; int sev_dev_init(struct psp_device *psp); void sev_dev_destroy(struct psp_device *psp); +int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); + void sev_pci_init(void); void sev_pci_exit(void); +struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages); +void snp_free_hv_fixed_pages(struct page *page); + +void sev_tsm_init_locked(struct sev_device *sev, void *tio_status_page); +void sev_tsm_uninit(struct sev_device *sev); +int sev_tio_cmd_buffer_len(int cmd); + #endif /* __SEV_DEV_H */ diff --git a/drivers/crypto/ccp/sfs.c b/drivers/crypto/ccp/sfs.c new file mode 100644 index 000000000000..2f4beaafe7ec --- /dev/null +++ b/drivers/crypto/ccp/sfs.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Processor Seamless Firmware Servicing support. + * + * Copyright (C) 2025 Advanced Micro Devices, Inc. + * + * Author: Ashish Kalra <ashish.kalra@amd.com> + */ + +#include <linux/firmware.h> + +#include "sfs.h" +#include "sev-dev.h" + +#define SFS_DEFAULT_TIMEOUT (10 * MSEC_PER_SEC) +#define SFS_MAX_PAYLOAD_SIZE (2 * 1024 * 1024) +#define SFS_NUM_2MB_PAGES_CMDBUF (SFS_MAX_PAYLOAD_SIZE / PMD_SIZE) +#define SFS_NUM_PAGES_CMDBUF (SFS_MAX_PAYLOAD_SIZE / PAGE_SIZE) + +static DEFINE_MUTEX(sfs_ioctl_mutex); + +static struct sfs_misc_dev *misc_dev; + +static int send_sfs_cmd(struct sfs_device *sfs_dev, int msg) +{ + int ret; + + sfs_dev->command_buf->hdr.status = 0; + sfs_dev->command_buf->hdr.sub_cmd_id = msg; + + ret = psp_extended_mailbox_cmd(sfs_dev->psp, + SFS_DEFAULT_TIMEOUT, + (struct psp_ext_request *)sfs_dev->command_buf); + if (ret == -EIO) { + dev_dbg(sfs_dev->dev, + "msg 0x%x failed with PSP error: 0x%x, extended status: 0x%x\n", + msg, sfs_dev->command_buf->hdr.status, + *(u32 *)sfs_dev->command_buf->buf); + } + + return ret; +} + +static int send_sfs_get_fw_versions(struct sfs_device *sfs_dev) +{ + /* + * SFS_GET_FW_VERSIONS command needs the output buffer to be + * initialized to 0xC7 in every byte. + */ + memset(sfs_dev->command_buf->sfs_buffer, 0xc7, PAGE_SIZE); + sfs_dev->command_buf->hdr.payload_size = 2 * PAGE_SIZE; + + return send_sfs_cmd(sfs_dev, PSP_SFS_GET_FW_VERSIONS); +} + +static int send_sfs_update_package(struct sfs_device *sfs_dev, const char *payload_name) +{ + char payload_path[PAYLOAD_NAME_SIZE + sizeof("amd/")]; + const struct firmware *firmware; + unsigned long package_size; + int ret; + + /* Sanitize userspace provided payload name */ + if (!strnchr(payload_name, PAYLOAD_NAME_SIZE, '\0')) + return -EINVAL; + + snprintf(payload_path, sizeof(payload_path), "amd/%s", payload_name); + + ret = firmware_request_nowarn(&firmware, payload_path, sfs_dev->dev); + if (ret < 0) { + dev_warn_ratelimited(sfs_dev->dev, "firmware request failed for %s (%d)\n", + payload_path, ret); + return -ENOENT; + } + + /* + * SFS Update Package command's input buffer contains TEE_EXT_CMD_BUFFER + * followed by the Update Package and it should be 64KB aligned. + */ + package_size = ALIGN(firmware->size + PAGE_SIZE, 0x10000U); + + /* + * SFS command buffer is a pre-allocated 2MB buffer, fail update package + * if SFS payload is larger than the pre-allocated command buffer. + */ + if (package_size > SFS_MAX_PAYLOAD_SIZE) { + dev_warn_ratelimited(sfs_dev->dev, + "SFS payload size %ld larger than maximum supported payload size of %u\n", + package_size, SFS_MAX_PAYLOAD_SIZE); + release_firmware(firmware); + return -E2BIG; + } + + /* + * Copy firmware data to a HV_Fixed memory region. + */ + memcpy(sfs_dev->command_buf->sfs_buffer, firmware->data, firmware->size); + sfs_dev->command_buf->hdr.payload_size = package_size; + + release_firmware(firmware); + + return send_sfs_cmd(sfs_dev, PSP_SFS_UPDATE); +} + +static long sfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct sfs_user_get_fw_versions __user *sfs_get_fw_versions; + struct sfs_user_update_package __user *sfs_update_package; + struct psp_device *psp_master = psp_get_master_device(); + char payload_name[PAYLOAD_NAME_SIZE]; + struct sfs_device *sfs_dev; + int ret = 0; + + if (!psp_master || !psp_master->sfs_data) + return -ENODEV; + + sfs_dev = psp_master->sfs_data; + + guard(mutex)(&sfs_ioctl_mutex); + + switch (cmd) { + case SFSIOCFWVERS: + dev_dbg(sfs_dev->dev, "in SFSIOCFWVERS\n"); + + sfs_get_fw_versions = (struct sfs_user_get_fw_versions __user *)arg; + + ret = send_sfs_get_fw_versions(sfs_dev); + if (ret && ret != -EIO) + return ret; + + /* + * Return SFS status and extended status back to userspace + * if PSP status indicated success or command error. + */ + if (copy_to_user(&sfs_get_fw_versions->blob, sfs_dev->command_buf->sfs_buffer, + PAGE_SIZE)) + return -EFAULT; + if (copy_to_user(&sfs_get_fw_versions->sfs_status, + &sfs_dev->command_buf->hdr.status, + sizeof(sfs_get_fw_versions->sfs_status))) + return -EFAULT; + if (copy_to_user(&sfs_get_fw_versions->sfs_extended_status, + &sfs_dev->command_buf->buf, + sizeof(sfs_get_fw_versions->sfs_extended_status))) + return -EFAULT; + break; + case SFSIOCUPDATEPKG: + dev_dbg(sfs_dev->dev, "in SFSIOCUPDATEPKG\n"); + + sfs_update_package = (struct sfs_user_update_package __user *)arg; + + if (copy_from_user(payload_name, sfs_update_package->payload_name, + PAYLOAD_NAME_SIZE)) + return -EFAULT; + + ret = send_sfs_update_package(sfs_dev, payload_name); + if (ret && ret != -EIO) + return ret; + + /* + * Return SFS status and extended status back to userspace + * if PSP status indicated success or command error. + */ + if (copy_to_user(&sfs_update_package->sfs_status, + &sfs_dev->command_buf->hdr.status, + sizeof(sfs_update_package->sfs_status))) + return -EFAULT; + if (copy_to_user(&sfs_update_package->sfs_extended_status, + &sfs_dev->command_buf->buf, + sizeof(sfs_update_package->sfs_extended_status))) + return -EFAULT; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static const struct file_operations sfs_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = sfs_ioctl, +}; + +static void sfs_exit(struct kref *ref) +{ + misc_deregister(&misc_dev->misc); + kfree(misc_dev); + misc_dev = NULL; +} + +void sfs_dev_destroy(struct psp_device *psp) +{ + struct sfs_device *sfs_dev = psp->sfs_data; + + if (!sfs_dev) + return; + + /* + * Change SFS command buffer back to the default "Write-Back" type. + */ + set_memory_wb((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); + + snp_free_hv_fixed_pages(sfs_dev->page); + + if (sfs_dev->misc) + kref_put(&misc_dev->refcount, sfs_exit); + + psp->sfs_data = NULL; +} + +/* Based on sev_misc_init() */ +static int sfs_misc_init(struct sfs_device *sfs) +{ + struct device *dev = sfs->dev; + int ret; + + /* + * SFS feature support can be detected on multiple devices but the SFS + * FW commands must be issued on the master. During probe, we do not + * know the master hence we create /dev/sfs on the first device probe. + */ + if (!misc_dev) { + struct miscdevice *misc; + + misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL); + if (!misc_dev) + return -ENOMEM; + + misc = &misc_dev->misc; + misc->minor = MISC_DYNAMIC_MINOR; + misc->name = "sfs"; + misc->fops = &sfs_fops; + misc->mode = 0600; + + ret = misc_register(misc); + if (ret) + return ret; + + kref_init(&misc_dev->refcount); + } else { + kref_get(&misc_dev->refcount); + } + + sfs->misc = misc_dev; + dev_dbg(dev, "registered SFS device\n"); + + return 0; +} + +int sfs_dev_init(struct psp_device *psp) +{ + struct device *dev = psp->dev; + struct sfs_device *sfs_dev; + struct page *page; + int ret = -ENOMEM; + + sfs_dev = devm_kzalloc(dev, sizeof(*sfs_dev), GFP_KERNEL); + if (!sfs_dev) + return -ENOMEM; + + /* + * Pre-allocate 2MB command buffer for all SFS commands using + * SNP HV_Fixed page allocator which also transitions the + * SFS command buffer to HV_Fixed page state if SNP is enabled. + */ + page = snp_alloc_hv_fixed_pages(SFS_NUM_2MB_PAGES_CMDBUF); + if (!page) { + dev_dbg(dev, "Command Buffer HV-Fixed page allocation failed\n"); + goto cleanup_dev; + } + sfs_dev->page = page; + sfs_dev->command_buf = page_address(page); + + dev_dbg(dev, "Command buffer 0x%px to be marked as HV_Fixed\n", sfs_dev->command_buf); + + /* + * SFS command buffer must be mapped as non-cacheable. + */ + ret = set_memory_uc((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); + if (ret) { + dev_dbg(dev, "Set memory uc failed\n"); + goto cleanup_cmd_buf; + } + + dev_dbg(dev, "Command buffer 0x%px marked uncacheable\n", sfs_dev->command_buf); + + psp->sfs_data = sfs_dev; + sfs_dev->dev = dev; + sfs_dev->psp = psp; + + ret = sfs_misc_init(sfs_dev); + if (ret) + goto cleanup_mem_attr; + + dev_notice(sfs_dev->dev, "SFS support is available\n"); + + return 0; + +cleanup_mem_attr: + set_memory_wb((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); + +cleanup_cmd_buf: + snp_free_hv_fixed_pages(page); + +cleanup_dev: + psp->sfs_data = NULL; + devm_kfree(dev, sfs_dev); + + return ret; +} diff --git a/drivers/crypto/ccp/sfs.h b/drivers/crypto/ccp/sfs.h new file mode 100644 index 000000000000..97704c210efd --- /dev/null +++ b/drivers/crypto/ccp/sfs.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Platform Security Processor (PSP) Seamless Firmware (SFS) Support. + * + * Copyright (C) 2025 Advanced Micro Devices, Inc. + * + * Author: Ashish Kalra <ashish.kalra@amd.com> + */ + +#ifndef __SFS_H__ +#define __SFS_H__ + +#include <uapi/linux/psp-sfs.h> + +#include <linux/device.h> +#include <linux/miscdevice.h> +#include <linux/psp-sev.h> +#include <linux/psp-platform-access.h> +#include <linux/set_memory.h> + +#include "psp-dev.h" + +struct sfs_misc_dev { + struct kref refcount; + struct miscdevice misc; +}; + +struct sfs_command { + struct psp_ext_req_buffer_hdr hdr; + u8 buf[PAGE_SIZE - sizeof(struct psp_ext_req_buffer_hdr)]; + u8 sfs_buffer[]; +} __packed; + +struct sfs_device { + struct device *dev; + struct psp_device *psp; + + struct page *page; + struct sfs_command *command_buf; + + struct sfs_misc_dev *misc; +}; + +void sfs_dev_destroy(struct psp_device *psp); +int sfs_dev_init(struct psp_device *psp); + +#endif /* __SFS_H__ */ diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c index 7eb3e4668286..3467f6db4f50 100644 --- a/drivers/crypto/ccp/sp-dev.c +++ b/drivers/crypto/ccp/sp-dev.c @@ -19,6 +19,7 @@ #include <linux/types.h> #include <linux/ccp.h> +#include "sev-dev.h" #include "ccp-dev.h" #include "sp-dev.h" @@ -253,8 +254,12 @@ unlock: static int __init sp_mod_init(void) { #ifdef CONFIG_X86 + static bool initialized; int ret; + if (initialized) + return 0; + ret = sp_pci_init(); if (ret) return ret; @@ -263,6 +268,8 @@ static int __init sp_mod_init(void) psp_pci_init(); #endif + initialized = true; + return 0; #endif @@ -279,6 +286,13 @@ static int __init sp_mod_init(void) return -ENODEV; } +#if IS_BUILTIN(CONFIG_KVM_AMD) && IS_ENABLED(CONFIG_KVM_AMD_SEV) +int __init sev_module_init(void) +{ + return sp_mod_init(); +} +#endif + static void __exit sp_mod_exit(void) { #ifdef CONFIG_X86 diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 6f9d7063257d..1335a83fe052 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -95,7 +95,7 @@ struct sp_device { struct device *dev; - struct sp_dev_vdata *dev_vdata; + const struct sp_dev_vdata *dev_vdata; unsigned int ord; char name[SP_MAX_NAME_LEN]; diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 248d98fd8c48..8891ceee1d7d 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -189,14 +189,17 @@ static bool sp_pci_is_master(struct sp_device *sp) pdev_new = to_pci_dev(dev_new); pdev_cur = to_pci_dev(dev_cur); - if (pdev_new->bus->number < pdev_cur->bus->number) - return true; + if (pci_domain_nr(pdev_new->bus) != pci_domain_nr(pdev_cur->bus)) + return pci_domain_nr(pdev_new->bus) < pci_domain_nr(pdev_cur->bus); - if (PCI_SLOT(pdev_new->devfn) < PCI_SLOT(pdev_cur->devfn)) - return true; + if (pdev_new->bus->number != pdev_cur->bus->number) + return pdev_new->bus->number < pdev_cur->bus->number; - if (PCI_FUNC(pdev_new->devfn) < PCI_FUNC(pdev_cur->devfn)) - return true; + if (PCI_SLOT(pdev_new->devfn) != PCI_SLOT(pdev_cur->devfn)) + return PCI_SLOT(pdev_new->devfn) < PCI_SLOT(pdev_cur->devfn); + + if (PCI_FUNC(pdev_new->devfn) != PCI_FUNC(pdev_cur->devfn)) + return PCI_FUNC(pdev_new->devfn) < PCI_FUNC(pdev_cur->devfn); return false; } @@ -372,6 +375,7 @@ static const struct tee_vdata teev1 = { static const struct tee_vdata teev2 = { .ring_wptr_reg = 0x10950, /* C2PMSG_20 */ .ring_rptr_reg = 0x10954, /* C2PMSG_21 */ + .info_reg = 0x109e8, /* C2PMSG_58 */ }; static const struct platform_access_vdata pa_v1 = { @@ -437,6 +441,7 @@ static const struct psp_vdata pspv5 = { .cmdresp_reg = 0x10944, /* C2PMSG_17 */ .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ .feature_reg = 0x109fc, /* C2PMSG_63 */ .inten_reg = 0x10510, /* P2CMSG_INTEN */ .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ @@ -448,11 +453,23 @@ static const struct psp_vdata pspv6 = { .cmdresp_reg = 0x10944, /* C2PMSG_17 */ .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ .feature_reg = 0x109fc, /* C2PMSG_63 */ .inten_reg = 0x10510, /* P2CMSG_INTEN */ .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ }; +static const struct psp_vdata pspv7 = { + .tee = &teev2, + .cmdresp_reg = 0x10944, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10510, /* P2CMSG_INTEN */ + .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ +}; + #endif static const struct sp_dev_vdata dev_vdata[] = { @@ -519,6 +536,13 @@ static const struct sp_dev_vdata dev_vdata[] = { .psp_vdata = &pspv6, #endif }, + { /* 9 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv7, +#endif + }, + }; static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] }, @@ -529,8 +553,11 @@ static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x14CA), (kernel_ulong_t)&dev_vdata[5] }, { PCI_VDEVICE(AMD, 0x15C7), (kernel_ulong_t)&dev_vdata[6] }, { PCI_VDEVICE(AMD, 0x1649), (kernel_ulong_t)&dev_vdata[6] }, + { PCI_VDEVICE(AMD, 0x1134), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x115A), (kernel_ulong_t)&dev_vdata[9] }, /* Last entry must be zero */ { 0, } }; diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c index 3933cac1694d..3f9843fa7782 100644 --- a/drivers/crypto/ccp/sp-platform.c +++ b/drivers/crypto/ccp/sp-platform.c @@ -52,24 +52,13 @@ static const struct of_device_id sp_of_match[] = { }; MODULE_DEVICE_TABLE(of, sp_of_match); -static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev) -{ - const struct of_device_id *match; - - match = of_match_node(sp_of_match, pdev->dev.of_node); - if (match && match->data) - return (struct sp_dev_vdata *)match->data; - - return NULL; -} - -static struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) +static const struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) { const struct acpi_device_id *match; match = acpi_match_device(sp_acpi_match, &pdev->dev); if (match && match->driver_data) - return (struct sp_dev_vdata *)match->driver_data; + return (const struct sp_dev_vdata *)match->driver_data; return NULL; } @@ -123,7 +112,7 @@ static int sp_platform_probe(struct platform_device *pdev) goto e_err; sp->dev_specific = sp_platform; - sp->dev_vdata = pdev->dev.of_node ? sp_get_of_version(pdev) + sp->dev_vdata = pdev->dev.of_node ? of_device_get_match_data(&pdev->dev) : sp_get_acpi_version(pdev); if (!sp->dev_vdata) { ret = -ENODEV; diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index bcca55bff910..dc7e0cd51c25 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -224,7 +224,7 @@ static int cc_generate_mlli(struct device *dev, struct buffer_array *sg_data, /* Set MLLI size for the bypass operation */ mlli_params->mlli_len = (total_nents * LLI_ENTRY_BYTE_SIZE); - dev_dbg(dev, "MLLI params: virt_addr=%pK dma_addr=%pad mlli_len=0x%X\n", + dev_dbg(dev, "MLLI params: virt_addr=%p dma_addr=%pad mlli_len=0x%X\n", mlli_params->mlli_virt_addr, &mlli_params->mlli_dma_addr, mlli_params->mlli_len); @@ -239,7 +239,7 @@ static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data, { unsigned int index = sgl_data->num_of_buffers; - dev_dbg(dev, "index=%u nents=%u sgl=%pK data_len=0x%08X is_last=%d\n", + dev_dbg(dev, "index=%u nents=%u sgl=%p data_len=0x%08X is_last=%d\n", index, nents, sgl, data_len, is_last_table); sgl_data->nents[index] = nents; sgl_data->entry[index].sgl = sgl; @@ -298,7 +298,7 @@ cc_set_aead_conf_buf(struct device *dev, struct aead_req_ctx *areq_ctx, dev_err(dev, "dma_map_sg() config buffer failed\n"); return -ENOMEM; } - dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n", + dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%p offset=%u length=%u\n", &sg_dma_address(&areq_ctx->ccm_adata_sg), sg_page(&areq_ctx->ccm_adata_sg), sg_virt(&areq_ctx->ccm_adata_sg), @@ -323,7 +323,7 @@ static int cc_set_hash_buf(struct device *dev, struct ahash_req_ctx *areq_ctx, dev_err(dev, "dma_map_sg() src buffer failed\n"); return -ENOMEM; } - dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n", + dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%p offset=%u length=%u\n", &sg_dma_address(areq_ctx->buff_sg), sg_page(areq_ctx->buff_sg), sg_virt(areq_ctx->buff_sg), areq_ctx->buff_sg->offset, areq_ctx->buff_sg->length); @@ -359,11 +359,11 @@ void cc_unmap_cipher_request(struct device *dev, void *ctx, if (src != dst) { dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_TO_DEVICE); dma_unmap_sg(dev, dst, req_ctx->out_nents, DMA_FROM_DEVICE); - dev_dbg(dev, "Unmapped req->dst=%pK\n", sg_virt(dst)); - dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src)); + dev_dbg(dev, "Unmapped req->dst=%p\n", sg_virt(dst)); + dev_dbg(dev, "Unmapped req->src=%p\n", sg_virt(src)); } else { dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_BIDIRECTIONAL); - dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src)); + dev_dbg(dev, "Unmapped req->src=%p\n", sg_virt(src)); } } @@ -391,11 +391,11 @@ int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx, req_ctx->gen_ctx.iv_dma_addr = dma_map_single(dev, info, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) { - dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping iv %u B at va=%p for DMA failed\n", ivsize, info); return -ENOMEM; } - dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped iv %u B at va=%p to dma=%pad\n", ivsize, info, &req_ctx->gen_ctx.iv_dma_addr); } else { req_ctx->gen_ctx.iv_dma_addr = 0; @@ -506,7 +506,7 @@ void cc_unmap_aead_request(struct device *dev, struct aead_request *req) if ((areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) && (areq_ctx->mlli_params.mlli_virt_addr)) { - dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n", + dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%p\n", &areq_ctx->mlli_params.mlli_dma_addr, areq_ctx->mlli_params.mlli_virt_addr); dma_pool_free(areq_ctx->mlli_params.curr_pool, @@ -514,13 +514,13 @@ void cc_unmap_aead_request(struct device *dev, struct aead_request *req) areq_ctx->mlli_params.mlli_dma_addr); } - dev_dbg(dev, "Unmapping src sgl: req->src=%pK areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n", + dev_dbg(dev, "Unmapping src sgl: req->src=%p areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n", sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents, areq_ctx->assoclen, req->cryptlen); dma_unmap_sg(dev, req->src, areq_ctx->src.mapped_nents, src_direction); if (req->src != req->dst) { - dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n", + dev_dbg(dev, "Unmapping dst sgl: req->dst=%p\n", sg_virt(req->dst)); dma_unmap_sg(dev, req->dst, areq_ctx->dst.mapped_nents, DMA_FROM_DEVICE); } @@ -566,7 +566,7 @@ static int cc_aead_chain_iv(struct cc_drvdata *drvdata, dma_map_single(dev, areq_ctx->gen_ctx.iv, hw_iv_size, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) { - dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping iv %u B at va=%p for DMA failed\n", hw_iv_size, req->iv); kfree_sensitive(areq_ctx->gen_ctx.iv); areq_ctx->gen_ctx.iv = NULL; @@ -574,7 +574,7 @@ static int cc_aead_chain_iv(struct cc_drvdata *drvdata, goto chain_iv_exit; } - dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped iv %u B at va=%p to dma=%pad\n", hw_iv_size, req->iv, &areq_ctx->gen_ctx.iv_dma_addr); chain_iv_exit: @@ -977,7 +977,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) dma_addr = dma_map_single(dev, areq_ctx->mac_buf, MAX_MAC_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, dma_addr)) { - dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping mac_buf %u B at va=%p for DMA failed\n", MAX_MAC_SIZE, areq_ctx->mac_buf); rc = -ENOMEM; goto aead_map_failure; @@ -991,7 +991,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_addr)) { - dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping mac_buf %u B at va=%p for DMA failed\n", AES_BLOCK_SIZE, addr); areq_ctx->ccm_iv0_dma_addr = 0; rc = -ENOMEM; @@ -1009,7 +1009,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) dma_addr = dma_map_single(dev, areq_ctx->hkey, AES_BLOCK_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, dma_addr)) { - dev_err(dev, "Mapping hkey %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping hkey %u B at va=%p for DMA failed\n", AES_BLOCK_SIZE, areq_ctx->hkey); rc = -ENOMEM; goto aead_map_failure; @@ -1019,7 +1019,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) dma_addr = dma_map_single(dev, &areq_ctx->gcm_len_block, AES_BLOCK_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_addr)) { - dev_err(dev, "Mapping gcm_len_block %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping gcm_len_block %u B at va=%p for DMA failed\n", AES_BLOCK_SIZE, &areq_ctx->gcm_len_block); rc = -ENOMEM; goto aead_map_failure; @@ -1030,7 +1030,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) AES_BLOCK_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_addr)) { - dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%p for DMA failed\n", AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc1)); areq_ctx->gcm_iv_inc1_dma_addr = 0; rc = -ENOMEM; @@ -1042,7 +1042,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) AES_BLOCK_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_addr)) { - dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%p for DMA failed\n", AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc2)); areq_ctx->gcm_iv_inc2_dma_addr = 0; rc = -ENOMEM; @@ -1152,7 +1152,7 @@ int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx, u32 dummy = 0; u32 mapped_nents = 0; - dev_dbg(dev, "final params : curr_buff=%pK curr_buff_cnt=0x%X nbytes = 0x%X src=%pK curr_index=%u\n", + dev_dbg(dev, "final params : curr_buff=%p curr_buff_cnt=0x%X nbytes = 0x%X src=%p curr_index=%u\n", curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); /* Init the type of the dma buffer */ areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL; @@ -1235,8 +1235,9 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, int rc = 0; u32 dummy = 0; u32 mapped_nents = 0; + int sg_nents; - dev_dbg(dev, " update params : curr_buff=%pK curr_buff_cnt=0x%X nbytes=0x%X src=%pK curr_index=%u\n", + dev_dbg(dev, " update params : curr_buff=%p curr_buff_cnt=0x%X nbytes=0x%X src=%p curr_index=%u\n", curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); /* Init the type of the dma buffer */ areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL; @@ -1246,9 +1247,12 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, areq_ctx->in_nents = 0; if (total_in_len < block_size) { - dev_dbg(dev, " less than one block: curr_buff=%pK *curr_buff_cnt=0x%X copy_to=%pK\n", + dev_dbg(dev, " less than one block: curr_buff=%p *curr_buff_cnt=0x%X copy_to=%p\n", curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]); - areq_ctx->in_nents = sg_nents_for_len(src, nbytes); + sg_nents = sg_nents_for_len(src, nbytes); + if (sg_nents < 0) + return sg_nents; + areq_ctx->in_nents = sg_nents; sg_copy_to_buffer(src, areq_ctx->in_nents, &curr_buff[*curr_buff_cnt], nbytes); *curr_buff_cnt += nbytes; @@ -1265,7 +1269,7 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, /* Copy the new residue to next buffer */ if (*next_buff_cnt) { - dev_dbg(dev, " handle residue: next buff %pK skip data %u residue %u\n", + dev_dbg(dev, " handle residue: next buff %p skip data %u residue %u\n", next_buff, (update_data_len - *curr_buff_cnt), *next_buff_cnt); cc_copy_sg_portion(dev, next_buff, src, @@ -1338,7 +1342,7 @@ void cc_unmap_hash_request(struct device *dev, void *ctx, *allocated and should be released */ if (areq_ctx->mlli_params.curr_pool) { - dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n", + dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%p\n", &areq_ctx->mlli_params.mlli_dma_addr, areq_ctx->mlli_params.mlli_virt_addr); dma_pool_free(areq_ctx->mlli_params.curr_pool, @@ -1347,14 +1351,14 @@ void cc_unmap_hash_request(struct device *dev, void *ctx, } if (src && areq_ctx->in_nents) { - dev_dbg(dev, "Unmapped sg src: virt=%pK dma=%pad len=0x%X\n", + dev_dbg(dev, "Unmapped sg src: virt=%p dma=%pad len=0x%X\n", sg_virt(src), &sg_dma_address(src), sg_dma_len(src)); dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE); } if (*prev_len) { - dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%pK dma=%pad len 0x%X\n", + dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%p dma=%pad len 0x%X\n", sg_virt(areq_ctx->buff_sg), &sg_dma_address(areq_ctx->buff_sg), sg_dma_len(areq_ctx->buff_sg)); diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index d39c067672fd..e2cbfdf7a0e4 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -211,11 +211,11 @@ static int cc_cipher_init(struct crypto_tfm *tfm) max_key_buf_size, DMA_TO_DEVICE); if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) { - dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping Key %u B at va=%p for DMA failed\n", max_key_buf_size, ctx_p->user.key); goto free_key; } - dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped key %u B at va=%p to dma=%pad\n", max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); return 0; diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c index d0612bec4d58..c6d085c8ff79 100644 --- a/drivers/crypto/ccree/cc_hash.c +++ b/drivers/crypto/ccree/cc_hash.c @@ -125,7 +125,7 @@ static int cc_map_result(struct device *dev, struct ahash_req_ctx *state, digestsize); return -ENOMEM; } - dev_dbg(dev, "Mapped digest result buffer %u B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped digest result buffer %u B at va=%p to dma=%pad\n", digestsize, state->digest_result_buff, &state->digest_result_dma_addr); @@ -184,11 +184,11 @@ static int cc_map_req(struct device *dev, struct ahash_req_ctx *state, dma_map_single(dev, state->digest_buff, ctx->inter_digestsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, state->digest_buff_dma_addr)) { - dev_err(dev, "Mapping digest len %d B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping digest len %d B at va=%p for DMA failed\n", ctx->inter_digestsize, state->digest_buff); return -EINVAL; } - dev_dbg(dev, "Mapped digest %d B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped digest %d B at va=%p to dma=%pad\n", ctx->inter_digestsize, state->digest_buff, &state->digest_buff_dma_addr); @@ -197,11 +197,11 @@ static int cc_map_req(struct device *dev, struct ahash_req_ctx *state, dma_map_single(dev, state->digest_bytes_len, HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) { - dev_err(dev, "Mapping digest len %u B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping digest len %u B at va=%p for DMA failed\n", HASH_MAX_LEN_SIZE, state->digest_bytes_len); goto unmap_digest_buf; } - dev_dbg(dev, "Mapped digest len %u B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped digest len %u B at va=%p to dma=%pad\n", HASH_MAX_LEN_SIZE, state->digest_bytes_len, &state->digest_bytes_len_dma_addr); } @@ -212,12 +212,12 @@ static int cc_map_req(struct device *dev, struct ahash_req_ctx *state, ctx->inter_digestsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, state->opad_digest_dma_addr)) { - dev_err(dev, "Mapping opad digest %d B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping opad digest %d B at va=%p for DMA failed\n", ctx->inter_digestsize, state->opad_digest_buff); goto unmap_digest_len; } - dev_dbg(dev, "Mapped opad digest %d B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped opad digest %d B at va=%p to dma=%pad\n", ctx->inter_digestsize, state->opad_digest_buff, &state->opad_digest_dma_addr); } @@ -272,7 +272,7 @@ static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state, if (state->digest_result_dma_addr) { dma_unmap_single(dev, state->digest_result_dma_addr, digestsize, DMA_BIDIRECTIONAL); - dev_dbg(dev, "unmpa digest result buffer va (%pK) pa (%pad) len %u\n", + dev_dbg(dev, "unmpa digest result buffer va (%p) pa (%pad) len %u\n", state->digest_result_buff, &state->digest_result_dma_addr, digestsize); memcpy(result, state->digest_result_buff, digestsize); @@ -287,7 +287,7 @@ static void cc_update_complete(struct device *dev, void *cc_req, int err) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); - dev_dbg(dev, "req=%pK\n", req); + dev_dbg(dev, "req=%p\n", req); if (err != -EINPROGRESS) { /* Not a BACKLOG notification */ @@ -306,7 +306,7 @@ static void cc_digest_complete(struct device *dev, void *cc_req, int err) struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); u32 digestsize = crypto_ahash_digestsize(tfm); - dev_dbg(dev, "req=%pK\n", req); + dev_dbg(dev, "req=%p\n", req); if (err != -EINPROGRESS) { /* Not a BACKLOG notification */ @@ -326,7 +326,7 @@ static void cc_hash_complete(struct device *dev, void *cc_req, int err) struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); u32 digestsize = crypto_ahash_digestsize(tfm); - dev_dbg(dev, "req=%pK\n", req); + dev_dbg(dev, "req=%p\n", req); if (err != -EINPROGRESS) { /* Not a BACKLOG notification */ @@ -1077,11 +1077,11 @@ static int cc_alloc_ctx(struct cc_hash_ctx *ctx) dma_map_single(dev, ctx->digest_buff, sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) { - dev_err(dev, "Mapping digest len %zu B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping digest len %zu B at va=%p for DMA failed\n", sizeof(ctx->digest_buff), ctx->digest_buff); goto fail; } - dev_dbg(dev, "Mapped digest %zu B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped digest %zu B at va=%p to dma=%pad\n", sizeof(ctx->digest_buff), ctx->digest_buff, &ctx->digest_buff_dma_addr); @@ -1090,12 +1090,12 @@ static int cc_alloc_ctx(struct cc_hash_ctx *ctx) sizeof(ctx->opad_tmp_keys_buff), DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) { - dev_err(dev, "Mapping opad digest %zu B at va=%pK for DMA failed\n", + dev_err(dev, "Mapping opad digest %zu B at va=%p for DMA failed\n", sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff); goto fail; } - dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n", + dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%p to dma=%pad\n", sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff, &ctx->opad_tmp_keys_dma_addr); diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 6124fbbbed94..bbd118f8de0e 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -77,6 +77,5 @@ int cc_pm_get(struct device *dev) void cc_pm_put_suspend(struct device *dev) { - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); } diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 5dd3f6a4781a..37294bb74003 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -4,9 +4,9 @@ config CRYPTO_DEV_CHELSIO depends on CHELSIO_T4 select CRYPTO_LIB_AES select CRYPTO_LIB_GF128MUL - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 select CRYPTO_AUTHENC help The Chelsio Crypto Co-processor driver for T6 adapters. diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index af37477ffd8d..22cbc343198a 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -51,7 +51,6 @@ #include <crypto/aes.h> #include <crypto/algapi.h> -#include <crypto/hash.h> #include <crypto/gcm.h> #include <crypto/sha1.h> #include <crypto/sha2.h> @@ -277,88 +276,60 @@ static void get_aes_decrypt_key(unsigned char *dec_key, } } -static struct crypto_shash *chcr_alloc_shash(unsigned int ds) +static int chcr_prepare_hmac_key(const u8 *raw_key, unsigned int raw_key_len, + int digestsize, void *istate, void *ostate) { - struct crypto_shash *base_hash = ERR_PTR(-EINVAL); - - switch (ds) { + __be32 *istate32 = istate, *ostate32 = ostate; + __be64 *istate64 = istate, *ostate64 = ostate; + union { + struct hmac_sha1_key sha1; + struct hmac_sha224_key sha224; + struct hmac_sha256_key sha256; + struct hmac_sha384_key sha384; + struct hmac_sha512_key sha512; + } k; + + switch (digestsize) { case SHA1_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha1", 0, 0); + hmac_sha1_preparekey(&k.sha1, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha1.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha1.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha1.ostate.h[i]); + } break; case SHA224_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha224", 0, 0); + hmac_sha224_preparekey(&k.sha224, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha224.key.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha224.key.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha224.key.ostate.h[i]); + } break; case SHA256_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha256", 0, 0); + hmac_sha256_preparekey(&k.sha256, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha256.key.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha256.key.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha256.key.ostate.h[i]); + } break; case SHA384_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha384", 0, 0); + hmac_sha384_preparekey(&k.sha384, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha384.key.istate.h); i++) { + istate64[i] = cpu_to_be64(k.sha384.key.istate.h[i]); + ostate64[i] = cpu_to_be64(k.sha384.key.ostate.h[i]); + } break; case SHA512_DIGEST_SIZE: - base_hash = crypto_alloc_shash("sha512", 0, 0); + hmac_sha512_preparekey(&k.sha512, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha512.key.istate.h); i++) { + istate64[i] = cpu_to_be64(k.sha512.key.istate.h[i]); + ostate64[i] = cpu_to_be64(k.sha512.key.ostate.h[i]); + } break; + default: + return -EINVAL; } - - return base_hash; -} - -static int chcr_compute_partial_hash(struct shash_desc *desc, - char *iopad, char *result_hash, - int digest_size) -{ - struct sha1_state sha1_st; - struct sha256_state sha256_st; - struct sha512_state sha512_st; - int error; - - if (digest_size == SHA1_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: - crypto_shash_export(desc, (void *)&sha1_st); - memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); - } else if (digest_size == SHA224_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: - crypto_shash_export(desc, (void *)&sha256_st); - memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); - - } else if (digest_size == SHA256_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: - crypto_shash_export(desc, (void *)&sha256_st); - memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); - - } else if (digest_size == SHA384_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: - crypto_shash_export(desc, (void *)&sha512_st); - memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); - - } else if (digest_size == SHA512_DIGEST_SIZE) { - error = crypto_shash_init(desc) ?: - crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: - crypto_shash_export(desc, (void *)&sha512_st); - memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); - } else { - error = -EINVAL; - pr_err("Unknown digest size %d\n", digest_size); - } - return error; -} - -static void chcr_change_order(char *buf, int ds) -{ - int i; - - if (ds == SHA512_DIGEST_SIZE) { - for (i = 0; i < (ds / sizeof(u64)); i++) - *((__be64 *)buf + i) = - cpu_to_be64(*((u64 *)buf + i)); - } else { - for (i = 0; i < (ds / sizeof(u32)); i++) - *((__be32 *)buf + i) = - cpu_to_be32(*((u32 *)buf + i)); - } + memzero_explicit(&k, sizeof(k)); + return 0; } static inline int is_hmac(struct crypto_tfm *tfm) @@ -1547,11 +1518,6 @@ static int get_alg_config(struct algo_param *params, return 0; } -static inline void chcr_free_shash(struct crypto_shash *base_hash) -{ - crypto_free_shash(base_hash); -} - /** * create_hash_wr - Create hash work request * @req: Cipher req base @@ -2202,53 +2168,13 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); - unsigned int digestsize = crypto_ahash_digestsize(tfm); - unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); - unsigned int i, err = 0, updated_digestsize; - - SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); /* use the key to calculate the ipad and opad. ipad will sent with the * first request's data. opad will be sent with the final hash result * ipad in hmacctx->ipad and opad in hmacctx->opad location */ - shash->tfm = hmacctx->base_hash; - if (keylen > bs) { - err = crypto_shash_digest(shash, key, keylen, - hmacctx->ipad); - if (err) - goto out; - keylen = digestsize; - } else { - memcpy(hmacctx->ipad, key, keylen); - } - memset(hmacctx->ipad + keylen, 0, bs - keylen); - unsafe_memcpy(hmacctx->opad, hmacctx->ipad, bs, - "fortified memcpy causes -Wrestrict warning"); - - for (i = 0; i < bs / sizeof(int); i++) { - *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; - *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; - } - - updated_digestsize = digestsize; - if (digestsize == SHA224_DIGEST_SIZE) - updated_digestsize = SHA256_DIGEST_SIZE; - else if (digestsize == SHA384_DIGEST_SIZE) - updated_digestsize = SHA512_DIGEST_SIZE; - err = chcr_compute_partial_hash(shash, hmacctx->ipad, - hmacctx->ipad, digestsize); - if (err) - goto out; - chcr_change_order(hmacctx->ipad, updated_digestsize); - - err = chcr_compute_partial_hash(shash, hmacctx->opad, - hmacctx->opad, digestsize); - if (err) - goto out; - chcr_change_order(hmacctx->opad, updated_digestsize); -out: - return err; + return chcr_prepare_hmac_key(key, keylen, crypto_ahash_digestsize(tfm), + hmacctx->ipad, hmacctx->opad); } static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, @@ -2344,30 +2270,11 @@ static int chcr_hmac_init(struct ahash_request *areq) static int chcr_hmac_cra_init(struct crypto_tfm *tfm) { - struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); - unsigned int digestsize = - crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct chcr_ahash_req_ctx)); - hmacctx->base_hash = chcr_alloc_shash(digestsize); - if (IS_ERR(hmacctx->base_hash)) - return PTR_ERR(hmacctx->base_hash); return chcr_device_init(crypto_tfm_ctx(tfm)); } -static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) -{ - struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); - - if (hmacctx->base_hash) { - chcr_free_shash(hmacctx->base_hash); - hmacctx->base_hash = NULL; - } -} - inline void chcr_aead_common_exit(struct aead_request *req) { struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); @@ -3557,15 +3464,12 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); /* it contains auth and cipher key both*/ struct crypto_authenc_keys keys; - unsigned int bs, subtype; + unsigned int subtype; unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; - int err = 0, i, key_ctx_len = 0; + int err = 0, key_ctx_len = 0; unsigned char ck_size = 0; - unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; - struct crypto_shash *base_hash = ERR_PTR(-EINVAL); struct algo_param param; int align; - u8 *o_ptr = NULL; crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) @@ -3613,68 +3517,26 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, aeadctx->enckey_len << 3); } - base_hash = chcr_alloc_shash(max_authsize); - if (IS_ERR(base_hash)) { - pr_err("Base driver cannot be loaded\n"); + + align = KEYCTX_ALIGN_PAD(max_authsize); + err = chcr_prepare_hmac_key(keys.authkey, keys.authkeylen, max_authsize, + actx->h_iopad, + actx->h_iopad + param.result_size + align); + if (err) goto out; - } - { - SHASH_DESC_ON_STACK(shash, base_hash); - - shash->tfm = base_hash; - bs = crypto_shash_blocksize(base_hash); - align = KEYCTX_ALIGN_PAD(max_authsize); - o_ptr = actx->h_iopad + param.result_size + align; - - if (keys.authkeylen > bs) { - err = crypto_shash_digest(shash, keys.authkey, - keys.authkeylen, - o_ptr); - if (err) { - pr_err("Base driver cannot be loaded\n"); - goto out; - } - keys.authkeylen = max_authsize; - } else - memcpy(o_ptr, keys.authkey, keys.authkeylen); - - /* Compute the ipad-digest*/ - memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); - memcpy(pad, o_ptr, keys.authkeylen); - for (i = 0; i < bs >> 2; i++) - *((unsigned int *)pad + i) ^= IPAD_DATA; - - if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, - max_authsize)) - goto out; - /* Compute the opad-digest */ - memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); - memcpy(pad, o_ptr, keys.authkeylen); - for (i = 0; i < bs >> 2; i++) - *((unsigned int *)pad + i) ^= OPAD_DATA; - if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) - goto out; + key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16) + + (param.result_size + align) * 2; + aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 0, 1, + key_ctx_len >> 4); + actx->auth_mode = param.auth_mode; + + memzero_explicit(&keys, sizeof(keys)); + return 0; - /* convert the ipad and opad digest to network order */ - chcr_change_order(actx->h_iopad, param.result_size); - chcr_change_order(o_ptr, param.result_size); - key_ctx_len = sizeof(struct _key_ctx) + - roundup(keys.enckeylen, 16) + - (param.result_size + align) * 2; - aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, - 0, 1, key_ctx_len >> 4); - actx->auth_mode = param.auth_mode; - chcr_free_shash(base_hash); - - memzero_explicit(&keys, sizeof(keys)); - return 0; - } out: aeadctx->enckey_len = 0; memzero_explicit(&keys, sizeof(keys)); - if (!IS_ERR(base_hash)) - chcr_free_shash(base_hash); return -EINVAL; } @@ -4490,7 +4352,6 @@ static int chcr_register_alg(void) if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { a_hash->halg.base.cra_init = chcr_hmac_cra_init; - a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; a_hash->init = chcr_hmac_init; a_hash->setkey = chcr_ahash_setkey; a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 1d693b8436e6..e1e79e5f01e7 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -241,7 +241,6 @@ struct chcr_aead_ctx { }; struct hmac_ctx { - struct crypto_shash *base_hash; u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128]; u8 opad[CHCR_HASH_MAX_BLOCK_SIZE_128]; }; diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c index fa5a9f207bc9..d933f26aeb3a 100644 --- a/drivers/crypto/geode-aes.c +++ b/drivers/crypto/geode-aes.c @@ -433,4 +433,4 @@ module_pci_driver(geode_aes_driver); MODULE_AUTHOR("Advanced Micro Devices, Inc."); MODULE_DESCRIPTION("Geode LX Hardware AES driver"); MODULE_LICENSE("GPL"); -MODULE_IMPORT_NS(CRYPTO_INTERNAL); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 925991526745..edf36f6add52 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -913,11 +913,10 @@ static void hifn_init_pll(struct hifn_device *dev) else pllcfg |= HIFN_PLL_REF_CLK_HBI; - if (hifn_pll_ref[3] != '\0') - freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); - else { + if (hifn_pll_ref[3] == '\0' || + kstrtouint(hifn_pll_ref + 3, 10, &freq)) { freq = 66; - dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", + dev_info(&dev->pdev->dev, "assuming %u MHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", freq, hifn_pll_ref); } diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 4137a8bf131f..4835bdebdbb3 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -69,7 +69,6 @@ config CRYPTO_DEV_HISI_HPRE select CRYPTO_DEV_HISI_QM select CRYPTO_DH select CRYPTO_RSA - select CRYPTO_CURVE25519 select CRYPTO_ECDH help Support for HiSilicon HPRE(High Performance RSA Engine) diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c index 1b9b7bccdeff..17eb236e9ee4 100644 --- a/drivers/crypto/hisilicon/debugfs.c +++ b/drivers/crypto/hisilicon/debugfs.c @@ -192,7 +192,7 @@ static int qm_sqc_dump(struct hisi_qm *qm, char *s, char *name) down_read(&qm->qps_lock); if (qm->sqc) { - memcpy(&sqc, qm->sqc + qp_id * sizeof(struct qm_sqc), sizeof(struct qm_sqc)); + memcpy(&sqc, qm->sqc + qp_id, sizeof(struct qm_sqc)); sqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); sqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); dump_show(qm, &sqc, sizeof(struct qm_sqc), "SOFT SQC"); @@ -229,7 +229,7 @@ static int qm_cqc_dump(struct hisi_qm *qm, char *s, char *name) down_read(&qm->qps_lock); if (qm->cqc) { - memcpy(&cqc, qm->cqc + qp_id * sizeof(struct qm_cqc), sizeof(struct qm_cqc)); + memcpy(&cqc, qm->cqc + qp_id, sizeof(struct qm_cqc)); cqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); cqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); dump_show(qm, &cqc, sizeof(struct qm_cqc), "SOFT CQC"); @@ -888,6 +888,7 @@ static int qm_diff_regs_init(struct hisi_qm *qm, dfx_regs_uninit(qm, qm->debug.qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); ret = PTR_ERR(qm->debug.acc_diff_regs); qm->debug.acc_diff_regs = NULL; + qm->debug.qm_diff_regs = NULL; return ret; } diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 2a2910261210..21ccf879f70c 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2019 HiSilicon Limited. */ #include <crypto/akcipher.h> -#include <crypto/curve25519.h> #include <crypto/dh.h> #include <crypto/ecc_curve.h> #include <crypto/ecdh.h> @@ -39,6 +38,8 @@ struct hpre_ctx; #define HPRE_DFX_SEC_TO_US 1000000 #define HPRE_DFX_US_TO_NS 1000 +#define HPRE_ENABLE_HPCORE_SHIFT 7 + /* due to nist p521 */ #define HPRE_ECC_MAX_KSZ 66 @@ -104,16 +105,6 @@ struct hpre_ecdh_ctx { dma_addr_t dma_g; }; -struct hpre_curve25519_ctx { - /* low address: p->a->k */ - unsigned char *p; - dma_addr_t dma_p; - - /* gx coordinate */ - unsigned char *g; - dma_addr_t dma_g; -}; - struct hpre_ctx { struct hisi_qp *qp; struct device *dev; @@ -127,10 +118,11 @@ struct hpre_ctx { struct hpre_rsa_ctx rsa; struct hpre_dh_ctx dh; struct hpre_ecdh_ctx ecdh; - struct hpre_curve25519_ctx curve25519; }; /* for ecc algorithms */ unsigned int curve_id; + /* for high performance core */ + u8 enable_hpcore; }; struct hpre_asym_request { @@ -142,7 +134,6 @@ struct hpre_asym_request { struct akcipher_request *rsa; struct kpp_request *dh; struct kpp_request *ecdh; - struct kpp_request *curve25519; } areq; int err; int req_id; @@ -1210,8 +1201,7 @@ static void hpre_key_to_big_end(u8 *data, int len) } } -static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, - bool is_ecdh) +static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) { struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; @@ -1220,17 +1210,11 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, if (is_clear_all) hisi_qm_stop_qp(ctx->qp); - if (is_ecdh && ctx->ecdh.p) { + if (ctx->ecdh.p) { /* ecdh: p->a->k->b */ memzero_explicit(ctx->ecdh.p + shift, sz); dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); ctx->ecdh.p = NULL; - } else if (!is_ecdh && ctx->curve25519.p) { - /* curve25519: p->a->k */ - memzero_explicit(ctx->curve25519.p + shift, sz); - dma_free_coherent(dev, sz << 2, ctx->curve25519.p, - ctx->curve25519.dma_p); - ctx->curve25519.p = NULL; } hpre_ctx_clear(ctx, is_clear_all); @@ -1428,7 +1412,7 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, return -EINVAL; } - hpre_ecc_clear_ctx(ctx, false, true); + hpre_ecc_clear_ctx(ctx, false); ret = hpre_ecdh_set_param(ctx, ¶ms); if (ret < 0) { @@ -1487,11 +1471,13 @@ static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + /* Do unmap before data processing */ + hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); + p = sg_virt(areq->dst); memmove(p, p + ctx->key_sz - curve_sz, curve_sz); memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); - hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); kpp_request_complete(areq, ret); atomic64_inc(&dfx[HPRE_RECV_CNT].value); @@ -1619,6 +1605,8 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) } msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); + msg->resv1 = ctx->enable_hpcore << HPRE_ENABLE_HPCORE_SHIFT; + ret = hpre_send(ctx, msg); if (likely(!ret)) return -EINPROGRESS; @@ -1653,6 +1641,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); ctx->curve_id = ECC_CURVE_NIST_P256; + ctx->enable_hpcore = 1; kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); @@ -1674,335 +1663,7 @@ static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - hpre_ecc_clear_ctx(ctx, true, true); -} - -static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf, - unsigned int len) -{ - u8 secret[CURVE25519_KEY_SIZE] = { 0 }; - unsigned int sz = ctx->key_sz; - const struct ecc_curve *curve; - unsigned int shift = sz << 1; - void *p; - - /* - * The key from 'buf' is in little-endian, we should preprocess it as - * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64", - * then convert it to big endian. Only in this way, the result can be - * the same as the software curve-25519 that exists in crypto. - */ - memcpy(secret, buf, len); - curve25519_clamp_secret(secret); - hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE); - - p = ctx->curve25519.p + sz - len; - - curve = ecc_get_curve25519(); - - /* fill curve parameters */ - fill_curve_param(p, curve->p, len, curve->g.ndigits); - fill_curve_param(p + sz, curve->a, len, curve->g.ndigits); - memcpy(p + shift, secret, len); - fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits); - memzero_explicit(secret, CURVE25519_KEY_SIZE); -} - -static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf, - unsigned int len) -{ - struct device *dev = ctx->dev; - unsigned int sz = ctx->key_sz; - unsigned int shift = sz << 1; - - /* p->a->k->gx */ - if (!ctx->curve25519.p) { - ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2, - &ctx->curve25519.dma_p, - GFP_KERNEL); - if (!ctx->curve25519.p) - return -ENOMEM; - } - - ctx->curve25519.g = ctx->curve25519.p + shift + sz; - ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz; - - hpre_curve25519_fill_curve(ctx, buf, len); - - return 0; -} - -static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, - unsigned int len) -{ - struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = ctx->dev; - int ret = -EINVAL; - - if (len != CURVE25519_KEY_SIZE || - !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) { - dev_err(dev, "key is null or key len is not 32bytes!\n"); - return ret; - } - - /* Free old secret if any */ - hpre_ecc_clear_ctx(ctx, false, false); - - ctx->key_sz = CURVE25519_KEY_SIZE; - ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE); - if (ret) { - dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret); - hpre_ecc_clear_ctx(ctx, false, false); - return ret; - } - - return 0; -} - -static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, - struct hpre_asym_request *req, - struct scatterlist *dst, - struct scatterlist *src) -{ - struct device *dev = ctx->dev; - struct hpre_sqe *sqe = &req->req; - dma_addr_t dma; - - dma = le64_to_cpu(sqe->in); - if (unlikely(dma_mapping_error(dev, dma))) - return; - - if (src && req->src) - dma_free_coherent(dev, ctx->key_sz, req->src, dma); - - dma = le64_to_cpu(sqe->out); - if (unlikely(dma_mapping_error(dev, dma))) - return; - - if (req->dst) - dma_free_coherent(dev, ctx->key_sz, req->dst, dma); - if (dst) - dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE); -} - -static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp) -{ - struct hpre_dfx *dfx = ctx->hpre->debug.dfx; - struct hpre_asym_request *req = NULL; - struct kpp_request *areq; - u64 overtime_thrhld; - int ret; - - ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); - areq = req->areq.curve25519; - areq->dst_len = ctx->key_sz; - - overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); - if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) - atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); - - hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE); - - hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); - kpp_request_complete(areq, ret); - - atomic64_inc(&dfx[HPRE_RECV_CNT].value); -} - -static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, - struct kpp_request *req) -{ - struct hpre_asym_request *h_req; - struct hpre_sqe *msg; - int req_id; - void *tmp; - - if (unlikely(req->dst_len < ctx->key_sz)) { - req->dst_len = ctx->key_sz; - return -EINVAL; - } - - tmp = kpp_request_ctx(req); - h_req = PTR_ALIGN(tmp, hpre_align_sz()); - h_req->cb = hpre_curve25519_cb; - h_req->areq.curve25519 = req; - msg = &h_req->req; - memset(msg, 0, sizeof(*msg)); - msg->in = cpu_to_le64(DMA_MAPPING_ERROR); - msg->out = cpu_to_le64(DMA_MAPPING_ERROR); - msg->key = cpu_to_le64(ctx->curve25519.dma_p); - - msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); - msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; - h_req->ctx = ctx; - - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); - return 0; -} - -static void hpre_curve25519_src_modulo_p(u8 *ptr) -{ - int i; - - for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++) - ptr[i] = 0; - - /* The modulus is ptr's last byte minus '0xed'(last byte of p) */ - ptr[i] -= 0xed; -} - -static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, - struct scatterlist *data, unsigned int len) -{ - struct hpre_sqe *msg = &hpre_req->req; - struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = ctx->dev; - u8 p[CURVE25519_KEY_SIZE] = { 0 }; - const struct ecc_curve *curve; - dma_addr_t dma = 0; - u8 *ptr; - - if (len != CURVE25519_KEY_SIZE) { - dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len); - return -EINVAL; - } - - ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL); - if (unlikely(!ptr)) - return -ENOMEM; - - scatterwalk_map_and_copy(ptr, data, 0, len, 0); - - if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) { - dev_err(dev, "gx is null!\n"); - goto err; - } - - /* - * Src_data(gx) is in little-endian order, MSB in the final byte should - * be masked as described in RFC7748, then transform it to big-endian - * form, then hisi_hpre can use the data. - */ - ptr[31] &= 0x7f; - hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE); - - curve = ecc_get_curve25519(); - - fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits); - - /* - * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p, - * we get its modulus to p, and then use it. - */ - if (memcmp(ptr, p, ctx->key_sz) == 0) { - dev_err(dev, "gx is p!\n"); - goto err; - } else if (memcmp(ptr, p, ctx->key_sz) > 0) { - hpre_curve25519_src_modulo_p(ptr); - } - - hpre_req->src = ptr; - msg->in = cpu_to_le64(dma); - return 0; - -err: - dma_free_coherent(dev, ctx->key_sz, ptr, dma); - return -EINVAL; -} - -static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req, - struct scatterlist *data, unsigned int len) -{ - struct hpre_sqe *msg = &hpre_req->req; - struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = ctx->dev; - dma_addr_t dma; - - if (!data || !sg_is_last(data) || len != ctx->key_sz) { - dev_err(dev, "data or data length is illegal!\n"); - return -EINVAL; - } - - hpre_req->dst = NULL; - dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, dma))) { - dev_err(dev, "dma map data err!\n"); - return -ENOMEM; - } - - msg->out = cpu_to_le64(dma); - return 0; -} - -static int hpre_curve25519_compute_value(struct kpp_request *req) -{ - struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); - struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = ctx->dev; - void *tmp = kpp_request_ctx(req); - struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); - struct hpre_sqe *msg = &hpre_req->req; - int ret; - - ret = hpre_curve25519_msg_request_set(ctx, req); - if (unlikely(ret)) { - dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret); - return ret; - } - - if (req->src) { - ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len); - if (unlikely(ret)) { - dev_err(dev, "failed to init src data, ret = %d!\n", - ret); - goto clear_all; - } - } else { - msg->in = cpu_to_le64(ctx->curve25519.dma_g); - } - - ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len); - if (unlikely(ret)) { - dev_err(dev, "failed to init dst data, ret = %d!\n", ret); - goto clear_all; - } - - msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL); - ret = hpre_send(ctx, msg); - if (likely(!ret)) - return -EINPROGRESS; - -clear_all: - hpre_rm_req_from_ctx(hpre_req); - hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); - return ret; -} - -static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm) -{ - struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - - return ctx->key_sz; -} - -static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm) -{ - struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); -} - -static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm) -{ - struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - - hpre_ecc_clear_ctx(ctx, true, false); + hpre_ecc_clear_ctx(ctx, true); } static struct akcipher_alg rsa = { @@ -2084,22 +1745,6 @@ static struct kpp_alg ecdh_curves[] = { } }; -static struct kpp_alg curve25519_alg = { - .set_secret = hpre_curve25519_set_secret, - .generate_public_key = hpre_curve25519_compute_value, - .compute_shared_secret = hpre_curve25519_compute_value, - .max_size = hpre_curve25519_max_size, - .init = hpre_curve25519_init_tfm, - .exit = hpre_curve25519_exit_tfm, - .base = { - .cra_ctxsize = sizeof(struct hpre_ctx), - .cra_priority = HPRE_CRYPTO_ALG_PRI, - .cra_name = "curve25519", - .cra_driver_name = "hpre-curve25519", - .cra_module = THIS_MODULE, - }, -}; - static int hpre_register_rsa(struct hisi_qm *qm) { int ret; @@ -2181,28 +1826,6 @@ static void hpre_unregister_ecdh(struct hisi_qm *qm) crypto_unregister_kpp(&ecdh_curves[i]); } -static int hpre_register_x25519(struct hisi_qm *qm) -{ - int ret; - - if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) - return 0; - - ret = crypto_register_kpp(&curve25519_alg); - if (ret) - dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret); - - return ret; -} - -static void hpre_unregister_x25519(struct hisi_qm *qm) -{ - if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) - return; - - crypto_unregister_kpp(&curve25519_alg); -} - int hpre_algs_register(struct hisi_qm *qm) { int ret = 0; @@ -2225,17 +1848,11 @@ int hpre_algs_register(struct hisi_qm *qm) if (ret) goto unreg_dh; - ret = hpre_register_x25519(qm); - if (ret) - goto unreg_ecdh; - hpre_available_devs++; mutex_unlock(&hpre_algs_lock); return ret; -unreg_ecdh: - hpre_unregister_ecdh(qm); unreg_dh: hpre_unregister_dh(qm); unreg_rsa: @@ -2251,7 +1868,6 @@ void hpre_algs_unregister(struct hisi_qm *qm) if (--hpre_available_devs) goto unlock; - hpre_unregister_x25519(qm); hpre_unregister_ecdh(qm); hpre_unregister_dh(qm); hpre_unregister_rsa(qm); diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 96fde9437b4b..b94fecd765ee 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -39,6 +39,7 @@ #define HPRE_HAC_RAS_NFE_ENB 0x301414 #define HPRE_HAC_RAS_FE_ENB 0x301418 #define HPRE_HAC_INT_SET 0x301500 +#define HPRE_AXI_ERROR_MASK GENMASK(21, 10) #define HPRE_RNG_TIMEOUT_NUM 0x301A34 #define HPRE_CORE_INT_ENABLE 0 #define HPRE_RDCHN_INI_ST 0x301a00 @@ -78,6 +79,11 @@ #define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30))) #define HPRE_PREFETCH_DISABLE BIT(30) #define HPRE_SVA_DISABLE_READY (BIT(4) | BIT(8)) +#define HPRE_SVA_PREFTCH_DFX4 0x301144 +#define HPRE_WAIT_SVA_READY 500000 +#define HPRE_READ_SVA_STATUS_TIMES 3 +#define HPRE_WAIT_US_MIN 10 +#define HPRE_WAIT_US_MAX 20 /* clock gate */ #define HPRE_CLKGATE_CTL 0x301a10 @@ -466,6 +472,33 @@ struct hisi_qp *hpre_create_qp(u8 type) return NULL; } +static int hpre_wait_sva_ready(struct hisi_qm *qm) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + HPRE_SVA_PREFTCH_DFX4); + if (val) + count = 0; + else if (++count == HPRE_READ_SVA_STATUS_TIMES) + break; + + usleep_range(HPRE_WAIT_US_MIN, HPRE_WAIT_US_MAX); + } while (++try_times < HPRE_WAIT_SVA_READY); + + if (try_times == HPRE_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + static void hpre_config_pasid(struct hisi_qm *qm) { u32 val1, val2; @@ -563,7 +596,7 @@ static void disable_flr_of_bme(struct hisi_qm *qm) writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); } -static void hpre_open_sva_prefetch(struct hisi_qm *qm) +static void hpre_close_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -571,20 +604,21 @@ static void hpre_open_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; - /* Enable prefetch */ val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); - val &= HPRE_PREFETCH_ENABLE; + val |= HPRE_PREFETCH_DISABLE; writel(val, qm->io_base + HPRE_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG, - val, !(val & HPRE_PREFETCH_DISABLE), + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX, + val, !(val & HPRE_SVA_DISABLE_READY), HPRE_REG_RD_INTVRL_US, HPRE_REG_RD_TMOUT_US); if (ret) - pci_err(qm->pdev, "failed to open sva prefetch\n"); + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)hpre_wait_sva_ready(qm); } -static void hpre_close_sva_prefetch(struct hisi_qm *qm) +static void hpre_open_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -592,16 +626,24 @@ static void hpre_close_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; + /* Enable prefetch */ val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); - val |= HPRE_PREFETCH_DISABLE; + val &= HPRE_PREFETCH_ENABLE; writel(val, qm->io_base + HPRE_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX, - val, !(val & HPRE_SVA_DISABLE_READY), + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG, + val, !(val & HPRE_PREFETCH_DISABLE), HPRE_REG_RD_INTVRL_US, HPRE_REG_RD_TMOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + hpre_close_sva_prefetch(qm); + return; + } + + ret = hpre_wait_sva_ready(qm); if (ret) - pci_err(qm->pdev, "failed to close sva prefetch\n"); + hpre_close_sva_prefetch(qm); } static void hpre_enable_clock_gate(struct hisi_qm *qm) @@ -721,6 +763,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) /* Config data buffer pasid needed by Kunpeng 920 */ hpre_config_pasid(qm); + hpre_open_sva_prefetch(qm); hpre_enable_clock_gate(qm); @@ -756,8 +799,7 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); if (enable) { val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE; - val2 = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + val2 = qm->err_info.dev_err.shutdown_mask; } else { val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; val2 = 0x0; @@ -771,38 +813,33 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hpre_hw_error_disable(struct hisi_qm *qm) { - u32 ce, nfe; - - ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); - nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; /* disable hpre hw error interrupts */ - writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_INT_MASK); + writel(err_mask, qm->io_base + HPRE_INT_MASK); /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ hpre_master_ooo_ctrl(qm, false); } static void hpre_hw_error_enable(struct hisi_qm *qm) { - u32 ce, nfe, err_en; - - ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); - nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; /* clear HPRE hw error source if having */ - writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_HAC_SOURCE_INT); + writel(err_mask, qm->io_base + HPRE_HAC_SOURCE_INT); /* configure error type */ - writel(ce, qm->io_base + HPRE_RAS_CE_ENB); - writel(nfe, qm->io_base + HPRE_RAS_NFE_ENB); - writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB); + writel(dev_err->ce, qm->io_base + HPRE_RAS_CE_ENB); + writel(dev_err->nfe, qm->io_base + HPRE_RAS_NFE_ENB); + writel(dev_err->fe, qm->io_base + HPRE_RAS_FE_ENB); /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */ hpre_master_ooo_ctrl(qm, true); /* enable hpre hw error interrupts */ - err_en = ce | nfe | HPRE_HAC_RAS_FE_ENABLE; - writel(~err_en, qm->io_base + HPRE_INT_MASK); + writel(~err_mask, qm->io_base + HPRE_INT_MASK); } static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) @@ -1171,7 +1208,7 @@ static int hpre_pre_store_cap_reg(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(hpre_cap_query_info); - hpre_cap = devm_kzalloc(dev, sizeof(*hpre_cap) * size, GFP_KERNEL); + hpre_cap = devm_kcalloc(dev, size, sizeof(*hpre_cap), GFP_KERNEL); if (!hpre_cap) return -ENOMEM; @@ -1209,7 +1246,6 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->mode = uacce_mode; qm->pdev = pdev; - qm->ver = pdev->revision; qm->sqe_size = HPRE_SQE_SIZE; qm->dev_name = hpre_name; @@ -1358,12 +1394,20 @@ static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) static void hpre_disable_error_report(struct hisi_qm *qm, u32 err_type) { - u32 nfe_mask; + u32 nfe_mask = qm->err_info.dev_err.nfe; - nfe_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); writel(nfe_mask & (~err_type), qm->io_base + HPRE_RAS_NFE_ENB); } +static void hpre_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + HPRE_RAS_NFE_ENB); + writel(ce_mask, qm->io_base + HPRE_RAS_CE_ENB); +} + static void hpre_open_axi_master_ooo(struct hisi_qm *qm) { u32 value; @@ -1381,37 +1425,86 @@ static enum acc_err_result hpre_get_err_result(struct hisi_qm *qm) err_status = hpre_get_hw_err_status(qm); if (err_status) { - if (err_status & qm->err_info.ecc_2bits_mask) + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; hpre_log_hw_error(qm, err_status); - if (err_status & qm->err_info.dev_reset_mask) { + if (err_status & qm->err_info.dev_err.reset_mask) { /* Disable the same error reporting until device is recovered. */ hpre_disable_error_report(qm, err_status); return ACC_ERR_NEED_RESET; } hpre_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hpre_enable_error_report(qm); } return ACC_ERR_RECOVERED; } +static bool hpre_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hpre_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_err.shutdown_mask) + return true; + + return false; +} + +static void hpre_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + u32 val; + + val = ~(err_mask & (~HPRE_AXI_ERROR_MASK)); + writel(val, qm->io_base + HPRE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~HPRE_AXI_ERROR_MASK), + qm->io_base + HPRE_OOO_SHUTDOWN_SEL); +} + +static void hpre_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(HPRE_AXI_ERROR_MASK, qm->io_base + HPRE_HAC_SOURCE_INT); + + writel(~err_mask, qm->io_base + HPRE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + HPRE_OOO_SHUTDOWN_SEL); +} + static void hpre_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = HPRE_HAC_RAS_FE_ENABLE; + qm_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + qm_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + + dev_err->fe = HPRE_HAC_RAS_FE_ENABLE; + dev_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_RESET_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR; - err_info->fe = HPRE_HAC_RAS_FE_ENABLE; - err_info->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver); - err_info->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver); - err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR; - err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_QM_RESET_MASK_CAP, qm->cap_ver); - err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, - HPRE_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HPRE_WR_MSI_PORT; err_info->acpi_rst = "HRST"; } @@ -1428,6 +1521,9 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .show_last_dfx_regs = hpre_show_last_dfx_regs, .err_info_init = hpre_err_info_init, .get_err_result = hpre_get_err_result, + .dev_is_abnormal = hpre_dev_is_abnormal, + .disable_axi_error = hpre_disable_axi_error, + .enable_axi_error = hpre_enable_axi_error, }; static int hpre_pf_probe_init(struct hpre *hpre) @@ -1439,8 +1535,6 @@ static int hpre_pf_probe_init(struct hpre *hpre) if (ret) return ret; - hpre_open_sva_prefetch(qm); - hisi_qm_dev_err_init(qm); ret = hpre_show_last_regs_init(qm); if (ret) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 19c1b5d3c954..f8bfff5dd0bd 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -30,8 +30,6 @@ /* mailbox */ #define QM_MB_PING_ALL_VFS 0xffff -#define QM_MB_CMD_DATA_SHIFT 32 -#define QM_MB_CMD_DATA_MASK GENMASK(31, 0) #define QM_MB_STATUS_MASK GENMASK(12, 9) /* sqc shift */ @@ -47,6 +45,8 @@ #define QM_SQ_TYPE_MASK GENMASK(3, 0) #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc).w11) >> 6) & 0x1) +#define QM_SQC_DISABLE_QP (1U << 6) +#define QM_XQC_RANDOM_DATA 0xaaaa /* cqc shift */ #define QM_CQ_HOP_NUM_SHIFT 0 @@ -64,10 +64,10 @@ #define QM_EQE_AEQE_SIZE (2UL << 12) #define QM_EQC_PHASE_SHIFT 16 -#define QM_EQE_PHASE(eqe) ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1) +#define QM_EQE_PHASE(dw0) (((dw0) >> 16) & 0x1) #define QM_EQE_CQN_MASK GENMASK(15, 0) -#define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1) +#define QM_AEQE_PHASE(dw0) (((dw0) >> 16) & 0x1) #define QM_AEQE_TYPE_SHIFT 17 #define QM_AEQE_TYPE_MASK 0xf #define QM_AEQE_CQN_MASK GENMASK(15, 0) @@ -102,6 +102,8 @@ #define QM_PM_CTRL 0x100148 #define QM_IDLE_DISABLE BIT(9) +#define QM_SUB_VERSION_ID 0x210 + #define QM_VFT_CFG_DATA_L 0x100064 #define QM_VFT_CFG_DATA_H 0x100068 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8) @@ -119,6 +121,7 @@ #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) #define QM_SQC_VFT_NUM_SHIFT_V2 45 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) +#define QM_MAX_QC_TYPE 2 #define QM_ABNORMAL_INT_SOURCE 0x100000 #define QM_ABNORMAL_INT_MASK 0x100004 @@ -144,9 +147,9 @@ #define QM_RAS_CE_TIMES_PER_IRQ 1 #define QM_OOO_SHUTDOWN_SEL 0x1040f8 #define QM_AXI_RRESP_ERR BIT(0) -#define QM_ECC_MBIT BIT(2) #define QM_DB_TIMEOUT BIT(10) #define QM_OF_FIFO_OF BIT(11) +#define QM_RAS_AXI_ERROR (BIT(0) | BIT(1) | BIT(12)) #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 @@ -162,7 +165,6 @@ #define ACC_MASTER_TRANS_RETURN 0x300150 #define ACC_MASTER_GLOBAL_CTRL 0x300000 #define ACC_AM_CFG_PORT_WR_EN 0x30001c -#define QM_RAS_NFE_MBIT_DISABLE ~QM_ECC_MBIT #define ACC_AM_ROB_ECC_INT_STS 0x300104 #define ACC_ROB_ECC_ERR_MULTPL BIT(1) #define QM_MSI_CAP_ENABLE BIT(16) @@ -176,6 +178,10 @@ #define QM_IFC_INT_MASK 0x0024 #define QM_IFC_INT_STATUS 0x0028 #define QM_IFC_INT_SET_V 0x002C +#define QM_PF2VF_PF_W 0x104700 +#define QM_VF2PF_PF_R 0x104800 +#define QM_VF2PF_VF_W 0x320 +#define QM_PF2VF_VF_R 0x380 #define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) #define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) #define QM_IFC_INT_SOURCE_MASK BIT(0) @@ -185,8 +191,11 @@ #define QM_WAIT_DST_ACK 10 #define QM_MAX_PF_WAIT_COUNT 10 #define QM_MAX_VF_WAIT_COUNT 40 -#define QM_VF_RESET_WAIT_US 20000 -#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF_RESET_WAIT_US 20000 +#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF2PF_REG_SIZE 4 +#define QM_IFC_CMD_MASK GENMASK(31, 0) +#define QM_IFC_DATA_SHIFT 32 #define QM_VF_RESET_WAIT_TIMEOUT_US \ (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) @@ -234,8 +243,6 @@ #define QM_QOS_MAX_CIR_U 6 #define QM_AUTOSUSPEND_DELAY 3000 -#define QM_DEV_ALG_MAX_LEN 256 - /* abnormal status value for stopping queue */ #define QM_STOP_QUEUE_FAIL 1 #define QM_DUMP_SQC_FAIL 3 @@ -276,7 +283,7 @@ enum qm_alg_type { ALG_TYPE_1, }; -enum qm_mb_cmd { +enum qm_ifc_cmd { QM_PF_FLR_PREPARE = 0x01, QM_PF_SRST_PREPARE, QM_PF_RESET_DONE, @@ -333,6 +340,7 @@ static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, + {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0}, }; static const struct hisi_qm_cap_info qm_cap_info_pf[] = { @@ -396,6 +404,11 @@ struct hisi_qm_hw_ops { void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); int (*set_msi)(struct hisi_qm *qm, bool set); + + /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */ + int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num); + void (*set_ifc_end)(struct hisi_qm *qm); + int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num); }; struct hisi_qm_hw_error { @@ -501,15 +514,20 @@ static u32 qm_get_dev_err_status(struct hisi_qm *qm) /* Check if the error causes the master ooo block */ static bool qm_check_dev_error(struct hisi_qm *qm) { - u32 val, dev_val; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + u32 err_status; - if (qm->fun_type == QM_HW_VF) + if (pf_qm->fun_type == QM_HW_VF) return false; - val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask; - dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask; + err_status = qm_get_hw_error_status(pf_qm); + if (err_status & pf_qm->err_info.qm_err.shutdown_mask) + return true; + + if (pf_qm->err_ini->dev_is_abnormal) + return pf_qm->err_ini->dev_is_abnormal(pf_qm); - return val || dev_val; + return false; } static int qm_wait_reset_finish(struct hisi_qm *qm) @@ -654,7 +672,6 @@ EXPORT_SYMBOL_GPL(hisi_qm_mb); /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) { - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); struct qm_mailbox mailbox; dma_addr_t xqc_dma; void *tmp_xqc; @@ -688,7 +705,7 @@ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op } /* Setting xqc will fail if master OOO is blocked. */ - if (qm_check_dev_error(pf_qm)) { + if (qm_check_dev_error(qm)) { dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n"); return -EIO; } @@ -846,7 +863,7 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d return -EINVAL; } - algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL); if (!algs) return -ENOMEM; @@ -855,10 +872,10 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d strcat(algs, dev_algs[i].alg); ptr = strrchr(algs, '\n'); - if (ptr) { + if (ptr) *ptr = '\0'; - qm->uacce->algs = algs; - } + + qm->uacce->algs = algs; return 0; } @@ -896,7 +913,6 @@ static void qm_pm_put_sync(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return; - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); } @@ -960,23 +976,23 @@ static void qm_get_complete_eqe_num(struct hisi_qm *qm) { struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; struct hisi_qm_poll_data *poll_data = NULL; + u32 dw0 = le32_to_cpu(eqe->dw0); u16 eq_depth = qm->eq_depth; u16 cqn, eqe_num = 0; - if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) { + if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { atomic64_inc(&qm->debug.dfx.err_irq_cnt); qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); return; } - cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; + cqn = dw0 & QM_EQE_CQN_MASK; if (unlikely(cqn >= qm->qp_num)) return; poll_data = &qm->poll_data[cqn]; - while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { - cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; - poll_data->qp_finish_id[eqe_num] = cqn; + while (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { + poll_data->qp_finish_id[eqe_num] = dw0 & QM_EQE_CQN_MASK; eqe_num++; if (qm->status.eq_head == eq_depth - 1) { @@ -990,6 +1006,8 @@ static void qm_get_complete_eqe_num(struct hisi_qm *qm) if (eqe_num == (eq_depth >> 1) - 1) break; + + dw0 = le32_to_cpu(eqe->dw0); } poll_data->eqe_num = eqe_num; @@ -1052,11 +1070,10 @@ static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) static void qm_reset_function(struct hisi_qm *qm) { - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); struct device *dev = &qm->pdev->dev; int ret; - if (qm_check_dev_error(pf_qm)) + if (qm_check_dev_error(qm)) return; ret = qm_reset_prepare_ready(qm); @@ -1083,15 +1100,15 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; + u32 dw0 = le32_to_cpu(aeqe->dw0); u16 aeq_depth = qm->aeq_depth; u32 type, qp_id; atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); - while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { - type = (le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT) & - QM_AEQE_TYPE_MASK; - qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK; + while (QM_AEQE_PHASE(dw0) == qm->status.aeqc_phase) { + type = (dw0 >> QM_AEQE_TYPE_SHIFT) & QM_AEQE_TYPE_MASK; + qp_id = dw0 & QM_AEQE_CQN_MASK; switch (type) { case QM_EQ_OVERFLOW: @@ -1119,6 +1136,7 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) aeqe++; qm->status.aeq_head++; } + dw0 = le32_to_cpu(aeqe->dw0); } qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); @@ -1268,6 +1286,13 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); } break; + /* + * Note: The current logic only needs to handle the above three types + * If new types are added, they need to be supplemented here, + * otherwise undefined behavior may occur. + */ + default: + break; } } @@ -1381,17 +1406,17 @@ static void qm_hw_error_init_v1(struct hisi_qm *qm) static void qm_hw_error_cfg(struct hisi_qm *qm) { - struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; - qm->error_mask = err_info->nfe | err_info->ce | err_info->fe; + qm->error_mask = qm_err->nfe | qm_err->ce | qm_err->fe; /* clear QM hw residual error source */ writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); /* configure error type */ - writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE); + writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); - writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE); - writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->fe, qm->io_base + QM_RAS_FE_ENABLE); } static void qm_hw_error_init_v2(struct hisi_qm *qm) @@ -1420,7 +1445,7 @@ static void qm_hw_error_init_v3(struct hisi_qm *qm) qm_hw_error_cfg(qm); /* enable close master ooo when hardware error happened */ - writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); irq_unmask = ~qm->error_mask; irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); @@ -1482,6 +1507,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; u32 error_status; error_status = qm_get_hw_error_status(qm); @@ -1490,17 +1516,16 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); - if (error_status & qm->err_info.qm_reset_mask) { + if (error_status & qm_err->reset_mask) { /* Disable the same error reporting until device is recovered. */ - writel(qm->err_info.nfe & (~error_status), - qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->nfe & (~error_status), qm->io_base + QM_RAS_NFE_ENABLE); return ACC_ERR_NEED_RESET; } /* Clear error source if not need reset. */ writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); - writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE); - writel(qm->err_info.ce, qm->io_base + QM_RAS_CE_ENABLE); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); } return ACC_ERR_RECOVERED; @@ -1540,17 +1565,15 @@ static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) { struct device *dev = &qm->pdev->dev; - u32 cmd; - u64 msg; + enum qm_ifc_cmd cmd; int ret; - ret = qm_get_mb_cmd(qm, &msg, vf_id); + ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id); if (ret) { - dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id); + dev_err(dev, "failed to get command from VF(%u)!\n", vf_id); return; } - cmd = msg & QM_MB_CMD_DATA_MASK; switch (cmd) { case QM_VF_PREPARE_FAIL: dev_err(dev, "failed to stop VF(%u)!\n", vf_id); @@ -1562,7 +1585,7 @@ static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) case QM_VF_START_DONE: break; default: - dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id); + dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id); break; } } @@ -1630,17 +1653,14 @@ static void qm_trigger_pf_interrupt(struct hisi_qm *qm) writel(val, qm->io_base + QM_IFC_INT_SET_V); } -static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) +static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) { struct device *dev = &qm->pdev->dev; - struct qm_mailbox mailbox; int cnt = 0; u64 val; int ret; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0); - mutex_lock(&qm->mailbox_lock); - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num); if (ret) { dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); goto err_unlock; @@ -1662,27 +1682,23 @@ static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) } err_unlock: - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); return ret; } -static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) +static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { struct device *dev = &qm->pdev->dev; u32 vfs_num = qm->vfs_num; - struct qm_mailbox mailbox; u64 val = 0; int cnt = 0; int ret; u32 i; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0); - mutex_lock(&qm->mailbox_lock); - /* PF sends command to all VFs by mailbox */ - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS); if (ret) { - dev_err(dev, "failed to send command to VFs!\n"); - mutex_unlock(&qm->mailbox_lock); + dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); + qm->ops->set_ifc_end(qm); return ret; } @@ -1692,7 +1708,7 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) val = readq(qm->io_base + QM_IFC_READY_STATUS); /* If all VFs acked, PF notifies VFs successfully. */ if (!(val & GENMASK(vfs_num, 1))) { - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); return 0; } @@ -1700,7 +1716,7 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) break; } - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); /* Check which vf respond timeout. */ for (i = 1; i <= vfs_num; i++) { @@ -1711,18 +1727,15 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) return -ETIMEDOUT; } -static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) +static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { - struct qm_mailbox mailbox; int cnt = 0; u32 val; int ret; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0); - mutex_lock(&qm->mailbox_lock); - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0); if (ret) { - dev_err(&qm->pdev->dev, "failed to send command to PF!\n"); + dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd); goto unlock; } @@ -1741,7 +1754,8 @@ static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) } unlock: - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); + return ret; } @@ -1842,6 +1856,94 @@ static int qm_set_msi_v3(struct hisi_qm *qm, bool set) return ret; } +static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + struct qm_mailbox mailbox; + u64 msg; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + return qm_mb_nolock(qm, &mailbox); +} + +static void qm_set_ifc_end_v3(struct hisi_qm *qm) +{ + mutex_unlock(&qm->mailbox_lock); +} + +static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + int ret; + + ret = qm_get_mb_cmd(qm, &msg, fun_num); + if (ret) + return ret; + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + +static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + uintptr_t offset; + u64 msg; + + if (qm->fun_type == QM_HW_PF) + offset = QM_PF2VF_PF_W; + else + offset = QM_VF2PF_VF_W; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + mutex_lock(&qm->ifc_lock); + writeq(msg, qm->io_base + offset); + + return 0; +} + +static void qm_set_ifc_end_v4(struct hisi_qm *qm) +{ + mutex_unlock(&qm->ifc_lock); +} + +static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num) +{ + uintptr_t offset; + + offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num; + + return (u64)readl(qm->io_base + offset); +} + +static u64 qm_get_ifc_vf(struct hisi_qm *qm) +{ + return readq(qm->io_base + QM_PF2VF_VF_R); +} + +static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + + if (qm->fun_type == QM_HW_PF) + msg = qm_get_ifc_pf(qm, fun_num); + else + msg = qm_get_ifc_vf(qm); + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, .hw_error_init = qm_hw_error_init_v1, @@ -1864,6 +1966,21 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v3, + .set_ifc_end = qm_set_ifc_end_v3, + .get_ifc = qm_get_ifc_v3, +}; + +static const struct hisi_qm_hw_ops qm_hw_ops_v4 = { + .get_vft = qm_get_vft_v2, + .qm_db = qm_db_v2, + .hw_error_init = qm_hw_error_init_v3, + .hw_error_uninit = qm_hw_error_uninit_v3, + .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v4, + .set_ifc_end = qm_set_ifc_end_v4, + .get_ifc = qm_get_ifc_v4, }; static void *qm_get_avail_sqe(struct hisi_qp *qp) @@ -2156,12 +2273,11 @@ static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) static int qm_drain_qp(struct hisi_qp *qp) { struct hisi_qm *qm = qp->qm; - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); u32 state = 0; int ret; /* No need to judge if master OOO is blocked. */ - if (qm_check_dev_error(pf_qm)) + if (qm_check_dev_error(qm)) return 0; /* HW V3 supports drain qp by device */ @@ -2475,7 +2591,7 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, sizeof(struct hisi_qp_ctx))) return -EFAULT; - if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1) + if (qp_ctx.qc_type > QM_MAX_QC_TYPE) return -EINVAL; qm_set_sqctype(q, qp_ctx.qc_type); @@ -2546,10 +2662,10 @@ static int qm_hw_err_isolate(struct hisi_qm *qm) } } list_add(&hw_err->list, &isolate->qm_hw_errs); - mutex_unlock(&isolate->isolate_lock); if (count >= isolate->err_threshold) isolate->is_isolate = true; + mutex_unlock(&isolate->isolate_lock); return 0; } @@ -2558,12 +2674,10 @@ static void qm_hw_err_destroy(struct hisi_qm *qm) { struct qm_hw_err *err, *tmp; - mutex_lock(&qm->isolate_data.isolate_lock); list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) { list_del(&err->list); kfree(err); } - mutex_unlock(&qm->isolate_data.isolate_lock); } static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce) @@ -2591,10 +2705,12 @@ static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) if (qm->isolate_data.is_isolate) return -EPERM; + mutex_lock(&qm->isolate_data.isolate_lock); qm->isolate_data.err_threshold = num; /* After the policy is updated, need to reset the hardware err list */ qm_hw_err_destroy(qm); + mutex_unlock(&qm->isolate_data.isolate_lock); return 0; } @@ -2631,12 +2747,36 @@ static void qm_remove_uacce(struct hisi_qm *qm) struct uacce_device *uacce = qm->uacce; if (qm->use_sva) { + mutex_lock(&qm->isolate_data.isolate_lock); qm_hw_err_destroy(qm); + mutex_unlock(&qm->isolate_data.isolate_lock); + uacce_remove(uacce); qm->uacce = NULL; } } +static void qm_uacce_api_ver_init(struct hisi_qm *qm) +{ + struct uacce_device *uacce = qm->uacce; + + switch (qm->ver) { + case QM_HW_V1: + uacce->api_ver = HISI_QM_API_VER_BASE; + break; + case QM_HW_V2: + uacce->api_ver = HISI_QM_API_VER2_BASE; + break; + case QM_HW_V3: + case QM_HW_V4: + uacce->api_ver = HISI_QM_API_VER3_BASE; + break; + default: + uacce->api_ver = HISI_QM_API_VER5_BASE; + break; + } +} + static int qm_alloc_uacce(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -2671,13 +2811,6 @@ static int qm_alloc_uacce(struct hisi_qm *qm) uacce->priv = qm; if (qm->ver == QM_HW_V1) - uacce->api_ver = HISI_QM_API_VER_BASE; - else if (qm->ver == QM_HW_V2) - uacce->api_ver = HISI_QM_API_VER2_BASE; - else - uacce->api_ver = HISI_QM_API_VER3_BASE; - - if (qm->ver == QM_HW_V1) mmio_page_nr = QM_DOORBELL_PAGE_NR; else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) mmio_page_nr = QM_DOORBELL_PAGE_NR + @@ -2696,6 +2829,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; qm->uacce = uacce; + qm_uacce_api_ver_init(qm); INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs); mutex_init(&qm->isolate_data.isolate_lock); @@ -2843,11 +2977,14 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->ops = &qm_hw_ops_v1; else if (qm->ver == QM_HW_V2) qm->ops = &qm_hw_ops_v2; - else + else if (qm->ver == QM_HW_V3) qm->ops = &qm_hw_ops_v3; + else + qm->ops = &qm_hw_ops_v4; pci_set_drvdata(pdev, qm); mutex_init(&qm->mailbox_lock); + mutex_init(&qm->ifc_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { @@ -2895,11 +3032,36 @@ static void qm_put_pci_res(struct hisi_qm *qm) pci_release_mem_regions(pdev); } +static void hisi_mig_region_clear(struct hisi_qm *qm) +{ + u32 val; + + /* Clear migration region set of PF */ + if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { + val = readl(qm->io_base + QM_MIG_REGION_SEL); + val &= ~QM_MIG_REGION_EN; + writel(val, qm->io_base + QM_MIG_REGION_SEL); + } +} + +static void hisi_mig_region_enable(struct hisi_qm *qm) +{ + u32 val; + + /* Select migration region of PF */ + if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { + val = readl(qm->io_base + QM_MIG_REGION_SEL); + val |= QM_MIG_REGION_EN; + writel(val, qm->io_base + QM_MIG_REGION_SEL); + } +} + static void hisi_qm_pci_uninit(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; pci_free_irq_vectors(pdev); + hisi_mig_region_clear(qm); qm_put_pci_res(qm); pci_disable_device(pdev); } @@ -3071,6 +3233,9 @@ static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) qm_init_eq_aeq_status(qm); + /* Before starting the dev, clear the memory and then configure to device using. */ + memset(qm->qdma.va, 0, qm->qdma.size); + ret = qm_eq_ctx_cfg(qm); if (ret) { dev_err(dev, "Set eqc failed!\n"); @@ -3082,9 +3247,13 @@ static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) static int __hisi_qm_start(struct hisi_qm *qm) { + struct device *dev = &qm->pdev->dev; int ret; - WARN_ON(!qm->qdma.va); + if (!qm->qdma.va) { + dev_err(dev, "qm qdma is NULL!\n"); + return -EINVAL; + } if (qm->fun_type == QM_HW_PF) { ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); @@ -3158,7 +3327,7 @@ static int qm_restart(struct hisi_qm *qm) for (i = 0; i < qm->qp_num; i++) { qp = &qm->qp_array[i]; if (atomic_read(&qp->qp_status.flags) == QP_STOP && - qp->is_resetting == true) { + qp->is_resetting == true && qp->is_in_kernel == true) { ret = qm_start_qp_nolock(qp, 0); if (ret < 0) { dev_err(dev, "Failed to start qp%d!\n", i); @@ -3190,24 +3359,44 @@ static void qm_stop_started_qp(struct hisi_qm *qm) } /** - * qm_clear_queues() - Clear all queues memory in a qm. - * @qm: The qm in which the queues will be cleared. + * qm_invalid_queues() - invalid all queues in use. + * @qm: The qm in which the queues will be invalidated. * - * This function clears all queues memory in a qm. Reset of accelerator can - * use this to clear queues. + * This function invalid all queues in use. If the doorbell command is sent + * to device in user space after the device is reset, the device discards + * the doorbell command. */ -static void qm_clear_queues(struct hisi_qm *qm) +static void qm_invalid_queues(struct hisi_qm *qm) { struct hisi_qp *qp; + struct qm_sqc *sqc; + struct qm_cqc *cqc; int i; + /* + * Normal stop queues is no longer used and does not need to be + * invalid queues. + */ + if (qm->status.stop_reason == QM_NORMAL) + return; + + if (qm->status.stop_reason == QM_DOWN) + hisi_qm_cache_wb(qm); + for (i = 0; i < qm->qp_num; i++) { qp = &qm->qp_array[i]; - if (qp->is_in_kernel && qp->is_resetting) + if (!qp->is_resetting) + continue; + + /* Modify random data and set sqc close bit to invalid queue. */ + sqc = qm->sqc + i; + cqc = qm->cqc + i; + sqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); + sqc->w13 = cpu_to_le16(QM_SQC_DISABLE_QP); + cqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); + if (qp->is_in_kernel) memset(qp->qdma.va, 0, qp->qdma.size); } - - memset(qm->qdma.va, 0, qm->qdma.size); } /** @@ -3264,7 +3453,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) } } - qm_clear_queues(qm); + qm_invalid_queues(qm); qm->status.stop_reason = QM_NORMAL; err_unlock: @@ -3509,24 +3698,25 @@ static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) return 0; } -static int qm_clear_vft_config(struct hisi_qm *qm) +static void qm_clear_vft_config(struct hisi_qm *qm) { - int ret; u32 i; - for (i = 1; i <= qm->vfs_num; i++) { - ret = hisi_qm_set_vft(qm, i, 0, 0); - if (ret) - return ret; - } - qm->vfs_num = 0; + /* + * When disabling SR-IOV, clear the configuration of each VF in the hardware + * sequentially. Failure to clear a single VF should not affect the clearing + * operation of other VFs. + */ + for (i = 1; i <= qm->vfs_num; i++) + (void)hisi_qm_set_vft(qm, i, 0, 0); - return 0; + qm->vfs_num = 0; } static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) { struct device *dev = &qm->pdev->dev; + struct qm_shaper_factor t_factor; u32 ir = qos * QM_QOS_RATE; int ret, total_vfs, i; @@ -3534,6 +3724,7 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) if (fun_index > total_vfs) return -EINVAL; + memcpy(&t_factor, &qm->factor[fun_index], sizeof(t_factor)); qm->factor[fun_index].func_qos = qos; ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); @@ -3547,11 +3738,21 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); if (ret) { dev_err(dev, "type: %d, failed to set shaper vft!\n", i); - return -EINVAL; + goto back_func_qos; } } return 0; + +back_func_qos: + memcpy(&qm->factor[fun_index], &t_factor, sizeof(t_factor)); + for (i--; i >= ALG_TYPE_0; i--) { + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) + dev_err(dev, "failed to restore shaper vft during rollback!\n"); + } + + return -EINVAL; } static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) @@ -3607,7 +3808,6 @@ static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) { struct device *dev = &qm->pdev->dev; - u64 mb_cmd; u32 qos; int ret; @@ -3617,10 +3817,9 @@ static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) return; } - mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT; - ret = qm_ping_single_vf(qm, mb_cmd, fun_num); + ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num); if (ret) - dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num); + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num); } static int qm_vf_read_qos(struct hisi_qm *qm) @@ -3720,8 +3919,14 @@ static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, } pdev = container_of(dev, struct pci_dev, dev); + if (pci_physfn(pdev) != qm->pdev) { + pci_err(qm->pdev, "the pdev input does not match the pf!\n"); + put_device(dev); + return -EINVAL; + } *fun_index = pdev->devfn; + put_device(dev); return 0; } @@ -3854,13 +4059,13 @@ int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) goto err_put_sync; } + qm->vfs_num = num_vfs; ret = pci_enable_sriov(pdev, num_vfs); if (ret) { pci_err(pdev, "Can't enable VF!\n"); qm_clear_vft_config(qm); goto err_put_sync; } - qm->vfs_num = num_vfs; pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs); @@ -3895,11 +4100,10 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) } pci_disable_sriov(pdev); - - qm->vfs_num = 0; + qm_clear_vft_config(qm); qm_pm_put_sync(qm); - return qm_clear_vft_config(qm); + return 0; } EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); @@ -4073,9 +4277,9 @@ static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) !qm->err_status.is_qm_ecc_mbit && !qm->err_ini->close_axi_master_ooo) { nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE); - writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE, + writel(nfe_enb & ~qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_RAS_NFE_ENABLE); - writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET); + writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SET); } } @@ -4109,7 +4313,7 @@ stop_fail: return ret; } -static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, +static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, enum qm_stop_reason stop_reason) { struct pci_dev *pdev = qm->pdev; @@ -4122,7 +4326,7 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { ret = qm_ping_all_vfs(qm, cmd); if (ret) - pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); + pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); } else { ret = qm_vf_reset_prepare(qm, stop_reason); if (ret) @@ -4137,6 +4341,12 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; int ret; + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + ret = qm_reset_prepare_ready(qm); if (ret) { pci_err(pdev, "Controller reset not ready!\n"); @@ -4298,7 +4508,7 @@ restart_fail: return ret; } -static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) +static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { struct pci_dev *pdev = qm->pdev; int ret; @@ -4335,9 +4545,6 @@ static void qm_restart_prepare(struct hisi_qm *qm) { u32 value; - if (qm->err_ini->open_sva_prefetch) - qm->err_ini->open_sva_prefetch(qm); - if (qm->ver >= QM_HW_V3) return; @@ -4351,12 +4558,12 @@ static void qm_restart_prepare(struct hisi_qm *qm) qm->io_base + ACC_AM_CFG_PORT_WR_EN); /* clear dev ecc 2bit error source if having */ - value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask; + value = qm_get_dev_err_status(qm) & qm->err_info.dev_err.ecc_2bits_mask; if (value && qm->err_ini->clear_dev_hw_err_status) qm->err_ini->clear_dev_hw_err_status(qm, value); /* clear QM ecc mbit error source */ - writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); /* clear AM Reorder Buffer ecc mbit source */ writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); @@ -4383,6 +4590,34 @@ clear_flags: qm->err_status.is_dev_ecc_mbit = false; } +static void qm_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; + u32 val; + + val = ~(qm->error_mask & (~QM_RAS_AXI_ERROR)); + writel(val, qm->io_base + QM_ABNORMAL_INT_MASK); + if (qm->ver > QM_HW_V2) + writel(qm_err->shutdown_mask & (~QM_RAS_AXI_ERROR), + qm->io_base + QM_OOO_SHUTDOWN_SEL); + + if (qm->err_ini->disable_axi_error) + qm->err_ini->disable_axi_error(qm); +} + +static void qm_enable_axi_error(struct hisi_qm *qm) +{ + /* clear axi error source */ + writel(QM_RAS_AXI_ERROR, qm->io_base + QM_ABNORMAL_INT_SOURCE); + + writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); + if (qm->ver > QM_HW_V2) + writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + + if (qm->err_ini->enable_axi_error) + qm->err_ini->enable_axi_error(qm); +} + static int qm_controller_reset_done(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -4416,6 +4651,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) qm_restart_prepare(qm); hisi_qm_dev_err_init(qm); + qm_disable_axi_error(qm); if (qm->err_ini->open_axi_master_ooo) qm->err_ini->open_axi_master_ooo(qm); @@ -4438,7 +4674,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) ret = qm_wait_vf_prepare_finish(qm); if (ret) pci_err(pdev, "failed to start by vfs in soft reset!\n"); - + qm_enable_axi_error(qm); qm_cmd_init(qm); qm_restart_done(qm); @@ -4527,7 +4763,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) * Check whether there is an ECC mbit error, If it occurs, need to * wait for soft reset to fix it. */ - while (qm_check_dev_error(pf_qm)) { + while (qm_check_dev_error(qm)) { msleep(++delay); if (delay > QM_RESET_WAIT_TIMEOUT) return; @@ -4619,6 +4855,15 @@ flr_done: } EXPORT_SYMBOL_GPL(hisi_qm_reset_done); +static irqreturn_t qm_rsvd_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + + dev_info(&qm->pdev->dev, "Reserved interrupt, ignore!\n"); + + return IRQ_HANDLED; +} + static irqreturn_t qm_abnormal_irq(int irq, void *data) { struct hisi_qm *qm = data; @@ -4648,8 +4893,6 @@ void hisi_qm_dev_shutdown(struct pci_dev *pdev) ret = hisi_qm_stop(qm, QM_DOWN); if (ret) dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); - - hisi_qm_cache_wb(qm); } EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); @@ -4675,7 +4918,7 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { - enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE; + enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE; struct pci_dev *pdev = qm->pdev; int ret; @@ -4709,7 +4952,7 @@ out: static void qm_pf_reset_vf_done(struct hisi_qm *qm) { - enum qm_mb_cmd cmd = QM_VF_START_DONE; + enum qm_ifc_cmd cmd = QM_VF_START_DONE; struct pci_dev *pdev = qm->pdev; int ret; @@ -4732,7 +4975,6 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; u32 val, cmd; - u64 msg; int ret; /* Wait for reset to finish */ @@ -4749,16 +4991,15 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) * Whether message is got successfully, * VF needs to ack PF by clearing the interrupt. */ - ret = qm_get_mb_cmd(qm, &msg, 0); + ret = qm->ops->get_ifc(qm, &cmd, NULL, 0); qm_clear_cmd_interrupt(qm, 0); if (ret) { - dev_err(dev, "failed to get msg from PF in reset done!\n"); + dev_err(dev, "failed to get command from PF in reset done!\n"); return ret; } - cmd = msg & QM_MB_CMD_DATA_MASK; if (cmd != QM_PF_RESET_DONE) { - dev_err(dev, "the cmd(%u) is not reset done!\n", cmd); + dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); ret = -EINVAL; } @@ -4795,22 +5036,21 @@ err_get_status: static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) { struct device *dev = &qm->pdev->dev; - u64 msg; - u32 cmd; + enum qm_ifc_cmd cmd; + u32 data; int ret; /* * Get the msg from source by sending mailbox. Whether message is got * successfully, destination needs to ack source by clearing the interrupt. */ - ret = qm_get_mb_cmd(qm, &msg, fun_num); + ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num); qm_clear_cmd_interrupt(qm, BIT(fun_num)); if (ret) { - dev_err(dev, "failed to get msg from source!\n"); + dev_err(dev, "failed to get command from source!\n"); return; } - cmd = msg & QM_MB_CMD_DATA_MASK; switch (cmd) { case QM_PF_FLR_PREPARE: qm_pf_reset_vf_process(qm, QM_DOWN); @@ -4822,10 +5062,10 @@ static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) qm_vf_get_qos(qm, fun_num); break; case QM_PF_SET_QOS: - qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT; + qm->mb_qos = data; break; default: - dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num); + dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); break; } } @@ -4905,7 +5145,7 @@ static void qm_unregister_abnormal_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - if (qm->fun_type == QM_HW_VF) + if (qm->fun_type == QM_HW_VF && qm->ver < QM_HW_V3) return; val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; @@ -4922,17 +5162,28 @@ static int qm_register_abnormal_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - if (qm->fun_type == QM_HW_VF) - return 0; - val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return 0; - irq_vector = val & QM_IRQ_VECTOR_MASK; + + /* For VF, this is a reserved interrupt in V3 version. */ + if (qm->fun_type == QM_HW_VF) { + if (qm->ver < QM_HW_V3) + return 0; + + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_rsvd_irq, + IRQF_NO_AUTOEN, qm->dev_name, qm); + if (ret) { + dev_err(&pdev->dev, "failed to request reserved irq, ret = %d!\n", ret); + return ret; + } + return 0; + } + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); if (ret) - dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret); + dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d!\n", ret); return ret; } @@ -5114,7 +5365,7 @@ static int qm_pre_store_caps(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(qm_cap_query_info); - qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL); + qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL); if (!qm_cap) return -ENOMEM; @@ -5167,6 +5418,20 @@ static int qm_get_hw_caps(struct hisi_qm *qm) return qm_pre_store_caps(qm); } +static void qm_get_version(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 sub_version_id; + + qm->ver = pdev->revision; + + if (pdev->revision == QM_HW_V3) { + sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID); + if (sub_version_id) + qm->ver = sub_version_id; + } +} + static int qm_get_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5186,6 +5451,8 @@ static int qm_get_pci_res(struct hisi_qm *qm) goto err_request_mem_regions; } + qm_get_version(qm); + ret = qm_get_hw_caps(qm); if (ret) goto err_ioremap; @@ -5205,6 +5472,7 @@ static int qm_get_pci_res(struct hisi_qm *qm) qm->db_interval = 0; } + hisi_qm_pre_init(qm); ret = qm_get_qp_num(qm); if (ret) goto err_db_ioremap; @@ -5247,6 +5515,14 @@ static int qm_clear_device(struct hisi_qm *qm) return ret; } + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) { + writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + return ret; + } + } + return qm_reset_device(qm); } @@ -5273,6 +5549,12 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) pci_set_master(pdev); num_vec = qm_get_irq_num(qm); + if (!num_vec) { + dev_err(dev, "Device irq num is zero!\n"); + ret = -EINVAL; + goto err_get_pci_res; + } + num_vec = roundup_pow_of_two(num_vec); ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to enable MSI vectors!\n"); @@ -5461,8 +5743,6 @@ int hisi_qm_init(struct hisi_qm *qm) struct device *dev = &pdev->dev; int ret; - hisi_qm_pre_init(qm); - ret = hisi_qm_pci_init(qm); if (ret) return ret; @@ -5497,6 +5777,7 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_free_qm_memory; qm_cmd_init(qm); + hisi_mig_region_enable(qm); return 0; @@ -5598,6 +5879,12 @@ static int qm_prepare_for_suspend(struct hisi_qm *qm) if (ret) return ret; + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + ret = qm_set_pf_mse(qm, false); if (ret) pci_err(pdev, "failed to disable MSE before suspending!\n"); @@ -5629,6 +5916,7 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm) } qm_cmd_init(qm); + hisi_mig_region_enable(qm); hisi_qm_dev_err_init(qm); /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c index ef0cb733c92c..129cb6faa0b7 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.c +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -922,7 +922,8 @@ static int sec_hw_init(struct sec_dev_info *info) struct iommu_domain *domain; u32 sec_ipv4_mask = 0; u32 sec_ipv6_mask[10] = {}; - u32 i, ret; + int ret; + u32 i; domain = iommu_get_domain_for_dev(info->dev); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 356188bee6fb..81d0beda93b2 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -7,6 +7,12 @@ #include <linux/hisi_acc_qm.h> #include "sec_crypto.h" +#define SEC_PBUF_SZ 512 +#define SEC_MAX_MAC_LEN 64 +#define SEC_IV_SIZE 24 +#define SEC_SGE_NR_NUM 4 +#define SEC_SGL_ALIGN_SIZE 64 + /* Algorithm resource per hardware SEC queue */ struct sec_alg_res { u8 *pbuf; @@ -20,6 +26,40 @@ struct sec_alg_res { u16 depth; }; +struct sec_hw_sge { + dma_addr_t buf; + void *page_ctrl; + __le32 len; + __le32 pad; + __le32 pad0; + __le32 pad1; +}; + +struct sec_hw_sgl { + dma_addr_t next_dma; + __le16 entry_sum_in_chain; + __le16 entry_sum_in_sgl; + __le16 entry_length_in_sgl; + __le16 pad0; + __le64 pad1[5]; + struct sec_hw_sgl *next; + struct sec_hw_sge sge_entries[SEC_SGE_NR_NUM]; +} __aligned(SEC_SGL_ALIGN_SIZE); + +struct sec_src_dst_buf { + struct sec_hw_sgl in; + struct sec_hw_sgl out; +}; + +struct sec_request_buf { + union { + struct sec_src_dst_buf data_buf; + __u8 pbuf[SEC_PBUF_SZ]; + }; + dma_addr_t in_dma; + dma_addr_t out_dma; +}; + /* Cipher request of SEC private */ struct sec_cipher_req { struct hisi_acc_hw_sgl *c_out; @@ -29,6 +69,7 @@ struct sec_cipher_req { struct skcipher_request *sk_req; u32 c_len; bool encrypt; + __u8 c_ivin_buf[SEC_IV_SIZE]; }; struct sec_aead_req { @@ -37,6 +78,13 @@ struct sec_aead_req { u8 *a_ivin; dma_addr_t a_ivin_dma; struct aead_request *aead_req; + __u8 a_ivin_buf[SEC_IV_SIZE]; + __u8 out_mac_buf[SEC_MAX_MAC_LEN]; +}; + +struct sec_instance_backlog { + struct list_head list; + spinlock_t lock; }; /* SEC request of Crypto */ @@ -55,15 +103,17 @@ struct sec_req { dma_addr_t in_dma; struct sec_cipher_req c_req; struct sec_aead_req aead_req; - struct list_head backlog_head; + struct crypto_async_request *base; int err_type; int req_id; u32 flag; - /* Status of the SEC request */ - bool fake_busy; bool use_pbuf; + + struct list_head list; + struct sec_instance_backlog *backlog; + struct sec_request_buf buf; }; /** @@ -90,9 +140,7 @@ struct sec_auth_ctx { dma_addr_t a_key_dma; u8 *a_key; u8 a_key_len; - u8 mac_len; u8 a_alg; - bool fallback; struct crypto_shash *hash_tfm; struct crypto_aead *fallback_aead_tfm; }; @@ -121,9 +169,11 @@ struct sec_qp_ctx { struct sec_alg_res *res; struct sec_ctx *ctx; spinlock_t req_lock; - struct list_head backlog; + spinlock_t id_lock; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; + struct sec_instance_backlog backlog; + u16 send_head; }; enum sec_alg_type { @@ -141,9 +191,6 @@ struct sec_ctx { /* Half queues for encipher, and half for decipher */ u32 hlf_q_num; - /* Threshold for fake busy, trigger to return -EBUSY to user */ - u32 fake_req_limit; - /* Current cyclic index to select a queue for encipher */ atomic_t enc_qcyclic; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index ae9ebbb4103d..31590d01139a 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -57,7 +57,6 @@ #define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 #define SEC_ICV_MASK 0x000E -#define SEC_SQE_LEN_RATE_MASK 0x3 #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) #define SEC_SGL_SGE_NR 128 @@ -68,7 +67,6 @@ #define SEC_MAX_CCM_AAD_LEN 65279 #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) -#define SEC_PBUF_SZ 512 #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ @@ -80,16 +78,16 @@ #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ SEC_PBUF_LEFT_SZ(depth)) -#define SEC_SQE_LEN_RATE 4 #define SEC_SQE_CFLAG 2 #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 #define SEC_ICV_ERR 0x2 -#define MIN_MAC_LEN 4 #define MAC_LEN_MASK 0x1U #define MAX_INPUT_DATA_LEN 0xFFFE00 #define BITS_MASK 0xFF +#define WORD_MASK 0x3 #define BYTE_BITS 0x8 +#define BYTES_TO_WORDS(bcount) ((bcount) >> 2) #define SEC_XTS_NAME_SZ 0x3 #define IV_CM_CAL_NUM 2 #define IV_CL_MASK 0x7 @@ -103,6 +101,8 @@ #define IV_LAST_BYTE_MASK 0xFF #define IV_CTR_INIT 0x1 #define IV_BYTE_OFFSET 0x8 +#define SEC_GCM_MIN_AUTH_SZ 0x8 +#define SEC_RETRY_MAX_CNT 5U static DEFINE_MUTEX(sec_algs_lock); static unsigned int sec_available_devs; @@ -117,40 +117,19 @@ struct sec_aead { struct aead_alg alg; }; -/* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ -static inline u32 sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) -{ - if (req->c_req.encrypt) - return (u32)atomic_inc_return(&ctx->enc_qcyclic) % - ctx->hlf_q_num; - - return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + - ctx->hlf_q_num; -} - -static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) -{ - if (req->c_req.encrypt) - atomic_dec(&ctx->enc_qcyclic); - else - atomic_dec(&ctx->dec_qcyclic); -} +static int sec_aead_soft_crypto(struct sec_ctx *ctx, + struct aead_request *aead_req, + bool encrypt); +static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, + struct skcipher_request *sreq, bool encrypt); static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) { int req_id; - spin_lock_bh(&qp_ctx->req_lock); + spin_lock_bh(&qp_ctx->id_lock); req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); - spin_unlock_bh(&qp_ctx->req_lock); - if (unlikely(req_id < 0)) { - dev_err(req->ctx->dev, "alloc req id fail!\n"); - return req_id; - } - - req->qp_ctx = qp_ctx; - qp_ctx->req_list[req_id] = req; - + spin_unlock_bh(&qp_ctx->id_lock); return req_id; } @@ -164,12 +143,9 @@ static void sec_free_req_id(struct sec_req *req) return; } - qp_ctx->req_list[req_id] = NULL; - req->qp_ctx = NULL; - - spin_lock_bh(&qp_ctx->req_lock); + spin_lock_bh(&qp_ctx->id_lock); idr_remove(&qp_ctx->req_idr, req_id); - spin_unlock_bh(&qp_ctx->req_lock); + spin_unlock_bh(&qp_ctx->id_lock); } static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) @@ -230,6 +206,90 @@ static int sec_cb_status_check(struct sec_req *req, return 0; } +static int qp_send_message(struct sec_req *req) +{ + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + int ret; + + if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) + return -EBUSY; + + spin_lock_bh(&qp_ctx->req_lock); + if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) { + spin_unlock_bh(&qp_ctx->req_lock); + return -EBUSY; + } + + if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) { + req->sec_sqe.type2.tag = cpu_to_le16((u16)qp_ctx->send_head); + qp_ctx->req_list[qp_ctx->send_head] = req; + } + + ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); + if (ret) { + spin_unlock_bh(&qp_ctx->req_lock); + return ret; + } + if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) + qp_ctx->send_head = (qp_ctx->send_head + 1) % qp_ctx->qp->sq_depth; + + spin_unlock_bh(&qp_ctx->req_lock); + + atomic64_inc(&req->ctx->sec->debug.dfx.send_cnt); + return -EINPROGRESS; +} + +static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) +{ + struct sec_req *req, *tmp; + int ret; + + list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + list_del(&req->list); + ctx->req_op->buf_unmap(ctx, req); + if (req->req_id >= 0) + sec_free_req_id(req); + + if (ctx->alg_type == SEC_AEAD) + ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, + req->c_req.encrypt); + else + ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, + req->c_req.encrypt); + + /* Wake up the busy thread first, then return the errno. */ + crypto_request_complete(req->base, -EINPROGRESS); + crypto_request_complete(req->base, ret); + } +} + +static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) +{ + struct sec_req *req, *tmp; + int ret; + + spin_lock_bh(&qp_ctx->backlog.lock); + list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + ret = qp_send_message(req); + switch (ret) { + case -EINPROGRESS: + list_del(&req->list); + crypto_request_complete(req->base, -EINPROGRESS); + break; + case -EBUSY: + /* Device is busy and stop send any request. */ + goto unlock; + default: + /* Release memory resources and send all requests through software. */ + sec_alg_send_backlog_soft(ctx, qp_ctx); + goto unlock; + } + } + +unlock: + spin_unlock_bh(&qp_ctx->backlog.lock); +} + static void sec_req_cb(struct hisi_qp *qp, void *resp) { struct sec_qp_ctx *qp_ctx = qp->qp_ctx; @@ -274,40 +334,54 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) ctx->req_op->callback(ctx, req, err); } -static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) +static int sec_alg_send_message_retry(struct sec_req *req) { - struct sec_qp_ctx *qp_ctx = req->qp_ctx; + int ctr = 0; int ret; - if (ctx->fake_req_limit <= - atomic_read(&qp_ctx->qp->qp_status.used) && - !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) - return -EBUSY; + do { + ret = qp_send_message(req); + } while (ret == -EBUSY && ctr++ < SEC_RETRY_MAX_CNT); - spin_lock_bh(&qp_ctx->req_lock); - ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); - if (ctx->fake_req_limit <= - atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { - list_add_tail(&req->backlog_head, &qp_ctx->backlog); - atomic64_inc(&ctx->sec->debug.dfx.send_cnt); - atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); - spin_unlock_bh(&qp_ctx->req_lock); + return ret; +} + +static int sec_alg_try_enqueue(struct sec_req *req) +{ + /* Check if any request is already backlogged */ + if (!list_empty(&req->backlog->list)) return -EBUSY; - } - spin_unlock_bh(&qp_ctx->req_lock); - if (unlikely(ret == -EBUSY)) - return -ENOBUFS; + /* Try to enqueue to HW ring */ + return qp_send_message(req); +} - if (likely(!ret)) { - ret = -EINPROGRESS; - atomic64_inc(&ctx->sec->debug.dfx.send_cnt); - } + +static int sec_alg_send_message_maybacklog(struct sec_req *req) +{ + int ret; + + ret = sec_alg_try_enqueue(req); + if (ret != -EBUSY) + return ret; + + spin_lock_bh(&req->backlog->lock); + ret = sec_alg_try_enqueue(req); + if (ret == -EBUSY) + list_add_tail(&req->list, &req->backlog->list); + spin_unlock_bh(&req->backlog->lock); return ret; } -/* Get DMA memory resources */ +static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) +{ + if (req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG) + return sec_alg_send_message_maybacklog(req); + + return sec_alg_send_message_retry(req); +} + static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) { u16 q_depth = res->depth; @@ -559,7 +633,10 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) spin_lock_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); - INIT_LIST_HEAD(&qp_ctx->backlog); + spin_lock_init(&qp_ctx->backlog.lock); + spin_lock_init(&qp_ctx->id_lock); + INIT_LIST_HEAD(&qp_ctx->backlog.list); + qp_ctx->send_head = 0; ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); if (ret) @@ -603,9 +680,6 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) ctx->hlf_q_num = sec->ctx_q_num >> 1; ctx->pbuf_supported = ctx->sec->iommu_used; - - /* Half of queue depth is taken as fake requests limit in the queue. */ - ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1; ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), GFP_KERNEL); if (!ctx->qp_ctx) { @@ -691,14 +765,10 @@ static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) c_ctx->fallback = false; - /* Currently, only XTS mode need fallback tfm when using 192bit key */ - if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) - return 0; - c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(c_ctx->fbtfm)) { - pr_err("failed to alloc xts mode fallback tfm!\n"); + pr_err("failed to alloc fallback tfm for %s!\n", alg); return PTR_ERR(c_ctx->fbtfm); } @@ -711,7 +781,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) int ret; ctx->alg_type = SEC_SKCIPHER; - crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); + crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct sec_req)); ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { pr_err("get error skcipher iv size!\n"); @@ -858,7 +928,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (c_ctx->fallback && c_ctx->fbtfm) { + if (c_ctx->fbtfm) { ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); if (ret) { dev_err(dev, "failed to set fallback skcipher key!\n"); @@ -888,24 +958,25 @@ GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src) { - struct sec_aead_req *a_req = &req->aead_req; - struct aead_request *aead_req = a_req->aead_req; + struct aead_request *aead_req = req->aead_req.aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct sec_request_buf *buf = &req->buf; struct device *dev = ctx->dev; int copy_size, pbuf_length; int req_id = req->req_id; struct crypto_aead *tfm; + u8 *mac_offset, *pbuf; size_t authsize; - u8 *mac_offset; if (ctx->alg_type == SEC_AEAD) copy_size = aead_req->cryptlen + aead_req->assoclen; else copy_size = c_req->c_len; - pbuf_length = sg_copy_to_buffer(src, sg_nents(src), - qp_ctx->res[req_id].pbuf, copy_size); + + pbuf = req->req_id < 0 ? buf->pbuf : qp_ctx->res[req_id].pbuf; + pbuf_length = sg_copy_to_buffer(src, sg_nents(src), pbuf, copy_size); if (unlikely(pbuf_length != copy_size)) { dev_err(dev, "copy src data to pbuf error!\n"); return -EINVAL; @@ -913,8 +984,17 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { tfm = crypto_aead_reqtfm(aead_req); authsize = crypto_aead_authsize(tfm); - mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize; - memcpy(a_req->out_mac, mac_offset, authsize); + mac_offset = pbuf + copy_size - authsize; + memcpy(req->aead_req.out_mac, mac_offset, authsize); + } + + if (req->req_id < 0) { + buf->in_dma = dma_map_single(dev, buf->pbuf, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, buf->in_dma))) + return -ENOMEM; + + buf->out_dma = buf->in_dma; + return 0; } req->in_dma = qp_ctx->res[req_id].pbuf_dma; @@ -929,6 +1009,7 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, struct aead_request *aead_req = req->aead_req.aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct sec_request_buf *buf = &req->buf; int copy_size, pbuf_length; int req_id = req->req_id; @@ -937,10 +1018,16 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, else copy_size = c_req->c_len; - pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), - qp_ctx->res[req_id].pbuf, copy_size); + if (req->req_id < 0) + pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), buf->pbuf, copy_size); + else + pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf, + copy_size); if (unlikely(pbuf_length != copy_size)) dev_err(ctx->dev, "copy pbuf data to dst error!\n"); + + if (req->req_id < 0) + dma_unmap_single(ctx->dev, buf->in_dma, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); } static int sec_aead_mac_init(struct sec_aead_req *req) @@ -948,29 +1035,109 @@ static int sec_aead_mac_init(struct sec_aead_req *req) struct aead_request *aead_req = req->aead_req; struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); size_t authsize = crypto_aead_authsize(tfm); - u8 *mac_out = req->out_mac; struct scatterlist *sgl = aead_req->src; + u8 *mac_out = req->out_mac; size_t copy_size; off_t skip_size; /* Copy input mac */ skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; - copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, - authsize, skip_size); + copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, authsize, skip_size); if (unlikely(copy_size != authsize)) return -EINVAL; return 0; } -static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, - struct scatterlist *src, struct scatterlist *dst) +static void fill_sg_to_hw_sge(struct scatterlist *sgl, struct sec_hw_sge *hw_sge) +{ + hw_sge->buf = sg_dma_address(sgl); + hw_sge->len = cpu_to_le32(sg_dma_len(sgl)); + hw_sge->page_ctrl = sg_virt(sgl); +} + +static int sec_cipher_to_hw_sgl(struct device *dev, struct scatterlist *src, + struct sec_hw_sgl *src_in, dma_addr_t *hw_sgl_dma, + int dma_dir) +{ + struct sec_hw_sge *curr_hw_sge = src_in->sge_entries; + u32 i, sg_n, sg_n_mapped; + struct scatterlist *sg; + u32 sge_var = 0; + + sg_n = sg_nents(src); + sg_n_mapped = dma_map_sg(dev, src, sg_n, dma_dir); + if (unlikely(!sg_n_mapped)) { + dev_err(dev, "dma mapping for SG error!\n"); + return -EINVAL; + } else if (unlikely(sg_n_mapped > SEC_SGE_NR_NUM)) { + dev_err(dev, "the number of entries in input scatterlist error!\n"); + dma_unmap_sg(dev, src, sg_n, dma_dir); + return -EINVAL; + } + + for_each_sg(src, sg, sg_n_mapped, i) { + fill_sg_to_hw_sge(sg, curr_hw_sge); + curr_hw_sge++; + sge_var++; + } + + src_in->entry_sum_in_sgl = cpu_to_le16(sge_var); + src_in->entry_sum_in_chain = cpu_to_le16(SEC_SGE_NR_NUM); + src_in->entry_length_in_sgl = cpu_to_le16(SEC_SGE_NR_NUM); + *hw_sgl_dma = dma_map_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); + if (unlikely(dma_mapping_error(dev, *hw_sgl_dma))) { + dma_unmap_sg(dev, src, sg_n, dma_dir); + return -ENOMEM; + } + + return 0; +} + +static void sec_cipher_put_hw_sgl(struct device *dev, struct scatterlist *src, + dma_addr_t src_in, int dma_dir) +{ + dma_unmap_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); + dma_unmap_sg(dev, src, sg_nents(src), dma_dir); +} + +static int sec_cipher_map_sgl(struct device *dev, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_hw_sgl *src_in = &req->buf.data_buf.in; + struct sec_hw_sgl *dst_out = &req->buf.data_buf.out; + int ret; + + if (dst == src) { + ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, + DMA_BIDIRECTIONAL); + req->buf.out_dma = req->buf.in_dma; + return ret; + } + + ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, DMA_TO_DEVICE); + if (unlikely(ret)) + return ret; + + ret = sec_cipher_to_hw_sgl(dev, dst, dst_out, &req->buf.out_dma, + DMA_FROM_DEVICE); + if (unlikely(ret)) { + sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); + return ret; + } + + return 0; +} + +static int sec_cipher_map_inner(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) { struct sec_cipher_req *c_req = &req->c_req; struct sec_aead_req *a_req = &req->aead_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; struct sec_alg_res *res = &qp_ctx->res[req->req_id]; struct device *dev = ctx->dev; + enum dma_data_direction src_direction; int ret; if (req->use_pbuf) { @@ -983,10 +1150,9 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, a_req->out_mac_dma = res->pbuf_dma + SEC_PBUF_MAC_OFFSET; } - ret = sec_cipher_pbuf_map(ctx, req, src); - - return ret; + return sec_cipher_pbuf_map(ctx, req, src); } + c_req->c_ivin = res->c_ivin; c_req->c_ivin_dma = res->c_ivin_dma; if (ctx->alg_type == SEC_AEAD) { @@ -996,10 +1162,11 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, a_req->out_mac_dma = res->out_mac_dma; } + src_direction = dst == src ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, qp_ctx->c_in_pool, req->req_id, - &req->in_dma); + &req->in_dma, src_direction); if (IS_ERR(req->in)) { dev_err(dev, "fail to dma map input sgl buffers!\n"); return PTR_ERR(req->in); @@ -1009,7 +1176,7 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, ret = sec_aead_mac_init(a_req); if (unlikely(ret)) { dev_err(dev, "fail to init mac data for ICV!\n"); - hisi_acc_sg_buf_unmap(dev, src, req->in); + hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); return ret; } } @@ -1021,11 +1188,12 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, qp_ctx->c_out_pool, req->req_id, - &c_req->c_out_dma); + &c_req->c_out_dma, + DMA_FROM_DEVICE); if (IS_ERR(c_req->c_out)) { dev_err(dev, "fail to dma map output sgl buffers!\n"); - hisi_acc_sg_buf_unmap(dev, src, req->in); + hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); return PTR_ERR(c_req->c_out); } } @@ -1033,19 +1201,108 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, return 0; } +static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + bool is_aead = (ctx->alg_type == SEC_AEAD); + struct device *dev = ctx->dev; + int ret = -ENOMEM; + + if (req->req_id >= 0) + return sec_cipher_map_inner(ctx, req, src, dst); + + c_req->c_ivin = c_req->c_ivin_buf; + c_req->c_ivin_dma = dma_map_single(dev, c_req->c_ivin, + SEC_IV_SIZE, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, c_req->c_ivin_dma))) + return -ENOMEM; + + if (is_aead) { + a_req->a_ivin = a_req->a_ivin_buf; + a_req->out_mac = a_req->out_mac_buf; + a_req->a_ivin_dma = dma_map_single(dev, a_req->a_ivin, + SEC_IV_SIZE, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, a_req->a_ivin_dma))) + goto free_c_ivin_dma; + + a_req->out_mac_dma = dma_map_single(dev, a_req->out_mac, + SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, a_req->out_mac_dma))) + goto free_a_ivin_dma; + } + if (req->use_pbuf) { + ret = sec_cipher_pbuf_map(ctx, req, src); + if (unlikely(ret)) + goto free_out_mac_dma; + + return 0; + } + + if (!c_req->encrypt && is_aead) { + ret = sec_aead_mac_init(a_req); + if (unlikely(ret)) { + dev_err(dev, "fail to init mac data for ICV!\n"); + goto free_out_mac_dma; + } + } + + ret = sec_cipher_map_sgl(dev, req, src, dst); + if (unlikely(ret)) { + dev_err(dev, "fail to dma map input sgl buffers!\n"); + goto free_out_mac_dma; + } + + return 0; + +free_out_mac_dma: + if (is_aead) + dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); +free_a_ivin_dma: + if (is_aead) + dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); +free_c_ivin_dma: + dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); + return ret; +} + static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src, struct scatterlist *dst) { + struct sec_aead_req *a_req = &req->aead_req; struct sec_cipher_req *c_req = &req->c_req; struct device *dev = ctx->dev; + if (req->req_id >= 0) { + if (req->use_pbuf) { + sec_cipher_pbuf_unmap(ctx, req, dst); + } else { + if (dst != src) { + hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE); + hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE); + } else { + hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL); + } + } + return; + } + if (req->use_pbuf) { sec_cipher_pbuf_unmap(ctx, req, dst); } else { - if (dst != src) - hisi_acc_sg_buf_unmap(dev, src, req->in); + if (dst != src) { + sec_cipher_put_hw_sgl(dev, dst, req->buf.out_dma, DMA_FROM_DEVICE); + sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); + } else { + sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_BIDIRECTIONAL); + } + } - hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); + dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); + if (ctx->alg_type == SEC_AEAD) { + dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); + dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); } } @@ -1091,11 +1348,6 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, struct crypto_shash *hash_tfm = ctx->hash_tfm; int blocksize, digestsize, ret; - if (!keys->authkeylen) { - pr_err("hisi_sec2: aead auth key error!\n"); - return -EINVAL; - } - blocksize = crypto_shash_blocksize(hash_tfm); digestsize = crypto_shash_digestsize(hash_tfm); if (keys->authkeylen > blocksize) { @@ -1107,7 +1359,8 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, } ctx->a_key_len = digestsize; } else { - memcpy(ctx->a_key, keys->authkey, keys->authkeylen); + if (keys->authkeylen) + memcpy(ctx->a_key, keys->authkey, keys->authkeylen); ctx->a_key_len = keys->authkeylen; } @@ -1120,10 +1373,7 @@ static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) struct sec_ctx *ctx = crypto_tfm_ctx(tfm); struct sec_auth_ctx *a_ctx = &ctx->a_ctx; - if (unlikely(a_ctx->fallback_aead_tfm)) - return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); - - return 0; + return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); } static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, @@ -1139,7 +1389,6 @@ static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, const u32 keylen, const enum sec_hash_alg a_alg, const enum sec_calg c_alg, - const enum sec_mac_len mac_len, const enum sec_cmode c_mode) { struct sec_ctx *ctx = crypto_aead_ctx(tfm); @@ -1151,7 +1400,6 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, ctx->a_ctx.a_alg = a_alg; ctx->c_ctx.c_alg = c_alg; - ctx->a_ctx.mac_len = mac_len; c_ctx->c_mode = c_mode; if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { @@ -1162,18 +1410,14 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (unlikely(a_ctx->fallback_aead_tfm)) { - ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); - if (ret) - return ret; - } - - return 0; + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); } ret = crypto_authenc_extractkeys(&keys, key, keylen); - if (ret) + if (ret) { + dev_err(dev, "sec extract aead keys err!\n"); goto bad_key; + } ret = sec_aead_aes_set_key(c_ctx, &keys); if (ret) { @@ -1187,10 +1431,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, goto bad_key; } - if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || - (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { - ret = -EINVAL; - dev_err(dev, "MAC or AUTH key length error!\n"); + ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + if (ret) { + dev_err(dev, "set sec fallback key err!\n"); goto bad_key; } @@ -1202,27 +1445,19 @@ bad_key: } -#define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ -static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ - u32 keylen) \ -{ \ - return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ -} - -GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, - SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) -GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, - SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) -GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, - SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) -GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, - SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) -GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, - SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) -GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, - SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) -GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, - SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) +#define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, cmode) \ +static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, u32 keylen) \ +{ \ + return sec_aead_setkey(tfm, key, keylen, aalg, calg, cmode); \ +} + +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, SEC_CMODE_GCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, SEC_CMODE_GCM) static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) { @@ -1285,8 +1520,15 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); - sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); - sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); + if (req->req_id < 0) { + sec_sqe->type2.data_src_addr = cpu_to_le64(req->buf.in_dma); + sec_sqe->type2.data_dst_addr = cpu_to_le64(req->buf.out_dma); + } else { + sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); + sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); + } + if (sec_sqe->type2.data_src_addr != sec_sqe->type2.data_dst_addr) + de = 0x1 << SEC_DE_OFFSET; sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << SEC_CMODE_OFFSET); @@ -1312,13 +1554,10 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) sec_sqe->sdm_addr_type |= da_type; scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; - if (req->in_dma != c_req->c_out_dma) - de = 0x1 << SEC_DE_OFFSET; sec_sqe->sds_sa_type = (de | scene | sa_type); sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); - sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); return 0; } @@ -1335,8 +1574,15 @@ static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); - sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); - sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); + if (req->req_id < 0) { + sec_sqe3->data_src_addr = cpu_to_le64(req->buf.in_dma); + sec_sqe3->data_dst_addr = cpu_to_le64(req->buf.out_dma); + } else { + sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); + sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); + } + if (sec_sqe3->data_src_addr != sec_sqe3->data_dst_addr) + bd_param |= 0x1 << SEC_DE_OFFSET_V3; sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | c_ctx->c_mode; @@ -1362,8 +1608,6 @@ static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) } bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; - if (req->in_dma != c_req->c_out_dma) - bd_param |= 0x1 << SEC_DE_OFFSET_V3; bd_param |= SEC_BD_TYPE3; sec_sqe3->bd_param = cpu_to_le32(bd_param); @@ -1395,15 +1639,12 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) size_t sz; u8 *iv; - if (req->c_req.encrypt) - sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; - else - sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; - if (alg_type == SEC_SKCIPHER) { + sgl = req->c_req.encrypt ? sk_req->dst : sk_req->src; iv = sk_req->iv; cryptlen = sk_req->cryptlen; } else { + sgl = req->c_req.encrypt ? aead_req->dst : aead_req->src; iv = aead_req->iv; cryptlen = aead_req->cryptlen; } @@ -1414,65 +1655,35 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) if (unlikely(sz != iv_size)) dev_err(req->ctx->dev, "copy output iv error!\n"); } else { - sz = cryptlen / iv_size; - if (cryptlen % iv_size) - sz += 1; + sz = (cryptlen + iv_size - 1) / iv_size; ctr_iv_inc(iv, iv_size, sz); } } -static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, - struct sec_qp_ctx *qp_ctx) -{ - struct sec_req *backlog_req = NULL; - - spin_lock_bh(&qp_ctx->req_lock); - if (ctx->fake_req_limit >= - atomic_read(&qp_ctx->qp->qp_status.used) && - !list_empty(&qp_ctx->backlog)) { - backlog_req = list_first_entry(&qp_ctx->backlog, - typeof(*backlog_req), backlog_head); - list_del(&backlog_req->backlog_head); - } - spin_unlock_bh(&qp_ctx->req_lock); - - return backlog_req; -} - static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, int err) { - struct skcipher_request *sk_req = req->c_req.sk_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct skcipher_request *backlog_sk_req; - struct sec_req *backlog_req; - sec_free_req_id(req); + if (req->req_id >= 0) + sec_free_req_id(req); /* IV output at encrypto of CBC/CTR mode */ if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) sec_update_iv(req, SEC_SKCIPHER); - while (1) { - backlog_req = sec_back_req_clear(ctx, qp_ctx); - if (!backlog_req) - break; - - backlog_sk_req = backlog_req->c_req.sk_req; - skcipher_request_complete(backlog_sk_req, -EINPROGRESS); - atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); - } - - skcipher_request_complete(sk_req, err); + crypto_request_complete(req->base, err); + sec_alg_send_backlog(ctx, qp_ctx); } static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) { struct aead_request *aead_req = req->aead_req.aead_req; - struct sec_cipher_req *c_req = &req->c_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); struct sec_aead_req *a_req = &req->aead_req; - size_t authsize = ctx->a_ctx.mac_len; + struct sec_cipher_req *c_req = &req->c_req; u32 data_size = aead_req->cryptlen; u8 flage = 0; u8 cm, cl; @@ -1513,10 +1724,8 @@ static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) { struct aead_request *aead_req = req->aead_req.aead_req; - struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); - size_t authsize = crypto_aead_authsize(tfm); - struct sec_cipher_req *c_req = &req->c_req; struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); @@ -1524,15 +1733,11 @@ static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) /* * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, * the counter must set to 0x01 + * CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ - ctx->a_ctx.mac_len = authsize; - /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ set_aead_auth_iv(ctx, req); - } - - /* GCM 12Byte Cipher_IV == Auth_IV */ - if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { - ctx->a_ctx.mac_len = authsize; + } else if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { + /* GCM 12Byte Cipher_IV == Auth_IV */ memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); } } @@ -1542,9 +1747,11 @@ static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, { struct sec_aead_req *a_req = &req->aead_req; struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ - sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len); + sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)authsize); /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; @@ -1568,9 +1775,11 @@ static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, { struct sec_aead_req *a_req = &req->aead_req; struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ - sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3); + sqe3->c_icv_key |= cpu_to_le16((u16)authsize << SEC_MAC_OFFSET_V3); /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ sqe3->a_key_addr = sqe3->c_key_addr; @@ -1594,15 +1803,15 @@ static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, struct sec_aead_req *a_req = &req->aead_req; struct sec_cipher_req *c_req = &req->c_req; struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); - sec_sqe->type2.mac_key_alg = - cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); + sec_sqe->type2.mac_key_alg = cpu_to_le32(BYTES_TO_WORDS(authsize)); sec_sqe->type2.mac_key_alg |= - cpu_to_le32((u32)((ctx->a_key_len) / - SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET); sec_sqe->type2.mac_key_alg |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); @@ -1648,16 +1857,16 @@ static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, struct sec_aead_req *a_req = &req->aead_req; struct sec_cipher_req *c_req = &req->c_req; struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); sqe3->auth_mac_key |= - cpu_to_le32((u32)(ctx->mac_len / - SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); + cpu_to_le32(BYTES_TO_WORDS(authsize) << SEC_MAC_OFFSET_V3); sqe3->auth_mac_key |= - cpu_to_le32((u32)(ctx->a_key_len / - SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET_V3); sqe3->auth_mac_key |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); @@ -1703,73 +1912,53 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) { struct aead_request *a_req = req->aead_req.aead_req; struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); - struct sec_aead_req *aead_req = &req->aead_req; - struct sec_cipher_req *c_req = &req->c_req; size_t authsize = crypto_aead_authsize(tfm); struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct aead_request *backlog_aead_req; - struct sec_req *backlog_req; size_t sz; - if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) - sec_update_iv(req, SEC_AEAD); + if (!err && req->c_req.encrypt) { + if (c->c_ctx.c_mode == SEC_CMODE_CBC) + sec_update_iv(req, SEC_AEAD); - /* Copy output mac */ - if (!err && c_req->encrypt) { - struct scatterlist *sgl = a_req->dst; - - sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), - aead_req->out_mac, - authsize, a_req->cryptlen + - a_req->assoclen); + sz = sg_pcopy_from_buffer(a_req->dst, sg_nents(a_req->dst), req->aead_req.out_mac, + authsize, a_req->cryptlen + a_req->assoclen); if (unlikely(sz != authsize)) { dev_err(c->dev, "copy out mac err!\n"); err = -EINVAL; } } - sec_free_req_id(req); - - while (1) { - backlog_req = sec_back_req_clear(c, qp_ctx); - if (!backlog_req) - break; - - backlog_aead_req = backlog_req->aead_req.aead_req; - aead_request_complete(backlog_aead_req, -EINPROGRESS); - atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); - } + if (req->req_id >= 0) + sec_free_req_id(req); - aead_request_complete(a_req, err); + crypto_request_complete(req->base, err); + sec_alg_send_backlog(c, qp_ctx); } -static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) +static void sec_request_uninit(struct sec_req *req) { - sec_free_req_id(req); - sec_free_queue_id(ctx, req); + if (req->req_id >= 0) + sec_free_req_id(req); } static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) { struct sec_qp_ctx *qp_ctx; - int queue_id; + int i = 0; - /* To load balance */ - queue_id = sec_alloc_queue_id(ctx, req); - qp_ctx = &ctx->qp_ctx[queue_id]; + do { + qp_ctx = &ctx->qp_ctx[i]; + req->req_id = sec_alloc_req_id(req, qp_ctx); + } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); - req->req_id = sec_alloc_req_id(req, qp_ctx); - if (unlikely(req->req_id < 0)) { - sec_free_queue_id(ctx, req); - return req->req_id; - } + req->qp_ctx = qp_ctx; + req->backlog = &qp_ctx->backlog; return 0; } static int sec_process(struct sec_ctx *ctx, struct sec_req *req) { - struct sec_cipher_req *c_req = &req->c_req; int ret; ret = sec_request_init(ctx, req); @@ -1786,8 +1975,7 @@ static int sec_process(struct sec_ctx *ctx, struct sec_req *req) sec_update_iv(req, ctx->alg_type); ret = ctx->req_op->bd_send(ctx, req); - if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || - (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { + if (unlikely((ret != -EBUSY && ret != -EINPROGRESS))) { dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); goto err_send_req; } @@ -1798,16 +1986,23 @@ err_send_req: /* As failing, restore the IV from user */ if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { if (ctx->alg_type == SEC_SKCIPHER) - memcpy(req->c_req.sk_req->iv, c_req->c_ivin, + memcpy(req->c_req.sk_req->iv, req->c_req.c_ivin, ctx->c_ctx.ivsize); else - memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, + memcpy(req->aead_req.aead_req->iv, req->c_req.c_ivin, ctx->c_ctx.ivsize); } sec_request_untransfer(ctx, req); + err_uninit_req: - sec_request_uninit(ctx, req); + sec_request_uninit(req); + if (ctx->alg_type == SEC_AEAD) + ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, + req->c_req.encrypt); + else + ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, + req->c_req.encrypt); return ret; } @@ -1881,7 +2076,7 @@ static int sec_aead_init(struct crypto_aead *tfm) struct sec_ctx *ctx = crypto_aead_ctx(tfm); int ret; - crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); + crypto_aead_set_reqsize_dma(tfm, sizeof(struct sec_req)); ctx->alg_type = SEC_AEAD; ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || @@ -1929,8 +2124,10 @@ static void sec_aead_exit(struct crypto_aead *tfm) static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) { + struct aead_alg *alg = crypto_aead_alg(tfm); struct sec_ctx *ctx = crypto_aead_ctx(tfm); - struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + const char *aead_name = alg->base.cra_name; int ret; ret = sec_aead_init(tfm); @@ -1939,11 +2136,20 @@ static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) return ret; } - auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); - if (IS_ERR(auth_ctx->hash_tfm)) { + a_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); + if (IS_ERR(a_ctx->hash_tfm)) { dev_err(ctx->dev, "aead alloc shash error!\n"); sec_aead_exit(tfm); - return PTR_ERR(auth_ctx->hash_tfm); + return PTR_ERR(a_ctx->hash_tfm); + } + + a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, + CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); + if (IS_ERR(a_ctx->fallback_aead_tfm)) { + dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); + crypto_free_shash(ctx->a_ctx.hash_tfm); + sec_aead_exit(tfm); + return PTR_ERR(a_ctx->fallback_aead_tfm); } return 0; @@ -1953,6 +2159,7 @@ static void sec_aead_ctx_exit(struct crypto_aead *tfm) { struct sec_ctx *ctx = crypto_aead_ctx(tfm); + crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); crypto_free_shash(ctx->a_ctx.hash_tfm); sec_aead_exit(tfm); } @@ -1979,7 +2186,6 @@ static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) sec_aead_exit(tfm); return PTR_ERR(a_ctx->fallback_aead_tfm); } - a_ctx->fallback = false; return 0; } @@ -2007,8 +2213,7 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) return sec_aead_ctx_init(tfm, "sha512"); } -static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, - struct sec_req *sreq) +static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq) { u32 cryptlen = sreq->c_req.sk_req->cryptlen; struct device *dev = ctx->dev; @@ -2030,10 +2235,6 @@ static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, } break; case SEC_CMODE_CTR: - if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { - dev_err(dev, "skcipher HW version error!\n"); - ret = -EINVAL; - } break; default: ret = -EINVAL; @@ -2042,17 +2243,21 @@ static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, return ret; } -static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +static int sec_skcipher_param_check(struct sec_ctx *ctx, + struct sec_req *sreq, bool *need_fallback) { struct skcipher_request *sk_req = sreq->c_req.sk_req; struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; - if (unlikely(!sk_req->src || !sk_req->dst || - sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { + if (unlikely(!sk_req->src || !sk_req->dst)) { dev_err(dev, "skcipher input param error!\n"); return -EINVAL; } + + if (sk_req->cryptlen > MAX_INPUT_DATA_LEN) + *need_fallback = true; + sreq->c_req.c_len = sk_req->cryptlen; if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) @@ -2108,8 +2313,9 @@ static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); - struct sec_req *req = skcipher_request_ctx(sk_req); + struct sec_req *req = skcipher_request_ctx_dma(sk_req); struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + bool need_fallback = false; int ret; if (!sk_req->cryptlen) { @@ -2122,12 +2328,13 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) req->c_req.sk_req = sk_req; req->c_req.encrypt = encrypt; req->ctx = ctx; + req->base = &sk_req->base; - ret = sec_skcipher_param_check(ctx, req); + ret = sec_skcipher_param_check(ctx, req, &need_fallback); if (unlikely(ret)) return -EINVAL; - if (unlikely(ctx->c_ctx.fallback)) + if (unlikely(ctx->c_ctx.fallback || need_fallback)) return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); return ctx->req_op->process(ctx, req); @@ -2233,55 +2440,40 @@ static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) { struct aead_request *req = sreq->aead_req.aead_req; struct crypto_aead *tfm = crypto_aead_reqtfm(req); - size_t authsize = crypto_aead_authsize(tfm); + size_t sz = crypto_aead_authsize(tfm); u8 c_mode = ctx->c_ctx.c_mode; - struct device *dev = ctx->dev; int ret; - if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || - req->assoclen > SEC_MAX_AAD_LEN)) { - dev_err(dev, "aead input spec error!\n"); + if (unlikely(ctx->sec->qm.ver == QM_HW_V2 && !sreq->c_req.c_len)) return -EINVAL; - } - if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) || - (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN || - authsize & MAC_LEN_MASK)))) { - dev_err(dev, "aead input mac length error!\n"); + if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || + req->assoclen > SEC_MAX_AAD_LEN)) return -EINVAL; - } if (c_mode == SEC_CMODE_CCM) { - if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) { - dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n"); + if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) return -EINVAL; - } - ret = aead_iv_demension_check(req); - if (ret) { - dev_err(dev, "aead input iv param error!\n"); - return ret; - } - } - if (sreq->c_req.encrypt) - sreq->c_req.c_len = req->cryptlen; - else - sreq->c_req.c_len = req->cryptlen - authsize; - if (c_mode == SEC_CMODE_CBC) { - if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { - dev_err(dev, "aead crypto length error!\n"); + ret = aead_iv_demension_check(req); + if (unlikely(ret)) + return -EINVAL; + } else if (c_mode == SEC_CMODE_CBC) { + if (unlikely(sz & WORD_MASK)) + return -EINVAL; + if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK)) + return -EINVAL; + } else if (c_mode == SEC_CMODE_GCM) { + if (unlikely(sz < SEC_GCM_MIN_AUTH_SZ)) return -EINVAL; - } } return 0; } -static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq, bool *need_fallback) { struct aead_request *req = sreq->aead_req.aead_req; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - size_t authsize = crypto_aead_authsize(tfm); struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; @@ -2290,12 +2482,10 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } - if (ctx->sec->qm.ver == QM_HW_V2) { - if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && - req->cryptlen <= authsize))) { - ctx->a_ctx.fallback = true; - return -EINVAL; - } + if (unlikely(ctx->c_ctx.c_mode == SEC_CMODE_CBC && + sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "aead cbc mode input data length error!\n"); + return -EINVAL; } /* Support AES or SM4 */ @@ -2304,8 +2494,10 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } - if (unlikely(sec_aead_spec_check(ctx, sreq))) + if (unlikely(sec_aead_spec_check(ctx, sreq))) { + *need_fallback = true; return -EINVAL; + } if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= SEC_PBUF_SZ) @@ -2321,16 +2513,9 @@ static int sec_aead_soft_crypto(struct sec_ctx *ctx, bool encrypt) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; - struct device *dev = ctx->dev; struct aead_request *subreq; int ret; - /* Kunpeng920 aead mode not support input 0 size */ - if (!a_ctx->fallback_aead_tfm) { - dev_err(dev, "aead fallback tfm is NULL!\n"); - return -EINVAL; - } - subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL); if (!subreq) return -ENOMEM; @@ -2354,18 +2539,22 @@ static int sec_aead_soft_crypto(struct sec_ctx *ctx, static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) { struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); - struct sec_req *req = aead_request_ctx(a_req); + struct sec_req *req = aead_request_ctx_dma(a_req); struct sec_ctx *ctx = crypto_aead_ctx(tfm); + size_t sz = crypto_aead_authsize(tfm); + bool need_fallback = false; int ret; req->flag = a_req->base.flags; req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; req->ctx = ctx; + req->base = &a_req->base; + req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz); - ret = sec_aead_param_check(ctx, req); + ret = sec_aead_param_check(ctx, req, &need_fallback); if (unlikely(ret)) { - if (ctx->a_ctx.fallback) + if (need_fallback) return sec_aead_soft_crypto(ctx, a_req, encrypt); return -EINVAL; } diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h index 27a0ee5ad913..04725b514382 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.h +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -23,17 +23,6 @@ enum sec_hash_alg { SEC_A_HMAC_SHA512 = 0x15, }; -enum sec_mac_len { - SEC_HMAC_CCM_MAC = 16, - SEC_HMAC_GCM_MAC = 16, - SEC_SM3_MAC = 32, - SEC_HMAC_SM3_MAC = 32, - SEC_HMAC_MD5_MAC = 16, - SEC_HMAC_SHA1_MAC = 20, - SEC_HMAC_SHA256_MAC = 32, - SEC_HMAC_SHA512_MAC = 64, -}; - enum sec_cmode { SEC_CMODE_ECB = 0x0, SEC_CMODE_CBC = 0x1, diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 8ec5333bb5aa..5eb2d6820742 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -47,6 +47,8 @@ #define SEC_RAS_FE_ENB_MSK 0x0 #define SEC_OOO_SHUTDOWN_SEL 0x301014 #define SEC_RAS_DISABLE 0x0 +#define SEC_AXI_ERROR_MASK (BIT(0) | BIT(1)) + #define SEC_MEM_START_INIT_REG 0x301100 #define SEC_MEM_INIT_DONE_REG 0x301104 @@ -93,6 +95,16 @@ #define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11))) #define SEC_PREFETCH_DISABLE BIT(1) #define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11)) +#define SEC_SVA_PREFETCH_INFO 0x301ED4 +#define SEC_SVA_STALL_NUM GENMASK(23, 8) +#define SEC_SVA_PREFETCH_NUM GENMASK(2, 0) +#define SEC_WAIT_SVA_READY 500000 +#define SEC_READ_SVA_STATUS_TIMES 3 +#define SEC_WAIT_US_MIN 10 +#define SEC_WAIT_US_MAX 20 +#define SEC_WAIT_QP_US_MIN 1000 +#define SEC_WAIT_QP_US_MAX 2000 +#define SEC_MAX_WAIT_TIMES 2000 #define SEC_DELAY_10_US 10 #define SEC_POLL_TIMEOUT_US 1000 @@ -464,6 +476,81 @@ static void sec_set_endian(struct hisi_qm *qm) writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); } +static int sec_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + offset); + if (val & mask) + count = 0; + else if (++count == SEC_READ_SVA_STATUS_TIMES) + break; + + usleep_range(SEC_WAIT_US_MIN, SEC_WAIT_US_MAX); + } while (++try_times < SEC_WAIT_SVA_READY); + + if (try_times == SEC_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static void sec_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val |= SEC_PREFETCH_DISABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, + val, !(val & SEC_SVA_DISABLE_READY), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)sec_wait_sva_ready(qm, SEC_SVA_PREFETCH_INFO, SEC_SVA_STALL_NUM); +} + +static void sec_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val &= SEC_PREFETCH_ENABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, + val, !(val & SEC_PREFETCH_DISABLE), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + sec_close_sva_prefetch(qm); + return; + } + + ret = sec_wait_sva_ready(qm, SEC_SVA_TRANS, SEC_SVA_PREFETCH_NUM); + if (ret) + sec_close_sva_prefetch(qm); +} + static void sec_engine_sva_config(struct hisi_qm *qm) { u32 reg; @@ -497,45 +584,7 @@ static void sec_engine_sva_config(struct hisi_qm *qm) writel_relaxed(reg, qm->io_base + SEC_INTERFACE_USER_CTRL1_REG); } -} - -static void sec_open_sva_prefetch(struct hisi_qm *qm) -{ - u32 val; - int ret; - - if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) - return; - - /* Enable prefetch */ - val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); - val &= SEC_PREFETCH_ENABLE; - writel(val, qm->io_base + SEC_PREFETCH_CFG); - - ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, - val, !(val & SEC_PREFETCH_DISABLE), - SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to open sva prefetch\n"); -} - -static void sec_close_sva_prefetch(struct hisi_qm *qm) -{ - u32 val; - int ret; - - if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) - return; - - val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); - val |= SEC_PREFETCH_DISABLE; - writel(val, qm->io_base + SEC_PREFETCH_CFG); - - ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, - val, !(val & SEC_SVA_DISABLE_READY), - SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to close sva prefetch\n"); + sec_open_sva_prefetch(qm); } static void sec_enable_clock_gate(struct hisi_qm *qm) @@ -666,8 +715,7 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + SEC_CONTROL_REG); if (enable) { val1 |= SEC_AXI_SHUTDOWN_ENABLE; - val2 = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + val2 = qm->err_info.dev_err.shutdown_mask; } else { val1 &= SEC_AXI_SHUTDOWN_DISABLE; val2 = 0x0; @@ -681,7 +729,8 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void sec_hw_error_enable(struct hisi_qm *qm) { - u32 ce, nfe; + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; if (qm->ver == QM_HW_V1) { writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); @@ -689,22 +738,19 @@ static void sec_hw_error_enable(struct hisi_qm *qm) return; } - ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); - nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); - /* clear SEC hw error source if having */ - writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_SOURCE); + writel(err_mask, qm->io_base + SEC_CORE_INT_SOURCE); /* enable RAS int */ - writel(ce, qm->io_base + SEC_RAS_CE_REG); - writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); - writel(nfe, qm->io_base + SEC_RAS_NFE_REG); + writel(dev_err->ce, qm->io_base + SEC_RAS_CE_REG); + writel(dev_err->fe, qm->io_base + SEC_RAS_FE_REG); + writel(dev_err->nfe, qm->io_base + SEC_RAS_NFE_REG); /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ sec_master_ooo_ctrl(qm, true); /* enable SEC hw error interrupts */ - writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_MASK); + writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); } static void sec_hw_error_disable(struct hisi_qm *qm) @@ -1061,12 +1107,20 @@ static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) static void sec_disable_error_report(struct hisi_qm *qm, u32 err_type) { - u32 nfe_mask; + u32 nfe_mask = qm->err_info.dev_err.nfe; - nfe_mask = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); writel(nfe_mask & (~err_type), qm->io_base + SEC_RAS_NFE_REG); } +static void sec_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + SEC_RAS_NFE_REG); + writel(ce_mask, qm->io_base + SEC_RAS_CE_REG); +} + static void sec_open_axi_master_ooo(struct hisi_qm *qm) { u32 val; @@ -1082,37 +1136,84 @@ static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) err_status = sec_get_hw_err_status(qm); if (err_status) { - if (err_status & qm->err_info.ecc_2bits_mask) + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; sec_log_hw_error(qm, err_status); - if (err_status & qm->err_info.dev_reset_mask) { + if (err_status & qm->err_info.dev_err.reset_mask) { /* Disable the same error reporting until device is recovered. */ sec_disable_error_report(qm, err_status); return ACC_ERR_NEED_RESET; } sec_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + sec_enable_error_report(qm); } return ACC_ERR_RECOVERED; } +static bool sec_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = sec_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_err.shutdown_mask) + return true; + + return false; +} + +static void sec_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + writel(err_mask & ~SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~SEC_AXI_ERROR_MASK), + qm->io_base + SEC_OOO_SHUTDOWN_SEL); +} + +static void sec_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_SOURCE); + + writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + SEC_OOO_SHUTDOWN_SEL); +} + static void sec_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = SEC_RAS_FE_ENB_MSK; + qm_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + qm_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + + dev_err->fe = SEC_RAS_FE_ENB_MSK; + dev_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_RESET_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; - err_info->fe = SEC_RAS_FE_ENB_MSK; - err_info->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); - err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); - err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; - err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_QM_RESET_MASK_CAP, qm->cap_ver); - err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = BIT(0); err_info->acpi_rst = "SRST"; } @@ -1129,6 +1230,9 @@ static const struct hisi_qm_err_ini sec_err_ini = { .show_last_dfx_regs = sec_show_last_dfx_regs, .err_info_init = sec_err_info_init, .get_err_result = sec_get_err_result, + .dev_is_abnormal = sec_dev_is_abnormal, + .disable_axi_error = sec_disable_axi_error, + .enable_axi_error = sec_enable_axi_error, }; static int sec_pf_probe_init(struct sec_dev *sec) @@ -1140,7 +1244,6 @@ static int sec_pf_probe_init(struct sec_dev *sec) if (ret) return ret; - sec_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); sec_debug_regs_clear(qm); ret = sec_show_last_regs_init(qm); @@ -1157,7 +1260,7 @@ static int sec_pre_store_cap_reg(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(sec_cap_query_info); - sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL); + sec_cap = devm_kcalloc(&pdev->dev, size, sizeof(*sec_cap), GFP_KERNEL); if (!sec_cap) return -ENOMEM; @@ -1180,7 +1283,6 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) int ret; qm->pdev = pdev; - qm->ver = pdev->revision; qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index c974f95cd126..24c7b6ab285b 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -210,15 +210,15 @@ static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) * @pool: Pool which hw sgl memory will be allocated in. * @index: Index of hisi_acc_hw_sgl in pool. * @hw_sgl_dma: The dma address of allocated hw sgl. + * @dir: DMA direction. * * This function builds hw sgl according input sgl, user can use hw_sgl_dma * as src/dst in its BD. Only support single hw sgl currently. */ struct hisi_acc_hw_sgl * -hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, - struct scatterlist *sgl, - struct hisi_acc_sgl_pool *pool, - u32 index, dma_addr_t *hw_sgl_dma) +hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, + struct hisi_acc_sgl_pool *pool, u32 index, + dma_addr_t *hw_sgl_dma, enum dma_data_direction dir) { struct hisi_acc_hw_sgl *curr_hw_sgl; unsigned int i, sg_n_mapped; @@ -232,7 +232,7 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, sg_n = sg_nents(sgl); - sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + sg_n_mapped = dma_map_sg(dev, sgl, sg_n, dir); if (!sg_n_mapped) { dev_err(dev, "DMA mapping for SG error!\n"); return ERR_PTR(-EINVAL); @@ -245,11 +245,6 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, } curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma); - if (IS_ERR(curr_hw_sgl)) { - dev_err(dev, "Get SGL error!\n"); - ret = -ENOMEM; - goto err_unmap; - } curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr); curr_hw_sge = curr_hw_sgl->sge_entries; @@ -276,16 +271,17 @@ EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl); * @dev: The device which hw sgl belongs to. * @sgl: Related scatterlist. * @hw_sgl: Virtual address of hw sgl. + * @dir: DMA direction. * * This function unmaps allocated hw sgl. */ void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, - struct hisi_acc_hw_sgl *hw_sgl) + struct hisi_acc_hw_sgl *hw_sgl, enum dma_data_direction dir) { if (!dev || !sgl || !hw_sgl) return; - dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, sgl, sg_nents(sgl), dir); clear_hw_sgl_sge(hw_sgl); hw_sgl->entry_sum_in_chain = 0; hw_sgl->entry_sum_in_sgl = 0; diff --git a/drivers/crypto/hisilicon/zip/Makefile b/drivers/crypto/hisilicon/zip/Makefile index a936f099ee22..13de020b77d6 100644 --- a/drivers/crypto/hisilicon/zip/Makefile +++ b/drivers/crypto/hisilicon/zip/Makefile @@ -1,2 +1,2 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o -hisi_zip-objs = zip_main.o zip_crypto.o +hisi_zip-objs = zip_main.o zip_crypto.o dae_main.o diff --git a/drivers/crypto/hisilicon/zip/dae_main.c b/drivers/crypto/hisilicon/zip/dae_main.c new file mode 100644 index 000000000000..68aebd02fc84 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/dae_main.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 HiSilicon Limited. */ + +#include <linux/bitops.h> +#include <linux/io.h> +#include <linux/uacce.h> +#include "zip.h" + +/* memory */ +#define DAE_MEM_START_OFFSET 0x331040 +#define DAE_MEM_DONE_OFFSET 0x331044 +#define DAE_MEM_START_MASK 0x1 +#define DAE_MEM_DONE_MASK 0x1 +#define DAE_REG_RD_INTVRL_US 10 +#define DAE_REG_RD_TMOUT_US USEC_PER_SEC + +#define DAE_ALG_NAME "hashagg" +#define DAE_V5_ALG_NAME "hashagg\nudma\nhashjoin\ngather" + +/* error */ +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_MASK (BIT(0) | BIT(5)) +#define DAE_ERR_SOURCE_OFFSET 0x331C84 +#define DAE_ERR_STATUS_OFFSET 0x331C88 +#define DAE_ERR_CE_OFFSET 0x331CA0 +#define DAE_ERR_CE_MASK BIT(3) +#define DAE_ERR_NFE_OFFSET 0x331CA4 +#define DAE_ERR_NFE_MASK 0x17 +#define DAE_ERR_FE_OFFSET 0x331CA8 +#define DAE_ERR_FE_MASK 0 +#define DAE_ECC_MBIT_MASK BIT(2) +#define DAE_ECC_INFO_OFFSET 0x33400C +#define DAE_ERR_SHUTDOWN_OFFSET 0x331CAC +#define DAE_ERR_SHUTDOWN_MASK 0x17 +#define DAE_ERR_ENABLE_OFFSET 0x331C80 +#define DAE_ERR_ENABLE_MASK (DAE_ERR_FE_MASK | DAE_ERR_NFE_MASK | DAE_ERR_CE_MASK) +#define DAE_AM_CTRL_GLOBAL_OFFSET 0x330000 +#define DAE_AM_RETURN_OFFSET 0x330150 +#define DAE_AM_RETURN_MASK 0x3 +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_EN_MASK (BIT(0) | BIT(5)) + +struct hisi_dae_hw_error { + u32 int_msk; + const char *msg; +}; + +static const struct hisi_dae_hw_error dae_hw_error[] = { + { .int_msk = BIT(0), .msg = "dae_axi_bus_err" }, + { .int_msk = BIT(1), .msg = "dae_axi_poison_err" }, + { .int_msk = BIT(2), .msg = "dae_ecc_2bit_err" }, + { .int_msk = BIT(3), .msg = "dae_ecc_1bit_err" }, + { .int_msk = BIT(4), .msg = "dae_fsm_hbeat_err" }, +}; + +static inline bool dae_is_support(struct hisi_qm *qm) +{ + if (test_bit(QM_SUPPORT_DAE, &qm->caps)) + return true; + + return false; +} + +int hisi_dae_set_user_domain(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_MEM_START_OFFSET); + val |= DAE_MEM_START_MASK; + writel(val, qm->io_base + DAE_MEM_START_OFFSET); + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_MEM_DONE_OFFSET, val, + val & DAE_MEM_DONE_MASK, + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to init dae memory!\n"); + + return ret; +} + +int hisi_dae_set_alg(struct hisi_qm *qm) +{ + const char *alg_name; + size_t len; + + if (!dae_is_support(qm)) + return 0; + + if (!qm->uacce) + return 0; + + if (qm->ver >= QM_HW_V5) + alg_name = DAE_V5_ALG_NAME; + else + alg_name = DAE_ALG_NAME; + + len = strlen(qm->uacce->algs); + /* A line break may be required */ + if (len + strlen(alg_name) + 1 >= QM_DEV_ALG_MAX_LEN) { + pci_err(qm->pdev, "algorithm name is too long!\n"); + return -EINVAL; + } + + if (len) + strcat((char *)qm->uacce->algs, "\n"); + + strcat((char *)qm->uacce->algs, alg_name); + + return 0; +} + +static void hisi_dae_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 axi_val, err_val; + + axi_val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + if (enable) { + axi_val |= DAE_AXI_SHUTDOWN_MASK; + err_val = DAE_ERR_SHUTDOWN_MASK; + } else { + axi_val &= ~DAE_AXI_SHUTDOWN_MASK; + err_val = 0; + } + + writel(axi_val, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(err_val, qm->io_base + DAE_ERR_SHUTDOWN_OFFSET); +} + +void hisi_dae_hw_error_enable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + /* clear dae hw error source if having */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_SOURCE_OFFSET); + + /* configure error type */ + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); + writel(DAE_ERR_FE_MASK, qm->io_base + DAE_ERR_FE_OFFSET); + + hisi_dae_master_ooo_ctrl(qm, true); + + /* enable dae hw error interrupts */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_ENABLE_OFFSET); +} + +void hisi_dae_hw_error_disable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + writel(0, qm->io_base + DAE_ERR_ENABLE_OFFSET); + hisi_dae_master_ooo_ctrl(qm, false); +} + +static u32 hisi_dae_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + DAE_ERR_STATUS_OFFSET); +} + +static void hisi_dae_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + if (!dae_is_support(qm)) + return; + + writel(err_sts, qm->io_base + DAE_ERR_SOURCE_OFFSET); +} + +static void hisi_dae_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + writel(DAE_ERR_NFE_MASK & (~err_type), qm->io_base + DAE_ERR_NFE_OFFSET); +} + +static void hisi_dae_enable_error_report(struct hisi_qm *qm) +{ + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); +} + +static void hisi_dae_log_hw_error(struct hisi_qm *qm, u32 err_type) +{ + const struct hisi_dae_hw_error *err = dae_hw_error; + struct device *dev = &qm->pdev->dev; + u32 ecc_info; + size_t i; + + for (i = 0; i < ARRAY_SIZE(dae_hw_error); i++) { + err = &dae_hw_error[i]; + if (!(err->int_msk & err_type)) + continue; + + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & DAE_ECC_MBIT_MASK) { + ecc_info = readl(qm->io_base + DAE_ECC_INFO_OFFSET); + dev_err(dev, "dae multi ecc sram info 0x%x\n", ecc_info); + } + } +} + +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return ACC_ERR_NONE; + + err_status = hisi_dae_get_hw_err_status(qm); + if (!err_status) + return ACC_ERR_NONE; + + hisi_dae_log_hw_error(qm, err_status); + + if (err_status & DAE_ERR_NFE_MASK) { + /* Disable the same error reporting until device is recovered. */ + hisi_dae_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + hisi_dae_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hisi_dae_enable_error_report(qm); + + return ACC_ERR_RECOVERED; +} + +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return false; + + err_status = hisi_dae_get_hw_err_status(qm); + if (err_status & DAE_ERR_NFE_MASK) + return true; + + return false; +} + +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + val |= BIT(0); + writel(val, qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_AM_RETURN_OFFSET, + val, (val == DAE_AM_RETURN_MASK), + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + dev_err(&qm->pdev->dev, "failed to close dae axi ooo!\n"); + + return ret; +} + +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + if (!dae_is_support(qm)) + return; + + val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + + writel(val & ~DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(val | DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); +} diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 2fecf346c3c9..9fb2a9c01132 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -103,4 +103,12 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); +int hisi_dae_set_user_domain(struct hisi_qm *qm); +int hisi_dae_set_alg(struct hisi_qm *qm); +void hisi_dae_hw_error_disable(struct hisi_qm *qm); +void hisi_dae_hw_error_enable(struct hisi_qm *qm); +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm); +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm); +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm); +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 7327f8f29b01..b97513981a3b 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -224,7 +224,8 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, return -EINVAL; req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, - req->req_id << 1, &req->dma_src); + req->req_id << 1, &req->dma_src, + DMA_TO_DEVICE); if (IS_ERR(req->hw_src)) { dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", PTR_ERR(req->hw_src)); @@ -233,7 +234,7 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, (req->req_id << 1) + 1, - &req->dma_dst); + &req->dma_dst, DMA_FROM_DEVICE); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", @@ -258,9 +259,9 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, return -EINPROGRESS; err_unmap_output: - hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); + hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE); err_unmap_input: - hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); + hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE); return ret; } @@ -303,8 +304,8 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) err = -EIO; } - hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); - hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); + hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst, DMA_FROM_DEVICE); + hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src, DMA_TO_DEVICE); acomp_req->dlen = ops->get_dstlen(sqe); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 9239b251c2d7..4fcbe6bada06 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -65,6 +65,7 @@ #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 #define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0) +#define HZIP_AXI_ERROR_MASK (BIT(2) | BIT(3)) #define HZIP_SQE_SIZE 128 #define HZIP_PF_DEF_Q_NUM 64 #define HZIP_PF_DEF_Q_BASE 0 @@ -80,6 +81,7 @@ #define HZIP_ALG_GZIP_BIT GENMASK(3, 2) #define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4) #define HZIP_ALG_LZ77_BIT GENMASK(7, 6) +#define HZIP_ALG_LZ4_BIT GENMASK(9, 8) #define HZIP_BUF_SIZE 22 #define HZIP_SQE_MASK_OFFSET 64 @@ -95,10 +97,16 @@ #define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0))) #define HZIP_SVA_PREFETCH_DISABLE BIT(26) #define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30)) +#define HZIP_SVA_PREFETCH_NUM GENMASK(18, 16) +#define HZIP_SVA_STALL_NUM GENMASK(15, 0) #define HZIP_SHAPER_RATE_COMPRESS 750 #define HZIP_SHAPER_RATE_DECOMPRESS 140 -#define HZIP_DELAY_1_US 1 -#define HZIP_POLL_TIMEOUT_US 1000 +#define HZIP_DELAY_1_US 1 +#define HZIP_POLL_TIMEOUT_US 1000 +#define HZIP_WAIT_SVA_READY 500000 +#define HZIP_READ_SVA_STATUS_TIMES 3 +#define HZIP_WAIT_US_MIN 10 +#define HZIP_WAIT_US_MAX 20 /* clock gating */ #define HZIP_PEH_CFG_AUTO_GATE 0x3011A8 @@ -111,6 +119,9 @@ /* zip comp high performance */ #define HZIP_HIGH_PERF_OFFSET 0x301208 +#define HZIP_LIT_LEN_EN_OFFSET 0x301204 +#define HZIP_LIT_LEN_EN_EN BIT(4) + enum { HZIP_HIGH_COMP_RATE, HZIP_HIGH_COMP_PERF, @@ -141,6 +152,12 @@ static const struct qm_dev_alg zip_dev_algs[] = { { }, { .alg_msk = HZIP_ALG_LZ77_BIT, .alg = "lz77_zstd\n", + }, { + .alg_msk = HZIP_ALG_LZ77_BIT, + .alg = "lz77_only\n", + }, { + .alg_msk = HZIP_ALG_LZ4_BIT, + .alg = "lz4\n", }, }; @@ -448,10 +465,23 @@ bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) return false; } -static int hisi_zip_set_high_perf(struct hisi_qm *qm) +static void hisi_zip_literal_set(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + HZIP_LIT_LEN_EN_OFFSET); + val &= ~HZIP_LIT_LEN_EN_EN; + + /* enable literal length in stream mode compression */ + writel(val, qm->io_base + HZIP_LIT_LEN_EN_OFFSET); +} + +static void hisi_zip_set_high_perf(struct hisi_qm *qm) { u32 val; - int ret; val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET); if (perf_mode == HZIP_HIGH_COMP_PERF) @@ -461,16 +491,36 @@ static int hisi_zip_set_high_perf(struct hisi_qm *qm) /* Set perf mode */ writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET); - ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_HIGH_PERF_OFFSET, - val, val == perf_mode, HZIP_DELAY_1_US, - HZIP_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to set perf mode\n"); +} - return ret; +static int hisi_zip_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + offset); + if (val & mask) + count = 0; + else if (++count == HZIP_READ_SVA_STATUS_TIMES) + break; + + usleep_range(HZIP_WAIT_US_MIN, HZIP_WAIT_US_MAX); + } while (++try_times < HZIP_WAIT_SVA_READY); + + if (try_times == HZIP_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; } -static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) +static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -478,19 +528,20 @@ static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; - /* Enable prefetch */ val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); - val &= HZIP_PREFETCH_ENABLE; + val |= HZIP_SVA_PREFETCH_DISABLE; writel(val, qm->io_base + HZIP_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG, - val, !(val & HZIP_SVA_PREFETCH_DISABLE), + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS, + val, !(val & HZIP_SVA_DISABLE_READY), HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); if (ret) - pci_err(qm->pdev, "failed to open sva prefetch\n"); + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_STALL_NUM); } -static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) +static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; @@ -498,15 +549,23 @@ static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; + /* Enable prefetch */ val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); - val |= HZIP_SVA_PREFETCH_DISABLE; + val &= HZIP_PREFETCH_ENABLE; writel(val, qm->io_base + HZIP_PREFETCH_CFG); - ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS, - val, !(val & HZIP_SVA_DISABLE_READY), + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG, + val, !(val & HZIP_SVA_PREFETCH_DISABLE), HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + hisi_zip_close_sva_prefetch(qm); + return; + } + + ret = hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_PREFETCH_NUM); if (ret) - pci_err(qm->pdev, "failed to close sva prefetch\n"); + hisi_zip_close_sva_prefetch(qm); } static void hisi_zip_enable_clock_gate(struct hisi_qm *qm) @@ -530,6 +589,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) void __iomem *base = qm->io_base; u32 dcomp_bm, comp_bm; u32 zip_core_en; + int ret; /* qm user domain */ writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); @@ -565,6 +625,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); } + hisi_zip_open_sva_prefetch(qm); /* let's open all compression/decompression cores */ @@ -580,9 +641,19 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); + hisi_zip_set_high_perf(qm); + hisi_zip_literal_set(qm); hisi_zip_enable_clock_gate(qm); + ret = hisi_dae_set_user_domain(qm); + if (ret) + goto close_sva_prefetch; + return 0; + +close_sva_prefetch: + hisi_zip_close_sva_prefetch(qm); + return ret; } static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) @@ -592,8 +663,7 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); if (enable) { val1 |= HZIP_AXI_SHUTDOWN_ENABLE; - val2 = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + val2 = qm->err_info.dev_err.shutdown_mask; } else { val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE; val2 = 0x0; @@ -607,7 +677,8 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hisi_zip_hw_error_enable(struct hisi_qm *qm) { - u32 nfe, ce; + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; if (qm->ver == QM_HW_V1) { writel(HZIP_CORE_INT_MASK_ALL, @@ -616,33 +687,33 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) return; } - nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); - ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); - /* clear ZIP hw error source if having */ - writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_SOURCE); + writel(err_mask, qm->io_base + HZIP_CORE_INT_SOURCE); /* configure error type */ - writel(ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); - writel(HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); - writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(dev_err->ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(dev_err->fe, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); + writel(dev_err->nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); hisi_zip_master_ooo_ctrl(qm, true); /* enable ZIP hw error interrupts */ - writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); + writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); + + hisi_dae_hw_error_enable(qm); } static void hisi_zip_hw_error_disable(struct hisi_qm *qm) { - u32 nfe, ce; + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; /* disable ZIP hw error interrupts */ - nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); - ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); - writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG); + writel(err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); hisi_zip_master_ooo_ctrl(qm, false); + + hisi_dae_hw_error_disable(qm); } static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) @@ -1112,12 +1183,20 @@ static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) static void hisi_zip_disable_error_report(struct hisi_qm *qm, u32 err_type) { - u32 nfe_mask; + u32 nfe_mask = qm->err_info.dev_err.nfe; - nfe_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); writel(nfe_mask & (~err_type), qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); } +static void hisi_zip_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(ce_mask, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); +} + static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) { u32 val; @@ -1129,6 +1208,8 @@ static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) writel(val | HZIP_AXI_SHUTDOWN_ENABLE, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + + hisi_dae_open_axi_master_ooo(qm); } static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) @@ -1147,42 +1228,104 @@ static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) { + enum acc_err_result zip_result = ACC_ERR_NONE; + enum acc_err_result dae_result; u32 err_status; + /* Get device hardware new error status */ err_status = hisi_zip_get_hw_err_status(qm); if (err_status) { - if (err_status & qm->err_info.ecc_2bits_mask) + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; hisi_zip_log_hw_error(qm, err_status); - if (err_status & qm->err_info.dev_reset_mask) { + if (err_status & qm->err_info.dev_err.reset_mask) { /* Disable the same error reporting until device is recovered. */ hisi_zip_disable_error_report(qm, err_status); - return ACC_ERR_NEED_RESET; + zip_result = ACC_ERR_NEED_RESET; + } else { + hisi_zip_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hisi_zip_enable_error_report(qm); } - hisi_zip_clear_hw_err_status(qm, err_status); } - return ACC_ERR_RECOVERED; + dae_result = hisi_dae_get_err_result(qm); + + return (zip_result == ACC_ERR_NEED_RESET || + dae_result == ACC_ERR_NEED_RESET) ? + ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; +} + +static bool hisi_zip_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hisi_zip_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_err.shutdown_mask) + return true; + + return hisi_dae_dev_is_abnormal(qm); +} + +static int hisi_zip_set_priv_status(struct hisi_qm *qm) +{ + return hisi_dae_close_axi_master_ooo(qm); +} + +static void hisi_zip_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + u32 val; + + val = ~(err_mask & (~HZIP_AXI_ERROR_MASK)); + writel(val, qm->io_base + HZIP_CORE_INT_MASK_REG); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~HZIP_AXI_ERROR_MASK), + qm->io_base + HZIP_OOO_SHUTDOWN_SEL); +} + +static void hisi_zip_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(HZIP_AXI_ERROR_MASK, qm->io_base + HZIP_CORE_INT_SOURCE); + + writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + HZIP_OOO_SHUTDOWN_SEL); } static void hisi_zip_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + qm_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + qm_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + + dev_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + dev_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_RESET_MASK_CAP, qm->cap_ver); - err_info->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; - err_info->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver); - err_info->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_QM_NFE_MASK_CAP, qm->cap_ver); - err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; - err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); - err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_QM_RESET_MASK_CAP, qm->cap_ver); - err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HZIP_WR_PORT; err_info->acpi_rst = "ZRST"; } @@ -1200,6 +1343,10 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .show_last_dfx_regs = hisi_zip_show_last_dfx_regs, .err_info_init = hisi_zip_err_info_init, .get_err_result = hisi_zip_get_err_result, + .set_priv_status = hisi_zip_set_priv_status, + .dev_is_abnormal = hisi_zip_dev_is_abnormal, + .disable_axi_error = hisi_zip_disable_axi_error, + .enable_axi_error = hisi_zip_enable_axi_error, }; static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) @@ -1219,11 +1366,6 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) if (ret) return ret; - ret = hisi_zip_set_high_perf(qm); - if (ret) - return ret; - - hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); @@ -1241,7 +1383,7 @@ static int zip_pre_store_cap_reg(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(zip_cap_query_info); - zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL); + zip_cap = devm_kcalloc(&pdev->dev, size, sizeof(*zip_cap), GFP_KERNEL); if (!zip_cap) return -ENOMEM; @@ -1264,7 +1406,6 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) int ret; qm->pdev = pdev; - qm->ver = pdev->revision; qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; @@ -1301,17 +1442,24 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) ret = zip_pre_store_cap_reg(qm); if (ret) { pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); - hisi_qm_uninit(qm); - return ret; + goto err_qm_uninit; } alg_msk = qm->cap_tables.dev_cap_table[ZIP_ALG_BITMAP].cap_val; ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set zip algs!\n"); - hisi_qm_uninit(qm); + goto err_qm_uninit; } + ret = hisi_dae_set_alg(qm); + if (ret) + goto err_qm_uninit; + + return 0; + +err_qm_uninit: + hisi_qm_uninit(qm); return ret; } diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 1dc2378aa88b..f22c12e36b56 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -436,7 +436,7 @@ static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev) struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); if (ctx->flags & DRIVER_FLAGS_SG) - dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE); + dma_unmap_sg(hdev->dev, ctx->sg, 1, DMA_TO_DEVICE); return 0; } @@ -491,8 +491,9 @@ static int img_hash_init(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -555,10 +556,10 @@ static int img_hash_update(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -570,9 +571,10 @@ static int img_hash_final(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -584,11 +586,12 @@ static int img_hash_finup(struct ahash_request *req) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); + return crypto_ahash_finup(&rctx->fallback_req); } @@ -600,8 +603,9 @@ static int img_hash_import(struct ahash_request *req, const void *in) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -613,8 +617,9 @@ static int img_hash_export(struct ahash_request *req, void *out) struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); - rctx->fallback_req.base.flags = req->base.flags - & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } @@ -695,22 +700,22 @@ static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name) static int img_hash_cra_md5_init(struct crypto_tfm *tfm) { - return img_hash_cra_init(tfm, "md5-generic"); + return img_hash_cra_init(tfm, "md5-lib"); } static int img_hash_cra_sha1_init(struct crypto_tfm *tfm) { - return img_hash_cra_init(tfm, "sha1-generic"); + return img_hash_cra_init(tfm, "sha1-lib"); } static int img_hash_cra_sha224_init(struct crypto_tfm *tfm) { - return img_hash_cra_init(tfm, "sha224-generic"); + return img_hash_cra_init(tfm, "sha224-lib"); } static int img_hash_cra_sha256_init(struct crypto_tfm *tfm) { - return img_hash_cra_init(tfm, "sha256-generic"); + return img_hash_cra_init(tfm, "sha256-lib"); } static void img_hash_cra_exit(struct crypto_tfm *tfm) diff --git a/drivers/crypto/inside-secure/Makefile b/drivers/crypto/inside-secure/Makefile index 13f64f96c626..30d13fd5d58e 100644 --- a/drivers/crypto/inside-secure/Makefile +++ b/drivers/crypto/inside-secure/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += crypto_safexcel.o crypto_safexcel-objs := safexcel.o safexcel_ring.o safexcel_cipher.o safexcel_hash.o +obj-y += eip93/ diff --git a/drivers/crypto/inside-secure/eip93/Kconfig b/drivers/crypto/inside-secure/eip93/Kconfig new file mode 100644 index 000000000000..8353d3d7ec9b --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/Kconfig @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0 +config CRYPTO_DEV_EIP93 + tristate "Support for EIP93 crypto HW accelerators" + depends on SOC_MT7621 || ARCH_AIROHA ||COMPILE_TEST + select CRYPTO_LIB_AES + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + help + EIP93 have various crypto HW accelerators. Select this if + you want to use the EIP93 modules for any of the crypto algorithms. + + If the IP supports it, this provide offload for AES - ECB, CBC and + CTR crypto. Also provide DES and 3DES ECB and CBC. + + Also provide AEAD authenc(hmac(x), cipher(y)) for supported algo. diff --git a/drivers/crypto/inside-secure/eip93/Makefile b/drivers/crypto/inside-secure/eip93/Makefile new file mode 100644 index 000000000000..a3d3d3677cdc --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/Makefile @@ -0,0 +1,5 @@ +obj-$(CONFIG_CRYPTO_DEV_EIP93) += crypto-hw-eip93.o + +crypto-hw-eip93-y += eip93-main.o eip93-common.o +crypto-hw-eip93-y += eip93-cipher.o eip93-aead.o +crypto-hw-eip93-y += eip93-hash.o diff --git a/drivers/crypto/inside-secure/eip93/eip93-aead.c b/drivers/crypto/inside-secure/eip93/eip93-aead.c new file mode 100644 index 000000000000..18dd8a9a5165 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-aead.c @@ -0,0 +1,711 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ + +#include <crypto/aead.h> +#include <crypto/aes.h> +#include <crypto/authenc.h> +#include <crypto/ctr.h> +#include <crypto/hmac.h> +#include <crypto/internal/aead.h> +#include <crypto/md5.h> +#include <crypto/null.h> +#include <crypto/sha1.h> +#include <crypto/sha2.h> + +#include <crypto/internal/des.h> + +#include <linux/crypto.h> +#include <linux/dma-mapping.h> + +#include "eip93-aead.h" +#include "eip93-cipher.h" +#include "eip93-common.h" +#include "eip93-regs.h" + +void eip93_aead_handle_result(struct crypto_async_request *async, int err) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(async->tfm); + struct eip93_device *eip93 = ctx->eip93; + struct aead_request *req = aead_request_cast(async); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + + eip93_unmap_dma(eip93, rctx, req->src, req->dst); + eip93_handle_result(eip93, rctx, req->iv); + + aead_request_complete(req, err); +} + +static int eip93_aead_send_req(struct crypto_async_request *async) +{ + struct aead_request *req = aead_request_cast(async); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + int err; + + err = check_valid_request(rctx); + if (err) { + aead_request_complete(req, err); + return err; + } + + return eip93_send_req(async, req->iv, rctx); +} + +/* Crypto aead API functions */ +static int eip93_aead_cra_init(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, alg.aead.base); + + crypto_aead_set_reqsize(__crypto_aead_cast(tfm), + sizeof(struct eip93_cipher_reqctx)); + + ctx->eip93 = tmpl->eip93; + ctx->flags = tmpl->flags; + ctx->type = tmpl->type; + ctx->set_assoc = true; + + ctx->sa_record = kzalloc(sizeof(*ctx->sa_record), GFP_KERNEL); + if (!ctx->sa_record) + return -ENOMEM; + + return 0; +} + +static void eip93_aead_cra_exit(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + dma_unmap_single(ctx->eip93->dev, ctx->sa_record_base, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + kfree(ctx->sa_record); +} + +static int eip93_aead_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_authenc_keys keys; + struct crypto_aes_ctx aes; + struct sa_record *sa_record = ctx->sa_record; + u32 nonce = 0; + int ret; + + if (crypto_authenc_extractkeys(&keys, key, len)) + return -EINVAL; + + if (IS_RFC3686(ctx->flags)) { + if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; + memcpy(&nonce, keys.enckey + keys.enckeylen, + CTR_RFC3686_NONCE_SIZE); + } + + switch ((ctx->flags & EIP93_ALG_MASK)) { + case EIP93_ALG_DES: + ret = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); + if (ret) + return ret; + + break; + case EIP93_ALG_3DES: + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + return -EINVAL; + + ret = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); + if (ret) + return ret; + + break; + case EIP93_ALG_AES: + ret = aes_expandkey(&aes, keys.enckey, keys.enckeylen); + if (ret) + return ret; + + break; + } + + ctx->blksize = crypto_aead_blocksize(ctfm); + /* Encryption key */ + eip93_set_sa_record(sa_record, keys.enckeylen, ctx->flags); + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_OPCODE; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_OPCODE, + EIP93_SA_CMD_OPCODE_BASIC_OUT_ENC_HASH); + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_DIGEST_LENGTH; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, + ctx->authsize / sizeof(u32)); + + memcpy(sa_record->sa_key, keys.enckey, keys.enckeylen); + ctx->sa_nonce = nonce; + sa_record->sa_nonce = nonce; + + /* authentication key */ + ret = eip93_hmac_setkey(ctx->flags, keys.authkey, keys.authkeylen, + ctx->authsize, sa_record->sa_i_digest, + sa_record->sa_o_digest, false); + + ctx->set_assoc = true; + + return ret; +} + +static int eip93_aead_setauthsize(struct crypto_aead *ctfm, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->authsize = authsize; + ctx->sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_DIGEST_LENGTH; + ctx->sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, + ctx->authsize / sizeof(u32)); + + return 0; +} + +static void eip93_aead_setassoc(struct eip93_crypto_ctx *ctx, + struct aead_request *req) +{ + struct sa_record *sa_record = ctx->sa_record; + + sa_record->sa_cmd1_word &= ~EIP93_SA_CMD_HASH_CRYPT_OFFSET; + sa_record->sa_cmd1_word |= FIELD_PREP(EIP93_SA_CMD_HASH_CRYPT_OFFSET, + req->assoclen / sizeof(u32)); + + ctx->assoclen = req->assoclen; +} + +static int eip93_aead_crypt(struct aead_request *req) +{ + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + struct crypto_async_request *async = &req->base; + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ret; + + ctx->sa_record_base = dma_map_single(ctx->eip93->dev, ctx->sa_record, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + ret = dma_mapping_error(ctx->eip93->dev, ctx->sa_record_base); + if (ret) + return ret; + + rctx->textsize = req->cryptlen; + rctx->blksize = ctx->blksize; + rctx->assoclen = req->assoclen; + rctx->authsize = ctx->authsize; + rctx->sg_src = req->src; + rctx->sg_dst = req->dst; + rctx->ivsize = crypto_aead_ivsize(aead); + rctx->desc_flags = EIP93_DESC_AEAD; + rctx->sa_record_base = ctx->sa_record_base; + + if (IS_DECRYPT(rctx->flags)) + rctx->textsize -= rctx->authsize; + + return eip93_aead_send_req(async); +} + +static int eip93_aead_encrypt(struct aead_request *req) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + + rctx->flags = ctx->flags; + rctx->flags |= EIP93_ENCRYPT; + if (ctx->set_assoc) { + eip93_aead_setassoc(ctx, req); + ctx->set_assoc = false; + } + + if (req->assoclen != ctx->assoclen) { + dev_err(ctx->eip93->dev, "Request AAD length error\n"); + return -EINVAL; + } + + return eip93_aead_crypt(req); +} + +static int eip93_aead_decrypt(struct aead_request *req) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + + ctx->sa_record->sa_cmd0_word |= EIP93_SA_CMD_DIRECTION_IN; + ctx->sa_record->sa_cmd1_word &= ~(EIP93_SA_CMD_COPY_PAD | + EIP93_SA_CMD_COPY_DIGEST); + + rctx->flags = ctx->flags; + rctx->flags |= EIP93_DECRYPT; + if (ctx->set_assoc) { + eip93_aead_setassoc(ctx, req); + ctx->set_assoc = false; + } + + if (req->assoclen != ctx->assoclen) { + dev_err(ctx->eip93->dev, "Request AAD length error\n"); + return -EINVAL; + } + + return eip93_aead_crypt(req); +} + +/* Available authenc algorithms in this module */ +struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = + "authenc(hmac(md5-eip93), cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_md5_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(md5-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = + "authenc(hmac(md5-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(md5-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/eip93/eip93-aead.h b/drivers/crypto/inside-secure/eip93/eip93-aead.h new file mode 100644 index 000000000000..e2fa8fd39c50 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-aead.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ +#ifndef _EIP93_AEAD_H_ +#define _EIP93_AEAD_H_ + +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_ecb_null; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_ecb_null; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_ecb_null; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_ecb_null; + +void eip93_aead_handle_result(struct crypto_async_request *async, int err); + +#endif /* _EIP93_AEAD_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-aes.h b/drivers/crypto/inside-secure/eip93/eip93-aes.h new file mode 100644 index 000000000000..1d83d39cab2a --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-aes.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ +#ifndef _EIP93_AES_H_ +#define _EIP93_AES_H_ + +extern struct eip93_alg_template eip93_alg_ecb_aes; +extern struct eip93_alg_template eip93_alg_cbc_aes; +extern struct eip93_alg_template eip93_alg_ctr_aes; +extern struct eip93_alg_template eip93_alg_rfc3686_aes; + +#endif /* _EIP93_AES_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-cipher.c b/drivers/crypto/inside-secure/eip93/eip93-cipher.c new file mode 100644 index 000000000000..1f2d6846610f --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-cipher.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ + +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/internal/des.h> +#include <linux/dma-mapping.h> + +#include "eip93-aes.h" +#include "eip93-cipher.h" +#include "eip93-common.h" +#include "eip93-des.h" +#include "eip93-regs.h" + +void eip93_skcipher_handle_result(struct crypto_async_request *async, int err) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(async->tfm); + struct eip93_device *eip93 = ctx->eip93; + struct skcipher_request *req = skcipher_request_cast(async); + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + + eip93_unmap_dma(eip93, rctx, req->src, req->dst); + eip93_handle_result(eip93, rctx, req->iv); + + skcipher_request_complete(req, err); +} + +static int eip93_skcipher_send_req(struct crypto_async_request *async) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + int err; + + err = check_valid_request(rctx); + + if (err) { + skcipher_request_complete(req, err); + return err; + } + + return eip93_send_req(async, req->iv, rctx); +} + +/* Crypto skcipher API functions */ +static int eip93_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, alg.skcipher.base); + + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct eip93_cipher_reqctx)); + + memset(ctx, 0, sizeof(*ctx)); + + ctx->eip93 = tmpl->eip93; + ctx->type = tmpl->type; + + ctx->sa_record = kzalloc(sizeof(*ctx->sa_record), GFP_KERNEL); + if (!ctx->sa_record) + return -ENOMEM; + + return 0; +} + +static void eip93_skcipher_cra_exit(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + dma_unmap_single(ctx->eip93->dev, ctx->sa_record_base, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + kfree(ctx->sa_record); +} + +static int eip93_skcipher_setkey(struct crypto_skcipher *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, + alg.skcipher.base); + struct sa_record *sa_record = ctx->sa_record; + unsigned int keylen = len; + u32 flags = tmpl->flags; + u32 nonce = 0; + int ret; + + if (!key || !keylen) + return -EINVAL; + + if (IS_RFC3686(flags)) { + if (len < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + keylen = len - CTR_RFC3686_NONCE_SIZE; + memcpy(&nonce, key + keylen, CTR_RFC3686_NONCE_SIZE); + } + + if (flags & EIP93_ALG_DES) { + ctx->blksize = DES_BLOCK_SIZE; + ret = verify_skcipher_des_key(ctfm, key); + if (ret) + return ret; + } + if (flags & EIP93_ALG_3DES) { + ctx->blksize = DES3_EDE_BLOCK_SIZE; + ret = verify_skcipher_des3_key(ctfm, key); + if (ret) + return ret; + } + + if (flags & EIP93_ALG_AES) { + struct crypto_aes_ctx aes; + + ctx->blksize = AES_BLOCK_SIZE; + ret = aes_expandkey(&aes, key, keylen); + if (ret) + return ret; + } + + eip93_set_sa_record(sa_record, keylen, flags); + + memcpy(sa_record->sa_key, key, keylen); + ctx->sa_nonce = nonce; + sa_record->sa_nonce = nonce; + + return 0; +} + +static int eip93_skcipher_crypt(struct skcipher_request *req) +{ + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_async_request *async = &req->base; + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + int ret; + + if (!req->cryptlen) + return 0; + + /* + * ECB and CBC algorithms require message lengths to be + * multiples of block size. + */ + if (IS_ECB(rctx->flags) || IS_CBC(rctx->flags)) + if (!IS_ALIGNED(req->cryptlen, + crypto_skcipher_blocksize(skcipher))) + return -EINVAL; + + ctx->sa_record_base = dma_map_single(ctx->eip93->dev, ctx->sa_record, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + ret = dma_mapping_error(ctx->eip93->dev, ctx->sa_record_base); + if (ret) + return ret; + + rctx->assoclen = 0; + rctx->textsize = req->cryptlen; + rctx->authsize = 0; + rctx->sg_src = req->src; + rctx->sg_dst = req->dst; + rctx->ivsize = crypto_skcipher_ivsize(skcipher); + rctx->blksize = ctx->blksize; + rctx->desc_flags = EIP93_DESC_SKCIPHER; + rctx->sa_record_base = ctx->sa_record_base; + + return eip93_skcipher_send_req(async); +} + +static int eip93_skcipher_encrypt(struct skcipher_request *req) +{ + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct eip93_alg_template *tmpl = container_of(req->base.tfm->__crt_alg, + struct eip93_alg_template, alg.skcipher.base); + + rctx->flags = tmpl->flags; + rctx->flags |= EIP93_ENCRYPT; + + return eip93_skcipher_crypt(req); +} + +static int eip93_skcipher_decrypt(struct skcipher_request *req) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct eip93_alg_template *tmpl = container_of(req->base.tfm->__crt_alg, + struct eip93_alg_template, alg.skcipher.base); + + ctx->sa_record->sa_cmd0_word |= EIP93_SA_CMD_DIRECTION_IN; + + rctx->flags = tmpl->flags; + rctx->flags |= EIP93_DECRYPT; + + return eip93_skcipher_crypt(req); +} + +/* Available algorithms in this module */ +struct eip93_alg_template eip93_alg_ecb_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_ECB | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = 0, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb(aes-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_cbc_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc(aes-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_ctr_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CTR | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr(aes-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_rfc3686_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686(ctr(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_ecb_des = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_ECB | EIP93_ALG_DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = 0, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "ebc(des-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_cbc_des = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc(des-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_ecb_des3_ede = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_ECB | EIP93_ALG_3DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb(des3_ede-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc(des3_ede-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/eip93/eip93-cipher.h b/drivers/crypto/inside-secure/eip93/eip93-cipher.h new file mode 100644 index 000000000000..6e2545ebd879 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-cipher.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ +#ifndef _EIP93_CIPHER_H_ +#define _EIP93_CIPHER_H_ + +#include "eip93-main.h" + +struct eip93_crypto_ctx { + struct eip93_device *eip93; + u32 flags; + struct sa_record *sa_record; + u32 sa_nonce; + int blksize; + dma_addr_t sa_record_base; + /* AEAD specific */ + unsigned int authsize; + unsigned int assoclen; + bool set_assoc; + enum eip93_alg_type type; +}; + +struct eip93_cipher_reqctx { + u16 desc_flags; + u16 flags; + unsigned int blksize; + unsigned int ivsize; + unsigned int textsize; + unsigned int assoclen; + unsigned int authsize; + dma_addr_t sa_record_base; + struct sa_state *sa_state; + dma_addr_t sa_state_base; + struct eip93_descriptor *cdesc; + struct scatterlist *sg_src; + struct scatterlist *sg_dst; + int src_nents; + int dst_nents; + struct sa_state *sa_state_ctr; + dma_addr_t sa_state_ctr_base; +}; + +int check_valid_request(struct eip93_cipher_reqctx *rctx); + +void eip93_unmap_dma(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + struct scatterlist *reqsrc, struct scatterlist *reqdst); + +void eip93_skcipher_handle_result(struct crypto_async_request *async, int err); + +int eip93_send_req(struct crypto_async_request *async, + const u8 *reqiv, struct eip93_cipher_reqctx *rctx); + +void eip93_handle_result(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + u8 *reqiv); + +#endif /* _EIP93_CIPHER_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-common.c b/drivers/crypto/inside-secure/eip93/eip93-common.c new file mode 100644 index 000000000000..66153aa2493f --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-common.c @@ -0,0 +1,822 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ + +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/hmac.h> +#include <crypto/sha1.h> +#include <crypto/sha2.h> +#include <linux/kernel.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/scatterlist.h> + +#include "eip93-cipher.h" +#include "eip93-hash.h" +#include "eip93-common.h" +#include "eip93-main.h" +#include "eip93-regs.h" + +int eip93_parse_ctrl_stat_err(struct eip93_device *eip93, int err) +{ + u32 ext_err; + + if (!err) + return 0; + + switch (err & ~EIP93_PE_CTRL_PE_EXT_ERR_CODE) { + case EIP93_PE_CTRL_PE_AUTH_ERR: + case EIP93_PE_CTRL_PE_PAD_ERR: + return -EBADMSG; + /* let software handle anti-replay errors */ + case EIP93_PE_CTRL_PE_SEQNUM_ERR: + return 0; + case EIP93_PE_CTRL_PE_EXT_ERR: + break; + default: + dev_err(eip93->dev, "Unhandled error 0x%08x\n", err); + return -EINVAL; + } + + /* Parse additional ext errors */ + ext_err = FIELD_GET(EIP93_PE_CTRL_PE_EXT_ERR_CODE, err); + switch (ext_err) { + case EIP93_PE_CTRL_PE_EXT_ERR_BUS: + case EIP93_PE_CTRL_PE_EXT_ERR_PROCESSING: + return -EIO; + case EIP93_PE_CTRL_PE_EXT_ERR_DESC_OWNER: + return -EACCES; + case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_OP: + case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_ALGO: + case EIP93_PE_CTRL_PE_EXT_ERR_SPI: + return -EINVAL; + case EIP93_PE_CTRL_PE_EXT_ERR_ZERO_LENGTH: + case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_PK_LENGTH: + case EIP93_PE_CTRL_PE_EXT_ERR_BLOCK_SIZE_ERR: + return -EBADMSG; + default: + dev_err(eip93->dev, "Unhandled ext error 0x%08x\n", ext_err); + return -EINVAL; + } +} + +static void *eip93_ring_next_wptr(struct eip93_device *eip93, + struct eip93_desc_ring *ring) +{ + void *ptr = ring->write; + + if ((ring->write == ring->read - ring->offset) || + (ring->read == ring->base && ring->write == ring->base_end)) + return ERR_PTR(-ENOMEM); + + if (ring->write == ring->base_end) + ring->write = ring->base; + else + ring->write += ring->offset; + + return ptr; +} + +static void *eip93_ring_next_rptr(struct eip93_device *eip93, + struct eip93_desc_ring *ring) +{ + void *ptr = ring->read; + + if (ring->write == ring->read) + return ERR_PTR(-ENOENT); + + if (ring->read == ring->base_end) + ring->read = ring->base; + else + ring->read += ring->offset; + + return ptr; +} + +int eip93_put_descriptor(struct eip93_device *eip93, + struct eip93_descriptor *desc) +{ + struct eip93_descriptor *cdesc; + struct eip93_descriptor *rdesc; + + rdesc = eip93_ring_next_wptr(eip93, &eip93->ring->rdr); + if (IS_ERR(rdesc)) + return -ENOENT; + + cdesc = eip93_ring_next_wptr(eip93, &eip93->ring->cdr); + if (IS_ERR(cdesc)) + return -ENOENT; + + memset(rdesc, 0, sizeof(struct eip93_descriptor)); + + memcpy(cdesc, desc, sizeof(struct eip93_descriptor)); + + return 0; +} + +void *eip93_get_descriptor(struct eip93_device *eip93) +{ + struct eip93_descriptor *cdesc; + void *ptr; + + cdesc = eip93_ring_next_rptr(eip93, &eip93->ring->cdr); + if (IS_ERR(cdesc)) + return ERR_PTR(-ENOENT); + + memset(cdesc, 0, sizeof(struct eip93_descriptor)); + + ptr = eip93_ring_next_rptr(eip93, &eip93->ring->rdr); + if (IS_ERR(ptr)) + return ERR_PTR(-ENOENT); + + return ptr; +} + +static void eip93_free_sg_copy(const int len, struct scatterlist **sg) +{ + if (!*sg || !len) + return; + + free_pages((unsigned long)sg_virt(*sg), get_order(len)); + kfree(*sg); + *sg = NULL; +} + +static int eip93_make_sg_copy(struct scatterlist *src, struct scatterlist **dst, + const u32 len, const bool copy) +{ + void *pages; + + *dst = kmalloc(sizeof(**dst), GFP_KERNEL); + if (!*dst) + return -ENOMEM; + + pages = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, + get_order(len)); + if (!pages) { + kfree(*dst); + *dst = NULL; + return -ENOMEM; + } + + sg_init_table(*dst, 1); + sg_set_buf(*dst, pages, len); + + /* copy only as requested */ + if (copy) + sg_copy_to_buffer(src, sg_nents(src), pages, len); + + return 0; +} + +static bool eip93_is_sg_aligned(struct scatterlist *sg, u32 len, + const int blksize) +{ + int nents; + + for (nents = 0; sg; sg = sg_next(sg), ++nents) { + if (!IS_ALIGNED(sg->offset, 4)) + return false; + + if (len <= sg->length) { + if (!IS_ALIGNED(len, blksize)) + return false; + + return true; + } + + if (!IS_ALIGNED(sg->length, blksize)) + return false; + + len -= sg->length; + } + return false; +} + +int check_valid_request(struct eip93_cipher_reqctx *rctx) +{ + struct scatterlist *src = rctx->sg_src; + struct scatterlist *dst = rctx->sg_dst; + u32 textsize = rctx->textsize; + u32 authsize = rctx->authsize; + u32 blksize = rctx->blksize; + u32 totlen_src = rctx->assoclen + rctx->textsize; + u32 totlen_dst = rctx->assoclen + rctx->textsize; + u32 copy_len; + bool src_align, dst_align; + int src_nents, dst_nents; + int err = -EINVAL; + + if (!IS_CTR(rctx->flags)) { + if (!IS_ALIGNED(textsize, blksize)) + return err; + } + + if (authsize) { + if (IS_ENCRYPT(rctx->flags)) + totlen_dst += authsize; + else + totlen_src += authsize; + } + + src_nents = sg_nents_for_len(src, totlen_src); + if (src_nents < 0) + return src_nents; + + dst_nents = sg_nents_for_len(dst, totlen_dst); + if (dst_nents < 0) + return dst_nents; + + if (src == dst) { + src_nents = max(src_nents, dst_nents); + dst_nents = src_nents; + if (unlikely((totlen_src || totlen_dst) && !src_nents)) + return err; + + } else { + if (unlikely(totlen_src && !src_nents)) + return err; + + if (unlikely(totlen_dst && !dst_nents)) + return err; + } + + if (authsize) { + if (dst_nents == 1 && src_nents == 1) { + src_align = eip93_is_sg_aligned(src, totlen_src, blksize); + if (src == dst) + dst_align = src_align; + else + dst_align = eip93_is_sg_aligned(dst, totlen_dst, blksize); + } else { + src_align = false; + dst_align = false; + } + } else { + src_align = eip93_is_sg_aligned(src, totlen_src, blksize); + if (src == dst) + dst_align = src_align; + else + dst_align = eip93_is_sg_aligned(dst, totlen_dst, blksize); + } + + copy_len = max(totlen_src, totlen_dst); + if (!src_align) { + err = eip93_make_sg_copy(src, &rctx->sg_src, copy_len, true); + if (err) + return err; + } + + if (!dst_align) { + err = eip93_make_sg_copy(dst, &rctx->sg_dst, copy_len, false); + if (err) + return err; + } + + src_nents = sg_nents_for_len(rctx->sg_src, totlen_src); + if (src_nents < 0) + return src_nents; + + dst_nents = sg_nents_for_len(rctx->sg_dst, totlen_dst); + if (dst_nents < 0) + return dst_nents; + + rctx->src_nents = src_nents; + rctx->dst_nents = dst_nents; + + return 0; +} + +/* + * Set sa_record function: + * Even sa_record is set to "0", keep " = 0" for readability. + */ +void eip93_set_sa_record(struct sa_record *sa_record, const unsigned int keylen, + const u32 flags) +{ + /* Reset cmd word */ + sa_record->sa_cmd0_word = 0; + sa_record->sa_cmd1_word = 0; + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_IV_FROM_STATE; + if (!IS_ECB(flags)) + sa_record->sa_cmd0_word |= EIP93_SA_CMD_SAVE_IV; + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_OP_BASIC; + + switch ((flags & EIP93_ALG_MASK)) { + case EIP93_ALG_AES: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_AES; + sa_record->sa_cmd1_word |= FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, + keylen >> 3); + break; + case EIP93_ALG_3DES: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_3DES; + break; + case EIP93_ALG_DES: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_DES; + break; + default: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_NULL; + } + + switch ((flags & EIP93_HASH_MASK)) { + case EIP93_HASH_SHA256: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA256; + break; + case EIP93_HASH_SHA224: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA224; + break; + case EIP93_HASH_SHA1: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA1; + break; + case EIP93_HASH_MD5: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_MD5; + break; + default: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_NULL; + } + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_PAD_ZERO; + + switch ((flags & EIP93_MODE_MASK)) { + case EIP93_MODE_CBC: + sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_CBC; + break; + case EIP93_MODE_CTR: + sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_CTR; + break; + case EIP93_MODE_ECB: + sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_ECB; + break; + } + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_DIGEST_3WORD; + if (IS_HASH(flags)) { + sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_PAD; + sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_DIGEST; + } + + if (IS_HMAC(flags)) { + sa_record->sa_cmd1_word |= EIP93_SA_CMD_HMAC; + sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_HEADER; + } + + sa_record->sa_spi = 0x0; + sa_record->sa_seqmum_mask[0] = 0xFFFFFFFF; + sa_record->sa_seqmum_mask[1] = 0x0; +} + +/* + * Poor mans Scatter/gather function: + * Create a Descriptor for every segment to avoid copying buffers. + * For performance better to wait for hardware to perform multiple DMA + */ +static int eip93_scatter_combine(struct eip93_device *eip93, + struct eip93_cipher_reqctx *rctx, + u32 datalen, u32 split, int offsetin) +{ + struct eip93_descriptor *cdesc = rctx->cdesc; + struct scatterlist *sgsrc = rctx->sg_src; + struct scatterlist *sgdst = rctx->sg_dst; + unsigned int remainin = sg_dma_len(sgsrc); + unsigned int remainout = sg_dma_len(sgdst); + dma_addr_t saddr = sg_dma_address(sgsrc); + dma_addr_t daddr = sg_dma_address(sgdst); + dma_addr_t state_addr; + u32 src_addr, dst_addr, len, n; + bool nextin = false; + bool nextout = false; + int offsetout = 0; + int err; + + if (IS_ECB(rctx->flags)) + rctx->sa_state_base = 0; + + if (split < datalen) { + state_addr = rctx->sa_state_ctr_base; + n = split; + } else { + state_addr = rctx->sa_state_base; + n = datalen; + } + + do { + if (nextin) { + sgsrc = sg_next(sgsrc); + remainin = sg_dma_len(sgsrc); + if (remainin == 0) + continue; + + saddr = sg_dma_address(sgsrc); + offsetin = 0; + nextin = false; + } + + if (nextout) { + sgdst = sg_next(sgdst); + remainout = sg_dma_len(sgdst); + if (remainout == 0) + continue; + + daddr = sg_dma_address(sgdst); + offsetout = 0; + nextout = false; + } + src_addr = saddr + offsetin; + dst_addr = daddr + offsetout; + + if (remainin == remainout) { + len = remainin; + if (len > n) { + len = n; + remainin -= n; + remainout -= n; + offsetin += n; + offsetout += n; + } else { + nextin = true; + nextout = true; + } + } else if (remainin < remainout) { + len = remainin; + if (len > n) { + len = n; + remainin -= n; + remainout -= n; + offsetin += n; + offsetout += n; + } else { + offsetout += len; + remainout -= len; + nextin = true; + } + } else { + len = remainout; + if (len > n) { + len = n; + remainin -= n; + remainout -= n; + offsetin += n; + offsetout += n; + } else { + offsetin += len; + remainin -= len; + nextout = true; + } + } + n -= len; + + cdesc->src_addr = src_addr; + cdesc->dst_addr = dst_addr; + cdesc->state_addr = state_addr; + cdesc->pe_length_word = FIELD_PREP(EIP93_PE_LENGTH_HOST_PE_READY, + EIP93_PE_LENGTH_HOST_READY); + cdesc->pe_length_word |= FIELD_PREP(EIP93_PE_LENGTH_LENGTH, len); + + if (n == 0) { + n = datalen - split; + split = datalen; + state_addr = rctx->sa_state_base; + } + + if (n == 0) + cdesc->user_id |= FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, + EIP93_DESC_LAST); + + /* + * Loop - Delay - No need to rollback + * Maybe refine by slowing down at EIP93_RING_BUSY + */ +again: + scoped_guard(spinlock_irqsave, &eip93->ring->write_lock) + err = eip93_put_descriptor(eip93, cdesc); + if (err) { + usleep_range(EIP93_RING_BUSY_DELAY, + EIP93_RING_BUSY_DELAY * 2); + goto again; + } + /* Writing new descriptor count starts DMA action */ + writel(1, eip93->base + EIP93_REG_PE_CD_COUNT); + } while (n); + + return -EINPROGRESS; +} + +int eip93_send_req(struct crypto_async_request *async, + const u8 *reqiv, struct eip93_cipher_reqctx *rctx) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(async->tfm); + struct eip93_device *eip93 = ctx->eip93; + struct scatterlist *src = rctx->sg_src; + struct scatterlist *dst = rctx->sg_dst; + struct sa_state *sa_state; + struct eip93_descriptor cdesc; + u32 flags = rctx->flags; + int offsetin = 0, err; + u32 datalen = rctx->assoclen + rctx->textsize; + u32 split = datalen; + u32 start, end, ctr, blocks; + u32 iv[AES_BLOCK_SIZE / sizeof(u32)]; + int crypto_async_idr; + + rctx->sa_state_ctr = NULL; + rctx->sa_state = NULL; + + if (IS_ECB(flags)) + goto skip_iv; + + memcpy(iv, reqiv, rctx->ivsize); + + rctx->sa_state = kzalloc(sizeof(*rctx->sa_state), GFP_KERNEL); + if (!rctx->sa_state) + return -ENOMEM; + + sa_state = rctx->sa_state; + + memcpy(sa_state->state_iv, iv, rctx->ivsize); + if (IS_RFC3686(flags)) { + sa_state->state_iv[0] = ctx->sa_nonce; + sa_state->state_iv[1] = iv[0]; + sa_state->state_iv[2] = iv[1]; + sa_state->state_iv[3] = (u32 __force)cpu_to_be32(0x1); + } else if (!IS_HMAC(flags) && IS_CTR(flags)) { + /* Compute data length. */ + blocks = DIV_ROUND_UP(rctx->textsize, AES_BLOCK_SIZE); + ctr = be32_to_cpu((__be32 __force)iv[3]); + /* Check 32bit counter overflow. */ + start = ctr; + end = start + blocks - 1; + if (end < start) { + split = AES_BLOCK_SIZE * -start; + /* + * Increment the counter manually to cope with + * the hardware counter overflow. + */ + iv[3] = 0xffffffff; + crypto_inc((u8 *)iv, AES_BLOCK_SIZE); + + rctx->sa_state_ctr = kzalloc(sizeof(*rctx->sa_state_ctr), + GFP_KERNEL); + if (!rctx->sa_state_ctr) { + err = -ENOMEM; + goto free_sa_state; + } + + memcpy(rctx->sa_state_ctr->state_iv, reqiv, rctx->ivsize); + memcpy(sa_state->state_iv, iv, rctx->ivsize); + + rctx->sa_state_ctr_base = dma_map_single(eip93->dev, rctx->sa_state_ctr, + sizeof(*rctx->sa_state_ctr), + DMA_TO_DEVICE); + err = dma_mapping_error(eip93->dev, rctx->sa_state_ctr_base); + if (err) + goto free_sa_state_ctr; + } + } + + rctx->sa_state_base = dma_map_single(eip93->dev, rctx->sa_state, + sizeof(*rctx->sa_state), DMA_TO_DEVICE); + err = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (err) + goto free_sa_state_ctr_dma; + +skip_iv: + + cdesc.pe_ctrl_stat_word = FIELD_PREP(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, + EIP93_PE_CTRL_HOST_READY); + cdesc.sa_addr = rctx->sa_record_base; + cdesc.arc4_addr = 0; + + scoped_guard(spinlock_bh, &eip93->ring->idr_lock) + crypto_async_idr = idr_alloc(&eip93->ring->crypto_async_idr, async, 0, + EIP93_RING_NUM - 1, GFP_ATOMIC); + + cdesc.user_id = FIELD_PREP(EIP93_PE_USER_ID_CRYPTO_IDR, (u16)crypto_async_idr) | + FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, rctx->desc_flags); + + rctx->cdesc = &cdesc; + + /* map DMA_BIDIRECTIONAL to invalidate cache on destination + * implies __dma_cache_wback_inv + */ + if (!dma_map_sg(eip93->dev, dst, rctx->dst_nents, DMA_BIDIRECTIONAL)) { + err = -ENOMEM; + goto free_sa_state_ctr_dma; + } + + if (src != dst && + !dma_map_sg(eip93->dev, src, rctx->src_nents, DMA_TO_DEVICE)) { + err = -ENOMEM; + goto free_sg_dma; + } + + return eip93_scatter_combine(eip93, rctx, datalen, split, offsetin); + +free_sg_dma: + dma_unmap_sg(eip93->dev, dst, rctx->dst_nents, DMA_BIDIRECTIONAL); +free_sa_state_ctr_dma: + if (rctx->sa_state_ctr) + dma_unmap_single(eip93->dev, rctx->sa_state_ctr_base, + sizeof(*rctx->sa_state_ctr), + DMA_TO_DEVICE); +free_sa_state_ctr: + kfree(rctx->sa_state_ctr); + if (rctx->sa_state) + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*rctx->sa_state), + DMA_TO_DEVICE); +free_sa_state: + kfree(rctx->sa_state); + + return err; +} + +void eip93_unmap_dma(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + struct scatterlist *reqsrc, struct scatterlist *reqdst) +{ + u32 len = rctx->assoclen + rctx->textsize; + u32 authsize = rctx->authsize; + u32 flags = rctx->flags; + u32 *otag; + int i; + + if (rctx->sg_src == rctx->sg_dst) { + dma_unmap_sg(eip93->dev, rctx->sg_dst, rctx->dst_nents, + DMA_BIDIRECTIONAL); + goto process_tag; + } + + dma_unmap_sg(eip93->dev, rctx->sg_src, rctx->src_nents, + DMA_TO_DEVICE); + + if (rctx->sg_src != reqsrc) + eip93_free_sg_copy(len + rctx->authsize, &rctx->sg_src); + + dma_unmap_sg(eip93->dev, rctx->sg_dst, rctx->dst_nents, + DMA_BIDIRECTIONAL); + + /* SHA tags need conversion from net-to-host */ +process_tag: + if (IS_DECRYPT(flags)) + authsize = 0; + + if (authsize) { + if (!IS_HASH_MD5(flags)) { + otag = sg_virt(rctx->sg_dst) + len; + for (i = 0; i < (authsize / 4); i++) + otag[i] = be32_to_cpu((__be32 __force)otag[i]); + } + } + + if (rctx->sg_dst != reqdst) { + sg_copy_from_buffer(reqdst, sg_nents(reqdst), + sg_virt(rctx->sg_dst), len + authsize); + eip93_free_sg_copy(len + rctx->authsize, &rctx->sg_dst); + } +} + +void eip93_handle_result(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + u8 *reqiv) +{ + if (rctx->sa_state_ctr) + dma_unmap_single(eip93->dev, rctx->sa_state_ctr_base, + sizeof(*rctx->sa_state_ctr), + DMA_FROM_DEVICE); + + if (rctx->sa_state) + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*rctx->sa_state), + DMA_FROM_DEVICE); + + if (!IS_ECB(rctx->flags)) + memcpy(reqiv, rctx->sa_state->state_iv, rctx->ivsize); + + kfree(rctx->sa_state_ctr); + kfree(rctx->sa_state); +} + +int eip93_hmac_setkey(u32 ctx_flags, const u8 *key, unsigned int keylen, + unsigned int hashlen, u8 *dest_ipad, u8 *dest_opad, + bool skip_ipad) +{ + u8 ipad[SHA256_BLOCK_SIZE], opad[SHA256_BLOCK_SIZE]; + struct crypto_ahash *ahash_tfm; + struct eip93_hash_reqctx *rctx; + struct ahash_request *req; + DECLARE_CRYPTO_WAIT(wait); + struct scatterlist sg[1]; + const char *alg_name; + int i, ret; + + switch (ctx_flags & EIP93_HASH_MASK) { + case EIP93_HASH_SHA256: + alg_name = "sha256-eip93"; + break; + case EIP93_HASH_SHA224: + alg_name = "sha224-eip93"; + break; + case EIP93_HASH_SHA1: + alg_name = "sha1-eip93"; + break; + case EIP93_HASH_MD5: + alg_name = "md5-eip93"; + break; + default: /* Impossible */ + return -EINVAL; + } + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_ATOMIC); + if (!req) { + ret = -ENOMEM; + goto err_ahash; + } + + rctx = ahash_request_ctx_dma(req); + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + /* Hash the key if > SHA256_BLOCK_SIZE */ + if (keylen > SHA256_BLOCK_SIZE) { + sg_init_one(&sg[0], key, keylen); + + ahash_request_set_crypt(req, sg, ipad, keylen); + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (ret) + goto err_req; + + keylen = hashlen; + } else { + memcpy(ipad, key, keylen); + } + + /* Copy to opad */ + memset(ipad + keylen, 0, SHA256_BLOCK_SIZE - keylen); + memcpy(opad, ipad, SHA256_BLOCK_SIZE); + + /* Pad with HMAC constants */ + for (i = 0; i < SHA256_BLOCK_SIZE; i++) { + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; + } + + if (skip_ipad) { + memcpy(dest_ipad, ipad, SHA256_BLOCK_SIZE); + } else { + /* Hash ipad */ + sg_init_one(&sg[0], ipad, SHA256_BLOCK_SIZE); + ahash_request_set_crypt(req, sg, dest_ipad, SHA256_BLOCK_SIZE); + ret = crypto_ahash_init(req); + if (ret) + goto err_req; + + /* Disable HASH_FINALIZE for ipad hash */ + rctx->partial_hash = true; + + ret = crypto_wait_req(crypto_ahash_finup(req), &wait); + if (ret) + goto err_req; + } + + /* Hash opad */ + sg_init_one(&sg[0], opad, SHA256_BLOCK_SIZE); + ahash_request_set_crypt(req, sg, dest_opad, SHA256_BLOCK_SIZE); + ret = crypto_ahash_init(req); + if (ret) + goto err_req; + + /* Disable HASH_FINALIZE for opad hash */ + rctx->partial_hash = true; + + ret = crypto_wait_req(crypto_ahash_finup(req), &wait); + if (ret) + goto err_req; + + if (!IS_HASH_MD5(ctx_flags)) { + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) { + u32 *ipad_hash = (u32 *)dest_ipad; + u32 *opad_hash = (u32 *)dest_opad; + + if (!skip_ipad) + ipad_hash[i] = (u32 __force)cpu_to_be32(ipad_hash[i]); + opad_hash[i] = (u32 __force)cpu_to_be32(opad_hash[i]); + } + } + +err_req: + ahash_request_free(req); +err_ahash: + crypto_free_ahash(ahash_tfm); + + return ret; +} diff --git a/drivers/crypto/inside-secure/eip93/eip93-common.h b/drivers/crypto/inside-secure/eip93/eip93-common.h new file mode 100644 index 000000000000..80964cfa34df --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-common.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ + +#ifndef _EIP93_COMMON_H_ +#define _EIP93_COMMON_H_ + +void *eip93_get_descriptor(struct eip93_device *eip93); +int eip93_put_descriptor(struct eip93_device *eip93, struct eip93_descriptor *desc); + +void eip93_set_sa_record(struct sa_record *sa_record, const unsigned int keylen, + const u32 flags); + +int eip93_parse_ctrl_stat_err(struct eip93_device *eip93, int err); + +int eip93_hmac_setkey(u32 ctx_flags, const u8 *key, unsigned int keylen, + unsigned int hashlen, u8 *ipad, u8 *opad, + bool skip_ipad); + +#endif /* _EIP93_COMMON_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-des.h b/drivers/crypto/inside-secure/eip93/eip93-des.h new file mode 100644 index 000000000000..74748df04acf --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-des.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ +#ifndef _EIP93_DES_H_ +#define _EIP93_DES_H_ + +extern struct eip93_alg_template eip93_alg_ecb_des; +extern struct eip93_alg_template eip93_alg_cbc_des; +extern struct eip93_alg_template eip93_alg_ecb_des3_ede; +extern struct eip93_alg_template eip93_alg_cbc_des3_ede; + +#endif /* _EIP93_DES_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-hash.c b/drivers/crypto/inside-secure/eip93/eip93-hash.c new file mode 100644 index 000000000000..ac13d90a2b7c --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-hash.c @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 + * + * Christian Marangi <ansuelsmth@gmail.com + */ + +#include <crypto/sha1.h> +#include <crypto/sha2.h> +#include <crypto/md5.h> +#include <crypto/hmac.h> +#include <linux/dma-mapping.h> +#include <linux/delay.h> + +#include "eip93-cipher.h" +#include "eip93-hash.h" +#include "eip93-main.h" +#include "eip93-common.h" +#include "eip93-regs.h" + +static void eip93_hash_free_data_blocks(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct eip93_device *eip93 = ctx->eip93; + struct mkt_hash_block *block, *tmp; + + list_for_each_entry_safe(block, tmp, &rctx->blocks, list) { + dma_unmap_single(eip93->dev, block->data_dma, + SHA256_BLOCK_SIZE, DMA_TO_DEVICE); + kfree(block); + } + if (!list_empty(&rctx->blocks)) + INIT_LIST_HEAD(&rctx->blocks); + + if (rctx->finalize) + dma_unmap_single(eip93->dev, rctx->data_dma, + rctx->data_used, + DMA_TO_DEVICE); +} + +static void eip93_hash_free_sa_record(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct eip93_device *eip93 = ctx->eip93; + + if (IS_HMAC(ctx->flags)) + dma_unmap_single(eip93->dev, rctx->sa_record_hmac_base, + sizeof(rctx->sa_record_hmac), DMA_TO_DEVICE); + + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(rctx->sa_record), DMA_TO_DEVICE); +} + +void eip93_hash_handle_result(struct crypto_async_request *async, int err) +{ + struct ahash_request *req = ahash_request_cast(async); + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int i; + + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_FROM_DEVICE); + + /* + * With partial_hash assume SHA256_DIGEST_SIZE buffer is passed. + * This is to handle SHA224 that have a 32 byte intermediate digest. + */ + if (rctx->partial_hash) + digestsize = SHA256_DIGEST_SIZE; + + if (rctx->finalize || rctx->partial_hash) { + /* bytes needs to be swapped for req->result */ + if (!IS_HASH_MD5(ctx->flags)) { + for (i = 0; i < digestsize / sizeof(u32); i++) { + u32 *digest = (u32 *)sa_state->state_i_digest; + + digest[i] = be32_to_cpu((__be32 __force)digest[i]); + } + } + + memcpy(req->result, sa_state->state_i_digest, digestsize); + } + + eip93_hash_free_sa_record(req); + eip93_hash_free_data_blocks(req); + + ahash_request_complete(req, err); +} + +static void eip93_hash_init_sa_state_digest(u32 hash, u8 *digest) +{ + static const u32 sha256_init[] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 + }; + static const u32 sha224_init[] = { + SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, + SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7 + }; + static const u32 sha1_init[] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 + }; + static const u32 md5_init[] = { + MD5_H0, MD5_H1, MD5_H2, MD5_H3 + }; + + /* Init HASH constant */ + switch (hash) { + case EIP93_HASH_SHA256: + memcpy(digest, sha256_init, sizeof(sha256_init)); + return; + case EIP93_HASH_SHA224: + memcpy(digest, sha224_init, sizeof(sha224_init)); + return; + case EIP93_HASH_SHA1: + memcpy(digest, sha1_init, sizeof(sha1_init)); + return; + case EIP93_HASH_MD5: + memcpy(digest, md5_init, sizeof(md5_init)); + return; + default: /* Impossible */ + return; + } +} + +static void eip93_hash_export_sa_state(struct ahash_request *req, + struct eip93_hash_export_state *state) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct sa_state *sa_state = &rctx->sa_state; + + /* + * EIP93 have special handling for state_byte_cnt in sa_state. + * Even if a zero packet is passed (and a BADMSG is returned), + * state_byte_cnt is incremented to the digest handled (with the hash + * primitive). This is problematic with export/import as EIP93 + * expect 0 state_byte_cnt for the very first iteration. + */ + if (!rctx->len) + memset(state->state_len, 0, sizeof(u32) * 2); + else + memcpy(state->state_len, sa_state->state_byte_cnt, + sizeof(u32) * 2); + memcpy(state->state_hash, sa_state->state_i_digest, + SHA256_DIGEST_SIZE); + state->len = rctx->len; + state->data_used = rctx->data_used; +} + +static void __eip93_hash_init(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_record *sa_record = &rctx->sa_record; + int digestsize; + + digestsize = crypto_ahash_digestsize(ahash); + + eip93_set_sa_record(sa_record, 0, ctx->flags); + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_FROM_STATE; + sa_record->sa_cmd0_word |= EIP93_SA_CMD_SAVE_HASH; + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_OPCODE; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_OPCODE, + EIP93_SA_CMD_OPCODE_BASIC_OUT_HASH); + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_DIGEST_LENGTH; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, + digestsize / sizeof(u32)); + + /* + * HMAC special handling + * Enabling CMD_HMAC force the inner hash to be always finalized. + * This cause problems on handling message > 64 byte as we + * need to produce intermediate inner hash on sending intermediate + * 64 bytes blocks. + * + * To handle this, enable CMD_HMAC only on the last block. + * We make a duplicate of sa_record and on the last descriptor, + * we pass a dedicated sa_record with CMD_HMAC enabled to make + * EIP93 apply the outer hash. + */ + if (IS_HMAC(ctx->flags)) { + struct sa_record *sa_record_hmac = &rctx->sa_record_hmac; + + memcpy(sa_record_hmac, sa_record, sizeof(*sa_record)); + /* Copy pre-hashed opad for HMAC */ + memcpy(sa_record_hmac->sa_o_digest, ctx->opad, SHA256_DIGEST_SIZE); + + /* Disable HMAC for hash normal sa_record */ + sa_record->sa_cmd1_word &= ~EIP93_SA_CMD_HMAC; + } + + rctx->len = 0; + rctx->data_used = 0; + rctx->partial_hash = false; + rctx->finalize = false; + INIT_LIST_HEAD(&rctx->blocks); +} + +static int eip93_send_hash_req(struct crypto_async_request *async, u8 *data, + dma_addr_t *data_dma, u32 len, bool last) +{ + struct ahash_request *req = ahash_request_cast(async); + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct eip93_device *eip93 = ctx->eip93; + struct eip93_descriptor cdesc = { }; + dma_addr_t src_addr; + int ret; + + /* Map block data to DMA */ + src_addr = dma_map_single(eip93->dev, data, len, DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, src_addr); + if (ret) + return ret; + + cdesc.pe_ctrl_stat_word = FIELD_PREP(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, + EIP93_PE_CTRL_HOST_READY); + cdesc.sa_addr = rctx->sa_record_base; + cdesc.arc4_addr = 0; + + cdesc.state_addr = rctx->sa_state_base; + cdesc.src_addr = src_addr; + cdesc.pe_length_word = FIELD_PREP(EIP93_PE_LENGTH_HOST_PE_READY, + EIP93_PE_LENGTH_HOST_READY); + cdesc.pe_length_word |= FIELD_PREP(EIP93_PE_LENGTH_LENGTH, + len); + + cdesc.user_id |= FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, EIP93_DESC_HASH); + + if (last) { + int crypto_async_idr; + + if (rctx->finalize && !rctx->partial_hash) { + /* For last block, pass sa_record with CMD_HMAC enabled */ + if (IS_HMAC(ctx->flags)) { + struct sa_record *sa_record_hmac = &rctx->sa_record_hmac; + + rctx->sa_record_hmac_base = dma_map_single(eip93->dev, + sa_record_hmac, + sizeof(*sa_record_hmac), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_hmac_base); + if (ret) + return ret; + + cdesc.sa_addr = rctx->sa_record_hmac_base; + } + + cdesc.pe_ctrl_stat_word |= EIP93_PE_CTRL_PE_HASH_FINAL; + } + + scoped_guard(spinlock_bh, &eip93->ring->idr_lock) + crypto_async_idr = idr_alloc(&eip93->ring->crypto_async_idr, async, 0, + EIP93_RING_NUM - 1, GFP_ATOMIC); + + cdesc.user_id |= FIELD_PREP(EIP93_PE_USER_ID_CRYPTO_IDR, (u16)crypto_async_idr) | + FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, EIP93_DESC_LAST); + } + +again: + scoped_guard(spinlock_irqsave, &eip93->ring->write_lock) + ret = eip93_put_descriptor(eip93, &cdesc); + if (ret) { + usleep_range(EIP93_RING_BUSY_DELAY, + EIP93_RING_BUSY_DELAY * 2); + goto again; + } + + /* Writing new descriptor count starts DMA action */ + writel(1, eip93->base + EIP93_REG_PE_CD_COUNT); + + *data_dma = src_addr; + return 0; +} + +static int eip93_hash_init(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_state *sa_state = &rctx->sa_state; + + memset(sa_state->state_byte_cnt, 0, sizeof(u32) * 2); + eip93_hash_init_sa_state_digest(ctx->flags & EIP93_HASH_MASK, + sa_state->state_i_digest); + + __eip93_hash_init(req); + + /* For HMAC setup the initial block for ipad */ + if (IS_HMAC(ctx->flags)) { + memcpy(rctx->data, ctx->ipad, SHA256_BLOCK_SIZE); + + rctx->data_used = SHA256_BLOCK_SIZE; + rctx->len += SHA256_BLOCK_SIZE; + } + + return 0; +} + +/* + * With complete_req true, we wait for the engine to consume all the block in list, + * else we just queue the block to the engine as final() will wait. This is useful + * for finup(). + */ +static int __eip93_hash_update(struct ahash_request *req, bool complete_req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_async_request *async = &req->base; + unsigned int read, to_consume = req->nbytes; + unsigned int max_read, consumed = 0; + struct mkt_hash_block *block; + bool wait_req = false; + int offset; + int ret; + + /* Get the offset and available space to fill req data */ + offset = rctx->data_used; + max_read = SHA256_BLOCK_SIZE - offset; + + /* Consume req in block of SHA256_BLOCK_SIZE. + * to_read is initially set to space available in the req data + * and then reset to SHA256_BLOCK_SIZE. + */ + while (to_consume > max_read) { + block = kzalloc(sizeof(*block), GFP_ATOMIC); + if (!block) { + ret = -ENOMEM; + goto free_blocks; + } + + read = sg_pcopy_to_buffer(req->src, sg_nents(req->src), + block->data + offset, + max_read, consumed); + + /* + * For first iteration only, copy req data to block + * and reset offset and max_read for next iteration. + */ + if (offset > 0) { + memcpy(block->data, rctx->data, offset); + offset = 0; + max_read = SHA256_BLOCK_SIZE; + } + + list_add(&block->list, &rctx->blocks); + to_consume -= read; + consumed += read; + } + + /* Write the remaining data to req data */ + read = sg_pcopy_to_buffer(req->src, sg_nents(req->src), + rctx->data + offset, to_consume, + consumed); + rctx->data_used = offset + read; + + /* Update counter with processed bytes */ + rctx->len += read + consumed; + + /* Consume all the block added to list */ + list_for_each_entry_reverse(block, &rctx->blocks, list) { + wait_req = complete_req && + list_is_first(&block->list, &rctx->blocks); + + ret = eip93_send_hash_req(async, block->data, + &block->data_dma, + SHA256_BLOCK_SIZE, wait_req); + if (ret) + goto free_blocks; + } + + return wait_req ? -EINPROGRESS : 0; + +free_blocks: + eip93_hash_free_data_blocks(req); + + return ret; +} + +static int eip93_hash_update(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_record *sa_record = &rctx->sa_record; + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int ret; + + if (!req->nbytes) + return 0; + + rctx->sa_state_base = dma_map_single(eip93->dev, sa_state, + sizeof(*sa_state), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (ret) + return ret; + + rctx->sa_record_base = dma_map_single(eip93->dev, sa_record, + sizeof(*sa_record), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_base); + if (ret) + goto free_sa_state; + + ret = __eip93_hash_update(req, true); + if (ret && ret != -EINPROGRESS) + goto free_sa_record; + + return ret; + +free_sa_record: + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(*sa_record), DMA_TO_DEVICE); + +free_sa_state: + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_TO_DEVICE); + + return ret; +} + +/* + * With map_data true, we map the sa_record and sa_state. This is needed + * for finup() as the they are mapped before calling update() + */ +static int __eip93_hash_final(struct ahash_request *req, bool map_dma) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct crypto_async_request *async = &req->base; + struct sa_record *sa_record = &rctx->sa_record; + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int ret; + + /* EIP93 can't handle zero bytes hash */ + if (!rctx->len && !IS_HMAC(ctx->flags)) { + switch ((ctx->flags & EIP93_HASH_MASK)) { + case EIP93_HASH_SHA256: + memcpy(req->result, sha256_zero_message_hash, + SHA256_DIGEST_SIZE); + break; + case EIP93_HASH_SHA224: + memcpy(req->result, sha224_zero_message_hash, + SHA224_DIGEST_SIZE); + break; + case EIP93_HASH_SHA1: + memcpy(req->result, sha1_zero_message_hash, + SHA1_DIGEST_SIZE); + break; + case EIP93_HASH_MD5: + memcpy(req->result, md5_zero_message_hash, + MD5_DIGEST_SIZE); + break; + default: /* Impossible */ + return -EINVAL; + } + + return 0; + } + + /* Signal interrupt from engine is for last block */ + rctx->finalize = true; + + if (map_dma) { + rctx->sa_state_base = dma_map_single(eip93->dev, sa_state, + sizeof(*sa_state), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (ret) + return ret; + + rctx->sa_record_base = dma_map_single(eip93->dev, sa_record, + sizeof(*sa_record), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_base); + if (ret) + goto free_sa_state; + } + + /* Send last block */ + ret = eip93_send_hash_req(async, rctx->data, &rctx->data_dma, + rctx->data_used, true); + if (ret) + goto free_blocks; + + return -EINPROGRESS; + +free_blocks: + eip93_hash_free_data_blocks(req); + + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(*sa_record), DMA_TO_DEVICE); + +free_sa_state: + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_TO_DEVICE); + + return ret; +} + +static int eip93_hash_final(struct ahash_request *req) +{ + return __eip93_hash_final(req, true); +} + +static int eip93_hash_finup(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_record *sa_record = &rctx->sa_record; + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int ret; + + if (rctx->len + req->nbytes || IS_HMAC(ctx->flags)) { + rctx->sa_state_base = dma_map_single(eip93->dev, sa_state, + sizeof(*sa_state), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (ret) + return ret; + + rctx->sa_record_base = dma_map_single(eip93->dev, sa_record, + sizeof(*sa_record), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_base); + if (ret) + goto free_sa_state; + + ret = __eip93_hash_update(req, false); + if (ret) + goto free_sa_record; + } + + return __eip93_hash_final(req, false); + +free_sa_record: + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(*sa_record), DMA_TO_DEVICE); +free_sa_state: + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_TO_DEVICE); + + return ret; +} + +static int eip93_hash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, + u32 keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(ahash); + struct crypto_tfm *tfm = crypto_ahash_tfm(ahash); + struct eip93_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + return eip93_hmac_setkey(ctx->flags, key, keylen, digestsize, + ctx->ipad, ctx->opad, true); +} + +static int eip93_hash_cra_init(struct crypto_tfm *tfm) +{ + struct eip93_hash_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, alg.ahash.halg.base); + + crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), + sizeof(struct eip93_hash_reqctx)); + + ctx->eip93 = tmpl->eip93; + ctx->flags = tmpl->flags; + + return 0; +} + +static int eip93_hash_digest(struct ahash_request *req) +{ + int ret; + + ret = eip93_hash_init(req); + if (ret) + return ret; + + return eip93_hash_finup(req); +} + +static int eip93_hash_import(struct ahash_request *req, const void *in) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + const struct eip93_hash_export_state *state = in; + struct sa_state *sa_state = &rctx->sa_state; + + memcpy(sa_state->state_byte_cnt, state->state_len, sizeof(u32) * 2); + memcpy(sa_state->state_i_digest, state->state_hash, SHA256_DIGEST_SIZE); + + __eip93_hash_init(req); + + rctx->len = state->len; + rctx->data_used = state->data_used; + + /* Skip copying data if we have nothing to copy */ + if (rctx->len) + memcpy(rctx->data, state->data, rctx->data_used); + + return 0; +} + +static int eip93_hash_export(struct ahash_request *req, void *out) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct eip93_hash_export_state *state = out; + + /* Save the first block in state data */ + if (rctx->len) + memcpy(state->data, rctx->data, rctx->data_used); + + eip93_hash_export_sa_state(req, state); + + return 0; +} + +struct eip93_alg_template eip93_alg_md5 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_MD5, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_sha1 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_SHA1, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_sha224 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_SHA224, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_sha256 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_SHA256, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_md5 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "hmac(md5-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_sha1 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "hmac(sha1-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_sha224 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac(sha224-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_sha256 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac(sha256-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/eip93/eip93-hash.h b/drivers/crypto/inside-secure/eip93/eip93-hash.h new file mode 100644 index 000000000000..556f22fc1dd0 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-hash.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ +#ifndef _EIP93_HASH_H_ +#define _EIP93_HASH_H_ + +#include <crypto/sha2.h> + +#include "eip93-main.h" +#include "eip93-regs.h" + +struct eip93_hash_ctx { + struct eip93_device *eip93; + u32 flags; + + u8 ipad[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 opad[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); +}; + +struct eip93_hash_reqctx { + /* Placement is important for DMA align */ + struct { + struct sa_record sa_record; + struct sa_record sa_record_hmac; + struct sa_state sa_state; + } __aligned(CRYPTO_DMA_ALIGN); + + dma_addr_t sa_record_base; + dma_addr_t sa_record_hmac_base; + dma_addr_t sa_state_base; + + /* Don't enable HASH_FINALIZE when last block is sent */ + bool partial_hash; + + /* Set to signal interrupt is for final packet */ + bool finalize; + + /* + * EIP93 requires data to be accumulated in block of 64 bytes + * for intermediate hash calculation. + */ + u64 len; + u32 data_used; + + u8 data[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); + dma_addr_t data_dma; + + struct list_head blocks; +}; + +struct mkt_hash_block { + struct list_head list; + u8 data[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); + dma_addr_t data_dma; +}; + +struct eip93_hash_export_state { + u64 len; + u32 data_used; + + u32 state_len[2]; + u8 state_hash[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); + + u8 data[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); +}; + +void eip93_hash_handle_result(struct crypto_async_request *async, int err); + +extern struct eip93_alg_template eip93_alg_md5; +extern struct eip93_alg_template eip93_alg_sha1; +extern struct eip93_alg_template eip93_alg_sha224; +extern struct eip93_alg_template eip93_alg_sha256; +extern struct eip93_alg_template eip93_alg_hmac_md5; +extern struct eip93_alg_template eip93_alg_hmac_sha1; +extern struct eip93_alg_template eip93_alg_hmac_sha224; +extern struct eip93_alg_template eip93_alg_hmac_sha256; + +#endif /* _EIP93_HASH_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c new file mode 100644 index 000000000000..0b38a567da0e --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ + +#include <linux/atomic.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> +#include <crypto/aes.h> +#include <crypto/ctr.h> + +#include "eip93-main.h" +#include "eip93-regs.h" +#include "eip93-common.h" +#include "eip93-cipher.h" +#include "eip93-aes.h" +#include "eip93-des.h" +#include "eip93-aead.h" +#include "eip93-hash.h" + +static struct eip93_alg_template *eip93_algs[] = { + &eip93_alg_ecb_des, + &eip93_alg_cbc_des, + &eip93_alg_ecb_des3_ede, + &eip93_alg_cbc_des3_ede, + &eip93_alg_ecb_aes, + &eip93_alg_cbc_aes, + &eip93_alg_ctr_aes, + &eip93_alg_rfc3686_aes, + &eip93_alg_authenc_hmac_md5_cbc_des, + &eip93_alg_authenc_hmac_sha1_cbc_des, + &eip93_alg_authenc_hmac_sha224_cbc_des, + &eip93_alg_authenc_hmac_sha256_cbc_des, + &eip93_alg_authenc_hmac_md5_cbc_des3_ede, + &eip93_alg_authenc_hmac_sha1_cbc_des3_ede, + &eip93_alg_authenc_hmac_sha224_cbc_des3_ede, + &eip93_alg_authenc_hmac_sha256_cbc_des3_ede, + &eip93_alg_authenc_hmac_md5_cbc_aes, + &eip93_alg_authenc_hmac_sha1_cbc_aes, + &eip93_alg_authenc_hmac_sha224_cbc_aes, + &eip93_alg_authenc_hmac_sha256_cbc_aes, + &eip93_alg_authenc_hmac_md5_rfc3686_aes, + &eip93_alg_authenc_hmac_sha1_rfc3686_aes, + &eip93_alg_authenc_hmac_sha224_rfc3686_aes, + &eip93_alg_authenc_hmac_sha256_rfc3686_aes, + &eip93_alg_md5, + &eip93_alg_sha1, + &eip93_alg_sha224, + &eip93_alg_sha256, + &eip93_alg_hmac_md5, + &eip93_alg_hmac_sha1, + &eip93_alg_hmac_sha224, + &eip93_alg_hmac_sha256, +}; + +inline void eip93_irq_disable(struct eip93_device *eip93, u32 mask) +{ + __raw_writel(mask, eip93->base + EIP93_REG_MASK_DISABLE); +} + +inline void eip93_irq_enable(struct eip93_device *eip93, u32 mask) +{ + __raw_writel(mask, eip93->base + EIP93_REG_MASK_ENABLE); +} + +inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) +{ + __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); +} + +static void eip93_unregister_algs(unsigned int i) +{ + unsigned int j; + + for (j = 0; j < i; j++) { + switch (eip93_algs[j]->type) { + case EIP93_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); + break; + case EIP93_ALG_TYPE_AEAD: + crypto_unregister_aead(&eip93_algs[j]->alg.aead); + break; + case EIP93_ALG_TYPE_HASH: + crypto_unregister_ahash(&eip93_algs[i]->alg.ahash); + break; + } + } +} + +static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_flags) +{ + unsigned int i; + int ret = 0; + + for (i = 0; i < ARRAY_SIZE(eip93_algs); i++) { + u32 alg_flags = eip93_algs[i]->flags; + + eip93_algs[i]->eip93 = eip93; + + if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && + !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + continue; + + if (IS_AES(alg_flags)) { + if (!(supported_algo_flags & EIP93_PE_OPTION_AES)) + continue; + + if (!IS_HMAC(alg_flags)) { + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_128; + + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_192; + + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_256; + + if (IS_RFC3686(alg_flags)) + eip93_algs[i]->alg.skcipher.max_keysize += + CTR_RFC3686_NONCE_SIZE; + } + } + + if (IS_HASH_MD5(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_MD5)) + continue; + + if (IS_HASH_SHA1(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) + continue; + + if (IS_HASH_SHA224(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) + continue; + + if (IS_HASH_SHA256(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) + continue; + + switch (eip93_algs[i]->type) { + case EIP93_ALG_TYPE_SKCIPHER: + ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); + break; + case EIP93_ALG_TYPE_AEAD: + ret = crypto_register_aead(&eip93_algs[i]->alg.aead); + break; + case EIP93_ALG_TYPE_HASH: + ret = crypto_register_ahash(&eip93_algs[i]->alg.ahash); + break; + } + if (ret) + goto fail; + } + + return 0; + +fail: + eip93_unregister_algs(i); + + return ret; +} + +static void eip93_handle_result_descriptor(struct eip93_device *eip93) +{ + struct crypto_async_request *async; + struct eip93_descriptor *rdesc; + u16 desc_flags, crypto_idr; + bool last_entry; + int handled, left, err; + u32 pe_ctrl_stat; + u32 pe_length; + +get_more: + handled = 0; + + left = readl(eip93->base + EIP93_REG_PE_RD_COUNT) & EIP93_PE_RD_COUNT; + + if (!left) { + eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); + eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); + return; + } + + last_entry = false; + + while (left) { + scoped_guard(spinlock_irqsave, &eip93->ring->read_lock) + rdesc = eip93_get_descriptor(eip93); + if (IS_ERR(rdesc)) { + dev_err(eip93->dev, "Ndesc: %d nreq: %d\n", + handled, left); + err = -EIO; + break; + } + /* make sure DMA is finished writing */ + do { + pe_ctrl_stat = READ_ONCE(rdesc->pe_ctrl_stat_word); + pe_length = READ_ONCE(rdesc->pe_length_word); + } while (FIELD_GET(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, pe_ctrl_stat) != + EIP93_PE_CTRL_PE_READY || + FIELD_GET(EIP93_PE_LENGTH_HOST_PE_READY, pe_length) != + EIP93_PE_LENGTH_PE_READY); + + err = rdesc->pe_ctrl_stat_word & (EIP93_PE_CTRL_PE_EXT_ERR_CODE | + EIP93_PE_CTRL_PE_EXT_ERR | + EIP93_PE_CTRL_PE_SEQNUM_ERR | + EIP93_PE_CTRL_PE_PAD_ERR | + EIP93_PE_CTRL_PE_AUTH_ERR); + + desc_flags = FIELD_GET(EIP93_PE_USER_ID_DESC_FLAGS, rdesc->user_id); + crypto_idr = FIELD_GET(EIP93_PE_USER_ID_CRYPTO_IDR, rdesc->user_id); + + writel(1, eip93->base + EIP93_REG_PE_RD_COUNT); + eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); + + handled++; + left--; + + if (desc_flags & EIP93_DESC_LAST) { + last_entry = true; + break; + } + } + + if (!last_entry) + goto get_more; + + /* Get crypto async ref only for last descriptor */ + scoped_guard(spinlock_bh, &eip93->ring->idr_lock) { + async = idr_find(&eip93->ring->crypto_async_idr, crypto_idr); + idr_remove(&eip93->ring->crypto_async_idr, crypto_idr); + } + + /* Parse error in ctrl stat word */ + err = eip93_parse_ctrl_stat_err(eip93, err); + + if (desc_flags & EIP93_DESC_SKCIPHER) + eip93_skcipher_handle_result(async, err); + + if (desc_flags & EIP93_DESC_AEAD) + eip93_aead_handle_result(async, err); + + if (desc_flags & EIP93_DESC_HASH) + eip93_hash_handle_result(async, err); + + goto get_more; +} + +static void eip93_done_task(unsigned long data) +{ + struct eip93_device *eip93 = (struct eip93_device *)data; + + eip93_handle_result_descriptor(eip93); +} + +static irqreturn_t eip93_irq_handler(int irq, void *data) +{ + struct eip93_device *eip93 = data; + u32 irq_status; + + irq_status = readl(eip93->base + EIP93_REG_INT_MASK_STAT); + if (FIELD_GET(EIP93_INT_RDR_THRESH, irq_status)) { + eip93_irq_disable(eip93, EIP93_INT_RDR_THRESH); + tasklet_schedule(&eip93->ring->done_task); + return IRQ_HANDLED; + } + + /* Ignore errors in AUTO mode, handled by the RDR */ + eip93_irq_clear(eip93, irq_status); + if (irq_status) + eip93_irq_disable(eip93, irq_status); + + return IRQ_NONE; +} + +static void eip93_initialize(struct eip93_device *eip93, u32 supported_algo_flags) +{ + u32 val; + + /* Reset PE and rings */ + val = EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING; + val |= EIP93_PE_TARGET_AUTO_RING_MODE; + /* For Auto more, update the CDR ring owner after processing */ + val |= EIP93_PE_CONFIG_EN_CDR_UPDATE; + writel(val, eip93->base + EIP93_REG_PE_CONFIG); + + /* Wait for PE and ring to reset */ + usleep_range(10, 20); + + /* Release PE and ring reset */ + val = readl(eip93->base + EIP93_REG_PE_CONFIG); + val &= ~(EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING); + writel(val, eip93->base + EIP93_REG_PE_CONFIG); + + /* Config Clocks */ + val = EIP93_PE_CLOCK_EN_PE_CLK; + if (supported_algo_flags & EIP93_PE_OPTION_TDES) + val |= EIP93_PE_CLOCK_EN_DES_CLK; + if (supported_algo_flags & EIP93_PE_OPTION_AES) + val |= EIP93_PE_CLOCK_EN_AES_CLK; + if (supported_algo_flags & + (EIP93_PE_OPTION_MD5 | EIP93_PE_OPTION_SHA_1 | EIP93_PE_OPTION_SHA_224 | + EIP93_PE_OPTION_SHA_256)) + val |= EIP93_PE_CLOCK_EN_HASH_CLK; + writel(val, eip93->base + EIP93_REG_PE_CLOCK_CTRL); + + /* Config DMA thresholds */ + val = FIELD_PREP(EIP93_PE_OUTBUF_THRESH, 128) | + FIELD_PREP(EIP93_PE_INBUF_THRESH, 128); + writel(val, eip93->base + EIP93_REG_PE_BUF_THRESH); + + /* Clear/ack all interrupts before disable all */ + eip93_irq_clear(eip93, EIP93_INT_ALL); + eip93_irq_disable(eip93, EIP93_INT_ALL); + + /* Setup CRD threshold to trigger interrupt */ + val = FIELD_PREP(EIPR93_PE_CDR_THRESH, EIP93_RING_NUM - EIP93_RING_BUSY); + /* + * Configure RDR interrupt to be triggered if RD counter is not 0 + * for more than 2^(N+10) system clocks. + */ + val |= FIELD_PREP(EIPR93_PE_RD_TIMEOUT, 5) | EIPR93_PE_TIMEROUT_EN; + writel(val, eip93->base + EIP93_REG_PE_RING_THRESH); +} + +static void eip93_desc_free(struct eip93_device *eip93) +{ + writel(0, eip93->base + EIP93_REG_PE_RING_CONFIG); + writel(0, eip93->base + EIP93_REG_PE_CDR_BASE); + writel(0, eip93->base + EIP93_REG_PE_RDR_BASE); +} + +static int eip93_set_ring(struct eip93_device *eip93, struct eip93_desc_ring *ring) +{ + ring->offset = sizeof(struct eip93_descriptor); + ring->base = dmam_alloc_coherent(eip93->dev, + sizeof(struct eip93_descriptor) * EIP93_RING_NUM, + &ring->base_dma, GFP_KERNEL); + if (!ring->base) + return -ENOMEM; + + ring->write = ring->base; + ring->base_end = ring->base + sizeof(struct eip93_descriptor) * (EIP93_RING_NUM - 1); + ring->read = ring->base; + + return 0; +} + +static int eip93_desc_init(struct eip93_device *eip93) +{ + struct eip93_desc_ring *cdr = &eip93->ring->cdr; + struct eip93_desc_ring *rdr = &eip93->ring->rdr; + int ret; + u32 val; + + ret = eip93_set_ring(eip93, cdr); + if (ret) + return ret; + + ret = eip93_set_ring(eip93, rdr); + if (ret) + return ret; + + writel((u32 __force)cdr->base_dma, eip93->base + EIP93_REG_PE_CDR_BASE); + writel((u32 __force)rdr->base_dma, eip93->base + EIP93_REG_PE_RDR_BASE); + + val = FIELD_PREP(EIP93_PE_RING_SIZE, EIP93_RING_NUM - 1); + writel(val, eip93->base + EIP93_REG_PE_RING_CONFIG); + + return 0; +} + +static void eip93_cleanup(struct eip93_device *eip93) +{ + tasklet_kill(&eip93->ring->done_task); + + /* Clear/ack all interrupts before disable all */ + eip93_irq_clear(eip93, EIP93_INT_ALL); + eip93_irq_disable(eip93, EIP93_INT_ALL); + + writel(0, eip93->base + EIP93_REG_PE_CLOCK_CTRL); + + eip93_desc_free(eip93); + + idr_destroy(&eip93->ring->crypto_async_idr); +} + +static int eip93_crypto_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct eip93_device *eip93; + u32 ver, algo_flags; + int ret; + + eip93 = devm_kzalloc(dev, sizeof(*eip93), GFP_KERNEL); + if (!eip93) + return -ENOMEM; + + eip93->dev = dev; + platform_set_drvdata(pdev, eip93); + + eip93->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(eip93->base)) + return PTR_ERR(eip93->base); + + eip93->irq = platform_get_irq(pdev, 0); + if (eip93->irq < 0) + return eip93->irq; + + ret = devm_request_threaded_irq(eip93->dev, eip93->irq, eip93_irq_handler, + NULL, IRQF_ONESHOT, + dev_name(eip93->dev), eip93); + + eip93->ring = devm_kcalloc(eip93->dev, 1, sizeof(*eip93->ring), GFP_KERNEL); + if (!eip93->ring) + return -ENOMEM; + + ret = eip93_desc_init(eip93); + + if (ret) + return ret; + + tasklet_init(&eip93->ring->done_task, eip93_done_task, (unsigned long)eip93); + + spin_lock_init(&eip93->ring->read_lock); + spin_lock_init(&eip93->ring->write_lock); + + spin_lock_init(&eip93->ring->idr_lock); + idr_init(&eip93->ring->crypto_async_idr); + + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); + + eip93_initialize(eip93, algo_flags); + + /* Init finished, enable RDR interrupt */ + eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); + + ret = eip93_register_algs(eip93, algo_flags); + if (ret) { + eip93_cleanup(eip93); + return ret; + } + + ver = readl(eip93->base + EIP93_REG_PE_REVISION); + /* EIP_EIP_NO:MAJOR_HW_REV:MINOR_HW_REV:HW_PATCH,PE(ALGO_FLAGS) */ + dev_info(eip93->dev, "EIP%lu:%lx:%lx:%lx,PE(0x%x:0x%x)\n", + FIELD_GET(EIP93_PE_REVISION_EIP_NO, ver), + FIELD_GET(EIP93_PE_REVISION_MAJ_HW_REV, ver), + FIELD_GET(EIP93_PE_REVISION_MIN_HW_REV, ver), + FIELD_GET(EIP93_PE_REVISION_HW_PATCH, ver), + algo_flags, + readl(eip93->base + EIP93_REG_PE_OPTION_0)); + + return 0; +} + +static void eip93_crypto_remove(struct platform_device *pdev) +{ + struct eip93_device *eip93 = platform_get_drvdata(pdev); + + eip93_unregister_algs(ARRAY_SIZE(eip93_algs)); + eip93_cleanup(eip93); +} + +static const struct of_device_id eip93_crypto_of_match[] = { + { .compatible = "inside-secure,safexcel-eip93i", }, + { .compatible = "inside-secure,safexcel-eip93ie", }, + { .compatible = "inside-secure,safexcel-eip93is", }, + { .compatible = "inside-secure,safexcel-eip93ies", }, + /* IW not supported currently, missing AES-XCB-MAC/AES-CCM */ + /* { .compatible = "inside-secure,safexcel-eip93iw", }, */ + {} +}; +MODULE_DEVICE_TABLE(of, eip93_crypto_of_match); + +static struct platform_driver eip93_crypto_driver = { + .probe = eip93_crypto_probe, + .remove = eip93_crypto_remove, + .driver = { + .name = "inside-secure-eip93", + .of_match_table = eip93_crypto_of_match, + }, +}; +module_platform_driver(eip93_crypto_driver); + +MODULE_AUTHOR("Richard van Schagen <vschagen@cs.com>"); +MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>"); +MODULE_DESCRIPTION("Mediatek EIP-93 crypto engine driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.h b/drivers/crypto/inside-secure/eip93/eip93-main.h new file mode 100644 index 000000000000..79b078f0e5da --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-main.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ +#ifndef _EIP93_MAIN_H_ +#define _EIP93_MAIN_H_ + +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <linux/bitfield.h> +#include <linux/interrupt.h> + +#define EIP93_RING_BUSY_DELAY 500 + +#define EIP93_RING_NUM 512 +#define EIP93_RING_BUSY 32 +#define EIP93_CRA_PRIORITY 1500 + +#define EIP93_RING_SA_STATE_ADDR(base, idx) ((base) + (idx)) +#define EIP93_RING_SA_STATE_DMA(dma_base, idx) ((u32 __force)(dma_base) + \ + ((idx) * sizeof(struct sa_state))) + +/* cipher algorithms */ +#define EIP93_ALG_DES BIT(0) +#define EIP93_ALG_3DES BIT(1) +#define EIP93_ALG_AES BIT(2) +#define EIP93_ALG_MASK GENMASK(2, 0) +/* hash and hmac algorithms */ +#define EIP93_HASH_MD5 BIT(3) +#define EIP93_HASH_SHA1 BIT(4) +#define EIP93_HASH_SHA224 BIT(5) +#define EIP93_HASH_SHA256 BIT(6) +#define EIP93_HASH_HMAC BIT(7) +#define EIP93_HASH_MASK GENMASK(6, 3) +/* cipher modes */ +#define EIP93_MODE_CBC BIT(8) +#define EIP93_MODE_ECB BIT(9) +#define EIP93_MODE_CTR BIT(10) +#define EIP93_MODE_RFC3686 BIT(11) +#define EIP93_MODE_MASK GENMASK(10, 8) + +/* cipher encryption/decryption operations */ +#define EIP93_ENCRYPT BIT(12) +#define EIP93_DECRYPT BIT(13) + +#define EIP93_BUSY BIT(14) + +/* descriptor flags */ +#define EIP93_DESC_DMA_IV BIT(0) +#define EIP93_DESC_IPSEC BIT(1) +#define EIP93_DESC_FINISH BIT(2) +#define EIP93_DESC_LAST BIT(3) +#define EIP93_DESC_FAKE_HMAC BIT(4) +#define EIP93_DESC_PRNG BIT(5) +#define EIP93_DESC_HASH BIT(6) +#define EIP93_DESC_AEAD BIT(7) +#define EIP93_DESC_SKCIPHER BIT(8) +#define EIP93_DESC_ASYNC BIT(9) + +#define IS_DMA_IV(desc_flags) ((desc_flags) & EIP93_DESC_DMA_IV) + +#define IS_DES(flags) ((flags) & EIP93_ALG_DES) +#define IS_3DES(flags) ((flags) & EIP93_ALG_3DES) +#define IS_AES(flags) ((flags) & EIP93_ALG_AES) + +#define IS_HASH_MD5(flags) ((flags) & EIP93_HASH_MD5) +#define IS_HASH_SHA1(flags) ((flags) & EIP93_HASH_SHA1) +#define IS_HASH_SHA224(flags) ((flags) & EIP93_HASH_SHA224) +#define IS_HASH_SHA256(flags) ((flags) & EIP93_HASH_SHA256) +#define IS_HMAC(flags) ((flags) & EIP93_HASH_HMAC) + +#define IS_CBC(mode) ((mode) & EIP93_MODE_CBC) +#define IS_ECB(mode) ((mode) & EIP93_MODE_ECB) +#define IS_CTR(mode) ((mode) & EIP93_MODE_CTR) +#define IS_RFC3686(mode) ((mode) & EIP93_MODE_RFC3686) + +#define IS_BUSY(flags) ((flags) & EIP93_BUSY) + +#define IS_ENCRYPT(dir) ((dir) & EIP93_ENCRYPT) +#define IS_DECRYPT(dir) ((dir) & EIP93_DECRYPT) + +#define IS_CIPHER(flags) ((flags) & (EIP93_ALG_DES | \ + EIP93_ALG_3DES | \ + EIP93_ALG_AES)) + +#define IS_HASH(flags) ((flags) & (EIP93_HASH_MD5 | \ + EIP93_HASH_SHA1 | \ + EIP93_HASH_SHA224 | \ + EIP93_HASH_SHA256)) + +/** + * struct eip93_device - crypto engine device structure + */ +struct eip93_device { + void __iomem *base; + struct device *dev; + struct clk *clk; + int irq; + struct eip93_ring *ring; +}; + +struct eip93_desc_ring { + void *base; + void *base_end; + dma_addr_t base_dma; + /* write and read pointers */ + void *read; + void *write; + /* descriptor element offset */ + u32 offset; +}; + +struct eip93_state_pool { + void *base; + dma_addr_t base_dma; +}; + +struct eip93_ring { + struct tasklet_struct done_task; + /* command/result rings */ + struct eip93_desc_ring cdr; + struct eip93_desc_ring rdr; + spinlock_t write_lock; + spinlock_t read_lock; + /* aync idr */ + spinlock_t idr_lock; + struct idr crypto_async_idr; +}; + +enum eip93_alg_type { + EIP93_ALG_TYPE_AEAD, + EIP93_ALG_TYPE_SKCIPHER, + EIP93_ALG_TYPE_HASH, +}; + +struct eip93_alg_template { + struct eip93_device *eip93; + enum eip93_alg_type type; + u32 flags; + union { + struct aead_alg aead; + struct skcipher_alg skcipher; + struct ahash_alg ahash; + } alg; +}; + +#endif /* _EIP93_MAIN_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-regs.h b/drivers/crypto/inside-secure/eip93/eip93-regs.h new file mode 100644 index 000000000000..0490b8d15131 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-regs.h @@ -0,0 +1,335 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen <vschagen@icloud.com> + * Christian Marangi <ansuelsmth@gmail.com + */ +#ifndef REG_EIP93_H +#define REG_EIP93_H + +#define EIP93_REG_PE_CTRL_STAT 0x0 +#define EIP93_PE_CTRL_PE_PAD_CTRL_STAT GENMASK(31, 24) +#define EIP93_PE_CTRL_PE_EXT_ERR_CODE GENMASK(23, 20) +#define EIP93_PE_CTRL_PE_EXT_ERR_PROCESSING 0x8 +#define EIP93_PE_CTRL_PE_EXT_ERR_BLOCK_SIZE_ERR 0x7 +#define EIP93_PE_CTRL_PE_EXT_ERR_INVALID_PK_LENGTH 0x6 +#define EIP93_PE_CTRL_PE_EXT_ERR_ZERO_LENGTH 0x5 +#define EIP93_PE_CTRL_PE_EXT_ERR_SPI 0x4 +#define EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_ALGO 0x3 +#define EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_OP 0x2 +#define EIP93_PE_CTRL_PE_EXT_ERR_DESC_OWNER 0x1 +#define EIP93_PE_CTRL_PE_EXT_ERR_BUS 0x0 +#define EIP93_PE_CTRL_PE_EXT_ERR BIT(19) +#define EIP93_PE_CTRL_PE_SEQNUM_ERR BIT(18) +#define EIP93_PE_CTRL_PE_PAD_ERR BIT(17) +#define EIP93_PE_CTRL_PE_AUTH_ERR BIT(16) +#define EIP93_PE_CTRL_PE_PAD_VALUE GENMASK(15, 8) +#define EIP93_PE_CTRL_PE_PRNG_MODE GENMASK(7, 6) +#define EIP93_PE_CTRL_PE_HASH_FINAL BIT(4) +#define EIP93_PE_CTRL_PE_INIT_ARC4 BIT(3) +#define EIP93_PE_CTRL_PE_READY_DES_TRING_OWN GENMASK(1, 0) +#define EIP93_PE_CTRL_PE_READY 0x2 +#define EIP93_PE_CTRL_HOST_READY 0x1 +#define EIP93_REG_PE_SOURCE_ADDR 0x4 +#define EIP93_REG_PE_DEST_ADDR 0x8 +#define EIP93_REG_PE_SA_ADDR 0xc +#define EIP93_REG_PE_ADDR 0x10 /* STATE_ADDR */ +/* + * Special implementation for user ID + * user_id in eip93_descriptor is used to identify the + * descriptor and is opaque and can be used by the driver + * in custom way. + * + * The usage of this should be to put an address to the crypto + * request struct from the kernel but this can't work in 64bit + * world. + * + * Also it's required to put some flags to identify the last + * descriptor. + * + * To handle this, split the u32 in 2 part: + * - 31:16 descriptor flags + * - 15:0 IDR to connect the crypto request address + */ +#define EIP93_REG_PE_USER_ID 0x18 +#define EIP93_PE_USER_ID_DESC_FLAGS GENMASK(31, 16) +#define EIP93_PE_USER_ID_CRYPTO_IDR GENMASK(15, 0) +#define EIP93_REG_PE_LENGTH 0x1c +#define EIP93_PE_LENGTH_BYPASS GENMASK(31, 24) +#define EIP93_PE_LENGTH_HOST_PE_READY GENMASK(23, 22) +#define EIP93_PE_LENGTH_PE_READY 0x2 +#define EIP93_PE_LENGTH_HOST_READY 0x1 +#define EIP93_PE_LENGTH_LENGTH GENMASK(19, 0) + +/* PACKET ENGINE RING configuration registers */ +#define EIP93_REG_PE_CDR_BASE 0x80 +#define EIP93_REG_PE_RDR_BASE 0x84 +#define EIP93_REG_PE_RING_CONFIG 0x88 +#define EIP93_PE_EN_EXT_TRIG BIT(31) +/* Absent in later revision of eip93 */ +/* #define EIP93_PE_RING_OFFSET GENMASK(23, 15) */ +#define EIP93_PE_RING_SIZE GENMASK(9, 0) +#define EIP93_REG_PE_RING_THRESH 0x8c +#define EIPR93_PE_TIMEROUT_EN BIT(31) +#define EIPR93_PE_RD_TIMEOUT GENMASK(29, 26) +#define EIPR93_PE_RDR_THRESH GENMASK(25, 16) +#define EIPR93_PE_CDR_THRESH GENMASK(9, 0) +#define EIP93_REG_PE_CD_COUNT 0x90 +#define EIP93_PE_CD_COUNT GENMASK(10, 0) +/* + * In the same register, writing a value in GENMASK(7, 0) will + * increment the descriptor count and start DMA action. + */ +#define EIP93_PE_CD_COUNT_INCR GENMASK(7, 0) +#define EIP93_REG_PE_RD_COUNT 0x94 +#define EIP93_PE_RD_COUNT GENMASK(10, 0) +/* + * In the same register, writing a value in GENMASK(7, 0) will + * increment the descriptor count and start DMA action. + */ +#define EIP93_PE_RD_COUNT_INCR GENMASK(7, 0) +#define EIP93_REG_PE_RING_RW_PNTR 0x98 /* RING_PNTR */ + +/* PACKET ENGINE configuration registers */ +#define EIP93_REG_PE_CONFIG 0x100 +#define EIP93_PE_CONFIG_SWAP_TARGET BIT(20) +#define EIP93_PE_CONFIG_SWAP_DATA BIT(18) +#define EIP93_PE_CONFIG_SWAP_SA BIT(17) +#define EIP93_PE_CONFIG_SWAP_CDRD BIT(16) +#define EIP93_PE_CONFIG_EN_CDR_UPDATE BIT(10) +#define EIP93_PE_CONFIG_PE_MODE GENMASK(9, 8) +#define EIP93_PE_TARGET_AUTO_RING_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x3) +#define EIP93_PE_TARGET_COMMAND_NO_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x2) +#define EIP93_PE_TARGET_COMMAND_WITH_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x1) +#define EIP93_PE_DIRECT_HOST_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x0) +#define EIP93_PE_CONFIG_RST_RING BIT(2) +#define EIP93_PE_CONFIG_RST_PE BIT(0) +#define EIP93_REG_PE_STATUS 0x104 +#define EIP93_REG_PE_BUF_THRESH 0x10c +#define EIP93_PE_OUTBUF_THRESH GENMASK(23, 16) +#define EIP93_PE_INBUF_THRESH GENMASK(7, 0) +#define EIP93_REG_PE_INBUF_COUNT 0x100 +#define EIP93_REG_PE_OUTBUF_COUNT 0x114 +#define EIP93_REG_PE_BUF_RW_PNTR 0x118 /* BUF_PNTR */ + +/* PACKET ENGINE endian config */ +#define EIP93_REG_PE_ENDIAN_CONFIG 0x1cc +#define EIP93_AIROHA_REG_PE_ENDIAN_CONFIG 0x1d0 +#define EIP93_PE_ENDIAN_TARGET_BYTE_SWAP GENMASK(23, 16) +#define EIP93_PE_ENDIAN_MASTER_BYTE_SWAP GENMASK(7, 0) +/* + * Byte goes 2 and 2 and are referenced by ID + * Split GENMASK(7, 0) in 4 part, one for each byte. + * Example LITTLE ENDIAN: Example BIG ENDIAN + * GENMASK(7, 6) 0x3 GENMASK(7, 6) 0x0 + * GENMASK(5, 4) 0x2 GENMASK(7, 6) 0x1 + * GENMASK(3, 2) 0x1 GENMASK(3, 2) 0x2 + * GENMASK(1, 0) 0x0 GENMASK(1, 0) 0x3 + */ +#define EIP93_PE_ENDIAN_BYTE0 0x0 +#define EIP93_PE_ENDIAN_BYTE1 0x1 +#define EIP93_PE_ENDIAN_BYTE2 0x2 +#define EIP93_PE_ENDIAN_BYTE3 0x3 + +/* EIP93 CLOCK control registers */ +#define EIP93_REG_PE_CLOCK_CTRL 0x1e8 +#define EIP93_PE_CLOCK_EN_HASH_CLK BIT(4) +#define EIP93_PE_CLOCK_EN_ARC4_CLK BIT(3) +#define EIP93_PE_CLOCK_EN_AES_CLK BIT(2) +#define EIP93_PE_CLOCK_EN_DES_CLK BIT(1) +#define EIP93_PE_CLOCK_EN_PE_CLK BIT(0) + +/* EIP93 Device Option and Revision Register */ +#define EIP93_REG_PE_OPTION_1 0x1f4 +#define EIP93_PE_OPTION_MAC_KEY256 BIT(31) +#define EIP93_PE_OPTION_MAC_KEY192 BIT(30) +#define EIP93_PE_OPTION_MAC_KEY128 BIT(29) +#define EIP93_PE_OPTION_AES_CBC_MAC BIT(28) +#define EIP93_PE_OPTION_AES_XCBX BIT(23) +#define EIP93_PE_OPTION_SHA_256 BIT(19) +#define EIP93_PE_OPTION_SHA_224 BIT(18) +#define EIP93_PE_OPTION_SHA_1 BIT(17) +#define EIP93_PE_OPTION_MD5 BIT(16) +#define EIP93_PE_OPTION_AES_KEY256 BIT(15) +#define EIP93_PE_OPTION_AES_KEY192 BIT(14) +#define EIP93_PE_OPTION_AES_KEY128 BIT(13) +#define EIP93_PE_OPTION_AES BIT(2) +#define EIP93_PE_OPTION_ARC4 BIT(1) +#define EIP93_PE_OPTION_TDES BIT(0) /* DES and TDES */ +#define EIP93_REG_PE_OPTION_0 0x1f8 +#define EIP93_REG_PE_REVISION 0x1fc +#define EIP93_PE_REVISION_MAJ_HW_REV GENMASK(27, 24) +#define EIP93_PE_REVISION_MIN_HW_REV GENMASK(23, 20) +#define EIP93_PE_REVISION_HW_PATCH GENMASK(19, 16) +#define EIP93_PE_REVISION_EIP_NO GENMASK(7, 0) + +/* EIP93 Interrupt Control Register */ +#define EIP93_REG_INT_UNMASK_STAT 0x200 +#define EIP93_REG_INT_MASK_STAT 0x204 +#define EIP93_REG_INT_CLR 0x204 +#define EIP93_REG_INT_MASK 0x208 /* INT_EN */ +/* Each int reg have the same bitmap */ +#define EIP93_INT_INTERFACE_ERR BIT(18) +#define EIP93_INT_RPOC_ERR BIT(17) +#define EIP93_INT_PE_RING_ERR BIT(16) +#define EIP93_INT_HALT BIT(15) +#define EIP93_INT_OUTBUF_THRESH BIT(11) +#define EIP93_INT_INBUF_THRESH BIT(10) +#define EIP93_INT_OPERATION_DONE BIT(9) +#define EIP93_INT_RDR_THRESH BIT(1) +#define EIP93_INT_CDR_THRESH BIT(0) +#define EIP93_INT_ALL (EIP93_INT_INTERFACE_ERR | \ + EIP93_INT_RPOC_ERR | \ + EIP93_INT_PE_RING_ERR | \ + EIP93_INT_HALT | \ + EIP93_INT_OUTBUF_THRESH | \ + EIP93_INT_INBUF_THRESH | \ + EIP93_INT_OPERATION_DONE | \ + EIP93_INT_RDR_THRESH | \ + EIP93_INT_CDR_THRESH) + +#define EIP93_REG_INT_CFG 0x20c +#define EIP93_INT_TYPE_PULSE BIT(0) +#define EIP93_REG_MASK_ENABLE 0x210 +#define EIP93_REG_MASK_DISABLE 0x214 + +/* EIP93 SA Record register */ +#define EIP93_REG_SA_CMD_0 0x400 +#define EIP93_SA_CMD_SAVE_HASH BIT(29) +#define EIP93_SA_CMD_SAVE_IV BIT(28) +#define EIP93_SA_CMD_HASH_SOURCE GENMASK(27, 26) +#define EIP93_SA_CMD_HASH_NO_LOAD FIELD_PREP(EIP93_SA_CMD_HASH_SOURCE, 0x3) +#define EIP93_SA_CMD_HASH_FROM_STATE FIELD_PREP(EIP93_SA_CMD_HASH_SOURCE, 0x2) +#define EIP93_SA_CMD_HASH_FROM_SA FIELD_PREP(EIP93_SA_CMD_HASH_SOURCE, 0x0) +#define EIP93_SA_CMD_IV_SOURCE GENMASK(25, 24) +#define EIP93_SA_CMD_IV_FROM_PRNG FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x3) +#define EIP93_SA_CMD_IV_FROM_STATE FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x2) +#define EIP93_SA_CMD_IV_FROM_INPUT FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x1) +#define EIP93_SA_CMD_IV_NO_LOAD FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x0) +#define EIP93_SA_CMD_DIGEST_LENGTH GENMASK(23, 20) +#define EIP93_SA_CMD_DIGEST_10WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0xa) /* SRTP and TLS */ +#define EIP93_SA_CMD_DIGEST_8WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x8) /* SHA-256 */ +#define EIP93_SA_CMD_DIGEST_7WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x7) /* SHA-224 */ +#define EIP93_SA_CMD_DIGEST_6WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x6) +#define EIP93_SA_CMD_DIGEST_5WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x5) /* SHA1 */ +#define EIP93_SA_CMD_DIGEST_4WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x4) /* MD5 and AES-based */ +#define EIP93_SA_CMD_DIGEST_3WORD_IPSEC FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x3) /* IPSEC */ +#define EIP93_SA_CMD_DIGEST_2WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x2) +#define EIP93_SA_CMD_DIGEST_1WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x1) +#define EIP93_SA_CMD_DIGEST_3WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x0) /* 96bit output */ +#define EIP93_SA_CMD_HDR_PROC BIT(19) +#define EIP93_SA_CMD_EXT_PAD BIT(18) +#define EIP93_SA_CMD_SCPAD BIT(17) +#define EIP93_SA_CMD_HASH GENMASK(15, 12) +#define EIP93_SA_CMD_HASH_NULL FIELD_PREP(EIP93_SA_CMD_HASH, 0xf) +#define EIP93_SA_CMD_HASH_SHA256 FIELD_PREP(EIP93_SA_CMD_HASH, 0x3) +#define EIP93_SA_CMD_HASH_SHA224 FIELD_PREP(EIP93_SA_CMD_HASH, 0x2) +#define EIP93_SA_CMD_HASH_SHA1 FIELD_PREP(EIP93_SA_CMD_HASH, 0x1) +#define EIP93_SA_CMD_HASH_MD5 FIELD_PREP(EIP93_SA_CMD_HASH, 0x0) +#define EIP93_SA_CMD_CIPHER GENMASK(11, 8) +#define EIP93_SA_CMD_CIPHER_NULL FIELD_PREP(EIP93_SA_CMD_CIPHER, 0xf) +#define EIP93_SA_CMD_CIPHER_AES FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x3) +#define EIP93_SA_CMD_CIPHER_ARC4 FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x2) +#define EIP93_SA_CMD_CIPHER_3DES FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x1) +#define EIP93_SA_CMD_CIPHER_DES FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x0) +#define EIP93_SA_CMD_PAD_TYPE GENMASK(7, 6) +#define EIP93_SA_CMD_PAD_CONST_SSL FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x6) +#define EIP93_SA_CMD_PAD_TLS_DTLS FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x5) +#define EIP93_SA_CMD_PAD_ZERO FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x3) +#define EIP93_SA_CMD_PAD_CONST FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x2) +#define EIP93_SA_CMD_PAD_PKCS7 FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x1) +#define EIP93_SA_CMD_PAD_IPSEC FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x0) +#define EIP93_SA_CMD_OPGROUP GENMASK(5, 4) +#define EIP93_SA_CMD_OP_EXT FIELD_PREP(EIP93_SA_CMD_OPGROUP, 0x2) +#define EIP93_SA_CMD_OP_PROTOCOL FIELD_PREP(EIP93_SA_CMD_OPGROUP, 0x1) +#define EIP93_SA_CMD_OP_BASIC FIELD_PREP(EIP93_SA_CMD_OPGROUP, 0x0) +#define EIP93_SA_CMD_DIRECTION_IN BIT(3) /* 0: outbount 1: inbound */ +#define EIP93_SA_CMD_OPCODE GENMASK(2, 0) +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_PRNG 0x7 +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_HASH 0x3 +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_ENC_HASH 0x1 +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_ENC 0x0 +#define EIP93_SA_CMD_OPCODE_BASIC_IN_HASH 0x3 +#define EIP93_SA_CMD_OPCODE_BASIC_IN_HASH_DEC 0x1 +#define EIP93_SA_CMD_OPCODE_BASIC_IN_DEC 0x0 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_ESP 0x0 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_SSL 0x4 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_TLS 0x5 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_SRTP 0x7 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_ESP 0x0 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_SSL 0x2 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_TLS 0x3 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_SRTP 0x7 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_DTSL 0x1 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_SSL 0x4 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_TLSV10 0x5 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_TLSV11 0x6 +#define EIP93_SA_CMD_OPCODE_EXT_IN_DTSL 0x1 +#define EIP93_SA_CMD_OPCODE_EXT_IN_SSL 0x4 +#define EIP93_SA_CMD_OPCODE_EXT_IN_TLSV10 0x5 +#define EIP93_SA_CMD_OPCODE_EXT_IN_TLSV11 0x6 +#define EIP93_REG_SA_CMD_1 0x404 +#define EIP93_SA_CMD_EN_SEQNUM_CHK BIT(29) +/* This mask can be either used for ARC4 or AES */ +#define EIP93_SA_CMD_ARC4_KEY_LENGHT GENMASK(28, 24) +#define EIP93_SA_CMD_AES_DEC_KEY BIT(28) /* 0: encrypt key 1: decrypt key */ +#define EIP93_SA_CMD_AES_KEY_LENGTH GENMASK(26, 24) +#define EIP93_SA_CMD_AES_KEY_256BIT FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, 0x4) +#define EIP93_SA_CMD_AES_KEY_192BIT FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, 0x3) +#define EIP93_SA_CMD_AES_KEY_128BIT FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, 0x2) +#define EIP93_SA_CMD_HASH_CRYPT_OFFSET GENMASK(23, 16) +#define EIP93_SA_CMD_BYTE_OFFSET BIT(13) /* 0: CRYPT_OFFSET in 32bit word 1: CRYPT_OFFSET in 8bit bytes */ +#define EIP93_SA_CMD_HMAC BIT(12) +#define EIP93_SA_CMD_SSL_MAC BIT(12) +/* This mask can be either used for ARC4 or AES */ +#define EIP93_SA_CMD_CHIPER_MODE GENMASK(9, 8) +/* AES or DES operations */ +#define EIP93_SA_CMD_CHIPER_MODE_ICM FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x3) +#define EIP93_SA_CMD_CHIPER_MODE_CTR FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x2) +#define EIP93_SA_CMD_CHIPER_MODE_CBC FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x1) +#define EIP93_SA_CMD_CHIPER_MODE_ECB FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x0) +/* ARC4 operations */ +#define EIP93_SA_CMD_CHIPER_MODE_STATEFULL FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x1) +#define EIP93_SA_CMD_CHIPER_MODE_STATELESS FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x0) +#define EIP93_SA_CMD_COPY_PAD BIT(3) +#define EIP93_SA_CMD_COPY_PAYLOAD BIT(2) +#define EIP93_SA_CMD_COPY_HEADER BIT(1) +#define EIP93_SA_CMD_COPY_DIGEST BIT(0) /* With this enabled, COPY_PAD is required */ + +/* State save register */ +#define EIP93_REG_STATE_IV_0 0x500 +#define EIP93_REG_STATE_IV_1 0x504 + +#define EIP93_REG_PE_ARC4STATE 0x700 + +struct sa_record { + u32 sa_cmd0_word; + u32 sa_cmd1_word; + u32 sa_key[8]; + u8 sa_i_digest[32]; + u8 sa_o_digest[32]; + u32 sa_spi; + u32 sa_seqnum[2]; + u32 sa_seqmum_mask[2]; + u32 sa_nonce; +} __packed; + +struct sa_state { + u32 state_iv[4]; + u32 state_byte_cnt[2]; + u8 state_i_digest[32]; +} __packed; + +struct eip93_descriptor { + u32 pe_ctrl_stat_word; + u32 src_addr; + u32 dst_addr; + u32 sa_addr; + u32 state_addr; + u32 arc4_addr; + u32 user_id; + u32 pe_length_word; +} __packed; + +#endif diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 45758c7aa80e..c3b2b22934b7 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1218,7 +1218,6 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_xts_aes, &safexcel_alg_gcm, &safexcel_alg_ccm, - &safexcel_alg_crc32, &safexcel_alg_cbcmac, &safexcel_alg_xcbcmac, &safexcel_alg_cmac, @@ -2031,7 +2030,7 @@ MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>"); MODULE_AUTHOR("Igal Liberman <igall@marvell.com>"); MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197"); MODULE_LICENSE("GPL v2"); -MODULE_IMPORT_NS(CRYPTO_INTERNAL); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_FIRMWARE("ifpp.bin"); MODULE_FIRMWARE("ipue.bin"); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 0c79ad78d1c0..0f27367a85fa 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -959,7 +959,6 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes; extern struct safexcel_alg_template safexcel_alg_xts_aes; extern struct safexcel_alg_template safexcel_alg_gcm; extern struct safexcel_alg_template safexcel_alg_ccm; -extern struct safexcel_alg_template safexcel_alg_crc32; extern struct safexcel_alg_template safexcel_alg_cbcmac; extern struct safexcel_alg_template safexcel_alg_xcbcmac; extern struct safexcel_alg_template safexcel_alg_cmac; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index f44c08f5f5ec..ef0ba4832928 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -249,7 +249,9 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, safexcel_complete(priv, ring); if (sreq->nents) { - dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); sreq->nents = 0; } @@ -289,14 +291,8 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, return 1; } - if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM && - ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) { - /* Undo final XOR with 0xffffffff ...*/ - *(__le32 *)areq->result = ~sreq->state[0]; - } else { - memcpy(areq->result, sreq->state, - crypto_ahash_digestsize(ahash)); - } + memcpy(areq->result, sreq->state, + crypto_ahash_digestsize(ahash)); } cache_len = safexcel_queued_len(sreq); @@ -497,7 +493,9 @@ unmap_result: DMA_FROM_DEVICE); unmap_sg: if (req->nents) { - dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); req->nents = 0; } cdesc_rollback: @@ -1881,88 +1879,6 @@ struct safexcel_alg_template safexcel_alg_hmac_md5 = { }, }; -static int safexcel_crc32_cra_init(struct crypto_tfm *tfm) -{ - struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); - int ret = safexcel_ahash_cra_init(tfm); - - /* Default 'key' is all zeroes */ - memset(&ctx->base.ipad, 0, sizeof(u32)); - return ret; -} - -static int safexcel_crc32_init(struct ahash_request *areq) -{ - struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); - struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); - - memset(req, 0, sizeof(*req)); - - /* Start from loaded key */ - req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]); - /* Set processed to non-zero to enable invalidation detection */ - req->len = sizeof(u32); - req->processed = sizeof(u32); - - ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32; - req->digest = CONTEXT_CONTROL_DIGEST_XCM; - req->state_sz = sizeof(u32); - req->digest_sz = sizeof(u32); - req->block_sz = sizeof(u32); - - return 0; -} - -static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) -{ - struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - - if (keylen != sizeof(u32)) - return -EINVAL; - - memcpy(&ctx->base.ipad, key, sizeof(u32)); - return 0; -} - -static int safexcel_crc32_digest(struct ahash_request *areq) -{ - return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq); -} - -struct safexcel_alg_template safexcel_alg_crc32 = { - .type = SAFEXCEL_ALG_TYPE_AHASH, - .algo_mask = 0, - .alg.ahash = { - .init = safexcel_crc32_init, - .update = safexcel_ahash_update, - .final = safexcel_ahash_final, - .finup = safexcel_ahash_finup, - .digest = safexcel_crc32_digest, - .setkey = safexcel_crc32_setkey, - .export = safexcel_ahash_export, - .import = safexcel_ahash_import, - .halg = { - .digestsize = sizeof(u32), - .statesize = sizeof(struct safexcel_ahash_export_state), - .base = { - .cra_name = "crc32", - .cra_driver_name = "safexcel-crc32", - .cra_priority = SAFEXCEL_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), - .cra_init = safexcel_crc32_cra_init, - .cra_exit = safexcel_ahash_cra_exit, - .cra_module = THIS_MODULE, - }, - }, - }, -}; - static int safexcel_cbcmac_init(struct ahash_request *areq) { struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); @@ -2043,7 +1959,7 @@ struct safexcel_alg_template safexcel_alg_cbcmac = { .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, + .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), .cra_init = safexcel_ahash_cra_init, .cra_exit = safexcel_ahash_cra_exit, diff --git a/drivers/crypto/intel/iaa/Makefile b/drivers/crypto/intel/iaa/Makefile index b64b208d2344..55bda7770fac 100644 --- a/drivers/crypto/intel/iaa/Makefile +++ b/drivers/crypto/intel/iaa/Makefile @@ -3,7 +3,7 @@ # Makefile for IAA crypto device drivers # -ccflags-y += -I $(srctree)/drivers/dma/idxd -DDEFAULT_SYMBOL_NAMESPACE=IDXD +ccflags-y += -I $(srctree)/drivers/dma/idxd -DDEFAULT_SYMBOL_NAMESPACE='"IDXD"' obj-$(CONFIG_CRYPTO_DEV_IAA_CRYPTO) := iaa_crypto.o diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 8fced88d3d06..d0058757b000 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -33,8 +33,6 @@ static unsigned int nr_cpus_per_node; /* Number of physical cpus sharing each iaa instance */ static unsigned int cpus_per_iaa; -static struct crypto_comp *deflate_generic_tfm; - /* Per-cpu lookup table for balanced wqs */ static struct wq_table_entry __percpu *wq_table; @@ -173,7 +171,7 @@ static int set_iaa_sync_mode(const char *name) async_mode = false; use_irq = false; } else if (sysfs_streq(name, "async")) { - async_mode = true; + async_mode = false; use_irq = false; } else if (sysfs_streq(name, "async_irq")) { async_mode = true; @@ -727,7 +725,7 @@ static int alloc_wq_table(int max_wqs) for (cpu = 0; cpu < nr_cpus; cpu++) { entry = per_cpu_ptr(wq_table, cpu); - entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL); + entry->wqs = kcalloc(max_wqs, sizeof(*entry->wqs), GFP_KERNEL); if (!entry->wqs) { free_wq_table(); return -ENOMEM; @@ -807,7 +805,7 @@ static int save_iaa_wq(struct idxd_wq *wq) if (!cpus_per_iaa) cpus_per_iaa = 1; out: - return 0; + return ret; } static void remove_iaa_wq(struct idxd_wq *wq) @@ -896,7 +894,7 @@ out: static void rebalance_wq_table(void) { const struct cpumask *node_cpus; - int node, cpu, iaa = -1; + int node_cpu, node, cpu, iaa = 0; if (nr_iaa == 0) return; @@ -907,36 +905,29 @@ static void rebalance_wq_table(void) clear_wq_table(); if (nr_iaa == 1) { - for (cpu = 0; cpu < nr_cpus; cpu++) { - if (WARN_ON(wq_table_add_wqs(0, cpu))) { - pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu); - return; - } + for_each_possible_cpu(cpu) { + if (WARN_ON(wq_table_add_wqs(0, cpu))) + goto err; } return; } for_each_node_with_cpus(node) { + cpu = 0; node_cpus = cpumask_of_node(node); - for (cpu = 0; cpu < cpumask_weight(node_cpus); cpu++) { - int node_cpu = cpumask_nth(cpu, node_cpus); - - if (WARN_ON(node_cpu >= nr_cpu_ids)) { - pr_debug("node_cpu %d doesn't exist!\n", node_cpu); - return; - } - - if ((cpu % cpus_per_iaa) == 0) - iaa++; - - if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) { - pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); - return; - } + for_each_cpu(node_cpu, node_cpus) { + iaa = cpu / cpus_per_iaa; + if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) + goto err; + cpu++; } } + + return; +err: + pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); } static inline int check_completion(struct device *dev, @@ -1001,17 +992,11 @@ out: static int deflate_generic_decompress(struct acomp_req *req) { - void *src, *dst; + ACOMP_FBREQ_ON_STACK(fbreq, req); int ret; - src = kmap_local_page(sg_page(req->src)) + req->src->offset; - dst = kmap_local_page(sg_page(req->dst)) + req->dst->offset; - - ret = crypto_comp_decompress(deflate_generic_tfm, - src, req->slen, dst, &req->dlen); - - kunmap_local(src); - kunmap_local(dst); + ret = crypto_acomp_decompress(fbreq); + req->dlen = fbreq->dlen; update_total_sw_decomp_calls(); @@ -1025,8 +1010,7 @@ static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc); + dma_addr_t dst_addr, unsigned int *dlen); static void iaa_desc_complete(struct idxd_desc *idxd_desc, enum idxd_complete_type comp_type, @@ -1092,10 +1076,10 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } if (ctx->compress && compression_ctx->verify_compress) { + u32 *compression_crc = acomp_request_ctx(ctx->req); dma_addr_t src_addr, dst_addr; - u32 compression_crc; - compression_crc = idxd_desc->iax_completion->crc; + *compression_crc = idxd_desc->iax_completion->crc; ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); if (ret) { @@ -1105,8 +1089,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, - ctx->req->slen, dst_addr, &ctx->req->dlen, - compression_crc); + ctx->req->slen, dst_addr, &ctx->req->dlen); if (ret) { dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); err = -EIO; @@ -1135,12 +1118,11 @@ out: static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 *compression_crc, - bool disable_async) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1180,7 +1162,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, desc->src2_size = sizeof(struct aecs_comp_table_record); desc->completion_addr = idxd_desc->compl_dma; - if (ctx->use_irq && !disable_async) { + if (ctx->use_irq) { desc->flags |= IDXD_OP_FLAG_RCI; idxd_desc->crypto.req = req; @@ -1193,8 +1175,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: compression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1214,7 +1195,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_comp_calls(); update_wq_comp_calls(wq); - if (ctx->async_mode && !disable_async) { + if (ctx->async_mode) { ret = -EINPROGRESS; dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); goto out; @@ -1234,7 +1215,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, *compression_crc = idxd_desc->iax_completion->crc; - if (!ctx->async_mode || disable_async) + if (!ctx->async_mode) idxd_free_desc(wq, idxd_desc); out: return ret; @@ -1288,11 +1269,11 @@ out: static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1352,10 +1333,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, goto err; } - if (compression_crc != idxd_desc->iax_completion->crc) { + if (*compression_crc != idxd_desc->iax_completion->crc) { ret = -EINVAL; dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" - " comp=0x%x, decomp=0x%x\n", compression_crc, + " comp=0x%x, decomp=0x%x\n", *compression_crc, idxd_desc->iax_completion->crc); print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, idxd_desc->iax_completion, 64, 0); @@ -1375,8 +1356,7 @@ err: static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - bool disable_async) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); @@ -1418,7 +1398,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, desc->src1_size = slen; desc->completion_addr = idxd_desc->compl_dma; - if (ctx->use_irq && !disable_async) { + if (ctx->use_irq) { desc->flags |= IDXD_OP_FLAG_RCI; idxd_desc->crypto.req = req; @@ -1431,8 +1411,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: decompression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1452,7 +1431,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_decomp_calls(); update_wq_decomp_calls(wq); - if (ctx->async_mode && !disable_async) { + if (ctx->async_mode) { ret = -EINPROGRESS; dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); goto out; @@ -1480,7 +1459,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, *dlen = req->dlen; - if (!ctx->async_mode || disable_async) + if (!ctx->async_mode) idxd_free_desc(wq, idxd_desc); /* Update stats */ @@ -1500,13 +1479,10 @@ static int iaa_comp_acompress(struct acomp_req *req) struct iaa_compression_ctx *compression_ctx; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - bool disable_async = false; int nr_sgs, cpu, ret = 0; struct iaa_wq *iaa_wq; - u32 compression_crc; struct idxd_wq *wq; struct device *dev; - int order = -1; compression_ctx = crypto_tfm_ctx(tfm); @@ -1536,21 +1512,6 @@ static int iaa_comp_acompress(struct acomp_req *req) iaa_wq = idxd_wq_get_private(wq); - if (!req->dst) { - gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - - /* incompressible data will always be < 2 * slen */ - req->dlen = 2 * req->slen; - order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); - req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); - if (!req->dst) { - ret = -ENOMEM; - order = -1; - goto out; - } - disable_async = true; - } - dev = &wq->idxd->pdev->dev; nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); @@ -1580,7 +1541,7 @@ static int iaa_comp_acompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, - &req->dlen, &compression_crc, disable_async); + &req->dlen); if (ret == -EINPROGRESS) return ret; @@ -1592,7 +1553,7 @@ static int iaa_comp_acompress(struct acomp_req *req) } ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, compression_crc); + dst_addr, &req->dlen); if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); @@ -1611,100 +1572,6 @@ err_map_dst: out: iaa_wq_put(wq); - if (order >= 0) - sgl_free_order(req->dst, order); - - return ret; -} - -static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req) -{ - gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? - GFP_KERNEL : GFP_ATOMIC; - struct crypto_tfm *tfm = req->base.tfm; - dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; - struct iaa_wq *iaa_wq; - struct device *dev; - struct idxd_wq *wq; - int order = -1; - - cpu = get_cpu(); - wq = wq_table_next_wq(cpu); - put_cpu(); - if (!wq) { - pr_debug("no wq configured for cpu=%d\n", cpu); - return -ENODEV; - } - - ret = iaa_wq_get(wq); - if (ret) { - pr_debug("no wq available for cpu=%d\n", cpu); - return -ENODEV; - } - - iaa_wq = idxd_wq_get_private(wq); - - dev = &wq->idxd->pdev->dev; - - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; - } - src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, - req->src, req->slen, sg_dma_len(req->src)); - - req->dlen = 4 * req->slen; /* start with ~avg comp rato */ -alloc_dest: - order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); - req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); - if (!req->dst) { - ret = -ENOMEM; - order = -1; - goto out; - } - - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; - } - - dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, - req->dst, req->dlen, sg_dma_len(req->dst)); - ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, true); - if (ret == -EOVERFLOW) { - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - req->dlen *= 2; - if (req->dlen > CRYPTO_ACOMP_DST_MAX) - goto err_map_dst; - goto alloc_dest; - } - - if (ret != 0) - dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); - - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); -out: - iaa_wq_put(wq); - - if (order >= 0) - sgl_free_order(req->dst, order); - return ret; } @@ -1727,9 +1594,6 @@ static int iaa_comp_adecompress(struct acomp_req *req) return -EINVAL; } - if (!req->dst) - return iaa_comp_adecompress_alloc_dest(req); - cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1775,7 +1639,7 @@ static int iaa_comp_adecompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, false); + dst_addr, &req->dlen); if (ret == -EINPROGRESS) return ret; @@ -1810,24 +1674,16 @@ static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm) return 0; } -static void dst_free(struct scatterlist *sgl) -{ - /* - * Called for req->dst = NULL cases but we free elsewhere - * using sgl_free_order(). - */ -} - static struct acomp_alg iaa_acomp_fixed_deflate = { .init = iaa_comp_init_fixed, .compress = iaa_comp_acompress, .decompress = iaa_comp_adecompress, - .dst_free = dst_free, .base = { .cra_name = "deflate", .cra_driver_name = "deflate-iaa", .cra_flags = CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct iaa_compression_ctx), + .cra_reqsize = sizeof(u32), .cra_module = THIS_MODULE, .cra_priority = IAA_ALG_PRIORITY, } @@ -2022,15 +1878,6 @@ static int __init iaa_crypto_init_module(void) } nr_cpus_per_node = nr_cpus / nr_nodes; - if (crypto_has_comp("deflate-generic", 0, 0)) - deflate_generic_tfm = crypto_alloc_comp("deflate-generic", 0, 0); - - if (IS_ERR_OR_NULL(deflate_generic_tfm)) { - pr_err("IAA could not alloc %s tfm: errcode = %ld\n", - "deflate-generic", PTR_ERR(deflate_generic_tfm)); - return -ENOMEM; - } - ret = iaa_aecs_init_fixed(); if (ret < 0) { pr_debug("IAA fixed compression mode init failed\n"); @@ -2072,7 +1919,6 @@ err_verify_attr_create: err_driver_reg: iaa_aecs_cleanup_fixed(); err_aecs_init: - crypto_free_comp(deflate_generic_tfm); goto out; } @@ -2089,12 +1935,11 @@ static void __exit iaa_crypto_cleanup_module(void) &driver_attr_verify_compress); idxd_driver_unregister(&iaa_crypto_driver); iaa_aecs_cleanup_fixed(); - crypto_free_comp(deflate_generic_tfm); pr_debug("cleaned up\n"); } -MODULE_IMPORT_NS(IDXD); +MODULE_IMPORT_NS("IDXD"); MODULE_LICENSE("GPL"); MODULE_ALIAS_IDXD_DEVICE(0); MODULE_AUTHOR("Intel Corporation"); diff --git a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c index 449c6d3ab2db..fcc0cf4df637 100644 --- a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c +++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c @@ -471,6 +471,7 @@ static int init_ixp_crypto(struct device *dev) return -ENODEV; } npe_id = npe_spec.args[0]; + of_node_put(npe_spec.np); ret = of_parse_phandle_with_fixed_args(np, "queue-rx", 1, 0, &queue_spec); @@ -479,6 +480,7 @@ static int init_ixp_crypto(struct device *dev) return -ENODEV; } recv_qid = queue_spec.args[0]; + of_node_put(queue_spec.np); ret = of_parse_phandle_with_fixed_args(np, "queue-txready", 1, 0, &queue_spec); @@ -487,6 +489,7 @@ static int init_ixp_crypto(struct device *dev) return -ENODEV; } send_qid = queue_spec.args[0]; + of_node_put(queue_spec.np); } else { /* * Hardcoded engine when using platform data, this goes away diff --git a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c index 95dc8979918d..48281d882260 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c @@ -68,6 +68,7 @@ struct ocs_hcu_ctx { * @sg_data_total: Total data in the SG list at any time. * @sg_data_offset: Offset into the data of the current individual SG node. * @sg_dma_nents: Number of sg entries mapped in dma_list. + * @nents: Number of entries in the scatterlist. */ struct ocs_hcu_rctx { struct ocs_hcu_dev *hcu_dev; @@ -91,6 +92,7 @@ struct ocs_hcu_rctx { unsigned int sg_data_total; unsigned int sg_data_offset; unsigned int sg_dma_nents; + unsigned int nents; }; /** @@ -199,7 +201,7 @@ static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, /* Unmap req->src (if mapped). */ if (rctx->sg_dma_nents) { - dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, rctx->nents, DMA_TO_DEVICE); rctx->sg_dma_nents = 0; } @@ -230,7 +232,7 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) struct device *dev = rctx->hcu_dev->dev; unsigned int remainder = 0; unsigned int total; - size_t nents; + int nents; size_t count; int rc; int i; @@ -251,6 +253,9 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) /* Determine the number of scatter gather list entries to process. */ nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); + if (nents < 0) + return nents; + /* If there are entries to process, map them. */ if (nents) { rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, @@ -260,6 +265,10 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) rc = -ENOMEM; goto cleanup; } + + /* Save the value of nents to pass to dma_unmap_sg. */ + rctx->nents = nents; + /* * The value returned by dma_map_sg() can be < nents; so update * nents accordingly. diff --git a/drivers/crypto/intel/keembay/ocs-aes.c b/drivers/crypto/intel/keembay/ocs-aes.c index be9f32fc8f42..bb6f33f6b4d3 100644 --- a/drivers/crypto/intel/keembay/ocs-aes.c +++ b/drivers/crypto/intel/keembay/ocs-aes.c @@ -7,6 +7,7 @@ #include <linux/dma-mapping.h> #include <linux/interrupt.h> +#include <linux/kernel.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/swab.h> @@ -1473,8 +1474,7 @@ int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, ll = dll_desc->vaddr; for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { ll[i].src_addr = sg_dma_address(sg) + data_offset; - ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? - (sg_dma_len(sg) - data_offset) : data_size; + ll[i].src_len = min(sg_dma_len(sg) - data_offset, data_size); data_offset = 0; data_size -= ll[i].src_len; /* Current element points to the DMA address of the next one. */ diff --git a/drivers/crypto/intel/qat/Kconfig b/drivers/crypto/intel/qat/Kconfig index 02fb8abe4e6e..4b4861460dd4 100644 --- a/drivers/crypto/intel/qat/Kconfig +++ b/drivers/crypto/intel/qat/Kconfig @@ -6,12 +6,11 @@ config CRYPTO_DEV_QAT select CRYPTO_SKCIPHER select CRYPTO_AKCIPHER select CRYPTO_DH - select CRYPTO_HMAC select CRYPTO_RSA - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 select CRYPTO_LIB_AES + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 select FW_LOADER select CRC8 @@ -70,6 +69,18 @@ config CRYPTO_DEV_QAT_420XX To compile this as a module, choose M here: the module will be called qat_420xx. +config CRYPTO_DEV_QAT_6XXX + tristate "Support for Intel(R) QuickAssist Technology QAT_6XXX" + depends on (X86 || COMPILE_TEST) + depends on PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_6xxx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_6xxx. + config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) diff --git a/drivers/crypto/intel/qat/Makefile b/drivers/crypto/intel/qat/Makefile index 235b69f4f3f7..abef14207afa 100644 --- a/drivers/crypto/intel/qat/Makefile +++ b/drivers/crypto/intel/qat/Makefile @@ -1,10 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 +subdir-ccflags-y := -I$(src)/qat_common obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_6XXX) += qat_6xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/intel/qat/qat_420xx/Makefile b/drivers/crypto/intel/qat/qat_420xx/Makefile index 45728659fbc4..f6df54d2993e 100644 --- a/drivers/crypto/intel/qat/qat_420xx/Makefile +++ b/drivers/crypto/intel/qat/qat_420xx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx.o -qat_420xx-objs := adf_drv.o adf_420xx_hw_data.o +qat_420xx-y := adf_drv.o adf_420xx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 9faef33e54bd..53fa91d577ed 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -9,15 +9,14 @@ #include <adf_common_drv.h> #include <adf_fw_config.h> #include <adf_gen4_config.h> -#include <adf_gen4_dc.h> #include <adf_gen4_hw_csr_data.h> #include <adf_gen4_hw_data.h> #include <adf_gen4_pfvf.h> #include <adf_gen4_pm.h> #include <adf_gen4_ras.h> -#include <adf_gen4_timer.h> #include <adf_gen4_tl.h> #include <adf_gen4_vf_mig.h> +#include <adf_timer.h> #include "adf_420xx_hw_data.h" #include "icp_qat_hw.h" @@ -93,12 +92,11 @@ static const struct adf_fw_config adf_fw_dcc_config[] = { static struct adf_hw_device_class adf_420xx_class = { .name = ADF_420XX_DEVICE_NAME, .type = DEV_420XX, - .instances = 0, }; static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + u32 me_disable = self->fuses[ADF_FUSECTL4]; return ~me_disable & ADF_420XX_ACCELENGINES_MASK; } @@ -106,8 +104,7 @@ static u32 get_ae_mask(struct adf_hw_device_data *self) static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) { switch (adf_get_service_enabled(accel_dev)) { - case SVC_CY: - case SVC_CY2: + case SVC_SYM_ASYM: return ARRAY_SIZE(adf_fw_cy_config); case SVC_DC: return ARRAY_SIZE(adf_fw_dc_config); @@ -118,10 +115,8 @@ static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) case SVC_ASYM: return ARRAY_SIZE(adf_fw_asym_config); case SVC_ASYM_DC: - case SVC_DC_ASYM: return ARRAY_SIZE(adf_fw_asym_dc_config); case SVC_SYM_DC: - case SVC_DC_SYM: return ARRAY_SIZE(adf_fw_sym_dc_config); default: return 0; @@ -131,8 +126,7 @@ static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev) { switch (adf_get_service_enabled(accel_dev)) { - case SVC_CY: - case SVC_CY2: + case SVC_SYM_ASYM: return adf_fw_cy_config; case SVC_DC: return adf_fw_dc_config; @@ -143,10 +137,8 @@ static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev case SVC_ASYM: return adf_fw_asym_config; case SVC_ASYM_DC: - case SVC_DC_ASYM: return adf_fw_asym_dc_config; case SVC_SYM_DC: - case SVC_DC_SYM: return adf_fw_sym_dc_config; default: return NULL; @@ -199,7 +191,6 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) ICP_ACCEL_CAPABILITIES_SM4 | ICP_ACCEL_CAPABILITIES_AES_V2 | ICP_ACCEL_CAPABILITIES_ZUC | - ICP_ACCEL_CAPABILITIES_ZUC_256 | ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT | ICP_ACCEL_CAPABILITIES_EXT_ALGCHAIN; @@ -231,17 +222,11 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) if (fusectl1 & ICP_ACCEL_GEN4_MASK_WCP_WAT_SLICE) { capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC; - capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT; } - if (fusectl1 & ICP_ACCEL_GEN4_MASK_EIA3_SLICE) { + if (fusectl1 & ICP_ACCEL_GEN4_MASK_EIA3_SLICE) capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC; - capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; - } - - if (fusectl1 & ICP_ACCEL_GEN4_MASK_ZUC_256_SLICE) - capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | ICP_ACCEL_CAPABILITIES_SM2 | @@ -266,8 +251,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) } switch (adf_get_service_enabled(accel_dev)) { - case SVC_CY: - case SVC_CY2: + case SVC_SYM_ASYM: return capabilities_sym | capabilities_asym; case SVC_DC: return capabilities_dc; @@ -284,10 +268,8 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) case SVC_ASYM: return capabilities_asym; case SVC_ASYM_DC: - case SVC_DC_ASYM: return capabilities_asym | capabilities_dc; case SVC_SYM_DC: - case SVC_DC_SYM: return capabilities_sym | capabilities_dc; default: return 0; @@ -314,11 +296,13 @@ static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) rl_data->pcie_scale_div = ADF_420XX_RL_PCIE_SCALE_FACTOR_DIV; rl_data->pcie_scale_mul = ADF_420XX_RL_PCIE_SCALE_FACTOR_MUL; rl_data->dcpr_correction = ADF_420XX_RL_DCPR_CORRECTION; - rl_data->max_tp[ADF_SVC_ASYM] = ADF_420XX_RL_MAX_TP_ASYM; - rl_data->max_tp[ADF_SVC_SYM] = ADF_420XX_RL_MAX_TP_SYM; - rl_data->max_tp[ADF_SVC_DC] = ADF_420XX_RL_MAX_TP_DC; + rl_data->max_tp[SVC_ASYM] = ADF_420XX_RL_MAX_TP_ASYM; + rl_data->max_tp[SVC_SYM] = ADF_420XX_RL_MAX_TP_SYM; + rl_data->max_tp[SVC_DC] = ADF_420XX_RL_MAX_TP_DC; rl_data->scan_interval = ADF_420XX_RL_SCANS_PER_SEC; rl_data->scale_ref = ADF_420XX_RL_SLICE_REF; + + adf_gen4_init_num_svc_aes(rl_data); } static int get_rp_group(struct adf_accel_dev *accel_dev, u32 ae_mask) @@ -420,6 +404,7 @@ static void adf_gen4_set_err_mask(struct adf_dev_err_mask *dev_err_mask) dev_err_mask->parerr_cpr_xlt_mask = ADF_420XX_PARITYERRORMASK_CPR_XLT_MASK; dev_err_mask->parerr_dcpr_ucs_mask = ADF_420XX_PARITYERRORMASK_DCPR_UCS_MASK; dev_err_mask->parerr_pke_mask = ADF_420XX_PARITYERRORMASK_PKE_MASK; + dev_err_mask->parerr_wat_wcp_mask = ADF_420XX_PARITYERRORMASK_WAT_WCP_MASK; dev_err_mask->ssmfeatren_mask = ADF_420XX_SSMFEATREN_MASK; } @@ -477,11 +462,13 @@ void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; - hw_data->start_timer = adf_gen4_timer_start; - hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; hw_data->clock_frequency = ADF_420XX_AE_FREQ; + hw_data->services_supported = adf_gen4_services_supported; + hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt; adf_gen4_set_err_mask(&hw_data->dev_err_mask); adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index f49818a13013..cfa00daeb4fb 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -14,7 +14,7 @@ #include "adf_420xx_hw_data.h" static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, ADF_420XX_PCI_DEVICE_ID), }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_420XX) }, { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -79,7 +79,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_420xx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); @@ -129,16 +129,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Find and map all the device's BARS */ bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; - ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + ret = pcim_request_all_regions(pdev, pci_name(pdev)); if (ret) { - dev_err(&pdev->dev, "Failed to map pci regions.\n"); + dev_err(&pdev->dev, "Failed to request PCI regions.\n"); goto out_err; } i = 0; for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { bar = &accel_pci_dev->pci_bars[i++]; - bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to ioremap PCI region.\n"); + ret = -ENOMEM; + goto out_err; + } } pci_set_master(pdev); @@ -181,11 +186,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_420XX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; @@ -199,4 +212,4 @@ MODULE_FIRMWARE(ADF_420XX_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); MODULE_SOFTDEP("pre: crypto-intel_qat"); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_4xxx/Makefile b/drivers/crypto/intel/qat/qat_4xxx/Makefile index 9ba202079a22..188b611445e6 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_4xxx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o -qat_4xxx-objs := adf_drv.o adf_4xxx_hw_data.o +qat_4xxx-y := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index bbd92c017c28..740f68a36ac5 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -3,21 +3,21 @@ #include <linux/iopoll.h> #include <adf_accel_devices.h> #include <adf_admin.h> +#include <adf_bank_state.h> #include <adf_cfg.h> #include <adf_cfg_services.h> #include <adf_clock.h> #include <adf_common_drv.h> #include <adf_fw_config.h> #include <adf_gen4_config.h> -#include <adf_gen4_dc.h> #include <adf_gen4_hw_csr_data.h> #include <adf_gen4_hw_data.h> #include <adf_gen4_pfvf.h> #include <adf_gen4_pm.h> #include "adf_gen4_ras.h" -#include <adf_gen4_timer.h> #include <adf_gen4_tl.h> #include <adf_gen4_vf_mig.h> +#include <adf_timer.h> #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -96,12 +96,11 @@ static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dcc_config)); static struct adf_hw_device_class adf_4xxx_class = { .name = ADF_4XXX_DEVICE_NAME, .type = DEV_4XXX, - .instances = 0, }; static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + u32 me_disable = self->fuses[ADF_FUSECTL4]; return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; } @@ -178,8 +177,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) } switch (adf_get_service_enabled(accel_dev)) { - case SVC_CY: - case SVC_CY2: + case SVC_SYM_ASYM: return capabilities_sym | capabilities_asym; case SVC_DC: return capabilities_dc; @@ -196,10 +194,8 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) case SVC_ASYM: return capabilities_asym; case SVC_ASYM_DC: - case SVC_DC_ASYM: return capabilities_asym | capabilities_dc; case SVC_SYM_DC: - case SVC_DC_SYM: return capabilities_sym | capabilities_dc; default: return 0; @@ -226,11 +222,13 @@ static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) rl_data->pcie_scale_div = ADF_4XXX_RL_PCIE_SCALE_FACTOR_DIV; rl_data->pcie_scale_mul = ADF_4XXX_RL_PCIE_SCALE_FACTOR_MUL; rl_data->dcpr_correction = ADF_4XXX_RL_DCPR_CORRECTION; - rl_data->max_tp[ADF_SVC_ASYM] = ADF_4XXX_RL_MAX_TP_ASYM; - rl_data->max_tp[ADF_SVC_SYM] = ADF_4XXX_RL_MAX_TP_SYM; - rl_data->max_tp[ADF_SVC_DC] = ADF_4XXX_RL_MAX_TP_DC; + rl_data->max_tp[SVC_ASYM] = ADF_4XXX_RL_MAX_TP_ASYM; + rl_data->max_tp[SVC_SYM] = ADF_4XXX_RL_MAX_TP_SYM; + rl_data->max_tp[SVC_DC] = ADF_4XXX_RL_MAX_TP_DC; rl_data->scan_interval = ADF_4XXX_RL_SCANS_PER_SEC; rl_data->scale_ref = ADF_4XXX_RL_SLICE_REF; + + adf_gen4_init_num_svc_aes(rl_data); } static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) @@ -241,8 +239,7 @@ static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev) { switch (adf_get_service_enabled(accel_dev)) { - case SVC_CY: - case SVC_CY2: + case SVC_SYM_ASYM: return adf_fw_cy_config; case SVC_DC: return adf_fw_dc_config; @@ -253,10 +250,8 @@ static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev case SVC_ASYM: return adf_fw_asym_config; case SVC_ASYM_DC: - case SVC_DC_ASYM: return adf_fw_asym_dc_config; case SVC_SYM_DC: - case SVC_DC_SYM: return adf_fw_sym_dc_config; default: return NULL; @@ -428,13 +423,13 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; hw_data->num_rps = ADF_GEN4_MAX_RPS; switch (dev_id) { - case ADF_402XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_402XX: hw_data->fw_name = ADF_402XX_FW; hw_data->fw_mmp_name = ADF_402XX_MMP; hw_data->uof_get_name = uof_get_name_402xx; hw_data->get_ena_thd_mask = get_ena_thd_mask; break; - case ADF_401XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_401XX: hw_data->fw_name = ADF_4XXX_FW; hw_data->fw_mmp_name = ADF_4XXX_MMP; hw_data->uof_get_name = uof_get_name_4xxx; @@ -456,16 +451,18 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->get_ring_to_svc_map = adf_gen4_get_ring_to_svc_map; hw_data->disable_iov = adf_disable_sriov; hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; - hw_data->bank_state_save = adf_gen4_bank_state_save; - hw_data->bank_state_restore = adf_gen4_bank_state_restore; + hw_data->bank_state_save = adf_bank_state_save; + hw_data->bank_state_restore = adf_bank_state_restore; hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; - hw_data->start_timer = adf_gen4_timer_start; - hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; hw_data->clock_frequency = ADF_4XXX_AE_FREQ; + hw_data->services_supported = adf_gen4_services_supported; + hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt; adf_gen4_set_err_mask(&hw_data->dev_err_mask); adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 659905e45950..c9be5dcddb27 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -14,9 +14,9 @@ #include "adf_4xxx_hw_data.h" static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_401XX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_402XX_PCI_DEVICE_ID), }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_4XXX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_401XX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_402XX) }, { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -81,7 +81,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); @@ -131,16 +131,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Find and map all the device's BARS */ bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; - ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + ret = pcim_request_all_regions(pdev, pci_name(pdev)); if (ret) { - dev_err(&pdev->dev, "Failed to map pci regions.\n"); + dev_err(&pdev->dev, "Failed to request PCI regions.\n"); goto out_err; } i = 0; for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { bar = &accel_pci_dev->pci_bars[i++]; - bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to ioremap PCI region.\n"); + ret = -ENOMEM; + goto out_err; + } } pci_set_master(pdev); @@ -183,11 +188,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_4XXX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; @@ -203,4 +216,4 @@ MODULE_FIRMWARE(ADF_402XX_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); MODULE_SOFTDEP("pre: crypto-intel_qat"); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_6xxx/Makefile b/drivers/crypto/intel/qat/qat_6xxx/Makefile new file mode 100644 index 000000000000..4b4de67cb0c2 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_6XXX) += qat_6xxx.o +qat_6xxx-y := adf_drv.o adf_6xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c new file mode 100644 index 000000000000..bed88d3ce8ca --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c @@ -0,0 +1,950 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include <linux/array_size.h> +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/bits.h> +#include <linux/iopoll.h> +#include <linux/pci.h> +#include <linux/types.h> + +#include <adf_accel_devices.h> +#include <adf_admin.h> +#include <adf_bank_state.h> +#include <adf_cfg.h> +#include <adf_cfg_services.h> +#include <adf_clock.h> +#include <adf_common_drv.h> +#include <adf_fw_config.h> +#include <adf_gen6_pm.h> +#include <adf_gen6_ras.h> +#include <adf_gen6_shared.h> +#include <adf_gen6_tl.h> +#include <adf_timer.h> +#include "adf_6xxx_hw_data.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw_51_comp.h" + +#define RP_GROUP_0_MASK (BIT(0) | BIT(2)) +#define RP_GROUP_1_MASK (BIT(1) | BIT(3)) +#define RP_GROUP_ALL_MASK (RP_GROUP_0_MASK | RP_GROUP_1_MASK) + +#define ADF_AE_GROUP_0 GENMASK(3, 0) +#define ADF_AE_GROUP_1 GENMASK(7, 4) +#define ADF_AE_GROUP_2 BIT(8) + +struct adf_ring_config { + u32 ring_mask; + enum adf_cfg_service_type ring_type; + const unsigned long *thrd_mask; +}; + +static u32 rmask_two_services[] = { + RP_GROUP_0_MASK, + RP_GROUP_1_MASK, +}; + +enum adf_gen6_rps { + RP0 = 0, + RP1 = 1, + RP2 = 2, + RP3 = 3, + RP_MAX = RP3 +}; + +/* + * thrd_mask_[sym|asym|cpr|dcc]: these static arrays define the thread + * configuration for handling requests of specific services across the + * accelerator engines. Each element in an array corresponds to an + * accelerator engine, with the value being a bitmask that specifies which + * threads within that engine are capable of processing the particular service. + * + * For example, a value of 0x0C means that threads 2 and 3 are enabled for the + * service in the respective accelerator engine. + */ +static const unsigned long thrd_mask_sym[ADF_6XXX_MAX_ACCELENGINES] = { + 0x0C, 0x0C, 0x0C, 0x0C, 0x1C, 0x1C, 0x1C, 0x1C, 0x00 +}; + +static const unsigned long thrd_mask_asym[ADF_6XXX_MAX_ACCELENGINES] = { + 0x70, 0x70, 0x70, 0x70, 0x60, 0x60, 0x60, 0x60, 0x00 +}; + +static const unsigned long thrd_mask_cpr[ADF_6XXX_MAX_ACCELENGINES] = { + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00 +}; + +static const unsigned long thrd_mask_dcc[ADF_6XXX_MAX_ACCELENGINES] = { + 0x00, 0x00, 0x00, 0x00, 0x07, 0x07, 0x03, 0x03, 0x00 +}; + +static const unsigned long thrd_mask_dcpr[ADF_6XXX_MAX_ACCELENGINES] = { + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x00 +}; + +static const char *const adf_6xxx_fw_objs[] = { + [ADF_FW_CY_OBJ] = ADF_6XXX_CY_OBJ, + [ADF_FW_DC_OBJ] = ADF_6XXX_DC_OBJ, + [ADF_FW_ADMIN_OBJ] = ADF_6XXX_ADMIN_OBJ, +}; + +static const struct adf_fw_config adf_default_fw_config[] = { + { ADF_AE_GROUP_1, ADF_FW_DC_OBJ }, + { ADF_AE_GROUP_0, ADF_FW_CY_OBJ }, + { ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ }, +}; + +static struct adf_hw_device_class adf_6xxx_class = { + .name = ADF_6XXX_DEVICE_NAME, + .type = DEV_6XXX, +}; + +static bool services_supported(unsigned long mask) +{ + int num_svc; + + if (mask >= BIT(SVC_COUNT)) + return false; + + num_svc = hweight_long(mask); + switch (num_svc) { + case ADF_ONE_SERVICE: + return true; + case ADF_TWO_SERVICES: + case ADF_THREE_SERVICES: + return !test_bit(SVC_DCC, &mask); + default: + return false; + } +} + +static int get_service(unsigned long *mask) +{ + if (test_and_clear_bit(SVC_ASYM, mask)) + return SVC_ASYM; + + if (test_and_clear_bit(SVC_SYM, mask)) + return SVC_SYM; + + if (test_and_clear_bit(SVC_DC, mask)) + return SVC_DC; + + if (test_and_clear_bit(SVC_DCC, mask)) + return SVC_DCC; + + if (test_and_clear_bit(SVC_DECOMP, mask)) + return SVC_DECOMP; + + return -EINVAL; +} + +static enum adf_cfg_service_type get_ring_type(unsigned int service) +{ + switch (service) { + case SVC_SYM: + return SYM; + case SVC_ASYM: + return ASYM; + case SVC_DC: + case SVC_DCC: + return COMP; + case SVC_DECOMP: + return DECOMP; + default: + return UNUSED; + } +} + +static const unsigned long *get_thrd_mask(unsigned int service) +{ + switch (service) { + case SVC_SYM: + return thrd_mask_sym; + case SVC_ASYM: + return thrd_mask_asym; + case SVC_DC: + return thrd_mask_cpr; + case SVC_DCC: + return thrd_mask_dcc; + case SVC_DECOMP: + return thrd_mask_dcpr; + default: + return NULL; + } +} + +static int get_rp_config(struct adf_accel_dev *accel_dev, struct adf_ring_config *rp_config, + unsigned int *num_services) +{ + unsigned int i, nservices; + unsigned long mask; + int ret, service; + + ret = adf_get_service_mask(accel_dev, &mask); + if (ret) + return ret; + + nservices = hweight_long(mask); + if (nservices > MAX_NUM_CONCURR_SVC) + return -EINVAL; + + for (i = 0; i < nservices; i++) { + service = get_service(&mask); + if (service < 0) + return service; + + rp_config[i].ring_type = get_ring_type(service); + rp_config[i].thrd_mask = get_thrd_mask(service); + + /* + * If there is only one service enabled, use all ring pairs for + * that service. + * If there are two services enabled, use ring pairs 0 and 2 for + * one service and ring pairs 1 and 3 for the other service. + */ + switch (nservices) { + case ADF_ONE_SERVICE: + rp_config[i].ring_mask = RP_GROUP_ALL_MASK; + break; + case ADF_TWO_SERVICES: + rp_config[i].ring_mask = rmask_two_services[i]; + break; + case ADF_THREE_SERVICES: + rp_config[i].ring_mask = BIT(i); + + /* If ASYM is enabled, use additional ring pair */ + if (service == SVC_ASYM) + rp_config[i].ring_mask |= BIT(RP3); + + break; + default: + return -EINVAL; + } + } + + *num_services = nservices; + + return 0; +} + +static u32 adf_gen6_get_arb_mask(struct adf_accel_dev *accel_dev, unsigned int ae) +{ + struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC]; + unsigned int num_services, i, thrd; + u32 ring_mask, thd2arb_mask = 0; + const unsigned long *p_mask; + + if (get_rp_config(accel_dev, rp_config, &num_services)) + return 0; + + /* + * The thd2arb_mask maps ring pairs to threads within an accelerator engine. + * It ensures that jobs submitted to ring pairs are scheduled on threads capable + * of handling the specified service type. + * + * Each group of 4 bits in the mask corresponds to a thread, with each bit + * indicating whether a job from a ring pair can be scheduled on that thread. + * The use of 4 bits is due to the organization of ring pairs into groups of + * four, where each group shares the same configuration. + */ + for (i = 0; i < num_services; i++) { + p_mask = &rp_config[i].thrd_mask[ae]; + ring_mask = rp_config[i].ring_mask; + + for_each_set_bit(thrd, p_mask, ADF_NUM_THREADS_PER_AE) + thd2arb_mask |= ring_mask << (thrd * 4); + } + + return thd2arb_mask; +} + +static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev) +{ + enum adf_cfg_service_type rps[ADF_GEN6_NUM_BANKS_PER_VF] = { }; + struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC]; + unsigned int num_services, rp_num, i; + unsigned long cfg_mask; + u16 ring_to_svc_map; + + if (get_rp_config(accel_dev, rp_config, &num_services)) + return 0; + + /* + * Loop through the configured services and populate the `rps` array that + * contains what service that particular ring pair can handle (i.e. symmetric + * crypto, asymmetric crypto, data compression or compression chaining). + */ + for (i = 0; i < num_services; i++) { + cfg_mask = rp_config[i].ring_mask; + for_each_set_bit(rp_num, &cfg_mask, ADF_GEN6_NUM_BANKS_PER_VF) + rps[rp_num] = rp_config[i].ring_type; + } + + /* + * The ring_mask is structured into segments of 3 bits, with each + * segment representing the service configuration for a specific ring pair. + * Since ring pairs are organized into groups of 4, the ring_mask contains 4 + * such 3-bit segments, each corresponding to one ring pair. + * + * The device has 64 ring pairs, which are organized in groups of 4, namely + * 16 groups. Each group has the same configuration, represented here by + * `ring_to_svc_map`. + */ + ring_to_svc_map = rps[RP0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT | + rps[RP1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT | + rps[RP2] << ADF_CFG_SERV_RING_PAIR_2_SHIFT | + rps[RP3] << ADF_CFG_SERV_RING_PAIR_3_SHIFT; + + return ring_to_svc_map; +} + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_GEN6_ACCELERATORS_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_GEN6_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return self ? hweight32(self->ae_mask) : 0; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_SRAM_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} + +static void get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_GEN6_ARB_CONFIG; + arb_info->arb_offset = ADF_GEN6_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET; +} + +static void get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_GEN6_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_GEN6_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_GEN6_ADMINMSGLR_OFFSET; +} + +static u32 get_heartbeat_clock(struct adf_hw_device_data *self) +{ + return ADF_GEN6_COUNTER_FREQ; +} + +static void enable_error_correction(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + /* + * Enable all error notification bits in errsou3 except VFLR + * notification on host. + */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_VFLNOTIFY); +} + +static void enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_MASK_OFFSET, 0); +} + +static void set_ssm_wdtimer(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + u64 val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; + u64 val = ADF_SSM_WDT_DEFAULT_VALUE; + + /* Enable watchdog timer for sym and dc */ + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTATHL_OFFSET, ADF_SSMWDTATHH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTCNVL_OFFSET, ADF_SSMWDTCNVH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTUCSL_OFFSET, ADF_SSMWDTUCSH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTDCPRL_OFFSET, ADF_SSMWDTDCPRH_OFFSET, val); + + /* Enable watchdog timer for pke */ + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, val_pke); +} + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_GEN6_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + unsigned int i; + + for (i = 0; i <= ADF_GEN6_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_GEN6_MSIX_RTTABLE_OFFSET(i), i); +} + +static int reset_ring_pair(void __iomem *csr, u32 bank_number) +{ + u32 status; + int ret; + + /* + * Write rpresetctl register BIT(0) as 1. + * Since rpresetctl registers have no RW fields, no need to preserve + * values for other bits. Just write directly. + */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), + ADF_WQM_CSR_RPRESETCTL_RESET); + + /* Read rpresetsts register and wait for rp reset to complete */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_WQM_CSR_RPRESETSTS_STATUS, + ADF_RPRESET_POLL_DELAY_US, + ADF_RPRESET_POLL_TIMEOUT_US, true, + csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); + if (ret) + return ret; + + /* When ring pair reset is done, clear rpresetsts */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), ADF_WQM_CSR_RPRESETSTS_STATUS); + + return 0; +} + +static int ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr = adf_get_etr_base(accel_dev); + int ret; + + if (bank_number >= hw_data->num_banks) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", bank_number); + + ret = reset_ring_pair(csr, bank_number); + if (ret) + dev_err(&GET_DEV(accel_dev), "ring pair reset failed (timeout)\n"); + else + dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n"); + + return ret; +} + +static int build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_hw_comp_51_config_csr_lower hw_comp_lower_csr = { }; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC; + break; + default: + return -EINVAL; + } + + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED; + hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1; + lower_val = ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(hw_comp_lower_csr); + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + return 0; +} + +static int build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = 0; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + return 0; +} + +static void adf_gen6_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = build_comp_block; + dc_ops->build_decomp_block = build_decomp_block; +} + +static int adf_gen6_init_thd2arb_map(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 *thd2arb_map = hw_data->thd_to_arb_map; + unsigned int i; + + for (i = 0; i < hw_data->num_engines; i++) { + thd2arb_map[i] = adf_gen6_get_arb_mask(accel_dev, i); + dev_dbg(&GET_DEV(accel_dev), "ME:%d arb_mask:%#x\n", i, thd2arb_map[i]); + } + + return 0; +} + +static void init_num_svc_aes(struct adf_rl_hw_data *device_data) +{ + enum adf_fw_objs obj_type, obj_iter; + unsigned int svc, i, num_grp; + u32 ae_mask; + + for (svc = 0; svc < SVC_BASE_COUNT; svc++) { + switch (svc) { + case SVC_SYM: + case SVC_ASYM: + obj_type = ADF_FW_CY_OBJ; + break; + case SVC_DC: + case SVC_DECOMP: + obj_type = ADF_FW_DC_OBJ; + break; + } + + num_grp = ARRAY_SIZE(adf_default_fw_config); + for (i = 0; i < num_grp; i++) { + obj_iter = adf_default_fw_config[i].obj; + if (obj_iter == obj_type) { + ae_mask = adf_default_fw_config[i].ae_mask; + device_data->svc_ae_mask[svc] = hweight32(ae_mask); + break; + } + } + } +} + +static u32 adf_gen6_get_svc_slice_cnt(struct adf_accel_dev *accel_dev, + enum adf_base_services svc) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + + switch (svc) { + case SVC_SYM: + return device_data->slices.cph_cnt; + case SVC_ASYM: + return device_data->slices.pke_cnt; + case SVC_DC: + return device_data->slices.cpr_cnt + device_data->slices.dcpr_cnt; + case SVC_DECOMP: + return device_data->slices.dcpr_cnt; + default: + return 0; + } +} + +static void set_vc_csr_for_bank(void __iomem *csr, u32 bank_number) +{ + u32 value; + + /* + * After each PF FLR, for each of the 64 ring pairs in the PF, the + * driver must program the ringmodectl CSRs. + */ + value = ADF_CSR_RD(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number)); + FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_MASK, &value, ADF_GEN6_RINGMODECTL_TC_DEFAULT); + FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_EN_MASK, &value, ADF_GEN6_RINGMODECTL_TC_EN_OP1); + ADF_CSR_WR(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number), value); +} + +static int set_vc_config(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + u32 value; + int err; + + /* + * After each PF FLR, the driver must program the Port Virtual Channel (VC) + * Control Registers. + * Read PVC0CTL then write the masked values. + */ + pci_read_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, &value); + FIELD_MODIFY(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT); + err = pci_write_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, value); + if (err) { + dev_err(&GET_DEV(accel_dev), "pci write to PVC0CTL failed\n"); + return pcibios_err_to_errno(err); + } + + /* Read PVC1CTL then write masked values */ + pci_read_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, &value); + FIELD_MODIFY(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT); + FIELD_MODIFY(ADF_GEN6_PVC1CTL_VCEN_MASK, &value, ADF_GEN6_PVC1CTL_VCEN_ON); + err = pci_write_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, value); + if (err) + dev_err(&GET_DEV(accel_dev), "pci write to PVC1CTL failed\n"); + + return pcibios_err_to_errno(err); +} + +static int adf_gen6_set_vc(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + void __iomem *csr = adf_get_etr_base(accel_dev); + u32 i; + + for (i = 0; i < hw_data->num_banks; i++) { + dev_dbg(&GET_DEV(accel_dev), "set virtual channels for bank:%d\n", i); + set_vc_csr_for_bank(csr, i); + } + + return set_vc_config(accel_dev); +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_6XXX_ACCELENGINES_MASK; + + /* + * If bit 0 is set in the fuses, the first 4 engines are disabled. + * If bit 4 is set, the second group of 4 engines are disabled. + * If bit 8 is set, the admin engine (bit 8) is disabled. + */ + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + return mask; +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + u32 capabilities_sym, capabilities_asym; + u32 capabilities_dc; + unsigned long mask; + u32 caps = 0; + u32 fusectl1; + + fusectl1 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1]; + + /* Read accelerator capabilities mask */ + capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_AES_V2; + + /* A set bit in fusectl1 means the corresponding feature is OFF in this SKU */ + if (fusectl1 & ICP_ACCEL_GEN6_MASK_UCS_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + } + if (fusectl1 & ICP_ACCEL_GEN6_MASK_AUTH_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_SM2 | + ICP_ACCEL_CAPABILITIES_ECEDMONT; + + if (fusectl1 & ICP_ACCEL_GEN6_MASK_PKE_SLICE) { + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; + } + + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + + if (fusectl1 & ICP_ACCEL_GEN6_MASK_CPR_SLICE) { + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + + if (adf_get_service_mask(accel_dev, &mask)) + return 0; + + if (test_bit(SVC_ASYM, &mask)) + caps |= capabilities_asym; + if (test_bit(SVC_SYM, &mask)) + caps |= capabilities_sym; + if (test_bit(SVC_DC, &mask) || test_bit(SVC_DECOMP, &mask)) + caps |= capabilities_dc; + if (test_bit(SVC_DCC, &mask)) { + /* + * Sym capabilities are available for chaining operations, + * but sym crypto instances cannot be supported + */ + caps = capabilities_dc | capabilities_sym; + caps &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + } + + return caps; +} + +static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) +{ + return ARRAY_SIZE(adf_default_fw_config); +} + +static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + int num_fw_objs = ARRAY_SIZE(adf_6xxx_fw_objs); + int id; + + id = adf_default_fw_config[obj_num].obj; + if (id >= num_fw_objs) + return NULL; + + return adf_6xxx_fw_objs[id]; +} + +static const char *uof_get_name_6xxx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + return uof_get_name(accel_dev, obj_num); +} + +static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + if (obj_num >= uof_get_num_objs(accel_dev)) + return -EINVAL; + + return adf_default_fw_config[obj_num].obj; +} + +static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + return adf_default_fw_config[obj_num].ae_mask; +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + if (adf_gen6_init_thd2arb_map(accel_dev)) + dev_warn(&GET_DEV(accel_dev), + "Failed to generate thread to arbiter mapping"); + + return GET_HW_DATA(accel_dev)->thd_to_arb_map; +} + +static int adf_init_device(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + u32 status; + u32 csr; + int ret; + + /* Temporarily mask PM interrupt */ + csr = ADF_CSR_RD(addr, ADF_GEN6_ERRMSK2); + csr |= ADF_GEN6_PM_SOU; + ADF_CSR_WR(addr, ADF_GEN6_ERRMSK2, csr); + + /* Set DRV_ACTIVE bit to power up the device */ + ADF_CSR_WR(addr, ADF_GEN6_PM_INTERRUPT, ADF_GEN6_PM_DRV_ACTIVE); + + /* Poll status register to make sure the device is powered up */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_GEN6_PM_INIT_STATE, + ADF_GEN6_PM_POLL_DELAY_US, + ADF_GEN6_PM_POLL_TIMEOUT_US, true, addr, + ADF_GEN6_PM_STATUS); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); + return ret; + } + + dev_dbg(&GET_DEV(accel_dev), "Setting virtual channels for device qat_dev%d\n", + accel_dev->accel_id); + + ret = adf_gen6_set_vc(accel_dev); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to set virtual channels\n"); + + return ret; +} + +static int enable_pm(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER); + if (ret) + return ret; + + /* Initialize PM internal data */ + adf_gen6_init_dev_pm_data(accel_dev); + + return 0; +} + +static int dev_config(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + return ret; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + return ret; + + switch (adf_get_service_enabled(accel_dev)) { + case SVC_DC: + case SVC_DCC: + ret = adf_gen6_comp_dev_config(accel_dev); + break; + default: + ret = adf_gen6_no_dev_config(accel_dev); + break; + } + if (ret) + return ret; + + __set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + return ret; +} + +static void adf_gen6_init_rl_data(struct adf_rl_hw_data *rl_data) +{ + rl_data->pciout_tb_offset = ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET; + rl_data->pciin_tb_offset = ADF_GEN6_RL_TOKEN_PCIEIN_BUCKET_OFFSET; + rl_data->r2l_offset = ADF_GEN6_RL_R2L_OFFSET; + rl_data->l2c_offset = ADF_GEN6_RL_L2C_OFFSET; + rl_data->c2s_offset = ADF_GEN6_RL_C2S_OFFSET; + rl_data->pcie_scale_div = ADF_6XXX_RL_PCIE_SCALE_FACTOR_DIV; + rl_data->pcie_scale_mul = ADF_6XXX_RL_PCIE_SCALE_FACTOR_MUL; + rl_data->max_tp[SVC_ASYM] = ADF_6XXX_RL_MAX_TP_ASYM; + rl_data->max_tp[SVC_SYM] = ADF_6XXX_RL_MAX_TP_SYM; + rl_data->max_tp[SVC_DC] = ADF_6XXX_RL_MAX_TP_DC; + rl_data->max_tp[SVC_DECOMP] = ADF_6XXX_RL_MAX_TP_DECOMP; + rl_data->scan_interval = ADF_6XXX_RL_SCANS_PER_SEC; + rl_data->scale_ref = ADF_6XXX_RL_SLICE_REF; + + init_num_svc_aes(rl_data); +} + +void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &adf_6xxx_class; + hw_data->instance_id = adf_6xxx_class.instances++; + hw_data->num_banks = ADF_GEN6_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_GEN6_NUM_BANKS_PER_VF; + hw_data->num_rings_per_bank = ADF_GEN6_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_GEN6_MAX_ACCELERATORS; + hw_data->num_engines = ADF_6XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_GEN6_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN6_TX_RINGS_MASK; + hw_data->ring_to_svc_map = 0; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_arb_info = get_arb_info; + hw_data->get_admin_info = get_admin_info; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_sku = get_sku; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = enable_ints; + hw_data->reset_device = adf_reset_flr; + hw_data->admin_ae_mask = ADF_6XXX_ADMIN_AE_MASK; + hw_data->fw_name = ADF_6XXX_FW; + hw_data->fw_mmp_name = ADF_6XXX_MMP; + hw_data->uof_get_name = uof_get_name_6xxx; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_obj_type = uof_get_obj_type; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->set_msix_rttable = set_msix_default_rttable; + hw_data->set_ssm_wdtimer = set_ssm_wdtimer; + hw_data->get_ring_to_svc_map = get_ring_to_svc_map; + hw_data->disable_iov = adf_disable_sriov; + hw_data->ring_pair_reset = ring_pair_reset; + hw_data->dev_config = dev_config; + hw_data->bank_state_save = adf_bank_state_save; + hw_data->bank_state_restore = adf_bank_state_restore; + hw_data->get_hb_clock = get_heartbeat_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; + hw_data->init_device = adf_init_device; + hw_data->enable_pm = enable_pm; + hw_data->services_supported = services_supported; + hw_data->num_rps = ADF_GEN6_ETR_MAX_BANKS; + hw_data->clock_frequency = ADF_6XXX_AE_FREQ; + hw_data->get_svc_slice_cnt = adf_gen6_get_svc_slice_cnt; + + adf_gen6_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen6_init_dc_ops(&hw_data->dc_ops); + adf_gen6_init_vf_mig_ops(&hw_data->vfmig_ops); + adf_gen6_init_ras_ops(&hw_data->ras_ops); + adf_gen6_init_tl_data(&hw_data->tl_data); + adf_gen6_init_rl_data(&hw_data->rl_data); +} + +void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data) +{ + if (hw_data->dev_class->instances) + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h new file mode 100644 index 000000000000..d822911fe68c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_6XXX_HW_DATA_H_ +#define ADF_6XXX_HW_DATA_H_ + +#include <linux/bits.h> +#include <linux/time.h> +#include <linux/units.h> + +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" +#include "adf_dc.h" + +/* PCIe configuration space */ +#define ADF_GEN6_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) +#define ADF_GEN6_SRAM_BAR 0 +#define ADF_GEN6_PMISC_BAR 1 +#define ADF_GEN6_ETR_BAR 2 +#define ADF_6XXX_MAX_ACCELENGINES 9 + +/* Clocks frequency */ +#define ADF_GEN6_COUNTER_FREQ (100 * HZ_PER_MHZ) + +/* Physical function fuses */ +#define ADF_GEN6_FUSECTL0_OFFSET 0x2C8 +#define ADF_GEN6_FUSECTL1_OFFSET 0x2CC +#define ADF_GEN6_FUSECTL4_OFFSET 0x2D8 + +/* Accelerators */ +#define ADF_GEN6_ACCELERATORS_MASK 0x1 +#define ADF_GEN6_MAX_ACCELERATORS 1 + +/* MSI-X interrupt */ +#define ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET 0x41A040 +#define ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET 0x41A044 +#define ADF_GEN6_SMIAPF_MASK_OFFSET 0x41A084 +#define ADF_GEN6_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 4)) + +/* Bank and ring configuration */ +#define ADF_GEN6_NUM_RINGS_PER_BANK 2 +#define ADF_GEN6_NUM_BANKS_PER_VF 4 +#define ADF_GEN6_ETR_MAX_BANKS 64 +#define ADF_GEN6_RX_RINGS_OFFSET 1 +#define ADF_GEN6_TX_RINGS_MASK 0x1 + +/* Arbiter configuration */ +#define ADF_GEN6_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_GEN6_ARB_OFFSET 0x000 +#define ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET 0x400 + +/* Admin interface configuration */ +#define ADF_GEN6_ADMINMSGUR_OFFSET 0x500574 +#define ADF_GEN6_ADMINMSGLR_OFFSET 0x500578 +#define ADF_GEN6_MAILBOX_BASE_OFFSET 0x600970 + +/* + * Watchdog timers + * Timeout is in cycles. Clock speed may vary across products but this + * value should be a few milli-seconds. + */ +#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL +#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000ULL +#define ADF_SSMWDTATHL_OFFSET 0x5208 +#define ADF_SSMWDTATHH_OFFSET 0x520C +#define ADF_SSMWDTCNVL_OFFSET 0x5408 +#define ADF_SSMWDTCNVH_OFFSET 0x540C +#define ADF_SSMWDTUCSL_OFFSET 0x5808 +#define ADF_SSMWDTUCSH_OFFSET 0x580C +#define ADF_SSMWDTDCPRL_OFFSET 0x5A08 +#define ADF_SSMWDTDCPRH_OFFSET 0x5A0C +#define ADF_SSMWDTPKEL_OFFSET 0x5E08 +#define ADF_SSMWDTPKEH_OFFSET 0x5E0C + +/* Ring reset */ +#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_RPRESET_POLL_DELAY_US 20 +#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) +#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + (bank) * 8) +#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) +#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) + +/* Controls and sets up the corresponding ring mode of operation */ +#define ADF_GEN6_CSR_RINGMODECTL(bank) (0x9000 + (bank) * 4) + +/* Specifies the traffic class to use for the transactions to/from the ring */ +#define ADF_GEN6_RINGMODECTL_TC_MASK GENMASK(18, 16) +#define ADF_GEN6_RINGMODECTL_TC_DEFAULT 0x7 + +/* Specifies usage of tc for the transactions to/from this ring */ +#define ADF_GEN6_RINGMODECTL_TC_EN_MASK GENMASK(20, 19) + +/* + * Use the value programmed in the tc field for request descriptor + * and metadata read transactions + */ +#define ADF_GEN6_RINGMODECTL_TC_EN_OP1 0x1 + +/* VC0 Resource Control Register */ +#define ADF_GEN6_PVC0CTL_OFFSET 0x204 +#define ADF_GEN6_PVC0CTL_TCVCMAP_OFFSET 1 +#define ADF_GEN6_PVC0CTL_TCVCMAP_MASK GENMASK(7, 1) +#define ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT 0x3F + +/* VC1 Resource Control Register */ +#define ADF_GEN6_PVC1CTL_OFFSET 0x210 +#define ADF_GEN6_PVC1CTL_TCVCMAP_OFFSET 1 +#define ADF_GEN6_PVC1CTL_TCVCMAP_MASK GENMASK(7, 1) +#define ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT 0x40 +#define ADF_GEN6_PVC1CTL_VCEN_OFFSET 31 +#define ADF_GEN6_PVC1CTL_VCEN_MASK BIT(31) +/* RW bit: 0x1 - enables a Virtual Channel, 0x0 - disables */ +#define ADF_GEN6_PVC1CTL_VCEN_ON 0x1 + +/* Error source mask registers */ +#define ADF_GEN6_ERRMSK0 0x41A210 +#define ADF_GEN6_ERRMSK1 0x41A214 +#define ADF_GEN6_ERRMSK2 0x41A218 +#define ADF_GEN6_ERRMSK3 0x41A21C + +#define ADF_GEN6_VFLNOTIFY BIT(7) + +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + +/* Rate Limiting */ +#define ADF_GEN6_RL_R2L_OFFSET 0x508000 +#define ADF_GEN6_RL_L2C_OFFSET 0x509000 +#define ADF_GEN6_RL_C2S_OFFSET 0x508818 +#define ADF_GEN6_RL_TOKEN_PCIEIN_BUCKET_OFFSET 0x508800 +#define ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET 0x508804 + +/* Physical function fuses */ +#define ADF_6XXX_ACCELENGINES_MASK GENMASK(8, 0) +#define ADF_6XXX_ADMIN_AE_MASK GENMASK(8, 8) + +/* Firmware binaries */ +#define ADF_6XXX_FW "qat_6xxx.bin" +#define ADF_6XXX_MMP "qat_6xxx_mmp.bin" +#define ADF_6XXX_CY_OBJ "qat_6xxx_cy.bin" +#define ADF_6XXX_DC_OBJ "qat_6xxx_dc.bin" +#define ADF_6XXX_ADMIN_OBJ "qat_6xxx_admin.bin" + +/* RL constants */ +#define ADF_6XXX_RL_PCIE_SCALE_FACTOR_DIV 100 +#define ADF_6XXX_RL_PCIE_SCALE_FACTOR_MUL 102 +#define ADF_6XXX_RL_SCANS_PER_SEC 954 +#define ADF_6XXX_RL_MAX_TP_ASYM 173750UL +#define ADF_6XXX_RL_MAX_TP_SYM 95000UL +#define ADF_6XXX_RL_MAX_TP_DC 40000UL +#define ADF_6XXX_RL_MAX_TP_DECOMP 40000UL +#define ADF_6XXX_RL_SLICE_REF 1000UL + +/* Clock frequency */ +#define ADF_6XXX_AE_FREQ (1000 * HZ_PER_MHZ) + +enum icp_qat_gen6_slice_mask { + ICP_ACCEL_GEN6_MASK_UCS_SLICE = BIT(0), + ICP_ACCEL_GEN6_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_GEN6_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_GEN6_MASK_CPR_SLICE = BIT(3), + ICP_ACCEL_GEN6_MASK_DCPRZ_SLICE = BIT(4), + ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE = BIT(6), +}; + +void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data); + +#endif /* ADF_6XXX_HW_DATA_H_ */ diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c new file mode 100644 index 000000000000..c1dc9c56fdf5 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include <linux/array_size.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/errno.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/types.h> + +#include <adf_accel_devices.h> +#include <adf_cfg.h> +#include <adf_common_drv.h> +#include <adf_dbgfs.h> + +#include "adf_gen6_shared.h" +#include "adf_6xxx_hw_data.h" + +static int bar_map[] = { + 0, /* SRAM */ + 2, /* PMISC */ + 4, /* ETR */ +}; + +static void adf_device_down(void *accel_dev) +{ + adf_dev_down(accel_dev); +} + +static void adf_dbgfs_cleanup(void *accel_dev) +{ + adf_dbgfs_exit(accel_dev); +} + +static void adf_cfg_device_remove(void *accel_dev) +{ + adf_cfg_dev_remove(accel_dev); +} + +static void adf_cleanup_hw_data(void *accel_dev) +{ + struct adf_accel_dev *accel_device = accel_dev; + + if (accel_device->hw_device) { + adf_clean_hw_data_6xxx(accel_device->hw_device); + accel_device->hw_device = NULL; + } +} + +static void adf_devmgr_remove(void *accel_dev) +{ + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + struct device *dev = &pdev->dev; + struct adf_accel_dev *accel_dev; + struct adf_bar *bar; + unsigned int i; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + return dev_err_probe(dev, -EINVAL, "Invalid NUMA configuration.\n"); + } + + accel_dev = devm_kzalloc(dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + INIT_LIST_HEAD(&accel_dev->list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + accel_dev->owner = THIS_MODULE; + + hw_data = devm_kzalloc(dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) + return -ENOMEM; + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL0_OFFSET, &hw_data->fuses[ADF_FUSECTL0]); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL1_OFFSET, &hw_data->fuses[ADF_FUSECTL1]); + + if (!(hw_data->fuses[ADF_FUSECTL1] & ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE)) + return dev_err_probe(dev, -EFAULT, "Wireless mode is not supported.\n"); + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) + return dev_err_probe(dev, ret, "Cannot enable PCI device.\n"); + + ret = adf_devmgr_add_dev(accel_dev, NULL); + if (ret) + return dev_err_probe(dev, ret, "Failed to add new accelerator device.\n"); + + ret = devm_add_action_or_reset(dev, adf_devmgr_remove, accel_dev); + if (ret) + return ret; + + accel_dev->hw_device = hw_data; + adf_init_hw_data_6xxx(accel_dev->hw_device); + + ret = devm_add_action_or_reset(dev, adf_cleanup_hw_data, accel_dev); + if (ret) + return ret; + + /* Get Accelerators and Accelerator Engine masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & ADF_GEN6_ACCELERATORS_MASK)) { + ret = -EFAULT; + return dev_err_probe(dev, ret, "No acceleration units were found.\n"); + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, adf_cfg_device_remove, accel_dev); + if (ret) + return ret; + + /* Set DMA identifier */ + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + return dev_err_probe(dev, ret, "No usable DMA configuration.\n"); + + ret = adf_gen6_cfg_dev_init(accel_dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to initialize configuration.\n"); + + /* Get accelerator capability mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) { + ret = -EINVAL; + return dev_err_probe(dev, ret, "Failed to get capabilities mask.\n"); + } + + for (i = 0; i < ARRAY_SIZE(bar_map); i++) { + bar = &accel_pci_dev->pci_bars[i]; + + /* Map 64-bit PCIe BAR */ + bar->virt_addr = pcim_iomap_region(pdev, bar_map[i], pci_name(pdev)); + if (IS_ERR(bar->virt_addr)) { + ret = PTR_ERR(bar->virt_addr); + return dev_err_probe(dev, ret, "Failed to ioremap PCI region.\n"); + } + } + + pci_set_master(pdev); + + /* + * The PCI config space is saved at this point and will be restored + * after a Function Level Reset (FLR) as the FLR does not completely + * restore it. + */ + ret = pci_save_state(pdev); + if (ret) + return dev_err_probe(dev, ret, "Failed to save pci state.\n"); + + accel_dev->ras_errors.enabled = true; + + adf_dbgfs_init(accel_dev); + + ret = devm_add_action_or_reset(dev, adf_dbgfs_cleanup, accel_dev); + if (ret) + return ret; + + ret = adf_dev_up(accel_dev, true); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, adf_device_down, accel_dev); + if (ret) + return ret; + + ret = adf_sysfs_init(accel_dev); + + return ret; +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_6XXX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_6XXX_DEVICE_NAME, + .probe = adf_probe, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; +module_pci_driver(adf_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_6XXX_FW); +MODULE_FIRMWARE(ADF_6XXX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology for GEN6 Devices"); +MODULE_SOFTDEP("pre: crypto-intel_qat"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c3xxx/Makefile b/drivers/crypto/intel/qat/qat_c3xxx/Makefile index 7a06ad519bfc..43604c025f0c 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o -qat_c3xxx-objs := adf_drv.o adf_c3xxx_hw_data.o +qat_c3xxx-y := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 201f9412c582..07f2c42a68f5 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -5,7 +5,6 @@ #include <adf_clock.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> -#include <adf_gen2_dc.h> #include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> @@ -22,13 +21,12 @@ static const u32 thrd_to_arb_map[ADF_C3XXX_MAX_ACCELENGINES] = { static struct adf_hw_device_class c3xxx_class = { .name = ADF_C3XXX_DEVICE_NAME, .type = DEV_C3XXX, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; @@ -39,8 +37,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index 4d18057745d4..bceb5dd8b148 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -19,24 +19,6 @@ #include <adf_dbgfs.h> #include "adf_c3xxx_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C3XXX_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -126,7 +108,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c3xxx(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); @@ -227,6 +209,29 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C3XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); @@ -252,4 +257,4 @@ MODULE_FIRMWARE(ADF_C3XXX_FW); MODULE_FIRMWARE(ADF_C3XXX_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile index 7ef633058c4f..03f6745b4aa2 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o -qat_c3xxxvf-objs := adf_drv.o adf_c3xxxvf_hw_data.o +qat_c3xxxvf-y := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index a512ca4efd3f..db3c33fa1881 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -3,7 +3,6 @@ #include <adf_accel_devices.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> -#include <adf_gen2_dc.h> #include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> @@ -13,7 +12,6 @@ static struct adf_hw_device_class c3xxxiov_class = { .name = ADF_C3XXXVF_DEVICE_NAME, .type = DEV_C3XXXVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c index f0023cfb234c..c622793e94a8 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c @@ -226,4 +226,4 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c62x/Makefile b/drivers/crypto/intel/qat/qat_c62x/Makefile index cc9255b3b198..f3d722bef088 100644 --- a/drivers/crypto/intel/qat/qat_c62x/Makefile +++ b/drivers/crypto/intel/qat/qat_c62x/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o -qat_c62x-objs := adf_drv.o adf_c62x_hw_data.o +qat_c62x-y := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 6b5b0cf9c7c7..0b410b41474d 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -5,7 +5,6 @@ #include <adf_clock.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> -#include <adf_gen2_dc.h> #include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> @@ -22,13 +21,12 @@ static const u32 thrd_to_arb_map[ADF_C62X_MAX_ACCELENGINES] = { static struct adf_hw_device_class c62x_class = { .name = ADF_C62X_DEVICE_NAME, .type = DEV_C62X, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; @@ -39,8 +37,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index e6b5de55434e..23ccb72b6ea2 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -19,24 +19,6 @@ #include <adf_dbgfs.h> #include "adf_c62x_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C62X_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -126,7 +108,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c62x(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); @@ -169,7 +151,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ - i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; + i = (hw_data->fuses[ADF_FUSECTL0] & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; @@ -227,6 +209,29 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C62X_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); @@ -252,4 +257,4 @@ MODULE_FIRMWARE(ADF_C62X_FW); MODULE_FIRMWARE(ADF_C62X_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c62xvf/Makefile b/drivers/crypto/intel/qat/qat_c62xvf/Makefile index 256786662d60..ed7f3f722d99 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c62xvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o -qat_c62xvf-objs := adf_drv.o adf_c62xvf_hw_data.o +qat_c62xvf-y := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 4aaaaf921734..7f00035d3661 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -3,7 +3,6 @@ #include <adf_accel_devices.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> -#include <adf_gen2_dc.h> #include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> @@ -13,7 +12,6 @@ static struct adf_hw_device_class c62xiov_class = { .name = ADF_C62XVF_DEVICE_NAME, .type = DEV_C62XVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c index 2bd5b0ff00e3..4840d44bbd5b 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c @@ -226,4 +226,4 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index eac73cbfdd38..89845754841b 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -1,62 +1,67 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o -ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CRYPTO_QAT -intel_qat-objs := adf_cfg.o \ - adf_isr.o \ - adf_ctl_drv.o \ +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE='"CRYPTO_QAT"' +intel_qat-y := adf_accel_engine.o \ + adf_admin.o \ + adf_aer.o \ + adf_bank_state.o \ + adf_cfg.o \ adf_cfg_services.o \ + adf_clock.o \ + adf_ctl_drv.o \ + adf_dc.o \ adf_dev_mgr.o \ - adf_init.o \ - adf_accel_engine.o \ - adf_aer.o \ - adf_transport.o \ - adf_admin.o \ - adf_hw_arbiter.o \ - adf_sysfs.o \ - adf_sysfs_ras_counters.o \ + adf_gen2_config.o \ adf_gen2_hw_csr_data.o \ adf_gen2_hw_data.o \ - adf_gen2_config.o \ adf_gen4_config.o \ adf_gen4_hw_csr_data.o \ adf_gen4_hw_data.o \ - adf_gen4_vf_mig.o \ adf_gen4_pm.o \ - adf_gen2_dc.o \ - adf_gen4_dc.o \ adf_gen4_ras.o \ - adf_gen4_timer.o \ - adf_clock.o \ + adf_gen4_vf_mig.o \ + adf_gen6_ras.o \ + adf_gen6_shared.o \ + adf_hw_arbiter.o \ + adf_init.o \ + adf_isr.o \ adf_mstate_mgr.o \ - qat_crypto.o \ - qat_compression.o \ - qat_comp_algs.o \ - qat_algs.o \ - qat_asym_algs.o \ - qat_algs_send.o \ - adf_rl.o \ adf_rl_admin.o \ + adf_rl.o \ + adf_sysfs.o \ + adf_sysfs_ras_counters.o \ adf_sysfs_rl.o \ - qat_uclo.o \ - qat_hal.o \ + adf_timer.o \ + adf_transport.o \ + qat_algs.o \ + qat_algs_send.o \ + qat_asym_algs.o \ qat_bl.o \ - qat_mig_dev.o + qat_comp_algs.o \ + qat_compression.o \ + qat_crypto.o \ + qat_hal.o \ + qat_mig_dev.o \ + qat_uclo.o -intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o \ +intel_qat-$(CONFIG_DEBUG_FS) += adf_cnv_dbgfs.o \ + adf_dbgfs.o \ adf_fw_counters.o \ - adf_cnv_dbgfs.o \ adf_gen4_pm_debugfs.o \ adf_gen4_tl.o \ - adf_heartbeat.o \ + adf_gen6_pm_dbgfs.o \ + adf_gen6_tl.o \ adf_heartbeat_dbgfs.o \ + adf_heartbeat.o \ adf_pm_dbgfs.o \ + adf_pm_dbgfs_utils.o \ adf_telemetry.o \ adf_tl_debugfs.o \ - adf_dbgfs.o + adf_transport_debug.o -intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ +intel_qat-$(CONFIG_PCI_IOV) += adf_gen2_pfvf.o adf_gen4_pfvf.o \ adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ - adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \ - adf_gen2_pfvf.o adf_gen4_pfvf.o + adf_pfvf_utils.o adf_pfvf_vf_msg.o \ + adf_pfvf_vf_proto.o adf_sriov.o adf_vf_isr.o intel_qat-$(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION) += adf_heartbeat_inject.o diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 7830ecb1a1f1..9fe3239f0114 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -10,7 +10,9 @@ #include <linux/ratelimit.h> #include <linux/types.h> #include <linux/qat/qat_mig_dev.h> +#include <linux/wordpart.h> #include "adf_cfg_common.h" +#include "adf_dc.h" #include "adf_rl.h" #include "adf_telemetry.h" #include "adf_pfvf_msg.h" @@ -24,14 +26,18 @@ #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_420XX_DEVICE_NAME "420xx" -#define ADF_4XXX_PCI_DEVICE_ID 0x4940 -#define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 -#define ADF_401XX_PCI_DEVICE_ID 0x4942 -#define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 -#define ADF_402XX_PCI_DEVICE_ID 0x4944 -#define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 -#define ADF_420XX_PCI_DEVICE_ID 0x4946 -#define ADF_420XXIOV_PCI_DEVICE_ID 0x4947 +#define ADF_6XXX_DEVICE_NAME "6xxx" +#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 +#define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941 +#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 +#define PCI_DEVICE_ID_INTEL_QAT_401XXIOV 0x4943 +#define PCI_DEVICE_ID_INTEL_QAT_402XX 0x4944 +#define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 +#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 +#define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX_IOV 0x4949 + #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_DEVICE_FUSECTL_MASK 0x80000000 @@ -52,6 +58,16 @@ enum adf_accel_capabilities { ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 }; +enum adf_fuses { + ADF_FUSECTL0, + ADF_FUSECTL1, + ADF_FUSECTL2, + ADF_FUSECTL3, + ADF_FUSECTL4, + ADF_FUSECTL5, + ADF_MAX_FUSES +}; + struct adf_bar { resource_size_t base_addr; void __iomem *virt_addr; @@ -141,39 +157,7 @@ struct admin_info { u32 mailbox_offset; }; -struct ring_config { - u64 base; - u32 config; - u32 head; - u32 tail; - u32 reserved0; -}; - -struct bank_state { - u32 ringstat0; - u32 ringstat1; - u32 ringuostat; - u32 ringestat; - u32 ringnestat; - u32 ringnfstat; - u32 ringfstat; - u32 ringcstat0; - u32 ringcstat1; - u32 ringcstat2; - u32 ringcstat3; - u32 iaintflagen; - u32 iaintflagreg; - u32 iaintflagsrcsel0; - u32 iaintflagsrcsel1; - u32 iaintcolen; - u32 iaintcolctl; - u32 iaintflagandcolen; - u32 ringexpstat; - u32 ringexpintenable; - u32 ringsrvarben; - u32 reserved0; - struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK]; -}; +struct adf_bank_state; struct adf_hw_csr_ops { u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); @@ -256,7 +240,8 @@ struct adf_pfvf_ops { }; struct adf_dc_ops { - void (*build_deflate_ctx)(void *ctx); + int (*build_comp_block)(void *ctx, enum adf_dc_algo algo); + int (*build_decomp_block)(void *ctx, enum adf_dc_algo algo); }; struct qat_migdev_ops { @@ -321,9 +306,9 @@ struct adf_hw_device_data { void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number, - struct bank_state *state); + struct adf_bank_state *state); int (*bank_state_restore)(struct adf_accel_dev *accel_dev, - u32 bank_number, struct bank_state *state); + u32 bank_number, struct adf_bank_state *state); void (*reset_device)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); @@ -333,6 +318,9 @@ struct adf_hw_device_data { int (*get_rp_group)(struct adf_accel_dev *accel_dev, u32 ae_mask); u32 (*get_ena_thd_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); int (*dev_config)(struct adf_accel_dev *accel_dev); + bool (*services_supported)(unsigned long mask); + u32 (*get_svc_slice_cnt)(struct adf_accel_dev *accel_dev, + enum adf_base_services svc); struct adf_pfvf_ops pfvf_ops; struct adf_hw_csr_ops csr_ops; struct adf_dc_ops dc_ops; @@ -343,7 +331,7 @@ struct adf_hw_device_data { struct qat_migdev_ops vfmig_ops; const char *fw_name; const char *fw_mmp_name; - u32 fuses; + u32 fuses[ADF_MAX_FUSES]; u32 straps; u32 accel_capabilities_mask; u32 extended_dc_capabilities; @@ -370,6 +358,15 @@ struct adf_hw_device_data { /* CSR write macro */ #define ADF_CSR_WR(csr_base, csr_offset, val) \ __raw_writel(val, csr_base + csr_offset) +/* + * CSR write macro to handle cases where the high and low + * offsets are sparsely located. + */ +#define ADF_CSR_WR64_LO_HI(csr_base, csr_low_offset, csr_high_offset, val) \ +do { \ + ADF_CSR_WR(csr_base, csr_low_offset, lower_32_bits(val)); \ + ADF_CSR_WR(csr_base, csr_high_offset, upper_32_bits(val)); \ +} while (0) /* CSR read macro */ #define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c index acad526eb741..573388c37100 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.c +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c @@ -449,6 +449,7 @@ int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay) return adf_send_admin(accel_dev, &req, &resp, ae_mask); } +EXPORT_SYMBOL_GPL(adf_init_admin_pm); int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr, size_t buff_size) diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 4cb8bd83f570..11728cf32653 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -105,7 +105,6 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) accel_dev->accel_id); hw_device->reset_device(accel_dev); pci_restore_state(pdev); - pci_save_state(pdev); } } @@ -204,7 +203,6 @@ static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) if (!pdev->is_busmaster) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); res = adf_dev_up(accel_dev, false); if (res && res != -EALREADY) return PCI_ERS_RESULT_DISCONNECT; @@ -229,7 +227,7 @@ const struct pci_error_handlers adf_err_handler = { }; EXPORT_SYMBOL_GPL(adf_err_handler); -int adf_dev_autoreset(struct adf_accel_dev *accel_dev) +static int adf_dev_autoreset(struct adf_accel_dev *accel_dev) { if (accel_dev->autoreset_on_error) return adf_dev_aer_schedule_reset(accel_dev, ADF_DEV_RESET_ASYNC); @@ -276,11 +274,11 @@ int adf_notify_fatal_error(struct adf_accel_dev *accel_dev) int adf_init_aer(void) { device_reset_wq = alloc_workqueue("qat_device_reset_wq", - WQ_MEM_RECLAIM, 0); + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!device_reset_wq) return -EFAULT; - device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", 0, 0); + device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", WQ_PERCPU, 0); if (!device_sriov_wq) { destroy_workqueue(device_reset_wq); device_reset_wq = NULL; diff --git a/drivers/crypto/intel/qat/qat_common/adf_bank_state.c b/drivers/crypto/intel/qat/qat_common/adf_bank_state.c new file mode 100644 index 000000000000..225d55d56a4b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_bank_state.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + +#include <linux/bits.h> +#include <linux/dev_printk.h> +#include <linux/printk.h> +#include "adf_accel_devices.h" +#include "adf_bank_state.h" +#include "adf_common_drv.h" + +/* Ring interrupt masks */ +#define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0) +#define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0) +#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4 + +static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val, + const char *name, void __iomem *base, u32 bank) +{ + u32 actual_val = op(base, bank); + + if (expect_val == actual_val) + return 0; + + pr_err("Fail to restore %s register. Expected %#x, actual %#x\n", + name, expect_val, actual_val); + + return -EINVAL; +} + +static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, + u32 bank, struct adf_bank_state *state, u32 num_rings) +{ + u32 i; + + state->ringstat0 = ops->read_csr_stat(base, bank); + state->ringuostat = ops->read_csr_uo_stat(base, bank); + state->ringestat = ops->read_csr_e_stat(base, bank); + state->ringnestat = ops->read_csr_ne_stat(base, bank); + state->ringnfstat = ops->read_csr_nf_stat(base, bank); + state->ringfstat = ops->read_csr_f_stat(base, bank); + state->ringcstat0 = ops->read_csr_c_stat(base, bank); + state->iaintflagen = ops->read_csr_int_en(base, bank); + state->iaintflagreg = ops->read_csr_int_flag(base, bank); + state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank); + state->iaintcolen = ops->read_csr_int_col_en(base, bank); + state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank); + state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank); + state->ringexpstat = ops->read_csr_exp_stat(base, bank); + state->ringexpintenable = ops->read_csr_exp_int_en(base, bank); + state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank); + + for (i = 0; i < num_rings; i++) { + state->rings[i].head = ops->read_csr_ring_head(base, bank, i); + state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i); + state->rings[i].config = ops->read_csr_ring_config(base, bank, i); + state->rings[i].base = ops->read_csr_ring_base(base, bank, i); + } +} + +static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base, + u32 bank, struct adf_bank_state *state, u32 num_rings, + int tx_rx_gap) +{ + u32 val, tmp_val, i; + int ret; + + for (i = 0; i < num_rings; i++) + ops->write_csr_ring_base(base, bank, i, state->rings[i].base); + + for (i = 0; i < num_rings; i++) + ops->write_csr_ring_config(base, bank, i, state->rings[i].config); + + for (i = 0; i < num_rings / 2; i++) { + int tx = i * (tx_rx_gap + 1); + int rx = tx + tx_rx_gap; + + ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); + ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail); + + /* + * The TX ring head needs to be updated again to make sure that + * the HW will not consider the ring as full when it is empty + * and the correct state flags are set to match the recovered state. + */ + if (state->ringestat & BIT(tx)) { + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK; + ops->write_csr_int_srcsel_w_val(base, bank, val); + ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); + } + + ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; + ops->write_csr_int_srcsel_w_val(base, bank, val); + + ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head); + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; + ops->write_csr_int_srcsel_w_val(base, bank, val); + + /* + * The RX ring tail needs to be updated again to make sure that + * the HW will not consider the ring as empty when it is full + * and the correct state flags are set to match the recovered state. + */ + if (state->ringfstat & BIT(rx)) + ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); + } + + ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen); + ops->write_csr_int_en(base, bank, state->iaintflagen); + ops->write_csr_int_col_en(base, bank, state->iaintcolen); + ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0); + ops->write_csr_exp_int_en(base, bank, state->ringexpintenable); + ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl); + + /* + * Verify whether any exceptions were raised during the bank save process. + * If exceptions occurred, the status and exception registers cannot + * be directly restored. Consequently, further restoration is not + * feasible, and the current state of the ring should be maintained. + */ + val = state->ringexpstat; + if (val) { + pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n", + bank, val); + return 0; + } + + /* Ensure that the restoration process completed without exceptions */ + tmp_val = ops->read_csr_exp_stat(base, bank); + if (tmp_val) { + pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val); + return -EFAULT; + } + + ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben); + + /* Check that all ring statuses match the saved state. */ + ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat", + base, bank); + if (ret) + return ret; + + return 0; +} + +/** + * adf_bank_state_save() - save state of bank-related registers + * @accel_dev: Pointer to the device structure + * @bank_number: Bank number + * @state: Pointer to bank state structure + * + * This function saves the state of a bank by reading the bank CSRs and + * writing them in the @state structure. + * + * Returns 0 on success, error code otherwise + */ +int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_base = adf_get_etr_base(accel_dev); + + if (bank_number >= hw_data->num_banks || !state) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number); + + bank_state_save(csr_ops, csr_base, bank_number, state, + hw_data->num_rings_per_bank); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_bank_state_save); + +/** + * adf_bank_state_restore() - restore state of bank-related registers + * @accel_dev: Pointer to the device structure + * @bank_number: Bank number + * @state: Pointer to bank state structure + * + * This function attempts to restore the state of a bank by writing the + * bank CSRs to the values in the state structure. + * + * Returns 0 on success, error code otherwise + */ +int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_base = adf_get_etr_base(accel_dev); + int ret; + + if (bank_number >= hw_data->num_banks || !state) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number); + + ret = bank_state_restore(csr_ops, csr_base, bank_number, state, + hw_data->num_rings_per_bank, hw_data->tx_rx_gap); + if (ret) + dev_err(&GET_DEV(accel_dev), + "Unable to restore state of bank %d\n", bank_number); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_bank_state_restore); diff --git a/drivers/crypto/intel/qat/qat_common/adf_bank_state.h b/drivers/crypto/intel/qat/qat_common/adf_bank_state.h new file mode 100644 index 000000000000..48b573d692dd --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_bank_state.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_BANK_STATE_H_ +#define ADF_BANK_STATE_H_ + +#include <linux/types.h> + +struct adf_accel_dev; + +struct ring_config { + u64 base; + u32 config; + u32 head; + u32 tail; + u32 reserved0; +}; + +struct adf_bank_state { + u32 ringstat0; + u32 ringstat1; + u32 ringuostat; + u32 ringestat; + u32 ringnestat; + u32 ringnfstat; + u32 ringfstat; + u32 ringcstat0; + u32 ringcstat1; + u32 ringcstat2; + u32 ringcstat3; + u32 iaintflagen; + u32 iaintflagreg; + u32 iaintflagsrcsel0; + u32 iaintflagsrcsel1; + u32 iaintcolen; + u32 iaintcolctl; + u32 iaintflagandcolen; + u32 ringexpstat; + u32 ringexpintenable; + u32 ringsrvarben; + u32 reserved0; + struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK]; +}; + +int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state); +int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index 89df3888d7ea..81e9e9d7eccd 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -29,6 +29,7 @@ enum adf_cfg_service_type { COMP, SYM, ASYM, + DECOMP, USED }; @@ -48,6 +49,7 @@ enum adf_device_type { DEV_C3XXXVF, DEV_4XXX, DEV_420XX, + DEV_6XXX, }; struct adf_dev_status_info { diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 268052294468..7d00bcb41ce7 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -1,47 +1,212 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2023 Intel Corporation */ +#include <linux/array_size.h> +#include <linux/bitops.h> #include <linux/export.h> #include <linux/pci.h> #include <linux/string.h> #include "adf_cfg.h" +#include "adf_cfg_common.h" #include "adf_cfg_services.h" #include "adf_cfg_strings.h" -const char *const adf_cfg_services[] = { - [SVC_CY] = ADF_CFG_CY, - [SVC_CY2] = ADF_CFG_ASYM_SYM, +static const char *const adf_cfg_services[] = { + [SVC_ASYM] = ADF_CFG_ASYM, + [SVC_SYM] = ADF_CFG_SYM, [SVC_DC] = ADF_CFG_DC, [SVC_DCC] = ADF_CFG_DCC, - [SVC_SYM] = ADF_CFG_SYM, - [SVC_ASYM] = ADF_CFG_ASYM, - [SVC_DC_ASYM] = ADF_CFG_DC_ASYM, - [SVC_ASYM_DC] = ADF_CFG_ASYM_DC, - [SVC_DC_SYM] = ADF_CFG_DC_SYM, - [SVC_SYM_DC] = ADF_CFG_SYM_DC, + [SVC_DECOMP] = ADF_CFG_DECOMP, }; -EXPORT_SYMBOL_GPL(adf_cfg_services); -int adf_get_service_enabled(struct adf_accel_dev *accel_dev) +/* + * Ensure that the size of the array matches the number of services, + * SVC_COUNT, that is used to size the bitmap. + */ +static_assert(ARRAY_SIZE(adf_cfg_services) == SVC_COUNT); + +/* + * Ensure that the maximum number of concurrent services that can be + * enabled on a device is less than or equal to the number of total + * supported services. + */ +static_assert(ARRAY_SIZE(adf_cfg_services) >= MAX_NUM_CONCURR_SVC); + +/* + * Ensure that the number of services fit a single unsigned long, as each + * service is represented by a bit in the mask. + */ +static_assert(BITS_PER_LONG >= SVC_COUNT); + +/* + * Ensure that size of the concatenation of all service strings is smaller + * than the size of the buffer that will contain them. + */ +static_assert(sizeof(ADF_CFG_SYM ADF_SERVICES_DELIMITER + ADF_CFG_ASYM ADF_SERVICES_DELIMITER + ADF_CFG_DC ADF_SERVICES_DELIMITER + ADF_CFG_DECOMP ADF_SERVICES_DELIMITER + ADF_CFG_DCC) < ADF_CFG_MAX_VAL_LEN_IN_BYTES); + +static int adf_service_string_to_mask(struct adf_accel_dev *accel_dev, const char *buf, + size_t len, unsigned long *out_mask) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + unsigned long mask = 0; + char *substr, *token; + int id, num_svc = 0; + + if (len > ADF_CFG_MAX_VAL_LEN_IN_BYTES - 1) + return -EINVAL; + + strscpy(services, buf, ADF_CFG_MAX_VAL_LEN_IN_BYTES); + substr = services; + + while ((token = strsep(&substr, ADF_SERVICES_DELIMITER))) { + id = sysfs_match_string(adf_cfg_services, token); + if (id < 0) + return id; + + if (test_and_set_bit(id, &mask)) + return -EINVAL; + + if (num_svc++ == MAX_NUM_CONCURR_SVC) + return -EINVAL; + } + + if (hw_data->services_supported && !hw_data->services_supported(mask)) + return -EINVAL; + + *out_mask = mask; + + return 0; +} + +static int adf_service_mask_to_string(unsigned long mask, char *buf, size_t len) +{ + int offset = 0; + int bit; + + if (len < ADF_CFG_MAX_VAL_LEN_IN_BYTES) + return -ENOSPC; + + for_each_set_bit(bit, &mask, SVC_COUNT) { + if (offset) + offset += scnprintf(buf + offset, len - offset, + ADF_SERVICES_DELIMITER); + + offset += scnprintf(buf + offset, len - offset, "%s", + adf_cfg_services[bit]); + } + + return 0; +} + +int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, + size_t in_len, char *out, size_t out_len) { - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + unsigned long mask; + int ret; + + ret = adf_service_string_to_mask(accel_dev, in, in_len, &mask); + if (ret) + return ret; + + if (!mask) + return -EINVAL; + + return adf_service_mask_to_string(mask, out, out_len); +} + +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + size_t len; int ret; ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, ADF_SERVICES_ENABLED, services); if (ret) { - dev_err(&GET_DEV(accel_dev), - ADF_SERVICES_ENABLED " param not found\n"); + dev_err(&GET_DEV(accel_dev), "%s param not found\n", + ADF_SERVICES_ENABLED); return ret; } - ret = match_string(adf_cfg_services, ARRAY_SIZE(adf_cfg_services), - services); - if (ret < 0) - dev_err(&GET_DEV(accel_dev), - "Invalid value of " ADF_SERVICES_ENABLED " param: %s\n", - services); + len = strnlen(services, ADF_CFG_MAX_VAL_LEN_IN_BYTES); + ret = adf_service_string_to_mask(accel_dev, services, len, mask); + if (ret) + dev_err(&GET_DEV(accel_dev), "Invalid value of %s param: %s\n", + ADF_SERVICES_ENABLED, services); return ret; } +EXPORT_SYMBOL_GPL(adf_get_service_mask); + +int adf_get_service_enabled(struct adf_accel_dev *accel_dev) +{ + unsigned long mask; + int ret; + + ret = adf_get_service_mask(accel_dev, &mask); + if (ret) + return ret; + + if (test_bit(SVC_SYM, &mask) && test_bit(SVC_ASYM, &mask)) + return SVC_SYM_ASYM; + + if (test_bit(SVC_SYM, &mask) && test_bit(SVC_DC, &mask)) + return SVC_SYM_DC; + + if (test_bit(SVC_ASYM, &mask) && test_bit(SVC_DC, &mask)) + return SVC_ASYM_DC; + + if (test_bit(SVC_SYM, &mask)) + return SVC_SYM; + + if (test_bit(SVC_ASYM, &mask)) + return SVC_ASYM; + + if (test_bit(SVC_DC, &mask)) + return SVC_DC; + + if (test_bit(SVC_DECOMP, &mask)) + return SVC_DECOMP; + + if (test_bit(SVC_DCC, &mask)) + return SVC_DCC; + + return -EINVAL; +} EXPORT_SYMBOL_GPL(adf_get_service_enabled); + +enum adf_cfg_service_type adf_srv_to_cfg_svc_type(enum adf_base_services svc) +{ + switch (svc) { + case SVC_ASYM: + return ASYM; + case SVC_SYM: + return SYM; + case SVC_DC: + return COMP; + case SVC_DECOMP: + return DECOMP; + default: + return UNUSED; + } +} + +bool adf_is_service_enabled(struct adf_accel_dev *accel_dev, enum adf_base_services svc) +{ + enum adf_cfg_service_type arb_srv = adf_srv_to_cfg_svc_type(svc); + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u8 rps_per_bundle = hw_data->num_banks_per_vf; + int i; + + for (i = 0; i < rps_per_bundle; i++) { + if (GET_SRV_TYPE(accel_dev, i) == arb_srv) + return true; + } + + return false; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h index c6b0328b0f5b..913d717280af 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h @@ -7,22 +7,38 @@ struct adf_accel_dev; -enum adf_services { - SVC_CY = 0, - SVC_CY2, - SVC_DC, - SVC_DCC, +enum adf_base_services { + SVC_ASYM = 0, SVC_SYM, - SVC_ASYM, - SVC_DC_ASYM, - SVC_ASYM_DC, - SVC_DC_SYM, - SVC_SYM_DC, + SVC_DC, + SVC_DECOMP, + SVC_BASE_COUNT +}; + +enum adf_extended_services { + SVC_DCC = SVC_BASE_COUNT, SVC_COUNT }; -extern const char *const adf_cfg_services[SVC_COUNT]; +enum adf_composed_services { + SVC_SYM_ASYM = SVC_COUNT, + SVC_SYM_DC, + SVC_ASYM_DC, +}; + +enum { + ADF_ONE_SERVICE = 1, + ADF_TWO_SERVICES, + ADF_THREE_SERVICES, +}; + +#define MAX_NUM_CONCURR_SVC ADF_THREE_SERVICES +int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, + size_t in_len, char *out, size_t out_len); int adf_get_service_enabled(struct adf_accel_dev *accel_dev); +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask); +enum adf_cfg_service_type adf_srv_to_cfg_svc_type(enum adf_base_services svc); +bool adf_is_service_enabled(struct adf_accel_dev *accel_dev, enum adf_base_services svc); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h index e015ad6cace2..30107a02ee7f 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h @@ -24,16 +24,13 @@ #define ADF_CY "Cy" #define ADF_DC "Dc" #define ADF_CFG_DC "dc" +#define ADF_CFG_DECOMP "decomp" #define ADF_CFG_CY "sym;asym" #define ADF_CFG_SYM "sym" #define ADF_CFG_ASYM "asym" -#define ADF_CFG_ASYM_SYM "asym;sym" -#define ADF_CFG_ASYM_DC "asym;dc" -#define ADF_CFG_DC_ASYM "dc;asym" -#define ADF_CFG_SYM_DC "sym;dc" -#define ADF_CFG_DC_SYM "dc;sym" #define ADF_CFG_DCC "dcc" #define ADF_SERVICES_ENABLED "ServicesEnabled" +#define ADF_SERVICES_DELIMITER ";" #define ADF_PM_IDLE_SUPPORT "PmIdleSupport" #define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" #define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index eaa6388a6678..6cf3a95489e8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -86,7 +86,6 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev); extern const struct pci_error_handlers adf_err_handler; void adf_reset_sbr(struct adf_accel_dev *accel_dev); void adf_reset_flr(struct adf_accel_dev *accel_dev); -int adf_dev_autoreset(struct adf_accel_dev *accel_dev); void adf_dev_restore(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); @@ -189,6 +188,7 @@ void adf_exit_misc_wq(void); bool adf_misc_wq_queue_work(struct work_struct *work); bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, unsigned long delay); +void adf_misc_wq_flush(void); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c index 70fa0f6497a9..c2e6f0cb7480 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c @@ -89,26 +89,14 @@ err_chrdev_unreg: return -EFAULT; } -static int adf_ctl_alloc_resources(struct adf_user_cfg_ctl_data **ctl_data, - unsigned long arg) +static struct adf_user_cfg_ctl_data *adf_ctl_alloc_resources(unsigned long arg) { struct adf_user_cfg_ctl_data *cfg_data; - cfg_data = kzalloc(sizeof(*cfg_data), GFP_KERNEL); - if (!cfg_data) - return -ENOMEM; - - /* Initialize device id to NO DEVICE as 0 is a valid device id */ - cfg_data->device_id = ADF_CFG_NO_DEVICE; - - if (copy_from_user(cfg_data, (void __user *)arg, sizeof(*cfg_data))) { + cfg_data = memdup_user((void __user *)arg, sizeof(*cfg_data)); + if (IS_ERR(cfg_data)) pr_err("QAT: failed to copy from user cfg_data.\n"); - kfree(cfg_data); - return -EIO; - } - - *ctl_data = cfg_data; - return 0; + return cfg_data; } static int adf_add_key_value_data(struct adf_accel_dev *accel_dev, @@ -188,13 +176,13 @@ out_err: static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd, unsigned long arg) { - int ret; struct adf_user_cfg_ctl_data *ctl_data; struct adf_accel_dev *accel_dev; + int ret = 0; - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; + ctl_data = adf_ctl_alloc_resources(arg); + if (IS_ERR(ctl_data)) + return PTR_ERR(ctl_data); accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); if (!accel_dev) { @@ -267,9 +255,9 @@ static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd, int ret; struct adf_user_cfg_ctl_data *ctl_data; - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; + ctl_data = adf_ctl_alloc_resources(arg); + if (IS_ERR(ctl_data)) + return PTR_ERR(ctl_data); if (adf_devmgr_verify_id(ctl_data->device_id)) { pr_err("QAT: Device %d not found\n", ctl_data->device_id); @@ -301,9 +289,9 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, struct adf_user_cfg_ctl_data *ctl_data; struct adf_accel_dev *accel_dev; - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; + ctl_data = adf_ctl_alloc_resources(arg); + if (IS_ERR(ctl_data)) + return PTR_ERR(ctl_data); ret = -ENODEV; accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); @@ -475,4 +463,4 @@ MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_ALIAS_CRYPTO("intel_qat"); MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_INTERNAL); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c b/drivers/crypto/intel/qat/qat_common/adf_dc.c index 47261b1c1da6..3e8fb4e3ed97 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.c @@ -1,22 +1,21 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2022 Intel Corporation */ #include "adf_accel_devices.h" -#include "adf_gen2_dc.h" +#include "adf_dc.h" #include "icp_qat_fw_comp.h" -static void qat_comp_build_deflate_ctx(void *ctx) +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo) { - struct icp_qat_fw_comp_req *req_tmpl = (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; + struct icp_qat_fw_comp_req *req_tmpl = ctx; struct icp_qat_fw_comp_cd_hdr *comp_cd_ctrl = &req_tmpl->comp_cd_ctrl; + struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + int ret; memset(req_tmpl, 0, sizeof(*req_tmpl)); header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, QAT_COMN_PTR_TYPE_SGL); @@ -26,12 +25,14 @@ static void qat_comp_build_deflate_ctx(void *ctx) ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + /* Build HW config block for compression */ + ret = GET_DC_OPS(accel_dev)->build_comp_block(ctx, algo); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to build compression block\n"); + return ret; + } + req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; req_pars->req_par_flags = @@ -45,26 +46,19 @@ static void qat_comp_build_deflate_ctx(void *ctx) ICP_QAT_FW_COMP_NO_XXHASH_ACC, ICP_QAT_FW_COMP_CNV_ERROR_NONE, ICP_QAT_FW_COMP_NO_APPEND_CRC, - ICP_QAT_FW_COMP_NO_DROP_DATA); + ICP_QAT_FW_COMP_NO_DROP_DATA, + ICP_QAT_FW_COMP_NO_PARTIAL_DECOMPRESS); ICP_QAT_FW_COMN_NEXT_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); ICP_QAT_FW_COMN_CURR_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_COMP); /* Fill second half of the template for decompression */ memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); -} -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate_ctx; + /* Build HW config block for decompression */ + ret = GET_DC_OPS(accel_dev)->build_decomp_block(req_tmpl, algo); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to build decompression block\n"); + + return ret; } -EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dc.h b/drivers/crypto/intel/qat/qat_common/adf_dc.h new file mode 100644 index 000000000000..6cb5e09054a6 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_DC_H +#define ADF_DC_H + +struct adf_accel_dev; + +enum adf_dc_algo { + QAT_DEFLATE, + QAT_LZ4, + QAT_LZ4S, + QAT_ZSTD, +}; + +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo); + +#endif /* ADF_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_config.h b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h index 4f86696800c9..78957fa900b7 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_fw_config.h +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h @@ -8,6 +8,7 @@ enum adf_fw_objs { ADF_FW_ASYM_OBJ, ADF_FW_DC_OBJ, ADF_FW_ADMIN_OBJ, + ADF_FW_CY_OBJ, }; struct adf_fw_config { diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h deleted file mode 100644 index 6eae023354d7..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN2_DC_H -#define ADF_GEN2_DC_H - -#include "adf_accel_devices.h" - -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN2_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c index 1f64bf49b221..6a505e9a5cf9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_common_drv.h" +#include "adf_dc.h" #include "adf_gen2_hw_data.h" +#include "icp_qat_fw_comp.h" #include "icp_qat_hw.h" #include <linux/pci.h> @@ -115,8 +117,8 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fuses = hw_data->fuses[ADF_FUSECTL0]; u32 straps = hw_data->straps; - u32 fuses = hw_data->fuses; u32 legfuses; u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | @@ -169,3 +171,58 @@ void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) } } EXPORT_SYMBOL_GPL(adf_gen2_set_ssm_wdtimer); + +static int adf_gen2_build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +static int adf_gen2_build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = adf_gen2_build_comp_block; + dc_ops->build_decomp_block = adf_gen2_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h index 708e9186127b..59bad368a921 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h @@ -88,5 +88,6 @@ void adf_gen2_get_arb_info(struct arb_info *arb_info); void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev); u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h index a716545a764c..34a63cf40db2 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h @@ -5,6 +5,7 @@ #include <linux/types.h> #include "adf_accel_devices.h" +#include "adf_common_drv.h" #define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) #define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c index fe1f3d727dc5..afcdfdd0a37a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c @@ -11,7 +11,7 @@ #include "qat_compression.h" #include "qat_crypto.h" -static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; int banks = GET_MAX_BANKS(accel_dev); @@ -117,7 +117,7 @@ err: return ret; } -static int adf_comp_dev_config(struct adf_accel_dev *accel_dev) +int adf_comp_dev_config(struct adf_accel_dev *accel_dev) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; int banks = GET_MAX_BANKS(accel_dev); @@ -187,7 +187,7 @@ err: return ret; } -static int adf_no_dev_config(struct adf_accel_dev *accel_dev) +int adf_no_dev_config(struct adf_accel_dev *accel_dev) { unsigned long val; int ret; @@ -213,7 +213,6 @@ static int adf_no_dev_config(struct adf_accel_dev *accel_dev) */ int adf_gen4_dev_config(struct adf_accel_dev *accel_dev) { - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; int ret; ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); @@ -224,18 +223,8 @@ int adf_gen4_dev_config(struct adf_accel_dev *accel_dev) if (ret) goto err; - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - if (ret) - goto err; - - ret = sysfs_match_string(adf_cfg_services, services); - if (ret < 0) - goto err; - - switch (ret) { - case SVC_CY: - case SVC_CY2: + switch (adf_get_service_enabled(accel_dev)) { + case SVC_SYM_ASYM: ret = adf_crypto_dev_config(accel_dev); break; case SVC_DC: diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h index bb87655f69a8..38a674c27e40 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h @@ -7,5 +7,8 @@ int adf_gen4_dev_config(struct adf_accel_dev *accel_dev); int adf_gen4_cfg_dev_init(struct adf_accel_dev *accel_dev); +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev); +int adf_comp_dev_config(struct adf_accel_dev *accel_dev); +int adf_no_dev_config(struct adf_accel_dev *accel_dev); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c deleted file mode 100644 index 5859238e37de..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include "adf_accel_devices.h" -#include "icp_qat_fw_comp.h" -#include "icp_qat_hw_20_comp.h" -#include "adf_gen4_dc.h" - -static void qat_comp_build_deflate(void *ctx) -{ - struct icp_qat_fw_comp_req *req_tmpl = - (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; - struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = {0}; - struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = {0}; - struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = {0}; - u32 upper_val; - u32 lower_val; - - memset(req_tmpl, 0, sizeof(*req_tmpl)); - header->hdr_flags = - ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); - header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; - header->comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, - QAT_COMN_PTR_TYPE_SGL); - header->serv_specif_flags = - ICP_QAT_FW_COMP_FLAGS_BUILD(ICP_QAT_FW_COMP_STATELESS_SESSION, - ICP_QAT_FW_COMP_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; - hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; - hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; - hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; - hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; - hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; - hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; - hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; - - upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); - lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; - - req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; - req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; - req_pars->req_par_flags = - ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(ICP_QAT_FW_COMP_SOP, - ICP_QAT_FW_COMP_EOP, - ICP_QAT_FW_COMP_BFINAL, - ICP_QAT_FW_COMP_CNV, - ICP_QAT_FW_COMP_CNV_RECOVERY, - ICP_QAT_FW_COMP_NO_CNV_DFX, - ICP_QAT_FW_COMP_CRC_MODE_LEGACY, - ICP_QAT_FW_COMP_NO_XXHASH_ACC, - ICP_QAT_FW_COMP_CNV_ERROR_NONE, - ICP_QAT_FW_COMP_NO_APPEND_CRC, - ICP_QAT_FW_COMP_NO_DROP_DATA); - - /* Fill second half of the template for decompression */ - memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); - req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - - hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; - lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = 0; -} - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate; -} -EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h deleted file mode 100644 index 0b1a6774412e..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN4_DC_H -#define ADF_GEN4_DC_H - -#include "adf_accel_devices.h" - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN4_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c index 41a0979e68c1..349fdb323763 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c @@ -1,5 +1,9 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + +#include <linux/bitops.h> #include <linux/iopoll.h> #include <asm/div64.h> #include "adf_accel_devices.h" @@ -8,6 +12,8 @@ #include "adf_fw_config.h" #include "adf_gen4_hw_data.h" #include "adf_gen4_pm.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw_20_comp.h" u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self) { @@ -134,36 +140,18 @@ int adf_gen4_init_device(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_gen4_init_device); -static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper, - u32 *lower) -{ - *lower = lower_32_bits(value); - *upper = upper_32_bits(value); -} - void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; - u32 ssm_wdt_pke_high = 0; - u32 ssm_wdt_pke_low = 0; - u32 ssm_wdt_high = 0; - u32 ssm_wdt_low = 0; - /* Convert 64bit WDT timer value into 32bit values for - * mmio write to 32bit CSRs. - */ - adf_gen4_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low); - adf_gen4_unpack_ssm_wdtimer(timer_val_pke, &ssm_wdt_pke_high, - &ssm_wdt_pke_low); - - /* Enable WDT for sym and dc */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTL_OFFSET, ssm_wdt_low); - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTH_OFFSET, ssm_wdt_high); - /* Enable WDT for pke */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ssm_wdt_pke_low); - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high); + /* Enable watchdog timer for sym and dc */ + ADF_CSR_WR64_LO_HI(pmisc_addr, ADF_SSMWDTL_OFFSET, ADF_SSMWDTH_OFFSET, timer_val); + + /* Enable watchdog timer for pke */ + ADF_CSR_WR64_LO_HI(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, + timer_val_pke); } EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer); @@ -265,18 +253,32 @@ static bool is_single_service(int service_id) case SVC_SYM: case SVC_ASYM: return true; - case SVC_CY: - case SVC_CY2: - case SVC_DCC: - case SVC_ASYM_DC: - case SVC_DC_ASYM: - case SVC_SYM_DC: - case SVC_DC_SYM: default: return false; } } +bool adf_gen4_services_supported(unsigned long mask) +{ + unsigned long num_svc = hweight_long(mask); + + if (mask >= BIT(SVC_COUNT)) + return false; + + if (test_bit(SVC_DECOMP, &mask)) + return false; + + switch (num_svc) { + case ADF_ONE_SERVICE: + return true; + case ADF_TWO_SERVICES: + return !test_bit(SVC_DCC, &mask); + default: + return false; + } +} +EXPORT_SYMBOL_GPL(adf_gen4_services_supported); + int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); @@ -489,183 +491,110 @@ int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, return ret; } -static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, - u32 bank, struct bank_state *state, u32 num_rings) +static int adf_gen4_build_comp_block(void *ctx, enum adf_dc_algo algo) { - u32 i; - - state->ringstat0 = ops->read_csr_stat(base, bank); - state->ringuostat = ops->read_csr_uo_stat(base, bank); - state->ringestat = ops->read_csr_e_stat(base, bank); - state->ringnestat = ops->read_csr_ne_stat(base, bank); - state->ringnfstat = ops->read_csr_nf_stat(base, bank); - state->ringfstat = ops->read_csr_f_stat(base, bank); - state->ringcstat0 = ops->read_csr_c_stat(base, bank); - state->iaintflagen = ops->read_csr_int_en(base, bank); - state->iaintflagreg = ops->read_csr_int_flag(base, bank); - state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank); - state->iaintcolen = ops->read_csr_int_col_en(base, bank); - state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank); - state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank); - state->ringexpstat = ops->read_csr_exp_stat(base, bank); - state->ringexpintenable = ops->read_csr_exp_int_en(base, bank); - state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank); - - for (i = 0; i < num_rings; i++) { - state->rings[i].head = ops->read_csr_ring_head(base, bank, i); - state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i); - state->rings[i].config = ops->read_csr_ring_config(base, bank, i); - state->rings[i].base = ops->read_csr_ring_base(base, bank, i); - } -} - -#define CHECK_STAT(op, expect_val, name, args...) \ -({ \ - u32 __expect_val = (expect_val); \ - u32 actual_val = op(args); \ - (__expect_val == actual_val) ? 0 : \ - (pr_err("QAT: Fail to restore %s register. Expected 0x%x, actual 0x%x\n", \ - name, __expect_val, actual_val), -EINVAL); \ -}) - -static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base, - u32 bank, struct bank_state *state, u32 num_rings, - int tx_rx_gap) -{ - u32 val, tmp_val, i; - int ret; - - for (i = 0; i < num_rings; i++) - ops->write_csr_ring_base(base, bank, i, state->rings[i].base); - - for (i = 0; i < num_rings; i++) - ops->write_csr_ring_config(base, bank, i, state->rings[i].config); - - for (i = 0; i < num_rings / 2; i++) { - int tx = i * (tx_rx_gap + 1); - int rx = tx + tx_rx_gap; - - ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); - ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail); - - /* - * The TX ring head needs to be updated again to make sure that - * the HW will not consider the ring as full when it is empty - * and the correct state flags are set to match the recovered state. - */ - if (state->ringestat & BIT(tx)) { - val = ops->read_csr_int_srcsel(base, bank); - val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK; - ops->write_csr_int_srcsel_w_val(base, bank, val); - ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); - } - - ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); - val = ops->read_csr_int_srcsel(base, bank); - val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; - ops->write_csr_int_srcsel_w_val(base, bank, val); - - ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head); - val = ops->read_csr_int_srcsel(base, bank); - val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; - ops->write_csr_int_srcsel_w_val(base, bank, val); - - /* - * The RX ring tail needs to be updated again to make sure that - * the HW will not consider the ring as empty when it is full - * and the correct state flags are set to match the recovered state. - */ - if (state->ringfstat & BIT(rx)) - ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = { }; + struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = { }; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 upper_val; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC; + break; + default: + return -EINVAL; } - ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen); - ops->write_csr_int_en(base, bank, state->iaintflagen); - ops->write_csr_int_col_en(base, bank, state->iaintcolen); - ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0); - ops->write_csr_exp_int_en(base, bank, state->ringexpintenable); - ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl); - ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben); - - /* Check that all ring statuses match the saved state. */ - ret = CHECK_STAT(ops->read_csr_stat, state->ringstat0, "ringstat", - base, bank); - if (ret) - return ret; - - ret = CHECK_STAT(ops->read_csr_e_stat, state->ringestat, "ringestat", - base, bank); - if (ret) - return ret; + hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; + hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; + hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; + hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; + hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; + hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; + hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; - ret = CHECK_STAT(ops->read_csr_ne_stat, state->ringnestat, "ringnestat", - base, bank); - if (ret) - return ret; + upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); + lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); - ret = CHECK_STAT(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat", - base, bank); - if (ret) - return ret; - - ret = CHECK_STAT(ops->read_csr_f_stat, state->ringfstat, "ringfstat", - base, bank); - if (ret) - return ret; - - ret = CHECK_STAT(ops->read_csr_c_stat, state->ringcstat0, "ringcstat", - base, bank); - if (ret) - return ret; - - tmp_val = ops->read_csr_exp_stat(base, bank); - val = state->ringexpstat; - if (tmp_val && !val) { - pr_err("QAT: Bank was restored with exception: 0x%x\n", val); - return -EINVAL; - } + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; return 0; } -int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, - struct bank_state *state) +static int adf_gen4_build_decomp_block(void *ctx, enum adf_dc_algo algo) { - struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); - void __iomem *csr_base = adf_get_etr_base(accel_dev); - - if (bank_number >= hw_data->num_banks || !state) + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = { }; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: return -EINVAL; + } - dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number); + hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; + lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); - bank_state_save(csr_ops, csr_base, bank_number, state, - hw_data->num_rings_per_bank); + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; return 0; } -EXPORT_SYMBOL_GPL(adf_gen4_bank_state_save); -int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, - struct bank_state *state) +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) { - struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); - void __iomem *csr_base = adf_get_etr_base(accel_dev); - int ret; + dc_ops->build_comp_block = adf_gen4_build_comp_block; + dc_ops->build_decomp_block = adf_gen4_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); - if (bank_number >= hw_data->num_banks || !state) - return -EINVAL; +void adf_gen4_init_num_svc_aes(struct adf_rl_hw_data *device_data) +{ + struct adf_hw_device_data *hw_data; + unsigned int i; + u32 ae_cnt; - dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number); + hw_data = container_of(device_data, struct adf_hw_device_data, rl_data); + ae_cnt = hweight32(hw_data->get_ae_mask(hw_data)); + if (!ae_cnt) + return; - ret = bank_state_restore(csr_ops, csr_base, bank_number, state, - hw_data->num_rings_per_bank, hw_data->tx_rx_gap); - if (ret) - dev_err(&GET_DEV(accel_dev), - "Unable to restore state of bank %d\n", bank_number); + for (i = 0; i < SVC_BASE_COUNT; i++) + device_data->svc_ae_mask[i] = ae_cnt - 1; - return ret; + /* + * The decompression service is not supported on QAT GEN4 devices. + * Therefore, set svc_ae_mask to 0. + */ + device_data->svc_ae_mask[SVC_DECOMP] = 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_num_svc_aes); + +u32 adf_gen4_get_svc_slice_cnt(struct adf_accel_dev *accel_dev, + enum adf_base_services svc) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + + switch (svc) { + case SVC_SYM: + return device_data->slices.cph_cnt; + case SVC_ASYM: + return device_data->slices.pke_cnt; + case SVC_DC: + return device_data->slices.dcpr_cnt; + default: + return 0; + } } -EXPORT_SYMBOL_GPL(adf_gen4_bank_state_restore); +EXPORT_SYMBOL_GPL(adf_gen4_get_svc_slice_cnt); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h index e8c53bd76f1b..cd26b6724c43 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -7,6 +7,7 @@ #include "adf_accel_devices.h" #include "adf_cfg_common.h" +#include "adf_dc.h" /* PCIe configuration space */ #define ADF_GEN4_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) @@ -83,9 +84,6 @@ #define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) /* Ring interrupt */ -#define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0) -#define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0) -#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4 #define ADF_COALESCED_POLL_TIMEOUT_US (1 * USEC_PER_SEC) #define ADF_COALESCED_POLL_DELAY_US 1000 #define ADF_WQM_CSR_RPINTSOU(bank) (0x200000 + ((bank) << 12)) @@ -175,9 +173,10 @@ int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, u32 bank_number, int timeout_us); void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, u32 bank_number); -int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, - struct bank_state *state); -int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, - u32 bank_number, struct bank_state *state); +bool adf_gen4_services_supported(unsigned long service_mask); +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); +void adf_gen4_init_num_svc_aes(struct adf_rl_hw_data *device_data); +u32 adf_gen4_get_svc_slice_cnt(struct adf_accel_dev *accel_dev, + enum adf_base_services svc); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h index 17d1b774d4a8..2c8708117f70 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h @@ -4,6 +4,7 @@ #define ADF_GEN4_PFVF_H #include "adf_accel_devices.h" +#include "adf_common_drv.h" #ifdef CONFIG_PCI_IOV void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c index 2e4095c4c12c..b7e38842a46d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c @@ -1,47 +1,18 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2023 Intel Corporation */ #include <linux/dma-mapping.h> -#include <linux/kernel.h> #include <linux/string_helpers.h> -#include <linux/stringify.h> #include "adf_accel_devices.h" #include "adf_admin.h" #include "adf_common_drv.h" #include "adf_gen4_pm.h" +#include "adf_pm_dbgfs_utils.h" #include "icp_qat_fw_init_admin.h" -/* - * This is needed because a variable is used to index the mask at - * pm_scnprint_table(), making it not compile time constant, so the compile - * asserts from FIELD_GET() or u32_get_bits() won't be fulfilled. - */ -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) - -#define PM_INFO_MEMBER_OFF(member) \ - (offsetof(struct icp_qat_fw_init_admin_pm_info, member) / sizeof(u32)) - -#define PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, _mask_) \ -{ \ - .reg_offset = PM_INFO_MEMBER_OFF(_reg_), \ - .key = __stringify(_field_), \ - .field_mask = _mask_, \ -} - -#define PM_INFO_REGSET_ENTRY32(_reg_, _field_) \ - PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, GENMASK(31, 0)) - #define PM_INFO_REGSET_ENTRY(_reg_, _field_) \ PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, ADF_GEN4_PM_##_field_##_MASK) -#define PM_INFO_MAX_KEY_LEN 21 - -struct pm_status_row { - int reg_offset; - u32 field_mask; - const char *key; -}; - static const struct pm_status_row pm_fuse_rows[] = { PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM), PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM_IDLE), @@ -109,44 +80,6 @@ static const struct pm_status_row pm_csrs_rows[] = { PM_INFO_REGSET_ENTRY32(pm.pwrreq, CPM_PM_PWRREQ), }; -static int pm_scnprint_table(char *buff, const struct pm_status_row *table, - u32 *pm_info_regs, size_t buff_size, int table_len, - bool lowercase) -{ - char key[PM_INFO_MAX_KEY_LEN]; - int wr = 0; - int i; - - for (i = 0; i < table_len; i++) { - if (lowercase) - string_lower(key, table[i].key); - else - string_upper(key, table[i].key); - - wr += scnprintf(&buff[wr], buff_size - wr, "%s: %#x\n", key, - field_get(table[i].field_mask, - pm_info_regs[table[i].reg_offset])); - } - - return wr; -} - -static int pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table, - u32 *pm_info_regs, size_t buff_size, - int table_len) -{ - return pm_scnprint_table(buff, table, pm_info_regs, buff_size, - table_len, false); -} - -static int pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table, - u32 *pm_info_regs, size_t buff_size, - int table_len) -{ - return pm_scnprint_table(buff, table, pm_info_regs, buff_size, - table_len, true); -} - static_assert(sizeof(struct icp_qat_fw_init_admin_pm_info) < PAGE_SIZE); static ssize_t adf_gen4_print_pm_status(struct adf_accel_dev *accel_dev, @@ -191,9 +124,9 @@ static ssize_t adf_gen4_print_pm_status(struct adf_accel_dev *accel_dev, /* Fusectl related */ len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "----------- PM Fuse info ---------\n"); - len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows, - pm_info_regs, PAGE_SIZE - len, - ARRAY_SIZE(pm_fuse_rows)); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_fuse_rows)); len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "max_pwrreq: %#x\n", pm_info->max_pwrreq); len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "min_pwrreq: %#x\n", @@ -204,28 +137,28 @@ static ssize_t adf_gen4_print_pm_status(struct adf_accel_dev *accel_dev, "------------ PM Info ------------\n"); len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "power_level: %s\n", pm_info->pwr_state == PM_SET_MIN ? "min" : "max"); - len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows, - pm_info_regs, PAGE_SIZE - len, - ARRAY_SIZE(pm_info_rows)); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_info_rows)); len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "pm_mode: STATIC\n"); /* SSM related */ len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "----------- SSM_PM Info ----------\n"); - len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows, - pm_info_regs, PAGE_SIZE - len, - ARRAY_SIZE(pm_ssm_rows)); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_ssm_rows)); /* Log related */ len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "------------- PM Log -------------\n"); - len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_log_rows, - pm_info_regs, PAGE_SIZE - len, - ARRAY_SIZE(pm_log_rows)); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_log_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_log_rows)); - len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_event_rows, - pm_info_regs, PAGE_SIZE - len, - ARRAY_SIZE(pm_event_rows)); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_event_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_event_rows)); len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "idle_irq_count: %#x\n", pm->idle_irq_counters); @@ -241,9 +174,9 @@ static ssize_t adf_gen4_print_pm_status(struct adf_accel_dev *accel_dev, /* CSRs content */ len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "----------- HW PM CSRs -----------\n"); - len += pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows, - pm_info_regs, PAGE_SIZE - len, - ARRAY_SIZE(pm_csrs_rows)); + len += adf_pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_csrs_rows)); val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_HOST_MSG); len += scnprintf(&pm_kv[len], PAGE_SIZE - len, diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c index 2dd3772bf58a..0f7f00a19e7d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c @@ -695,7 +695,7 @@ static bool adf_handle_slice_hang_error(struct adf_accel_dev *accel_dev, if (err_mask->parerr_wat_wcp_mask) adf_poll_slicehang_csr(accel_dev, csr, ADF_GEN4_SLICEHANGSTATUS_WAT_WCP, - "ath_cph"); + "wat_wcp"); return false; } @@ -1043,63 +1043,16 @@ static bool adf_handle_ssmcpppar_err(struct adf_accel_dev *accel_dev, return reset_required; } -static bool adf_handle_rf_parr_err(struct adf_accel_dev *accel_dev, +static void adf_handle_rf_parr_err(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 iastatssm) { - struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); - u32 reg; - if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_SSMSOFTERRORPARITY_BIT)) - return false; - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_SRC); - reg &= ADF_GEN4_SSMSOFTERRORPARITY_SRC_BIT; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_SRC, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_ATH_CPH); - reg &= err_mask->parerr_ath_cph_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_ATH_CPH, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_CPR_XLT); - reg &= err_mask->parerr_cpr_xlt_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_CPR_XLT, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_DCPR_UCS); - reg &= err_mask->parerr_dcpr_ucs_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_DCPR_UCS, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_PKE); - reg &= err_mask->parerr_pke_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_PKE, reg); - } - - if (err_mask->parerr_wat_wcp_mask) { - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_WAT_WCP); - reg &= err_mask->parerr_wat_wcp_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_WAT_WCP, - reg); - } - } + return; + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); dev_err(&GET_DEV(accel_dev), "Slice ssm soft parity error reported"); - return false; + return; } static bool adf_handle_ser_err_ssmsh(struct adf_accel_dev *accel_dev, @@ -1171,8 +1124,8 @@ static bool adf_handle_iaintstatssm(struct adf_accel_dev *accel_dev, reset_required |= adf_handle_slice_hang_error(accel_dev, csr, iastatssm); reset_required |= adf_handle_spppar_err(accel_dev, csr, iastatssm); reset_required |= adf_handle_ssmcpppar_err(accel_dev, csr, iastatssm); - reset_required |= adf_handle_rf_parr_err(accel_dev, csr, iastatssm); reset_required |= adf_handle_ser_err_ssmsh(accel_dev, csr, iastatssm); + adf_handle_rf_parr_err(accel_dev, csr, iastatssm); ADF_CSR_WR(csr, ADF_GEN4_IAINTSTATSSM, iastatssm); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c index a62eb5e8dbe6..adb21656a3ba 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c @@ -9,6 +9,7 @@ #include <asm/errno.h> #include "adf_accel_devices.h" +#include "adf_bank_state.h" #include "adf_common_drv.h" #include "adf_gen4_hw_data.h" #include "adf_gen4_pfvf.h" @@ -358,7 +359,7 @@ static int adf_gen4_vfmig_load_etr_regs(struct adf_mstate_mgr *sub_mgr, pf_bank_nr = vf_bank_info->bank_nr + vf_bank_info->vf_nr * hw_data->num_banks_per_vf; ret = hw_data->bank_state_restore(accel_dev, pf_bank_nr, - (struct bank_state *)state); + (struct adf_bank_state *)state); if (ret) { dev_err(&GET_DEV(accel_dev), "Failed to load regs for vf%d bank%d\n", @@ -585,7 +586,7 @@ static int adf_gen4_vfmig_save_etr_regs(struct adf_mstate_mgr *subs, u8 *state, pf_bank_nr += vf_bank_info->vf_nr * hw_data->num_banks_per_vf; ret = hw_data->bank_state_save(accel_dev, pf_bank_nr, - (struct bank_state *)state); + (struct adf_bank_state *)state); if (ret) { dev_err(&GET_DEV(accel_dev), "Failed to save regs for vf%d bank%d\n", @@ -593,7 +594,7 @@ static int adf_gen4_vfmig_save_etr_regs(struct adf_mstate_mgr *subs, u8 *state, return ret; } - return sizeof(struct bank_state); + return sizeof(struct adf_bank_state); } static int adf_gen4_vfmig_save_etr_bank(struct adf_accel_dev *accel_dev, diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h new file mode 100644 index 000000000000..4c0d576e8c21 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_PM_H +#define ADF_GEN6_PM_H + +#include <linux/bits.h> +#include <linux/time.h> + +struct adf_accel_dev; + +/* Power management */ +#define ADF_GEN6_PM_POLL_DELAY_US 20 +#define ADF_GEN6_PM_POLL_TIMEOUT_US USEC_PER_SEC +#define ADF_GEN6_PM_STATUS 0x50A00C +#define ADF_GEN6_PM_INTERRUPT 0x50A028 + +/* Power management source in ERRSOU2 and ERRMSK2 */ +#define ADF_GEN6_PM_SOU BIT(18) + +/* cpm_pm_interrupt bitfields */ +#define ADF_GEN6_PM_DRV_ACTIVE BIT(20) + +#define ADF_GEN6_PM_DEFAULT_IDLE_FILTER 0x6 + +/* cpm_pm_status bitfields */ +#define ADF_GEN6_PM_INIT_STATE BIT(21) +#define ADF_GEN6_PM_CPM_PM_STATE_MASK GENMASK(22, 20) + +/* fusectl0 bitfields */ +#define ADF_GEN6_PM_ENABLE_PM_MASK BIT(21) +#define ADF_GEN6_PM_ENABLE_PM_IDLE_MASK BIT(22) +#define ADF_GEN6_PM_ENABLE_DEEP_PM_IDLE_MASK BIT(23) + +/* cpm_pm_fw_init bitfields */ +#define ADF_GEN6_PM_IDLE_FILTER_MASK GENMASK(5, 3) +#define ADF_GEN6_PM_IDLE_ENABLE_MASK BIT(2) + +/* ssm_pm_enable bitfield */ +#define ADF_GEN6_PM_SSM_PM_ENABLE_MASK BIT(0) + +/* ssm_pm_domain_status bitfield */ +#define ADF_GEN6_PM_DOMAIN_POWERED_UP_MASK BIT(0) + +#ifdef CONFIG_DEBUG_FS +void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev); +#else +static inline void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#endif /* ADF_GEN6_PM_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c new file mode 100644 index 000000000000..603aefba0fdb --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include <linux/dma-mapping.h> +#include <linux/export.h> +#include <linux/string_helpers.h> + +#include "adf_admin.h" +#include "adf_common_drv.h" +#include "adf_gen6_pm.h" +#include "adf_pm_dbgfs_utils.h" +#include "icp_qat_fw_init_admin.h" + +#define PM_INFO_REGSET_ENTRY(_reg_, _field_) \ + PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, ADF_GEN6_PM_##_field_##_MASK) + +static struct pm_status_row pm_fuse_rows[] = { + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM), + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM_IDLE), + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_DEEP_PM_IDLE), +}; + +static struct pm_status_row pm_info_rows[] = { + PM_INFO_REGSET_ENTRY(pm.status, CPM_PM_STATE), + PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_ENABLE), + PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_FILTER), +}; + +static struct pm_status_row pm_ssm_rows[] = { + PM_INFO_REGSET_ENTRY(ssm.pm_enable, SSM_PM_ENABLE), + PM_INFO_REGSET_ENTRY(ssm.pm_domain_status, DOMAIN_POWERED_UP), +}; + +static struct pm_status_row pm_csrs_rows[] = { + PM_INFO_REGSET_ENTRY32(pm.fw_init, CPM_PM_FW_INIT), + PM_INFO_REGSET_ENTRY32(pm.status, CPM_PM_STATUS), +}; + +static_assert(sizeof(struct icp_qat_fw_init_admin_pm_info) < PAGE_SIZE); + +static ssize_t adf_gen6_print_pm_status(struct adf_accel_dev *accel_dev, + char __user *buf, size_t count, + loff_t *pos) +{ + void __iomem *pmisc = adf_get_pmisc_base(accel_dev); + struct icp_qat_fw_init_admin_pm_info *pm_info; + dma_addr_t p_state_addr; + u32 *pm_info_regs; + size_t len = 0; + char *pm_kv; + u32 val; + int ret; + + pm_info = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!pm_info) + return -ENOMEM; + + pm_kv = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!pm_kv) { + kfree(pm_info); + return -ENOMEM; + } + + p_state_addr = dma_map_single(&GET_DEV(accel_dev), pm_info, PAGE_SIZE, + DMA_FROM_DEVICE); + ret = dma_mapping_error(&GET_DEV(accel_dev), p_state_addr); + if (ret) + goto out_free; + + /* Query power management information from QAT FW */ + ret = adf_get_pm_info(accel_dev, p_state_addr, PAGE_SIZE); + dma_unmap_single(&GET_DEV(accel_dev), p_state_addr, PAGE_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto out_free; + + pm_info_regs = (u32 *)pm_info; + + /* Fuse control register */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- PM Fuse info ---------\n"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_fuse_rows)); + + /* Power management */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- PM Info --------------\n"); + + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_info_rows)); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "pm_mode: ACTIVE\n"); + + /* Shared Slice Module */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- SSM_PM Info ----------\n"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_ssm_rows)); + + /* Control status register content */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- HW PM CSRs -----------\n"); + len += adf_pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_csrs_rows)); + + val = ADF_CSR_RD(pmisc, ADF_GEN6_PM_INTERRUPT); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "CPM_PM_INTERRUPT: %#x\n", val); + ret = simple_read_from_buffer(buf, count, pos, pm_kv, len); + +out_free: + kfree(pm_info); + kfree(pm_kv); + + return ret; +} + +void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev) +{ + accel_dev->power_management.print_pm_status = adf_gen6_print_pm_status; + accel_dev->power_management.present = true; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_dev_pm_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c new file mode 100644 index 000000000000..967253082a98 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c @@ -0,0 +1,818 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include <linux/bitfield.h> +#include <linux/types.h> + +#include "adf_common_drv.h" +#include "adf_gen6_ras.h" +#include "adf_sysfs_ras_counters.h" + +static void enable_errsou_reporting(void __iomem *csr) +{ + /* Enable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, 0); + + /* Enable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, 0); + + /* + * Enable uncorrectable error reporting in ERRSOU2 + * but disable PM interrupt by default + */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, ADF_GEN6_ERRSOU2_PM_INT_BIT); + + /* Enable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, 0); +} + +static void enable_ae_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; + + /* Enable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, ae_mask); + + /* Enable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, ae_mask); +} + +static void enable_cpp_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + /* Enable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, + ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_MASK); +} + +static void enable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg, mask; + + /* Enable RI memory error reporting */ + mask = ADF_GEN6_RIMEM_PARERR_FATAL_MASK | ADF_GEN6_RIMEM_PARERR_CERR_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, mask); + + /* Enable IOSF primary command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, ADF_GEN6_RIMISCSTS_BIT); + + /* Enable TI internal memory parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CI_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CD_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, reg); + + /* Enable error handling in RI, TI CPP interface control registers */ + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, ADF_GEN6_RICPPINTCTL_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, ADF_GEN6_TICPPINTCTL_MASK); + + /* + * Enable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + reg |= ADF_GEN6_TIMISCCTL_BIT; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void enable_ssm_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + /* Enable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, 0); +} + +static void adf_gen6_enable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + enable_errsou_reporting(csr); + enable_ae_error_reporting(accel_dev, csr); + enable_cpp_error_reporting(accel_dev, csr); + enable_ti_ri_error_reporting(csr); + enable_ssm_error_reporting(accel_dev, csr); +} + +static void disable_errsou_reporting(void __iomem *csr) +{ + u32 val; + + /* Disable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, ADF_GEN6_ERRSOU0_MASK); + + /* Disable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, ADF_GEN6_ERRMSK1_MASK); + + /* Disable uncorrectable error reporting in ERRSOU2 */ + val = ADF_CSR_RD(csr, ADF_GEN6_ERRMSK2); + val |= ADF_GEN6_ERRSOU2_DIS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, val); + + /* Disable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_ERRSOU3_DIS_MASK); +} + +static void disable_ae_error_reporting(void __iomem *csr) +{ + /* Disable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, 0); + + /* Disable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, 0); +} + +static void disable_cpp_error_reporting(void __iomem *csr) +{ + /* Disable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, 0); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK); +} + +static void disable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg; + + /* Disable RI memory error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, 0); + + /* Disable IOSF primary command parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RIMISCCTL); + reg &= ~ADF_GEN6_RIMISCSTS_BIT; + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, reg); + + /* Disable TI internal memory parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, ADF_GEN6_TI_CI_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, ADF_GEN6_TI_CD_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, ADF_GEN6_TI_TRNSB_PAR_STS_MASK); + + /* Disable error handling in RI, TI CPP interface control registers */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTCTL); + reg &= ~ADF_GEN6_RICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTCTL); + reg &= ~ADF_GEN6_TICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, reg); + + /* + * Disable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void disable_ssm_error_reporting(void __iomem *csr) +{ + /* Disable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, ADF_GEN6_INTMASKSSM_MASK); +} + +static void adf_gen6_disable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + disable_errsou_reporting(csr); + disable_ae_error_reporting(csr); + disable_cpp_error_reporting(csr); + disable_ti_ri_error_reporting(csr); + disable_ssm_error_reporting(csr); +} + +static void adf_gen6_process_errsou0(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae, errsou; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAECORERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + + dev_warn(&GET_DEV(accel_dev), "Correctable error detected: %#x\n", ae); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + + /* Clear interrupt from ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOG_CPP0, ae); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou0 still set: %#x\n", errsou); +} + +static void adf_handle_cpp_ae_unc(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ae; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT)) + return; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + if (ae) { + dev_err(&GET_DEV(accel_dev), "Uncorrectable error detected: %#x\n", ae); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0, ae); + } +} + +static void adf_handle_cpp_cmd_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 cmd_par_err; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT)) + return; + + cmd_par_err = ADF_CSR_RD(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG); + cmd_par_err &= ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK; + if (cmd_par_err) { + dev_err(&GET_DEV(accel_dev), "HI CPP agent command parity error: %#x\n", + cmd_par_err); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG, cmd_par_err); + } +} + +static void adf_handle_ri_mem_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rimem_parerr_sts; + + if (!(errsou & ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT)) + return; + + rimem_parerr_sts = ADF_CSR_RD(csr, ADF_GEN6_RIMEM_PARERR_STS); + rimem_parerr_sts &= ADF_GEN6_RIMEM_PARERR_CERR_MASK | + ADF_GEN6_RIMEM_PARERR_FATAL_MASK; + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_CERR_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity correctable error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + } + + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_FATAL_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity fatal error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_RIMEM_PARERR_STS, rimem_parerr_sts); +} + +static void adf_handle_ti_ci_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_ci_par_sts; + + ti_ci_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_STS); + ti_ci_par_sts &= ADF_GEN6_TI_CI_PAR_STS_MASK; + if (ti_ci_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI memory parity error: %#x\n", ti_ci_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_STS, ti_ci_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pullfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pullfub_par_sts; + + ti_pullfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS); + ti_pullfub_par_sts &= ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + if (ti_pullfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI pull parity error: %#x\n", ti_pullfub_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS, ti_pullfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pushfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pushfub_par_sts; + + ti_pushfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS); + ti_pushfub_par_sts &= ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + if (ti_pushfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI push parity error: %#x\n", ti_pushfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS, ti_pushfub_par_sts); + } +} + +static void adf_handle_ti_cd_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_cd_par_sts; + + ti_cd_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_STS); + ti_cd_par_sts &= ADF_GEN6_TI_CD_PAR_STS_MASK; + if (ti_cd_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI CD parity error: %#x\n", ti_cd_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_STS, ti_cd_par_sts); + } +} + +static void adf_handle_ti_trnsb_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_trnsb_par_sts; + + ti_trnsb_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_STS); + ti_trnsb_par_sts &= ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + if (ti_trnsb_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI TRNSB parity error: %#x\n", ti_trnsb_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_STS, ti_trnsb_par_sts); + } +} + +static void adf_handle_iosfp_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 rimiscsts; + + rimiscsts = ADF_CSR_RD(csr, ADF_GEN6_RIMISCSTS); + rimiscsts &= ADF_GEN6_RIMISCSTS_BIT; + if (rimiscsts) { + dev_err(&GET_DEV(accel_dev), "Command parity error detected on IOSFP: %#x\n", + rimiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_RIMISCSTS, rimiscsts); + } +} + +static void adf_handle_ti_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return; + + adf_handle_ti_ci_par_sts(accel_dev, csr); + adf_handle_ti_pullfub_par_sts(accel_dev, csr); + adf_handle_ti_pushfub_par_sts(accel_dev, csr); + adf_handle_ti_cd_par_sts(accel_dev, csr); + adf_handle_ti_trnsb_par_sts(accel_dev, csr); + adf_handle_iosfp_cmd_parerr(accel_dev, csr); +} + +static void adf_handle_sfi_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Command parity error detected on streaming fabric interface\n"); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou1(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_cpp_ae_unc(accel_dev, csr, errsou); + adf_handle_cpp_cmd_par_err(accel_dev, csr, errsou); + adf_handle_ri_mem_par_err(accel_dev, csr, errsou); + adf_handle_ti_err(accel_dev, csr, errsou); + adf_handle_sfi_cmd_parerr(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou1 still set: %#x\n", errsou); +} + +static void adf_handle_cerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CERRSSMSH); + reg &= ADF_GEN6_CERRSSMSH_ERROR_BIT; + if (reg) { + dev_warn(&GET_DEV(accel_dev), + "Correctable error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + ADF_CSR_WR(csr, ADF_GEN6_CERRSSMSH, reg); + } +} + +static void adf_handle_uerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_UERRSSMSH); + reg &= ADF_GEN6_UERRSSMSH_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_UERRSSMSH, reg); + } +} + +static void adf_handle_pperr_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_PPERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_PPERR); + reg &= ADF_GEN6_PPERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal push or pull data error: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_PPERR, reg); + } +} + +static void adf_handle_scmpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_SCM_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on SCM: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_cpppar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_CPP_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on CPP: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_rfpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_RF_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on RF Parity: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_unexp_cpl_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_UNEXP_CPL_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error for AXI unexpected tag/length: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_iaintstatssm(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 iastatssm = ADF_CSR_RD(csr, ADF_GEN6_IAINTSTATSSM); + + iastatssm &= ADF_GEN6_IAINTSTATSSM_MASK; + if (!iastatssm) + return; + + adf_handle_uerrssmsh(accel_dev, csr, iastatssm); + adf_handle_pperr_err(accel_dev, csr, iastatssm); + adf_handle_scmpar_err(accel_dev, csr, iastatssm); + adf_handle_cpppar_err(accel_dev, csr, iastatssm); + adf_handle_rfpar_err(accel_dev, csr, iastatssm); + adf_handle_unexp_cpl_err(accel_dev, csr, iastatssm); + + ADF_CSR_WR(csr, ADF_GEN6_IAINTSTATSSM, iastatssm); +} + +static void adf_handle_ssm(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU2_SSM_ERR_BIT)) + return; + + adf_handle_cerrssmsh(accel_dev, csr); + adf_handle_iaintstatssm(accel_dev, csr); +} + +static void adf_handle_cpp_cfc_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 reg; + + if (!(errsou & ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CPP_CFC_ERR_STATUS); + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: data parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: command parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + if (reg & ADF_GEN6_CPP_CFC_FATAL_ERR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: errors: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_STATUS_CLR, + ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK); +} + +static void adf_gen6_process_errsou2(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_ssm(accel_dev, csr, errsou); + adf_handle_cpp_cfc_err(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou2 still set: %#x\n", errsou); +} + +static void adf_handle_timiscsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 timiscsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TIMISCSTS_BIT)) + return; + + timiscsts = ADF_CSR_RD(csr, ADF_GEN6_TIMISCSTS); + if (timiscsts) { + dev_err(&GET_DEV(accel_dev), "Fatal error in transmit interface: %#x\n", + timiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } +} + +static void adf_handle_ricppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ricppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK)) + return; + + ricppintsts = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTSTS); + ricppintsts &= ADF_GEN6_RICPPINTSTS_MASK; + if (ricppintsts) { + dev_err(&GET_DEV(accel_dev), "RI push pull error: %#x\n", ricppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTSTS, ricppintsts); + } +} + +static void adf_handle_ticppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ticppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK)) + return; + + ticppintsts = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTSTS); + ticppintsts &= ADF_GEN6_TICPPINTSTS_MASK; + if (ticppintsts) { + dev_err(&GET_DEV(accel_dev), "TI push pull error: %#x\n", ticppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTSTS, ticppintsts); + } +} + +static void adf_handle_atufaultstatus(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 max_rp_num = GET_HW_DATA(accel_dev)->num_banks; + u32 atufaultstatus; + u32 i; + + if (!(errsou & ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT)) + return; + + for (i = 0; i < max_rp_num; i++) { + atufaultstatus = ADF_CSR_RD(csr, ADF_GEN6_ATUFAULTSTATUS(i)); + + atufaultstatus &= ADF_GEN6_ATUFAULTSTATUS_BIT; + if (atufaultstatus) { + dev_err(&GET_DEV(accel_dev), "Ring pair (%u) ATU detected fault: %#x\n", i, + atufaultstatus); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_ATUFAULTSTATUS(i), atufaultstatus); + } + } +} + +static void adf_handle_rlterror(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rlterror; + + if (!(errsou & ADF_GEN6_ERRSOU3_RLTERROR_BIT)) + return; + + rlterror = ADF_CSR_RD(csr, ADF_GEN6_RLT_ERRLOG); + rlterror &= ADF_GEN6_RLT_ERRLOG_MASK; + if (rlterror) { + dev_err(&GET_DEV(accel_dev), "Error in rate limiting block: %#x\n", rlterror); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RLT_ERRLOG, rlterror); + } +} + +static void adf_handle_vflr(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Uncorrectable error in VF\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); +} + +static void adf_handle_tc_vc_map_error(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Violation of PCIe TC VC mapping\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pcie_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "DEVHALT due to an error in an incoming transaction\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pg_req_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Error due to response failure in response to a page request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_xlt_cpl_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Error status for a address translation request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_ti_int_err_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "DEVHALT due to a TI internal memory error\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou3(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_timiscsts(accel_dev, csr, errsou); + adf_handle_ricppintsts(accel_dev, csr, errsou); + adf_handle_ticppintsts(accel_dev, csr, errsou); + adf_handle_atufaultstatus(accel_dev, csr, errsou); + adf_handle_rlterror(accel_dev, csr, errsou); + adf_handle_vflr(accel_dev, csr, errsou); + adf_handle_tc_vc_map_error(accel_dev, csr, errsou); + adf_handle_pcie_devhalt(accel_dev, csr, errsou); + adf_handle_pg_req_devhalt(accel_dev, csr, errsou); + adf_handle_xlt_cpl_devhalt(accel_dev, csr, errsou); + adf_handle_ti_int_err_devhalt(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou3 still set: %#x\n", errsou); +} + +static void adf_gen6_is_reset_required(struct adf_accel_dev *accel_dev, void __iomem *csr, + bool *reset_required) +{ + u8 reset, dev_state; + u32 gensts; + + gensts = ADF_CSR_RD(csr, ADF_GEN6_GENSTS); + dev_state = FIELD_GET(ADF_GEN6_GENSTS_DEVICE_STATE_MASK, gensts); + reset = FIELD_GET(ADF_GEN6_GENSTS_RESET_TYPE_MASK, gensts); + if (dev_state == ADF_GEN6_GENSTS_DEVHALT && reset == ADF_GEN6_GENSTS_PFLR) { + *reset_required = true; + return; + } + + if (reset == ADF_GEN6_GENSTS_COLD_RESET) + dev_err(&GET_DEV(accel_dev), "Fatal error, cold reset required\n"); + + *reset_required = false; +} + +static bool adf_gen6_handle_interrupt(struct adf_accel_dev *accel_dev, bool *reset_required) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + bool handled = false; + u32 errsou; + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) { + adf_gen6_process_errsou0(accel_dev, csr); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) { + adf_gen6_process_errsou1(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) { + adf_gen6_process_errsou2(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) { + adf_gen6_process_errsou3(accel_dev, csr, errsou); + handled = true; + } + + adf_gen6_is_reset_required(accel_dev, csr, reset_required); + + return handled; +} + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops) +{ + ras_ops->enable_ras_errors = adf_gen6_enable_ras; + ras_ops->disable_ras_errors = adf_gen6_disable_ras; + ras_ops->handle_interrupt = adf_gen6_handle_interrupt; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_ras_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h new file mode 100644 index 000000000000..66ced271d173 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h @@ -0,0 +1,504 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_RAS_H_ +#define ADF_GEN6_RAS_H_ + +#include <linux/bits.h> + +struct adf_ras_ops; + +/* Error source registers */ +#define ADF_GEN6_ERRSOU0 0x41A200 +#define ADF_GEN6_ERRSOU1 0x41A204 +#define ADF_GEN6_ERRSOU2 0x41A208 +#define ADF_GEN6_ERRSOU3 0x41A20C + +/* Error source mask registers */ +#define ADF_GEN6_ERRMSK0 0x41A210 +#define ADF_GEN6_ERRMSK1 0x41A214 +#define ADF_GEN6_ERRMSK2 0x41A218 +#define ADF_GEN6_ERRMSK3 0x41A21C + +/* ERRSOU0 Correctable error mask */ +#define ADF_GEN6_ERRSOU0_MASK BIT(0) + +#define ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRSOU1_MASK ( \ + (ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + +#define ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRMSK1_MASK ( \ + (ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT)) + +/* HI AE Uncorrectable error log */ +#define ADF_GEN6_HIAEUNCERRLOG_CPP0 0x41A300 + +/* HI AE Uncorrectable error log enable */ +#define ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0 0x41A320 + +/* HI AE Correctable error log */ +#define ADF_GEN6_HIAECORERRLOG_CPP0 0x41A308 + +/* HI AE Correctable error log enable */ +#define ADF_GEN6_HIAECORERRLOGENABLE_CPP0 0x41A318 + +/* HI CPP Agent Command parity error log */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOG 0x41A310 + +/* HI CPP Agent command parity error logging enable */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE 0x41A314 + +#define ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK 0x1B + +/* RI Memory parity error status register */ +#define ADF_GEN6_RIMEM_PARERR_STS 0x41B128 + +/* RI Memory parity error reporting enable */ +#define ADF_GEN6_RI_MEM_PAR_ERR_EN0 0x41B12C + +/* + * RI Memory parity error mask + * BIT(4) - ri_tlq_phdr parity error + * BIT(5) - ri_tlq_pdata parity error + * BIT(6) - ri_tlq_nphdr parity error + * BIT(7) - ri_tlq_npdata parity error + * BIT(8) - ri_tlq_cplhdr parity error + * BIT(10) - BIT(13) - ri_tlq_cpldata[0:3] parity error + * BIT(19) - ri_cds_cmd_fifo parity error + * BIT(20) - ri_obc_ricpl_fifo parity error + * BIT(21) - ri_obc_tiricpl_fifo parity error + * BIT(22) - ri_obc_cppcpl_fifo parity error + * BIT(23) - ri_obc_pendcpl_fifo parity error + * BIT(24) - ri_cpp_cmd_fifo parity error + * BIT(25) - ri_cds_ticmd_fifo parity error + * BIT(26) - riti_cmd_fifo parity error + * BIT(27) - ri_int_msixtbl parity error + * BIT(28) - ri_int_imstbl parity error + * BIT(30) - ri_kpt_fuses parity error + */ +#define ADF_GEN6_RIMEM_PARERR_FATAL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(18) | BIT(19) | BIT(20) | BIT(21) | \ + BIT(22) | BIT(23) | BIT(24) | BIT(25) | BIT(26) | BIT(27) | \ + BIT(28) | BIT(30)) + +#define ADF_GEN6_RIMEM_PARERR_CERR_MASK \ + (BIT(10) | BIT(11) | BIT(12) | BIT(13)) + +/* TI CI parity status */ +#define ADF_GEN6_TI_CI_PAR_STS 0x50060C + +/* TI CI parity reporting mask */ +#define ADF_GEN6_TI_CI_PAR_ERR_MASK 0x500608 + +/* + * TI CI parity status mask + * BIT(0) - CdCmdQ_sts patiry error status + * BIT(1) - CdDataQ_sts parity error status + * BIT(3) - CPP_SkidQ_sts parity error status + */ +#define ADF_GEN6_TI_CI_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(3)) + +/* TI PULLFUB parity status */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS 0x500618 + +/* TI PULLFUB parity error reporting mask */ +#define ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK 0x500614 + +/* + * TI PULLFUB parity status mask + * BIT(0) - TrnPullReqQ_sts parity status + * BIT(1) - TrnSharedDataQ_sts parity status + * BIT(2) - TrnPullReqDataQ_sts parity status + * BIT(4) - CPP_CiPullReqQ_sts parity status + * BIT(5) - CPP_TrnPullReqQ_sts parity status + * BIT(6) - CPP_PullidQ_sts parity status + * BIT(7) - CPP_WaitDataQ_sts parity status + * BIT(8) - CPP_CdDataQ_sts parity status + * BIT(9) - CPP_TrnDataQP0_sts parity status + * BIT(10) - BIT(11) - CPP_TrnDataQRF[00:01]_sts parity status + * BIT(12) - CPP_TrnDataQP1_sts parity status + * BIT(13) - BIT(14) - CPP_TrnDataQRF[10:11]_sts parity status + */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7) | \ + BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | BIT(14)) + +/* TI PUSHUB parity status */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS 0x500630 + +/* TI PUSHFUB parity error reporting mask */ +#define ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK 0x50062C + +/* + * TI PUSHUB parity status mask + * BIT(0) - SbPushReqQ_sts parity status + * BIT(1) - BIT(2) - SbPushDataQ[0:1]_sts parity status + * BIT(4) - CPP_CdPushReqQ_sts parity status + * BIT(5) - BIT(6) - CPP_CdPushDataQ[0:1]_sts parity status + * BIT(7) - CPP_SbPushReqQ_sts parity status + * BIT(8) - CPP_SbPushDataQP_sts parity status + * BIT(9) - BIT(10) - CPP_SbPushDataQRF[0:1]_sts parity status + */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | \ + BIT(6) | BIT(7) | BIT(8) | BIT(9) | BIT(10)) + +/* TI CD parity status */ +#define ADF_GEN6_TI_CD_PAR_STS 0x50063C + +/* TI CD parity error mask */ +#define ADF_GEN6_TI_CD_PAR_ERR_MASK 0x500638 + +/* + * TI CD parity status mask + * BIT(0) - BIT(15) - CtxMdRam[0:15]_sts parity status + * BIT(16) - Leaf2ClusterRam_sts parity status + * BIT(17) - BIT(18) - Ring2LeafRam[0:1]_sts parity status + * BIT(19) - VirtualQ_sts parity status + * BIT(20) - DtRdQ_sts parity status + * BIT(21) - DtWrQ_sts parity status + * BIT(22) - RiCmdQ_sts parity status + * BIT(23) - BypassQ_sts parity status + * BIT(24) - DtRdQ_sc_sts parity status + * BIT(25) - DtWrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_CD_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | \ + BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25)) + +/* TI TRNSB parity status */ +#define ADF_GEN6_TI_TRNSB_PAR_STS 0x500648 + +/* TI TRNSB parity error reporting mask */ +#define ADF_GEN6_TI_TRNSB_PAR_ERR_MASK 0x500644 + +/* + * TI TRNSB parity status mask + * BIT(0) - TrnPHdrQP_sts parity status + * BIT(1) - TrnPHdrQRF_sts parity status + * BIT(2) - TrnPDataQP_sts parity status + * BIT(3) - BIT(6) - TrnPDataQRF[0:3]_sts parity status + * BIT(7) - TrnNpHdrQP_sts parity status + * BIT(8) - BIT(9) - TrnNpHdrQRF[0:1]_sts parity status + * BIT(10) - TrnCplHdrQ_sts parity status + * BIT(11) - TrnPutObsReqQ_sts parity status + * BIT(12) - TrnPushReqQ_sts parity status + * BIT(13) - SbSplitIdRam_sts parity status + * BIT(14) - SbReqCountQ_sts parity status + * BIT(15) - SbCplTrkRam_sts parity status + * BIT(16) - SbGetObsReqQ_sts parity status + * BIT(17) - SbEpochIdQ_sts parity status + * BIT(18) - SbAtCplHdrQ_sts parity status + * BIT(19) - SbAtCplDataQ_sts parity status + * BIT(20) - SbReqCountRam_sts parity status + * BIT(21) - SbAtCplHdrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_TRNSB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | \ + BIT(13) | BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | \ + BIT(19) | BIT(20) | BIT(21)) + +/* Status register to log misc error on RI */ +#define ADF_GEN6_RIMISCSTS 0x41B1B8 + +/* Status control register to log misc RI error */ +#define ADF_GEN6_RIMISCCTL 0x41B1BC + +/* + * ERRSOU2 bit mask + * BIT(0) - SSM Interrupt Mask + * BIT(1) - CFC on CPP. ORed of CFC Push error and Pull error + * BIT(2) - BIT(4) - CPP attention interrupts + * BIT(18) - PM interrupt + */ +#define ADF_GEN6_ERRSOU2_SSM_ERR_BIT BIT(0) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT BIT(1) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK \ + (BIT(2) | BIT(3) | BIT(4)) + +#define ADF_GEN6_ERRSOU2_PM_INT_BIT BIT(18) + +#define ADF_GEN6_ERRSOU2_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT) + +#define ADF_GEN6_ERRSOU2_DIS_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK) + +#define ADF_GEN6_IAINTSTATSSM 0x28 + +/* IAINTSTATSSM error bit mask definitions */ +#define ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT BIT(0) +#define ADF_GEN6_IAINTSTATSSM_PPERR_BIT BIT(2) +#define ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT BIT(4) +#define ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT BIT(5) +#define ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT BIT(6) +#define ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT BIT(7) + +#define ADF_GEN6_IAINTSTATSSM_MASK \ + (ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_PPERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT) + +#define ADF_GEN6_UERRSSMSH 0x18 + +/* + * UERRSSMSH error bit mask definitions + * + * BIT(0) - Indicates one uncorrectable error + * BIT(15) - Indicates multiple uncorrectable errors + * in device shared memory + */ +#define ADF_GEN6_UERRSSMSH_MASK (BIT(0) | BIT(15)) + +/* + * CERRSSMSH error bit + * BIT(0) - Indicates one correctable error + */ +#define ADF_GEN6_CERRSSMSH_ERROR_BIT (BIT(0) | BIT(15) | BIT(24)) +#define ADF_GEN6_CERRSSMSH 0x10 + +#define ADF_GEN6_INTMASKSSM 0x0 + +/* + * Error reporting mask in INTMASKSSM + * BIT(0) - Shared memory uncorrectable interrupt mask + * BIT(2) - PPERR interrupt mask + * BIT(4) - SCM parity error interrupt mask + * BIT(5) - CPP parity error interrupt mask + * BIT(6) - SHRAM RF parity error interrupt mask + * BIT(7) - AXI unexpected completion error mask + */ +#define ADF_GEN6_INTMASKSSM_MASK \ + (BIT(0) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7)) + +/* CPP push or pull error */ +#define ADF_GEN6_PPERR 0x8 + +#define ADF_GEN6_PPERR_MASK (BIT(0) | BIT(1)) + +/* + * SSM_FERR_STATUS error bit mask definitions + */ +#define ADF_GEN6_SCM_PAR_ERR_MASK BIT(5) +#define ADF_GEN6_CPP_PAR_ERR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_UNEXP_CPL_ERR_MASK (BIT(3) | BIT(4) | BIT(10) | BIT(11)) +#define ADF_GEN6_RF_PAR_ERR_MASK BIT(16) + +#define ADF_GEN6_SSM_FERR_STATUS 0x9C + +#define ADF_GEN6_CPP_CFC_ERR_STATUS 0x640C04 + +/* + * BIT(0) - Indicates one or more CPP CFC errors + * BIT(1) - Indicates multiple CPP CFC errors + * BIT(7) - Indicates CPP CFC command parity error type + * BIT(8) - Indicates CPP CFC data parity error type + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT BIT(0) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT BIT(1) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT BIT(7) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT BIT(8) +#define ADF_GEN6_CPP_CFC_FATAL_ERR_BIT \ + (ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT | \ + ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT) + +/* + * BIT(0) - Enables CFC to detect and log a push/pull data error + * BIT(1) - Enables CFC to generate interrupt to PCIEP for a CPP error + * BIT(4) - When 1 parity detection is disabled + * BIT(5) - When 1 parity detection is disabled on CPP command bus + * BIT(6) - When 1 parity detection is disabled on CPP push/pull bus + * BIT(9) - When 1 RF parity error detection is disabled + */ +#define ADF_GEN6_CPP_CFC_ERR_CTRL_MASK (BIT(0) | BIT(1)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK \ + (BIT(4) | BIT(5) | BIT(6) | BIT(9) | BIT(10)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL 0x640C00 + +/* + * BIT(0) - Clears bit(0) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when an error is reported on CPP + * BIT(1) - Clears bit(1) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when multiple errors are reported on CPP + * BIT(2) - Clears bit(2) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when attention interrupt is reported + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR 0x640C08 + +/* + * ERRSOU3 bit masks + * BIT(0) - indicates error response order overflow and/or BME error + * BIT(1) - indicates RI push/pull error + * BIT(2) - indicates TI push/pull error + * BIT(5) - indicates TI pull parity error + * BIT(6) - indicates RI push parity error + * BIT(7) - indicates VFLR interrupt + * BIT(8) - indicates ring pair interrupts for ATU detected fault + * BIT(9) - indicates rate limiting error + */ +#define ADF_GEN6_ERRSOU3_TIMISCSTS_BIT BIT(0) +#define ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK (BIT(1) | BIT(6)) +#define ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK (BIT(2) | BIT(5)) +#define ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT BIT(7) +#define ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT BIT(8) +#define ADF_GEN6_ERRSOU3_RLTERROR_BIT BIT(9) +#define ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT BIT(16) +#define ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT BIT(17) +#define ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT BIT(18) +#define ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT BIT(19) +#define ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT BIT(20) + +#define ADF_GEN6_ERRSOU3_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + +#define ADF_GEN6_ERRSOU3_DIS_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + +/* Rate limiting error log register */ +#define ADF_GEN6_RLT_ERRLOG 0x508814 + +#define ADF_GEN6_RLT_ERRLOG_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* TI misc status register */ +#define ADF_GEN6_TIMISCSTS 0x50054C + +/* TI misc error reporting mask */ +#define ADF_GEN6_TIMISCCTL 0x500548 + +/* + * TI Misc error reporting control mask + * BIT(0) - Enables error detection and logging in TIMISCSTS register + * BIT(1) - It has effect only when SRIOV enabled, this bit is 0 by default + * BIT(2) - Enables the D-F-x counter within the dispatch arbiter + * to start based on the command triggered from + * BIT(30) - Disables VFLR functionality + * bits 1, 2 and 30 value should be preserved and not meant to be changed + * within RAS. + */ +#define ADF_GEN6_TIMISCCTL_BIT BIT(0) +#define ADF_GEN6_TIMSCCTL_RELAY_MASK (BIT(1) | BIT(2) | BIT(30)) + +/* RI CPP interface status register */ +#define ADF_GEN6_RICPPINTSTS 0x41A330 + +/* + * Uncorrectable error mask in RICPPINTSTS register + * BIT(0) - RI asserted the CPP error signal during a push + * BIT(1) - RI detected the CPP error signal asserted during a pull + * BIT(2) - RI detected a push data parity error + * BIT(3) - RI detected a push valid parity error + */ +#define ADF_GEN6_RICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* RI CPP interface register control */ +#define ADF_GEN6_RICPPINTCTL 0x41A32C + +/* + * Control bit mask for RICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting + * on the RI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting + * on the RI CPP Pull interface + * BIT(2) - value of 1 enables error detection and reporting + * on the RI Parity + * BIT(3) - value of 1 enable checking parity on CPP + */ +#define ADF_GEN6_RICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* TI CPP interface status register */ +#define ADF_GEN6_TICPPINTSTS 0x50053C + +/* + * Uncorrectable error mask in TICPPINTSTS register + * BIT(0) - value of 1 indicates that the TI asserted + * the CPP error signal during a push + * BIT(1) - value of 1 indicates that the TI detected + * the CPP error signal asserted during a pull + * BIT(2) - value of 1 indicates that the TI detected + * a pull data parity error + */ +#define ADF_GEN6_TICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2)) + +/* TI CPP interface status register control */ +#define ADF_GEN6_TICPPINTCTL 0x500538 + +/* + * Control bit mask for TICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(2) - value of 1 enables parity error detection and logging on + * the TI CPP Pull interface + * BIT(3) - value of 1 enables CPP CMD and Pull Data parity checking + */ +#define ADF_GEN6_TICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* ATU fault status register */ +#define ADF_GEN6_ATUFAULTSTATUS(i) (0x506000 + ((i) * 0x4)) + +#define ADF_GEN6_ATUFAULTSTATUS_BIT BIT(0) + +/* Command parity error detected on IOSFP command to QAT */ +#define ADF_GEN6_RIMISCSTS_BIT BIT(0) + +#define ADF_GEN6_GENSTS 0x41A220 +#define ADF_GEN6_GENSTS_DEVICE_STATE_MASK GENMASK(1, 0) +#define ADF_GEN6_GENSTS_RESET_TYPE_MASK GENMASK(3, 2) +#define ADF_GEN6_GENSTS_PFLR 0x1 +#define ADF_GEN6_GENSTS_COLD_RESET 0x3 +#define ADF_GEN6_GENSTS_DEVHALT 0x1 + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops); + +#endif /* ADF_GEN6_RAS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c new file mode 100644 index 000000000000..c9b151006dca --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include <linux/export.h> + +#include "adf_gen4_config.h" +#include "adf_gen4_hw_csr_data.h" +#include "adf_gen4_pfvf.h" +#include "adf_gen4_vf_mig.h" +#include "adf_gen6_shared.h" + +struct adf_accel_dev; +struct adf_pfvf_ops; +struct adf_hw_csr_ops; + +/* + * QAT GEN4 and GEN6 devices often differ in terms of supported features, + * options and internal logic. However, some of the mechanisms and register + * layout are shared between those two GENs. This file serves as an abstraction + * layer that allows to use existing GEN4 implementation that is also + * applicable to GEN6 without additional overhead and complexity. + */ +void adf_gen6_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + adf_gen4_init_pf_pfvf_ops(pfvf_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_pf_pfvf_ops); + +void adf_gen6_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + return adf_gen4_init_hw_csr_ops(csr_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_hw_csr_ops); + +int adf_gen6_cfg_dev_init(struct adf_accel_dev *accel_dev) +{ + return adf_gen4_cfg_dev_init(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_cfg_dev_init); + +int adf_gen6_comp_dev_config(struct adf_accel_dev *accel_dev) +{ + return adf_comp_dev_config(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_comp_dev_config); + +int adf_gen6_no_dev_config(struct adf_accel_dev *accel_dev) +{ + return adf_no_dev_config(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_no_dev_config); + +void adf_gen6_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops) +{ + adf_gen4_init_vf_mig_ops(vfmig_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_vf_mig_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h new file mode 100644 index 000000000000..fc6fad029a70 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_SHARED_H_ +#define ADF_GEN6_SHARED_H_ + +struct adf_hw_csr_ops; +struct qat_migdev_ops; +struct adf_accel_dev; +struct adf_pfvf_ops; + +void adf_gen6_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +void adf_gen6_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); +int adf_gen6_cfg_dev_init(struct adf_accel_dev *accel_dev); +int adf_gen6_comp_dev_config(struct adf_accel_dev *accel_dev); +int adf_gen6_no_dev_config(struct adf_accel_dev *accel_dev); +void adf_gen6_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops); +#endif/* ADF_GEN6_SHARED_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c new file mode 100644 index 000000000000..faa60b04c406 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Intel Corporation. */ +#include <linux/export.h> + +#include "adf_gen6_tl.h" +#include "adf_telemetry.h" +#include "adf_tl_debugfs.h" +#include "icp_qat_fw_init_admin.h" + +#define ADF_GEN6_TL_DEV_REG_OFF(reg) ADF_TL_DEV_REG_OFF(reg, gen6) + +#define ADF_GEN6_TL_RP_REG_OFF(reg) ADF_TL_RP_REG_OFF(reg, gen6) + +#define ADF_GEN6_TL_SL_UTIL_COUNTER(_name) \ + ADF_TL_COUNTER("util_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_util, gen6)) + +#define ADF_GEN6_TL_SL_EXEC_COUNTER(_name) \ + ADF_TL_COUNTER("exec_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_exec_cnt, gen6)) + +#define SLICE_IDX(sl) offsetof(struct icp_qat_fw_init_admin_slice_cnt, sl##_cnt) + +#define ADF_GEN6_TL_CMDQ_WAIT_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_wait_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_wait_cnt, gen6)) +#define ADF_GEN6_TL_CMDQ_EXEC_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_exec_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_exec_cnt, gen6)) +#define ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_drain_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_drain_cnt, \ + gen6)) + +#define CPR_QUEUE_COUNT 5 +#define DCPR_QUEUE_COUNT 3 +#define PKE_QUEUE_COUNT 1 +#define WAT_QUEUE_COUNT 7 +#define WCP_QUEUE_COUNT 7 +#define USC_QUEUE_COUNT 3 +#define ATH_QUEUE_COUNT 2 + +/* Device level counters. */ +static const struct adf_tl_dbg_counter dev_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_prt_trans_cnt)), + /* Max read latency[ns]. */ + ADF_TL_COUNTER(MAX_RD_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_lat_max)), + /* Read latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(RD_LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_cmpl_cnt)), + /* Max "get to put" latency[ns]. */ + ADF_TL_COUNTER(MAX_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_gp_lat_max)), + /* "Get to put" latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_bw_out)), + /* Page request latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(PAGE_REQ_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_page_req_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_page_req_cnt)), + /* Page translation latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(AT_TRANS_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_trans_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_trans_lat_cnt)), + /* Maximum uTLB used. */ + ADF_TL_COUNTER(AT_MAX_UTLB_USED_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_max_utlb_used)), + /* Ring Empty average[ns] across all rings */ + ADF_TL_COUNTER_LATENCY(RE_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_re_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_re_cnt)), +}; + +/* Accelerator utilization counters */ +static const struct adf_tl_dbg_counter sl_util_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression accelerator utilization. */ + [SLICE_IDX(cpr)] = ADF_GEN6_TL_SL_UTIL_COUNTER(cnv), + /* Decompression accelerator utilization. */ + [SLICE_IDX(dcpr)] = ADF_GEN6_TL_SL_UTIL_COUNTER(dcprz), + /* PKE accelerator utilization. */ + [SLICE_IDX(pke)] = ADF_GEN6_TL_SL_UTIL_COUNTER(pke), + /* Wireless Authentication accelerator utilization. */ + [SLICE_IDX(wat)] = ADF_GEN6_TL_SL_UTIL_COUNTER(wat), + /* Wireless Cipher accelerator utilization. */ + [SLICE_IDX(wcp)] = ADF_GEN6_TL_SL_UTIL_COUNTER(wcp), + /* UCS accelerator utilization. */ + [SLICE_IDX(ucs)] = ADF_GEN6_TL_SL_UTIL_COUNTER(ucs), + /* Authentication accelerator utilization. */ + [SLICE_IDX(ath)] = ADF_GEN6_TL_SL_UTIL_COUNTER(ath), +}; + +/* Accelerator execution counters */ +static const struct adf_tl_dbg_counter sl_exec_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression accelerator execution count. */ + [SLICE_IDX(cpr)] = ADF_GEN6_TL_SL_EXEC_COUNTER(cnv), + /* Decompression accelerator execution count. */ + [SLICE_IDX(dcpr)] = ADF_GEN6_TL_SL_EXEC_COUNTER(dcprz), + /* PKE execution count. */ + [SLICE_IDX(pke)] = ADF_GEN6_TL_SL_EXEC_COUNTER(pke), + /* Wireless Authentication accelerator execution count. */ + [SLICE_IDX(wat)] = ADF_GEN6_TL_SL_EXEC_COUNTER(wat), + /* Wireless Cipher accelerator execution count. */ + [SLICE_IDX(wcp)] = ADF_GEN6_TL_SL_EXEC_COUNTER(wcp), + /* UCS accelerator execution count. */ + [SLICE_IDX(ucs)] = ADF_GEN6_TL_SL_EXEC_COUNTER(ucs), + /* Authentication accelerator execution count. */ + [SLICE_IDX(ath)] = ADF_GEN6_TL_SL_EXEC_COUNTER(ath), +}; + +static const struct adf_tl_dbg_counter cnv_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(cnv), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(cnv), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(cnv) +}; + +#define NUM_CMDQ_COUNTERS ARRAY_SIZE(cnv_cmdq_counters) + +static const struct adf_tl_dbg_counter dcprz_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(dcprz), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(dcprz), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(dcprz) +}; + +static_assert(ARRAY_SIZE(dcprz_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter pke_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(pke), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(pke), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(pke) +}; + +static_assert(ARRAY_SIZE(pke_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter wat_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(wat), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(wat), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(wat) +}; + +static_assert(ARRAY_SIZE(wat_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter wcp_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(wcp), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(wcp), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(wcp) +}; + +static_assert(ARRAY_SIZE(wcp_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter ucs_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(ucs), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(ucs), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(ucs) +}; + +static_assert(ARRAY_SIZE(ucs_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter ath_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(ath), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(ath), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(ath) +}; + +static_assert(ARRAY_SIZE(ath_cmdq_counters) == NUM_CMDQ_COUNTERS); + +/* CMDQ drain counters. */ +static const struct adf_tl_dbg_counter *cmdq_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression accelerator execution count. */ + [SLICE_IDX(cpr)] = cnv_cmdq_counters, + /* Decompression accelerator execution count. */ + [SLICE_IDX(dcpr)] = dcprz_cmdq_counters, + /* PKE execution count. */ + [SLICE_IDX(pke)] = pke_cmdq_counters, + /* Wireless Authentication accelerator execution count. */ + [SLICE_IDX(wat)] = wat_cmdq_counters, + /* Wireless Cipher accelerator execution count. */ + [SLICE_IDX(wcp)] = wcp_cmdq_counters, + /* UCS accelerator execution count. */ + [SLICE_IDX(ucs)] = ucs_cmdq_counters, + /* Authentication accelerator execution count. */ + [SLICE_IDX(ath)] = ath_cmdq_counters, +}; + +/* Ring pair counters. */ +static const struct adf_tl_dbg_counter rp_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_prt_trans_cnt)), + /* "Get to put" latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN6_TL_RP_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_bw_out)), + /* Message descriptor DevTLB hit rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_hit)), + /* Message descriptor DevTLB miss rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_miss)), + /* Payload DevTLB hit rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_hit)), + /* Payload DevTLB miss rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_miss)), + /* Ring Empty average[ns]. */ + ADF_TL_COUNTER_LATENCY(RE_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_re_acc), + ADF_GEN6_TL_RP_REG_OFF(reg_tl_re_cnt)), +}; + +void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data) +{ + tl_data->layout_sz = ADF_GEN6_TL_LAYOUT_SZ; + tl_data->slice_reg_sz = ADF_GEN6_TL_SLICE_REG_SZ; + tl_data->cmdq_reg_sz = ADF_GEN6_TL_CMDQ_REG_SZ; + tl_data->rp_reg_sz = ADF_GEN6_TL_RP_REG_SZ; + tl_data->num_hbuff = ADF_GEN6_TL_NUM_HIST_BUFFS; + tl_data->max_rp = ADF_GEN6_TL_MAX_RP_NUM; + tl_data->msg_cnt_off = ADF_GEN6_TL_MSG_CNT_OFF; + tl_data->cpp_ns_per_cycle = ADF_GEN6_CPP_NS_PER_CYCLE; + tl_data->bw_units_to_bytes = ADF_GEN6_TL_BW_HW_UNITS_TO_BYTES; + + tl_data->dev_counters = dev_counters; + tl_data->num_dev_counters = ARRAY_SIZE(dev_counters); + tl_data->sl_util_counters = sl_util_counters; + tl_data->sl_exec_counters = sl_exec_counters; + tl_data->cmdq_counters = cmdq_counters; + tl_data->num_cmdq_counters = NUM_CMDQ_COUNTERS; + tl_data->rp_counters = rp_counters; + tl_data->num_rp_counters = ARRAY_SIZE(rp_counters); + tl_data->max_sl_cnt = ADF_GEN6_TL_MAX_SLICES_PER_TYPE; + + tl_data->multiplier.cpr_cnt = CPR_QUEUE_COUNT; + tl_data->multiplier.dcpr_cnt = DCPR_QUEUE_COUNT; + tl_data->multiplier.pke_cnt = PKE_QUEUE_COUNT; + tl_data->multiplier.wat_cnt = WAT_QUEUE_COUNT; + tl_data->multiplier.wcp_cnt = WCP_QUEUE_COUNT; + tl_data->multiplier.ucs_cnt = USC_QUEUE_COUNT; + tl_data->multiplier.ath_cnt = ATH_QUEUE_COUNT; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_tl_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h new file mode 100644 index 000000000000..49db660b8eb9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2025 Intel Corporation. */ +#ifndef ADF_GEN6_TL_H +#define ADF_GEN6_TL_H + +#include <linux/types.h> + +struct adf_tl_hw_data; + +/* Computation constants. */ +#define ADF_GEN6_CPP_NS_PER_CYCLE 2 +#define ADF_GEN6_TL_BW_HW_UNITS_TO_BYTES 64 + +/* Maximum aggregation time. Value is in milliseconds. */ +#define ADF_GEN6_TL_MAX_AGGR_TIME_MS 4000 +/* Number of buffers to store historic values. */ +#define ADF_GEN6_TL_NUM_HIST_BUFFS \ + (ADF_GEN6_TL_MAX_AGGR_TIME_MS / ADF_TL_DATA_WR_INTERVAL_MS) + +/* Max number of HW resources of one type */ +#define ADF_GEN6_TL_MAX_SLICES_PER_TYPE 32 +#define MAX_ATH_SL_COUNT 7 +#define MAX_CNV_SL_COUNT 2 +#define MAX_DCPRZ_SL_COUNT 2 +#define MAX_PKE_SL_COUNT 32 +#define MAX_UCS_SL_COUNT 4 +#define MAX_WAT_SL_COUNT 5 +#define MAX_WCP_SL_COUNT 5 + +#define MAX_ATH_CMDQ_COUNT 14 +#define MAX_CNV_CMDQ_COUNT 6 +#define MAX_DCPRZ_CMDQ_COUNT 6 +#define MAX_PKE_CMDQ_COUNT 32 +#define MAX_UCS_CMDQ_COUNT 12 +#define MAX_WAT_CMDQ_COUNT 35 +#define MAX_WCP_CMDQ_COUNT 35 + +/* Max number of simultaneously monitored ring pairs. */ +#define ADF_GEN6_TL_MAX_RP_NUM 4 + +/** + * struct adf_gen6_tl_slice_data_regs - HW slice data as populated by FW. + * @reg_tm_slice_exec_cnt: Slice execution count. + * @reg_tm_slice_util: Slice utilization. + */ +struct adf_gen6_tl_slice_data_regs { + __u32 reg_tm_slice_exec_cnt; + __u32 reg_tm_slice_util; +}; + +#define ADF_GEN6_TL_SLICE_REG_SZ sizeof(struct adf_gen6_tl_slice_data_regs) + +/** + * struct adf_gen6_tl_cmdq_data_regs - HW CMDQ data as populated by FW. + * @reg_tm_cmdq_wait_cnt: CMDQ wait count. + * @reg_tm_cmdq_exec_cnt: CMDQ execution count. + * @reg_tm_cmdq_drain_cnt: CMDQ drain count. + */ +struct adf_gen6_tl_cmdq_data_regs { + __u32 reg_tm_cmdq_wait_cnt; + __u32 reg_tm_cmdq_exec_cnt; + __u32 reg_tm_cmdq_drain_cnt; + __u32 reserved; +}; + +#define ADF_GEN6_TL_CMDQ_REG_SZ sizeof(struct adf_gen6_tl_cmdq_data_regs) + +/** + * struct adf_gen6_tl_device_data_regs - This structure stores device telemetry + * counter values as are being populated periodically by device. + * @reg_tl_rd_lat_acc: read latency accumulator + * @reg_tl_gp_lat_acc: "get to put" latency accumulator + * @reg_tl_at_page_req_lat_acc: AT/DevTLB page request latency accumulator + * @reg_tl_at_trans_lat_acc: DevTLB transaction latency accumulator + * @reg_tl_re_acc: accumulated ring empty time + * @reg_tl_prt_trans_cnt: PCIe partial transactions + * @reg_tl_rd_lat_max: maximum logged read latency + * @reg_tl_rd_cmpl_cnt: read requests completed count + * @reg_tl_gp_lat_max: maximum logged get to put latency + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_page_req_cnt: DevTLB page requests count + * @reg_tl_at_trans_lat_cnt: DevTLB transaction latency samples count + * @reg_tl_at_max_utlb_used: maximum uTLB used + * @reg_tl_re_cnt: ring empty time samples count + * @reserved: reserved + * @ath_slices: array of Authentication slices utilization registers + * @cnv_slices: array of Compression slices utilization registers + * @dcprz_slices: array of Decompression slices utilization registers + * @pke_slices: array of PKE slices utilization registers + * @ucs_slices: array of UCS slices utilization registers + * @wat_slices: array of Wireless Authentication slices utilization registers + * @wcp_slices: array of Wireless Cipher slices utilization registers + * @ath_cmdq: array of Authentication cmdq telemetry registers + * @cnv_cmdq: array of Compression cmdq telemetry registers + * @dcprz_cmdq: array of Decomopression cmdq telemetry registers + * @pke_cmdq: array of PKE cmdq telemetry registers + * @ucs_cmdq: array of UCS cmdq telemetry registers + * @wat_cmdq: array of Wireless Authentication cmdq telemetry registers + * @wcp_cmdq: array of Wireless Cipher cmdq telemetry registers + */ +struct adf_gen6_tl_device_data_regs { + __u64 reg_tl_rd_lat_acc; + __u64 reg_tl_gp_lat_acc; + __u64 reg_tl_at_page_req_lat_acc; + __u64 reg_tl_at_trans_lat_acc; + __u64 reg_tl_re_acc; + __u32 reg_tl_prt_trans_cnt; + __u32 reg_tl_rd_lat_max; + __u32 reg_tl_rd_cmpl_cnt; + __u32 reg_tl_gp_lat_max; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_page_req_cnt; + __u32 reg_tl_at_trans_lat_cnt; + __u32 reg_tl_at_max_utlb_used; + __u32 reg_tl_re_cnt; + __u32 reserved; + struct adf_gen6_tl_slice_data_regs ath_slices[MAX_ATH_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs cnv_slices[MAX_CNV_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs dcprz_slices[MAX_DCPRZ_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs pke_slices[MAX_PKE_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs ucs_slices[MAX_UCS_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs wat_slices[MAX_WAT_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs wcp_slices[MAX_WCP_SL_COUNT]; + struct adf_gen6_tl_cmdq_data_regs ath_cmdq[MAX_ATH_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs cnv_cmdq[MAX_CNV_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs dcprz_cmdq[MAX_DCPRZ_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs pke_cmdq[MAX_PKE_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs ucs_cmdq[MAX_UCS_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs wat_cmdq[MAX_WAT_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs wcp_cmdq[MAX_WCP_CMDQ_COUNT]; +}; + +/** + * struct adf_gen6_tl_ring_pair_data_regs - This structure stores ring pair + * telemetry counter values as they are being populated periodically by device. + * @reg_tl_gp_lat_acc: get-put latency accumulator + * @reg_tl_re_acc: accumulated ring empty time + * @reg_tl_pci_trans_cnt: PCIe partial transactions + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_glob_devtlb_hit: Message descriptor DevTLB hit rate + * @reg_tl_at_glob_devtlb_miss: Message descriptor DevTLB miss rate + * @reg_tl_at_payld_devtlb_hit: Payload DevTLB hit rate + * @reg_tl_at_payld_devtlb_miss: Payload DevTLB miss rate + * @reg_tl_re_cnt: ring empty time samples count + * @reserved1: reserved + */ +struct adf_gen6_tl_ring_pair_data_regs { + __u64 reg_tl_gp_lat_acc; + __u64 reg_tl_re_acc; + __u32 reg_tl_prt_trans_cnt; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_glob_devtlb_hit; + __u32 reg_tl_at_glob_devtlb_miss; + __u32 reg_tl_at_payld_devtlb_hit; + __u32 reg_tl_at_payld_devtlb_miss; + __u32 reg_tl_re_cnt; + __u32 reserved1; +}; + +#define ADF_GEN6_TL_RP_REG_SZ sizeof(struct adf_gen6_tl_ring_pair_data_regs) + +/** + * struct adf_gen6_tl_layout - This structure represents the entire telemetry + * counters data: Device + 4 Ring Pairs as they are being populated periodically + * by device. + * @tl_device_data_regs: structure of device telemetry registers + * @tl_ring_pairs_data_regs: array of ring pairs telemetry registers + * @reg_tl_msg_cnt: telemetry message counter + * @reserved: reserved + */ +struct adf_gen6_tl_layout { + struct adf_gen6_tl_device_data_regs tl_device_data_regs; + struct adf_gen6_tl_ring_pair_data_regs + tl_ring_pairs_data_regs[ADF_GEN6_TL_MAX_RP_NUM]; + __u32 reg_tl_msg_cnt; + __u32 reserved; +}; + +#define ADF_GEN6_TL_LAYOUT_SZ sizeof(struct adf_gen6_tl_layout) +#define ADF_GEN6_TL_MSG_CNT_OFF \ + offsetof(struct adf_gen6_tl_layout, reg_tl_msg_cnt) + +#ifdef CONFIG_DEBUG_FS +void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data); +#else +static inline void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* ADF_GEN6_TL_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c index f189cce7d153..46491048e0bb 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_init.c +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -404,6 +404,7 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) hw_data->exit_admin_comms(accel_dev); adf_cleanup_etr_data(accel_dev); + adf_misc_wq_flush(); adf_dev_restore(accel_dev); } diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index cae1aee5479a..4639d7fd93e6 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -384,7 +384,8 @@ EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); */ int __init adf_init_misc_wq(void) { - adf_misc_wq = alloc_workqueue("qat_misc_wq", WQ_MEM_RECLAIM, 0); + adf_misc_wq = alloc_workqueue("qat_misc_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !adf_misc_wq ? -ENOMEM : 0; } @@ -407,3 +408,8 @@ bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, { return queue_delayed_work(adf_misc_wq, work, delay); } + +void adf_misc_wq_flush(void) +{ + flush_workqueue(adf_misc_wq); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c new file mode 100644 index 000000000000..4ccc94ed9493 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/sprintf.h> +#include <linux/string_helpers.h> + +#include "adf_pm_dbgfs_utils.h" + +#define PM_INFO_MAX_KEY_LEN 21 + +static int pm_scnprint_table(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len, + bool lowercase) +{ + char key[PM_INFO_MAX_KEY_LEN]; + int wr = 0; + int i; + + for (i = 0; i < table_len; i++) { + if (lowercase) + string_lower(key, table[i].key); + else + string_upper(key, table[i].key); + + wr += scnprintf(&buff[wr], buff_size - wr, "%s: %#x\n", key, + field_get(table[i].field_mask, + pm_info_regs[table[i].reg_offset])); + } + + return wr; +} + +int adf_pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len) +{ + return pm_scnprint_table(buff, table, pm_info_regs, buff_size, + table_len, false); +} + +int adf_pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len) +{ + return pm_scnprint_table(buff, table, pm_info_regs, buff_size, + table_len, true); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h new file mode 100644 index 000000000000..854f058b35ed --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_PM_DBGFS_UTILS_H_ +#define ADF_PM_DBGFS_UTILS_H_ + +#include <linux/stddef.h> +#include <linux/stringify.h> +#include <linux/types.h> +#include "icp_qat_fw_init_admin.h" + +#define PM_INFO_MEMBER_OFF(member) \ + (offsetof(struct icp_qat_fw_init_admin_pm_info, member) / sizeof(u32)) + +#define PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, _mask_) \ +{ \ + .reg_offset = PM_INFO_MEMBER_OFF(_reg_), \ + .key = __stringify(_field_), \ + .field_mask = _mask_, \ +} + +#define PM_INFO_REGSET_ENTRY32(_reg_, _field_) \ + PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, GENMASK(31, 0)) + +struct pm_status_row { + int reg_offset; + u32 field_mask; + const char *key; +}; + +int adf_pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len); + +int adf_pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len); + +#endif /* ADF_PM_DBGFS_UTILS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.c b/drivers/crypto/intel/qat/qat_common/adf_rl.c index e782c23fc1bf..c6a54e465931 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_rl.c +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.c @@ -13,6 +13,7 @@ #include <linux/units.h> #include "adf_accel_devices.h" +#include "adf_cfg_services.h" #include "adf_common_drv.h" #include "adf_rl_admin.h" #include "adf_rl.h" @@ -55,7 +56,7 @@ static int validate_user_input(struct adf_accel_dev *accel_dev, } } - if (sla_in->srv >= ADF_SVC_NONE) { + if (sla_in->srv >= SVC_BASE_COUNT) { dev_notice(&GET_DEV(accel_dev), "Wrong service type\n"); return -EINVAL; @@ -168,20 +169,6 @@ static struct rl_sla *find_parent(struct adf_rl *rl_data, return NULL; } -static enum adf_cfg_service_type srv_to_cfg_svc_type(enum adf_base_services rl_srv) -{ - switch (rl_srv) { - case ADF_SVC_ASYM: - return ASYM; - case ADF_SVC_SYM: - return SYM; - case ADF_SVC_DC: - return COMP; - default: - return UNUSED; - } -} - /** * adf_rl_get_sla_arr_of_type() - Returns a pointer to SLA type specific array * @rl_data: pointer to ratelimiting data @@ -209,22 +196,6 @@ u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, } } -static bool is_service_enabled(struct adf_accel_dev *accel_dev, - enum adf_base_services rl_srv) -{ - enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(rl_srv); - struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); - u8 rps_per_bundle = hw_data->num_banks_per_vf; - int i; - - for (i = 0; i < rps_per_bundle; i++) { - if (GET_SRV_TYPE(accel_dev, i) == arb_srv) - return true; - } - - return false; -} - /** * prepare_rp_ids() - Creates an array of ring pair IDs from bitmask * @accel_dev: pointer to acceleration device structure @@ -243,7 +214,7 @@ static bool is_service_enabled(struct adf_accel_dev *accel_dev, static int prepare_rp_ids(struct adf_accel_dev *accel_dev, struct rl_sla *sla, const unsigned long rp_mask) { - enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(sla->srv); + enum adf_cfg_service_type arb_srv = adf_srv_to_cfg_svc_type(sla->srv); u16 rps_per_bundle = GET_HW_DATA(accel_dev)->num_banks_per_vf; bool *rp_in_use = accel_dev->rate_limiting->rp_in_use; size_t rp_cnt_max = ARRAY_SIZE(sla->ring_pairs_ids); @@ -558,21 +529,9 @@ u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val, if (!sla_val) return 0; + /* Handle generation specific slice count adjustment */ avail_slice_cycles = hw_data->clock_frequency; - - switch (svc_type) { - case ADF_SVC_ASYM: - avail_slice_cycles *= device_data->slices.pke_cnt; - break; - case ADF_SVC_SYM: - avail_slice_cycles *= device_data->slices.cph_cnt; - break; - case ADF_SVC_DC: - avail_slice_cycles *= device_data->slices.dcpr_cnt; - break; - default: - break; - } + avail_slice_cycles *= hw_data->get_svc_slice_cnt(accel_dev, svc_type); do_div(avail_slice_cycles, device_data->scan_interval); allocated_tokens = avail_slice_cycles * sla_val; @@ -581,6 +540,17 @@ u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val, return allocated_tokens; } +static u32 adf_rl_get_num_svc_aes(struct adf_accel_dev *accel_dev, + enum adf_base_services svc) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + + if (svc >= SVC_BASE_COUNT) + return 0; + + return device_data->svc_ae_mask[svc]; +} + u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val, enum adf_base_services svc_type) { @@ -592,7 +562,7 @@ u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val, return 0; avail_ae_cycles = hw_data->clock_frequency; - avail_ae_cycles *= hw_data->get_num_aes(hw_data) - 1; + avail_ae_cycles *= adf_rl_get_num_svc_aes(accel_dev, svc_type); do_div(avail_ae_cycles, device_data->scan_interval); sla_val *= device_data->max_tp[svc_type]; @@ -617,9 +587,8 @@ u32 adf_rl_calculate_pci_bw(struct adf_accel_dev *accel_dev, u32 sla_val, sla_to_bytes *= device_data->max_tp[svc_type]; do_div(sla_to_bytes, device_data->scale_ref); - sla_to_bytes *= (svc_type == ADF_SVC_ASYM) ? RL_TOKEN_ASYM_SIZE : - BYTES_PER_MBIT; - if (svc_type == ADF_SVC_DC && is_bw_out) + sla_to_bytes *= (svc_type == SVC_ASYM) ? RL_TOKEN_ASYM_SIZE : BYTES_PER_MBIT; + if (svc_type == SVC_DC && is_bw_out) sla_to_bytes *= device_data->slices.dcpr_cnt - device_data->dcpr_correction; @@ -660,7 +629,7 @@ static int add_new_sla_entry(struct adf_accel_dev *accel_dev, } *sla_out = sla; - if (!is_service_enabled(accel_dev, sla_in->srv)) { + if (!adf_is_service_enabled(accel_dev, sla_in->srv)) { dev_notice(&GET_DEV(accel_dev), "Provided service is not enabled\n"); ret = -EINVAL; @@ -730,8 +699,8 @@ static int initialize_default_nodes(struct adf_accel_dev *accel_dev) sla_in.type = RL_ROOT; sla_in.parent_id = RL_PARENT_DEFAULT_ID; - for (i = 0; i < ADF_SVC_NONE; i++) { - if (!is_service_enabled(accel_dev, i)) + for (i = 0; i < SVC_BASE_COUNT; i++) { + if (!adf_is_service_enabled(accel_dev, i)) continue; sla_in.cir = device_data->scale_ref; @@ -745,10 +714,9 @@ static int initialize_default_nodes(struct adf_accel_dev *accel_dev) /* Init default cluster for each root */ sla_in.type = RL_CLUSTER; - for (i = 0; i < ADF_SVC_NONE; i++) { + for (i = 0; i < SVC_BASE_COUNT; i++) { if (!rl_data->root[i]) continue; - sla_in.cir = rl_data->root[i]->cir; sla_in.pir = sla_in.cir; sla_in.srv = rl_data->root[i]->srv; @@ -987,7 +955,7 @@ int adf_rl_get_capability_remaining(struct adf_accel_dev *accel_dev, struct rl_sla *sla = NULL; int i; - if (srv >= ADF_SVC_NONE) + if (srv >= SVC_BASE_COUNT) return -EINVAL; if (sla_id > RL_SLA_EMPTY_ID && !validate_sla_id(accel_dev, sla_id)) { @@ -1086,9 +1054,9 @@ int adf_rl_init(struct adf_accel_dev *accel_dev) int ret = 0; /* Validate device parameters */ - if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_ASYM]) || - RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_SYM]) || - RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_DC]) || + if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_ASYM]) || + RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_SYM]) || + RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_DC]) || RL_VALIDATE_NON_ZERO(rl_hw_data->scan_interval) || RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_div) || RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_mul) || diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.h b/drivers/crypto/intel/qat/qat_common/adf_rl.h index bfe750ea0e83..c1f3f9a51195 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_rl.h +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.h @@ -7,6 +7,8 @@ #include <linux/mutex.h> #include <linux/types.h> +#include "adf_cfg_services.h" + struct adf_accel_dev; #define RL_ROOT_MAX 4 @@ -24,13 +26,6 @@ enum rl_node_type { RL_LEAF, }; -enum adf_base_services { - ADF_SVC_ASYM = 0, - ADF_SVC_SYM, - ADF_SVC_DC, - ADF_SVC_NONE, -}; - /** * struct adf_rl_sla_input_data - ratelimiting user input data structure * @rp_mask: 64 bit bitmask of ring pair IDs which will be assigned to SLA. @@ -73,6 +68,7 @@ struct rl_slice_cnt { u8 dcpr_cnt; u8 pke_cnt; u8 cph_cnt; + u8 cpr_cnt; }; struct adf_rl_interface_data { @@ -94,6 +90,7 @@ struct adf_rl_hw_data { u32 pcie_scale_div; u32 dcpr_correction; u32 max_tp[RL_ROOT_MAX]; + u32 svc_ae_mask[SVC_BASE_COUNT]; struct rl_slice_cnt slices; }; diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl_admin.c b/drivers/crypto/intel/qat/qat_common/adf_rl_admin.c index 698a14f4ce66..4a3e0591fdba 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_rl_admin.c +++ b/drivers/crypto/intel/qat/qat_common/adf_rl_admin.c @@ -63,6 +63,7 @@ int adf_rl_send_admin_init_msg(struct adf_accel_dev *accel_dev, slices_int->pke_cnt = slices_resp.pke_cnt; /* For symmetric crypto, slice tokens are relative to the UCS slice */ slices_int->cph_cnt = slices_resp.ucs_cnt; + slices_int->cpr_cnt = slices_resp.cpr_cnt; return 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c index c75d0b6cb0ad..bb904ba4bf84 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c @@ -155,7 +155,6 @@ static int adf_do_enable_sriov(struct adf_accel_dev *accel_dev) if (!device_iommu_mapped(&GET_DEV(accel_dev))) { dev_warn(&GET_DEV(accel_dev), "IOMMU should be enabled for SR-IOV to work correctly\n"); - return -EINVAL; } if (adf_dev_started(accel_dev)) { @@ -300,7 +299,8 @@ EXPORT_SYMBOL_GPL(adf_sriov_configure); int __init adf_init_pf_wq(void) { /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", WQ_MEM_RECLAIM, 0); + pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !pf2vf_resp_wq ? -ENOMEM : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c index 4fcd61ff70d1..79c63dfa8ff3 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c @@ -3,6 +3,7 @@ #include <linux/device.h> #include <linux/errno.h> #include <linux/pci.h> +#include <linux/string_choices.h> #include "adf_accel_devices.h" #include "adf_cfg.h" #include "adf_cfg_services.h" @@ -19,14 +20,12 @@ static ssize_t state_show(struct device *dev, struct device_attribute *attr, char *buf) { struct adf_accel_dev *accel_dev; - char *state; accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); if (!accel_dev) return -EINVAL; - state = adf_dev_started(accel_dev) ? "up" : "down"; - return sysfs_emit(buf, "%s\n", state); + return sysfs_emit(buf, "%s\n", str_up_down(adf_dev_started(accel_dev))); } static ssize_t state_store(struct device *dev, struct device_attribute *attr, @@ -117,25 +116,27 @@ static int adf_sysfs_update_dev_config(struct adf_accel_dev *accel_dev, static ssize_t cfg_services_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; struct adf_hw_device_data *hw_data; struct adf_accel_dev *accel_dev; int ret; - ret = sysfs_match_string(adf_cfg_services, buf); - if (ret < 0) - return ret; - accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); if (!accel_dev) return -EINVAL; + ret = adf_parse_service_string(accel_dev, buf, count, services, + ADF_CFG_MAX_VAL_LEN_IN_BYTES); + if (ret) + return ret; + if (adf_dev_started(accel_dev)) { dev_info(dev, "Device qat_dev%d must be down to reconfigure the service.\n", accel_dev->accel_id); return -EINVAL; } - ret = adf_sysfs_update_dev_config(accel_dev, adf_cfg_services[ret]); + ret = adf_sysfs_update_dev_config(accel_dev, services); if (ret < 0) return ret; @@ -207,16 +208,13 @@ static DEVICE_ATTR_RW(pm_idle_enabled); static ssize_t auto_reset_show(struct device *dev, struct device_attribute *attr, char *buf) { - char *auto_reset; struct adf_accel_dev *accel_dev; accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); if (!accel_dev) return -EINVAL; - auto_reset = accel_dev->autoreset_on_error ? "on" : "off"; - - return sysfs_emit(buf, "%s\n", auto_reset); + return sysfs_emit(buf, "%s\n", str_on_off(accel_dev->autoreset_on_error)); } static ssize_t auto_reset_store(struct device *dev, struct device_attribute *attr, @@ -271,6 +269,8 @@ static ssize_t rp2srv_show(struct device *dev, struct device_attribute *attr, return sysfs_emit(buf, "%s\n", ADF_CFG_SYM); case ASYM: return sysfs_emit(buf, "%s\n", ADF_CFG_ASYM); + case DECOMP: + return sysfs_emit(buf, "%s\n", ADF_CFG_DECOMP); default: break; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c index bedb514d4e30..f31556beed8b 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c @@ -32,9 +32,10 @@ enum rl_params { }; static const char *const rl_services[] = { - [ADF_SVC_ASYM] = "asym", - [ADF_SVC_SYM] = "sym", - [ADF_SVC_DC] = "dc", + [SVC_ASYM] = "asym", + [SVC_SYM] = "sym", + [SVC_DC] = "dc", + [SVC_DECOMP] = "decomp", }; static const char *const rl_operations[] = { @@ -282,7 +283,7 @@ static ssize_t srv_show(struct device *dev, struct device_attribute *attr, if (ret) return ret; - if (get == ADF_SVC_NONE) + if (get == SVC_BASE_COUNT) return -EINVAL; return sysfs_emit(buf, "%s\n", rl_services[get]); @@ -291,14 +292,22 @@ static ssize_t srv_show(struct device *dev, struct device_attribute *attr, static ssize_t srv_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { + struct adf_accel_dev *accel_dev; unsigned int val; int ret; + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + ret = sysfs_match_string(rl_services, buf); if (ret < 0) return ret; val = ret; + if (!adf_is_service_enabled(accel_dev, val)) + return -EINVAL; + ret = set_param_u(dev, SRV, val); if (ret) return ret; @@ -439,8 +448,8 @@ int adf_sysfs_rl_add(struct adf_accel_dev *accel_dev) dev_err(&GET_DEV(accel_dev), "Failed to create qat_rl attribute group\n"); - data->cap_rem_srv = ADF_SVC_NONE; - data->input.srv = ADF_SVC_NONE; + data->cap_rem_srv = SVC_BASE_COUNT; + data->input.srv = SVC_BASE_COUNT; data->sysfs_added = true; return ret; diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c index 74fb0c2ed241..b64142db1f0d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c @@ -212,6 +212,23 @@ int adf_tl_halt(struct adf_accel_dev *accel_dev) return ret; } +static void adf_set_cmdq_cnt(struct adf_accel_dev *accel_dev, + struct adf_tl_hw_data *tl_data) +{ + struct icp_qat_fw_init_admin_slice_cnt *slice_cnt, *cmdq_cnt; + + slice_cnt = &accel_dev->telemetry->slice_cnt; + cmdq_cnt = &accel_dev->telemetry->cmdq_cnt; + + cmdq_cnt->cpr_cnt = slice_cnt->cpr_cnt * tl_data->multiplier.cpr_cnt; + cmdq_cnt->dcpr_cnt = slice_cnt->dcpr_cnt * tl_data->multiplier.dcpr_cnt; + cmdq_cnt->pke_cnt = slice_cnt->pke_cnt * tl_data->multiplier.pke_cnt; + cmdq_cnt->wat_cnt = slice_cnt->wat_cnt * tl_data->multiplier.wat_cnt; + cmdq_cnt->wcp_cnt = slice_cnt->wcp_cnt * tl_data->multiplier.wcp_cnt; + cmdq_cnt->ucs_cnt = slice_cnt->ucs_cnt * tl_data->multiplier.ucs_cnt; + cmdq_cnt->ath_cnt = slice_cnt->ath_cnt * tl_data->multiplier.ath_cnt; +} + int adf_tl_run(struct adf_accel_dev *accel_dev, int state) { struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); @@ -235,6 +252,8 @@ int adf_tl_run(struct adf_accel_dev *accel_dev, int state) return ret; } + adf_set_cmdq_cnt(accel_dev, tl_data); + telemetry->hbuffs = state; atomic_set(&telemetry->state, state); diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h index e54a406cc1b4..02d75c3c214a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h @@ -28,19 +28,23 @@ struct dentry; struct adf_tl_hw_data { size_t layout_sz; size_t slice_reg_sz; + size_t cmdq_reg_sz; size_t rp_reg_sz; size_t msg_cnt_off; const struct adf_tl_dbg_counter *dev_counters; const struct adf_tl_dbg_counter *sl_util_counters; const struct adf_tl_dbg_counter *sl_exec_counters; + const struct adf_tl_dbg_counter **cmdq_counters; const struct adf_tl_dbg_counter *rp_counters; u8 num_hbuff; u8 cpp_ns_per_cycle; u8 bw_units_to_bytes; u8 num_dev_counters; u8 num_rp_counters; + u8 num_cmdq_counters; u8 max_rp; u8 max_sl_cnt; + struct icp_qat_fw_init_admin_slice_cnt multiplier; }; struct adf_telemetry { @@ -69,6 +73,7 @@ struct adf_telemetry { struct mutex wr_lock; struct delayed_work work_ctx; struct icp_qat_fw_init_admin_slice_cnt slice_cnt; + struct icp_qat_fw_init_admin_slice_cnt cmdq_cnt; }; #ifdef CONFIG_DEBUG_FS diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c b/drivers/crypto/intel/qat/qat_common/adf_timer.c index 35ccb91d6ec1..8962a49f145a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.c @@ -12,9 +12,9 @@ #include "adf_admin.h" #include "adf_accel_devices.h" #include "adf_common_drv.h" -#include "adf_gen4_timer.h" +#include "adf_timer.h" -#define ADF_GEN4_TIMER_PERIOD_MS 200 +#define ADF_DEFAULT_TIMER_PERIOD_MS 200 /* This periodic update is used to trigger HB, RL & TL fw events */ static void work_handler(struct work_struct *work) @@ -27,16 +27,16 @@ static void work_handler(struct work_struct *work) accel_dev = timer_ctx->accel_dev; adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, - msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); time_periods = div_u64(ktime_ms_delta(ktime_get_real(), timer_ctx->initial_ktime), - ADF_GEN4_TIMER_PERIOD_MS); + ADF_DEFAULT_TIMER_PERIOD_MS); if (adf_send_admin_tim_sync(accel_dev, time_periods)) dev_err(&GET_DEV(accel_dev), "Failed to synchronize qat timer\n"); } -int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) +int adf_timer_start(struct adf_accel_dev *accel_dev) { struct adf_timer *timer_ctx; @@ -50,13 +50,13 @@ int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) INIT_DELAYED_WORK(&timer_ctx->work_ctx, work_handler); adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, - msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); return 0; } -EXPORT_SYMBOL_GPL(adf_gen4_timer_start); +EXPORT_SYMBOL_GPL(adf_timer_start); -void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) +void adf_timer_stop(struct adf_accel_dev *accel_dev) { struct adf_timer *timer_ctx = accel_dev->timer; @@ -68,4 +68,4 @@ void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) kfree(timer_ctx); accel_dev->timer = NULL; } -EXPORT_SYMBOL_GPL(adf_gen4_timer_stop); +EXPORT_SYMBOL_GPL(adf_timer_stop); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h b/drivers/crypto/intel/qat/qat_common/adf_timer.h index 66a709e7b358..68e5136d6ba1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* Copyright(c) 2023 Intel Corporation */ -#ifndef ADF_GEN4_TIMER_H_ -#define ADF_GEN4_TIMER_H_ +#ifndef ADF_TIMER_H_ +#define ADF_TIMER_H_ #include <linux/ktime.h> #include <linux/workqueue.h> @@ -15,7 +15,7 @@ struct adf_timer { ktime_t initial_ktime; }; -int adf_gen4_timer_start(struct adf_accel_dev *accel_dev); -void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev); +int adf_timer_start(struct adf_accel_dev *accel_dev); +void adf_timer_stop(struct adf_accel_dev *accel_dev); -#endif /* ADF_GEN4_TIMER_H_ */ +#endif /* ADF_TIMER_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c index c8241f5a0a26..b81f70576683 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c @@ -339,6 +339,48 @@ static int tl_calc_and_print_sl_counters(struct adf_accel_dev *accel_dev, return 0; } +static int tl_print_cmdq_counter(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct seq_file *s, u8 cnt_id, u8 counter) +{ + size_t cmdq_regs_sz = GET_TL_DATA(telemetry->accel_dev).cmdq_reg_sz; + size_t offset_inc = cnt_id * cmdq_regs_sz; + struct adf_tl_dbg_counter slice_ctr; + char cnt_name[MAX_COUNT_NAME_SIZE]; + + slice_ctr = *(ctr + counter); + slice_ctr.offset1 += offset_inc; + snprintf(cnt_name, MAX_COUNT_NAME_SIZE, "%s%d", slice_ctr.name, cnt_id); + + return tl_calc_and_print_counter(telemetry, s, &slice_ctr, cnt_name); +} + +static int tl_calc_and_print_cmdq_counters(struct adf_accel_dev *accel_dev, + struct seq_file *s, u8 cnt_type, + u8 cnt_id) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter **cmdq_tl_counters; + const struct adf_tl_dbg_counter *ctr; + u8 counter; + int ret; + + cmdq_tl_counters = tl_data->cmdq_counters; + ctr = cmdq_tl_counters[cnt_type]; + + for (counter = 0; counter < tl_data->num_cmdq_counters; counter++) { + ret = tl_print_cmdq_counter(telemetry, ctr, s, cnt_id, counter); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid slice utilization counter type\n"); + return ret; + } + } + + return 0; +} + static void tl_print_msg_cnt(struct seq_file *s, u32 msg_cnt) { seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, SNAPSHOT_CNT_MSG); @@ -352,6 +394,7 @@ static int tl_print_dev_data(struct adf_accel_dev *accel_dev, struct adf_telemetry *telemetry = accel_dev->telemetry; const struct adf_tl_dbg_counter *dev_tl_counters; u8 num_dev_counters = tl_data->num_dev_counters; + u8 *cmdq_cnt = (u8 *)&telemetry->cmdq_cnt; u8 *sl_cnt = (u8 *)&telemetry->slice_cnt; const struct adf_tl_dbg_counter *ctr; unsigned int i; @@ -387,6 +430,15 @@ static int tl_print_dev_data(struct adf_accel_dev *accel_dev, } } + /* Print per command queue telemetry. */ + for (i = 0; i < ADF_TL_SL_CNT_COUNT; i++) { + for (j = 0; j < cmdq_cnt[i]; j++) { + ret = tl_calc_and_print_cmdq_counters(accel_dev, s, i, j); + if (ret) + return ret; + } + } + return 0; } @@ -473,22 +525,6 @@ unlock_and_exit: } DEFINE_SHOW_STORE_ATTRIBUTE(tl_control); -static int get_rp_index_from_file(const struct file *f, u8 *rp_id, u8 rp_num) -{ - char alpha; - u8 index; - int ret; - - ret = sscanf(f->f_path.dentry->d_name.name, ADF_TL_RP_REGS_FNAME, &alpha); - if (ret != 1) - return -EINVAL; - - index = ADF_TL_DBG_RP_INDEX_ALPHA(alpha); - *rp_id = index; - - return 0; -} - static int adf_tl_dbg_change_rp_index(struct adf_accel_dev *accel_dev, unsigned int new_rp_num, unsigned int rp_regs_index) @@ -554,6 +590,9 @@ static void tl_print_rp_srv(struct adf_accel_dev *accel_dev, struct seq_file *s, case ASYM: seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_ASYM); break; + case DECOMP: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_DECOMP); + break; default: seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, TL_RP_SRV_UNKNOWN); break; @@ -611,18 +650,11 @@ static int tl_rp_data_show(struct seq_file *s, void *unused) { struct adf_accel_dev *accel_dev = s->private; u8 rp_regs_index; - u8 max_rp; - int ret; if (!accel_dev) return -EINVAL; - max_rp = GET_TL_DATA(accel_dev).max_rp; - ret = get_rp_index_from_file(s->file, &rp_regs_index, max_rp); - if (ret) { - dev_dbg(&GET_DEV(accel_dev), "invalid RP data file name\n"); - return ret; - } + rp_regs_index = debugfs_get_aux_num(s->file); return tl_print_rp_data(accel_dev, s, rp_regs_index); } @@ -635,7 +667,6 @@ static ssize_t tl_rp_data_write(struct file *file, const char __user *userbuf, struct adf_telemetry *telemetry; unsigned int new_rp_num; u8 rp_regs_index; - u8 max_rp; int ret; accel_dev = seq_f->private; @@ -643,15 +674,10 @@ static ssize_t tl_rp_data_write(struct file *file, const char __user *userbuf, return -EINVAL; telemetry = accel_dev->telemetry; - max_rp = GET_TL_DATA(accel_dev).max_rp; mutex_lock(&telemetry->wr_lock); - ret = get_rp_index_from_file(file, &rp_regs_index, max_rp); - if (ret) { - dev_dbg(&GET_DEV(accel_dev), "invalid RP data file name\n"); - goto unlock_and_exit; - } + rp_regs_index = debugfs_get_aux_num(file); ret = kstrtou32_from_user(userbuf, count, 10, &new_rp_num); if (ret) @@ -689,7 +715,8 @@ void adf_tl_dbgfs_add(struct adf_accel_dev *accel_dev) for (i = 0; i < max_rp; i++) { snprintf(name, sizeof(name), ADF_TL_RP_REGS_FNAME, ADF_TL_DBG_RP_ALPHA_INDEX(i)); - debugfs_create_file(name, 0644, dir, accel_dev, &tl_rp_data_fops); + debugfs_create_file_aux_num(name, 0644, dir, accel_dev, i, + &tl_rp_data_fops); } } diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h index 11cc9eae19b3..97c5eeaa1b17 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h @@ -17,6 +17,7 @@ struct adf_accel_dev; #define LAT_ACC_NAME "gp_lat_acc_avg" #define BW_IN_NAME "bw_in" #define BW_OUT_NAME "bw_out" +#define RE_ACC_NAME "re_acc_avg" #define PAGE_REQ_LAT_NAME "at_page_req_lat_avg" #define AT_TRANS_LAT_NAME "at_trans_lat_avg" #define AT_MAX_UTLB_USED_NAME "at_max_tlb_used" @@ -43,6 +44,10 @@ struct adf_accel_dev; (ADF_TL_DEV_REG_OFF(slice##_slices[0], qat_gen) + \ offsetof(struct adf_##qat_gen##_tl_slice_data_regs, reg)) +#define ADF_TL_CMDQ_REG_OFF(slice, reg, qat_gen) \ + (ADF_TL_DEV_REG_OFF(slice##_cmdq[0], qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_cmdq_data_regs, reg)) + #define ADF_TL_RP_REG_OFF(reg, qat_gen) \ (ADF_TL_DATA_REG_OFF(tl_ring_pairs_data_regs[0], qat_gen) + \ offsetof(struct adf_##qat_gen##_tl_ring_pair_data_regs, reg)) diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c index e2dd568b87b5..6c22bc9b28e4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c @@ -10,16 +10,21 @@ static DEFINE_MUTEX(ring_read_lock); static DEFINE_MUTEX(bank_read_lock); +#define ADF_RING_NUM_MSGS(ring) \ + (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / \ + ADF_MSG_SIZE_TO_BYTES(ring->msg_size)) + static void *adf_ring_start(struct seq_file *sfile, loff_t *pos) { struct adf_etr_ring_data *ring = sfile->private; + unsigned int num_msg = ADF_RING_NUM_MSGS(ring); + loff_t val = *pos; mutex_lock(&ring_read_lock); - if (*pos == 0) + if (val == 0) return SEQ_START_TOKEN; - if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / - ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) + if (val >= num_msg) return NULL; return ring->base_addr + @@ -29,13 +34,15 @@ static void *adf_ring_start(struct seq_file *sfile, loff_t *pos) static void *adf_ring_next(struct seq_file *sfile, void *v, loff_t *pos) { struct adf_etr_ring_data *ring = sfile->private; + unsigned int num_msg = ADF_RING_NUM_MSGS(ring); + loff_t val = *pos; + + (*pos)++; - if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / - ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) + if (val >= num_msg) return NULL; - return ring->base_addr + - (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++); + return ring->base_addr + (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * val); } static int adf_ring_show(struct seq_file *sfile, void *v) diff --git a/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c index a4636ec9f9ca..d0fef20a3df4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c @@ -299,7 +299,8 @@ EXPORT_SYMBOL_GPL(adf_flush_vf_wq); */ int __init adf_init_vf_wq(void) { - adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); + adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !adf_vf_stop_wq ? -EFAULT : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h index a03d43fef2b3..81969c515a17 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h @@ -16,8 +16,8 @@ enum icp_qat_fw_comp_20_cmd_id { ICP_QAT_FW_COMP_20_CMD_LZ4_DECOMPRESS = 4, ICP_QAT_FW_COMP_20_CMD_LZ4S_COMPRESS = 5, ICP_QAT_FW_COMP_20_CMD_LZ4S_DECOMPRESS = 6, - ICP_QAT_FW_COMP_20_CMD_XP10_COMPRESS = 7, - ICP_QAT_FW_COMP_20_CMD_XP10_DECOMPRESS = 8, + ICP_QAT_FW_COMP_20_CMD_RESERVED_7 = 7, + ICP_QAT_FW_COMP_20_CMD_RESERVED_8 = 8, ICP_QAT_FW_COMP_20_CMD_RESERVED_9 = 9, ICP_QAT_FW_COMP_23_CMD_ZSTD_COMPRESS = 10, ICP_QAT_FW_COMP_23_CMD_ZSTD_DECOMPRESS = 11, @@ -44,6 +44,7 @@ enum icp_qat_fw_comp_20_cmd_id { #define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK 0x1 #define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS 7 #define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK 0x1 +#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MAX_VALUE 0xFFFFFFFF #define ICP_QAT_FW_COMP_FLAGS_BUILD(sesstype, autoselect, enhanced_asb, \ ret_uncomp, secure_ram) \ @@ -117,7 +118,7 @@ struct icp_qat_fw_comp_req_params { #define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop, eop, bfinal, cnv, cnvnr, \ cnvdfx, crc, xxhash_acc, \ cnv_error_type, append_crc, \ - drop_data) \ + drop_data, partial_decomp) \ ((((sop) & ICP_QAT_FW_COMP_SOP_MASK) << \ ICP_QAT_FW_COMP_SOP_BITPOS) | \ (((eop) & ICP_QAT_FW_COMP_EOP_MASK) << \ @@ -139,7 +140,9 @@ struct icp_qat_fw_comp_req_params { (((append_crc) & ICP_QAT_FW_COMP_APPEND_CRC_MASK) \ << ICP_QAT_FW_COMP_APPEND_CRC_BITPOS) | \ (((drop_data) & ICP_QAT_FW_COMP_DROP_DATA_MASK) \ - << ICP_QAT_FW_COMP_DROP_DATA_BITPOS)) + << ICP_QAT_FW_COMP_DROP_DATA_BITPOS) | \ + (((partial_decomp) & ICP_QAT_FW_COMP_PARTIAL_DECOMP_MASK) \ + << ICP_QAT_FW_COMP_PARTIAL_DECOMP_BITPOS)) #define ICP_QAT_FW_COMP_NOT_SOP 0 #define ICP_QAT_FW_COMP_SOP 1 @@ -161,6 +164,8 @@ struct icp_qat_fw_comp_req_params { #define ICP_QAT_FW_COMP_NO_APPEND_CRC 0 #define ICP_QAT_FW_COMP_DROP_DATA 1 #define ICP_QAT_FW_COMP_NO_DROP_DATA 0 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMPRESS 1 +#define ICP_QAT_FW_COMP_NO_PARTIAL_DECOMPRESS 0 #define ICP_QAT_FW_COMP_SOP_BITPOS 0 #define ICP_QAT_FW_COMP_SOP_MASK 0x1 #define ICP_QAT_FW_COMP_EOP_BITPOS 1 @@ -189,6 +194,8 @@ struct icp_qat_fw_comp_req_params { #define ICP_QAT_FW_COMP_APPEND_CRC_MASK 0x1 #define ICP_QAT_FW_COMP_DROP_DATA_BITPOS 25 #define ICP_QAT_FW_COMP_DROP_DATA_MASK 0x1 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMP_BITPOS 27 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMP_MASK 0x1 #define ICP_QAT_FW_COMP_SOP_GET(flags) \ QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_SOP_BITPOS, \ @@ -281,8 +288,18 @@ struct icp_qat_fw_comp_req { union { struct icp_qat_fw_xlt_req_params xlt_pars; __u32 resrvd1[ICP_QAT_FW_NUM_LONGWORDS_2]; + struct { + __u32 partial_decompress_length; + __u32 partial_decompress_offset; + } partial_decompress; } u1; - __u32 resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2]; + union { + __u32 resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2]; + struct { + __u32 asb_value; + __u32 reserved; + } asb_threshold; + } u3; struct icp_qat_fw_comp_cd_hdr comp_cd_ctrl; union { struct icp_qat_fw_xlt_cd_hdr xlt_cd_ctrl; diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h index 7eb5daef4f88..6887930c7995 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h @@ -35,6 +35,7 @@ struct icp_qat_fw_loader_chip_info { u32 wakeup_event_val; bool fw_auth; bool css_3k; + bool dual_sign; bool tgroup_share_ustore; u32 fcu_ctl_csr; u32 fcu_sts_csr; diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h new file mode 100644 index 000000000000..dce639152345 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ICP_QAT_HW_51_COMP_H_ +#define ICP_QAT_HW_51_COMP_H_ + +#include <linux/types.h> + +#include "icp_qat_fw.h" +#include "icp_qat_hw_51_comp_defs.h" + +struct icp_qat_hw_comp_51_config_csr_lower { + enum icp_qat_hw_comp_51_abd abd; + enum icp_qat_hw_comp_51_lllbd_ctrl lllbd; + enum icp_qat_hw_comp_51_search_depth sd; + enum icp_qat_hw_comp_51_min_match_control mmctrl; + enum icp_qat_hw_comp_51_lz4_block_checksum lbc; +}; + +static inline u32 +ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(struct icp_qat_hw_comp_51_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.abd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_MASK); + QAT_FIELD_SET(val32, csr.lllbd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_MASK); + QAT_FIELD_SET(val32, csr.sd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_MASK); + QAT_FIELD_SET(val32, csr.mmctrl, + ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.lbc, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK); + + return val32; +} + +struct icp_qat_hw_comp_51_config_csr_upper { + enum icp_qat_hw_comp_51_dmm_algorithm edmm; + enum icp_qat_hw_comp_51_bms bms; + enum icp_qat_hw_comp_51_scb_mode_reset_mask scb_mode_reset; +}; + +static inline u32 +ICP_QAT_FW_COMP_51_BUILD_CONFIG_UPPER(struct icp_qat_hw_comp_51_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.edmm, + ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_MASK); + QAT_FIELD_SET(val32, csr.bms, + ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_MASK); + QAT_FIELD_SET(val32, csr.scb_mode_reset, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK); + + return val32; +} + +struct icp_qat_hw_decomp_51_config_csr_lower { + enum icp_qat_hw_decomp_51_lz4_block_checksum lbc; +}; + +static inline u32 +ICP_QAT_FW_DECOMP_51_BUILD_CONFIG_LOWER(struct icp_qat_hw_decomp_51_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.lbc, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK); + + return val32; +} + +struct icp_qat_hw_decomp_51_config_csr_upper { + enum icp_qat_hw_decomp_51_bms bms; +}; + +static inline u32 +ICP_QAT_FW_DECOMP_51_BUILD_CONFIG_UPPER(struct icp_qat_hw_decomp_51_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.bms, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_BITPOS, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_MASK); + + return val32; +} + +#endif /* ICP_QAT_HW_51_COMP_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h new file mode 100644 index 000000000000..e745688c5da4 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ICP_QAT_HW_51_COMP_DEFS_H_ +#define ICP_QAT_HW_51_COMP_DEFS_H_ + +#include <linux/bits.h> + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_BITPOS 28 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_MASK GENMASK(1, 0) +enum icp_qat_hw_comp_51_som_control { + ICP_QAT_HW_COMP_51_SOM_CONTROL_NORMAL_MODE = 0x0, + ICP_QAT_HW_COMP_51_SOM_CONTROL_DICTIONARY_MODE = 0x1, + ICP_QAT_HW_COMP_51_SOM_CONTROL_INPUT_CRC = 0x2, + ICP_QAT_HW_COMP_51_SOM_CONTROL_RESERVED_MODE = 0x3, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SOM_CONTROL_NORMAL_MODE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS 27 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_rd_control { + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_NO_SKIP = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_SKIP_HASH_READS = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_NO_SKIP +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_BITPOS 25 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_bypass_compression { + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_DISABLED = 0x0, + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_ENABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_DISABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_BITPOS 22 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_dmm_algorithm { + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_EDMM_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_ZSTD_DMM_LITE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_EDMM_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_token_fusion_internal_only { + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_BITPOS 19 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_MASK GENMASK(1, 0) +enum icp_qat_hw_comp_51_bms { + ICP_QAT_HW_COMP_51_BMS_BMS_64KB = 0x0, + ICP_QAT_HW_COMP_51_BMS_BMS_256KB = 0x1, + ICP_QAT_HW_COMP_51_BMS_BMS_1MB = 0x2, + ICP_QAT_HW_COMP_51_BMS_BMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BMS_BMS_64KB +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_scb_mode_reset_mask { + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_DO_NOT_RESET_HB_HT = 0x0, + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_RESET_HB_HT = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_DO_NOT_RESET_HB_HT +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_BITPOS 2 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_zstd_frame_gen_dec_en { + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_DISABLE = 0x0, + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_BITPOS 1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_cnv_disable { + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_BITPOS 0 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_asb_disable { + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_spec_decoder_internal_only { + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_BITPOS 20 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_mini_xcam_internal_only { + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_BITPOS 19 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_rep_off_enc_internal_only { + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_BITPOS 18 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_prog_block_drop_internal_only { + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_DISABLE = 0x0, + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_DISABLE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_BITPOS 17 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_override_internal_only { + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DETERMINE_HASH_PARAMS = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_OVERRIDE_HASH_PARAMS = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DETERMINE_HASH_PARAMS +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_BITPOS 14 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_MASK GENMASK(2, 0) +enum icp_qat_hw_comp_51_hbs { + ICP_QAT_HW_COMP_51_HBS_32KB = 0x0, + ICP_QAT_HW_COMP_51_HBS_64KB = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_HBS_32KB +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_BITPOS 13 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_abd { + ICP_QAT_HW_COMP_51_ABD_ABD_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_ABD_ABD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ABD_ABD_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_BITPOS 12 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_lllbd_ctrl { + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_BITPOS 8 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_MASK GENMASK(3, 0) +enum icp_qat_hw_comp_51_search_depth { + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1 = 0x1, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_6 = 0x3, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_9 = 0x4, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_10 = 0x4, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_BITPOS 5 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_MASK GENMASK(2, 0) +enum icp_qat_hw_comp_51_format { + ICP_QAT_HW_COMP_51_FORMAT_ILZ77 = 0x1, + ICP_QAT_HW_COMP_51_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_COMP_51_FORMAT_LZ4s = 0x3, + ICP_QAT_HW_COMP_51_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_FORMAT_ILZ77 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_min_match_control { + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_3B = 0x0, + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_4B = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_3B +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_collision { + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_ALLOW = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_ALLOW +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS 2 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_update { + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_ALLOW = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_ALLOW +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_BITPOS 1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_byte_skip { + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_TOKEN = 0x0, + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_LITERAL = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_TOKEN +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS 0 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_lz4_block_checksum { + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_ABSENT = 0x0, + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_PRESENT = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_ABSENT +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_BITPOS 26 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_discard_data { + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_DISABLED = 0x0, + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_ENABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_DISABLED +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_BITPOS 19 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_MASK GENMASK(1, 0) +enum icp_qat_hw_decomp_51_bms { + ICP_QAT_HW_DECOMP_51_BMS_BMS_64KB = 0x0, + ICP_QAT_HW_DECOMP_51_BMS_BMS_256KB = 0x1, + ICP_QAT_HW_DECOMP_51_BMS_BMS_1MB = 0x2, + ICP_QAT_HW_DECOMP_51_BMS_BMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_BMS_BMS_64KB +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_BITPOS 2 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_zstd_frame_gen_dec_en { + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_DISABLE = 0x0, + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_spec_decoder_internal_only { + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_BITPOS 20 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_mini_xcam_internal_only { + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_BITPOS 14 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_MASK GENMASK(2, 0) +enum icp_qat_hw_decomp_51_hbs { + ICP_QAT_HW_DECOMP_51_HBS_32KB = 0x0, + ICP_QAT_HW_DECOMP_51_HBS_64KB = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_HBS_32KB +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_BITPOS 5 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_MASK GENMASK(2, 0) +enum icp_qat_hw_decomp_51_format { + ICP_QAT_HW_DECOMP_51_FORMAT_ILZ77 = 0x1, + ICP_QAT_HW_DECOMP_51_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_DECOMP_51_FORMAT_RESERVED = 0x3, + ICP_QAT_HW_DECOMP_51_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_FORMAT_ILZ77 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS 0 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_lz4_block_checksum { + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_ABSENT = 0x0, + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_PRESENT = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_ABSENT + +#endif /* ICP_QAT_HW_51_COMP_DEFS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h index e28241bdd0f4..6313c35eff0c 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h @@ -7,6 +7,7 @@ #define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 #define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 +#define ICP_QAT_AC_6XXX_DEV_TYPE 0x80000000 #define ICP_QAT_UCLO_MAX_AE 17 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) @@ -43,7 +44,6 @@ #define ICP_QAT_SUOF_OBJS "SUF_OBJS" #define ICP_QAT_SUOF_IMAG "SUF_IMAG" #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) -#define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) #define DSS_FWSK_MODULUS_LEN 384 /* RSA3K */ #define DSS_FWSK_EXPONENT_LEN 4 @@ -75,13 +75,6 @@ DSS_SIGNATURE_LEN : \ CSS_SIGNATURE_LEN) -#define ICP_QAT_CSS_AE_IMG_LEN (sizeof(struct icp_qat_simg_ae_mode) + \ - ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \ - ICP_QAT_SIMG_AE_INSTS_LEN) -#define ICP_QAT_CSS_AE_SIMG_LEN(handle) (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_PUB_LEN(handle) + \ - ICP_QAT_CSS_SIGNATURE_LEN(handle) + \ - ICP_QAT_CSS_AE_IMG_LEN) #define ICP_QAT_AE_IMG_OFFSET(handle) (sizeof(struct icp_qat_css_hdr) + \ ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ @@ -89,6 +82,21 @@ #define ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN 0x30000 +/* All lengths below are in bytes */ +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN 12 +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN 16 +#define ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN 3540 +#define ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN 64 +#define ICP_QAT_DUALSIGN_XMSS_SIG_LEN 2692 +#define ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN 2696 +#define ICP_QAT_DUALSIGN_MISC_INFO_LEN 16 +#define ICP_QAT_DUALSIGN_FW_TYPE_LEN 7 +#define ICP_QAT_DUALSIGN_MODULE_TYPE 0x14 +#define ICP_QAT_DUALSIGN_HDR_LEN 0x375 +#define ICP_QAT_DUALSIGN_HDR_VER 0x40001 +#define ICP_QAT_DUALSIGN_HDR_LEN_OFFSET 4 +#define ICP_QAT_DUALSIGN_HDR_VER_OFFSET 8 + #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) #define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) #define ICP_QAT_SHARED_USTORE_MODE(ae_mode) (((ae_mode) >> 0xb) & 0x1) @@ -404,8 +412,6 @@ struct icp_qat_suof_img_hdr { char *simg_buf; unsigned long simg_len; char *css_header; - char *css_key; - char *css_signature; char *css_simg; unsigned long simg_size; unsigned int ae_num; @@ -450,6 +456,13 @@ struct icp_qat_fw_auth_desc { unsigned int img_ae_init_data_low; unsigned int img_ae_insts_high; unsigned int img_ae_insts_low; + unsigned int cpp_mask; + unsigned int reserved; + unsigned int xmss_pubkey_high; + unsigned int xmss_pubkey_low; + unsigned int xmss_sig_high; + unsigned int xmss_sig_low; + unsigned int reserved2[2]; }; struct icp_qat_auth_chunk { diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c index 3c4bba4a8779..7f638a62e3ad 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c @@ -9,8 +9,6 @@ #include <crypto/aes.h> #include <crypto/sha1.h> #include <crypto/sha2.h> -#include <crypto/hash.h> -#include <crypto/hmac.h> #include <crypto/algapi.h> #include <crypto/authenc.h> #include <crypto/scatterwalk.h> @@ -68,16 +66,10 @@ struct qat_alg_aead_ctx { dma_addr_t dec_cd_paddr; struct icp_qat_fw_la_bulk_req enc_fw_req; struct icp_qat_fw_la_bulk_req dec_fw_req; - struct crypto_shash *hash_tfm; enum icp_qat_hw_auth_algo qat_hash_alg; + unsigned int hash_digestsize; + unsigned int hash_blocksize; struct qat_crypto_instance *inst; - union { - struct sha1_state sha1; - struct sha256_state sha256; - struct sha512_state sha512; - }; - char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */ - char opad[SHA512_BLOCK_SIZE]; }; struct qat_alg_skcipher_ctx { @@ -94,125 +86,57 @@ struct qat_alg_skcipher_ctx { int mode; }; -static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) -{ - switch (qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - return ICP_QAT_HW_SHA1_STATE1_SZ; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - return ICP_QAT_HW_SHA256_STATE1_SZ; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - return ICP_QAT_HW_SHA512_STATE1_SZ; - default: - return -EFAULT; - } -} - static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, struct qat_alg_aead_ctx *ctx, const u8 *auth_key, unsigned int auth_keylen) { - SHASH_DESC_ON_STACK(shash, ctx->hash_tfm); - int block_size = crypto_shash_blocksize(ctx->hash_tfm); - int digest_size = crypto_shash_digestsize(ctx->hash_tfm); - __be32 *hash_state_out; - __be64 *hash512_state_out; - int i, offset; - - memset(ctx->ipad, 0, block_size); - memset(ctx->opad, 0, block_size); - shash->tfm = ctx->hash_tfm; - - if (auth_keylen > block_size) { - int ret = crypto_shash_digest(shash, auth_key, - auth_keylen, ctx->ipad); - if (ret) - return ret; - - memcpy(ctx->opad, ctx->ipad, digest_size); - } else { - memcpy(ctx->ipad, auth_key, auth_keylen); - memcpy(ctx->opad, auth_key, auth_keylen); + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: { + struct hmac_sha1_key key; + __be32 *istate = (__be32 *)hash->sha.state1; + __be32 *ostate = (__be32 *)(hash->sha.state1 + + round_up(sizeof(key.istate.h), 8)); + + hmac_sha1_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.istate.h); i++) { + istate[i] = cpu_to_be32(key.istate.h[i]); + ostate[i] = cpu_to_be32(key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; } - - for (i = 0; i < block_size; i++) { - char *ipad_ptr = ctx->ipad + i; - char *opad_ptr = ctx->opad + i; - *ipad_ptr ^= HMAC_IPAD_VALUE; - *opad_ptr ^= HMAC_OPAD_VALUE; + case ICP_QAT_HW_AUTH_ALGO_SHA256: { + struct hmac_sha256_key key; + __be32 *istate = (__be32 *)hash->sha.state1; + __be32 *ostate = (__be32 *)(hash->sha.state1 + + sizeof(key.key.istate.h)); + + hmac_sha256_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) { + istate[i] = cpu_to_be32(key.key.istate.h[i]); + ostate[i] = cpu_to_be32(key.key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; } - - if (crypto_shash_init(shash)) - return -EFAULT; - - if (crypto_shash_update(shash, ctx->ipad, block_size)) - return -EFAULT; - - hash_state_out = (__be32 *)hash->sha.state1; - hash512_state_out = (__be64 *)hash_state_out; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - if (crypto_shash_export(shash, &ctx->sha1)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha1.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - if (crypto_shash_export(shash, &ctx->sha256)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha256.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - if (crypto_shash_export(shash, &ctx->sha512)) - return -EFAULT; - for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) - *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]); - break; - default: - return -EFAULT; + case ICP_QAT_HW_AUTH_ALGO_SHA512: { + struct hmac_sha512_key key; + __be64 *istate = (__be64 *)hash->sha.state1; + __be64 *ostate = (__be64 *)(hash->sha.state1 + + sizeof(key.key.istate.h)); + + hmac_sha512_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) { + istate[i] = cpu_to_be64(key.key.istate.h[i]); + ostate[i] = cpu_to_be64(key.key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; } - - if (crypto_shash_init(shash)) - return -EFAULT; - - if (crypto_shash_update(shash, ctx->opad, block_size)) - return -EFAULT; - - offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8); - if (offset < 0) - return -EFAULT; - - hash_state_out = (__be32 *)(hash->sha.state1 + offset); - hash512_state_out = (__be64 *)hash_state_out; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - if (crypto_shash_export(shash, &ctx->sha1)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha1.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - if (crypto_shash_export(shash, &ctx->sha256)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha256.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - if (crypto_shash_export(shash, &ctx->sha512)) - return -EFAULT; - for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) - *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]); - break; default: return -EFAULT; } - memzero_explicit(ctx->ipad, block_size); - memzero_explicit(ctx->opad, block_size); - return 0; } static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) @@ -259,7 +183,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, ctx->qat_hash_alg, digestsize); hash->sha.inner_setup.auth_counter.counter = - cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); + cpu_to_be32(ctx->hash_blocksize); if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) return -EFAULT; @@ -326,7 +250,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, struct icp_qat_hw_cipher_algo_blk *cipher = (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + sizeof(struct icp_qat_hw_auth_setup) + - roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2); + roundup(ctx->hash_digestsize, 8) * 2); struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; @@ -346,7 +270,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, ctx->qat_hash_alg, digestsize); hash->sha.inner_setup.auth_counter.counter = - cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); + cpu_to_be32(ctx->hash_blocksize); if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) return -EFAULT; @@ -368,7 +292,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; cipher_cd_ctrl->cipher_cfg_offset = (sizeof(struct icp_qat_hw_auth_setup) + - roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3; + roundup(ctx->hash_digestsize, 8) * 2) >> 3; ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); @@ -1150,32 +1074,35 @@ static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req) } static int qat_alg_aead_init(struct crypto_aead *tfm, - enum icp_qat_hw_auth_algo hash, - const char *hash_name) + enum icp_qat_hw_auth_algo hash_alg, + unsigned int hash_digestsize, + unsigned int hash_blocksize) { struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); - ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); - if (IS_ERR(ctx->hash_tfm)) - return PTR_ERR(ctx->hash_tfm); - ctx->qat_hash_alg = hash; + ctx->qat_hash_alg = hash_alg; + ctx->hash_digestsize = hash_digestsize; + ctx->hash_blocksize = hash_blocksize; crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request)); return 0; } static int qat_alg_aead_sha1_init(struct crypto_aead *tfm) { - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, + SHA1_DIGEST_SIZE, SHA1_BLOCK_SIZE); } static int qat_alg_aead_sha256_init(struct crypto_aead *tfm) { - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, + SHA256_DIGEST_SIZE, SHA256_BLOCK_SIZE); } static int qat_alg_aead_sha512_init(struct crypto_aead *tfm) { - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, + SHA512_DIGEST_SIZE, SHA512_BLOCK_SIZE); } static void qat_alg_aead_exit(struct crypto_aead *tfm) @@ -1184,8 +1111,6 @@ static void qat_alg_aead_exit(struct crypto_aead *tfm) struct qat_crypto_instance *inst = ctx->inst; struct device *dev; - crypto_free_shash(ctx->hash_tfm); - if (!inst) return; @@ -1277,7 +1202,7 @@ static struct aead_alg qat_aeads[] = { { .base = { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha1", - .cra_priority = 4001, + .cra_priority = 100, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), @@ -1294,7 +1219,7 @@ static struct aead_alg qat_aeads[] = { { .base = { .cra_name = "authenc(hmac(sha256),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha256", - .cra_priority = 4001, + .cra_priority = 100, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), @@ -1311,7 +1236,7 @@ static struct aead_alg qat_aeads[] = { { .base = { .cra_name = "authenc(hmac(sha512),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha512", - .cra_priority = 4001, + .cra_priority = 100, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), @@ -1329,7 +1254,7 @@ static struct aead_alg qat_aeads[] = { { static struct skcipher_alg qat_skciphers[] = { { .base.cra_name = "cbc(aes)", .base.cra_driver_name = "qat_aes_cbc", - .base.cra_priority = 4001, + .base.cra_priority = 100, .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), @@ -1347,7 +1272,7 @@ static struct skcipher_alg qat_skciphers[] = { { }, { .base.cra_name = "ctr(aes)", .base.cra_driver_name = "qat_aes_ctr", - .base.cra_priority = 4001, + .base.cra_priority = 100, .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), @@ -1365,7 +1290,7 @@ static struct skcipher_alg qat_skciphers[] = { { }, { .base.cra_name = "xts(aes)", .base.cra_driver_name = "qat_aes_xts", - .base.cra_priority = 4001, + .base.cra_priority = 100, .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ALLOCATES_MEMORY, .base.cra_blocksize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/intel/qat/qat_common/qat_bl.c b/drivers/crypto/intel/qat/qat_common/qat_bl.c index 338acf29c487..9b2338f58d97 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_bl.c +++ b/drivers/crypto/intel/qat/qat_common/qat_bl.c @@ -38,7 +38,7 @@ void qat_bl_free_bufl(struct adf_accel_dev *accel_dev, for (i = 0; i < blout->num_mapped_bufs; i++) { dma_unmap_single(dev, blout->buffers[i].addr, blout->buffers[i].len, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); } dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); @@ -162,7 +162,7 @@ static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, } buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, sg->length - left, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(dev, buffers[y].addr))) goto err_out; buffers[y].len = sg->length; @@ -204,7 +204,7 @@ err_out: if (!dma_mapping_error(dev, buflout->buffers[i].addr)) dma_unmap_single(dev, buflout->buffers[i].addr, buflout->buffers[i].len, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); } if (!buf->sgl_dst_valid) @@ -251,162 +251,3 @@ int qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, extra_dst_buff, sz_extra_dst_buff, sskip, dskip, flags); } - -static void qat_bl_sgl_unmap(struct adf_accel_dev *accel_dev, - struct qat_alg_buf_list *bl) -{ - struct device *dev = &GET_DEV(accel_dev); - int n = bl->num_bufs; - int i; - - for (i = 0; i < n; i++) - if (!dma_mapping_error(dev, bl->buffers[i].addr)) - dma_unmap_single(dev, bl->buffers[i].addr, - bl->buffers[i].len, DMA_FROM_DEVICE); -} - -static int qat_bl_sgl_map(struct adf_accel_dev *accel_dev, - struct scatterlist *sgl, - struct qat_alg_buf_list **bl) -{ - struct device *dev = &GET_DEV(accel_dev); - struct qat_alg_buf_list *bufl; - int node = dev_to_node(dev); - struct scatterlist *sg; - int n, i, sg_nctr; - size_t sz; - - n = sg_nents(sgl); - sz = struct_size(bufl, buffers, n); - bufl = kzalloc_node(sz, GFP_KERNEL, node); - if (unlikely(!bufl)) - return -ENOMEM; - - for (i = 0; i < n; i++) - bufl->buffers[i].addr = DMA_MAPPING_ERROR; - - sg_nctr = 0; - for_each_sg(sgl, sg, n, i) { - int y = sg_nctr; - - if (!sg->length) - continue; - - bufl->buffers[y].addr = dma_map_single(dev, sg_virt(sg), - sg->length, - DMA_FROM_DEVICE); - bufl->buffers[y].len = sg->length; - if (unlikely(dma_mapping_error(dev, bufl->buffers[y].addr))) - goto err_map; - sg_nctr++; - } - bufl->num_bufs = sg_nctr; - bufl->num_mapped_bufs = sg_nctr; - - *bl = bufl; - - return 0; - -err_map: - for (i = 0; i < n; i++) - if (!dma_mapping_error(dev, bufl->buffers[i].addr)) - dma_unmap_single(dev, bufl->buffers[i].addr, - bufl->buffers[i].len, - DMA_FROM_DEVICE); - kfree(bufl); - *bl = NULL; - - return -ENOMEM; -} - -static void qat_bl_sgl_free_unmap(struct adf_accel_dev *accel_dev, - struct scatterlist *sgl, - struct qat_alg_buf_list *bl, - bool free_bl) -{ - if (bl) { - qat_bl_sgl_unmap(accel_dev, bl); - - if (free_bl) - kfree(bl); - } - if (sgl) - sgl_free(sgl); -} - -static int qat_bl_sgl_alloc_map(struct adf_accel_dev *accel_dev, - struct scatterlist **sgl, - struct qat_alg_buf_list **bl, - unsigned int dlen, - gfp_t gfp) -{ - struct scatterlist *dst; - int ret; - - dst = sgl_alloc(dlen, gfp, NULL); - if (!dst) { - dev_err(&GET_DEV(accel_dev), "sg_alloc failed\n"); - return -ENOMEM; - } - - ret = qat_bl_sgl_map(accel_dev, dst, bl); - if (ret) - goto err; - - *sgl = dst; - - return 0; - -err: - sgl_free(dst); - *sgl = NULL; - return ret; -} - -int qat_bl_realloc_map_new_dst(struct adf_accel_dev *accel_dev, - struct scatterlist **sg, - unsigned int dlen, - struct qat_request_buffs *qat_bufs, - gfp_t gfp) -{ - struct device *dev = &GET_DEV(accel_dev); - dma_addr_t new_blp = DMA_MAPPING_ERROR; - struct qat_alg_buf_list *new_bl; - struct scatterlist *new_sg; - size_t new_bl_size; - int ret; - - ret = qat_bl_sgl_alloc_map(accel_dev, &new_sg, &new_bl, dlen, gfp); - if (ret) - return ret; - - new_bl_size = struct_size(new_bl, buffers, new_bl->num_bufs); - - /* Map new firmware SGL descriptor */ - new_blp = dma_map_single(dev, new_bl, new_bl_size, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, new_blp))) - goto err; - - /* Unmap old firmware SGL descriptor */ - dma_unmap_single(dev, qat_bufs->bloutp, qat_bufs->sz_out, DMA_TO_DEVICE); - - /* Free and unmap old scatterlist */ - qat_bl_sgl_free_unmap(accel_dev, *sg, qat_bufs->blout, - !qat_bufs->sgl_dst_valid); - - qat_bufs->sgl_dst_valid = false; - qat_bufs->blout = new_bl; - qat_bufs->bloutp = new_blp; - qat_bufs->sz_out = new_bl_size; - - *sg = new_sg; - - return 0; -err: - qat_bl_sgl_free_unmap(accel_dev, new_sg, new_bl, true); - - if (!dma_mapping_error(dev, new_blp)) - dma_unmap_single(dev, new_blp, new_bl_size, DMA_TO_DEVICE); - - return -ENOMEM; -} diff --git a/drivers/crypto/intel/qat/qat_common/qat_bl.h b/drivers/crypto/intel/qat/qat_common/qat_bl.h index 3f5b79015400..2827d5055d3c 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_bl.h +++ b/drivers/crypto/intel/qat/qat_common/qat_bl.h @@ -65,10 +65,4 @@ static inline gfp_t qat_algs_alloc_flags(struct crypto_async_request *req) return req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; } -int qat_bl_realloc_map_new_dst(struct adf_accel_dev *accel_dev, - struct scatterlist **newd, - unsigned int dlen, - struct qat_request_buffs *qat_bufs, - gfp_t gfp); - #endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c index 2ba4aa22e092..8b123472b71c 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c @@ -8,6 +8,7 @@ #include <linux/workqueue.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" +#include "adf_dc.h" #include "qat_bl.h" #include "qat_comp_req.h" #include "qat_compression.h" @@ -29,11 +30,6 @@ struct qat_compression_ctx { int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp); }; -struct qat_dst { - bool is_null; - int resubmitted; -}; - struct qat_compression_req { u8 req[QAT_COMP_REQ_SIZE]; struct qat_compression_ctx *qat_compression_ctx; @@ -42,8 +38,6 @@ struct qat_compression_req { enum direction dir; int actual_dlen; struct qat_alg_req alg_req; - struct work_struct resubmit; - struct qat_dst dst; }; static int qat_alg_send_dc_message(struct qat_compression_req *qat_req, @@ -60,46 +54,6 @@ static int qat_alg_send_dc_message(struct qat_compression_req *qat_req, return qat_alg_send_message(alg_req); } -static void qat_comp_resubmit(struct work_struct *work) -{ - struct qat_compression_req *qat_req = - container_of(work, struct qat_compression_req, resubmit); - struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx; - struct adf_accel_dev *accel_dev = ctx->inst->accel_dev; - struct qat_request_buffs *qat_bufs = &qat_req->buf; - struct qat_compression_instance *inst = ctx->inst; - struct acomp_req *areq = qat_req->acompress_req; - struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq); - unsigned int dlen = CRYPTO_ACOMP_DST_MAX; - u8 *req = qat_req->req; - dma_addr_t dfbuf; - int ret; - - areq->dlen = dlen; - - dev_dbg(&GET_DEV(accel_dev), "[%s][%s] retry NULL dst request - dlen = %d\n", - crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)), - qat_req->dir == COMPRESSION ? "comp" : "decomp", dlen); - - ret = qat_bl_realloc_map_new_dst(accel_dev, &areq->dst, dlen, qat_bufs, - qat_algs_alloc_flags(&areq->base)); - if (ret) - goto err; - - qat_req->dst.resubmitted = true; - - dfbuf = qat_req->buf.bloutp; - qat_comp_override_dst(req, dfbuf, dlen); - - ret = qat_alg_send_dc_message(qat_req, inst, &areq->base); - if (ret != -ENOSPC) - return; - -err: - qat_bl_free_bufl(accel_dev, qat_bufs); - acomp_request_complete(areq, ret); -} - static void qat_comp_generic_callback(struct qat_compression_req *qat_req, void *resp) { @@ -131,21 +85,6 @@ static void qat_comp_generic_callback(struct qat_compression_req *qat_req, areq->dlen = 0; - if (qat_req->dir == DECOMPRESSION && qat_req->dst.is_null) { - if (cmp_err == ERR_CODE_OVERFLOW_ERROR) { - if (qat_req->dst.resubmitted) { - dev_dbg(&GET_DEV(accel_dev), - "Output does not fit destination buffer\n"); - res = -EOVERFLOW; - goto end; - } - - INIT_WORK(&qat_req->resubmit, qat_comp_resubmit); - adf_misc_wq_queue_work(&qat_req->resubmit); - return; - } - } - if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) goto end; @@ -207,9 +146,7 @@ static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm) return -EINVAL; ctx->inst = inst; - ctx->inst->build_deflate_ctx(ctx->comp_ctx); - - return 0; + return qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, QAT_DEFLATE); } static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm) @@ -245,29 +182,9 @@ static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum directi if (!areq->src || !slen) return -EINVAL; - if (areq->dst && !dlen) + if (!areq->dst || !dlen) return -EINVAL; - qat_req->dst.is_null = false; - - /* Handle acomp requests that require the allocation of a destination - * buffer. The size of the destination buffer is double the source - * buffer (rounded up to the size of a page) to fit the decompressed - * output or an expansion on the data for compression. - */ - if (!areq->dst) { - qat_req->dst.is_null = true; - - dlen = round_up(2 * slen, PAGE_SIZE); - areq->dst = sgl_alloc(dlen, f, NULL); - if (!areq->dst) - return -ENOMEM; - - dlen -= dhdr + dftr; - areq->dlen = dlen; - qat_req->dst.resubmitted = false; - } - if (dir == COMPRESSION) { params.extra_dst_buff = inst->dc_data->ovf_buff_p; ovf_buff_sz = inst->dc_data->ovf_buff_sz; @@ -323,14 +240,13 @@ static struct acomp_alg qat_acomp[] = { { .cra_priority = 4001, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_ctxsize = sizeof(struct qat_compression_ctx), + .cra_reqsize = sizeof(struct qat_compression_req), .cra_module = THIS_MODULE, }, .init = qat_comp_alg_init_tfm, .exit = qat_comp_alg_exit_tfm, .compress = qat_comp_alg_compress, .decompress = qat_comp_alg_decompress, - .dst_free = sgl_free, - .reqsize = sizeof(struct qat_compression_req), }}; int qat_comp_algs_register(void) diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_req.h b/drivers/crypto/intel/qat/qat_common/qat_comp_req.h index 404e32c5e778..18a1f33a6db9 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_comp_req.h +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_req.h @@ -25,16 +25,6 @@ static inline void qat_comp_create_req(void *ctx, void *req, u64 src, u32 slen, req_pars->out_buffer_sz = dlen; } -static inline void qat_comp_override_dst(void *req, u64 dst, u32 dlen) -{ - struct icp_qat_fw_comp_req *fw_req = req; - struct icp_qat_fw_comp_req_params *req_pars = &fw_req->comp_pars; - - fw_req->comn_mid.dest_data_addr = dst; - fw_req->comn_mid.dst_length = dlen; - req_pars->out_buffer_sz = dlen; -} - static inline void qat_comp_create_compression_req(void *ctx, void *req, u64 src, u32 slen, u64 dst, u32 dlen, diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 7842a9f22178..53a4db5507ec 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -144,7 +144,6 @@ static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; - inst->build_deflate_ctx = GET_DC_OPS(accel_dev)->build_deflate_ctx; snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); @@ -197,7 +196,7 @@ static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) struct adf_dc_data *dc_data = NULL; u8 *obuff = NULL; - dc_data = devm_kzalloc(dev, sizeof(*dc_data), GFP_KERNEL); + dc_data = kzalloc_node(sizeof(*dc_data), GFP_KERNEL, dev_to_node(dev)); if (!dc_data) goto err; @@ -205,7 +204,7 @@ static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) if (!obuff) goto err; - obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_FROM_DEVICE); + obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(dev, obuff_p))) goto err; @@ -233,9 +232,9 @@ static void qat_free_dc_data(struct adf_accel_dev *accel_dev) return; dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); kfree_sensitive(dc_data->ovf_buff); - devm_kfree(dev, dc_data); + kfree(dc_data); accel_dev->dc_data = NULL; } diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.h b/drivers/crypto/intel/qat/qat_common/qat_compression.h index aebac2302dcf..5ced3ed0e5ea 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.h +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.h @@ -20,7 +20,6 @@ struct qat_compression_instance { atomic_t refctr; struct qat_instance_backlog backlog; struct adf_dc_data *dc_data; - void (*build_deflate_ctx)(void *ctx); }; static inline bool adf_hw_dev_has_compression(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index ef8a9cf74f0c..da4eca6e1633 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -694,16 +694,17 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; switch (handle->pci_dev->device) { - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - case ADF_420XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: + case PCI_DEVICE_ID_INTEL_QAT_6XXX: handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = false; handle->chip_info->lm2lm3 = true; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; - if (handle->pci_dev->device == ADF_420XX_PCI_DEVICE_ID) + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_420XX) handle->chip_info->icp_rst_mask = 0x100155; else handle->chip_info->icp_rst_mask = 0x100015; @@ -712,6 +713,8 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = 0x80000000; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = true; + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX) + handle->chip_info->dual_sign = true; handle->chip_info->tgroup_share_ustore = true; handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 7ea40b4f6e5b..06d49cb781ae 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1,10 +1,16 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + +#include <linux/align.h> +#include <linux/bitops.h> #include <linux/slab.h> #include <linux/ctype.h> #include <linux/kernel.h> #include <linux/delay.h> #include <linux/pci_ids.h> +#include <linux/wordpart.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_uclo.h" @@ -58,7 +64,7 @@ static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) unsigned int i; if (!ae_data) { - pr_err("QAT: bad argument, ae_data is NULL\n"); + pr_err("bad argument, ae_data is NULL\n"); return -EINVAL; } @@ -85,12 +91,11 @@ static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) int min = hdr->min_ver & 0xff; if (hdr->file_id != ICP_QAT_UOF_FID) { - pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); + pr_err("Invalid header 0x%x\n", hdr->file_id); return -EINVAL; } if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { - pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad UOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; @@ -102,20 +107,19 @@ static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) int min = suof_hdr->min_ver & 0xff; if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { - pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); + pr_err("invalid header 0x%x\n", suof_hdr->file_id); return -EINVAL; } if (suof_hdr->fw_type != 0) { - pr_err("QAT: unsupported firmware type\n"); + pr_err("unsupported firmware type\n"); return -EINVAL; } if (suof_hdr->num_chunks <= 0x1) { - pr_err("QAT: SUOF chunk amount is incorrect\n"); + pr_err("SUOF chunk amount is incorrect\n"); return -EINVAL; } if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { - pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad SUOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; @@ -196,20 +200,12 @@ qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_parse_num(char *str, unsigned int *num) { - char buf[16] = {0}; - unsigned long ae = 0; - int i; - - strscpy(buf, str, sizeof(buf)); - for (i = 0; i < 16; i++) { - if (!isdigit(buf[i])) { - buf[i] = '\0'; - break; - } - } - if ((kstrtoul(buf, 10, &ae))) - return -EFAULT; + unsigned long long ae; + char *end; + ae = simple_strtoull(str, &end, 10); + if (ae > UINT_MAX || str == end || (end - str) > 19) + return -EINVAL; *num = (unsigned int)ae; return 0; } @@ -222,24 +218,24 @@ static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, char *str; if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { - pr_err("QAT: initmem is out of range"); + pr_err("initmem is out of range"); return -EINVAL; } if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { - pr_err("QAT: Memory scope for init_mem error\n"); + pr_err("Memory scope for init_mem error\n"); return -EINVAL; } str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); if (!str) { - pr_err("QAT: AE name assigned in UOF init table is NULL\n"); + pr_err("AE name assigned in UOF init table is NULL\n"); return -EINVAL; } if (qat_uclo_parse_num(str, ae)) { - pr_err("QAT: Parse num for AE number failed\n"); + pr_err("Parse num for AE number failed\n"); return -EINVAL; } if (*ae >= ICP_QAT_UCLO_MAX_AE) { - pr_err("QAT: ae %d out of range\n", *ae); + pr_err("ae %d out of range\n", *ae); return -EINVAL; } return 0; @@ -355,8 +351,7 @@ static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, return -EINVAL; break; default: - pr_err("QAT: initmem region error. region type=0x%x\n", - init_mem->region); + pr_err("initmem region error. region type=0x%x\n", init_mem->region); return -EINVAL; } return 0; @@ -430,7 +425,7 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_batch_wr_lm(handle, ae, obj_handle->lm_init_tab[ae])) { - pr_err("QAT: fail to batch init lmem for AE %d\n", ae); + pr_err("fail to batch init lmem for AE %d\n", ae); return -EINVAL; } qat_uclo_cleanup_batch_init_list(handle, @@ -538,26 +533,26 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, code_page->imp_expr_tab_offset); if (uc_var_tab->entry_num || imp_var_tab->entry_num || imp_expr_tab->entry_num) { - pr_err("QAT: UOF can't contain imported variable to be parsed\n"); + pr_err("UOF can't contain imported variable to be parsed\n"); return -EINVAL; } neigh_reg_tab = (struct icp_qat_uof_objtable *) (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain neighbor register table\n"); + pr_err("UOF can't contain neighbor register table\n"); return -EINVAL; } if (image->numpages > 1) { - pr_err("QAT: UOF can't contain multiple pages\n"); + pr_err("UOF can't contain multiple pages\n"); return -EINVAL; } if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use shared control store feature\n"); + pr_err("UOF can't use shared control store feature\n"); return -EFAULT; } if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use reloadable feature\n"); + pr_err("UOF can't use reloadable feature\n"); return -EFAULT; } return 0; @@ -676,7 +671,7 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) } } if (!mflag) { - pr_err("QAT: uimage uses AE not set\n"); + pr_err("uimage uses AE not set\n"); return -EINVAL; } return 0; @@ -730,14 +725,15 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) return ICP_QAT_AC_C62X_DEV_TYPE; case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - case ADF_420XX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: return ICP_QAT_AC_4XXX_A_DEV_TYPE; + case PCI_DEVICE_ID_INTEL_QAT_6XXX: + return ICP_QAT_AC_6XXX_DEV_TYPE; default: - pr_err("QAT: unsupported device 0x%x\n", - handle->pci_dev->device); + pr_err("unsupported device 0x%x\n", handle->pci_dev->device); return 0; } } @@ -747,7 +743,7 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) unsigned int maj_ver, prod_type = obj_handle->prod_type; if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { - pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", + pr_err("UOF type 0x%x doesn't match with platform 0x%x\n", obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, prod_type); return -EINVAL; @@ -755,7 +751,7 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) maj_ver = obj_handle->prod_rev & 0xff; if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { - pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); + pr_err("UOF majVer 0x%x out of range\n", maj_ver); return -EINVAL; } return 0; @@ -798,7 +794,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_NEIGH_REL: return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); default: - pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); + pr_err("UOF uses not supported reg type 0x%x\n", reg_type); return -EFAULT; } return 0; @@ -834,8 +830,7 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, case ICP_QAT_UOF_INIT_REG_CTX: /* check if ctx is appropriate for the ctxMode */ if (!((1 << init_regsym->ctx) & ctx_mask)) { - pr_err("QAT: invalid ctx num = 0x%x\n", - init_regsym->ctx); + pr_err("invalid ctx num = 0x%x\n", init_regsym->ctx); return -EINVAL; } qat_uclo_init_reg(handle, ae, @@ -847,10 +842,10 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, exp_res); break; case ICP_QAT_UOF_INIT_EXPR: - pr_err("QAT: INIT_EXPR feature not supported\n"); + pr_err("INIT_EXPR feature not supported\n"); return -EINVAL; case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: - pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); + pr_err("INIT_EXPR_ENDIAN_SWAP feature not supported\n"); return -EINVAL; default: break; @@ -870,7 +865,7 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return 0; if (obj_handle->init_mem_tab.entry_num) { if (qat_uclo_init_memory(handle)) { - pr_err("QAT: initialize memory failed\n"); + pr_err("initialize memory failed\n"); return -EINVAL; } } @@ -899,40 +894,40 @@ static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); + pr_err("qat_hal_set_ae_ctx_mode error\n"); return ret; } if (handle->chip_info->nn) { mode = ICP_QAT_NN_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_nn_mode(handle, ae, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + pr_err("qat_hal_set_ae_nn_mode error\n"); return ret; } } mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM0 error\n"); return ret; } mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM1 error\n"); return ret; } if (handle->chip_info->lm2lm3) { mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM2 error\n"); return ret; } mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM3 error\n"); return ret; } mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); @@ -996,7 +991,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) obj_handle->prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (qat_uclo_check_uof_compat(obj_handle)) { - pr_err("QAT: UOF incompatible\n"); + pr_err("UOF incompatible\n"); return -EINVAL; } obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), @@ -1007,7 +1002,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->obj_hdr->file_buff || !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, &obj_handle->str_table)) { - pr_err("QAT: UOF doesn't have effective images\n"); + pr_err("UOF doesn't have effective images\n"); goto out_err; } obj_handle->uimage_num = @@ -1016,7 +1011,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->uimage_num) goto out_err; if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { - pr_err("QAT: Bad object\n"); + pr_err("Bad object\n"); goto out_check_uof_aemask_err; } qat_uclo_init_uword_num(handle); @@ -1033,6 +1028,36 @@ out_err: return -EFAULT; } +static unsigned int qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle *handle) +{ + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN; + + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle *handle) +{ + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN + ICP_QAT_DUALSIGN_MISC_INFO_LEN; + + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle *handle, void *img_ptr) +{ + struct icp_qat_css_hdr *hdr = img_ptr; + char *fw_hdr = img_ptr; + unsigned int offset; + + if (handle->chip_info->dual_sign) { + offset = qat_uclo_simg_hdr2sign_len(handle) + ICP_QAT_DUALSIGN_FW_TYPE_LEN; + return *(fw_hdr + offset); + } + + return hdr->fw_type; +} + static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_filehdr *suof_ptr, int suof_size) @@ -1049,7 +1074,7 @@ static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, min_ver_offset); if (check_sum != suof_ptr->check_sum) { - pr_err("QAT: incorrect SUOF checksum\n"); + pr_err("incorrect SUOF checksum\n"); return -EINVAL; } suof_handle->check_sum = suof_ptr->check_sum; @@ -1064,8 +1089,9 @@ static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; - struct icp_qat_simg_ae_mode *ae_mode; + unsigned int offset = qat_uclo_simg_hdr2cont_len(handle); struct icp_qat_suof_objhdr *suof_objhdr; + struct icp_qat_simg_ae_mode *ae_mode; suof_img_hdr->simg_buf = (suof_handle->suof_buf + suof_chunk_hdr->offset + @@ -1075,13 +1101,7 @@ static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, suof_chunk_hdr->offset))->img_length; suof_img_hdr->css_header = suof_img_hdr->simg_buf; - suof_img_hdr->css_key = (suof_img_hdr->css_header + - sizeof(struct icp_qat_css_hdr)); - suof_img_hdr->css_signature = suof_img_hdr->css_key + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); - suof_img_hdr->css_simg = suof_img_hdr->css_signature + - ICP_QAT_CSS_SIGNATURE_LEN(handle); + suof_img_hdr->css_simg = suof_img_hdr->css_header + offset; ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); suof_img_hdr->ae_mask = ae_mode->ae_mask; @@ -1116,14 +1136,13 @@ static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (img_ae_mode->dev_type != prod_type) { - pr_err("QAT: incompatible product type %x\n", - img_ae_mode->dev_type); + pr_err("incompatible product type %x\n", img_ae_mode->dev_type); return -EINVAL; } maj_ver = prod_rev & 0xff; if (maj_ver > img_ae_mode->devmax_ver || maj_ver < img_ae_mode->devmin_ver) { - pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); + pr_err("incompatible device majver 0x%x\n", maj_ver); return -EINVAL; } return 0; @@ -1166,7 +1185,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr img_header; if (!suof_ptr || suof_size == 0) { - pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); + pr_err("input parameter SUOF pointer/size is NULL\n"); return -EINVAL; } if (qat_uclo_check_suof_format(suof_ptr)) @@ -1209,7 +1228,6 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, } #define ADD_ADDR(high, low) ((((u64)high) << 32) + low) -#define BITS_IN_DWORD 32 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) @@ -1227,7 +1245,7 @@ static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; - SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); + SET_CAP_CSR(handle, fcu_dram_hi_csr, bus_addr >> BITS_PER_TYPE(u32)); SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); @@ -1241,7 +1259,7 @@ static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, return 0; } while (retry++ < FW_AUTH_MAX_RETRY); auth_fail: - pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", + pr_err("authentication error (FCU_STATUS = 0x%x),retry = %d\n", fcu_sts & FCU_AUTH_STS_MASK, retry); return -EINVAL; } @@ -1277,14 +1295,13 @@ static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, fcu_sts_csr = handle->chip_info->fcu_sts_csr; fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; } else { - pr_err("Chip 0x%x doesn't support broadcast load\n", - handle->pci_dev->device); + pr_err("Chip 0x%x doesn't support broadcast load\n", handle->pci_dev->device); return -EINVAL; } for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { - pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); + pr_err("Broadcast load failed. AE is not enabled or active.\n"); return -EINVAL; } @@ -1316,7 +1333,7 @@ static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: broadcast load failed timeout %d\n", retry); + pr_err("broadcast load failed timeout %d\n", retry); return -EINVAL; } } @@ -1370,24 +1387,38 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, } static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, + void *image, unsigned int size, unsigned int fw_type) { char *fw_type_name = fw_type ? "MMP" : "AE"; unsigned int css_dword_size = sizeof(u32); + unsigned int header_len, simg_type; + struct icp_qat_css_hdr *css_hdr; if (handle->chip_info->fw_auth) { - struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); + header_len = qat_uclo_simg_hdr2sign_len(handle); + simg_type = qat_uclo_simg_fw_type(handle, image); + css_hdr = image; + + if (handle->chip_info->dual_sign) { + if (css_hdr->module_type != ICP_QAT_DUALSIGN_MODULE_TYPE) + goto err; + if (css_hdr->header_len != ICP_QAT_DUALSIGN_HDR_LEN) + goto err; + if (css_hdr->header_ver != ICP_QAT_DUALSIGN_HDR_VER) + goto err; + } else { + if (css_hdr->header_len * css_dword_size != header_len) + goto err; + if (css_hdr->size * css_dword_size != size) + goto err; + if (size <= header_len) + goto err; + } - if ((css_hdr->header_len * css_dword_size) != header_len) - goto err; - if ((css_hdr->size * css_dword_size) != size) - goto err; - if (fw_type != css_hdr->fw_type) - goto err; - if (size <= header_len) + if (fw_type != simg_type) goto err; + size -= header_len; } @@ -1401,115 +1432,95 @@ static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) goto err; } else { - pr_err("QAT: Unsupported firmware type\n"); + pr_err("Unsupported firmware type\n"); return -EINVAL; } return 0; err: - pr_err("QAT: Invalid %s firmware image\n", fw_type_name); + pr_err("Invalid %s firmware image\n", fw_type_name); return -EINVAL; } -static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, - struct icp_qat_fw_auth_desc **desc) +static int qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, struct icp_qat_fw_auth_desc **desc) { struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - struct icp_qat_fw_auth_desc *auth_desc; - struct icp_qat_auth_chunk *auth_chunk; - u64 virt_addr, bus_addr, virt_base; - unsigned int length, simg_offset = sizeof(*auth_chunk); struct icp_qat_simg_ae_mode *simg_ae_mode; - struct icp_firml_dram_desc img_desc; - - if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)) { - pr_err("QAT: error, input image size overflow %d\n", size); - return -EINVAL; - } - length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? - ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset : - size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset; - if (qat_uclo_simg_alloc(handle, &img_desc, length)) { - pr_err("QAT: error, allocate continuous dram fail\n"); - return -ENOMEM; - } + struct icp_qat_fw_auth_desc *auth_desc; + char *virt_addr, *virt_base; + u64 bus_addr; - auth_chunk = img_desc.dram_base_addr_v; - auth_chunk->chunk_size = img_desc.dram_size; - auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; - virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; - bus_addr = img_desc.dram_bus_addr + simg_offset; - auth_desc = img_desc.dram_base_addr_v; - auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->css_hdr_low = (unsigned int)bus_addr; + virt_base = dram_desc->dram_base_addr_v; + virt_base += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); virt_addr = virt_base; - memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); + memcpy(virt_addr, image, sizeof(*css_hdr)); /* pub key */ bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + sizeof(*css_hdr); virt_addr = virt_addr + sizeof(*css_hdr); - auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->fwsk_pub_low = (unsigned int)bus_addr; + auth_desc->fwsk_pub_high = upper_32_bits(bus_addr); + auth_desc->fwsk_pub_low = lower_32_bits(bus_addr); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + memcpy(virt_addr, image + sizeof(*css_hdr), ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); /* padding */ memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_PAD_LEN(handle)), - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), - sizeof(unsigned int)); + memcpy(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_PAD_LEN(handle), image + sizeof(*css_hdr) + + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); - auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->signature_low = (unsigned int)bus_addr; + auth_desc->signature_high = upper_32_bits(bus_addr); + auth_desc->signature_low = lower_32_bits(bus_addr); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), - ICP_QAT_CSS_SIGNATURE_LEN(handle)); + memcpy(virt_addr, image + sizeof(*css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle), ICP_QAT_CSS_SIGNATURE_LEN(handle)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + ICP_QAT_CSS_SIGNATURE_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); - auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), - auth_desc->img_len); + auth_desc->img_high = upper_32_bits(bus_addr); + auth_desc->img_low = lower_32_bits(bus_addr); + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + if (bus_addr + auth_desc->img_len > + dram_desc->dram_bus_addr + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { + pr_err("insufficient memory size for authentication data\n"); + qat_uclo_simg_free(handle, dram_desc); + return -ENOMEM; + } + + memcpy(virt_addr, image + qat_uclo_simg_hdr2sign_len(handle), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ - if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == - CSS_AE_FIRMWARE) { + if (fw_type == CSS_AE_FIRMWARE) { auth_desc->img_ae_mode_data_high = auth_desc->img_high; auth_desc->img_ae_mode_data_low = auth_desc->img_low; bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, auth_desc->img_ae_mode_data_low) + sizeof(struct icp_qat_simg_ae_mode); - auth_desc->img_ae_init_data_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); - auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; + auth_desc->img_ae_init_data_high = upper_32_bits(bus_addr); + auth_desc->img_ae_init_data_low = lower_32_bits(bus_addr); bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; - auth_desc->img_ae_insts_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); - auth_desc->img_ae_insts_low = (unsigned int)bus_addr; + auth_desc->img_ae_insts_high = upper_32_bits(bus_addr); + auth_desc->img_ae_insts_low = lower_32_bits(bus_addr); virt_addr += sizeof(struct icp_qat_css_hdr); virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); @@ -1523,6 +1534,141 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, return 0; } +static int qat_uclo_build_auth_desc_dualsign(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_simg_ae_mode *simg_ae_mode; + struct icp_qat_fw_auth_desc *auth_desc; + unsigned int chunk_offset, img_offset; + u64 bus_addr, addr; + char *virt_addr; + + virt_addr = dram_desc->dram_base_addr_v; + virt_addr += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); + memcpy(virt_addr, image, ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN); + + img_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN; + chunk_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN; + + /* RSA pub key */ + addr = bus_addr + chunk_offset; + auth_desc->fwsk_pub_high = upper_32_bits(addr); + auth_desc->fwsk_pub_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + /* RSA padding */ + memset(virt_addr + chunk_offset, 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); + + chunk_offset += ICP_QAT_CSS_FWSK_PAD_LEN(handle); + /* RSA exponent */ + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + /* RSA signature */ + addr = bus_addr + chunk_offset; + auth_desc->signature_high = upper_32_bits(addr); + auth_desc->signature_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_SIGNATURE_LEN(handle)); + + img_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + chunk_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + /* XMSS pubkey */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_pubkey_high = upper_32_bits(addr); + auth_desc->xmss_pubkey_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + /* XMSS signature */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_sig_high = upper_32_bits(addr); + auth_desc->xmss_sig_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_SIG_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_SIG_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN; + + if (dram_desc->dram_size < (chunk_offset + auth_desc->img_len)) { + pr_err("auth chunk memory size is not enough to store data\n"); + return -ENOMEM; + } + + /* Signed data */ + addr = bus_addr + chunk_offset; + auth_desc->img_high = upper_32_bits(addr); + auth_desc->img_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, auth_desc->img_len); + + chunk_offset += ICP_QAT_DUALSIGN_MISC_INFO_LEN; + /* AE firmware */ + if (fw_type == CSS_AE_FIRMWARE) { + /* AE mode data */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_mode_data_high = upper_32_bits(addr); + auth_desc->img_ae_mode_data_low = lower_32_bits(addr); + simg_ae_mode = + (struct icp_qat_simg_ae_mode *)(virt_addr + chunk_offset); + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; + + chunk_offset += sizeof(struct icp_qat_simg_ae_mode); + /* AE init seq */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_init_data_high = upper_32_bits(addr); + auth_desc->img_ae_init_data_low = lower_32_bits(addr); + + chunk_offset += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; + /* AE instructions */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } else { + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } + *desc = auth_desc; + return 0; +} + +static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_auth_chunk *auth_chunk; + struct icp_firml_dram_desc img_desc; + unsigned int simg_fw_type; + int ret; + + ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); + if (ret) + return ret; + + simg_fw_type = qat_uclo_simg_fw_type(handle, image); + auth_chunk = img_desc.dram_base_addr_v; + auth_chunk->chunk_size = img_desc.dram_size; + auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; + + if (handle->chip_info->dual_sign) + return qat_uclo_build_auth_desc_dualsign(handle, image, size, &img_desc, + simg_fw_type, desc); + + return qat_uclo_build_auth_desc_RSA(handle, image, size, &img_desc, + simg_fw_type, desc); +} + static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { @@ -1542,7 +1688,7 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, if (!((desc->ae_mask >> i) & 0x1)) continue; if (qat_hal_check_ae_active(handle, i)) { - pr_err("QAT: AE %d is active\n", i); + pr_err("AE %d is active\n", i); return -EINVAL; } SET_CAP_CSR(handle, fcu_ctl_csr, @@ -1562,7 +1708,7 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: firmware load failed timeout %x\n", retry); + pr_err("firmware load failed timeout %x\n", retry); return -EINVAL; } } @@ -1580,7 +1726,7 @@ static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, handle->sobj_handle = suof_handle; if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { qat_uclo_del_suof(handle); - pr_err("QAT: map SUOF failed\n"); + pr_err("map SUOF failed\n"); return -EINVAL; } return 0; @@ -1604,7 +1750,7 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, qat_uclo_ummap_auth_fw(handle, &desc); } else { if (handle->chip_info->mmp_sram_size < mem_size) { - pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); + pr_err("MMP size is too large: 0x%x\n", mem_size); return -EFBIG; } qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); @@ -1630,7 +1776,7 @@ static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, ICP_QAT_UOF_OBJS); if (!objhdl->obj_hdr) { - pr_err("QAT: object file chunk is null\n"); + pr_err("object file chunk is null\n"); goto out_objhdr_err; } handle->obj_handle = objhdl; @@ -1665,7 +1811,7 @@ static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, min_ver_offset); if (checksum != mof_ptr->checksum) { - pr_err("QAT: incorrect MOF checksum\n"); + pr_err("incorrect MOF checksum\n"); return -EINVAL; } @@ -1701,7 +1847,7 @@ static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, } } - pr_err("QAT: object %s is not found inside MOF\n", obj_name); + pr_err("object %s is not found inside MOF\n", obj_name); return -EINVAL; } @@ -1718,7 +1864,7 @@ static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; } else { - pr_err("QAT: unsupported chunk id\n"); + pr_err("unsupported chunk id\n"); return -EINVAL; } mobj_hdr->obj_buf = obj; @@ -1746,7 +1892,7 @@ static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) if (sobj_hdr) sobj_chunk_num = sobj_hdr->num_chunks; - mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * + mobj_hdr = kcalloc(size_add(uobj_chunk_num, sobj_chunk_num), sizeof(*mobj_hdr), GFP_KERNEL); if (!mobj_hdr) return -ENOMEM; @@ -1779,7 +1925,7 @@ static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) } if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { - pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); + pr_err("inconsistent UOF/SUOF chunk amount\n"); return -EINVAL; } return 0; @@ -1820,17 +1966,16 @@ static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) int min = mof_hdr->min_ver & 0xff; if (mof_hdr->file_id != ICP_QAT_MOF_FID) { - pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); + pr_err("invalid header 0x%x\n", mof_hdr->file_id); return -EINVAL; } if (mof_hdr->num_chunks <= 0x1) { - pr_err("QAT: MOF chunk amount is incorrect\n"); + pr_err("MOF chunk amount is incorrect\n"); return -EINVAL; } if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { - pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad MOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile index cfd3bd757715..1427fe76f171 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o -qat_dh895xcc-objs := adf_drv.o adf_dh895xcc_hw_data.o +qat_dh895xcc-y := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index c0661ff5e929..5b4bd0ba1ccb 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -4,7 +4,6 @@ #include <adf_admin.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> -#include <adf_gen2_dc.h> #include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> @@ -24,12 +23,11 @@ static const u32 thrd_to_arb_map[ADF_DH895XCC_MAX_ACCELENGINES] = { static struct adf_hw_device_class dh895xcc_class = { .name = ADF_DH895XCC_DEVICE_NAME, .type = DEV_DH895XCC, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & ADF_DH895XCC_ACCELERATORS_MASK; @@ -37,7 +35,7 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } @@ -99,7 +97,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { - int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) + int sku = (self->fuses[ADF_FUSECTL0] & ADF_DH895XCC_FUSECTL_SKU_MASK) >> ADF_DH895XCC_FUSECTL_SKU_SHIFT; switch (sku) { diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 2a50cce41515..b59e0cc49e52 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -19,24 +19,6 @@ #include <adf_dbgfs.h> #include "adf_dh895xcc_hw_data.h" -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_DH895XCC_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) { pci_release_regions(accel_dev->accel_pci_dev.pci_dev); @@ -126,7 +108,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_dh895xcc(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); @@ -227,6 +209,29 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_DH895XCC_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + static int __init adfdrv_init(void) { request_module("intel_qat"); @@ -252,4 +257,4 @@ MODULE_FIRMWARE(ADF_DH895XCC_FW); MODULE_FIRMWARE(ADF_DH895XCC_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile index 64b54e92b2b4..c2fdb6e0f68f 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o -qat_dh895xccvf-objs := adf_drv.o adf_dh895xccvf_hw_data.o +qat_dh895xccvf-y := adf_drv.o adf_dh895xccvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index f4ee4c2e00da..828456c43b76 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -3,7 +3,6 @@ #include <adf_accel_devices.h> #include <adf_common_drv.h> #include <adf_gen2_config.h> -#include <adf_gen2_dc.h> #include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> @@ -13,7 +12,6 @@ static struct adf_hw_device_class dh895xcciov_class = { .name = ADF_DH895XCCVF_DEVICE_NAME, .type = DEV_DH895XCCVF, - .instances = 0 }; static u32 get_accel_mask(struct adf_hw_device_data *self) diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c index 7cb015b55122..7cd528ee31e7 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c @@ -226,4 +226,4 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_QAT); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/loongson/Kconfig b/drivers/crypto/loongson/Kconfig new file mode 100644 index 000000000000..15475da8fc11 --- /dev/null +++ b/drivers/crypto/loongson/Kconfig @@ -0,0 +1,5 @@ +config CRYPTO_DEV_LOONGSON_RNG + tristate "Support for Loongson RNG Driver" + depends on MFD_LOONGSON_SE + help + Support for Loongson RNG Driver. diff --git a/drivers/crypto/loongson/Makefile b/drivers/crypto/loongson/Makefile new file mode 100644 index 000000000000..1ce5ec32b553 --- /dev/null +++ b/drivers/crypto/loongson/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_CRYPTO_DEV_LOONGSON_RNG) += loongson-rng.o diff --git a/drivers/crypto/loongson/loongson-rng.c b/drivers/crypto/loongson/loongson-rng.c new file mode 100644 index 000000000000..3a4940260f9e --- /dev/null +++ b/drivers/crypto/loongson/loongson-rng.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ +/* Copyright (c) 2025 Loongson Technology Corporation Limited. */ + +#include <linux/crypto.h> +#include <linux/err.h> +#include <linux/hw_random.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/mfd/loongson-se.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/platform_device.h> +#include <linux/random.h> +#include <crypto/internal/rng.h> + +#define SE_SEED_SIZE 32 + +struct loongson_rng_list { + struct mutex lock; + struct list_head list; + int registered; +}; + +struct loongson_rng { + u32 used; + struct loongson_se_engine *engine; + struct list_head list; + struct mutex lock; +}; + +struct loongson_rng_ctx { + struct loongson_rng *rng; +}; + +struct loongson_rng_cmd { + u32 cmd_id; + union { + u32 len; + u32 ret; + } u; + u32 seed_off; + u32 out_off; + u32 pad[4]; +}; + +static struct loongson_rng_list rng_devices = { + .lock = __MUTEX_INITIALIZER(rng_devices.lock), + .list = LIST_HEAD_INIT(rng_devices.list), +}; + +static int loongson_rng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct loongson_rng_ctx *ctx = crypto_rng_ctx(tfm); + struct loongson_rng *rng = ctx->rng; + struct loongson_rng_cmd *cmd = rng->engine->command; + int err, len; + + mutex_lock(&rng->lock); + cmd->seed_off = 0; + do { + len = min(dlen, rng->engine->buffer_size); + cmd = rng->engine->command; + cmd->u.len = len; + err = loongson_se_send_engine_cmd(rng->engine); + if (err) + break; + + cmd = rng->engine->command_ret; + if (cmd->u.ret) { + err = -EIO; + break; + } + + memcpy(dstn, rng->engine->data_buffer, len); + dlen -= len; + dstn += len; + } while (dlen > 0); + mutex_unlock(&rng->lock); + + return err; +} + +static int loongson_rng_init(struct crypto_tfm *tfm) +{ + struct loongson_rng_ctx *ctx = crypto_tfm_ctx(tfm); + struct loongson_rng *rng; + u32 min_used = U32_MAX; + + mutex_lock(&rng_devices.lock); + list_for_each_entry(rng, &rng_devices.list, list) { + if (rng->used < min_used) { + ctx->rng = rng; + min_used = rng->used; + } + } + ctx->rng->used++; + mutex_unlock(&rng_devices.lock); + + return 0; +} + +static void loongson_rng_exit(struct crypto_tfm *tfm) +{ + struct loongson_rng_ctx *ctx = crypto_tfm_ctx(tfm); + + mutex_lock(&rng_devices.lock); + ctx->rng->used--; + mutex_unlock(&rng_devices.lock); +} + +static int loongson_rng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct loongson_rng_ctx *ctx = crypto_rng_ctx(tfm); + struct loongson_rng *rng = ctx->rng; + struct loongson_rng_cmd *cmd; + int err; + + if (slen < SE_SEED_SIZE) + return -EINVAL; + + slen = min(slen, rng->engine->buffer_size); + + mutex_lock(&rng->lock); + cmd = rng->engine->command; + cmd->u.len = slen; + cmd->seed_off = rng->engine->buffer_off; + memcpy(rng->engine->data_buffer, seed, slen); + err = loongson_se_send_engine_cmd(rng->engine); + if (err) + goto out; + + cmd = rng->engine->command_ret; + if (cmd->u.ret) + err = -EIO; +out: + mutex_unlock(&rng->lock); + + return err; +} + +static struct rng_alg loongson_rng_alg = { + .generate = loongson_rng_generate, + .seed = loongson_rng_seed, + .seedsize = SE_SEED_SIZE, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "loongson_stdrng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct loongson_rng_ctx), + .cra_module = THIS_MODULE, + .cra_init = loongson_rng_init, + .cra_exit = loongson_rng_exit, + }, +}; + +static int loongson_rng_probe(struct platform_device *pdev) +{ + struct loongson_rng_cmd *cmd; + struct loongson_rng *rng; + int ret = 0; + + rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); + if (!rng) + return -ENOMEM; + + rng->engine = loongson_se_init_engine(pdev->dev.parent, SE_ENGINE_RNG); + if (!rng->engine) + return -ENODEV; + cmd = rng->engine->command; + cmd->cmd_id = SE_CMD_RNG; + cmd->out_off = rng->engine->buffer_off; + mutex_init(&rng->lock); + + mutex_lock(&rng_devices.lock); + + if (!rng_devices.registered) { + ret = crypto_register_rng(&loongson_rng_alg); + if (ret) { + dev_err(&pdev->dev, "failed to register crypto(%d)\n", ret); + goto out; + } + rng_devices.registered = 1; + } + + list_add_tail(&rng->list, &rng_devices.list); +out: + mutex_unlock(&rng_devices.lock); + + return ret; +} + +static struct platform_driver loongson_rng_driver = { + .probe = loongson_rng_probe, + .driver = { + .name = "loongson-rng", + }, +}; +module_platform_driver(loongson_rng_driver); + +MODULE_ALIAS("platform:loongson-rng"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Yinggang Gu <guyinggang@loongson.cn>"); +MODULE_AUTHOR("Qunqin Zhao <zhaoqunqin@loongson.cn>"); +MODULE_DESCRIPTION("Loongson Random Number Generator driver"); diff --git a/drivers/crypto/marvell/Kconfig b/drivers/crypto/marvell/Kconfig index 4c25a78ab3ed..aa269abb0499 100644 --- a/drivers/crypto/marvell/Kconfig +++ b/drivers/crypto/marvell/Kconfig @@ -24,7 +24,7 @@ config CRYPTO_DEV_OCTEONTX_CPT tristate "Support for Marvell OcteonTX CPT driver" depends on ARCH_THUNDER || COMPILE_TEST depends on PCI_MSI && 64BIT - depends on CRYPTO_LIB_AES + select CRYPTO_LIB_AES select CRYPTO_SKCIPHER select CRYPTO_HASH select CRYPTO_AEAD @@ -41,10 +41,10 @@ config CRYPTO_DEV_OCTEONTX2_CPT tristate "Marvell OcteonTX2 CPT driver" depends on ARCH_THUNDER2 || COMPILE_TEST depends on PCI_MSI && 64BIT - depends on CRYPTO_LIB_AES depends on NET_VENDOR_MARVELL select OCTEONTX2_MBOX select CRYPTO_DEV_MARVELL + select CRYPTO_LIB_AES select CRYPTO_SKCIPHER select CRYPTO_HASH select CRYPTO_AEAD diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index fa08f10e6f3f..301bdf239e7d 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -94,7 +94,7 @@ static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) { - if (engine->chain.first && engine->chain.last) + if (engine->chain_hw.first && engine->chain_hw.last) return mv_cesa_tdma_process(engine, status); return mv_cesa_std_process(engine, status); @@ -420,7 +420,6 @@ static int mv_cesa_probe(struct platform_device *pdev) { const struct mv_cesa_caps *caps = &orion_caps; const struct mbus_dram_target_info *dram; - const struct of_device_id *match; struct device *dev = &pdev->dev; struct mv_cesa_dev *cesa; struct mv_cesa_engine *engines; @@ -433,11 +432,9 @@ static int mv_cesa_probe(struct platform_device *pdev) } if (dev->of_node) { - match = of_match_node(mv_cesa_of_match_table, dev->of_node); - if (!match || !match->data) + caps = of_device_get_match_data(dev); + if (!caps) return -ENOTSUPP; - - caps = match->data; } cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index d215a6bed6bc..50ca1039fdaa 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -440,8 +440,10 @@ struct mv_cesa_dev { * SRAM * @queue: fifo of the pending crypto requests * @load: engine load counter, useful for load balancing - * @chain: list of the current tdma descriptors being processed - * by this engine. + * @chain_hw: list of the current tdma descriptors being processed + * by the hardware. + * @chain_sw: list of the current tdma descriptors that will be + * submitted to the hardware. * @complete_queue: fifo of the processed requests by the engine * * Structure storing CESA engine information. @@ -463,7 +465,8 @@ struct mv_cesa_engine { struct gen_pool *pool; struct crypto_queue queue; atomic_t load; - struct mv_cesa_tdma_chain chain; + struct mv_cesa_tdma_chain chain_hw; + struct mv_cesa_tdma_chain chain_sw; struct list_head complete_queue; int irq; }; diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index cf62db50f958..3fe0fd9226cf 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -75,9 +75,12 @@ mv_cesa_skcipher_dma_cleanup(struct skcipher_request *req) static inline void mv_cesa_skcipher_cleanup(struct skcipher_request *req) { struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_engine *engine = creq->base.engine; if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_skcipher_dma_cleanup(req); + + atomic_sub(req->cryptlen, &engine->load); } static void mv_cesa_skcipher_std_step(struct skcipher_request *req) @@ -212,7 +215,6 @@ mv_cesa_skcipher_complete(struct crypto_async_request *req) struct mv_cesa_engine *engine = creq->base.engine; unsigned int ivsize; - atomic_sub(skreq->cryptlen, &engine->load); ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(skreq)); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) { @@ -459,6 +461,9 @@ static int mv_cesa_skcipher_queue_req(struct skcipher_request *req, struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_engine *engine; + if (!req->cryptlen) + return 0; + ret = mv_cesa_skcipher_req_init(req, tmpl); if (ret) return ret; diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index f150861ceaf6..5103d36cdfdb 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -110,9 +110,12 @@ static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) static inline void mv_cesa_ahash_cleanup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_engine *engine = creq->base.engine; if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_cleanup(req); + + atomic_sub(req->nbytes, &engine->load); } static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) @@ -362,16 +365,13 @@ static void mv_cesa_ahash_complete(struct crypto_async_request *req) if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ && (creq->base.chain.last->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_RESULT) { - __le32 *data = NULL; + const void *data; /* * Result is already in the correct endianness when the SA is * used */ data = creq->base.chain.last->op->ctx.hash.hash; - for (i = 0; i < digsize / 4; i++) - creq->state[i] = le32_to_cpu(data[i]); - memcpy(ahashreq->result, data, digsize); } else { for (i = 0; i < digsize / 4; i++) @@ -395,8 +395,6 @@ static void mv_cesa_ahash_complete(struct crypto_async_request *req) } } } - - atomic_sub(ahashreq->nbytes, &engine->load); } static void mv_cesa_ahash_prepare(struct crypto_async_request *req, @@ -663,7 +661,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (ret) goto err_free_tdma; - if (iter.src.sg) { + if (iter.base.len > iter.src.op_offset) { /* * Add all the new data, inserting an operation block and * launch command between each full SRAM block-worth of diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index 388a06e180d6..243305354420 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -38,6 +38,15 @@ void mv_cesa_dma_step(struct mv_cesa_req *dreq) { struct mv_cesa_engine *engine = dreq->engine; + spin_lock_bh(&engine->lock); + if (engine->chain_sw.first == dreq->chain.first) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; + } + engine->chain_hw.first = dreq->chain.first; + engine->chain_hw.last = dreq->chain.last; + spin_unlock_bh(&engine->lock); + writel_relaxed(0, engine->regs + CESA_SA_CFG); mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); @@ -96,25 +105,27 @@ void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, struct mv_cesa_req *dreq) { - if (engine->chain.first == NULL && engine->chain.last == NULL) { - engine->chain.first = dreq->chain.first; - engine->chain.last = dreq->chain.last; - } else { - struct mv_cesa_tdma_desc *last; + struct mv_cesa_tdma_desc *last = engine->chain_sw.last; - last = engine->chain.last; + /* + * Break the DMA chain if the request being queued needs the IV + * regs to be set before lauching the request. + */ + if (!last || dreq->chain.first->flags & CESA_TDMA_SET_STATE) + engine->chain_sw.first = dreq->chain.first; + else { last->next = dreq->chain.first; - engine->chain.last = dreq->chain.last; - - /* - * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on - * the last element of the current chain, or if the request - * being queued needs the IV regs to be set before lauching - * the request. - */ - if (!(last->flags & CESA_TDMA_BREAK_CHAIN) && - !(dreq->chain.first->flags & CESA_TDMA_SET_STATE)) - last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + } + last = dreq->chain.last; + engine->chain_sw.last = last; + /* + * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on + * the last element of the current chain. + */ + if (last->flags & CESA_TDMA_BREAK_CHAIN) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; } } @@ -127,7 +138,7 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) tdma_cur = readl(engine->regs + CESA_TDMA_CUR); - for (tdma = engine->chain.first; tdma; tdma = next) { + for (tdma = engine->chain_hw.first; tdma; tdma = next) { spin_lock_bh(&engine->lock); next = tdma->next; spin_unlock_bh(&engine->lock); @@ -149,12 +160,12 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) &backlog); /* Re-chaining to the next request */ - engine->chain.first = tdma->next; + engine->chain_hw.first = tdma->next; tdma->next = NULL; /* If this is the last request, clear the chain */ - if (engine->chain.first == NULL) - engine->chain.last = NULL; + if (engine->chain_hw.first == NULL) + engine->chain_hw.last = NULL; spin_unlock_bh(&engine->lock); ctx = crypto_tfm_ctx(req->tfm); diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index c4250e5fcf8f..9f5601c0280b 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -10,6 +10,7 @@ #include <linux/ctype.h> #include <linux/firmware.h> +#include <linux/string_choices.h> #include "otx_cpt_common.h" #include "otx_cptpf_ucode.h" #include "otx_cptpf.h" @@ -505,17 +506,6 @@ int otx_cpt_uc_supports_eng_type(struct otx_cpt_ucode *ucode, int eng_type) } EXPORT_SYMBOL_GPL(otx_cpt_uc_supports_eng_type); -int otx_cpt_eng_grp_has_eng_type(struct otx_cpt_eng_grp_info *eng_grp, - int eng_type) -{ - struct otx_cpt_engs_rsvd *engs; - - engs = find_engines_by_type(eng_grp, eng_type); - - return (engs != NULL ? 1 : 0); -} -EXPORT_SYMBOL_GPL(otx_cpt_eng_grp_has_eng_type); - static void print_ucode_info(struct otx_cpt_eng_grp_info *eng_grp, char *buf, int size) { @@ -614,8 +604,8 @@ static void print_dbg_info(struct device *dev, for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) { grp = &eng_grps->grp[i]; - pr_debug("engine_group%d, state %s\n", i, grp->is_enabled ? - "enabled" : "disabled"); + pr_debug("engine_group%d, state %s\n", i, + str_enabled_disabled(grp->is_enabled)); if (grp->is_enabled) { mirrored_grp = &eng_grps->grp[grp->mirror.idx]; pr_debug("Ucode0 filename %s, version %s\n", diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h index 8620ac87a447..df79ee416c0d 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h @@ -174,7 +174,5 @@ int otx_cpt_try_create_default_eng_grps(struct pci_dev *pdev, void otx_cpt_set_eng_grps_is_rdonly(struct otx_cpt_eng_grps *eng_grps, bool is_rdonly); int otx_cpt_uc_supports_eng_type(struct otx_cpt_ucode *ucode, int eng_type); -int otx_cpt_eng_grp_has_eng_type(struct otx_cpt_eng_grp_info *eng_grp, - int eng_type); #endif /* __OTX_CPTPF_UCODE_H */ diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c index 6bfc59e67747..d4aab9e20f2a 100644 --- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c @@ -6,6 +6,7 @@ #include "otx2_cptvf.h" #include "otx2_cptlf.h" #include "cn10k_cpt.h" +#include "otx2_cpt_common.h" static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, struct otx2_cptlf_info *lf); @@ -27,7 +28,7 @@ static struct cpt_hw_ops cn10k_hw_ops = { static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, struct otx2_cptlf_info *lf) { - void __iomem *lmtline = lf->lmtline; + void *lmtline = lf->lfs->lmt_info.base + (lf->slot * LMTLINE_SIZE); u64 val = (lf->slot & 0x7FF); u64 tar_addr = 0; @@ -41,15 +42,49 @@ static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, dma_wmb(); /* Copy CPT command to LMTLINE */ - memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); + memcpy(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); cn10k_lmt_flush(val, tar_addr); } +void cn10k_cpt_lmtst_free(struct pci_dev *pdev, struct otx2_cptlfs_info *lfs) +{ + struct otx2_lmt_info *lmt_info = &lfs->lmt_info; + + if (!lmt_info->base) + return; + + dma_free_attrs(&pdev->dev, lmt_info->size, + lmt_info->base - lmt_info->align, + lmt_info->iova - lmt_info->align, + DMA_ATTR_FORCE_CONTIGUOUS); +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_lmtst_free, "CRYPTO_DEV_OCTEONTX2_CPT"); + +static int cn10k_cpt_lmtst_alloc(struct pci_dev *pdev, + struct otx2_cptlfs_info *lfs, u32 size) +{ + struct otx2_lmt_info *lmt_info = &lfs->lmt_info; + dma_addr_t align_iova; + dma_addr_t iova; + + lmt_info->base = dma_alloc_attrs(&pdev->dev, size, &iova, GFP_KERNEL, + DMA_ATTR_FORCE_CONTIGUOUS); + if (!lmt_info->base) + return -ENOMEM; + + align_iova = ALIGN((u64)iova, LMTLINE_ALIGN); + lmt_info->iova = align_iova; + lmt_info->align = align_iova - iova; + lmt_info->size = size; + lmt_info->base += lmt_info->align; + return 0; +} + int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf) { struct pci_dev *pdev = cptpf->pdev; - resource_size_t size; - u64 lmt_base; + u32 size; + int ret; if (!test_bit(CN10K_LMTST, &cptpf->cap_flag)) { cptpf->lfs.ops = &otx2_hw_ops; @@ -57,44 +92,52 @@ int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf) } cptpf->lfs.ops = &cn10k_hw_ops; - lmt_base = readq(cptpf->reg_base + RVU_PF_LMTLINE_ADDR); - if (!lmt_base) { - dev_err(&pdev->dev, "PF LMTLINE address not configured\n"); - return -ENOMEM; + size = OTX2_CPT_MAX_VFS_NUM * LMTLINE_SIZE + LMTLINE_ALIGN; + ret = cn10k_cpt_lmtst_alloc(pdev, &cptpf->lfs, size); + if (ret) { + dev_err(&pdev->dev, "PF-%d LMTLINE memory allocation failed\n", + cptpf->pf_id); + return ret; } - size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); - size -= ((1 + cptpf->max_vfs) * MBOX_SIZE); - cptpf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, lmt_base, size); - if (!cptpf->lfs.lmt_base) { - dev_err(&pdev->dev, - "Mapping of PF LMTLINE address failed\n"); - return -ENOMEM; + + ret = otx2_cpt_lmtst_tbl_setup_msg(&cptpf->lfs); + if (ret) { + dev_err(&pdev->dev, "PF-%d: LMTST Table setup failed\n", + cptpf->pf_id); + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); } return 0; } -EXPORT_SYMBOL_NS_GPL(cn10k_cptpf_lmtst_init, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(cn10k_cptpf_lmtst_init, "CRYPTO_DEV_OCTEONTX2_CPT"); int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) { struct pci_dev *pdev = cptvf->pdev; - resource_size_t offset, size; + u32 size; + int ret; if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) return 0; - offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); - size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); - /* Map VF LMILINE region */ - cptvf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, offset, size); - if (!cptvf->lfs.lmt_base) { - dev_err(&pdev->dev, "Unable to map BAR4\n"); - return -ENOMEM; + size = cptvf->lfs.lfs_num * LMTLINE_SIZE + LMTLINE_ALIGN; + ret = cn10k_cpt_lmtst_alloc(pdev, &cptvf->lfs, size); + if (ret) { + dev_err(&pdev->dev, "VF-%d LMTLINE memory allocation failed\n", + cptvf->vf_id); + return ret; + } + + ret = otx2_cpt_lmtst_tbl_setup_msg(&cptvf->lfs); + if (ret) { + dev_err(&pdev->dev, "VF-%d: LMTST Table setup failed\n", + cptvf->vf_id); + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); } return 0; } -EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, "CRYPTO_DEV_OCTEONTX2_CPT"); void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx) @@ -110,7 +153,7 @@ void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, DMA_BIDIRECTIONAL); kfree(er_ctx->hw_ctx); } -EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_clear, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_clear, "CRYPTO_DEV_OCTEONTX2_CPT"); void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz) { @@ -119,7 +162,7 @@ void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz) hctx->w0.ctx_sz = ctx_sz; hctx->w0.ctx_push_sz = 1; } -EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_set, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_set, "CRYPTO_DEV_OCTEONTX2_CPT"); int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx) @@ -149,7 +192,7 @@ int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, return 0; } -EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_init, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_init, "CRYPTO_DEV_OCTEONTX2_CPT"); void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval) { @@ -168,7 +211,7 @@ void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval) otx2_cpt_read64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot, OTX2_CPT_LF_CTX_ERR); } -EXPORT_SYMBOL_NS_GPL(cn10k_cpt_ctx_flush, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_ctx_flush, "CRYPTO_DEV_OCTEONTX2_CPT"); void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf) { @@ -177,4 +220,4 @@ void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf) else cptvf->lfs.ops = &otx2_hw_ops; } -EXPORT_SYMBOL_NS_GPL(cptvf_hw_ops_get, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(cptvf_hw_ops_get, "CRYPTO_DEV_OCTEONTX2_CPT"); diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h index 92be3ecf570f..ea5990048c21 100644 --- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h @@ -50,6 +50,7 @@ static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result) int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf); int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf); +void cn10k_cpt_lmtst_free(struct pci_dev *pdev, struct otx2_cptlfs_info *lfs); void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval); int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h index c5b7c57574ef..062def303dce 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -18,9 +18,8 @@ #define OTX2_CPT_MAX_VFS_NUM 128 #define OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs) \ (((blk) << 20) | ((slot) << 12) | (offs)) -#define OTX2_CPT_RVU_PFFUNC(pf, func) \ - ((((pf) & RVU_PFVF_PF_MASK) << RVU_PFVF_PF_SHIFT) | \ - (((func) & RVU_PFVF_FUNC_MASK) << RVU_PFVF_FUNC_SHIFT)) + +#define OTX2_CPT_RVU_PFFUNC(pdev, pf, func) rvu_make_pcifunc(pdev, pf, func) #define OTX2_CPT_INVALID_CRYPTO_ENG_GRP 0xFF #define OTX2_CPT_NAME_LENGTH 64 @@ -145,11 +144,8 @@ static inline u64 otx2_cpt_read64(void __iomem *reg_base, u64 blk, u64 slot, static inline bool is_dev_otx2(struct pci_dev *pdev) { - if (pdev->device == OTX2_CPT_PCI_PF_DEVICE_ID || - pdev->device == OTX2_CPT_PCI_VF_DEVICE_ID) - return true; - - return false; + return pdev->device == OTX2_CPT_PCI_PF_DEVICE_ID || + pdev->device == OTX2_CPT_PCI_VF_DEVICE_ID; } static inline bool is_dev_cn10ka(struct pci_dev *pdev) @@ -159,12 +155,10 @@ static inline bool is_dev_cn10ka(struct pci_dev *pdev) static inline bool is_dev_cn10ka_ax(struct pci_dev *pdev) { - if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && - ((pdev->revision & 0xFF) == 4 || (pdev->revision & 0xFF) == 0x50 || - (pdev->revision & 0xff) == 0x51)) - return true; - - return false; + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + ((pdev->revision & 0xFF) == 4 || + (pdev->revision & 0xFF) == 0x50 || + (pdev->revision & 0xFF) == 0x51); } static inline bool is_dev_cn10kb(struct pci_dev *pdev) @@ -174,11 +168,8 @@ static inline bool is_dev_cn10kb(struct pci_dev *pdev) static inline bool is_dev_cn10ka_b0(struct pci_dev *pdev) { - if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && - (pdev->revision & 0xFF) == 0x54) - return true; - - return false; + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + (pdev->revision & 0xFF) == 0x54; } static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, @@ -192,18 +183,12 @@ static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, static inline bool cpt_is_errata_38550_exists(struct pci_dev *pdev) { - if (is_dev_otx2(pdev) || is_dev_cn10ka_ax(pdev)) - return true; - - return false; + return is_dev_otx2(pdev) || is_dev_cn10ka_ax(pdev); } static inline bool cpt_feature_sgv2(struct pci_dev *pdev) { - if (!is_dev_otx2(pdev) && !is_dev_cn10ka_ax(pdev)) - return true; - - return false; + return !is_dev_otx2(pdev) && !is_dev_cn10ka_ax(pdev); } int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); @@ -223,5 +208,6 @@ int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs); int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs); int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox); int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot); +int otx2_cpt_lmtst_tbl_setup_msg(struct otx2_cptlfs_info *lfs); #endif /* __OTX2_CPT_COMMON_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c index 215a1a8ba7e9..07a74f702c3a 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c @@ -24,7 +24,8 @@ static int otx2_cpt_dl_egrp_delete(struct devlink *dl, u32 id, } static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id, - struct devlink_param_gset_ctx *ctx) + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) { ctx->val.vstr[0] = '\0'; @@ -32,7 +33,8 @@ static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id, } static int otx2_cpt_dl_t106_mode_get(struct devlink *dl, u32 id, - struct devlink_param_gset_ctx *ctx) + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) { struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c index 5be0103c1fb8..95f3de3a34eb 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c @@ -19,7 +19,7 @@ int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) } return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_mbox_msg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_mbox_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) { @@ -37,13 +37,13 @@ int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) return otx2_cpt_send_mbox_msg(mbox, pdev); } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_ready_msg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_ready_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_send_af_reg_requests(struct otx2_mbox *mbox, struct pci_dev *pdev) { return otx2_cpt_send_mbox_msg(mbox, pdev); } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_af_reg_requests, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_af_reg_requests, "CRYPTO_DEV_OCTEONTX2_CPT"); static int otx2_cpt_add_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, u64 reg, @@ -95,7 +95,7 @@ int otx2_cpt_add_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, return 0; } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_add_write_af_reg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_add_write_af_reg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, u64 reg, u64 *val, int blkaddr) @@ -108,7 +108,7 @@ int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, return otx2_cpt_send_mbox_msg(mbox, pdev); } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_read_af_reg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_read_af_reg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, u64 reg, u64 val, int blkaddr) @@ -121,7 +121,7 @@ int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, return otx2_cpt_send_mbox_msg(mbox, pdev); } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_write_af_reg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_write_af_reg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs) { @@ -180,7 +180,7 @@ int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs) return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_detach_rsrcs_msg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_detach_rsrcs_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs) { @@ -213,7 +213,7 @@ int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs) } return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_msix_offset_msg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_msix_offset_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox) { @@ -228,7 +228,7 @@ int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox) return otx2_mbox_check_rsp_msgs(mbox, 0); } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_sync_mbox_msg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_sync_mbox_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot) { @@ -254,4 +254,29 @@ int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot) return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cpt_lf_reset_msg, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cpt_lf_reset_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_lmtst_tbl_setup_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct lmtst_tbl_setup_req *req; + + req = (struct lmtst_tbl_setup_req *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) { + dev_err(&pdev->dev, "RVU MBOX failed to alloc message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_LMTST_TBL_SETUP; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + + req->use_local_lmt_region = true; + req->lmt_iova = lfs->lmt_info.iova; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_lmtst_tbl_setup_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h index e27e849b01df..e64ca30335de 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h @@ -34,6 +34,9 @@ #define SG_COMP_2 2 #define SG_COMP_1 1 +#define OTX2_CPT_DPTR_RPTR_ALIGN 8 +#define OTX2_CPT_RES_ADDR_ALIGN 32 + union otx2_cpt_opcode { u16 flags; struct { @@ -347,22 +350,48 @@ static inline struct otx2_cpt_inst_info * cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, gfp_t gfp) { - u32 dlen = 0, g_len, sg_len, info_len; - int align = OTX2_CPT_DMA_MINALIGN; + u32 dlen = 0, g_len, s_len, sg_len, info_len; struct otx2_cpt_inst_info *info; - u16 g_sz_bytes, s_sz_bytes; u32 total_mem_len; int i; - g_sz_bytes = ((req->in_cnt + 2) / 3) * - sizeof(struct cn10kb_cpt_sglist_component); - s_sz_bytes = ((req->out_cnt + 2) / 3) * - sizeof(struct cn10kb_cpt_sglist_component); + /* Allocate memory to meet below alignment requirement: + * ------------------------------------ + * | struct otx2_cpt_inst_info | + * | (No alignment required) | + * | --------------------------------| + * | | padding for ARCH_DMA_MINALIGN | + * | | alignment | + * |------------------------------------| + * | SG List Gather/Input memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * |---------------------------------- | + * | SG List Scatter/Output memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * | -------------------------------| + * | | padding for 32B alignment | + * |------------------------------------| + * | Result response memory | + * | Alignment = 32Byte | + * ------------------------------------ + */ + + info_len = sizeof(*info); + + g_len = ((req->in_cnt + 2) / 3) * + sizeof(struct cn10kb_cpt_sglist_component); + s_len = ((req->out_cnt + 2) / 3) * + sizeof(struct cn10kb_cpt_sglist_component); + sg_len = g_len + s_len; - g_len = ALIGN(g_sz_bytes, align); - sg_len = ALIGN(g_len + s_sz_bytes, align); - info_len = ALIGN(sizeof(*info), align); - total_mem_len = sg_len + info_len + sizeof(union otx2_cpt_res_s); + /* Allocate extra memory for SG and response address alignment */ + total_mem_len = ALIGN(info_len, OTX2_CPT_DPTR_RPTR_ALIGN); + total_mem_len += (ARCH_DMA_MINALIGN - 1) & + ~(OTX2_CPT_DPTR_RPTR_ALIGN - 1); + total_mem_len += ALIGN(sg_len, OTX2_CPT_RES_ADDR_ALIGN); + total_mem_len += sizeof(union otx2_cpt_res_s); info = kzalloc(total_mem_len, gfp); if (unlikely(!info)) @@ -372,7 +401,8 @@ cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, dlen += req->in[i].size; info->dlen = dlen; - info->in_buffer = (u8 *)info + info_len; + info->in_buffer = PTR_ALIGN((u8 *)info + info_len, ARCH_DMA_MINALIGN); + info->out_buffer = info->in_buffer + g_len; info->gthr_sz = req->in_cnt; info->sctr_sz = req->out_cnt; @@ -384,7 +414,7 @@ cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, } if (sgv2io_components_setup(pdev, req->out, req->out_cnt, - &info->in_buffer[g_len])) { + info->out_buffer)) { dev_err(&pdev->dev, "Failed to setup scatter list\n"); goto destroy_info; } @@ -401,8 +431,10 @@ cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, * Get buffer for union otx2_cpt_res_s response * structure and its physical address */ - info->completion_addr = info->in_buffer + sg_len; - info->comp_baddr = info->dptr_baddr + sg_len; + info->completion_addr = PTR_ALIGN((info->in_buffer + sg_len), + OTX2_CPT_RES_ADDR_ALIGN); + info->comp_baddr = ALIGN((info->dptr_baddr + sg_len), + OTX2_CPT_RES_ADDR_ALIGN); return info; @@ -417,10 +449,9 @@ static inline struct otx2_cpt_inst_info * otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, gfp_t gfp) { - int align = OTX2_CPT_DMA_MINALIGN; struct otx2_cpt_inst_info *info; - u32 dlen, align_dlen, info_len; - u16 g_sz_bytes, s_sz_bytes; + u32 dlen, info_len; + u16 g_len, s_len; u32 total_mem_len; if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || @@ -429,22 +460,54 @@ otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, return NULL; } - g_sz_bytes = ((req->in_cnt + 3) / 4) * - sizeof(struct otx2_cpt_sglist_component); - s_sz_bytes = ((req->out_cnt + 3) / 4) * - sizeof(struct otx2_cpt_sglist_component); + /* Allocate memory to meet below alignment requirement: + * ------------------------------------ + * | struct otx2_cpt_inst_info | + * | (No alignment required) | + * | --------------------------------| + * | | padding for ARCH_DMA_MINALIGN | + * | | alignment | + * |------------------------------------| + * | SG List Header of 8 Byte | + * |------------------------------------| + * | SG List Gather/Input memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * |---------------------------------- | + * | SG List Scatter/Output memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * | -------------------------------| + * | | padding for 32B alignment | + * |------------------------------------| + * | Result response memory | + * | Alignment = 32Byte | + * ------------------------------------ + */ + + info_len = sizeof(*info); + + g_len = ((req->in_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + s_len = ((req->out_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + + dlen = g_len + s_len + SG_LIST_HDR_SIZE; - dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; - align_dlen = ALIGN(dlen, align); - info_len = ALIGN(sizeof(*info), align); - total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); + /* Allocate extra memory for SG and response address alignment */ + total_mem_len = ALIGN(info_len, OTX2_CPT_DPTR_RPTR_ALIGN); + total_mem_len += (ARCH_DMA_MINALIGN - 1) & + ~(OTX2_CPT_DPTR_RPTR_ALIGN - 1); + total_mem_len += ALIGN(dlen, OTX2_CPT_RES_ADDR_ALIGN); + total_mem_len += sizeof(union otx2_cpt_res_s); info = kzalloc(total_mem_len, gfp); if (unlikely(!info)) return NULL; info->dlen = dlen; - info->in_buffer = (u8 *)info + info_len; + info->in_buffer = PTR_ALIGN((u8 *)info + info_len, ARCH_DMA_MINALIGN); + info->out_buffer = info->in_buffer + SG_LIST_HDR_SIZE + g_len; ((u16 *)info->in_buffer)[0] = req->out_cnt; ((u16 *)info->in_buffer)[1] = req->in_cnt; @@ -460,7 +523,7 @@ otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, } if (setup_sgio_components(pdev, req->out, req->out_cnt, - &info->in_buffer[8 + g_sz_bytes])) { + info->out_buffer)) { dev_err(&pdev->dev, "Failed to setup scatter list\n"); goto destroy_info; } @@ -476,8 +539,10 @@ otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, * Get buffer for union otx2_cpt_res_s response * structure and its physical address */ - info->completion_addr = info->in_buffer + align_dlen; - info->comp_baddr = info->dptr_baddr + align_dlen; + info->completion_addr = PTR_ALIGN((info->in_buffer + dlen), + OTX2_CPT_RES_ADDR_ALIGN); + info->comp_baddr = ALIGN((info->dptr_baddr + dlen), + OTX2_CPT_RES_ADDR_ALIGN); return info; @@ -490,6 +555,7 @@ struct otx2_cptlf_wqe; int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, int cpu_num); void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe); -int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev); +int otx2_cpt_get_eng_grp_num(struct pci_dev *pdev, + enum otx2_cpt_eng_type); #endif /* __OTX2_CPT_REQMGR_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c index b52728e3c0d1..dc7c7a2650a5 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c @@ -288,8 +288,7 @@ void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs) cptlf_set_misc_intrs(lfs, false); } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_misc_interrupts, - CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_misc_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs) { @@ -308,8 +307,7 @@ void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs) cptlf_set_done_intrs(lfs, false); } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_done_interrupts, - CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_done_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, int lf_num, int irq_offset, @@ -351,8 +349,7 @@ free_irq: otx2_cptlf_unregister_misc_interrupts(lfs); return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_misc_interrupts, - CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_misc_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs) { @@ -375,8 +372,7 @@ free_irq: otx2_cptlf_unregister_done_interrupts(lfs); return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_done_interrupts, - CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_done_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) { @@ -390,7 +386,7 @@ void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) free_cpumask_var(lfs->lf[slot].affinity_mask); } } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_free_irqs_affinity, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_free_irqs_affinity, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs) { @@ -423,7 +419,7 @@ free_affinity_mask: otx2_cptlf_free_irqs_affinity(lfs); return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_set_irqs_affinity, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_set_irqs_affinity, "CRYPTO_DEV_OCTEONTX2_CPT"); int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, int lfs_num) @@ -437,10 +433,7 @@ int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, for (slot = 0; slot < lfs->lfs_num; slot++) { lfs->lf[slot].lfs = lfs; lfs->lf[slot].slot = slot; - if (lfs->lmt_base) - lfs->lf[slot].lmtline = lfs->lmt_base + - (slot * LMTLINE_SIZE); - else + if (!lfs->lmt_info.base) lfs->lf[slot].lmtline = lfs->reg_base + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, OTX2_CPT_LMT_LF_LMTLINEX(0)); @@ -486,7 +479,7 @@ clear_lfs_num: lfs->lfs_num = 0; return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_init, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_init, "CRYPTO_DEV_OCTEONTX2_CPT"); void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) { @@ -498,7 +491,7 @@ void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) otx2_cpt_detach_rsrcs_msg(lfs); lfs->lfs_num = 0; } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_shutdown, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_shutdown, "CRYPTO_DEV_OCTEONTX2_CPT"); MODULE_AUTHOR("Marvell"); MODULE_DESCRIPTION("Marvell RVU CPT Common module"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h index bd8604be2952..1b9f75214d18 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h @@ -105,18 +105,27 @@ struct cpt_hw_ops { gfp_t gfp); }; +#define LMTLINE_SIZE 128 +#define LMTLINE_ALIGN 128 +struct otx2_lmt_info { + void *base; + dma_addr_t iova; + u32 size; + u8 align; +}; + struct otx2_cptlfs_info { /* Registers start address of VF/PF LFs are attached to */ void __iomem *reg_base; -#define LMTLINE_SIZE 128 - void __iomem *lmt_base; + struct otx2_lmt_info lmt_info; struct pci_dev *pdev; /* Device LFs are attached to */ struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; struct otx2_mbox *mbox; struct cpt_hw_ops *ops; u8 are_lfs_attached; /* Whether CPT LFs are attached */ u8 lfs_num; /* Number of CPT LFs */ - u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */ + u8 kcrypto_se_eng_grp_num; /* Crypto symmetric engine group number */ + u8 kcrypto_ae_eng_grp_num; /* Crypto asymmetric engine group number */ u8 kvf_limits; /* Kernel crypto limits */ atomic_t state; /* LF's state. started/reset */ int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index 400e36d9908f..1c5c262af48d 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -639,6 +639,12 @@ static int cptpf_device_init(struct otx2_cptpf_dev *cptpf) /* Disable all cores */ ret = otx2_cpt_disable_all_cores(cptpf); + otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base, + &cptpf->afpf_mbox, BLKADDR_CPT0); + if (cptpf->has_cpt1) + otx2_cptlf_set_dev_info(&cptpf->cpt1_lfs, cptpf->pdev, + cptpf->reg_base, &cptpf->afpf_mbox, + BLKADDR_CPT1); return ret; } @@ -739,18 +745,22 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, dev_err(dev, "Unable to get usable DMA configuration\n"); goto clear_drvdata; } - /* Map PF's configuration registers */ - err = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, - OTX2_CPT_DRV_NAME); + err = pcim_request_all_regions(pdev, OTX2_CPT_DRV_NAME); if (err) { - dev_err(dev, "Couldn't get PCI resources 0x%x\n", err); + dev_err(dev, "Couldn't request PCI resources 0x%x\n", err); goto clear_drvdata; } pci_set_master(pdev); pci_set_drvdata(pdev, cptpf); cptpf->pdev = pdev; - cptpf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + /* Map PF's configuration registers */ + cptpf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0); + if (!cptpf->reg_base) { + err = -ENOMEM; + dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", err); + goto clear_drvdata; + } /* Check if AF driver is up, otherwise defer probe */ err = cpt_is_pf_usable(cptpf); @@ -782,19 +792,19 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, cptpf->max_vfs = pci_sriov_get_totalvfs(pdev); cptpf->kvf_limits = 1; - err = cn10k_cptpf_lmtst_init(cptpf); + /* Initialize CPT PF device */ + err = cptpf_device_init(cptpf); if (err) goto unregister_intr; - /* Initialize CPT PF device */ - err = cptpf_device_init(cptpf); + err = cn10k_cptpf_lmtst_init(cptpf); if (err) goto unregister_intr; /* Initialize engine groups */ err = otx2_cpt_init_eng_grps(pdev, &cptpf->eng_grps); if (err) - goto unregister_intr; + goto free_lmtst; err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group); if (err) @@ -810,6 +820,8 @@ sysfs_grp_del: sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group); cleanup_eng_grps: otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); +free_lmtst: + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); unregister_intr: cptpf_disable_afpf_mbox_intr(cptpf); destroy_afpf_mbox: @@ -844,6 +856,8 @@ static void otx2_cptpf_remove(struct pci_dev *pdev) cptpf_disable_afpf_mbox_intr(cptpf); /* Destroy AF-PF mbox */ cptpf_afpf_mbox_destroy(cptpf); + /* Free LMTST memory */ + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); pci_set_drvdata(pdev, NULL); } @@ -864,7 +878,7 @@ static struct pci_driver otx2_cpt_pci_driver = { module_pci_driver(otx2_cpt_pci_driver); -MODULE_IMPORT_NS(CRYPTO_DEV_OCTEONTX2_CPT); +MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT"); MODULE_AUTHOR("Marvell"); MODULE_DESCRIPTION(OTX2_CPT_DRV_STRING); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c index ec1ac7e836a3..b4b2d3d1cbc2 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c @@ -142,7 +142,7 @@ static int send_inline_ipsec_inbound_msg(struct otx2_cptpf_dev *cptpf, memset(req, 0, sizeof(*req)); req->hdr.id = MBOX_MSG_CPT_INLINE_IPSEC_CFG; req->hdr.sig = OTX2_MBOX_REQ_SIG; - req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptpf->pf_id, 0); + req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptpf->pdev, cptpf->pf_id, 0); req->dir = CPT_INLINE_INBOUND; req->slot = slot; req->sso_pf_func_ovrd = cptpf->sso_pf_func_ovrd; @@ -184,7 +184,8 @@ static int rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, u8 egrp, nix_req->gen_cfg.opcode = cpt_inline_rx_opcode(pdev); nix_req->gen_cfg.param1 = req->param1; nix_req->gen_cfg.param2 = req->param2; - nix_req->inst_qsel.cpt_pf_func = OTX2_CPT_RVU_PFFUNC(cptpf->pf_id, 0); + nix_req->inst_qsel.cpt_pf_func = + OTX2_CPT_RVU_PFFUNC(cptpf->pdev, cptpf->pf_id, 0); nix_req->inst_qsel.cpt_slot = 0; ret = otx2_cpt_send_mbox_msg(&cptpf->afpf_mbox, pdev); if (ret) @@ -264,8 +265,6 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, return -ENOENT; } - otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base, - &cptpf->afpf_mbox, BLKADDR_CPT0); cptpf->lfs.global_slot = 0; cptpf->lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; cptpf->lfs.ctx_ilen = cfg_req->ctx_ilen; @@ -278,9 +277,6 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, if (cptpf->has_cpt1) { cptpf->rsrc_req_blkaddr = BLKADDR_CPT1; - otx2_cptlf_set_dev_info(&cptpf->cpt1_lfs, cptpf->pdev, - cptpf->reg_base, &cptpf->afpf_mbox, - BLKADDR_CPT1); cptpf->cpt1_lfs.global_slot = num_lfs; cptpf->cpt1_lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; cptpf->cpt1_lfs.ctx_ilen = cfg_req->ctx_ilen; @@ -397,9 +393,8 @@ void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work) msg = (struct mbox_msghdr *)(mdev->mbase + offset); /* Set which VF sent this message based on mbox IRQ */ - msg->pcifunc = ((u16)cptpf->pf_id << RVU_PFVF_PF_SHIFT) | - ((vf->vf_id + 1) & RVU_PFVF_FUNC_MASK); - + msg->pcifunc = rvu_make_pcifunc(cptpf->pdev, cptpf->pf_id, + (vf->vf_id + 1)); err = cptpf_handle_vf_req(cptpf, vf, msg, msg->next_msgoff - offset); /* @@ -474,8 +469,7 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, switch (msg->id) { case MBOX_MSG_READY: - cptpf->pf_id = (msg->pcifunc >> RVU_PFVF_PF_SHIFT) & - RVU_PFVF_PF_MASK; + cptpf->pf_id = rvu_get_pf(cptpf->pdev, msg->pcifunc); break; case MBOX_MSG_MSIX_OFFSET: rsp_msix = (struct msix_offset_rsp *) msg; @@ -507,6 +501,7 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, case MBOX_MSG_CPT_INLINE_IPSEC_CFG: case MBOX_MSG_NIX_INLINE_IPSEC_CFG: case MBOX_MSG_CPT_LF_RESET: + case MBOX_MSG_LMTST_TBL_SETUP: break; default: diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 5c9484646172..b5cc5401f704 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -3,6 +3,8 @@ #include <linux/ctype.h> #include <linux/firmware.h> +#include <linux/string.h> +#include <linux/string_choices.h> #include "otx2_cptpf_ucode.h" #include "otx2_cpt_common.h" #include "otx2_cptpf.h" @@ -175,7 +177,9 @@ static int cptx_set_ucode_base(struct otx2_cpt_eng_grp_info *eng_grp, /* Set PF number for microcode fetches */ ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, CPT_AF_PF_FUNC, - cptpf->pf_id << RVU_PFVF_PF_SHIFT, blkaddr); + rvu_make_pcifunc(cptpf->pdev, + cptpf->pf_id, 0), + blkaddr); if (ret) return ret; @@ -455,13 +459,13 @@ static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info, u16 rid) { char filename[OTX2_CPT_NAME_LENGTH]; - char eng_type[8] = {0}; + char eng_type[8]; int ret, e, i; INIT_LIST_HEAD(&fw_info->ucodes); for (e = 1; e < OTX2_CPT_MAX_ENG_TYPES; e++) { - strcpy(eng_type, get_eng_type_str(e)); + strscpy(eng_type, get_eng_type_str(e)); for (i = 0; i < strlen(eng_type); i++) eng_type[i] = tolower(eng_type[i]); @@ -1490,11 +1494,13 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) union otx2_cpt_opcode opcode; union otx2_cpt_res_s *result; union otx2_cpt_inst_s inst; + dma_addr_t result_baddr; dma_addr_t rptr_baddr; struct pci_dev *pdev; - u32 len, compl_rlen; + int timeout = 10000; + void *base, *rptr; int ret, etype; - void *rptr; + u32 len; /* * We don't get capabilities if it was already done @@ -1512,29 +1518,33 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) if (ret) goto delete_grps; - otx2_cptlf_set_dev_info(lfs, cptpf->pdev, cptpf->reg_base, - &cptpf->afpf_mbox, BLKADDR_CPT0); ret = otx2_cptlf_init(lfs, OTX2_CPT_ALL_ENG_GRPS_MASK, OTX2_CPT_QUEUE_HI_PRIO, 1); if (ret) goto delete_grps; - compl_rlen = ALIGN(sizeof(union otx2_cpt_res_s), OTX2_CPT_DMA_MINALIGN); - len = compl_rlen + LOADFVC_RLEN; + /* Allocate extra memory for "rptr" and "result" pointer alignment */ + len = LOADFVC_RLEN + ARCH_DMA_MINALIGN + + sizeof(union otx2_cpt_res_s) + OTX2_CPT_RES_ADDR_ALIGN; - result = kzalloc(len, GFP_KERNEL); - if (!result) { + base = kzalloc(len, GFP_KERNEL); + if (!base) { ret = -ENOMEM; goto lf_cleanup; } - rptr_baddr = dma_map_single(&pdev->dev, (void *)result, len, - DMA_BIDIRECTIONAL); + + rptr = PTR_ALIGN(base, ARCH_DMA_MINALIGN); + rptr_baddr = dma_map_single(&pdev->dev, rptr, len, DMA_BIDIRECTIONAL); if (dma_mapping_error(&pdev->dev, rptr_baddr)) { dev_err(&pdev->dev, "DMA mapping failed\n"); ret = -EFAULT; - goto free_result; + goto free_rptr; } - rptr = (u8 *)result + compl_rlen; + + result = (union otx2_cpt_res_s *)PTR_ALIGN(rptr + LOADFVC_RLEN, + OTX2_CPT_RES_ADDR_ALIGN); + result_baddr = ALIGN(rptr_baddr + LOADFVC_RLEN, + OTX2_CPT_RES_ADDR_ALIGN); /* Fill in the command */ opcode.s.major = LOADFVC_MAJOR_OP; @@ -1546,27 +1556,38 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) /* 64-bit swap for microcode data reads, not needed for addresses */ cpu_to_be64s(&iq_cmd.cmd.u); iq_cmd.dptr = 0; - iq_cmd.rptr = rptr_baddr + compl_rlen; + iq_cmd.rptr = rptr_baddr; iq_cmd.cptr.u = 0; for (etype = 1; etype < OTX2_CPT_MAX_ENG_TYPES; etype++) { result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(&cptpf->eng_grps, etype); - otx2_cpt_fill_inst(&inst, &iq_cmd, rptr_baddr); + otx2_cpt_fill_inst(&inst, &iq_cmd, result_baddr); lfs->ops->send_cmd(&inst, 1, &cptpf->lfs.lf[0]); + timeout = 10000; while (lfs->ops->cpt_get_compcode(result) == - OTX2_CPT_COMPLETION_CODE_INIT) + OTX2_CPT_COMPLETION_CODE_INIT) { cpu_relax(); + udelay(1); + timeout--; + if (!timeout) { + ret = -ENODEV; + cptpf->is_eng_caps_discovered = false; + dev_warn(&pdev->dev, "Timeout on CPT load_fvc completion poll\n"); + goto error_no_response; + } + } cptpf->eng_caps[etype].u = be64_to_cpup(rptr); } - dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL); cptpf->is_eng_caps_discovered = true; -free_result: - kfree(result); +error_no_response: + dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL); +free_rptr: + kfree(base); lf_cleanup: otx2_cptlf_shutdown(lfs); delete_grps: @@ -1595,7 +1616,7 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, return -EINVAL; } err_msg = "Invalid engine group format"; - strscpy(tmp_buf, ctx->val.vstr, strlen(ctx->val.vstr) + 1); + strscpy(tmp_buf, ctx->val.vstr); start = tmp_buf; has_se = has_ie = has_ae = false; @@ -1774,102 +1795,3 @@ err_print: dev_err(dev, "%s\n", err_msg); return -EINVAL; } - -static void get_engs_info(struct otx2_cpt_eng_grp_info *eng_grp, char *buf, - int size, int idx) -{ - struct otx2_cpt_engs_rsvd *mirrored_engs = NULL; - struct otx2_cpt_engs_rsvd *engs; - int len, i; - - buf[0] = '\0'; - for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { - engs = &eng_grp->engs[i]; - if (!engs->type) - continue; - if (idx != -1 && idx != i) - continue; - - if (eng_grp->mirror.is_ena) - mirrored_engs = find_engines_by_type( - &eng_grp->g->grp[eng_grp->mirror.idx], - engs->type); - if (i > 0 && idx == -1) { - len = strlen(buf); - scnprintf(buf + len, size - len, ", "); - } - - len = strlen(buf); - scnprintf(buf + len, size - len, "%d %s ", - mirrored_engs ? engs->count + mirrored_engs->count : - engs->count, - get_eng_type_str(engs->type)); - if (mirrored_engs) { - len = strlen(buf); - scnprintf(buf + len, size - len, - "(%d shared with engine_group%d) ", - engs->count <= 0 ? - engs->count + mirrored_engs->count : - mirrored_engs->count, - eng_grp->mirror.idx); - } - } -} - -void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf) -{ - struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; - struct otx2_cpt_eng_grp_info *mirrored_grp; - char engs_info[2 * OTX2_CPT_NAME_LENGTH]; - struct otx2_cpt_eng_grp_info *grp; - struct otx2_cpt_engs_rsvd *engs; - int i, j; - - pr_debug("Engine groups global info"); - pr_debug("max SE %d, max IE %d, max AE %d", eng_grps->avail.max_se_cnt, - eng_grps->avail.max_ie_cnt, eng_grps->avail.max_ae_cnt); - pr_debug("free SE %d", eng_grps->avail.se_cnt); - pr_debug("free IE %d", eng_grps->avail.ie_cnt); - pr_debug("free AE %d", eng_grps->avail.ae_cnt); - - for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { - grp = &eng_grps->grp[i]; - pr_debug("engine_group%d, state %s", i, - grp->is_enabled ? "enabled" : "disabled"); - if (grp->is_enabled) { - mirrored_grp = &eng_grps->grp[grp->mirror.idx]; - pr_debug("Ucode0 filename %s, version %s", - grp->mirror.is_ena ? - mirrored_grp->ucode[0].filename : - grp->ucode[0].filename, - grp->mirror.is_ena ? - mirrored_grp->ucode[0].ver_str : - grp->ucode[0].ver_str); - if (is_2nd_ucode_used(grp)) - pr_debug("Ucode1 filename %s, version %s", - grp->ucode[1].filename, - grp->ucode[1].ver_str); - } - - for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { - engs = &grp->engs[j]; - if (engs->type) { - u32 mask[5] = { }; - - get_engs_info(grp, engs_info, - 2 * OTX2_CPT_NAME_LENGTH, j); - pr_debug("Slot%d: %s", j, engs_info); - bitmap_to_arr32(mask, engs->bmap, - eng_grps->engs_num); - if (is_dev_otx2(cptpf->pdev)) - pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x", - mask[3], mask[2], mask[1], - mask[0]); - else - pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x %8.8x", - mask[4], mask[3], mask[2], mask[1], - mask[0]); - } - } - } -} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h index 365fe8943bd9..7e6a6a4ec37c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -166,7 +166,6 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, struct devlink_param_gset_ctx *ctx); int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf, struct devlink_param_gset_ctx *ctx); -void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf); struct otx2_cpt_engs_rsvd *find_engines_by_type( struct otx2_cpt_eng_grp_info *eng_grp, int eng_type); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c index 7eb0bc13994d..8d9f394d6b50 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c @@ -384,7 +384,8 @@ static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) req_info->req_type = OTX2_CPT_ENC_DEC_REQ; req_info->is_enc = enc; req_info->is_trunc_hmac = false; - req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); + req_info->ctrl.s.grp = otx2_cpt_get_eng_grp_num(pdev, + OTX2_CPT_SE_TYPES); req_info->req.cptr = ctx->er_ctx.hw_ctx; req_info->req.cptr_dma = ctx->er_ctx.cptr_dma; @@ -1288,7 +1289,8 @@ static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) if (status) return status; - req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); + req_info->ctrl.s.grp = otx2_cpt_get_eng_grp_num(pdev, + OTX2_CPT_SE_TYPES); /* * We perform an asynchronous send and once diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c index 527d34cc258b..c1c44a7b89fa 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -265,17 +265,33 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) u8 eng_grp_msk; /* Get engine group number for symmetric crypto */ - cptvf->lfs.kcrypto_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + cptvf->lfs.kcrypto_se_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_SE_TYPES); if (ret) return ret; - if (cptvf->lfs.kcrypto_eng_grp_num == OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { - dev_err(dev, "Engine group for kernel crypto not available\n"); - ret = -ENOENT; + if (cptvf->lfs.kcrypto_se_eng_grp_num == + OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { + dev_err(dev, + "Symmetric Engine group for crypto not available\n"); + return -ENOENT; + } + + /* Get engine group number for asymmetric crypto */ + cptvf->lfs.kcrypto_ae_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_AE_TYPES); + if (ret) return ret; + + if (cptvf->lfs.kcrypto_ae_eng_grp_num == + OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { + dev_err(dev, + "Asymmetric Engine group for crypto not available\n"); + return -ENOENT; } - eng_grp_msk = 1 << cptvf->lfs.kcrypto_eng_grp_num; + + eng_grp_msk = BIT(cptvf->lfs.kcrypto_se_eng_grp_num) | + BIT(cptvf->lfs.kcrypto_ae_eng_grp_num); ret = otx2_cptvf_send_kvf_limits_msg(cptvf); if (ret) @@ -283,8 +299,6 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) lfs_num = cptvf->lfs.kvf_limits; - otx2_cptlf_set_dev_info(lfs, cptvf->pdev, cptvf->reg_base, - &cptvf->pfvf_mbox, cptvf->blkaddr); ret = otx2_cptlf_init(lfs, eng_grp_msk, OTX2_CPT_QUEUE_HI_PRIO, lfs_num); if (ret) @@ -358,9 +372,8 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, dev_err(dev, "Unable to get usable DMA configuration\n"); goto clear_drvdata; } - /* Map VF's configuration registers */ - ret = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, - OTX2_CPTVF_DRV_NAME); + + ret = pcim_request_all_regions(pdev, OTX2_CPTVF_DRV_NAME); if (ret) { dev_err(dev, "Couldn't get PCI resources 0x%x\n", ret); goto clear_drvdata; @@ -369,14 +382,16 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, cptvf); cptvf->pdev = pdev; - cptvf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + /* Map VF's configuration registers */ + cptvf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0); + if (!cptvf->reg_base) { + ret = -ENOMEM; + dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", ret); + goto clear_drvdata; + } otx2_cpt_set_hw_caps(pdev, &cptvf->cap_flag); - ret = cn10k_cptvf_lmtst_init(cptvf); - if (ret) - goto clear_drvdata; - /* Initialize PF<=>VF mailbox */ ret = cptvf_pfvf_mbox_init(cptvf); if (ret) @@ -391,6 +406,9 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, cptvf_hw_ops_get(cptvf); + otx2_cptlf_set_dev_info(&cptvf->lfs, cptvf->pdev, cptvf->reg_base, + &cptvf->pfvf_mbox, cptvf->blkaddr); + ret = otx2_cptvf_send_caps_msg(cptvf); if (ret) { dev_err(&pdev->dev, "Couldn't get CPT engine capabilities.\n"); @@ -399,13 +417,19 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, if (cptvf->eng_caps[OTX2_CPT_SE_TYPES] & BIT_ULL(35)) cptvf->lfs.ops->cpt_sg_info_create = cn10k_sgv2_info_create; + ret = cn10k_cptvf_lmtst_init(cptvf); + if (ret) + goto unregister_interrupts; + /* Initialize CPT LFs */ ret = cptvf_lf_init(cptvf); if (ret) - goto unregister_interrupts; + goto free_lmtst; return 0; +free_lmtst: + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); unregister_interrupts: cptvf_disable_pfvf_mbox_intrs(cptvf); destroy_pfvf_mbox: @@ -429,6 +453,8 @@ static void otx2_cptvf_remove(struct pci_dev *pdev) cptvf_disable_pfvf_mbox_intrs(cptvf); /* Destroy PF-VF mbox */ cptvf_pfvf_mbox_destroy(cptvf); + /* Free LMTST memory */ + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); pci_set_drvdata(pdev, NULL); } @@ -448,7 +474,7 @@ static struct pci_driver otx2_cptvf_pci_driver = { module_pci_driver(otx2_cptvf_pci_driver); -MODULE_IMPORT_NS(CRYPTO_DEV_OCTEONTX2_CPT); +MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT"); MODULE_AUTHOR("Marvell"); MODULE_DESCRIPTION("Marvell RVU CPT Virtual Function Driver"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c index d9fa5f6e204d..5277bcfa275e 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -75,6 +75,7 @@ static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, struct otx2_cpt_caps_rsp *eng_caps; struct cpt_rd_wr_reg_msg *rsp_reg; struct msix_offset_rsp *rsp_msix; + u8 grp_num; int i; if (msg->id >= MBOX_MSG_MAX) { @@ -122,7 +123,11 @@ static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, break; case MBOX_MSG_GET_ENG_GRP_NUM: rsp_grp = (struct otx2_cpt_egrp_num_rsp *) msg; - cptvf->lfs.kcrypto_eng_grp_num = rsp_grp->eng_grp_num; + grp_num = rsp_grp->eng_grp_num; + if (rsp_grp->eng_type == OTX2_CPT_SE_TYPES) + cptvf->lfs.kcrypto_se_eng_grp_num = grp_num; + else if (rsp_grp->eng_type == OTX2_CPT_AE_TYPES) + cptvf->lfs.kcrypto_ae_eng_grp_num = grp_num; break; case MBOX_MSG_GET_KVF_LIMITS: rsp_limits = (struct otx2_cpt_kvf_limits_rsp *) msg; @@ -134,6 +139,7 @@ static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, sizeof(cptvf->eng_caps)); break; case MBOX_MSG_CPT_LF_RESET: + case MBOX_MSG_LMTST_TBL_SETUP: break; default: dev_err(&cptvf->pdev->dev, "Unsupported msg %d received.\n", @@ -188,7 +194,7 @@ int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type) } req->hdr.id = MBOX_MSG_GET_ENG_GRP_NUM; req->hdr.sig = OTX2_MBOX_REQ_SIG; - req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->pdev, cptvf->vf_id, 0); req->eng_type = eng_type; return otx2_cpt_send_mbox_msg(mbox, pdev); @@ -209,7 +215,7 @@ int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) } req->id = MBOX_MSG_GET_KVF_LIMITS; req->sig = OTX2_MBOX_REQ_SIG; - req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->pdev, cptvf->vf_id, 0); return otx2_cpt_send_mbox_msg(mbox, pdev); } @@ -229,7 +235,7 @@ int otx2_cptvf_send_caps_msg(struct otx2_cptvf_dev *cptvf) } req->id = MBOX_MSG_GET_CAPS; req->sig = OTX2_MBOX_REQ_SIG; - req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->pdev, cptvf->vf_id, 0); return otx2_cpt_send_mbox_msg(mbox, pdev); } diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c index 5387c68f3c9d..e71494486c64 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c @@ -264,9 +264,10 @@ static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, break; } - dev_err(&pdev->dev, - "Request failed with software error code 0x%x\n", - cpt_status->s.uc_compcode); + pr_debug("Request failed with software error code 0x%x: algo = %s driver = %s\n", + cpt_status->s.uc_compcode, + info->req->areq->tfm->__crt_alg->cra_name, + info->req->areq->tfm->__crt_alg->cra_driver_name); otx2_cpt_dump_sg_list(pdev, info->req); break; } @@ -390,9 +391,19 @@ void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) &wqe->lfs->lf[wqe->lf_num].pqueue); } -int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev) +int otx2_cpt_get_eng_grp_num(struct pci_dev *pdev, + enum otx2_cpt_eng_type eng_type) { struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); - return cptvf->lfs.kcrypto_eng_grp_num; + switch (eng_type) { + case OTX2_CPT_SE_TYPES: + return cptvf->lfs.kcrypto_se_eng_grp_num; + case OTX2_CPT_AE_TYPES: + return cptvf->lfs.kcrypto_ae_eng_grp_num; + default: + dev_err(&cptvf->pdev->dev, "Unsupported engine type"); + break; + } + return -ENXIO; } diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index d94a26c3541a..133ebc998236 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -265,12 +265,12 @@ static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, MXS_DCP_CONTROL0_INTERRUPT | MXS_DCP_CONTROL0_ENABLE_CIPHER; - if (key_referenced) - /* Set OTP key bit to select the key via KEY_SELECT. */ - desc->control0 |= MXS_DCP_CONTROL0_OTP_KEY; - else + if (!key_referenced) /* Payload contains the key. */ desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY; + else if (actx->key[0] == DCP_PAES_KEY_OTP) + /* Set OTP key bit to select the key via KEY_SELECT. */ + desc->control0 |= MXS_DCP_CONTROL0_OTP_KEY; if (rctx->enc) desc->control0 |= MXS_DCP_CONTROL0_CIPHER_ENCRYPT; diff --git a/drivers/crypto/n2_asm.S b/drivers/crypto/n2_asm.S deleted file mode 100644 index 9a67dbf340f4..000000000000 --- a/drivers/crypto/n2_asm.S +++ /dev/null @@ -1,96 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* n2_asm.S: Hypervisor calls for NCS support. - * - * Copyright (C) 2009 David S. Miller <davem@davemloft.net> - */ - -#include <linux/linkage.h> -#include <asm/hypervisor.h> -#include "n2_core.h" - - /* o0: queue type - * o1: RA of queue - * o2: num entries in queue - * o3: address of queue handle return - */ -ENTRY(sun4v_ncs_qconf) - mov HV_FAST_NCS_QCONF, %o5 - ta HV_FAST_TRAP - stx %o1, [%o3] - retl - nop -ENDPROC(sun4v_ncs_qconf) - - /* %o0: queue handle - * %o1: address of queue type return - * %o2: address of queue base address return - * %o3: address of queue num entries return - */ -ENTRY(sun4v_ncs_qinfo) - mov %o1, %g1 - mov %o2, %g2 - mov %o3, %g3 - mov HV_FAST_NCS_QINFO, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - stx %o3, [%g3] - retl - nop -ENDPROC(sun4v_ncs_qinfo) - - /* %o0: queue handle - * %o1: address of head offset return - */ -ENTRY(sun4v_ncs_gethead) - mov %o1, %o2 - mov HV_FAST_NCS_GETHEAD, %o5 - ta HV_FAST_TRAP - stx %o1, [%o2] - retl - nop -ENDPROC(sun4v_ncs_gethead) - - /* %o0: queue handle - * %o1: address of tail offset return - */ -ENTRY(sun4v_ncs_gettail) - mov %o1, %o2 - mov HV_FAST_NCS_GETTAIL, %o5 - ta HV_FAST_TRAP - stx %o1, [%o2] - retl - nop -ENDPROC(sun4v_ncs_gettail) - - /* %o0: queue handle - * %o1: new tail offset - */ -ENTRY(sun4v_ncs_settail) - mov HV_FAST_NCS_SETTAIL, %o5 - ta HV_FAST_TRAP - retl - nop -ENDPROC(sun4v_ncs_settail) - - /* %o0: queue handle - * %o1: address of devino return - */ -ENTRY(sun4v_ncs_qhandle_to_devino) - mov %o1, %o2 - mov HV_FAST_NCS_QHANDLE_TO_DEVINO, %o5 - ta HV_FAST_TRAP - stx %o1, [%o2] - retl - nop -ENDPROC(sun4v_ncs_qhandle_to_devino) - - /* %o0: queue handle - * %o1: new head offset - */ -ENTRY(sun4v_ncs_sethead_marker) - mov HV_FAST_NCS_SETHEAD_MARKER, %o5 - ta HV_FAST_TRAP - retl - nop -ENDPROC(sun4v_ncs_sethead_marker) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c deleted file mode 100644 index 14c302d2db79..000000000000 --- a/drivers/crypto/n2_core.c +++ /dev/null @@ -1,2168 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support. - * - * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/platform_device.h> -#include <linux/cpumask.h> -#include <linux/slab.h> -#include <linux/interrupt.h> -#include <linux/crypto.h> -#include <crypto/md5.h> -#include <crypto/sha1.h> -#include <crypto/sha2.h> -#include <crypto/aes.h> -#include <crypto/internal/des.h> -#include <linux/mutex.h> -#include <linux/delay.h> -#include <linux/sched.h> - -#include <crypto/internal/hash.h> -#include <crypto/internal/skcipher.h> -#include <crypto/scatterwalk.h> -#include <crypto/algapi.h> - -#include <asm/hypervisor.h> -#include <asm/mdesc.h> - -#include "n2_core.h" - -#define DRV_MODULE_NAME "n2_crypto" -#define DRV_MODULE_VERSION "0.2" -#define DRV_MODULE_RELDATE "July 28, 2011" - -static const char version[] = - DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; - -MODULE_AUTHOR("David S. Miller <davem@davemloft.net>"); -MODULE_DESCRIPTION("Niagara2 Crypto driver"); -MODULE_LICENSE("GPL"); -MODULE_VERSION(DRV_MODULE_VERSION); - -#define N2_CRA_PRIORITY 200 - -static DEFINE_MUTEX(spu_lock); - -struct spu_queue { - cpumask_t sharing; - unsigned long qhandle; - - spinlock_t lock; - u8 q_type; - void *q; - unsigned long head; - unsigned long tail; - struct list_head jobs; - - unsigned long devino; - - char irq_name[32]; - unsigned int irq; - - struct list_head list; -}; - -struct spu_qreg { - struct spu_queue *queue; - unsigned long type; -}; - -static struct spu_queue **cpu_to_cwq; -static struct spu_queue **cpu_to_mau; - -static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off) -{ - if (q->q_type == HV_NCS_QTYPE_MAU) { - off += MAU_ENTRY_SIZE; - if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES)) - off = 0; - } else { - off += CWQ_ENTRY_SIZE; - if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES)) - off = 0; - } - return off; -} - -struct n2_request_common { - struct list_head entry; - unsigned int offset; -}; -#define OFFSET_NOT_RUNNING (~(unsigned int)0) - -/* An async job request records the final tail value it used in - * n2_request_common->offset, test to see if that offset is in - * the range old_head, new_head, inclusive. - */ -static inline bool job_finished(struct spu_queue *q, unsigned int offset, - unsigned long old_head, unsigned long new_head) -{ - if (old_head <= new_head) { - if (offset > old_head && offset <= new_head) - return true; - } else { - if (offset > old_head || offset <= new_head) - return true; - } - return false; -} - -/* When the HEAD marker is unequal to the actual HEAD, we get - * a virtual device INO interrupt. We should process the - * completed CWQ entries and adjust the HEAD marker to clear - * the IRQ. - */ -static irqreturn_t cwq_intr(int irq, void *dev_id) -{ - unsigned long off, new_head, hv_ret; - struct spu_queue *q = dev_id; - - pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n", - smp_processor_id(), q->qhandle); - - spin_lock(&q->lock); - - hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head); - - pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n", - smp_processor_id(), new_head, hv_ret); - - for (off = q->head; off != new_head; off = spu_next_offset(q, off)) { - /* XXX ... XXX */ - } - - hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head); - if (hv_ret == HV_EOK) - q->head = new_head; - - spin_unlock(&q->lock); - - return IRQ_HANDLED; -} - -static irqreturn_t mau_intr(int irq, void *dev_id) -{ - struct spu_queue *q = dev_id; - unsigned long head, hv_ret; - - spin_lock(&q->lock); - - pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n", - smp_processor_id(), q->qhandle); - - hv_ret = sun4v_ncs_gethead(q->qhandle, &head); - - pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n", - smp_processor_id(), head, hv_ret); - - sun4v_ncs_sethead_marker(q->qhandle, head); - - spin_unlock(&q->lock); - - return IRQ_HANDLED; -} - -static void *spu_queue_next(struct spu_queue *q, void *cur) -{ - return q->q + spu_next_offset(q, cur - q->q); -} - -static int spu_queue_num_free(struct spu_queue *q) -{ - unsigned long head = q->head; - unsigned long tail = q->tail; - unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES); - unsigned long diff; - - if (head > tail) - diff = head - tail; - else - diff = (end - tail) + head; - - return (diff / CWQ_ENTRY_SIZE) - 1; -} - -static void *spu_queue_alloc(struct spu_queue *q, int num_entries) -{ - int avail = spu_queue_num_free(q); - - if (avail >= num_entries) - return q->q + q->tail; - - return NULL; -} - -static unsigned long spu_queue_submit(struct spu_queue *q, void *last) -{ - unsigned long hv_ret, new_tail; - - new_tail = spu_next_offset(q, last - q->q); - - hv_ret = sun4v_ncs_settail(q->qhandle, new_tail); - if (hv_ret == HV_EOK) - q->tail = new_tail; - return hv_ret; -} - -static u64 control_word_base(unsigned int len, unsigned int hmac_key_len, - int enc_type, int auth_type, - unsigned int hash_len, - bool sfas, bool sob, bool eob, bool encrypt, - int opcode) -{ - u64 word = (len - 1) & CONTROL_LEN; - - word |= ((u64) opcode << CONTROL_OPCODE_SHIFT); - word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT); - word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT); - if (sfas) - word |= CONTROL_STORE_FINAL_AUTH_STATE; - if (sob) - word |= CONTROL_START_OF_BLOCK; - if (eob) - word |= CONTROL_END_OF_BLOCK; - if (encrypt) - word |= CONTROL_ENCRYPT; - if (hmac_key_len) - word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT; - if (hash_len) - word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT; - - return word; -} - -#if 0 -static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) -{ - if (this_len >= 64 || - qp->head != qp->tail) - return true; - return false; -} -#endif - -struct n2_ahash_alg { - struct list_head entry; - const u8 *hash_zero; - const u8 *hash_init; - u8 hw_op_hashsz; - u8 digest_size; - u8 auth_type; - u8 hmac_type; - struct ahash_alg alg; -}; - -static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) -{ - struct crypto_alg *alg = tfm->__crt_alg; - struct ahash_alg *ahash_alg; - - ahash_alg = container_of(alg, struct ahash_alg, halg.base); - - return container_of(ahash_alg, struct n2_ahash_alg, alg); -} - -struct n2_hmac_alg { - const char *child_alg; - struct n2_ahash_alg derived; -}; - -static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm) -{ - struct crypto_alg *alg = tfm->__crt_alg; - struct ahash_alg *ahash_alg; - - ahash_alg = container_of(alg, struct ahash_alg, halg.base); - - return container_of(ahash_alg, struct n2_hmac_alg, derived.alg); -} - -struct n2_hash_ctx { - struct crypto_ahash *fallback_tfm; -}; - -#define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */ - -struct n2_hmac_ctx { - struct n2_hash_ctx base; - - struct crypto_shash *child_shash; - - int hash_key_len; - unsigned char hash_key[N2_HASH_KEY_MAX]; -}; - -struct n2_hash_req_ctx { - union { - struct md5_state md5; - struct sha1_state sha1; - struct sha256_state sha256; - } u; - - struct ahash_request fallback_req; -}; - -static int n2_hash_async_init(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - - return crypto_ahash_init(&rctx->fallback_req); -} - -static int n2_hash_async_update(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - - return crypto_ahash_update(&rctx->fallback_req); -} - -static int n2_hash_async_final(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; - - return crypto_ahash_final(&rctx->fallback_req); -} - -static int n2_hash_async_finup(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; - - return crypto_ahash_finup(&rctx->fallback_req); -} - -static int n2_hash_async_noimport(struct ahash_request *req, const void *in) -{ - return -ENOSYS; -} - -static int n2_hash_async_noexport(struct ahash_request *req, void *out) -{ - return -ENOSYS; -} - -static int n2_hash_cra_init(struct crypto_tfm *tfm) -{ - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct crypto_ahash *fallback_tfm; - int err; - - fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback_tfm)) { - pr_warn("Fallback driver '%s' could not be loaded!\n", - fallback_driver_name); - err = PTR_ERR(fallback_tfm); - goto out; - } - - crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + - crypto_ahash_reqsize(fallback_tfm))); - - ctx->fallback_tfm = fallback_tfm; - return 0; - -out: - return err; -} - -static void n2_hash_cra_exit(struct crypto_tfm *tfm) -{ - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); - - crypto_free_ahash(ctx->fallback_tfm); -} - -static int n2_hmac_cra_init(struct crypto_tfm *tfm) -{ - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); - struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); - struct crypto_ahash *fallback_tfm; - struct crypto_shash *child_shash; - int err; - - fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback_tfm)) { - pr_warn("Fallback driver '%s' could not be loaded!\n", - fallback_driver_name); - err = PTR_ERR(fallback_tfm); - goto out; - } - - child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0); - if (IS_ERR(child_shash)) { - pr_warn("Child shash '%s' could not be loaded!\n", - n2alg->child_alg); - err = PTR_ERR(child_shash); - goto out_free_fallback; - } - - crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + - crypto_ahash_reqsize(fallback_tfm))); - - ctx->child_shash = child_shash; - ctx->base.fallback_tfm = fallback_tfm; - return 0; - -out_free_fallback: - crypto_free_ahash(fallback_tfm); - -out: - return err; -} - -static void n2_hmac_cra_exit(struct crypto_tfm *tfm) -{ - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); - - crypto_free_ahash(ctx->base.fallback_tfm); - crypto_free_shash(ctx->child_shash); -} - -static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) -{ - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); - struct crypto_shash *child_shash = ctx->child_shash; - struct crypto_ahash *fallback_tfm; - int err, bs, ds; - - fallback_tfm = ctx->base.fallback_tfm; - err = crypto_ahash_setkey(fallback_tfm, key, keylen); - if (err) - return err; - - bs = crypto_shash_blocksize(child_shash); - ds = crypto_shash_digestsize(child_shash); - BUG_ON(ds > N2_HASH_KEY_MAX); - if (keylen > bs) { - err = crypto_shash_tfm_digest(child_shash, key, keylen, - ctx->hash_key); - if (err) - return err; - keylen = ds; - } else if (keylen <= N2_HASH_KEY_MAX) - memcpy(ctx->hash_key, key, keylen); - - ctx->hash_key_len = keylen; - - return err; -} - -static unsigned long wait_for_tail(struct spu_queue *qp) -{ - unsigned long head, hv_ret; - - do { - hv_ret = sun4v_ncs_gethead(qp->qhandle, &head); - if (hv_ret != HV_EOK) { - pr_err("Hypervisor error on gethead\n"); - break; - } - if (head == qp->tail) { - qp->head = head; - break; - } - } while (1); - return hv_ret; -} - -static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, - struct cwq_initial_entry *ent) -{ - unsigned long hv_ret = spu_queue_submit(qp, ent); - - if (hv_ret == HV_EOK) - hv_ret = wait_for_tail(qp); - - return hv_ret; -} - -static int n2_do_async_digest(struct ahash_request *req, - unsigned int auth_type, unsigned int digest_size, - unsigned int result_size, void *hash_loc, - unsigned long auth_key, unsigned int auth_key_len) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct cwq_initial_entry *ent; - struct crypto_hash_walk walk; - struct spu_queue *qp; - unsigned long flags; - int err = -ENODEV; - int nbytes, cpu; - - /* The total effective length of the operation may not - * exceed 2^16. - */ - if (unlikely(req->nbytes > (1 << 16))) { - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; - - return crypto_ahash_digest(&rctx->fallback_req); - } - - nbytes = crypto_hash_walk_first(req, &walk); - - cpu = get_cpu(); - qp = cpu_to_cwq[cpu]; - if (!qp) - goto out; - - spin_lock_irqsave(&qp->lock, flags); - - /* XXX can do better, improve this later by doing a by-hand scatterlist - * XXX walk, etc. - */ - ent = qp->q + qp->tail; - - ent->control = control_word_base(nbytes, auth_key_len, 0, - auth_type, digest_size, - false, true, false, false, - OPCODE_INPLACE_BIT | - OPCODE_AUTH_MAC); - ent->src_addr = __pa(walk.data); - ent->auth_key_addr = auth_key; - ent->auth_iv_addr = __pa(hash_loc); - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = 0UL; - ent->enc_iv_addr = 0UL; - ent->dest_addr = __pa(hash_loc); - - nbytes = crypto_hash_walk_done(&walk, 0); - while (nbytes > 0) { - ent = spu_queue_next(qp, ent); - - ent->control = (nbytes - 1); - ent->src_addr = __pa(walk.data); - ent->auth_key_addr = 0UL; - ent->auth_iv_addr = 0UL; - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = 0UL; - ent->enc_iv_addr = 0UL; - ent->dest_addr = 0UL; - - nbytes = crypto_hash_walk_done(&walk, 0); - } - ent->control |= CONTROL_END_OF_BLOCK; - - if (submit_and_wait_for_tail(qp, ent) != HV_EOK) - err = -EINVAL; - else - err = 0; - - spin_unlock_irqrestore(&qp->lock, flags); - - if (!err) - memcpy(req->result, hash_loc, result_size); -out: - put_cpu(); - - return err; -} - -static int n2_hash_async_digest(struct ahash_request *req) -{ - struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm); - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - int ds; - - ds = n2alg->digest_size; - if (unlikely(req->nbytes == 0)) { - memcpy(req->result, n2alg->hash_zero, ds); - return 0; - } - memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz); - - return n2_do_async_digest(req, n2alg->auth_type, - n2alg->hw_op_hashsz, ds, - &rctx->u, 0UL, 0); -} - -static int n2_hmac_async_digest(struct ahash_request *req) -{ - struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm); - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); - int ds; - - ds = n2alg->derived.digest_size; - if (unlikely(req->nbytes == 0) || - unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) { - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; - - return crypto_ahash_digest(&rctx->fallback_req); - } - memcpy(&rctx->u, n2alg->derived.hash_init, - n2alg->derived.hw_op_hashsz); - - return n2_do_async_digest(req, n2alg->derived.hmac_type, - n2alg->derived.hw_op_hashsz, ds, - &rctx->u, - __pa(&ctx->hash_key), - ctx->hash_key_len); -} - -struct n2_skcipher_context { - int key_len; - int enc_type; - union { - u8 aes[AES_MAX_KEY_SIZE]; - u8 des[DES_KEY_SIZE]; - u8 des3[3 * DES_KEY_SIZE]; - } key; -}; - -#define N2_CHUNK_ARR_LEN 16 - -struct n2_crypto_chunk { - struct list_head entry; - unsigned long iv_paddr : 44; - unsigned long arr_len : 20; - unsigned long dest_paddr; - unsigned long dest_final; - struct { - unsigned long src_paddr : 44; - unsigned long src_len : 20; - } arr[N2_CHUNK_ARR_LEN]; -}; - -struct n2_request_context { - struct skcipher_walk walk; - struct list_head chunk_list; - struct n2_crypto_chunk chunk; - u8 temp_iv[16]; -}; - -/* The SPU allows some level of flexibility for partial cipher blocks - * being specified in a descriptor. - * - * It merely requires that every descriptor's length field is at least - * as large as the cipher block size. This means that a cipher block - * can span at most 2 descriptors. However, this does not allow a - * partial block to span into the final descriptor as that would - * violate the rule (since every descriptor's length must be at lest - * the block size). So, for example, assuming an 8 byte block size: - * - * 0xe --> 0xa --> 0x8 - * - * is a valid length sequence, whereas: - * - * 0xe --> 0xb --> 0x7 - * - * is not a valid sequence. - */ - -struct n2_skcipher_alg { - struct list_head entry; - u8 enc_type; - struct skcipher_alg skcipher; -}; - -static inline struct n2_skcipher_alg *n2_skcipher_alg(struct crypto_skcipher *tfm) -{ - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - - return container_of(alg, struct n2_skcipher_alg, skcipher); -} - -static int n2_aes_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - - ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK); - - switch (keylen) { - case AES_KEYSIZE_128: - ctx->enc_type |= ENC_TYPE_ALG_AES128; - break; - case AES_KEYSIZE_192: - ctx->enc_type |= ENC_TYPE_ALG_AES192; - break; - case AES_KEYSIZE_256: - ctx->enc_type |= ENC_TYPE_ALG_AES256; - break; - default: - return -EINVAL; - } - - ctx->key_len = keylen; - memcpy(ctx->key.aes, key, keylen); - return 0; -} - -static int n2_des_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - int err; - - err = verify_skcipher_des_key(skcipher, key); - if (err) - return err; - - ctx->enc_type = n2alg->enc_type; - - ctx->key_len = keylen; - memcpy(ctx->key.des, key, keylen); - return 0; -} - -static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - int err; - - err = verify_skcipher_des3_key(skcipher, key); - if (err) - return err; - - ctx->enc_type = n2alg->enc_type; - - ctx->key_len = keylen; - memcpy(ctx->key.des3, key, keylen); - return 0; -} - -static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size) -{ - int this_len = nbytes; - - this_len -= (nbytes & (block_size - 1)); - return this_len > (1 << 16) ? (1 << 16) : this_len; -} - -static int __n2_crypt_chunk(struct crypto_skcipher *skcipher, - struct n2_crypto_chunk *cp, - struct spu_queue *qp, bool encrypt) -{ - struct n2_skcipher_context *ctx = crypto_skcipher_ctx(skcipher); - struct cwq_initial_entry *ent; - bool in_place; - int i; - - ent = spu_queue_alloc(qp, cp->arr_len); - if (!ent) { - pr_info("queue_alloc() of %d fails\n", - cp->arr_len); - return -EBUSY; - } - - in_place = (cp->dest_paddr == cp->arr[0].src_paddr); - - ent->control = control_word_base(cp->arr[0].src_len, - 0, ctx->enc_type, 0, 0, - false, true, false, encrypt, - OPCODE_ENCRYPT | - (in_place ? OPCODE_INPLACE_BIT : 0)); - ent->src_addr = cp->arr[0].src_paddr; - ent->auth_key_addr = 0UL; - ent->auth_iv_addr = 0UL; - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = __pa(&ctx->key); - ent->enc_iv_addr = cp->iv_paddr; - ent->dest_addr = (in_place ? 0UL : cp->dest_paddr); - - for (i = 1; i < cp->arr_len; i++) { - ent = spu_queue_next(qp, ent); - - ent->control = cp->arr[i].src_len - 1; - ent->src_addr = cp->arr[i].src_paddr; - ent->auth_key_addr = 0UL; - ent->auth_iv_addr = 0UL; - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = 0UL; - ent->enc_iv_addr = 0UL; - ent->dest_addr = 0UL; - } - ent->control |= CONTROL_END_OF_BLOCK; - - return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; -} - -static int n2_compute_chunks(struct skcipher_request *req) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct skcipher_walk *walk = &rctx->walk; - struct n2_crypto_chunk *chunk; - unsigned long dest_prev; - unsigned int tot_len; - bool prev_in_place; - int err, nbytes; - - err = skcipher_walk_async(walk, req); - if (err) - return err; - - INIT_LIST_HEAD(&rctx->chunk_list); - - chunk = &rctx->chunk; - INIT_LIST_HEAD(&chunk->entry); - - chunk->iv_paddr = 0UL; - chunk->arr_len = 0; - chunk->dest_paddr = 0UL; - - prev_in_place = false; - dest_prev = ~0UL; - tot_len = 0; - - while ((nbytes = walk->nbytes) != 0) { - unsigned long dest_paddr, src_paddr; - bool in_place; - int this_len; - - src_paddr = (page_to_phys(walk->src.phys.page) + - walk->src.phys.offset); - dest_paddr = (page_to_phys(walk->dst.phys.page) + - walk->dst.phys.offset); - in_place = (src_paddr == dest_paddr); - this_len = skcipher_descriptor_len(nbytes, walk->blocksize); - - if (chunk->arr_len != 0) { - if (in_place != prev_in_place || - (!prev_in_place && - dest_paddr != dest_prev) || - chunk->arr_len == N2_CHUNK_ARR_LEN || - tot_len + this_len > (1 << 16)) { - chunk->dest_final = dest_prev; - list_add_tail(&chunk->entry, - &rctx->chunk_list); - chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC); - if (!chunk) { - err = -ENOMEM; - break; - } - INIT_LIST_HEAD(&chunk->entry); - } - } - if (chunk->arr_len == 0) { - chunk->dest_paddr = dest_paddr; - tot_len = 0; - } - chunk->arr[chunk->arr_len].src_paddr = src_paddr; - chunk->arr[chunk->arr_len].src_len = this_len; - chunk->arr_len++; - - dest_prev = dest_paddr + this_len; - prev_in_place = in_place; - tot_len += this_len; - - err = skcipher_walk_done(walk, nbytes - this_len); - if (err) - break; - } - if (!err && chunk->arr_len != 0) { - chunk->dest_final = dest_prev; - list_add_tail(&chunk->entry, &rctx->chunk_list); - } - - return err; -} - -static void n2_chunk_complete(struct skcipher_request *req, void *final_iv) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct n2_crypto_chunk *c, *tmp; - - if (final_iv) - memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize); - - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - -} - -static int n2_do_ecb(struct skcipher_request *req, bool encrypt) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - int err = n2_compute_chunks(req); - struct n2_crypto_chunk *c, *tmp; - unsigned long flags, hv_ret; - struct spu_queue *qp; - - if (err) - return err; - - qp = cpu_to_cwq[get_cpu()]; - err = -ENODEV; - if (!qp) - goto out; - - spin_lock_irqsave(&qp->lock, flags); - - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { - err = __n2_crypt_chunk(tfm, c, qp, encrypt); - if (err) - break; - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - if (!err) { - hv_ret = wait_for_tail(qp); - if (hv_ret != HV_EOK) - err = -EINVAL; - } - - spin_unlock_irqrestore(&qp->lock, flags); - -out: - put_cpu(); - - n2_chunk_complete(req, NULL); - return err; -} - -static int n2_encrypt_ecb(struct skcipher_request *req) -{ - return n2_do_ecb(req, true); -} - -static int n2_decrypt_ecb(struct skcipher_request *req) -{ - return n2_do_ecb(req, false); -} - -static int n2_do_chaining(struct skcipher_request *req, bool encrypt) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - unsigned long flags, hv_ret, iv_paddr; - int err = n2_compute_chunks(req); - struct n2_crypto_chunk *c, *tmp; - struct spu_queue *qp; - void *final_iv_addr; - - final_iv_addr = NULL; - - if (err) - return err; - - qp = cpu_to_cwq[get_cpu()]; - err = -ENODEV; - if (!qp) - goto out; - - spin_lock_irqsave(&qp->lock, flags); - - if (encrypt) { - iv_paddr = __pa(rctx->walk.iv); - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, - entry) { - c->iv_paddr = iv_paddr; - err = __n2_crypt_chunk(tfm, c, qp, true); - if (err) - break; - iv_paddr = c->dest_final - rctx->walk.blocksize; - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - final_iv_addr = __va(iv_paddr); - } else { - list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list, - entry) { - if (c == &rctx->chunk) { - iv_paddr = __pa(rctx->walk.iv); - } else { - iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr + - tmp->arr[tmp->arr_len-1].src_len - - rctx->walk.blocksize); - } - if (!final_iv_addr) { - unsigned long pa; - - pa = (c->arr[c->arr_len-1].src_paddr + - c->arr[c->arr_len-1].src_len - - rctx->walk.blocksize); - final_iv_addr = rctx->temp_iv; - memcpy(rctx->temp_iv, __va(pa), - rctx->walk.blocksize); - } - c->iv_paddr = iv_paddr; - err = __n2_crypt_chunk(tfm, c, qp, false); - if (err) - break; - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - } - if (!err) { - hv_ret = wait_for_tail(qp); - if (hv_ret != HV_EOK) - err = -EINVAL; - } - - spin_unlock_irqrestore(&qp->lock, flags); - -out: - put_cpu(); - - n2_chunk_complete(req, err ? NULL : final_iv_addr); - return err; -} - -static int n2_encrypt_chaining(struct skcipher_request *req) -{ - return n2_do_chaining(req, true); -} - -static int n2_decrypt_chaining(struct skcipher_request *req) -{ - return n2_do_chaining(req, false); -} - -struct n2_skcipher_tmpl { - const char *name; - const char *drv_name; - u8 block_size; - u8 enc_type; - struct skcipher_alg skcipher; -}; - -static const struct n2_skcipher_tmpl skcipher_tmpls[] = { - /* DES: ECB CBC and CFB are supported */ - { .name = "ecb(des)", - .drv_name = "ecb-des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_DES | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = n2_des_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - { .name = "cbc(des)", - .drv_name = "cbc-des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_DES | - ENC_TYPE_CHAINING_CBC), - .skcipher = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = n2_des_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, - - /* 3DES: ECB CBC and CFB are supported */ - { .name = "ecb(des3_ede)", - .drv_name = "ecb-3des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_3DES | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = 3 * DES_KEY_SIZE, - .max_keysize = 3 * DES_KEY_SIZE, - .setkey = n2_3des_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - { .name = "cbc(des3_ede)", - .drv_name = "cbc-3des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_3DES | - ENC_TYPE_CHAINING_CBC), - .skcipher = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = 3 * DES_KEY_SIZE, - .max_keysize = 3 * DES_KEY_SIZE, - .setkey = n2_3des_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, - - /* AES: ECB CBC and CTR are supported */ - { .name = "ecb(aes)", - .drv_name = "ecb-aes", - .block_size = AES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_AES128 | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = n2_aes_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - { .name = "cbc(aes)", - .drv_name = "cbc-aes", - .block_size = AES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_AES128 | - ENC_TYPE_CHAINING_CBC), - .skcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = n2_aes_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, - { .name = "ctr(aes)", - .drv_name = "ctr-aes", - .block_size = AES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_AES128 | - ENC_TYPE_CHAINING_COUNTER), - .skcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = n2_aes_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_encrypt_chaining, - }, - }, - -}; -#define NUM_CIPHER_TMPLS ARRAY_SIZE(skcipher_tmpls) - -static LIST_HEAD(skcipher_algs); - -struct n2_hash_tmpl { - const char *name; - const u8 *hash_zero; - const u8 *hash_init; - u8 hw_op_hashsz; - u8 digest_size; - u8 statesize; - u8 block_size; - u8 auth_type; - u8 hmac_type; -}; - -static const __le32 n2_md5_init[MD5_HASH_WORDS] = { - cpu_to_le32(MD5_H0), - cpu_to_le32(MD5_H1), - cpu_to_le32(MD5_H2), - cpu_to_le32(MD5_H3), -}; -static const u32 n2_sha1_init[SHA1_DIGEST_SIZE / 4] = { - SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, -}; -static const u32 n2_sha256_init[SHA256_DIGEST_SIZE / 4] = { - SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, -}; -static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = { - SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, - SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, -}; - -static const struct n2_hash_tmpl hash_tmpls[] = { - { .name = "md5", - .hash_zero = md5_zero_message_hash, - .hash_init = (u8 *)n2_md5_init, - .auth_type = AUTH_TYPE_MD5, - .hmac_type = AUTH_TYPE_HMAC_MD5, - .hw_op_hashsz = MD5_DIGEST_SIZE, - .digest_size = MD5_DIGEST_SIZE, - .statesize = sizeof(struct md5_state), - .block_size = MD5_HMAC_BLOCK_SIZE }, - { .name = "sha1", - .hash_zero = sha1_zero_message_hash, - .hash_init = (u8 *)n2_sha1_init, - .auth_type = AUTH_TYPE_SHA1, - .hmac_type = AUTH_TYPE_HMAC_SHA1, - .hw_op_hashsz = SHA1_DIGEST_SIZE, - .digest_size = SHA1_DIGEST_SIZE, - .statesize = sizeof(struct sha1_state), - .block_size = SHA1_BLOCK_SIZE }, - { .name = "sha256", - .hash_zero = sha256_zero_message_hash, - .hash_init = (u8 *)n2_sha256_init, - .auth_type = AUTH_TYPE_SHA256, - .hmac_type = AUTH_TYPE_HMAC_SHA256, - .hw_op_hashsz = SHA256_DIGEST_SIZE, - .digest_size = SHA256_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), - .block_size = SHA256_BLOCK_SIZE }, - { .name = "sha224", - .hash_zero = sha224_zero_message_hash, - .hash_init = (u8 *)n2_sha224_init, - .auth_type = AUTH_TYPE_SHA256, - .hmac_type = AUTH_TYPE_RESERVED, - .hw_op_hashsz = SHA256_DIGEST_SIZE, - .digest_size = SHA224_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), - .block_size = SHA224_BLOCK_SIZE }, -}; -#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) - -static LIST_HEAD(ahash_algs); -static LIST_HEAD(hmac_algs); - -static int algs_registered; - -static void __n2_unregister_algs(void) -{ - struct n2_skcipher_alg *skcipher, *skcipher_tmp; - struct n2_ahash_alg *alg, *alg_tmp; - struct n2_hmac_alg *hmac, *hmac_tmp; - - list_for_each_entry_safe(skcipher, skcipher_tmp, &skcipher_algs, entry) { - crypto_unregister_skcipher(&skcipher->skcipher); - list_del(&skcipher->entry); - kfree(skcipher); - } - list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) { - crypto_unregister_ahash(&hmac->derived.alg); - list_del(&hmac->derived.entry); - kfree(hmac); - } - list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { - crypto_unregister_ahash(&alg->alg); - list_del(&alg->entry); - kfree(alg); - } -} - -static int n2_skcipher_init_tfm(struct crypto_skcipher *tfm) -{ - crypto_skcipher_set_reqsize(tfm, sizeof(struct n2_request_context)); - return 0; -} - -static int __n2_register_one_skcipher(const struct n2_skcipher_tmpl *tmpl) -{ - struct n2_skcipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); - struct skcipher_alg *alg; - int err; - - if (!p) - return -ENOMEM; - - alg = &p->skcipher; - *alg = tmpl->skcipher; - - snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); - snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name); - alg->base.cra_priority = N2_CRA_PRIORITY; - alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY; - alg->base.cra_blocksize = tmpl->block_size; - p->enc_type = tmpl->enc_type; - alg->base.cra_ctxsize = sizeof(struct n2_skcipher_context); - alg->base.cra_module = THIS_MODULE; - alg->init = n2_skcipher_init_tfm; - - list_add(&p->entry, &skcipher_algs); - err = crypto_register_skcipher(alg); - if (err) { - pr_err("%s alg registration failed\n", alg->base.cra_name); - list_del(&p->entry); - kfree(p); - } else { - pr_info("%s alg registered\n", alg->base.cra_name); - } - return err; -} - -static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) -{ - struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); - struct ahash_alg *ahash; - struct crypto_alg *base; - int err; - - if (!p) - return -ENOMEM; - - p->child_alg = n2ahash->alg.halg.base.cra_name; - memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg)); - INIT_LIST_HEAD(&p->derived.entry); - - ahash = &p->derived.alg; - ahash->digest = n2_hmac_async_digest; - ahash->setkey = n2_hmac_async_setkey; - - base = &ahash->halg.base; - err = -EINVAL; - if (snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", - p->child_alg) >= CRYPTO_MAX_ALG_NAME) - goto out_free_p; - if (snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", - p->child_alg) >= CRYPTO_MAX_ALG_NAME) - goto out_free_p; - - base->cra_ctxsize = sizeof(struct n2_hmac_ctx); - base->cra_init = n2_hmac_cra_init; - base->cra_exit = n2_hmac_cra_exit; - - list_add(&p->derived.entry, &hmac_algs); - err = crypto_register_ahash(ahash); - if (err) { - pr_err("%s alg registration failed\n", base->cra_name); - list_del(&p->derived.entry); -out_free_p: - kfree(p); - } else { - pr_info("%s alg registered\n", base->cra_name); - } - return err; -} - -static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) -{ - struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); - struct hash_alg_common *halg; - struct crypto_alg *base; - struct ahash_alg *ahash; - int err; - - if (!p) - return -ENOMEM; - - p->hash_zero = tmpl->hash_zero; - p->hash_init = tmpl->hash_init; - p->auth_type = tmpl->auth_type; - p->hmac_type = tmpl->hmac_type; - p->hw_op_hashsz = tmpl->hw_op_hashsz; - p->digest_size = tmpl->digest_size; - - ahash = &p->alg; - ahash->init = n2_hash_async_init; - ahash->update = n2_hash_async_update; - ahash->final = n2_hash_async_final; - ahash->finup = n2_hash_async_finup; - ahash->digest = n2_hash_async_digest; - ahash->export = n2_hash_async_noexport; - ahash->import = n2_hash_async_noimport; - - halg = &ahash->halg; - halg->digestsize = tmpl->digest_size; - halg->statesize = tmpl->statesize; - - base = &halg->base; - snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); - snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name); - base->cra_priority = N2_CRA_PRIORITY; - base->cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK; - base->cra_blocksize = tmpl->block_size; - base->cra_ctxsize = sizeof(struct n2_hash_ctx); - base->cra_module = THIS_MODULE; - base->cra_init = n2_hash_cra_init; - base->cra_exit = n2_hash_cra_exit; - - list_add(&p->entry, &ahash_algs); - err = crypto_register_ahash(ahash); - if (err) { - pr_err("%s alg registration failed\n", base->cra_name); - list_del(&p->entry); - kfree(p); - } else { - pr_info("%s alg registered\n", base->cra_name); - } - if (!err && p->hmac_type != AUTH_TYPE_RESERVED) - err = __n2_register_one_hmac(p); - return err; -} - -static int n2_register_algs(void) -{ - int i, err = 0; - - mutex_lock(&spu_lock); - if (algs_registered++) - goto out; - - for (i = 0; i < NUM_HASH_TMPLS; i++) { - err = __n2_register_one_ahash(&hash_tmpls[i]); - if (err) { - __n2_unregister_algs(); - goto out; - } - } - for (i = 0; i < NUM_CIPHER_TMPLS; i++) { - err = __n2_register_one_skcipher(&skcipher_tmpls[i]); - if (err) { - __n2_unregister_algs(); - goto out; - } - } - -out: - mutex_unlock(&spu_lock); - return err; -} - -static void n2_unregister_algs(void) -{ - mutex_lock(&spu_lock); - if (!--algs_registered) - __n2_unregister_algs(); - mutex_unlock(&spu_lock); -} - -/* To map CWQ queues to interrupt sources, the hypervisor API provides - * a devino. This isn't very useful to us because all of the - * interrupts listed in the device_node have been translated to - * Linux virtual IRQ cookie numbers. - * - * So we have to back-translate, going through the 'intr' and 'ino' - * property tables of the n2cp MDESC node, matching it with the OF - * 'interrupts' property entries, in order to figure out which - * devino goes to which already-translated IRQ. - */ -static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, - unsigned long dev_ino) -{ - const unsigned int *dev_intrs; - unsigned int intr; - int i; - - for (i = 0; i < ip->num_intrs; i++) { - if (ip->ino_table[i].ino == dev_ino) - break; - } - if (i == ip->num_intrs) - return -ENODEV; - - intr = ip->ino_table[i].intr; - - dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL); - if (!dev_intrs) - return -ENODEV; - - for (i = 0; i < dev->archdata.num_irqs; i++) { - if (dev_intrs[i] == intr) - return i; - } - - return -ENODEV; -} - -static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip, - const char *irq_name, struct spu_queue *p, - irq_handler_t handler) -{ - unsigned long herr; - int index; - - herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino); - if (herr) - return -EINVAL; - - index = find_devino_index(dev, ip, p->devino); - if (index < 0) - return index; - - p->irq = dev->archdata.irqs[index]; - - sprintf(p->irq_name, "%s-%d", irq_name, index); - - return request_irq(p->irq, handler, 0, p->irq_name, p); -} - -static struct kmem_cache *queue_cache[2]; - -static void *new_queue(unsigned long q_type) -{ - return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL); -} - -static void free_queue(void *p, unsigned long q_type) -{ - kmem_cache_free(queue_cache[q_type - 1], p); -} - -static int queue_cache_init(void) -{ - if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) - queue_cache[HV_NCS_QTYPE_MAU - 1] = - kmem_cache_create("mau_queue", - (MAU_NUM_ENTRIES * - MAU_ENTRY_SIZE), - MAU_ENTRY_SIZE, 0, NULL); - if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) - return -ENOMEM; - - if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) - queue_cache[HV_NCS_QTYPE_CWQ - 1] = - kmem_cache_create("cwq_queue", - (CWQ_NUM_ENTRIES * - CWQ_ENTRY_SIZE), - CWQ_ENTRY_SIZE, 0, NULL); - if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) { - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); - queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; - return -ENOMEM; - } - return 0; -} - -static void queue_cache_destroy(void) -{ - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]); - queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; - queue_cache[HV_NCS_QTYPE_CWQ - 1] = NULL; -} - -static long spu_queue_register_workfn(void *arg) -{ - struct spu_qreg *qr = arg; - struct spu_queue *p = qr->queue; - unsigned long q_type = qr->type; - unsigned long hv_ret; - - hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q), - CWQ_NUM_ENTRIES, &p->qhandle); - if (!hv_ret) - sun4v_ncs_sethead_marker(p->qhandle, 0); - - return hv_ret ? -EINVAL : 0; -} - -static int spu_queue_register(struct spu_queue *p, unsigned long q_type) -{ - int cpu = cpumask_any_and(&p->sharing, cpu_online_mask); - struct spu_qreg qr = { .queue = p, .type = q_type }; - - return work_on_cpu_safe(cpu, spu_queue_register_workfn, &qr); -} - -static int spu_queue_setup(struct spu_queue *p) -{ - int err; - - p->q = new_queue(p->q_type); - if (!p->q) - return -ENOMEM; - - err = spu_queue_register(p, p->q_type); - if (err) { - free_queue(p->q, p->q_type); - p->q = NULL; - } - - return err; -} - -static void spu_queue_destroy(struct spu_queue *p) -{ - unsigned long hv_ret; - - if (!p->q) - return; - - hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle); - - if (!hv_ret) - free_queue(p->q, p->q_type); -} - -static void spu_list_destroy(struct list_head *list) -{ - struct spu_queue *p, *n; - - list_for_each_entry_safe(p, n, list, list) { - int i; - - for (i = 0; i < NR_CPUS; i++) { - if (cpu_to_cwq[i] == p) - cpu_to_cwq[i] = NULL; - } - - if (p->irq) { - free_irq(p->irq, p); - p->irq = 0; - } - spu_queue_destroy(p); - list_del(&p->list); - kfree(p); - } -} - -/* Walk the backward arcs of a CWQ 'exec-unit' node, - * gathering cpu membership information. - */ -static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, - struct platform_device *dev, - u64 node, struct spu_queue *p, - struct spu_queue **table) -{ - u64 arc; - - mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) { - u64 tgt = mdesc_arc_target(mdesc, arc); - const char *name = mdesc_node_name(mdesc, tgt); - const u64 *id; - - if (strcmp(name, "cpu")) - continue; - id = mdesc_get_property(mdesc, tgt, "id", NULL); - if (table[*id] != NULL) { - dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n", - dev->dev.of_node); - return -EINVAL; - } - cpumask_set_cpu(*id, &p->sharing); - table[*id] = p; - } - return 0; -} - -/* Process an 'exec-unit' MDESC node of type 'cwq'. */ -static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, - struct platform_device *dev, struct mdesc_handle *mdesc, - u64 node, const char *iname, unsigned long q_type, - irq_handler_t handler, struct spu_queue **table) -{ - struct spu_queue *p; - int err; - - p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); - if (!p) { - dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n", - dev->dev.of_node); - return -ENOMEM; - } - - cpumask_clear(&p->sharing); - spin_lock_init(&p->lock); - p->q_type = q_type; - INIT_LIST_HEAD(&p->jobs); - list_add(&p->list, list); - - err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table); - if (err) - return err; - - err = spu_queue_setup(p); - if (err) - return err; - - return spu_map_ino(dev, ip, iname, p, handler); -} - -static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev, - struct spu_mdesc_info *ip, struct list_head *list, - const char *exec_name, unsigned long q_type, - irq_handler_t handler, struct spu_queue **table) -{ - int err = 0; - u64 node; - - mdesc_for_each_node_by_name(mdesc, node, "exec-unit") { - const char *type; - - type = mdesc_get_property(mdesc, node, "type", NULL); - if (!type || strcmp(type, exec_name)) - continue; - - err = handle_exec_unit(ip, list, dev, mdesc, node, - exec_name, q_type, handler, table); - if (err) { - spu_list_destroy(list); - break; - } - } - - return err; -} - -static int get_irq_props(struct mdesc_handle *mdesc, u64 node, - struct spu_mdesc_info *ip) -{ - const u64 *ino; - int ino_len; - int i; - - ino = mdesc_get_property(mdesc, node, "ino", &ino_len); - if (!ino) { - printk("NO 'ino'\n"); - return -ENODEV; - } - - ip->num_intrs = ino_len / sizeof(u64); - ip->ino_table = kzalloc((sizeof(struct ino_blob) * - ip->num_intrs), - GFP_KERNEL); - if (!ip->ino_table) - return -ENOMEM; - - for (i = 0; i < ip->num_intrs; i++) { - struct ino_blob *b = &ip->ino_table[i]; - b->intr = i + 1; - b->ino = ino[i]; - } - - return 0; -} - -static int grab_mdesc_irq_props(struct mdesc_handle *mdesc, - struct platform_device *dev, - struct spu_mdesc_info *ip, - const char *node_name) -{ - u64 node, reg; - - if (of_property_read_reg(dev->dev.of_node, 0, ®, NULL) < 0) - return -ENODEV; - - mdesc_for_each_node_by_name(mdesc, node, "virtual-device") { - const char *name; - const u64 *chdl; - - name = mdesc_get_property(mdesc, node, "name", NULL); - if (!name || strcmp(name, node_name)) - continue; - chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL); - if (!chdl || (*chdl != reg)) - continue; - ip->cfg_handle = *chdl; - return get_irq_props(mdesc, node, ip); - } - - return -ENODEV; -} - -static unsigned long n2_spu_hvapi_major; -static unsigned long n2_spu_hvapi_minor; - -static int n2_spu_hvapi_register(void) -{ - int err; - - n2_spu_hvapi_major = 2; - n2_spu_hvapi_minor = 0; - - err = sun4v_hvapi_register(HV_GRP_NCS, - n2_spu_hvapi_major, - &n2_spu_hvapi_minor); - - if (!err) - pr_info("Registered NCS HVAPI version %lu.%lu\n", - n2_spu_hvapi_major, - n2_spu_hvapi_minor); - - return err; -} - -static void n2_spu_hvapi_unregister(void) -{ - sun4v_hvapi_unregister(HV_GRP_NCS); -} - -static int global_ref; - -static int grab_global_resources(void) -{ - int err = 0; - - mutex_lock(&spu_lock); - - if (global_ref++) - goto out; - - err = n2_spu_hvapi_register(); - if (err) - goto out; - - err = queue_cache_init(); - if (err) - goto out_hvapi_release; - - err = -ENOMEM; - cpu_to_cwq = kcalloc(NR_CPUS, sizeof(struct spu_queue *), - GFP_KERNEL); - if (!cpu_to_cwq) - goto out_queue_cache_destroy; - - cpu_to_mau = kcalloc(NR_CPUS, sizeof(struct spu_queue *), - GFP_KERNEL); - if (!cpu_to_mau) - goto out_free_cwq_table; - - err = 0; - -out: - if (err) - global_ref--; - mutex_unlock(&spu_lock); - return err; - -out_free_cwq_table: - kfree(cpu_to_cwq); - cpu_to_cwq = NULL; - -out_queue_cache_destroy: - queue_cache_destroy(); - -out_hvapi_release: - n2_spu_hvapi_unregister(); - goto out; -} - -static void release_global_resources(void) -{ - mutex_lock(&spu_lock); - if (!--global_ref) { - kfree(cpu_to_cwq); - cpu_to_cwq = NULL; - - kfree(cpu_to_mau); - cpu_to_mau = NULL; - - queue_cache_destroy(); - n2_spu_hvapi_unregister(); - } - mutex_unlock(&spu_lock); -} - -static struct n2_crypto *alloc_n2cp(void) -{ - struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL); - - if (np) - INIT_LIST_HEAD(&np->cwq_list); - - return np; -} - -static void free_n2cp(struct n2_crypto *np) -{ - kfree(np->cwq_info.ino_table); - np->cwq_info.ino_table = NULL; - - kfree(np); -} - -static void n2_spu_driver_version(void) -{ - static int n2_spu_version_printed; - - if (n2_spu_version_printed++ == 0) - pr_info("%s", version); -} - -static int n2_crypto_probe(struct platform_device *dev) -{ - struct mdesc_handle *mdesc; - struct n2_crypto *np; - int err; - - n2_spu_driver_version(); - - pr_info("Found N2CP at %pOF\n", dev->dev.of_node); - - np = alloc_n2cp(); - if (!np) { - dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n", - dev->dev.of_node); - return -ENOMEM; - } - - err = grab_global_resources(); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", - dev->dev.of_node); - goto out_free_n2cp; - } - - mdesc = mdesc_grab(); - - if (!mdesc) { - dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", - dev->dev.of_node); - err = -ENODEV; - goto out_free_global; - } - err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", - dev->dev.of_node); - mdesc_release(mdesc); - goto out_free_global; - } - - err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list, - "cwq", HV_NCS_QTYPE_CWQ, cwq_intr, - cpu_to_cwq); - mdesc_release(mdesc); - - if (err) { - dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n", - dev->dev.of_node); - goto out_free_global; - } - - err = n2_register_algs(); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n", - dev->dev.of_node); - goto out_free_spu_list; - } - - dev_set_drvdata(&dev->dev, np); - - return 0; - -out_free_spu_list: - spu_list_destroy(&np->cwq_list); - -out_free_global: - release_global_resources(); - -out_free_n2cp: - free_n2cp(np); - - return err; -} - -static void n2_crypto_remove(struct platform_device *dev) -{ - struct n2_crypto *np = dev_get_drvdata(&dev->dev); - - n2_unregister_algs(); - - spu_list_destroy(&np->cwq_list); - - release_global_resources(); - - free_n2cp(np); -} - -static struct n2_mau *alloc_ncp(void) -{ - struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL); - - if (mp) - INIT_LIST_HEAD(&mp->mau_list); - - return mp; -} - -static void free_ncp(struct n2_mau *mp) -{ - kfree(mp->mau_info.ino_table); - mp->mau_info.ino_table = NULL; - - kfree(mp); -} - -static int n2_mau_probe(struct platform_device *dev) -{ - struct mdesc_handle *mdesc; - struct n2_mau *mp; - int err; - - n2_spu_driver_version(); - - pr_info("Found NCP at %pOF\n", dev->dev.of_node); - - mp = alloc_ncp(); - if (!mp) { - dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n", - dev->dev.of_node); - return -ENOMEM; - } - - err = grab_global_resources(); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", - dev->dev.of_node); - goto out_free_ncp; - } - - mdesc = mdesc_grab(); - - if (!mdesc) { - dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", - dev->dev.of_node); - err = -ENODEV; - goto out_free_global; - } - - err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", - dev->dev.of_node); - mdesc_release(mdesc); - goto out_free_global; - } - - err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list, - "mau", HV_NCS_QTYPE_MAU, mau_intr, - cpu_to_mau); - mdesc_release(mdesc); - - if (err) { - dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n", - dev->dev.of_node); - goto out_free_global; - } - - dev_set_drvdata(&dev->dev, mp); - - return 0; - -out_free_global: - release_global_resources(); - -out_free_ncp: - free_ncp(mp); - - return err; -} - -static void n2_mau_remove(struct platform_device *dev) -{ - struct n2_mau *mp = dev_get_drvdata(&dev->dev); - - spu_list_destroy(&mp->mau_list); - - release_global_resources(); - - free_ncp(mp); -} - -static const struct of_device_id n2_crypto_match[] = { - { - .name = "n2cp", - .compatible = "SUNW,n2-cwq", - }, - { - .name = "n2cp", - .compatible = "SUNW,vf-cwq", - }, - { - .name = "n2cp", - .compatible = "SUNW,kt-cwq", - }, - {}, -}; - -MODULE_DEVICE_TABLE(of, n2_crypto_match); - -static struct platform_driver n2_crypto_driver = { - .driver = { - .name = "n2cp", - .of_match_table = n2_crypto_match, - }, - .probe = n2_crypto_probe, - .remove = n2_crypto_remove, -}; - -static const struct of_device_id n2_mau_match[] = { - { - .name = "ncp", - .compatible = "SUNW,n2-mau", - }, - { - .name = "ncp", - .compatible = "SUNW,vf-mau", - }, - { - .name = "ncp", - .compatible = "SUNW,kt-mau", - }, - {}, -}; - -MODULE_DEVICE_TABLE(of, n2_mau_match); - -static struct platform_driver n2_mau_driver = { - .driver = { - .name = "ncp", - .of_match_table = n2_mau_match, - }, - .probe = n2_mau_probe, - .remove = n2_mau_remove, -}; - -static struct platform_driver * const drivers[] = { - &n2_crypto_driver, - &n2_mau_driver, -}; - -static int __init n2_init(void) -{ - return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); -} - -static void __exit n2_exit(void) -{ - platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); -} - -module_init(n2_init); -module_exit(n2_exit); diff --git a/drivers/crypto/n2_core.h b/drivers/crypto/n2_core.h deleted file mode 100644 index 2406763b0306..000000000000 --- a/drivers/crypto/n2_core.h +++ /dev/null @@ -1,232 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _N2_CORE_H -#define _N2_CORE_H - -#ifndef __ASSEMBLY__ - -struct ino_blob { - u64 intr; - u64 ino; -}; - -struct spu_mdesc_info { - u64 cfg_handle; - struct ino_blob *ino_table; - int num_intrs; -}; - -struct n2_crypto { - struct spu_mdesc_info cwq_info; - struct list_head cwq_list; -}; - -struct n2_mau { - struct spu_mdesc_info mau_info; - struct list_head mau_list; -}; - -#define CWQ_ENTRY_SIZE 64 -#define CWQ_NUM_ENTRIES 64 - -#define MAU_ENTRY_SIZE 64 -#define MAU_NUM_ENTRIES 64 - -struct cwq_initial_entry { - u64 control; - u64 src_addr; - u64 auth_key_addr; - u64 auth_iv_addr; - u64 final_auth_state_addr; - u64 enc_key_addr; - u64 enc_iv_addr; - u64 dest_addr; -}; - -struct cwq_ext_entry { - u64 len; - u64 src_addr; - u64 resv1; - u64 resv2; - u64 resv3; - u64 resv4; - u64 resv5; - u64 resv6; -}; - -struct cwq_final_entry { - u64 control; - u64 src_addr; - u64 resv1; - u64 resv2; - u64 resv3; - u64 resv4; - u64 resv5; - u64 resv6; -}; - -#define CONTROL_LEN 0x000000000000ffffULL -#define CONTROL_LEN_SHIFT 0 -#define CONTROL_HMAC_KEY_LEN 0x0000000000ff0000ULL -#define CONTROL_HMAC_KEY_LEN_SHIFT 16 -#define CONTROL_ENC_TYPE 0x00000000ff000000ULL -#define CONTROL_ENC_TYPE_SHIFT 24 -#define ENC_TYPE_ALG_RC4_STREAM 0x00ULL -#define ENC_TYPE_ALG_RC4_NOSTREAM 0x04ULL -#define ENC_TYPE_ALG_DES 0x08ULL -#define ENC_TYPE_ALG_3DES 0x0cULL -#define ENC_TYPE_ALG_AES128 0x10ULL -#define ENC_TYPE_ALG_AES192 0x14ULL -#define ENC_TYPE_ALG_AES256 0x18ULL -#define ENC_TYPE_ALG_RESERVED 0x1cULL -#define ENC_TYPE_ALG_MASK 0x1cULL -#define ENC_TYPE_CHAINING_ECB 0x00ULL -#define ENC_TYPE_CHAINING_CBC 0x01ULL -#define ENC_TYPE_CHAINING_CFB 0x02ULL -#define ENC_TYPE_CHAINING_COUNTER 0x03ULL -#define ENC_TYPE_CHAINING_MASK 0x03ULL -#define CONTROL_AUTH_TYPE 0x0000001f00000000ULL -#define CONTROL_AUTH_TYPE_SHIFT 32 -#define AUTH_TYPE_RESERVED 0x00ULL -#define AUTH_TYPE_MD5 0x01ULL -#define AUTH_TYPE_SHA1 0x02ULL -#define AUTH_TYPE_SHA256 0x03ULL -#define AUTH_TYPE_CRC32 0x04ULL -#define AUTH_TYPE_HMAC_MD5 0x05ULL -#define AUTH_TYPE_HMAC_SHA1 0x06ULL -#define AUTH_TYPE_HMAC_SHA256 0x07ULL -#define AUTH_TYPE_TCP_CHECKSUM 0x08ULL -#define AUTH_TYPE_SSL_HMAC_MD5 0x09ULL -#define AUTH_TYPE_SSL_HMAC_SHA1 0x0aULL -#define AUTH_TYPE_SSL_HMAC_SHA256 0x0bULL -#define CONTROL_STRAND 0x000000e000000000ULL -#define CONTROL_STRAND_SHIFT 37 -#define CONTROL_HASH_LEN 0x0000ff0000000000ULL -#define CONTROL_HASH_LEN_SHIFT 40 -#define CONTROL_INTERRUPT 0x0001000000000000ULL -#define CONTROL_STORE_FINAL_AUTH_STATE 0x0002000000000000ULL -#define CONTROL_RESERVED 0x001c000000000000ULL -#define CONTROL_HV_DONE 0x0004000000000000ULL -#define CONTROL_HV_PROTOCOL_ERROR 0x0008000000000000ULL -#define CONTROL_HV_HARDWARE_ERROR 0x0010000000000000ULL -#define CONTROL_END_OF_BLOCK 0x0020000000000000ULL -#define CONTROL_START_OF_BLOCK 0x0040000000000000ULL -#define CONTROL_ENCRYPT 0x0080000000000000ULL -#define CONTROL_OPCODE 0xff00000000000000ULL -#define CONTROL_OPCODE_SHIFT 56 -#define OPCODE_INPLACE_BIT 0x80ULL -#define OPCODE_SSL_KEYBLOCK 0x10ULL -#define OPCODE_COPY 0x20ULL -#define OPCODE_ENCRYPT 0x40ULL -#define OPCODE_AUTH_MAC 0x41ULL - -#endif /* !(__ASSEMBLY__) */ - -/* NCS v2.0 hypervisor interfaces */ -#define HV_NCS_QTYPE_MAU 0x01 -#define HV_NCS_QTYPE_CWQ 0x02 - -/* ncs_qconf() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_QCONF - * ARG0: Queue type (HV_NCS_QTYPE_{MAU,CWQ}) - * ARG1: Real address of queue, or handle for unconfigure - * ARG2: Number of entries in queue, zero for unconfigure - * RET0: status - * RET1: queue handle - * - * Configure a queue in the stream processing unit. - * - * The real address given as the base must be 64-byte - * aligned. - * - * The queue size can range from a minimum of 2 to a maximum - * of 64. The queue size must be a power of two. - * - * To unconfigure a queue, specify a length of zero and place - * the queue handle into ARG1. - * - * On configure success the hypervisor will set the FIRST, HEAD, - * and TAIL registers to the address of the first entry in the - * queue. The LAST register will be set to point to the last - * entry in the queue. - */ -#define HV_FAST_NCS_QCONF 0x111 - -/* ncs_qinfo() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_QINFO - * ARG0: Queue handle - * RET0: status - * RET1: Queue type (HV_NCS_QTYPE_{MAU,CWQ}) - * RET2: Queue base address - * RET3: Number of entries - */ -#define HV_FAST_NCS_QINFO 0x112 - -/* ncs_gethead() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_GETHEAD - * ARG0: Queue handle - * RET0: status - * RET1: queue head offset - */ -#define HV_FAST_NCS_GETHEAD 0x113 - -/* ncs_gettail() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_GETTAIL - * ARG0: Queue handle - * RET0: status - * RET1: queue tail offset - */ -#define HV_FAST_NCS_GETTAIL 0x114 - -/* ncs_settail() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_SETTAIL - * ARG0: Queue handle - * ARG1: New tail offset - * RET0: status - */ -#define HV_FAST_NCS_SETTAIL 0x115 - -/* ncs_qhandle_to_devino() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_QHANDLE_TO_DEVINO - * ARG0: Queue handle - * RET0: status - * RET1: devino - */ -#define HV_FAST_NCS_QHANDLE_TO_DEVINO 0x116 - -/* ncs_sethead_marker() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_SETHEAD_MARKER - * ARG0: Queue handle - * ARG1: New head offset - * RET0: status - */ -#define HV_FAST_NCS_SETHEAD_MARKER 0x117 - -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_ncs_qconf(unsigned long queue_type, - unsigned long queue_ra, - unsigned long num_entries, - unsigned long *qhandle); -extern unsigned long sun4v_ncs_qinfo(unsigned long qhandle, - unsigned long *queue_type, - unsigned long *queue_ra, - unsigned long *num_entries); -extern unsigned long sun4v_ncs_gethead(unsigned long qhandle, - unsigned long *head); -extern unsigned long sun4v_ncs_gettail(unsigned long qhandle, - unsigned long *tail); -extern unsigned long sun4v_ncs_settail(unsigned long qhandle, - unsigned long tail); -extern unsigned long sun4v_ncs_qhandle_to_devino(unsigned long qhandle, - unsigned long *devino); -extern unsigned long sun4v_ncs_sethead_marker(unsigned long qhandle, - unsigned long head); -#endif /* !(__ASSEMBLY__) */ - -#endif /* _N2_CORE_H */ diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index 82214cde2bcd..b950fcce8a9b 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -101,9 +101,13 @@ static int update_param(struct nx842_crypto_param *p, return 0; } -int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver) +void *nx842_crypto_alloc_ctx(struct nx842_driver *driver) { - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_ctx *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); spin_lock_init(&ctx->lock); ctx->driver = driver; @@ -114,22 +118,23 @@ int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver) kfree(ctx->wmem); free_page((unsigned long)ctx->sbounce); free_page((unsigned long)ctx->dbounce); - return -ENOMEM; + kfree(ctx); + return ERR_PTR(-ENOMEM); } - return 0; + return ctx; } -EXPORT_SYMBOL_GPL(nx842_crypto_init); +EXPORT_SYMBOL_GPL(nx842_crypto_alloc_ctx); -void nx842_crypto_exit(struct crypto_tfm *tfm) +void nx842_crypto_free_ctx(void *p) { - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_ctx *ctx = p; kfree(ctx->wmem); free_page((unsigned long)ctx->sbounce); free_page((unsigned long)ctx->dbounce); } -EXPORT_SYMBOL_GPL(nx842_crypto_exit); +EXPORT_SYMBOL_GPL(nx842_crypto_free_ctx); static void check_constraints(struct nx842_constraints *c) { @@ -246,11 +251,11 @@ nospc: return update_param(p, slen, dskip + dlen); } -int nx842_crypto_compress(struct crypto_tfm *tfm, +int nx842_crypto_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) + u8 *dst, unsigned int *dlen, void *pctx) { - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_ctx *ctx = pctx; struct nx842_crypto_header *hdr = container_of(&ctx->header, struct nx842_crypto_header, hdr); @@ -431,11 +436,11 @@ usesw: return update_param(p, slen + padding, dlen); } -int nx842_crypto_decompress(struct crypto_tfm *tfm, +int nx842_crypto_decompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) + u8 *dst, unsigned int *dlen, void *pctx) { - struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_ctx *ctx = pctx; struct nx842_crypto_header *hdr; struct nx842_crypto_param p; struct nx842_constraints c = *ctx->driver->constraints; diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h index 887d4ce3cb49..f5e2c82ba876 100644 --- a/drivers/crypto/nx/nx-842.h +++ b/drivers/crypto/nx/nx-842.h @@ -3,7 +3,6 @@ #ifndef __NX_842_H__ #define __NX_842_H__ -#include <crypto/algapi.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/module.h> @@ -101,6 +100,8 @@ #define LEN_ON_SIZE(pa, size) ((size) - ((pa) & ((size) - 1))) #define LEN_ON_PAGE(pa) LEN_ON_SIZE(pa, PAGE_SIZE) +struct crypto_scomp; + static inline unsigned long nx842_get_pa(void *addr) { if (!is_vmalloc_addr(addr)) @@ -182,13 +183,13 @@ struct nx842_crypto_ctx { struct nx842_driver *driver; }; -int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver); -void nx842_crypto_exit(struct crypto_tfm *tfm); -int nx842_crypto_compress(struct crypto_tfm *tfm, +void *nx842_crypto_alloc_ctx(struct nx842_driver *driver); +void nx842_crypto_free_ctx(void *ctx); +int nx842_crypto_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen); -int nx842_crypto_decompress(struct crypto_tfm *tfm, + u8 *dst, unsigned int *dlen, void *ctx); +int nx842_crypto_decompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen); + u8 *dst, unsigned int *dlen, void *ctx); #endif /* __NX_842_H__ */ diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c index 0e440f704a8f..35fa5bad1d9f 100644 --- a/drivers/crypto/nx/nx-aes-cbc.c +++ b/drivers/crypto/nx/nx-aes-cbc.c @@ -8,10 +8,12 @@ */ #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include <asm/vio.h> #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c index c843f4c6f684..56a0b3a67c33 100644 --- a/drivers/crypto/nx/nx-aes-ccm.c +++ b/drivers/crypto/nx/nx-aes-ccm.c @@ -217,13 +217,11 @@ static int generate_pat(u8 *iv, memset(b1, 0, 16); if (assoclen <= 65280) { *(u16 *)b1 = assoclen; - scatterwalk_map_and_copy(b1 + 2, req->src, 0, - iauth_len, SCATTERWALK_FROM_SG); + memcpy_from_sglist(b1 + 2, req->src, 0, iauth_len); } else { *(u16 *)b1 = (u16)(0xfffe); *(u32 *)&b1[2] = assoclen; - scatterwalk_map_and_copy(b1 + 6, req->src, 0, - iauth_len, SCATTERWALK_FROM_SG); + memcpy_from_sglist(b1 + 6, req->src, 0, iauth_len); } } @@ -341,9 +339,8 @@ static int ccm_nx_decrypt(struct aead_request *req, nbytes -= authsize; /* copy out the auth tag to compare with later */ - scatterwalk_map_and_copy(priv->oauth_tag, - req->src, nbytes + req->assoclen, authsize, - SCATTERWALK_FROM_SG); + memcpy_from_sglist(priv->oauth_tag, req->src, nbytes + req->assoclen, + authsize); rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen, csbcpb->cpb.aes_ccm.in_pat_or_b0); @@ -465,9 +462,8 @@ static int ccm_nx_encrypt(struct aead_request *req, } while (processed < nbytes); /* copy out the auth tag */ - scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac, - req->dst, nbytes + req->assoclen, authsize, - SCATTERWALK_TO_SG); + memcpy_to_sglist(req->dst, nbytes + req->assoclen, + csbcpb->cpb.aes_ccm.out_pat_or_mac, authsize); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c index dfa3ad1a12f2..709b3ee74657 100644 --- a/drivers/crypto/nx/nx-aes-ctr.c +++ b/drivers/crypto/nx/nx-aes-ctr.c @@ -9,10 +9,12 @@ #include <crypto/aes.h> #include <crypto/ctr.h> -#include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include <asm/vio.h> #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c index 502a565074e9..4039cf3b22d4 100644 --- a/drivers/crypto/nx/nx-aes-ecb.c +++ b/drivers/crypto/nx/nx-aes-ecb.c @@ -8,10 +8,12 @@ */ #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include <asm/vio.h> #include "nx_csbcpb.h" diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c index 4a796318b430..b7fe2de96d96 100644 --- a/drivers/crypto/nx/nx-aes-gcm.c +++ b/drivers/crypto/nx/nx-aes-gcm.c @@ -103,16 +103,13 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx, { int rc; struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; - struct scatter_walk walk; struct nx_sg *nx_sg = nx_ctx->in_sg; unsigned int nbytes = assoclen; unsigned int processed = 0, to_process; unsigned int max_sg_len; if (nbytes <= AES_BLOCK_SIZE) { - scatterwalk_start(&walk, req->src); - scatterwalk_copychunks(out, &walk, nbytes, SCATTERWALK_FROM_SG); - scatterwalk_done(&walk, SCATTERWALK_FROM_SG, 0); + memcpy_from_sglist(out, req->src, 0, nbytes); return 0; } @@ -391,19 +388,17 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc, mac: if (enc) { /* copy out the auth tag */ - scatterwalk_map_and_copy( - csbcpb->cpb.aes_gcm.out_pat_or_mac, + memcpy_to_sglist( req->dst, req->assoclen + nbytes, - crypto_aead_authsize(crypto_aead_reqtfm(req)), - SCATTERWALK_TO_SG); + csbcpb->cpb.aes_gcm.out_pat_or_mac, + crypto_aead_authsize(crypto_aead_reqtfm(req))); } else { u8 *itag = nx_ctx->priv.gcm.iauth_tag; u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac; - scatterwalk_map_and_copy( + memcpy_from_sglist( itag, req->src, req->assoclen + nbytes, - crypto_aead_authsize(crypto_aead_reqtfm(req)), - SCATTERWALK_FROM_SG); + crypto_aead_authsize(crypto_aead_reqtfm(req))); rc = crypto_memneq(itag, otag, crypto_aead_authsize(crypto_aead_reqtfm(req))) ? -EBADMSG : 0; diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c index eb5c8f689360..bf465d824e2c 100644 --- a/drivers/crypto/nx/nx-aes-xcbc.c +++ b/drivers/crypto/nx/nx-aes-xcbc.c @@ -7,13 +7,14 @@ * Author: Kent Yoder <yoder1@us.ibm.com> */ -#include <crypto/internal/hash.h> #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/internal/hash.h> +#include <linux/atomic.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <linux/types.h> -#include <linux/crypto.h> -#include <asm/vio.h> +#include <linux/spinlock.h> +#include <linux/string.h> #include "nx_csbcpb.h" #include "nx.h" @@ -21,8 +22,6 @@ struct xcbc_state { u8 state[AES_BLOCK_SIZE]; - unsigned int count; - u8 buffer[AES_BLOCK_SIZE]; }; static int nx_xcbc_set_key(struct crypto_shash *desc, @@ -58,7 +57,7 @@ static int nx_xcbc_set_key(struct crypto_shash *desc, */ static int nx_xcbc_empty(struct shash_desc *desc, u8 *out) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; u8 keys[2][AES_BLOCK_SIZE]; @@ -135,9 +134,9 @@ out: return rc; } -static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_tfm *tfm) +static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; int err; @@ -166,31 +165,24 @@ static int nx_xcbc_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct xcbc_state *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg; struct nx_sg *out_sg; - u32 to_process = 0, leftover, total; unsigned int max_sg_len; unsigned long irq_flags; + u32 to_process, total; int rc = 0; int data_len; spin_lock_irqsave(&nx_ctx->lock, irq_flags); + memcpy(csbcpb->cpb.aes_xcbc.out_cv_mac, sctx->state, AES_BLOCK_SIZE); + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - total = sctx->count + len; - - /* 2 cases for total data len: - * 1: <= AES_BLOCK_SIZE: copy into state, return 0 - * 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover - */ - if (total <= AES_BLOCK_SIZE) { - memcpy(sctx->buffer + sctx->count, data, len); - sctx->count += len; - goto out; - } + total = len; in_sg = nx_ctx->in_sg; max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg), @@ -200,7 +192,7 @@ static int nx_xcbc_update(struct shash_desc *desc, data_len = AES_BLOCK_SIZE; out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state, - &len, nx_ctx->ap->sglen); + &data_len, nx_ctx->ap->sglen); if (data_len != AES_BLOCK_SIZE) { rc = -EINVAL; @@ -210,56 +202,21 @@ static int nx_xcbc_update(struct shash_desc *desc, nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); do { - to_process = total - to_process; - to_process = to_process & ~(AES_BLOCK_SIZE - 1); - - leftover = total - to_process; - - /* the hardware will not accept a 0 byte operation for this - * algorithm and the operation MUST be finalized to be correct. - * So if we happen to get an update that falls on a block sized - * boundary, we must save off the last block to finalize with - * later. */ - if (!leftover) { - to_process -= AES_BLOCK_SIZE; - leftover = AES_BLOCK_SIZE; - } - - if (sctx->count) { - data_len = sctx->count; - in_sg = nx_build_sg_list(nx_ctx->in_sg, - (u8 *) sctx->buffer, - &data_len, - max_sg_len); - if (data_len != sctx->count) { - rc = -EINVAL; - goto out; - } - } + to_process = total & ~(AES_BLOCK_SIZE - 1); - data_len = to_process - sctx->count; in_sg = nx_build_sg_list(in_sg, (u8 *) data, - &data_len, + &to_process, max_sg_len); - if (data_len != to_process - sctx->count) { - rc = -EINVAL; - goto out; - } - nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); /* we've hit the nx chip previously and we're updating again, * so copy over the partial digest */ - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { - memcpy(csbcpb->cpb.aes_xcbc.cv, - csbcpb->cpb.aes_xcbc.out_cv_mac, - AES_BLOCK_SIZE); - } + memcpy(csbcpb->cpb.aes_xcbc.cv, + csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); - NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { rc = -EINVAL; goto out; @@ -271,28 +228,24 @@ static int nx_xcbc_update(struct shash_desc *desc, atomic_inc(&(nx_ctx->stats->aes_ops)); - /* everything after the first update is continuation */ - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - total -= to_process; - data += to_process - sctx->count; - sctx->count = 0; + data += to_process; in_sg = nx_ctx->in_sg; - } while (leftover > AES_BLOCK_SIZE); + } while (total >= AES_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - memcpy(sctx->buffer, data, leftover); - sctx->count = leftover; + rc = total; + memcpy(sctx->state, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_xcbc_final(struct shash_desc *desc, u8 *out) +static int nx_xcbc_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct xcbc_state *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; unsigned long irq_flags; @@ -301,12 +254,10 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out) spin_lock_irqsave(&nx_ctx->lock, irq_flags); - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.aes_xcbc.cv, - csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); - } else if (sctx->count == 0) { + if (nbytes) { + /* non-zero final, so copy over the partial digest */ + memcpy(csbcpb->cpb.aes_xcbc.cv, sctx->state, AES_BLOCK_SIZE); + } else { /* * we've never seen an update, so this is a 0 byte op. The * hardware cannot handle a 0 byte op, so just ECB to @@ -320,11 +271,11 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out) * this is not an intermediate operation */ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - len = sctx->count; - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buffer, - &len, nx_ctx->ap->sglen); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, + nx_ctx->ap->sglen); - if (len != sctx->count) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -362,18 +313,19 @@ struct shash_alg nx_shash_aes_xcbc_alg = { .digestsize = AES_BLOCK_SIZE, .init = nx_xcbc_init, .update = nx_xcbc_update, - .final = nx_xcbc_final, + .finup = nx_xcbc_finup, .setkey = nx_xcbc_set_key, .descsize = sizeof(struct xcbc_state), - .statesize = sizeof(struct xcbc_state), + .init_tfm = nx_crypto_ctx_aes_xcbc_init2, + .exit_tfm = nx_crypto_ctx_shash_exit, .base = { .cra_name = "xcbc(aes)", .cra_driver_name = "xcbc-aes-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, .cra_blocksize = AES_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_aes_xcbc_init2, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c index 8c859872c183..0493041ea088 100644 --- a/drivers/crypto/nx/nx-common-powernv.c +++ b/drivers/crypto/nx/nx-common-powernv.c @@ -9,6 +9,7 @@ #include "nx-842.h" +#include <crypto/internal/scompress.h> #include <linux/timer.h> #include <asm/prom.h> @@ -1031,23 +1032,23 @@ static struct nx842_driver nx842_powernv_driver = { .decompress = nx842_powernv_decompress, }; -static int nx842_powernv_crypto_init(struct crypto_tfm *tfm) +static void *nx842_powernv_crypto_alloc_ctx(void) { - return nx842_crypto_init(tfm, &nx842_powernv_driver); + return nx842_crypto_alloc_ctx(&nx842_powernv_driver); } -static struct crypto_alg nx842_powernv_alg = { - .cra_name = "842", - .cra_driver_name = "842-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct nx842_crypto_ctx), - .cra_module = THIS_MODULE, - .cra_init = nx842_powernv_crypto_init, - .cra_exit = nx842_crypto_exit, - .cra_u = { .compress = { - .coa_compress = nx842_crypto_compress, - .coa_decompress = nx842_crypto_decompress } } +static struct scomp_alg nx842_powernv_alg = { + .base.cra_name = "842", + .base.cra_driver_name = "842-nx", + .base.cra_priority = 300, + .base.cra_module = THIS_MODULE, + + .streams = { + .alloc_ctx = nx842_powernv_crypto_alloc_ctx, + .free_ctx = nx842_crypto_free_ctx, + }, + .compress = nx842_crypto_compress, + .decompress = nx842_crypto_decompress, }; static __init int nx_compress_powernv_init(void) @@ -1107,7 +1108,7 @@ static __init int nx_compress_powernv_init(void) nx842_powernv_exec = nx842_exec_vas; } - ret = crypto_register_alg(&nx842_powernv_alg); + ret = crypto_register_scomp(&nx842_powernv_alg); if (ret) { nx_delete_coprocs(); return ret; @@ -1128,7 +1129,7 @@ static void __exit nx_compress_powernv_exit(void) if (!nx842_ct) vas_unregister_api_powernv(); - crypto_unregister_alg(&nx842_powernv_alg); + crypto_unregister_scomp(&nx842_powernv_alg); nx_delete_coprocs(); } diff --git a/drivers/crypto/nx/nx-common-pseries.c b/drivers/crypto/nx/nx-common-pseries.c index 1660c5cf3641..fc0222ebe807 100644 --- a/drivers/crypto/nx/nx-common-pseries.c +++ b/drivers/crypto/nx/nx-common-pseries.c @@ -11,6 +11,7 @@ #include <asm/vio.h> #include <asm/hvcall.h> #include <asm/vas.h> +#include <crypto/internal/scompress.h> #include "nx-842.h" #include "nx_csbcpb.h" /* struct nx_csbcpb */ @@ -1008,23 +1009,23 @@ static struct nx842_driver nx842_pseries_driver = { .decompress = nx842_pseries_decompress, }; -static int nx842_pseries_crypto_init(struct crypto_tfm *tfm) +static void *nx842_pseries_crypto_alloc_ctx(void) { - return nx842_crypto_init(tfm, &nx842_pseries_driver); + return nx842_crypto_alloc_ctx(&nx842_pseries_driver); } -static struct crypto_alg nx842_pseries_alg = { - .cra_name = "842", - .cra_driver_name = "842-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct nx842_crypto_ctx), - .cra_module = THIS_MODULE, - .cra_init = nx842_pseries_crypto_init, - .cra_exit = nx842_crypto_exit, - .cra_u = { .compress = { - .coa_compress = nx842_crypto_compress, - .coa_decompress = nx842_crypto_decompress } } +static struct scomp_alg nx842_pseries_alg = { + .base.cra_name = "842", + .base.cra_driver_name = "842-nx", + .base.cra_priority = 300, + .base.cra_module = THIS_MODULE, + + .streams = { + .alloc_ctx = nx842_pseries_crypto_alloc_ctx, + .free_ctx = nx842_crypto_free_ctx, + }, + .compress = nx842_crypto_compress, + .decompress = nx842_crypto_decompress, }; static int nx842_probe(struct vio_dev *viodev, @@ -1072,7 +1073,7 @@ static int nx842_probe(struct vio_dev *viodev, if (ret) goto error; - ret = crypto_register_alg(&nx842_pseries_alg); + ret = crypto_register_scomp(&nx842_pseries_alg); if (ret) { dev_err(&viodev->dev, "could not register comp alg: %d\n", ret); goto error; @@ -1120,7 +1121,7 @@ static void nx842_remove(struct vio_dev *viodev) if (caps_feat) sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group); - crypto_unregister_alg(&nx842_pseries_alg); + crypto_unregister_scomp(&nx842_pseries_alg); of_reconfig_notifier_unregister(&nx842_of_nb); @@ -1145,6 +1146,7 @@ static void __init nxcop_get_capabilities(void) { struct hv_vas_all_caps *hv_caps; struct hv_nx_cop_caps *hv_nxc; + u64 feat; int rc; hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL); @@ -1155,27 +1157,26 @@ static void __init nxcop_get_capabilities(void) */ rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0, (u64)virt_to_phys(hv_caps)); + if (!rc) + feat = be64_to_cpu(hv_caps->feat_type); + kfree(hv_caps); if (rc) - goto out; + return; + if (!(feat & VAS_NX_GZIP_FEAT_BIT)) + return; - caps_feat = be64_to_cpu(hv_caps->feat_type); /* * NX-GZIP feature available */ - if (caps_feat & VAS_NX_GZIP_FEAT_BIT) { - hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL); - if (!hv_nxc) - goto out; - /* - * Get capabilities for NX-GZIP feature - */ - rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, - VAS_NX_GZIP_FEAT, - (u64)virt_to_phys(hv_nxc)); - } else { - pr_err("NX-GZIP feature is not available\n"); - rc = -EINVAL; - } + hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL); + if (!hv_nxc) + return; + /* + * Get capabilities for NX-GZIP feature + */ + rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, + VAS_NX_GZIP_FEAT, + (u64)virt_to_phys(hv_nxc)); if (!rc) { nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor); @@ -1185,13 +1186,10 @@ static void __init nxcop_get_capabilities(void) be64_to_cpu(hv_nxc->min_compress_len); nx_cop_caps.min_decompress_len = be64_to_cpu(hv_nxc->min_decompress_len); - } else { - caps_feat = 0; + caps_feat = feat; } kfree(hv_nxc); -out: - kfree(hv_caps); } static const struct vio_device_id nx842_vio_driver_ids[] = { @@ -1256,7 +1254,7 @@ static void __exit nx842_pseries_exit(void) vas_unregister_api_pseries(); - crypto_unregister_alg(&nx842_pseries_alg); + crypto_unregister_scomp(&nx842_pseries_alg); spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index c3bebf0feabe..5b29dd026df2 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -9,9 +9,12 @@ #include <crypto/internal/hash.h> #include <crypto/sha2.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/vio.h> -#include <asm/byteorder.h> +#include <linux/spinlock.h> +#include <linux/string.h> +#include <linux/unaligned.h> #include "nx_csbcpb.h" #include "nx.h" @@ -19,12 +22,11 @@ struct sha256_state_be { __be32 state[SHA256_DIGEST_SIZE / 4]; u64 count; - u8 buf[SHA256_BLOCK_SIZE]; }; -static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) +static int nx_crypto_ctx_sha256_init(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); int err; err = nx_crypto_ctx_sha_init(tfm); @@ -40,11 +42,10 @@ static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) return 0; } -static int nx_sha256_init(struct shash_desc *desc) { +static int nx_sha256_init(struct shash_desc *desc) +{ struct sha256_state_be *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof *sctx); - sctx->state[0] = __cpu_to_be32(SHA256_H0); sctx->state[1] = __cpu_to_be32(SHA256_H1); sctx->state[2] = __cpu_to_be32(SHA256_H2); @@ -61,30 +62,18 @@ static int nx_sha256_init(struct shash_desc *desc) { static int nx_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha256_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; struct nx_sg *out_sg; - u64 to_process = 0, leftover, total; unsigned long irq_flags; int rc = 0; int data_len; u32 max_sg_len; - u64 buf_len = (sctx->count % SHA256_BLOCK_SIZE); spin_lock_irqsave(&nx_ctx->lock, irq_flags); - /* 2 cases for total data len: - * 1: < SHA256_BLOCK_SIZE: copy into state, return 0 - * 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover - */ - total = (sctx->count % SHA256_BLOCK_SIZE) + len; - if (total < SHA256_BLOCK_SIZE) { - memcpy(sctx->buf + buf_len, data, len); - sctx->count += len; - goto out; - } - memcpy(csbcpb->cpb.sha256.message_digest, sctx->state, SHA256_DIGEST_SIZE); NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; @@ -105,41 +94,17 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, } do { - int used_sgs = 0; struct nx_sg *in_sg = nx_ctx->in_sg; - if (buf_len) { - data_len = buf_len; - in_sg = nx_build_sg_list(in_sg, - (u8 *) sctx->buf, - &data_len, - max_sg_len); - - if (data_len != buf_len) { - rc = -EINVAL; - goto out; - } - used_sgs = in_sg - nx_ctx->in_sg; - } + to_process = total & ~(SHA256_BLOCK_SIZE - 1); - /* to_process: SHA256_BLOCK_SIZE aligned chunk to be - * processed in this iteration. This value is restricted - * by sg list limits and number of sgs we already used - * for leftover data. (see above) - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, - * but because data may not be aligned, we need to account - * for that too. */ - to_process = min_t(u64, total, - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); - to_process = to_process & ~(SHA256_BLOCK_SIZE - 1); - - data_len = to_process - buf_len; + data_len = to_process; in_sg = nx_build_sg_list(in_sg, (u8 *) data, &data_len, max_sg_len); nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); - to_process = data_len + buf_len; + to_process = data_len; leftover = total - to_process; /* @@ -162,26 +127,22 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, atomic_inc(&(nx_ctx->stats->sha256_ops)); total -= to_process; - data += to_process - buf_len; - buf_len = 0; - + data += to_process; + sctx->count += to_process; } while (leftover >= SHA256_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - if (leftover) - memcpy(sctx->buf, data, leftover); - - sctx->count += len; + rc = leftover; memcpy(sctx->state, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_sha256_final(struct shash_desc *desc, u8 *out) +static int nx_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha256_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; unsigned long irq_flags; @@ -197,25 +158,19 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out) nx_ctx->ap->databytelen/NX_PAGE_SIZE); /* final is represented by continuing the operation and indicating that - * this is not an intermediate operation */ - if (sctx->count >= SHA256_BLOCK_SIZE) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - } else { - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; - } + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + sctx->count += nbytes; csbcpb->cpb.sha256.message_bit_length = (u64) (sctx->count * 8); - len = sctx->count & (SHA256_BLOCK_SIZE - 1); - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) sctx->buf, - &len, max_sg_len); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); - if (len != (sctx->count & (SHA256_BLOCK_SIZE - 1))) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -251,18 +206,34 @@ out: static int nx_sha256_export(struct shash_desc *desc, void *out) { struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; - memcpy(out, sctx, sizeof(*sctx)); + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + put_unaligned(be32_to_cpu(sctx->state[i]), p.u32++); + put_unaligned(sctx->count, p.u64++); return 0; } static int nx_sha256_import(struct shash_desc *desc, const void *in) { struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; - memcpy(sctx, in, sizeof(*sctx)); + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + sctx->state[i] = cpu_to_be32(get_unaligned(p.u32++)); + sctx->count = get_unaligned(p.u64++); return 0; } @@ -270,19 +241,20 @@ struct shash_alg nx_shash_sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = nx_sha256_init, .update = nx_sha256_update, - .final = nx_sha256_final, + .finup = nx_sha256_finup, .export = nx_sha256_export, .import = nx_sha256_import, + .init_tfm = nx_crypto_ctx_sha256_init, + .exit_tfm = nx_crypto_ctx_shash_exit, .descsize = sizeof(struct sha256_state_be), .statesize = sizeof(struct sha256_state_be), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_sha256_init, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 1ffb40d2c324..f74776b7d7d7 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -9,8 +9,12 @@ #include <crypto/internal/hash.h> #include <crypto/sha2.h> +#include <linux/errno.h> +#include <linux/kernel.h> #include <linux/module.h> -#include <asm/vio.h> +#include <linux/spinlock.h> +#include <linux/string.h> +#include <linux/unaligned.h> #include "nx_csbcpb.h" #include "nx.h" @@ -18,12 +22,11 @@ struct sha512_state_be { __be64 state[SHA512_DIGEST_SIZE / 8]; u64 count[2]; - u8 buf[SHA512_BLOCK_SIZE]; }; -static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm) +static int nx_crypto_ctx_sha512_init(struct crypto_shash *tfm) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); int err; err = nx_crypto_ctx_sha_init(tfm); @@ -43,8 +46,6 @@ static int nx_sha512_init(struct shash_desc *desc) { struct sha512_state_be *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof *sctx); - sctx->state[0] = __cpu_to_be64(SHA512_H0); sctx->state[1] = __cpu_to_be64(SHA512_H1); sctx->state[2] = __cpu_to_be64(SHA512_H2); @@ -54,6 +55,7 @@ static int nx_sha512_init(struct shash_desc *desc) sctx->state[6] = __cpu_to_be64(SHA512_H6); sctx->state[7] = __cpu_to_be64(SHA512_H7); sctx->count[0] = 0; + sctx->count[1] = 0; return 0; } @@ -61,30 +63,18 @@ static int nx_sha512_init(struct shash_desc *desc) static int nx_sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct sha512_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; struct nx_sg *out_sg; - u64 to_process, leftover = 0, total; unsigned long irq_flags; int rc = 0; int data_len; u32 max_sg_len; - u64 buf_len = (sctx->count[0] % SHA512_BLOCK_SIZE); spin_lock_irqsave(&nx_ctx->lock, irq_flags); - /* 2 cases for total data len: - * 1: < SHA512_BLOCK_SIZE: copy into state, return 0 - * 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover - */ - total = (sctx->count[0] % SHA512_BLOCK_SIZE) + len; - if (total < SHA512_BLOCK_SIZE) { - memcpy(sctx->buf + buf_len, data, len); - sctx->count[0] += len; - goto out; - } - memcpy(csbcpb->cpb.sha512.message_digest, sctx->state, SHA512_DIGEST_SIZE); NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; @@ -105,45 +95,17 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, } do { - int used_sgs = 0; struct nx_sg *in_sg = nx_ctx->in_sg; - if (buf_len) { - data_len = buf_len; - in_sg = nx_build_sg_list(in_sg, - (u8 *) sctx->buf, - &data_len, max_sg_len); - - if (data_len != buf_len) { - rc = -EINVAL; - goto out; - } - used_sgs = in_sg - nx_ctx->in_sg; - } + to_process = total & ~(SHA512_BLOCK_SIZE - 1); - /* to_process: SHA512_BLOCK_SIZE aligned chunk to be - * processed in this iteration. This value is restricted - * by sg list limits and number of sgs we already used - * for leftover data. (see above) - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, - * but because data may not be aligned, we need to account - * for that too. */ - to_process = min_t(u64, total, - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); - to_process = to_process & ~(SHA512_BLOCK_SIZE - 1); - - data_len = to_process - buf_len; + data_len = to_process; in_sg = nx_build_sg_list(in_sg, (u8 *) data, &data_len, max_sg_len); nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); - if (data_len != (to_process - buf_len)) { - rc = -EINVAL; - goto out; - } - - to_process = data_len + buf_len; + to_process = data_len; leftover = total - to_process; /* @@ -166,30 +128,29 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, atomic_inc(&(nx_ctx->stats->sha512_ops)); total -= to_process; - data += to_process - buf_len; - buf_len = 0; - + data += to_process; + sctx->count[0] += to_process; + if (sctx->count[0] < to_process) + sctx->count[1]++; } while (leftover >= SHA512_BLOCK_SIZE); - /* copy the leftover back into the state struct */ - if (leftover) - memcpy(sctx->buf, data, leftover); - sctx->count[0] += len; + rc = leftover; memcpy(sctx->state, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); return rc; } -static int nx_sha512_final(struct shash_desc *desc, u8 *out) +static int nx_sha512_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) { struct sha512_state_be *sctx = shash_desc_ctx(desc); - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; u32 max_sg_len; - u64 count0; unsigned long irq_flags; + u64 count0, count1; int rc = 0; int len; @@ -201,30 +162,23 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) nx_ctx->ap->databytelen/NX_PAGE_SIZE); /* final is represented by continuing the operation and indicating that - * this is not an intermediate operation */ - if (sctx->count[0] >= SHA512_BLOCK_SIZE) { - /* we've hit the nx chip previously, now we're finalizing, - * so copy over the partial digest */ - memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, - SHA512_DIGEST_SIZE); - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - } else { - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; - } - + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, SHA512_DIGEST_SIZE); NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; - count0 = sctx->count[0] * 8; + count0 = sctx->count[0] + nbytes; + count1 = sctx->count[1]; - csbcpb->cpb.sha512.message_bit_length_lo = count0; + csbcpb->cpb.sha512.message_bit_length_lo = count0 << 3; + csbcpb->cpb.sha512.message_bit_length_hi = (count1 << 3) | + (count0 >> 61); - len = sctx->count[0] & (SHA512_BLOCK_SIZE - 1); - in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buf, &len, - max_sg_len); + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); - if (len != (sctx->count[0] & (SHA512_BLOCK_SIZE - 1))) { + if (len != nbytes) { rc = -EINVAL; goto out; } @@ -246,7 +200,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) goto out; atomic_inc(&(nx_ctx->stats->sha512_ops)); - atomic64_add(sctx->count[0], &(nx_ctx->stats->sha512_bytes)); + atomic64_add(count0, &(nx_ctx->stats->sha512_bytes)); memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); out: @@ -257,18 +211,34 @@ out: static int nx_sha512_export(struct shash_desc *desc, void *out) { struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u64 *u64; + } p = { .u8 = out }; + int i; - memcpy(out, sctx, sizeof(*sctx)); + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + put_unaligned(be64_to_cpu(sctx->state[i]), p.u64++); + put_unaligned(sctx->count[0], p.u64++); + put_unaligned(sctx->count[1], p.u64++); return 0; } static int nx_sha512_import(struct shash_desc *desc, const void *in) { struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u64 *u64; + } p = { .u8 = in }; + int i; - memcpy(sctx, in, sizeof(*sctx)); + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + sctx->state[i] = cpu_to_be64(get_unaligned(p.u64++)); + sctx->count[0] = get_unaligned(p.u64++); + sctx->count[1] = get_unaligned(p.u64++); return 0; } @@ -276,19 +246,20 @@ struct shash_alg nx_shash_sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, .init = nx_sha512_init, .update = nx_sha512_update, - .final = nx_sha512_final, + .finup = nx_sha512_finup, .export = nx_sha512_export, .import = nx_sha512_import, + .init_tfm = nx_crypto_ctx_sha512_init, + .exit_tfm = nx_crypto_ctx_shash_exit, .descsize = sizeof(struct sha512_state_be), .statesize = sizeof(struct sha512_state_be), .base = { .cra_name = "sha512", .cra_driver_name = "sha512-nx", .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_init = nx_crypto_ctx_sha512_init, - .cra_exit = nx_crypto_ctx_exit, } }; diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 010e87d9da36..78135fb13f5c 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -7,11 +7,11 @@ * Author: Kent Yoder <yoder1@us.ibm.com> */ +#include <crypto/aes.h> #include <crypto/internal/aead.h> #include <crypto/internal/hash.h> -#include <crypto/aes.h> +#include <crypto/internal/skcipher.h> #include <crypto/sha2.h> -#include <crypto/algapi.h> #include <crypto/scatterwalk.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -124,8 +124,6 @@ struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head, } if ((sg - sg_head) == sgmax) { - pr_err("nx: scatter/gather list overflow, pid: %d\n", - current->pid); sg++; break; } @@ -153,40 +151,18 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, { struct scatter_walk walk; struct nx_sg *nx_sg = nx_dst; - unsigned int n, offset = 0, len = *src_len; - char *dst; + unsigned int n, len = *src_len; /* we need to fast forward through @start bytes first */ - for (;;) { - scatterwalk_start(&walk, sg_src); - - if (start < offset + sg_src->length) - break; - - offset += sg_src->length; - sg_src = sg_next(sg_src); - } - - /* start - offset is the number of bytes to advance in the scatterlist - * element we're currently looking at */ - scatterwalk_advance(&walk, start - offset); + scatterwalk_start_at_pos(&walk, sg_src, start); while (len && (nx_sg - nx_dst) < sglen) { - n = scatterwalk_clamp(&walk, len); - if (!n) { - /* In cases where we have scatterlist chain sg_next - * handles with it properly */ - scatterwalk_start(&walk, sg_next(walk.sg)); - n = scatterwalk_clamp(&walk, len); - } - dst = scatterwalk_map(&walk); + n = scatterwalk_next(&walk, len); - nx_sg = nx_build_sg_list(nx_sg, dst, &n, sglen - (nx_sg - nx_dst)); - len -= n; + nx_sg = nx_build_sg_list(nx_sg, walk.addr, &n, sglen - (nx_sg - nx_dst)); - scatterwalk_unmap(dst); - scatterwalk_advance(&walk, n); - scatterwalk_done(&walk, SCATTERWALK_FROM_SG, len); + scatterwalk_done_src(&walk, n); + len -= n; } /* update to_process */ *src_len -= len; @@ -724,14 +700,14 @@ int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm) NX_MODE_AES_ECB); } -int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm) { - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); } -int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm) { - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES, + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_AES, NX_MODE_AES_XCBC_MAC); } @@ -766,6 +742,11 @@ void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm) kfree_sensitive(nx_ctx->kmem); } +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm) +{ + nx_crypto_ctx_exit(crypto_shash_ctx(tfm)); +} + static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id) { dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n", diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index 2697baebb6a3..36974f08490a 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -3,7 +3,11 @@ #ifndef __NX_H__ #define __NX_H__ +#include <asm/vio.h> #include <crypto/ctr.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> #define NX_NAME "nx-crypto" #define NX_STRING "IBM Power7+ Nest Accelerator Crypto Driver" @@ -139,19 +143,20 @@ struct nx_crypto_ctx { } priv; }; -struct crypto_aead; +struct scatterlist; /* prototypes */ int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm); -int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm); int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm); -int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm); void nx_crypto_ctx_exit(struct crypto_tfm *tfm); void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm); void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm); void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function); int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op, u32 may_sleep); @@ -189,7 +194,4 @@ extern struct shash_alg nx_shash_sha256_alg; extern struct nx_crypto_driver nx_driver; -#define SCATTERWALK_TO_SG 1 -#define SCATTERWALK_FROM_SG 0 - #endif diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index c498950402e8..1f4586509ca4 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -38,7 +38,6 @@ static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) crypto_finalize_aead_request(dd->engine, req, ret); - pm_runtime_mark_last_busy(dd->dev); pm_runtime_put_autosuspend(dd->dev); } diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index e27b84616743..3cc802622dd5 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -18,7 +18,6 @@ #include <crypto/internal/aead.h> #include <crypto/internal/engine.h> #include <crypto/internal/skcipher.h> -#include <crypto/scatterwalk.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/err.h> @@ -33,6 +32,7 @@ #include <linux/pm_runtime.h> #include <linux/scatterlist.h> #include <linux/string.h> +#include <linux/workqueue.h> #include "omap-crypto.h" #include "omap-aes.h" @@ -222,7 +222,7 @@ static void omap_aes_dma_out_callback(void *data) struct omap_aes_dev *dd = data; /* dma_lch_out - completed */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); } static int omap_aes_dma_init(struct omap_aes_dev *dd) @@ -272,9 +272,9 @@ static int omap_aes_crypt_dma(struct omap_aes_dev *dd, int ret; if (dd->pio_only) { - scatterwalk_start(&dd->in_walk, dd->in_sg); + dd->in_sg_offset = 0; if (out_sg_len) - scatterwalk_start(&dd->out_walk, dd->out_sg); + dd->out_sg_offset = 0; /* Enable DATAIN interrupt and let it take care of the rest */ @@ -401,7 +401,6 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) crypto_finalize_skcipher_request(dd->engine, req, err); - pm_runtime_mark_last_busy(dd->dev); pm_runtime_put_autosuspend(dd->dev); } @@ -496,9 +495,9 @@ static void omap_aes_copy_ivout(struct omap_aes_dev *dd, u8 *ivbuf) ((u32 *)ivbuf)[i] = omap_aes_read(dd, AES_REG_IV(dd, i)); } -static void omap_aes_done_task(unsigned long data) +static void omap_aes_done_task(struct work_struct *t) { - struct omap_aes_dev *dd = (struct omap_aes_dev *)data; + struct omap_aes_dev *dd = from_work(dd, t, done_task); pr_debug("enter done_task\n"); @@ -871,21 +870,18 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) BUG_ON(!dd->in_sg); - BUG_ON(_calc_walked(in) > dd->in_sg->length); + BUG_ON(dd->in_sg_offset > dd->in_sg->length); - src = sg_virt(dd->in_sg) + _calc_walked(in); + src = sg_virt(dd->in_sg) + dd->in_sg_offset; for (i = 0; i < AES_BLOCK_WORDS; i++) { omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src); - - scatterwalk_advance(&dd->in_walk, 4); - if (dd->in_sg->length == _calc_walked(in)) { + dd->in_sg_offset += 4; + if (dd->in_sg_offset == dd->in_sg->length) { dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { - scatterwalk_start(&dd->in_walk, - dd->in_sg); - src = sg_virt(dd->in_sg) + - _calc_walked(in); + dd->in_sg_offset = 0; + src = sg_virt(dd->in_sg); } } else { src++; @@ -904,20 +900,18 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) BUG_ON(!dd->out_sg); - BUG_ON(_calc_walked(out) > dd->out_sg->length); + BUG_ON(dd->out_sg_offset > dd->out_sg->length); - dst = sg_virt(dd->out_sg) + _calc_walked(out); + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; for (i = 0; i < AES_BLOCK_WORDS; i++) { *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i)); - scatterwalk_advance(&dd->out_walk, 4); - if (dd->out_sg->length == _calc_walked(out)) { + dd->out_sg_offset += 4; + if (dd->out_sg_offset == dd->out_sg->length) { dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { - scatterwalk_start(&dd->out_walk, - dd->out_sg); - dst = sg_virt(dd->out_sg) + - _calc_walked(out); + dd->out_sg_offset = 0; + dst = sg_virt(dd->out_sg); } } else { dst++; @@ -932,7 +926,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) if (!dd->total) /* All bytes read! */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); else /* Enable DATA_IN interrupt for next block */ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2); @@ -1092,10 +1086,7 @@ static struct attribute *omap_aes_attrs[] = { &dev_attr_fallback.attr, NULL, }; - -static const struct attribute_group omap_aes_attr_group = { - .attrs = omap_aes_attrs, -}; +ATTRIBUTE_GROUPS(omap_aes); static int omap_aes_probe(struct platform_device *pdev) { @@ -1150,7 +1141,7 @@ static int omap_aes_probe(struct platform_device *pdev) (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); - tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd); + INIT_WORK(&dd->done_task, omap_aes_done_task); err = omap_aes_dma_init(dd); if (err == -EPROBE_DEFER) { @@ -1221,12 +1212,6 @@ static int omap_aes_probe(struct platform_device *pdev) } } - err = sysfs_create_group(&dev->kobj, &omap_aes_attr_group); - if (err) { - dev_err(dev, "could not create sysfs device attrs\n"); - goto err_aead_algs; - } - return 0; err_aead_algs: for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { @@ -1245,7 +1230,7 @@ err_engine: omap_aes_dma_cleanup(dd); err_irq: - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); err_pm_disable: pm_runtime_disable(dev); err_res: @@ -1280,11 +1265,9 @@ static void omap_aes_remove(struct platform_device *pdev) crypto_engine_exit(dd->engine); - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); omap_aes_dma_cleanup(dd); pm_runtime_disable(dd->dev); - - sysfs_remove_group(&dd->dev->kobj, &omap_aes_attr_group); } #ifdef CONFIG_PM_SLEEP @@ -1310,6 +1293,7 @@ static struct platform_driver omap_aes_driver = { .name = "omap-aes", .pm = &omap_aes_pm_ops, .of_match_table = omap_aes_of_match, + .dev_groups = omap_aes_groups, }, }; diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h index 0f35c9164764..99c36a777e97 100644 --- a/drivers/crypto/omap-aes.h +++ b/drivers/crypto/omap-aes.h @@ -14,8 +14,6 @@ #define DST_MAXBURST 4 #define DMA_MIN (DST_MAXBURST * sizeof(u32)) -#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) - /* * OMAP TRM gives bitfields as start:end, where start is the higher bit * number. For example 7:0 @@ -161,7 +159,7 @@ struct omap_aes_dev { unsigned long flags; int err; - struct tasklet_struct done_task; + struct work_struct done_task; struct aead_queue aead_queue; spinlock_t lock; @@ -186,8 +184,8 @@ struct omap_aes_dev { struct scatterlist out_sgl; struct scatterlist *orig_out; - struct scatter_walk in_walk; - struct scatter_walk out_walk; + unsigned int in_sg_offset; + unsigned int out_sg_offset; struct dma_chan *dma_lch_in; struct dma_chan *dma_lch_out; int in_sg_len; diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 498cbd585ed1..149ebd77710b 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -19,7 +19,6 @@ #include <crypto/engine.h> #include <crypto/internal/des.h> #include <crypto/internal/skcipher.h> -#include <crypto/scatterwalk.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/err.h> @@ -33,6 +32,7 @@ #include <linux/pm_runtime.h> #include <linux/scatterlist.h> #include <linux/string.h> +#include <linux/workqueue.h> #include "omap-crypto.h" @@ -40,8 +40,6 @@ #define DES_BLOCK_WORDS (DES_BLOCK_SIZE >> 2) -#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) - #define DES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ ((x ^ 0x01) * 0x04)) @@ -133,7 +131,7 @@ struct omap_des_dev { unsigned long flags; int err; - struct tasklet_struct done_task; + struct work_struct done_task; struct skcipher_request *req; struct crypto_engine *engine; @@ -152,8 +150,8 @@ struct omap_des_dev { struct scatterlist out_sgl; struct scatterlist *orig_out; - struct scatter_walk in_walk; - struct scatter_walk out_walk; + unsigned int in_sg_offset; + unsigned int out_sg_offset; struct dma_chan *dma_lch_in; struct dma_chan *dma_lch_out; int in_sg_len; @@ -328,7 +326,7 @@ static void omap_des_dma_out_callback(void *data) struct omap_des_dev *dd = data; /* dma_lch_out - completed */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); } static int omap_des_dma_init(struct omap_des_dev *dd) @@ -379,8 +377,8 @@ static int omap_des_crypt_dma(struct crypto_tfm *tfm, int ret; if (dd->pio_only) { - scatterwalk_start(&dd->in_walk, dd->in_sg); - scatterwalk_start(&dd->out_walk, dd->out_sg); + dd->in_sg_offset = 0; + dd->out_sg_offset = 0; /* Enable DATAIN interrupt and let it take care of the rest */ @@ -492,7 +490,6 @@ static void omap_des_finish_req(struct omap_des_dev *dd, int err) crypto_finalize_skcipher_request(dd->engine, req, err); - pm_runtime_mark_last_busy(dd->dev); pm_runtime_put_autosuspend(dd->dev); } @@ -584,9 +581,9 @@ static int omap_des_crypt_req(struct crypto_engine *engine, omap_des_crypt_dma_start(dd); } -static void omap_des_done_task(unsigned long data) +static void omap_des_done_task(struct work_struct *t) { - struct omap_des_dev *dd = (struct omap_des_dev *)data; + struct omap_des_dev *dd = from_work(dd, t, done_task); int i; pr_debug("enter done_task\n"); @@ -836,21 +833,18 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) BUG_ON(!dd->in_sg); - BUG_ON(_calc_walked(in) > dd->in_sg->length); + BUG_ON(dd->in_sg_offset > dd->in_sg->length); - src = sg_virt(dd->in_sg) + _calc_walked(in); + src = sg_virt(dd->in_sg) + dd->in_sg_offset; for (i = 0; i < DES_BLOCK_WORDS; i++) { omap_des_write(dd, DES_REG_DATA_N(dd, i), *src); - - scatterwalk_advance(&dd->in_walk, 4); - if (dd->in_sg->length == _calc_walked(in)) { + dd->in_sg_offset += 4; + if (dd->in_sg_offset == dd->in_sg->length) { dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { - scatterwalk_start(&dd->in_walk, - dd->in_sg); - src = sg_virt(dd->in_sg) + - _calc_walked(in); + dd->in_sg_offset = 0; + src = sg_virt(dd->in_sg); } } else { src++; @@ -869,20 +863,18 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) BUG_ON(!dd->out_sg); - BUG_ON(_calc_walked(out) > dd->out_sg->length); + BUG_ON(dd->out_sg_offset > dd->out_sg->length); - dst = sg_virt(dd->out_sg) + _calc_walked(out); + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; for (i = 0; i < DES_BLOCK_WORDS; i++) { *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i)); - scatterwalk_advance(&dd->out_walk, 4); - if (dd->out_sg->length == _calc_walked(out)) { + dd->out_sg_offset += 4; + if (dd->out_sg_offset == dd->out_sg->length) { dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { - scatterwalk_start(&dd->out_walk, - dd->out_sg); - dst = sg_virt(dd->out_sg) + - _calc_walked(out); + dd->out_sg_offset = 0; + dst = sg_virt(dd->out_sg); } } else { dst++; @@ -899,7 +891,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) if (!dd->total) /* All bytes read! */ - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); else /* Enable DATA_IN interrupt for next block */ omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2); @@ -995,7 +987,7 @@ static int omap_des_probe(struct platform_device *pdev) (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); - tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd); + INIT_WORK(&dd->done_task, omap_des_done_task); err = omap_des_dma_init(dd); if (err == -EPROBE_DEFER) { @@ -1062,7 +1054,7 @@ err_engine: omap_des_dma_cleanup(dd); err_irq: - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); err_get: pm_runtime_disable(dev); err_res: @@ -1086,7 +1078,7 @@ static void omap_des_remove(struct platform_device *pdev) crypto_engine_unregister_skcipher( &dd->pdata->algs_info[i].algs_list[j]); - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); omap_des_dma_cleanup(dd); pm_runtime_disable(dd->dev); } diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 7021481bf027..ff8aac02994a 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -37,6 +37,7 @@ #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/workqueue.h> #define MD5_DIGEST_SIZE 16 @@ -217,7 +218,7 @@ struct omap_sham_dev { int irq; int err; struct dma_chan *dma_lch; - struct tasklet_struct done_task; + struct work_struct done_task; u8 polling_mode; u8 xmit_buf[BUFLEN] OMAP_ALIGNED; @@ -561,7 +562,7 @@ static void omap_sham_dma_callback(void *param) struct omap_sham_dev *dd = param; set_bit(FLAGS_DMA_READY, &dd->flags); - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); } static int omap_sham_xmit_dma(struct omap_sham_dev *dd, size_t length, @@ -1167,7 +1168,6 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); - pm_runtime_mark_last_busy(dd->dev); pm_runtime_put_autosuspend(dd->dev); ctx->offset = 0; @@ -1704,9 +1704,9 @@ static struct ahash_engine_alg algs_sha384_sha512[] = { }, }; -static void omap_sham_done_task(unsigned long data) +static void omap_sham_done_task(struct work_struct *t) { - struct omap_sham_dev *dd = (struct omap_sham_dev *)data; + struct omap_sham_dev *dd = from_work(dd, t, done_task); int err = 0; dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags); @@ -1740,7 +1740,7 @@ finish: static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd) { set_bit(FLAGS_OUTPUT_READY, &dd->flags); - tasklet_schedule(&dd->done_task); + queue_work(system_bh_wq, &dd->done_task); return IRQ_HANDLED; } @@ -2039,10 +2039,7 @@ static struct attribute *omap_sham_attrs[] = { &dev_attr_fallback.attr, NULL, }; - -static const struct attribute_group omap_sham_attr_group = { - .attrs = omap_sham_attrs, -}; +ATTRIBUTE_GROUPS(omap_sham); static int omap_sham_probe(struct platform_device *pdev) { @@ -2063,7 +2060,7 @@ static int omap_sham_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dd); INIT_LIST_HEAD(&dd->list); - tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); + INIT_WORK(&dd->done_task, omap_sham_done_task); crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) : @@ -2158,12 +2155,6 @@ static int omap_sham_probe(struct platform_device *pdev) } } - err = sysfs_create_group(&dev->kobj, &omap_sham_attr_group); - if (err) { - dev_err(dev, "could not create sysfs device attrs\n"); - goto err_algs; - } - return 0; err_algs: @@ -2204,14 +2195,12 @@ static void omap_sham_remove(struct platform_device *pdev) &dd->pdata->algs_info[i].algs_list[j]); dd->pdata->algs_info[i].registered--; } - tasklet_kill(&dd->done_task); + cancel_work_sync(&dd->done_task); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); if (!dd->polling_mode) dma_release_channel(dd->dma_lch); - - sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group); } static struct platform_driver omap_sham_driver = { @@ -2220,6 +2209,7 @@ static struct platform_driver omap_sham_driver = { .driver = { .name = "omap-sham", .of_match_table = omap_sham_of_match, + .dev_groups = omap_sham_groups, }, }; diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index 6865c7f1fc1a..329f60ad422e 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -7,59 +7,89 @@ * Copyright (c) 2006 Michal Ludvig <michal@logix.cz> */ +#include <asm/cpu_device_id.h> #include <crypto/internal/hash.h> #include <crypto/padlock.h> #include <crypto/sha1.h> #include <crypto/sha2.h> +#include <linux/cpufeature.h> #include <linux/err.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/errno.h> -#include <linux/interrupt.h> #include <linux/kernel.h> -#include <linux/scatterlist.h> -#include <asm/cpu_device_id.h> -#include <asm/fpu/api.h> +#include <linux/module.h> -struct padlock_sha_desc { - struct shash_desc fallback; -}; +#define PADLOCK_SHA_DESCSIZE (128 + ((PADLOCK_ALIGNMENT - 1) & \ + ~(CRYPTO_MINALIGN - 1))) struct padlock_sha_ctx { - struct crypto_shash *fallback; + struct crypto_ahash *fallback; }; -static int padlock_sha_init(struct shash_desc *desc) +static inline void *padlock_shash_desc_ctx(struct shash_desc *desc) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + return PTR_ALIGN(shash_desc_ctx(desc), PADLOCK_ALIGNMENT); +} + +static int padlock_sha1_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = padlock_shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static int padlock_sha256_init(struct shash_desc *desc) +{ + struct crypto_sha256_state *sctx = padlock_shash_desc_ctx(desc); - dctx->fallback.tfm = ctx->fallback; - return crypto_shash_init(&dctx->fallback); + sha256_block_init(sctx); + return 0; } static int padlock_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); + u8 *state = padlock_shash_desc_ctx(desc); + struct crypto_shash *tfm = desc->tfm; + int err, remain; + + remain = length - round_down(length, crypto_shash_blocksize(tfm)); + { + struct padlock_sha_ctx *ctx = crypto_shash_ctx(tfm); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, data, NULL, length - remain); + err = crypto_ahash_import_core(req, state) ?: + crypto_ahash_update(req) ?: + crypto_ahash_export_core(req, state); + HASH_REQUEST_ZERO(req); + } - return crypto_shash_update(&dctx->fallback, data, length); + return err ?: remain; } static int padlock_sha_export(struct shash_desc *desc, void *out) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - - return crypto_shash_export(&dctx->fallback, out); + memcpy(out, padlock_shash_desc_ctx(desc), + crypto_shash_coresize(desc->tfm)); + return 0; } static int padlock_sha_import(struct shash_desc *desc, const void *in) { - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + unsigned int ss = crypto_shash_coresize(desc->tfm); + u64 *state = padlock_shash_desc_ctx(desc); + + memcpy(state, in, ss); + + /* Stop evil imports from generating a fault. */ + state[ss / 8 - 1] &= ~(bs - 1); - dctx->fallback.tfm = ctx->fallback; - return crypto_shash_import(&dctx->fallback, in); + return 0; } static inline void padlock_output_block(uint32_t *src, @@ -69,65 +99,38 @@ static inline void padlock_output_block(uint32_t *src, *dst++ = swab32(*src++); } +static int padlock_sha_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) +{ + struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, in, out, count); + return crypto_ahash_import_core(req, padlock_shash_desc_ctx(desc)) ?: + crypto_ahash_finup(req); +} + static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, unsigned int count, u8 *out) { /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct sha1_state state; - unsigned int space; - unsigned int leftover; - int err; - - err = crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; + struct sha1_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; - if (state.count + count > ULONG_MAX) - return crypto_shash_finup(&dctx->fallback, in, count, out); - - leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1; - space = SHA1_BLOCK_SIZE - leftover; - if (space) { - if (count > space) { - err = crypto_shash_update(&dctx->fallback, in, space) ?: - crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - count -= space; - in += space; - } else { - memcpy(state.buffer + leftover, in, count); - in = state.buffer; - count += leftover; - state.count &= ~(SHA1_BLOCK_SIZE - 1); - } - } - - memcpy(result, &state.state, SHA1_DIGEST_SIZE); + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */ : \ - : "c"((unsigned long)state.count + count), \ - "a"((unsigned long)state.count), \ - "S"(in), "D"(result)); - - padlock_output_block((uint32_t *)result, (uint32_t *)out, 5); + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); -out: - return err; -} - -static int padlock_sha1_final(struct shash_desc *desc, u8 *out) -{ - u8 buf[4]; - - return padlock_sha1_finup(desc, buf, 0, out); + padlock_output_block(state->state, (uint32_t *)out, 5); + return 0; } static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, @@ -136,78 +139,46 @@ static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - struct sha256_state state; - unsigned int space; - unsigned int leftover; - int err; - - err = crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; + struct sha256_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; - if (state.count + count > ULONG_MAX) - return crypto_shash_finup(&dctx->fallback, in, count, out); - - leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1; - space = SHA256_BLOCK_SIZE - leftover; - if (space) { - if (count > space) { - err = crypto_shash_update(&dctx->fallback, in, space) ?: - crypto_shash_export(&dctx->fallback, &state); - if (err) - goto out; - count -= space; - in += space; - } else { - memcpy(state.buf + leftover, in, count); - in = state.buf; - count += leftover; - state.count &= ~(SHA1_BLOCK_SIZE - 1); - } - } - - memcpy(result, &state.state, SHA256_DIGEST_SIZE); + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */ : \ - : "c"((unsigned long)state.count + count), \ - "a"((unsigned long)state.count), \ - "S"(in), "D"(result)); + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); - padlock_output_block((uint32_t *)result, (uint32_t *)out, 8); - -out: - return err; -} - -static int padlock_sha256_final(struct shash_desc *desc, u8 *out) -{ - u8 buf[4]; - - return padlock_sha256_finup(desc, buf, 0, out); + padlock_output_block(state->state, (uint32_t *)out, 8); + return 0; } static int padlock_init_tfm(struct crypto_shash *hash) { const char *fallback_driver_name = crypto_shash_alg_name(hash); struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); - struct crypto_shash *fallback_tfm; + struct crypto_ahash *fallback_tfm; /* Allocate a fallback and abort if it failed. */ - fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); + fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback_tfm)) { printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n", fallback_driver_name); return PTR_ERR(fallback_tfm); } + if (crypto_shash_statesize(hash) != + crypto_ahash_statesize(fallback_tfm)) { + crypto_free_ahash(fallback_tfm); + return -EINVAL; + } + ctx->fallback = fallback_tfm; - hash->descsize += crypto_shash_descsize(fallback_tfm); + return 0; } @@ -215,26 +186,27 @@ static void padlock_exit_tfm(struct crypto_shash *hash) { struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); - crypto_free_shash(ctx->fallback); + crypto_free_ahash(ctx->fallback); } static struct shash_alg sha1_alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = padlock_sha_init, + .init = padlock_sha1_init, .update = padlock_sha_update, .finup = padlock_sha1_finup, - .final = padlock_sha1_final, .export = padlock_sha_export, .import = padlock_sha_import, .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc), - .statesize = sizeof(struct sha1_state), + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-padlock", .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct padlock_sha_ctx), .cra_module = THIS_MODULE, @@ -243,21 +215,22 @@ static struct shash_alg sha1_alg = { static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, - .init = padlock_sha_init, + .init = padlock_sha256_init, .update = padlock_sha_update, .finup = padlock_sha256_finup, - .final = padlock_sha256_final, + .init_tfm = padlock_init_tfm, .export = padlock_sha_export, .import = padlock_sha_import, - .init_tfm = padlock_init_tfm, .exit_tfm = padlock_exit_tfm, - .descsize = sizeof(struct padlock_sha_desc), - .statesize = sizeof(struct sha256_state), + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-padlock", .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct padlock_sha_ctx), .cra_module = THIS_MODULE, @@ -266,207 +239,58 @@ static struct shash_alg sha256_alg = { /* Add two shash_alg instance for hardware-implemented * * multiple-parts hash supported by VIA Nano Processor.*/ -static int padlock_sha1_init_nano(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} static int padlock_sha1_update_nano(struct shash_desc *desc, - const u8 *data, unsigned int len) + const u8 *src, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ - u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE); - - if ((partial + len) >= SHA1_BLOCK_SIZE) { - - /* Append the bytes in state's buffer to a block to handle */ - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, - done + SHA1_BLOCK_SIZE); - src = sctx->buffer; - asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" - : "+S"(src), "+D"(dst) \ - : "a"((long)-1), "c"((unsigned long)1)); - done += SHA1_BLOCK_SIZE; - src = data + done; - } - - /* Process the left bytes from the input data */ - if (len - done >= SHA1_BLOCK_SIZE) { - asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" - : "+S"(src), "+D"(dst) - : "a"((long)-1), - "c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE))); - done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE); - src = data + done; - } - partial = 0; - } - memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE); - memcpy(sctx->buffer + partial, src, len - done); - - return 0; -} - -static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc); - unsigned int partial, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(state->count << 3); - - /* Pad out to 56 mod 64 */ - partial = state->count & 0x3f; - padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); - padlock_sha1_update_nano(desc, padding, padlen); - - /* Append length field bytes */ - padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits)); - - /* Swap to output */ - padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5); - - return 0; -} - -static int padlock_sha256_init_nano(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha256_state){ - .state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \ - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7}, - }; - - return 0; + struct sha1_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA1_BLOCK_SIZE; + + len -= blocks * SHA1_BLOCK_SIZE; + state->count += blocks * SHA1_BLOCK_SIZE; + + /* Process the left bytes from the input data */ + asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; } -static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data, +static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *src, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ - u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ - ((aligned(STACK_ALIGN))); - u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); - - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE); - - if ((partial + len) >= SHA256_BLOCK_SIZE) { - - /* Append the bytes in state's buffer to a block to handle */ - if (partial) { - done = -partial; - memcpy(sctx->buf + partial, data, - done + SHA256_BLOCK_SIZE); - src = sctx->buf; - asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" - : "+S"(src), "+D"(dst) - : "a"((long)-1), "c"((unsigned long)1)); - done += SHA256_BLOCK_SIZE; - src = data + done; - } - - /* Process the left bytes from input data*/ - if (len - done >= SHA256_BLOCK_SIZE) { - asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" - : "+S"(src), "+D"(dst) - : "a"((long)-1), - "c"((unsigned long)((len - done) / 64))); - done += ((len - done) - (len - done) % 64); - src = data + done; - } - partial = 0; - } - memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE); - memcpy(sctx->buf + partial, src, len - done); - - return 0; -} - -static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *state = - (struct sha256_state *)shash_desc_ctx(desc); - unsigned int partial, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(state->count << 3); - - /* Pad out to 56 mod 64 */ - partial = state->count & 0x3f; - padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); - padlock_sha256_update_nano(desc, padding, padlen); - - /* Append length field bytes */ - padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits)); - - /* Swap to output */ - padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8); - - return 0; -} - -static int padlock_sha_export_nano(struct shash_desc *desc, - void *out) -{ - int statesize = crypto_shash_statesize(desc->tfm); - void *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, statesize); - return 0; -} - -static int padlock_sha_import_nano(struct shash_desc *desc, - const void *in) -{ - int statesize = crypto_shash_statesize(desc->tfm); - void *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, statesize); - return 0; + struct crypto_sha256_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA256_BLOCK_SIZE; + + len -= blocks * SHA256_BLOCK_SIZE; + state->count += blocks * SHA256_BLOCK_SIZE; + + /* Process the left bytes from input data*/ + asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; } static struct shash_alg sha1_alg_nano = { .digestsize = SHA1_DIGEST_SIZE, - .init = padlock_sha1_init_nano, + .init = padlock_sha1_init, .update = padlock_sha1_update_nano, - .final = padlock_sha1_final_nano, - .export = padlock_sha_export_nano, - .import = padlock_sha_import_nano, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), + .finup = padlock_sha1_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, .base = { .cra_name = "sha1", .cra_driver_name = "sha1-padlock-nano", .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -474,17 +298,19 @@ static struct shash_alg sha1_alg_nano = { static struct shash_alg sha256_alg_nano = { .digestsize = SHA256_DIGEST_SIZE, - .init = padlock_sha256_init_nano, + .init = padlock_sha256_init, .update = padlock_sha256_update_nano, - .final = padlock_sha256_final_nano, - .export = padlock_sha_export_nano, - .import = padlock_sha_import_nano, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .finup = padlock_sha256_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-padlock-nano", .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c index 7d811728f047..97b56e92ea33 100644 --- a/drivers/crypto/qce/aead.c +++ b/drivers/crypto/qce/aead.c @@ -786,7 +786,7 @@ static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_devi alg->init = qce_aead_init; alg->exit = qce_aead_exit; - alg->base.cra_priority = 300; + alg->base.cra_priority = 275; alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | CRYPTO_ALG_KERN_DRIVER_ONLY | diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index e228a31fe28d..b966f3365b7d 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -3,14 +3,15 @@ * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. */ +#include <linux/cleanup.h> #include <linux/clk.h> +#include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/interconnect.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/mod_devicetable.h> #include <linux/platform_device.h> -#include <linux/spinlock.h> #include <linux/types.h> #include <crypto/algapi.h> #include <crypto/internal/hash.h> @@ -20,7 +21,6 @@ #include "sha.h" #include "aead.h" -#define QCE_MAJOR_VERSION5 0x05 #define QCE_QUEUE_LENGTH 1 #define QCE_DEFAULT_MEM_BANDWIDTH 393600 @@ -37,9 +37,10 @@ static const struct qce_algo_ops *qce_ops[] = { #endif }; -static void qce_unregister_algs(struct qce_device *qce) +static void qce_unregister_algs(void *data) { const struct qce_algo_ops *ops; + struct qce_device *qce = data; int i; for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { @@ -48,19 +49,22 @@ static void qce_unregister_algs(struct qce_device *qce) } } -static int qce_register_algs(struct qce_device *qce) +static int devm_qce_register_algs(struct qce_device *qce) { const struct qce_algo_ops *ops; - int i, ret = -ENODEV; + int i, j, ret = -ENODEV; for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { ops = qce_ops[i]; ret = ops->register_algs(qce); - if (ret) - break; + if (ret) { + for (j = i - 1; j >= 0; j--) + ops->unregister_algs(qce); + return ret; + } } - return ret; + return devm_add_action_or_reset(qce->dev, qce_unregister_algs, qce); } static int qce_handle_request(struct crypto_async_request *async_req) @@ -84,55 +88,49 @@ static int qce_handle_queue(struct qce_device *qce, struct crypto_async_request *req) { struct crypto_async_request *async_req, *backlog; - unsigned long flags; int ret = 0, err; - spin_lock_irqsave(&qce->lock, flags); + scoped_guard(mutex, &qce->lock) { + if (req) + ret = crypto_enqueue_request(&qce->queue, req); - if (req) - ret = crypto_enqueue_request(&qce->queue, req); + /* busy, do not dequeue request */ + if (qce->req) + return ret; - /* busy, do not dequeue request */ - if (qce->req) { - spin_unlock_irqrestore(&qce->lock, flags); - return ret; + backlog = crypto_get_backlog(&qce->queue); + async_req = crypto_dequeue_request(&qce->queue); + if (async_req) + qce->req = async_req; } - backlog = crypto_get_backlog(&qce->queue); - async_req = crypto_dequeue_request(&qce->queue); - if (async_req) - qce->req = async_req; - - spin_unlock_irqrestore(&qce->lock, flags); - if (!async_req) return ret; if (backlog) { - spin_lock_bh(&qce->lock); - crypto_request_complete(backlog, -EINPROGRESS); - spin_unlock_bh(&qce->lock); + scoped_guard(mutex, &qce->lock) + crypto_request_complete(backlog, -EINPROGRESS); } err = qce_handle_request(async_req); if (err) { qce->result = err; - tasklet_schedule(&qce->done_tasklet); + schedule_work(&qce->done_work); } return ret; } -static void qce_tasklet_req_done(unsigned long data) +static void qce_req_done_work(struct work_struct *work) { - struct qce_device *qce = (struct qce_device *)data; + struct qce_device *qce = container_of(work, struct qce_device, + done_work); struct crypto_async_request *req; - unsigned long flags; - spin_lock_irqsave(&qce->lock, flags); - req = qce->req; - qce->req = NULL; - spin_unlock_irqrestore(&qce->lock, flags); + scoped_guard(mutex, &qce->lock) { + req = qce->req; + qce->req = NULL; + } if (req) crypto_request_complete(req, qce->result); @@ -149,7 +147,7 @@ static int qce_async_request_enqueue(struct qce_device *qce, static void qce_async_request_done(struct qce_device *qce, int ret) { qce->result = ret; - tasklet_schedule(&qce->done_tasklet); + schedule_work(&qce->done_work); } static int qce_check_version(struct qce_device *qce) @@ -162,7 +160,7 @@ static int qce_check_version(struct qce_device *qce) * the driver does not support v5 with minor 0 because it has special * alignment requirements. */ - if (major != QCE_MAJOR_VERSION5 || minor == 0) + if (major == 5 && minor == 0) return -ENODEV; qce->burst_size = QCE_BAM_BURST_SIZE; @@ -209,15 +207,15 @@ static int qce_crypto_probe(struct platform_device *pdev) if (ret < 0) return ret; - qce->core = devm_clk_get_optional(qce->dev, "core"); + qce->core = devm_clk_get_optional_enabled(qce->dev, "core"); if (IS_ERR(qce->core)) return PTR_ERR(qce->core); - qce->iface = devm_clk_get_optional(qce->dev, "iface"); + qce->iface = devm_clk_get_optional_enabled(qce->dev, "iface"); if (IS_ERR(qce->iface)) return PTR_ERR(qce->iface); - qce->bus = devm_clk_get_optional(qce->dev, "bus"); + qce->bus = devm_clk_get_optional_enabled(qce->dev, "bus"); if (IS_ERR(qce->bus)) return PTR_ERR(qce->bus); @@ -229,64 +227,25 @@ static int qce_crypto_probe(struct platform_device *pdev) if (ret) return ret; - ret = clk_prepare_enable(qce->core); + ret = devm_qce_dma_request(qce->dev, &qce->dma); if (ret) - goto err_mem_path_disable; - - ret = clk_prepare_enable(qce->iface); - if (ret) - goto err_clks_core; - - ret = clk_prepare_enable(qce->bus); - if (ret) - goto err_clks_iface; + return ret; - ret = qce_dma_request(qce->dev, &qce->dma); + ret = qce_check_version(qce); if (ret) - goto err_clks; + return ret; - ret = qce_check_version(qce); + ret = devm_mutex_init(qce->dev, &qce->lock); if (ret) - goto err_clks; + return ret; - spin_lock_init(&qce->lock); - tasklet_init(&qce->done_tasklet, qce_tasklet_req_done, - (unsigned long)qce); + INIT_WORK(&qce->done_work, qce_req_done_work); crypto_init_queue(&qce->queue, QCE_QUEUE_LENGTH); qce->async_req_enqueue = qce_async_request_enqueue; qce->async_req_done = qce_async_request_done; - ret = qce_register_algs(qce); - if (ret) - goto err_dma; - - return 0; - -err_dma: - qce_dma_release(&qce->dma); -err_clks: - clk_disable_unprepare(qce->bus); -err_clks_iface: - clk_disable_unprepare(qce->iface); -err_clks_core: - clk_disable_unprepare(qce->core); -err_mem_path_disable: - icc_set_bw(qce->mem_path, 0, 0); - - return ret; -} - -static void qce_crypto_remove(struct platform_device *pdev) -{ - struct qce_device *qce = platform_get_drvdata(pdev); - - tasklet_kill(&qce->done_tasklet); - qce_unregister_algs(qce); - qce_dma_release(&qce->dma); - clk_disable_unprepare(qce->bus); - clk_disable_unprepare(qce->iface); - clk_disable_unprepare(qce->core); + return devm_qce_register_algs(qce); } static const struct of_device_id qce_crypto_of_match[] = { @@ -299,7 +258,6 @@ MODULE_DEVICE_TABLE(of, qce_crypto_of_match); static struct platform_driver qce_crypto_driver = { .probe = qce_crypto_probe, - .remove = qce_crypto_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = qce_crypto_of_match, diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h index 228fcd69ec51..eb6fa7a8b64a 100644 --- a/drivers/crypto/qce/core.h +++ b/drivers/crypto/qce/core.h @@ -6,13 +6,16 @@ #ifndef _CORE_H_ #define _CORE_H_ +#include <linux/mutex.h> +#include <linux/workqueue.h> + #include "dma.h" /** * struct qce_device - crypto engine device structure * @queue: crypto request queue * @lock: the lock protects queue and req - * @done_tasklet: done tasklet object + * @done_work: workqueue context * @req: current active request * @result: result of current transform * @base: virtual IO base @@ -28,8 +31,8 @@ */ struct qce_device { struct crypto_queue queue; - spinlock_t lock; - struct tasklet_struct done_tasklet; + struct mutex lock; + struct work_struct done_work; struct crypto_async_request *req; int result; void __iomem *base; diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c index 46db5bf366b4..68cafd4741ad 100644 --- a/drivers/crypto/qce/dma.c +++ b/drivers/crypto/qce/dma.c @@ -3,22 +3,34 @@ * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. */ +#include <linux/device.h> #include <linux/dmaengine.h> #include <crypto/scatterwalk.h> #include "dma.h" -int qce_dma_request(struct device *dev, struct qce_dma_data *dma) +static void qce_dma_release(void *data) +{ + struct qce_dma_data *dma = data; + + dma_release_channel(dma->txchan); + dma_release_channel(dma->rxchan); + kfree(dma->result_buf); +} + +int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma) { int ret; dma->txchan = dma_request_chan(dev, "tx"); if (IS_ERR(dma->txchan)) - return PTR_ERR(dma->txchan); + return dev_err_probe(dev, PTR_ERR(dma->txchan), + "Failed to get TX DMA channel\n"); dma->rxchan = dma_request_chan(dev, "rx"); if (IS_ERR(dma->rxchan)) { - ret = PTR_ERR(dma->rxchan); + ret = dev_err_probe(dev, PTR_ERR(dma->rxchan), + "Failed to get RX DMA channel\n"); goto error_rx; } @@ -31,7 +43,8 @@ int qce_dma_request(struct device *dev, struct qce_dma_data *dma) dma->ignore_buf = dma->result_buf + QCE_RESULT_BUF_SZ; - return 0; + return devm_add_action_or_reset(dev, qce_dma_release, dma); + error_nomem: dma_release_channel(dma->rxchan); error_rx: @@ -39,13 +52,6 @@ error_rx: return ret; } -void qce_dma_release(struct qce_dma_data *dma) -{ - dma_release_channel(dma->txchan); - dma_release_channel(dma->rxchan); - kfree(dma->result_buf); -} - struct scatterlist * qce_sgtable_add(struct sg_table *sgt, struct scatterlist *new_sgl, unsigned int max_len) diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h index 786402169360..31629185000e 100644 --- a/drivers/crypto/qce/dma.h +++ b/drivers/crypto/qce/dma.h @@ -34,8 +34,7 @@ struct qce_dma_data { void *ignore_buf; }; -int qce_dma_request(struct device *dev, struct qce_dma_data *dma); -void qce_dma_release(struct qce_dma_data *dma); +int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma); int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in, int in_ents, struct scatterlist *sg_out, int out_ents, dma_async_tx_callback cb, void *cb_param); diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index fc72af8aa9a7..71b748183cfa 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -482,7 +482,7 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def, base = &alg->halg.base; base->cra_blocksize = def->blocksize; - base->cra_priority = 300; + base->cra_priority = 175; base->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; base->cra_ctxsize = sizeof(struct qce_sha_ctx); base->cra_alignmask = 0; diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index 5b493fdc1e74..ffb334eb5b34 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -461,7 +461,7 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def, alg->encrypt = qce_skcipher_encrypt; alg->decrypt = qce_skcipher_decrypt; - alg->base.cra_priority = 300; + alg->base.cra_priority = 275; alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | CRYPTO_ALG_KERN_DRIVER_ONLY; diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index 69d6019d8abc..b9f5a8b42e66 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -52,12 +52,11 @@ static int rk_ahash_digest_fb(struct ahash_request *areq) algt->stat_fb++; ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); - rctx->fallback_req.base.flags = areq->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = areq->nbytes; - rctx->fallback_req.src = areq->src; - rctx->fallback_req.result = areq->result; + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -124,8 +123,9 @@ static int rk_ahash_init(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -137,10 +137,10 @@ static int rk_ahash_update(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -152,9 +152,10 @@ static int rk_ahash_final(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -166,12 +167,11 @@ static int rk_ahash_finup(struct ahash_request *req) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -183,8 +183,9 @@ static int rk_ahash_import(struct ahash_request *req, const void *in) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -196,8 +197,9 @@ static int rk_ahash_export(struct ahash_request *req, void *out) struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } @@ -252,7 +254,7 @@ static void rk_hash_unprepare(struct crypto_engine *engine, void *breq) struct rk_ahash_rctx *rctx = ahash_request_ctx(areq); struct rk_crypto_info *rkc = rctx->dev; - dma_unmap_sg(rkc->dev, areq->src, rctx->nrsg, DMA_TO_DEVICE); + dma_unmap_sg(rkc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); } static int rk_hash_run(struct crypto_engine *engine, void *breq) diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c index 9393e10671c2..e80f9148c012 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c @@ -321,8 +321,7 @@ static int rk_cipher_run(struct crypto_engine *engine, void *async_req) algt->stat_req++; rkc->nreq++; - ivsize = crypto_skcipher_ivsize(tfm); - if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { + if (areq->iv && ivsize > 0) { if (rctx->mode & RK_CRYPTO_DEC) { offset = areq->cryptlen - ivsize; scatterwalk_map_and_copy(rctx->backup_iv, areq->src, diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 57ab237e899e..b829c84f60f2 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -9,11 +9,17 @@ // // Hash part based on omap-sham.c driver. +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <crypto/md5.h> +#include <crypto/scatterwalk.h> +#include <crypto/sha1.h> +#include <crypto/sha2.h> #include <linux/clk.h> -#include <linux/crypto.h> #include <linux/dma-mapping.h> #include <linux/err.h> -#include <linux/errno.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -22,17 +28,9 @@ #include <linux/of.h> #include <linux/platform_device.h> #include <linux/scatterlist.h> - -#include <crypto/ctr.h> -#include <crypto/aes.h> -#include <crypto/algapi.h> -#include <crypto/scatterwalk.h> - -#include <crypto/hash.h> -#include <crypto/md5.h> -#include <crypto/sha1.h> -#include <crypto/sha2.h> -#include <crypto/internal/hash.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/string.h> #define _SBF(s, v) ((v) << (s)) @@ -458,19 +456,6 @@ static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg) *sg = NULL; } -static void s5p_sg_copy_buf(void *buf, struct scatterlist *sg, - unsigned int nbytes, int out) -{ - struct scatter_walk walk; - - if (!nbytes) - return; - - scatterwalk_start(&walk, sg); - scatterwalk_copychunks(buf, &walk, nbytes, out); - scatterwalk_done(&walk, out, 0); -} - static void s5p_sg_done(struct s5p_aes_dev *dev) { struct skcipher_request *req = dev->req; @@ -480,8 +465,8 @@ static void s5p_sg_done(struct s5p_aes_dev *dev) dev_dbg(dev->dev, "Copying %d bytes of output data back to original place\n", dev->req->cryptlen); - s5p_sg_copy_buf(sg_virt(dev->sg_dst_cpy), dev->req->dst, - dev->req->cryptlen, 1); + memcpy_to_sglist(dev->req->dst, 0, sg_virt(dev->sg_dst_cpy), + dev->req->cryptlen); } s5p_free_sg_cpy(dev, &dev->sg_src_cpy); s5p_free_sg_cpy(dev, &dev->sg_dst_cpy); @@ -526,7 +511,7 @@ static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src, return -ENOMEM; } - s5p_sg_copy_buf(pages, src, dev->req->cryptlen, 0); + memcpy_from_sglist(pages, src, 0, dev->req->cryptlen); sg_init_table(*dst, 1); sg_set_buf(*dst, pages, len); @@ -1035,8 +1020,7 @@ static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx, if (ctx->bufcnt) memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt); - scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip, - new_len, 0); + memcpy_from_sglist(buf + ctx->bufcnt, sg, ctx->skip, new_len); sg_init_table(ctx->sgl, 1); sg_set_buf(ctx->sgl, buf, len); ctx->sg = ctx->sgl; @@ -1229,8 +1213,7 @@ static int s5p_hash_prepare_request(struct ahash_request *req, bool update) if (len > nbytes) len = nbytes; - scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, - 0, len, 0); + memcpy_from_sglist(ctx->buffer + ctx->bufcnt, req->src, 0, len); ctx->bufcnt += len; nbytes -= len; ctx->skip = len; @@ -1253,9 +1236,8 @@ static int s5p_hash_prepare_request(struct ahash_request *req, bool update) hash_later = ctx->total - xmit_len; /* copy hash_later bytes from end of req->src */ /* previous bytes are in xmit_buf, so no overwrite */ - scatterwalk_map_and_copy(ctx->buffer, req->src, - req->nbytes - hash_later, - hash_later, 0); + memcpy_from_sglist(ctx->buffer, req->src, + req->nbytes - hash_later, hash_later); } if (xmit_len > BUFLEN) { @@ -1267,8 +1249,8 @@ static int s5p_hash_prepare_request(struct ahash_request *req, bool update) /* have buffered data only */ if (unlikely(!ctx->bufcnt)) { /* first update didn't fill up buffer */ - scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src, - 0, xmit_len, 0); + memcpy_from_sglist(ctx->dd->xmit_buf, req->src, + 0, xmit_len); } sg_init_table(ctx->sgl, 1); @@ -1506,8 +1488,8 @@ static int s5p_hash_update(struct ahash_request *req) return 0; if (ctx->bufcnt + req->nbytes <= BUFLEN) { - scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, - 0, req->nbytes, 0); + memcpy_from_sglist(ctx->buffer + ctx->bufcnt, req->src, + 0, req->nbytes); ctx->bufcnt += req->nbytes; return 0; } diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 091612b066f1..fdc0b2486069 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -1415,22 +1415,13 @@ static int sa_sha_run(struct ahash_request *req) (auth_len >= SA_UNSAFE_DATA_SZ_MIN && auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { struct ahash_request *subreq = &rctx->fallback_req; - int ret = 0; + int ret; ahash_request_set_tfm(subreq, ctx->fallback.ahash); - subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - - crypto_ahash_init(subreq); - - subreq->nbytes = auth_len; - subreq->src = req->src; - subreq->result = req->result; - - ret |= crypto_ahash_update(subreq); - - subreq->nbytes = 0; + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(subreq, req->src, req->result, auth_len); - ret |= crypto_ahash_final(subreq); + ret = crypto_ahash_digest(subreq); return ret; } @@ -1502,8 +1493,7 @@ static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) return ret; if (alg_base) { - ctx->shash = crypto_alloc_shash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); + ctx->shash = crypto_alloc_shash(alg_base, 0, 0); if (IS_ERR(ctx->shash)) { dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", alg_base); @@ -1511,8 +1501,7 @@ static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) } /* for fallback */ ctx->fallback.ahash = - crypto_alloc_ahash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); + crypto_alloc_ahash(alg_base, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(ctx->fallback.ahash)) { dev_err(ctx->dev_data->dev, "Could not load fallback driver\n"); @@ -1546,54 +1535,38 @@ static int sa_sha_init(struct ahash_request *req) crypto_ahash_digestsize(tfm), rctx); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, NULL, 0); return crypto_ahash_init(&rctx->fallback_req); } static int sa_sha_update(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } static int sa_sha_final(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } static int sa_sha_finup(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -1605,8 +1578,7 @@ static int sa_sha_import(struct ahash_request *req, const void *in) struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -1614,12 +1586,9 @@ static int sa_sha_import(struct ahash_request *req, const void *in) static int sa_sha_export(struct ahash_request *req, void *out) { struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); struct ahash_request *subreq = &rctx->fallback_req; - ahash_request_set_tfm(subreq, ctx->fallback.ahash); - subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); return crypto_ahash_export(subreq, out); } diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 86a1a1fa9f8f..426b24889af8 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -511,8 +511,7 @@ static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp, stsg = sg_next(stsg), dtsg = sg_next(dtsg)) { src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); if (src_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg error\n"); + return -ENOMEM; dst_nents = src_nents; len = min(sg_dma_len(stsg), remain); @@ -528,13 +527,11 @@ static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp, for (stsg = src, dtsg = dst;;) { src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); if (src_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg src error\n"); + return -ENOMEM; dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); if (dst_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg dst error\n"); + return -ENOMEM; len = min(sg_dma_len(stsg), sg_dma_len(dtsg)); len = min(len, remain); @@ -669,8 +666,7 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq if (cryp->assoclen) { rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); if (!rctx->adata) - return dev_err_probe(cryp->dev, -ENOMEM, - "Failed to alloc memory for adata"); + return -ENOMEM; if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), rctx->adata, cryp->assoclen) != cryp->assoclen) diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 2c60a1047bc3..54b7af4a7aee 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -229,8 +229,7 @@ static int starfive_hash_one_request(struct crypto_engine *engine, void *areq) for_each_sg(rctx->in_sg, tsg, rctx->in_sg_len, i) { src_nents = dma_map_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); if (src_nents == 0) - return dev_err_probe(cryp->dev, -ENOMEM, - "dma_map_sg error\n"); + return -ENOMEM; ret = starfive_hash_dma_xfer(cryp, tsg); dma_unmap_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); @@ -326,6 +325,7 @@ static int starfive_hash_digest(struct ahash_request *req) struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); struct starfive_cryp_dev *cryp = ctx->cryp; + int sg_len; memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx)); @@ -334,7 +334,10 @@ static int starfive_hash_digest(struct ahash_request *req) rctx->in_sg = req->src; rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); rctx->digsize = crypto_ahash_digestsize(tfm); - rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + if (sg_len < 0) + return sg_len; + rctx->in_sg_len = sg_len; ctx->rctx = rctx; return crypto_transfer_hash_request_to_engine(cryp->engine, req); @@ -493,25 +496,25 @@ static int starfive_hash_setkey(struct crypto_ahash *hash, static int starfive_sha224_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha224-generic", + return starfive_hash_init_tfm(hash, "sha224-lib", STARFIVE_HASH_SHA224, 0); } static int starfive_sha256_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha256-generic", + return starfive_hash_init_tfm(hash, "sha256-lib", STARFIVE_HASH_SHA256, 0); } static int starfive_sha384_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha384-generic", + return starfive_hash_init_tfm(hash, "sha384-lib", STARFIVE_HASH_SHA384, 0); } static int starfive_sha512_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha512-generic", + return starfive_hash_init_tfm(hash, "sha512-lib", STARFIVE_HASH_SHA512, 0); } @@ -523,25 +526,25 @@ static int starfive_sm3_init_tfm(struct crypto_ahash *hash) static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha224-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha224-lib", STARFIVE_HASH_SHA224, 1); } static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha256-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha256-lib", STARFIVE_HASH_SHA256, 1); } static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha384-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha384-lib", STARFIVE_HASH_SHA384, 1); } static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha512-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha512-lib", STARFIVE_HASH_SHA512, 1); } diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig index 49dfd161e9b9..d6dc848c82ee 100644 --- a/drivers/crypto/stm32/Kconfig +++ b/drivers/crypto/stm32/Kconfig @@ -1,13 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -config CRYPTO_DEV_STM32_CRC - tristate "Support for STM32 crc accelerators" - depends on ARCH_STM32 - select CRYPTO_HASH - select CRC32 - help - This enables support for the CRC32 hw accelerator which can be found - on STMicroelectronics STM32 SOC. - config CRYPTO_DEV_STM32_HASH tristate "Support for STM32 hash accelerators" depends on ARCH_STM32 || ARCH_U8500 diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile index 518e0e0b11a9..c63004026afb 100644 --- a/drivers/crypto/stm32/Makefile +++ b/drivers/crypto/stm32/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32-crc32.o obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c deleted file mode 100644 index de4d0402f133..000000000000 --- a/drivers/crypto/stm32/stm32-crc32.c +++ /dev/null @@ -1,480 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) STMicroelectronics SA 2017 - * Author: Fabien Dessenne <fabien.dessenne@st.com> - */ - -#include <linux/bitrev.h> -#include <linux/clk.h> -#include <linux/crc32.h> -#include <linux/crc32poly.h> -#include <linux/io.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/mod_devicetable.h> -#include <linux/platform_device.h> -#include <linux/pm_runtime.h> - -#include <crypto/internal/hash.h> - -#include <linux/unaligned.h> - -#define DRIVER_NAME "stm32-crc32" -#define CHKSUM_DIGEST_SIZE 4 -#define CHKSUM_BLOCK_SIZE 1 - -/* Registers */ -#define CRC_DR 0x00000000 -#define CRC_CR 0x00000008 -#define CRC_INIT 0x00000010 -#define CRC_POL 0x00000014 - -/* Registers values */ -#define CRC_CR_RESET BIT(0) -#define CRC_CR_REV_IN_WORD (BIT(6) | BIT(5)) -#define CRC_CR_REV_IN_BYTE BIT(5) -#define CRC_CR_REV_OUT BIT(7) -#define CRC32C_INIT_DEFAULT 0xFFFFFFFF - -#define CRC_AUTOSUSPEND_DELAY 50 - -static unsigned int burst_size; -module_param(burst_size, uint, 0644); -MODULE_PARM_DESC(burst_size, "Select burst byte size (0 unlimited)"); - -struct stm32_crc { - struct list_head list; - struct device *dev; - void __iomem *regs; - struct clk *clk; - spinlock_t lock; -}; - -struct stm32_crc_list { - struct list_head dev_list; - spinlock_t lock; /* protect dev_list */ -}; - -static struct stm32_crc_list crc_list = { - .dev_list = LIST_HEAD_INIT(crc_list.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(crc_list.lock), -}; - -struct stm32_crc_ctx { - u32 key; - u32 poly; -}; - -struct stm32_crc_desc_ctx { - u32 partial; /* crc32c: partial in first 4 bytes of that struct */ -}; - -static int stm32_crc32_cra_init(struct crypto_tfm *tfm) -{ - struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); - - mctx->key = 0; - mctx->poly = CRC32_POLY_LE; - return 0; -} - -static int stm32_crc32c_cra_init(struct crypto_tfm *tfm) -{ - struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); - - mctx->key = CRC32C_INIT_DEFAULT; - mctx->poly = CRC32C_POLY_LE; - return 0; -} - -static int stm32_crc_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct stm32_crc_ctx *mctx = crypto_shash_ctx(tfm); - - if (keylen != sizeof(u32)) - return -EINVAL; - - mctx->key = get_unaligned_le32(key); - return 0; -} - -static struct stm32_crc *stm32_crc_get_next_crc(void) -{ - struct stm32_crc *crc; - - spin_lock_bh(&crc_list.lock); - crc = list_first_entry_or_null(&crc_list.dev_list, struct stm32_crc, list); - if (crc) - list_move_tail(&crc->list, &crc_list.dev_list); - spin_unlock_bh(&crc_list.lock); - - return crc; -} - -static int stm32_crc_init(struct shash_desc *desc) -{ - struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); - struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); - struct stm32_crc *crc; - unsigned long flags; - - crc = stm32_crc_get_next_crc(); - if (!crc) - return -ENODEV; - - pm_runtime_get_sync(crc->dev); - - spin_lock_irqsave(&crc->lock, flags); - - /* Reset, set key, poly and configure in bit reverse mode */ - writel_relaxed(bitrev32(mctx->key), crc->regs + CRC_INIT); - writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL); - writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, - crc->regs + CRC_CR); - - /* Store partial result */ - ctx->partial = readl_relaxed(crc->regs + CRC_DR); - - spin_unlock_irqrestore(&crc->lock, flags); - - pm_runtime_mark_last_busy(crc->dev); - pm_runtime_put_autosuspend(crc->dev); - - return 0; -} - -static int burst_update(struct shash_desc *desc, const u8 *d8, - size_t length) -{ - struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); - struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); - struct stm32_crc *crc; - - crc = stm32_crc_get_next_crc(); - if (!crc) - return -ENODEV; - - pm_runtime_get_sync(crc->dev); - - if (!spin_trylock(&crc->lock)) { - /* Hardware is busy, calculate crc32 by software */ - if (mctx->poly == CRC32_POLY_LE) - ctx->partial = crc32_le(ctx->partial, d8, length); - else - ctx->partial = __crc32c_le(ctx->partial, d8, length); - - goto pm_out; - } - - /* - * Restore previously calculated CRC for this context as init value - * Restore polynomial configuration - * Configure in register for word input data, - * Configure out register in reversed bit mode data. - */ - writel_relaxed(bitrev32(ctx->partial), crc->regs + CRC_INIT); - writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL); - writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, - crc->regs + CRC_CR); - - if (d8 != PTR_ALIGN(d8, sizeof(u32))) { - /* Configure for byte data */ - writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT, - crc->regs + CRC_CR); - while (d8 != PTR_ALIGN(d8, sizeof(u32)) && length) { - writeb_relaxed(*d8++, crc->regs + CRC_DR); - length--; - } - /* Configure for word data */ - writel_relaxed(CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, - crc->regs + CRC_CR); - } - - for (; length >= sizeof(u32); d8 += sizeof(u32), length -= sizeof(u32)) - writel_relaxed(*((u32 *)d8), crc->regs + CRC_DR); - - if (length) { - /* Configure for byte data */ - writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT, - crc->regs + CRC_CR); - while (length--) - writeb_relaxed(*d8++, crc->regs + CRC_DR); - } - - /* Store partial result */ - ctx->partial = readl_relaxed(crc->regs + CRC_DR); - - spin_unlock(&crc->lock); - -pm_out: - pm_runtime_mark_last_busy(crc->dev); - pm_runtime_put_autosuspend(crc->dev); - - return 0; -} - -static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, - unsigned int length) -{ - const unsigned int burst_sz = burst_size; - unsigned int rem_sz; - const u8 *cur; - size_t size; - int ret; - - if (!burst_sz) - return burst_update(desc, d8, length); - - /* Digest first bytes not 32bit aligned at first pass in the loop */ - size = min_t(size_t, length, burst_sz + (size_t)d8 - - ALIGN_DOWN((size_t)d8, sizeof(u32))); - for (rem_sz = length, cur = d8; rem_sz; - rem_sz -= size, cur += size, size = min(rem_sz, burst_sz)) { - ret = burst_update(desc, cur, size); - if (ret) - return ret; - } - - return 0; -} - -static int stm32_crc_final(struct shash_desc *desc, u8 *out) -{ - struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); - struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); - - /* Send computed CRC */ - put_unaligned_le32(mctx->poly == CRC32C_POLY_LE ? - ~ctx->partial : ctx->partial, out); - - return 0; -} - -static int stm32_crc_finup(struct shash_desc *desc, const u8 *data, - unsigned int length, u8 *out) -{ - return stm32_crc_update(desc, data, length) ?: - stm32_crc_final(desc, out); -} - -static int stm32_crc_digest(struct shash_desc *desc, const u8 *data, - unsigned int length, u8 *out) -{ - return stm32_crc_init(desc) ?: stm32_crc_finup(desc, data, length, out); -} - -static unsigned int refcnt; -static DEFINE_MUTEX(refcnt_lock); -static struct shash_alg algs[] = { - /* CRC-32 */ - { - .setkey = stm32_crc_setkey, - .init = stm32_crc_init, - .update = stm32_crc_update, - .final = stm32_crc_final, - .finup = stm32_crc_finup, - .digest = stm32_crc_digest, - .descsize = sizeof(struct stm32_crc_desc_ctx), - .digestsize = CHKSUM_DIGEST_SIZE, - .base = { - .cra_name = "crc32", - .cra_driver_name = "stm32-crc32-crc32", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .cra_blocksize = CHKSUM_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct stm32_crc_ctx), - .cra_module = THIS_MODULE, - .cra_init = stm32_crc32_cra_init, - } - }, - /* CRC-32Castagnoli */ - { - .setkey = stm32_crc_setkey, - .init = stm32_crc_init, - .update = stm32_crc_update, - .final = stm32_crc_final, - .finup = stm32_crc_finup, - .digest = stm32_crc_digest, - .descsize = sizeof(struct stm32_crc_desc_ctx), - .digestsize = CHKSUM_DIGEST_SIZE, - .base = { - .cra_name = "crc32c", - .cra_driver_name = "stm32-crc32-crc32c", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .cra_blocksize = CHKSUM_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct stm32_crc_ctx), - .cra_module = THIS_MODULE, - .cra_init = stm32_crc32c_cra_init, - } - } -}; - -static int stm32_crc_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct stm32_crc *crc; - int ret; - - crc = devm_kzalloc(dev, sizeof(*crc), GFP_KERNEL); - if (!crc) - return -ENOMEM; - - crc->dev = dev; - - crc->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(crc->regs)) { - dev_err(dev, "Cannot map CRC IO\n"); - return PTR_ERR(crc->regs); - } - - crc->clk = devm_clk_get(dev, NULL); - if (IS_ERR(crc->clk)) { - dev_err(dev, "Could not get clock\n"); - return PTR_ERR(crc->clk); - } - - ret = clk_prepare_enable(crc->clk); - if (ret) { - dev_err(crc->dev, "Failed to enable clock\n"); - return ret; - } - - pm_runtime_set_autosuspend_delay(dev, CRC_AUTOSUSPEND_DELAY); - pm_runtime_use_autosuspend(dev); - - pm_runtime_get_noresume(dev); - pm_runtime_set_active(dev); - pm_runtime_irq_safe(dev); - pm_runtime_enable(dev); - - spin_lock_init(&crc->lock); - - platform_set_drvdata(pdev, crc); - - spin_lock(&crc_list.lock); - list_add(&crc->list, &crc_list.dev_list); - spin_unlock(&crc_list.lock); - - mutex_lock(&refcnt_lock); - if (!refcnt) { - ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); - if (ret) { - mutex_unlock(&refcnt_lock); - dev_err(dev, "Failed to register\n"); - clk_disable_unprepare(crc->clk); - return ret; - } - } - refcnt++; - mutex_unlock(&refcnt_lock); - - dev_info(dev, "Initialized\n"); - - pm_runtime_put_sync(dev); - - return 0; -} - -static void stm32_crc_remove(struct platform_device *pdev) -{ - struct stm32_crc *crc = platform_get_drvdata(pdev); - int ret = pm_runtime_get_sync(crc->dev); - - spin_lock(&crc_list.lock); - list_del(&crc->list); - spin_unlock(&crc_list.lock); - - mutex_lock(&refcnt_lock); - if (!--refcnt) - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); - mutex_unlock(&refcnt_lock); - - pm_runtime_disable(crc->dev); - pm_runtime_put_noidle(crc->dev); - - if (ret >= 0) - clk_disable(crc->clk); - clk_unprepare(crc->clk); -} - -static int __maybe_unused stm32_crc_suspend(struct device *dev) -{ - struct stm32_crc *crc = dev_get_drvdata(dev); - int ret; - - ret = pm_runtime_force_suspend(dev); - if (ret) - return ret; - - clk_unprepare(crc->clk); - - return 0; -} - -static int __maybe_unused stm32_crc_resume(struct device *dev) -{ - struct stm32_crc *crc = dev_get_drvdata(dev); - int ret; - - ret = clk_prepare(crc->clk); - if (ret) { - dev_err(crc->dev, "Failed to prepare clock\n"); - return ret; - } - - return pm_runtime_force_resume(dev); -} - -static int __maybe_unused stm32_crc_runtime_suspend(struct device *dev) -{ - struct stm32_crc *crc = dev_get_drvdata(dev); - - clk_disable(crc->clk); - - return 0; -} - -static int __maybe_unused stm32_crc_runtime_resume(struct device *dev) -{ - struct stm32_crc *crc = dev_get_drvdata(dev); - int ret; - - ret = clk_enable(crc->clk); - if (ret) { - dev_err(crc->dev, "Failed to enable clock\n"); - return ret; - } - - return 0; -} - -static const struct dev_pm_ops stm32_crc_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(stm32_crc_suspend, - stm32_crc_resume) - SET_RUNTIME_PM_OPS(stm32_crc_runtime_suspend, - stm32_crc_runtime_resume, NULL) -}; - -static const struct of_device_id stm32_dt_ids[] = { - { .compatible = "st,stm32f7-crc", }, - {}, -}; -MODULE_DEVICE_TABLE(of, stm32_dt_ids); - -static struct platform_driver stm32_crc_driver = { - .probe = stm32_crc_probe, - .remove = stm32_crc_remove, - .driver = { - .name = DRIVER_NAME, - .pm = &stm32_crc_pm_ops, - .of_match_table = stm32_dt_ids, - }, -}; - -module_platform_driver(stm32_crc_driver); - -MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); -MODULE_DESCRIPTION("STMicrolectronics STM32 CRC32 hardware driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 14c6339c2e43..5e82e8a1f71a 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -666,7 +666,7 @@ static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp) written = min_t(size_t, AES_BLOCK_SIZE - len, alen); - scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0); + memcpy_from_scatterwalk((char *)block + len, &cryp->in_walk, written); writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); @@ -851,7 +851,6 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) stm32_cryp_get_iv(cryp); - pm_runtime_mark_last_busy(cryp->dev); pm_runtime_put_autosuspend(cryp->dev); if (is_gcm(cryp) || is_ccm(cryp)) @@ -993,7 +992,7 @@ static int stm32_cryp_header_dma_start(struct stm32_cryp *cryp) /* Advance scatterwalk to not DMA'ed data */ align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize); - scatterwalk_copychunks(NULL, &cryp->in_walk, align_size, 2); + scatterwalk_skip(&cryp->in_walk, align_size); cryp->header_in -= align_size; ret = dma_submit_error(dmaengine_submit(tx_in)); @@ -1056,7 +1055,7 @@ static int stm32_cryp_dma_start(struct stm32_cryp *cryp) /* Advance scatterwalk to not DMA'ed data */ align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); - scatterwalk_copychunks(NULL, &cryp->in_walk, align_size, 2); + scatterwalk_skip(&cryp->in_walk, align_size); cryp->payload_in -= align_size; ret = dma_submit_error(dmaengine_submit(tx_in)); @@ -1067,7 +1066,7 @@ static int stm32_cryp_dma_start(struct stm32_cryp *cryp) dma_async_issue_pending(cryp->dma_lch_in); /* Advance scatterwalk to not DMA'ed data */ - scatterwalk_copychunks(NULL, &cryp->out_walk, align_size, 2); + scatterwalk_skip(&cryp->out_walk, align_size); cryp->payload_out -= align_size; ret = dma_submit_error(dmaengine_submit(tx_out)); if (ret < 0) { @@ -1737,9 +1736,9 @@ static int stm32_cryp_prepare_req(struct skcipher_request *req, out_sg = areq->dst; scatterwalk_start(&cryp->in_walk, in_sg); - scatterwalk_start(&cryp->out_walk, out_sg); /* In output, jump after assoc data */ - scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2); + scatterwalk_start_at_pos(&cryp->out_walk, out_sg, + areq->assoclen); ret = stm32_cryp_hw_init(cryp); if (ret) @@ -1873,12 +1872,12 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) /* Get and write tag */ readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); - scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1); + memcpy_to_scatterwalk(&cryp->out_walk, out_tag, cryp->authsize); } else { /* Get and check tag */ u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32]; - scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0); + memcpy_from_scatterwalk(in_tag, &cryp->in_walk, cryp->authsize); readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); if (crypto_memneq(in_tag, out_tag, cryp->authsize)) @@ -1923,8 +1922,8 @@ static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp) u32 block[AES_BLOCK_32]; readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); - scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, - cryp->payload_out), 1); + memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, + cryp->payload_out)); cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); } @@ -1933,8 +1932,8 @@ static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) { u32 block[AES_BLOCK_32] = {0}; - scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize, - cryp->payload_in), 0); + memcpy_from_scatterwalk(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize, + cryp->payload_in)); writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32)); cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in); } @@ -1981,8 +1980,8 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) */ readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); - scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, - cryp->payload_out), 1); + memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, + cryp->payload_out)); cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); @@ -2079,8 +2078,8 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) */ readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); - scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, - cryp->payload_out), 1); + memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, + cryp->payload_out)); cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); /* d) Load again CRYP_CSGCMCCMxR */ @@ -2161,7 +2160,7 @@ static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp) written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in); - scatterwalk_copychunks(block, &cryp->in_walk, written, 0); + memcpy_from_scatterwalk(block, &cryp->in_walk, written); writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); @@ -2782,5 +2781,5 @@ static struct platform_driver stm32_cryp_driver = { module_platform_driver(stm32_cryp_driver); MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); -MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver"); +MODULE_DESCRIPTION("STMicroelectronics STM32 CRYP hardware driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 768b27de4737..a4436728b0db 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -1373,7 +1373,6 @@ static void stm32_hash_unprepare_request(struct ahash_request *req) *preg++ = stm32_hash_read(hdev, HASH_CSR(i)); pm_runtime: - pm_runtime_mark_last_busy(hdev->dev); pm_runtime_put_autosuspend(hdev->dev); } diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 9d130592cc0a..0e07d0523291 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -28,6 +28,9 @@ struct tegra_aes_ctx { u32 ivsize; u32 key1_id; u32 key2_id; + u32 keylen; + u8 key1[AES_MAX_KEY_SIZE]; + u8 key2[AES_MAX_KEY_SIZE]; }; struct tegra_aes_reqctx { @@ -43,8 +46,9 @@ struct tegra_aead_ctx { struct tegra_se *se; unsigned int authsize; u32 alg; - u32 keylen; u32 key_id; + u32 keylen; + u8 key[AES_MAX_KEY_SIZE]; }; struct tegra_aead_reqctx { @@ -56,8 +60,8 @@ struct tegra_aead_reqctx { unsigned int cryptlen; unsigned int authsize; bool encrypt; - u32 config; u32 crypto_config; + u32 config; u32 key_id; u32 iv[4]; u8 authdata[16]; @@ -67,6 +71,8 @@ struct tegra_cmac_ctx { struct tegra_se *se; unsigned int alg; u32 key_id; + u32 keylen; + u8 key[AES_MAX_KEY_SIZE]; struct crypto_shash *fallback_tfm; }; @@ -260,17 +266,13 @@ static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); struct tegra_se *se = ctx->se; - unsigned int cmdlen; + unsigned int cmdlen, key1_id, key2_id; int ret; - rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_AES_BUFLEN, - &rctx->datbuf.addr, GFP_KERNEL); - if (!rctx->datbuf.buf) - return -ENOMEM; - - rctx->datbuf.size = SE_AES_BUFLEN; - rctx->iv = (u32 *)req->iv; + rctx->iv = (ctx->alg == SE_ALG_ECB) ? NULL : (u32 *)req->iv; rctx->len = req->cryptlen; + key1_id = ctx->key1_id; + key2_id = ctx->key2_id; /* Pad input to AES Block size */ if (ctx->alg != SE_ALG_XTS) { @@ -278,20 +280,59 @@ static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) rctx->len += AES_BLOCK_SIZE - (rctx->len % AES_BLOCK_SIZE); } + rctx->datbuf.size = rctx->len; + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_finalize; + } + scatterwalk_map_and_copy(rctx->datbuf.buf, req->src, 0, req->cryptlen, 0); + rctx->config = tegra234_aes_cfg(ctx->alg, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, rctx->encrypt); + + if (!key1_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key1, + ctx->keylen, ctx->alg, &key1_id); + if (ret) + goto out; + } + + rctx->crypto_config |= SE_AES_KEY_INDEX(key1_id); + + if (ctx->alg == SE_ALG_XTS) { + if (!key2_id) { + ret = tegra_key_submit_reserved_xts(ctx->se, ctx->key2, + ctx->keylen, ctx->alg, &key2_id); + if (ret) + goto out; + } + + rctx->crypto_config |= SE_AES_KEY2_INDEX(key2_id); + } + /* Prepare the command and submit for execution */ cmdlen = tegra_aes_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); /* Copy the result */ tegra_aes_update_iv(req, ctx); scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1); +out: /* Free the buffer */ - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + dma_free_coherent(ctx->se->dev, rctx->datbuf.size, rctx->datbuf.buf, rctx->datbuf.addr); + if (tegra_key_is_reserved(key1_id)) + tegra_key_invalidate_reserved(ctx->se, key1_id, ctx->alg); + + if (tegra_key_is_reserved(key2_id)) + tegra_key_invalidate_reserved(ctx->se, key2_id, ctx->alg); + +out_finalize: crypto_finalize_skcipher_request(se->engine, req, ret); return 0; @@ -313,6 +354,7 @@ static int tegra_aes_cra_init(struct crypto_skcipher *tfm) ctx->se = se_alg->se_dev; ctx->key1_id = 0; ctx->key2_id = 0; + ctx->keylen = 0; algname = crypto_tfm_alg_name(&tfm->base); ret = se_algname_to_algid(algname); @@ -341,13 +383,20 @@ static int tegra_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, u32 keylen) { struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) { dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); return -EINVAL; } - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key1, key, keylen); + } + + return 0; } static int tegra_xts_setkey(struct crypto_skcipher *tfm, @@ -365,11 +414,17 @@ static int tegra_xts_setkey(struct crypto_skcipher *tfm, ret = tegra_key_submit(ctx->se, key, len, ctx->alg, &ctx->key1_id); - if (ret) - return ret; + if (ret) { + ctx->keylen = len; + memcpy(ctx->key1, key, len); + } - return tegra_key_submit(ctx->se, key + len, len, + ret = tegra_key_submit(ctx->se, key + len, len, ctx->alg, &ctx->key2_id); + if (ret) { + ctx->keylen = len; + memcpy(ctx->key2, key + len, len); + } return 0; } @@ -444,12 +499,6 @@ static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt) return 0; rctx->encrypt = encrypt; - rctx->config = tegra234_aes_cfg(ctx->alg, encrypt); - rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, encrypt); - rctx->crypto_config |= SE_AES_KEY_INDEX(ctx->key1_id); - - if (ctx->key2_id) - rctx->crypto_config |= SE_AES_KEY2_INDEX(ctx->key2_id); return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req); } @@ -715,11 +764,11 @@ static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqct rctx->config = tegra234_aes_cfg(SE_ALG_GMAC, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GMAC, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); cmdlen = tegra_gmac_prep_cmd(ctx, rctx); - return tegra_se_host1x_submit(se, cmdlen); + return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); } static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) @@ -732,11 +781,11 @@ static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc rctx->config = tegra234_aes_cfg(SE_ALG_GCM, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Prepare command and submit */ cmdlen = tegra_gcm_crypt_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) return ret; @@ -755,11 +804,11 @@ static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM_FINAL, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Prepare command and submit */ cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) return ret; @@ -886,12 +935,12 @@ static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_req rctx->config = tegra234_aes_cfg(SE_ALG_CBC_MAC, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CBC_MAC, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Prepare command and submit */ cmdlen = tegra_cbcmac_prep_cmd(ctx, rctx); - return tegra_se_host1x_submit(se, cmdlen); + return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); } static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize) @@ -1073,7 +1122,7 @@ static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx rctx->config = tegra234_aes_cfg(SE_ALG_CTR, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CTR, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Copy authdata in the top of buffer for encryption/decryption */ if (rctx->encrypt) @@ -1098,7 +1147,7 @@ static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx /* Prepare command and submit */ cmdlen = tegra_ctr_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) return ret; @@ -1117,6 +1166,11 @@ static int tegra_ccm_crypt_init(struct aead_request *req, struct tegra_se *se, rctx->assoclen = req->assoclen; rctx->authsize = crypto_aead_authsize(tfm); + if (rctx->encrypt) + rctx->cryptlen = req->cryptlen; + else + rctx->cryptlen = req->cryptlen - rctx->authsize; + memcpy(iv, req->iv, 16); ret = tegra_ccm_check_iv(iv); @@ -1145,30 +1199,35 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_se *se = ctx->se; int ret; + ret = tegra_ccm_crypt_init(req, se, rctx); + if (ret) + goto out_finalize; + + rctx->key_id = ctx->key_id; + /* Allocate buffers required */ - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, &rctx->inbuf.addr, GFP_KERNEL); if (!rctx->inbuf.buf) - return -ENOMEM; - - rctx->inbuf.size = SE_AES_BUFLEN; + goto out_finalize; - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, &rctx->outbuf.addr, GFP_KERNEL); if (!rctx->outbuf.buf) { ret = -ENOMEM; - goto outbuf_err; + goto out_free_inbuf; } - rctx->outbuf.size = SE_AES_BUFLEN; - - ret = tegra_ccm_crypt_init(req, se, rctx); - if (ret) - goto out; + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } if (rctx->encrypt) { - rctx->cryptlen = req->cryptlen; - /* CBC MAC Operation */ ret = tegra_ccm_compute_auth(ctx, rctx); if (ret) @@ -1179,8 +1238,6 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) if (ret) goto out; } else { - rctx->cryptlen = req->cryptlen - ctx->authsize; - /* CTR operation */ ret = tegra_ccm_do_ctr(ctx, rctx); if (ret) @@ -1193,13 +1250,17 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) } out: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + dma_free_coherent(ctx->se->dev, rctx->inbuf.size, rctx->outbuf.buf, rctx->outbuf.addr); -outbuf_err: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, +out_free_inbuf: + dma_free_coherent(ctx->se->dev, rctx->outbuf.size, rctx->inbuf.buf, rctx->inbuf.addr); + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + +out_finalize: crypto_finalize_aead_request(ctx->se->engine, req, ret); return 0; @@ -1213,23 +1274,6 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_aead_reqctx *rctx = aead_request_ctx(req); int ret; - /* Allocate buffers required */ - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, - &rctx->inbuf.addr, GFP_KERNEL); - if (!rctx->inbuf.buf) - return -ENOMEM; - - rctx->inbuf.size = SE_AES_BUFLEN; - - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, - &rctx->outbuf.addr, GFP_KERNEL); - if (!rctx->outbuf.buf) { - ret = -ENOMEM; - goto outbuf_err; - } - - rctx->outbuf.size = SE_AES_BUFLEN; - rctx->src_sg = req->src; rctx->dst_sg = req->dst; rctx->assoclen = req->assoclen; @@ -1243,6 +1287,32 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); rctx->iv[3] = (1 << 24); + rctx->key_id = ctx->key_id; + + /* Allocate buffers required */ + rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, + &rctx->inbuf.addr, GFP_KERNEL); + if (!rctx->inbuf.buf) { + ret = -ENOMEM; + goto out_finalize; + } + + rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, + &rctx->outbuf.addr, GFP_KERNEL); + if (!rctx->outbuf.buf) { + ret = -ENOMEM; + goto out_free_inbuf; + } + + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } + /* If there is associated data perform GMAC operation */ if (rctx->assoclen) { ret = tegra_gcm_do_gmac(ctx, rctx); @@ -1266,14 +1336,17 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) ret = tegra_gcm_do_verify(ctx->se, rctx); out: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + dma_free_coherent(ctx->se->dev, rctx->outbuf.size, rctx->outbuf.buf, rctx->outbuf.addr); -outbuf_err: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, +out_free_inbuf: + dma_free_coherent(ctx->se->dev, rctx->inbuf.size, rctx->inbuf.buf, rctx->inbuf.addr); - /* Finalize the request if there are no errors */ + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + +out_finalize: crypto_finalize_aead_request(ctx->se->engine, req, ret); return 0; @@ -1295,6 +1368,7 @@ static int tegra_aead_cra_init(struct crypto_aead *tfm) ctx->se = se_alg->se_dev; ctx->key_id = 0; + ctx->keylen = 0; ret = se_algname_to_algid(algname); if (ret < 0) { @@ -1376,13 +1450,20 @@ static int tegra_aead_setkey(struct crypto_aead *tfm, const u8 *key, u32 keylen) { struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) { dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); return -EINVAL; } - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + } + + return 0; } static unsigned int tegra_cmac_prep_cmd(struct tegra_cmac_ctx *ctx, @@ -1456,6 +1537,35 @@ static void tegra_cmac_paste_result(struct tegra_se *se, struct tegra_cmac_reqct se->base + se->hw->regs->result + (i * 4)); } +static int tegra_cmac_do_init(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int i; + + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->key_id = ctx->key_id; + rctx->task |= SHA_FIRST; + rctx->blk_size = crypto_ahash_blocksize(tfm); + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + return -ENOMEM; + + rctx->residue.size = 0; + + /* Clear any previous result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + return 0; +} + static int tegra_cmac_do_update(struct ahash_request *req) { struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); @@ -1483,7 +1593,7 @@ static int tegra_cmac_do_update(struct ahash_request *req) rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; rctx->total_len += rctx->datbuf.size; rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); - rctx->crypto_config = SE_AES_KEY_INDEX(ctx->key_id); + rctx->crypto_config = SE_AES_KEY_INDEX(rctx->key_id); /* * Keep one block and residue bytes in residue and @@ -1497,6 +1607,11 @@ static int tegra_cmac_do_update(struct ahash_request *req) return 0; } + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + return -ENOMEM; + /* Copy the previous residue first */ if (rctx->residue.size) memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); @@ -1511,23 +1626,19 @@ static int tegra_cmac_do_update(struct ahash_request *req) rctx->residue.size = nresidue; /* - * If this is not the first 'update' call, paste the previous copied + * If this is not the first task, paste the previous copied * intermediate results to the registers so that it gets picked up. - * This is to support the import/export functionality. */ if (!(rctx->task & SHA_FIRST)) tegra_cmac_paste_result(ctx->se, rctx); cmdlen = tegra_cmac_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); - ret = tegra_se_host1x_submit(se, cmdlen); - /* - * If this is not the final update, copy the intermediate results - * from the registers so that it can be used in the next 'update' - * call. This is to support the import/export functionality. - */ - if (!(rctx->task & SHA_FINAL)) - tegra_cmac_copy_result(ctx->se, rctx); + tegra_cmac_copy_result(ctx->se, rctx); + + dma_free_coherent(ctx->se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); return ret; } @@ -1543,17 +1654,34 @@ static int tegra_cmac_do_final(struct ahash_request *req) if (!req->nbytes && !rctx->total_len && ctx->fallback_tfm) { return crypto_shash_tfm_digest(ctx->fallback_tfm, - rctx->datbuf.buf, 0, req->result); + NULL, 0, req->result); + } + + if (rctx->residue.size) { + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); } - memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); rctx->datbuf.size = rctx->residue.size; rctx->total_len += rctx->residue.size; rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); + /* + * If this is not the first task, paste the previous copied + * intermediate results to the registers so that it gets picked up. + */ + if (!(rctx->task & SHA_FIRST)) + tegra_cmac_paste_result(ctx->se, rctx); + /* Prepare command and submit */ cmdlen = tegra_cmac_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) goto out; @@ -1565,8 +1693,10 @@ static int tegra_cmac_do_final(struct ahash_request *req) writel(0, se->base + se->hw->regs->result + (i * 4)); out: - dma_free_coherent(se->dev, SE_SHA_BUFLEN, - rctx->datbuf.buf, rctx->datbuf.addr); + if (rctx->residue.size) + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); +out_free: dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm) * 2, rctx->residue.buf, rctx->residue.addr); return ret; @@ -1579,17 +1709,41 @@ static int tegra_cmac_do_one_req(struct crypto_engine *engine, void *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); struct tegra_se *se = ctx->se; - int ret; + int ret = 0; + + if (rctx->task & SHA_INIT) { + ret = tegra_cmac_do_init(req); + if (ret) + goto out; + + rctx->task &= ~SHA_INIT; + } + + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } if (rctx->task & SHA_UPDATE) { ret = tegra_cmac_do_update(req); + if (ret) + goto out; + rctx->task &= ~SHA_UPDATE; } if (rctx->task & SHA_FINAL) { ret = tegra_cmac_do_final(req); + if (ret) + goto out; + rctx->task &= ~SHA_FINAL; } +out: + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); crypto_finalize_hash_request(se->engine, req, ret); @@ -1631,6 +1785,7 @@ static int tegra_cmac_cra_init(struct crypto_tfm *tfm) ctx->se = se_alg->se_dev; ctx->key_id = 0; + ctx->keylen = 0; ret = se_algname_to_algid(algname); if (ret < 0) { @@ -1655,51 +1810,11 @@ static void tegra_cmac_cra_exit(struct crypto_tfm *tfm) tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); } -static int tegra_cmac_init(struct ahash_request *req) -{ - struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); - struct tegra_se *se = ctx->se; - int i; - - rctx->total_len = 0; - rctx->datbuf.size = 0; - rctx->residue.size = 0; - rctx->task = SHA_FIRST; - rctx->blk_size = crypto_ahash_blocksize(tfm); - - rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, - &rctx->residue.addr, GFP_KERNEL); - if (!rctx->residue.buf) - goto resbuf_fail; - - rctx->residue.size = 0; - - rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_SHA_BUFLEN, - &rctx->datbuf.addr, GFP_KERNEL); - if (!rctx->datbuf.buf) - goto datbuf_fail; - - rctx->datbuf.size = 0; - - /* Clear any previous result */ - for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) - writel(0, se->base + se->hw->regs->result + (i * 4)); - - return 0; - -datbuf_fail: - dma_free_coherent(se->dev, rctx->blk_size, rctx->residue.buf, - rctx->residue.addr); -resbuf_fail: - return -ENOMEM; -} - static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) { dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); @@ -1709,7 +1824,24 @@ static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, if (ctx->fallback_tfm) crypto_shash_setkey(ctx->fallback_tfm, key, keylen); - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + } + + return 0; +} + +static int tegra_cmac_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task = SHA_INIT; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } static int tegra_cmac_update(struct ahash_request *req) @@ -1751,8 +1883,7 @@ static int tegra_cmac_digest(struct ahash_request *req) struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); - tegra_cmac_init(req); - rctx->task |= SHA_UPDATE | SHA_FINAL; + rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL; return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 4d4bd727f498..4a298ace6e9f 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -34,6 +34,7 @@ struct tegra_sha_reqctx { struct tegra_se_datbuf datbuf; struct tegra_se_datbuf residue; struct tegra_se_datbuf digest; + struct tegra_se_datbuf intr_res; unsigned int alg; unsigned int config; unsigned int total_len; @@ -116,8 +117,9 @@ static int tegra_sha_fallback_init(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_init(&rctx->fallback_req); } @@ -129,10 +131,10 @@ static int tegra_sha_fallback_update(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); return crypto_ahash_update(&rctx->fallback_req); } @@ -144,9 +146,10 @@ static int tegra_sha_fallback_final(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); return crypto_ahash_final(&rctx->fallback_req); } @@ -158,12 +161,11 @@ static int tegra_sha_fallback_finup(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_finup(&rctx->fallback_req); } @@ -175,12 +177,11 @@ static int tegra_sha_fallback_digest(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); return crypto_ahash_digest(&rctx->fallback_req); } @@ -192,8 +193,9 @@ static int tegra_sha_fallback_import(struct ahash_request *req, const void *in) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_import(&rctx->fallback_req, in); } @@ -205,15 +207,69 @@ static int tegra_sha_fallback_export(struct ahash_request *req, void *out) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); return crypto_ahash_export(&rctx->fallback_req, out); } -static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, +static int tegra_se_insert_hash_result(struct tegra_sha_ctx *ctx, u32 *cpuvaddr, + struct tegra_sha_reqctx *rctx) +{ + __be32 *res_be = (__be32 *)rctx->intr_res.buf; + u32 *res = (u32 *)rctx->intr_res.buf; + int i = 0, j; + + cpuvaddr[i++] = 0; + cpuvaddr[i++] = host1x_opcode_setpayload(HASH_RESULT_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_HASH_RESULT); + + for (j = 0; j < HASH_RESULT_REG_COUNT; j++) { + int idx = j; + + /* + * The initial, intermediate and final hash value of SHA-384, SHA-512 + * in SHA_HASH_RESULT registers follow the below layout of bytes. + * + * +---------------+------------+ + * | HASH_RESULT_0 | B4...B7 | + * +---------------+------------+ + * | HASH_RESULT_1 | B0...B3 | + * +---------------+------------+ + * | HASH_RESULT_2 | B12...B15 | + * +---------------+------------+ + * | HASH_RESULT_3 | B8...B11 | + * +---------------+------------+ + * | ...... | + * +---------------+------------+ + * | HASH_RESULT_14| B60...B63 | + * +---------------+------------+ + * | HASH_RESULT_15| B56...B59 | + * +---------------+------------+ + * + */ + if (ctx->alg == SE_ALG_SHA384 || ctx->alg == SE_ALG_SHA512) + idx = (j % 2) ? j - 1 : j + 1; + + /* For SHA-1, SHA-224, SHA-256, SHA-384, SHA-512 the initial + * intermediate and final hash value when stored in + * SHA_HASH_RESULT registers, the byte order is NOT in + * little-endian. + */ + if (ctx->alg <= SE_ALG_SHA512) + cpuvaddr[i++] = be32_to_cpu(res_be[idx]); + else + cpuvaddr[i++] = res[idx]; + } + + return i; +} + +static int tegra_sha_prep_cmd(struct tegra_sha_ctx *ctx, u32 *cpuvaddr, struct tegra_sha_reqctx *rctx) { + struct tegra_se *se = ctx->se; u64 msg_len, msg_left; int i = 0; @@ -241,7 +297,7 @@ static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, cpuvaddr[i++] = upper_32_bits(msg_left); cpuvaddr[i++] = 0; cpuvaddr[i++] = 0; - cpuvaddr[i++] = host1x_opcode_setpayload(6); + cpuvaddr[i++] = host1x_opcode_setpayload(2); cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_CFG); cpuvaddr[i++] = rctx->config; @@ -249,15 +305,29 @@ static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, cpuvaddr[i++] = SE_SHA_TASK_HASH_INIT; rctx->task &= ~SHA_FIRST; } else { - cpuvaddr[i++] = 0; + /* + * If it isn't the first task, program the HASH_RESULT register + * with the intermediate result from the previous task + */ + i += tegra_se_insert_hash_result(ctx, cpuvaddr + i, rctx); } + cpuvaddr[i++] = host1x_opcode_setpayload(4); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_IN_ADDR); cpuvaddr[i++] = rctx->datbuf.addr; cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) | SE_ADDR_HI_SZ(rctx->datbuf.size)); - cpuvaddr[i++] = rctx->digest.addr; - cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->digest.addr)) | - SE_ADDR_HI_SZ(rctx->digest.size)); + + if (rctx->task & SHA_UPDATE) { + cpuvaddr[i++] = rctx->intr_res.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->intr_res.addr)) | + SE_ADDR_HI_SZ(rctx->intr_res.size)); + } else { + cpuvaddr[i++] = rctx->digest.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->digest.addr)) | + SE_ADDR_HI_SZ(rctx->digest.size)); + } + if (rctx->key_id) { cpuvaddr[i++] = host1x_opcode_setpayload(1); cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_CRYPTO_CFG); @@ -266,42 +336,73 @@ static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, cpuvaddr[i++] = host1x_opcode_setpayload(1); cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_OPERATION); - cpuvaddr[i++] = SE_SHA_OP_WRSTALL | - SE_SHA_OP_START | + cpuvaddr[i++] = SE_SHA_OP_WRSTALL | SE_SHA_OP_START | SE_SHA_OP_LASTBUF; cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); - dev_dbg(se->dev, "msg len %llu msg left %llu cfg %#x", - msg_len, msg_left, rctx->config); + dev_dbg(se->dev, "msg len %llu msg left %llu sz %zd cfg %#x", + msg_len, msg_left, rctx->datbuf.size, rctx->config); return i; } -static void tegra_sha_copy_hash_result(struct tegra_se *se, struct tegra_sha_reqctx *rctx) +static int tegra_sha_do_init(struct ahash_request *req) { - int i; + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; - for (i = 0; i < HASH_RESULT_REG_COUNT; i++) - rctx->result[i] = readl(se->base + se->hw->regs->result + (i * 4)); -} + if (ctx->fallback) + return tegra_sha_fallback_init(req); -static void tegra_sha_paste_hash_result(struct tegra_se *se, struct tegra_sha_reqctx *rctx) -{ - int i; + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->key_id = ctx->key_id; + rctx->task |= SHA_FIRST; + rctx->alg = ctx->alg; + rctx->blk_size = crypto_ahash_blocksize(tfm); + rctx->digest.size = crypto_ahash_digestsize(tfm); + + rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, + &rctx->digest.addr, GFP_KERNEL); + if (!rctx->digest.buf) + goto digbuf_fail; + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + goto resbuf_fail; + + rctx->intr_res.size = HASH_RESULT_REG_COUNT * 4; + rctx->intr_res.buf = dma_alloc_coherent(se->dev, rctx->intr_res.size, + &rctx->intr_res.addr, GFP_KERNEL); + if (!rctx->intr_res.buf) + goto intr_res_fail; - for (i = 0; i < HASH_RESULT_REG_COUNT; i++) - writel(rctx->result[i], - se->base + se->hw->regs->result + (i * 4)); + return 0; + +intr_res_fail: + dma_free_coherent(se->dev, rctx->residue.size, rctx->residue.buf, + rctx->residue.addr); +resbuf_fail: + dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, + rctx->digest.addr); +digbuf_fail: + return -ENOMEM; } static int tegra_sha_do_update(struct ahash_request *req) { struct tegra_sha_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); - unsigned int nblks, nresidue, size, ret; - u32 *cpuvaddr = ctx->se->cmdbuf->addr; + struct tegra_se *se = ctx->se; + unsigned int nblks, nresidue, size; + u32 *cpuvaddr = se->cmdbuf->addr; + int ret; nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; @@ -317,7 +418,6 @@ static int tegra_sha_do_update(struct ahash_request *req) rctx->src_sg = req->src; rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; - rctx->total_len += rctx->datbuf.size; /* * If nbytes are less than a block size, copy it residue and @@ -326,11 +426,16 @@ static int tegra_sha_do_update(struct ahash_request *req) if (nblks < 1) { scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size, rctx->src_sg, 0, req->nbytes, 0); - rctx->residue.size += req->nbytes; + return 0; } + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + return -ENOMEM; + /* Copy the previous residue first */ if (rctx->residue.size) memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); @@ -343,29 +448,16 @@ static int tegra_sha_do_update(struct ahash_request *req) /* Update residue value with the residue after current block */ rctx->residue.size = nresidue; + rctx->total_len += rctx->datbuf.size; rctx->config = tegra_sha_get_config(rctx->alg) | - SE_SHA_DST_HASH_REG; + SE_SHA_DST_MEMORY; - /* - * If this is not the first 'update' call, paste the previous copied - * intermediate results to the registers so that it gets picked up. - * This is to support the import/export functionality. - */ - if (!(rctx->task & SHA_FIRST)) - tegra_sha_paste_hash_result(ctx->se, rctx); - - size = tegra_sha_prep_cmd(ctx->se, cpuvaddr, rctx); - - ret = tegra_se_host1x_submit(ctx->se, size); + size = tegra_sha_prep_cmd(ctx, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, size); - /* - * If this is not the final update, copy the intermediate results - * from the registers so that it can be used in the next 'update' - * call. This is to support the import/export functionality. - */ - if (!(rctx->task & SHA_FINAL)) - tegra_sha_copy_hash_result(ctx->se, rctx); + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); return ret; } @@ -379,16 +471,25 @@ static int tegra_sha_do_final(struct ahash_request *req) u32 *cpuvaddr = se->cmdbuf->addr; int size, ret = 0; - memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + if (rctx->residue.size) { + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + } + rctx->datbuf.size = rctx->residue.size; rctx->total_len += rctx->residue.size; rctx->config = tegra_sha_get_config(rctx->alg) | SE_SHA_DST_MEMORY; - size = tegra_sha_prep_cmd(se, cpuvaddr, rctx); - - ret = tegra_se_host1x_submit(se, size); + size = tegra_sha_prep_cmd(ctx, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, size); if (ret) goto out; @@ -396,12 +497,18 @@ static int tegra_sha_do_final(struct ahash_request *req) memcpy(req->result, rctx->digest.buf, rctx->digest.size); out: - dma_free_coherent(se->dev, SE_SHA_BUFLEN, - rctx->datbuf.buf, rctx->datbuf.addr); + if (rctx->residue.size) + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); +out_free: dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm), rctx->residue.buf, rctx->residue.addr); dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, rctx->digest.addr); + + dma_free_coherent(se->dev, rctx->intr_res.size, rctx->intr_res.buf, + rctx->intr_res.addr); + return ret; } @@ -414,16 +521,31 @@ static int tegra_sha_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_se *se = ctx->se; int ret = 0; + if (rctx->task & SHA_INIT) { + ret = tegra_sha_do_init(req); + if (ret) + goto out; + + rctx->task &= ~SHA_INIT; + } + if (rctx->task & SHA_UPDATE) { ret = tegra_sha_do_update(req); + if (ret) + goto out; + rctx->task &= ~SHA_UPDATE; } if (rctx->task & SHA_FINAL) { ret = tegra_sha_do_final(req); + if (ret) + goto out; + rctx->task &= ~SHA_FINAL; } +out: crypto_finalize_hash_request(se->engine, req, ret); return 0; @@ -497,52 +619,6 @@ static void tegra_sha_cra_exit(struct crypto_tfm *tfm) tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); } -static int tegra_sha_init(struct ahash_request *req) -{ - struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); - struct tegra_se *se = ctx->se; - - if (ctx->fallback) - return tegra_sha_fallback_init(req); - - rctx->total_len = 0; - rctx->datbuf.size = 0; - rctx->residue.size = 0; - rctx->key_id = ctx->key_id; - rctx->task = SHA_FIRST; - rctx->alg = ctx->alg; - rctx->blk_size = crypto_ahash_blocksize(tfm); - rctx->digest.size = crypto_ahash_digestsize(tfm); - - rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, - &rctx->digest.addr, GFP_KERNEL); - if (!rctx->digest.buf) - goto digbuf_fail; - - rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size, - &rctx->residue.addr, GFP_KERNEL); - if (!rctx->residue.buf) - goto resbuf_fail; - - rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_SHA_BUFLEN, - &rctx->datbuf.addr, GFP_KERNEL); - if (!rctx->datbuf.buf) - goto datbuf_fail; - - return 0; - -datbuf_fail: - dma_free_coherent(se->dev, rctx->blk_size, rctx->residue.buf, - rctx->residue.addr); -resbuf_fail: - dma_free_coherent(se->dev, SE_SHA_BUFLEN, rctx->datbuf.buf, - rctx->datbuf.addr); -digbuf_fail: - return -ENOMEM; -} - static int tegra_hmac_fallback_setkey(struct tegra_sha_ctx *ctx, const u8 *key, unsigned int keylen) { @@ -559,13 +635,29 @@ static int tegra_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) return tegra_hmac_fallback_setkey(ctx, key, keylen); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) + return tegra_hmac_fallback_setkey(ctx, key, keylen); + ctx->fallback = false; - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + return 0; +} + +static int tegra_sha_init(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + rctx->task = SHA_INIT; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } static int tegra_sha_update(struct ahash_request *req) @@ -619,8 +711,7 @@ static int tegra_sha_digest(struct ahash_request *req) if (ctx->fallback) return tegra_sha_fallback_digest(req); - tegra_sha_init(req); - rctx->task |= SHA_UPDATE | SHA_FINAL; + rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL; return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } diff --git a/drivers/crypto/tegra/tegra-se-key.c b/drivers/crypto/tegra/tegra-se-key.c index ac14678dbd30..956fa9b4e9b1 100644 --- a/drivers/crypto/tegra/tegra-se-key.c +++ b/drivers/crypto/tegra/tegra-se-key.c @@ -115,11 +115,17 @@ static int tegra_key_insert(struct tegra_se *se, const u8 *key, u32 keylen, u16 slot, u32 alg) { const u32 *keyval = (u32 *)key; - u32 *addr = se->cmdbuf->addr, size; + u32 *addr = se->keybuf->addr, size; + int ret; + + mutex_lock(&kslt_lock); size = tegra_key_prep_ins_cmd(se, addr, keyval, keylen, slot, alg); + ret = tegra_se_host1x_submit(se, se->keybuf, size); + + mutex_unlock(&kslt_lock); - return tegra_se_host1x_submit(se, size); + return ret; } void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg) @@ -135,6 +141,23 @@ void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg) tegra_keyslot_free(keyid); } +void tegra_key_invalidate_reserved(struct tegra_se *se, u32 keyid, u32 alg) +{ + u8 zkey[AES_MAX_KEY_SIZE] = {0}; + + if (!keyid) + return; + + /* Overwrite the key with 0s */ + tegra_key_insert(se, zkey, AES_MAX_KEY_SIZE, keyid, alg); +} + +inline int tegra_key_submit_reserved(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + return tegra_key_insert(se, key, keylen, *keyid, alg); +} + int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u32 *keyid) { int ret; @@ -143,7 +166,7 @@ int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u3 if (!tegra_key_in_kslt(*keyid)) { *keyid = tegra_keyslot_alloc(); if (!(*keyid)) { - dev_err(se->dev, "failed to allocate key slot\n"); + dev_dbg(se->dev, "failed to allocate key slot\n"); return -ENOMEM; } } diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index 918c0b10614d..7237f14eaf5a 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -141,7 +141,7 @@ static struct tegra_se_cmdbuf *tegra_se_host1x_bo_alloc(struct tegra_se *se, ssi return cmdbuf; } -int tegra_se_host1x_submit(struct tegra_se *se, u32 size) +int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, u32 size) { struct host1x_job *job; int ret; @@ -160,9 +160,9 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size) job->engine_fallback_streamid = se->stream_id; job->engine_streamid_offset = SE_STREAM_ID; - se->cmdbuf->words = size; + cmdbuf->words = size; - host1x_job_add_gather(job, &se->cmdbuf->bo, size, 0); + host1x_job_add_gather(job, &cmdbuf->bo, size, 0); ret = host1x_job_pin(job, se->dev); if (ret) { @@ -220,14 +220,22 @@ static int tegra_se_client_init(struct host1x_client *client) goto syncpt_put; } + se->keybuf = tegra_se_host1x_bo_alloc(se, SZ_4K); + if (!se->keybuf) { + ret = -ENOMEM; + goto cmdbuf_put; + } + ret = se->hw->init_alg(se); if (ret) { dev_err(se->dev, "failed to register algorithms\n"); - goto cmdbuf_put; + goto keybuf_put; } return 0; +keybuf_put: + tegra_se_cmdbuf_put(&se->keybuf->bo); cmdbuf_put: tegra_se_cmdbuf_put(&se->cmdbuf->bo); syncpt_put: @@ -302,7 +310,7 @@ static int tegra_se_probe(struct platform_device *pdev) se->engine = crypto_engine_alloc_init(dev, 0); if (!se->engine) - return dev_err_probe(dev, -ENOMEM, "failed to init crypto engine\n"); + return -ENOMEM; ret = crypto_engine_start(se->engine); if (ret) { diff --git a/drivers/crypto/tegra/tegra-se.h b/drivers/crypto/tegra/tegra-se.h index b9dd7ceb8783..b6cac9384f66 100644 --- a/drivers/crypto/tegra/tegra-se.h +++ b/drivers/crypto/tegra/tegra-se.h @@ -24,6 +24,7 @@ #define SE_STREAM_ID 0x90 #define SE_SHA_CFG 0x4004 +#define SE_SHA_IN_ADDR 0x400c #define SE_SHA_KEY_ADDR 0x4094 #define SE_SHA_KEY_DATA 0x4098 #define SE_SHA_KEYMANIFEST 0x409c @@ -340,12 +341,14 @@ #define SE_CRYPTO_CTR_REG_COUNT 4 #define SE_MAX_KEYSLOT 15 #define SE_MAX_MEM_ALLOC SZ_4M -#define SE_AES_BUFLEN 0x8000 -#define SE_SHA_BUFLEN 0x2000 + +#define TEGRA_AES_RESERVED_KSLT 14 +#define TEGRA_XTS_RESERVED_KSLT 15 #define SHA_FIRST BIT(0) -#define SHA_UPDATE BIT(1) -#define SHA_FINAL BIT(2) +#define SHA_INIT BIT(1) +#define SHA_UPDATE BIT(2) +#define SHA_FINAL BIT(3) /* Security Engine operation modes */ enum se_aes_alg { @@ -420,6 +423,7 @@ struct tegra_se { struct host1x_client client; struct host1x_channel *channel; struct tegra_se_cmdbuf *cmdbuf; + struct tegra_se_cmdbuf *keybuf; struct crypto_engine *engine; struct host1x_syncpt *syncpt; struct device *dev; @@ -501,8 +505,33 @@ void tegra_deinit_aes(struct tegra_se *se); void tegra_deinit_hash(struct tegra_se *se); int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u32 *keyid); + +int tegra_key_submit_reserved(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid); + void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg); -int tegra_se_host1x_submit(struct tegra_se *se, u32 size); +void tegra_key_invalidate_reserved(struct tegra_se *se, u32 keyid, u32 alg); +int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, u32 size); + +static inline int tegra_key_submit_reserved_aes(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + *keyid = TEGRA_AES_RESERVED_KSLT; + return tegra_key_submit_reserved(se, key, keylen, alg, keyid); +} + +static inline int tegra_key_submit_reserved_xts(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + *keyid = TEGRA_XTS_RESERVED_KSLT; + return tegra_key_submit_reserved(se, key, keylen, alg, keyid); +} + +static inline bool tegra_key_is_reserved(u32 keyid) +{ + return ((keyid == TEGRA_AES_RESERVED_KSLT) || + (keyid == TEGRA_XTS_RESERVED_KSLT)); +} /* HOST1x OPCODES */ static inline u32 host1x_opcode_setpayload(unsigned int payload) diff --git a/drivers/crypto/ti/Kconfig b/drivers/crypto/ti/Kconfig new file mode 100644 index 000000000000..a3692ceec49b --- /dev/null +++ b/drivers/crypto/ti/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_TI_DTHEV2 + tristate "Support for TI DTHE V2 cryptography engine" + depends on ARCH_K3 || COMPILE_TEST + select CRYPTO_ENGINE + select CRYPTO_SKCIPHER + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_XTS + help + This enables support for the TI DTHE V2 hw cryptography engine + which can be found on TI K3 SOCs. Selecting this enables use + of hardware offloading for cryptographic algorithms on + these devices, providing enhanced resistance against side-channel + attacks. diff --git a/drivers/crypto/ti/Makefile b/drivers/crypto/ti/Makefile new file mode 100644 index 000000000000..b883078f203d --- /dev/null +++ b/drivers/crypto/ti/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_TI_DTHEV2) += dthev2.o +dthev2-objs := dthev2-common.o dthev2-aes.o diff --git a/drivers/crypto/ti/dthev2-aes.c b/drivers/crypto/ti/dthev2-aes.c new file mode 100644 index 000000000000..156729ccc50e --- /dev/null +++ b/drivers/crypto/ti/dthev2-aes.c @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham <t-pratham@ti.com> + */ + +#include <crypto/aead.h> +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/engine.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/skcipher.h> + +#include "dthev2-common.h" + +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/scatterlist.h> + +/* Registers */ + +// AES Engine +#define DTHE_P_AES_BASE 0x7000 + +#define DTHE_P_AES_KEY1_0 0x0038 +#define DTHE_P_AES_KEY1_1 0x003C +#define DTHE_P_AES_KEY1_2 0x0030 +#define DTHE_P_AES_KEY1_3 0x0034 +#define DTHE_P_AES_KEY1_4 0x0028 +#define DTHE_P_AES_KEY1_5 0x002C +#define DTHE_P_AES_KEY1_6 0x0020 +#define DTHE_P_AES_KEY1_7 0x0024 + +#define DTHE_P_AES_KEY2_0 0x0018 +#define DTHE_P_AES_KEY2_1 0x001C +#define DTHE_P_AES_KEY2_2 0x0010 +#define DTHE_P_AES_KEY2_3 0x0014 +#define DTHE_P_AES_KEY2_4 0x0008 +#define DTHE_P_AES_KEY2_5 0x000C +#define DTHE_P_AES_KEY2_6 0x0000 +#define DTHE_P_AES_KEY2_7 0x0004 + +#define DTHE_P_AES_IV_IN_0 0x0040 +#define DTHE_P_AES_IV_IN_1 0x0044 +#define DTHE_P_AES_IV_IN_2 0x0048 +#define DTHE_P_AES_IV_IN_3 0x004C +#define DTHE_P_AES_CTRL 0x0050 +#define DTHE_P_AES_C_LENGTH_0 0x0054 +#define DTHE_P_AES_C_LENGTH_1 0x0058 +#define DTHE_P_AES_AUTH_LENGTH 0x005C +#define DTHE_P_AES_DATA_IN_OUT 0x0060 + +#define DTHE_P_AES_SYSCONFIG 0x0084 +#define DTHE_P_AES_IRQSTATUS 0x008C +#define DTHE_P_AES_IRQENABLE 0x0090 + +/* Register write values and macros */ + +enum aes_ctrl_mode_masks { + AES_CTRL_ECB_MASK = 0x00, + AES_CTRL_CBC_MASK = BIT(5), + AES_CTRL_XTS_MASK = BIT(12) | BIT(11), +}; + +#define DTHE_AES_CTRL_MODE_CLEAR_MASK ~GENMASK(28, 5) + +#define DTHE_AES_CTRL_DIR_ENC BIT(2) + +#define DTHE_AES_CTRL_KEYSIZE_16B BIT(3) +#define DTHE_AES_CTRL_KEYSIZE_24B BIT(4) +#define DTHE_AES_CTRL_KEYSIZE_32B (BIT(3) | BIT(4)) + +#define DTHE_AES_CTRL_SAVE_CTX_SET BIT(29) + +#define DTHE_AES_CTRL_OUTPUT_READY BIT_MASK(0) +#define DTHE_AES_CTRL_INPUT_READY BIT_MASK(1) +#define DTHE_AES_CTRL_SAVED_CTX_READY BIT_MASK(30) +#define DTHE_AES_CTRL_CTX_READY BIT_MASK(31) + +#define DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN GENMASK(6, 5) +#define DTHE_AES_IRQENABLE_EN_ALL GENMASK(3, 0) + +/* Misc */ +#define AES_IV_SIZE AES_BLOCK_SIZE +#define AES_BLOCK_WORDS (AES_BLOCK_SIZE / sizeof(u32)) +#define AES_IV_WORDS AES_BLOCK_WORDS + +static int dthe_cipher_init_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + ctx->dev_data = dev_data; + ctx->keylen = 0; + + return 0; +} + +static int dthe_cipher_xts_init_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + ctx->dev_data = dev_data; + ctx->keylen = 0; + + ctx->skcipher_fb = crypto_alloc_sync_skcipher("xts(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->skcipher_fb)) { + dev_err(dev_data->dev, "fallback driver xts(aes) couldn't be loaded\n"); + return PTR_ERR(ctx->skcipher_fb); + } + + return 0; +} + +static void dthe_cipher_xts_exit_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_sync_skcipher(ctx->skcipher_fb); +} + +static int dthe_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + return -EINVAL; + + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return 0; +} + +static int dthe_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->aes_mode = DTHE_AES_ECB; + + return dthe_aes_setkey(tfm, key, keylen); +} + +static int dthe_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->aes_mode = DTHE_AES_CBC; + + return dthe_aes_setkey(tfm, key, keylen); +} + +static int dthe_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != 2 * AES_KEYSIZE_128 && + keylen != 2 * AES_KEYSIZE_192 && + keylen != 2 * AES_KEYSIZE_256) + return -EINVAL; + + ctx->aes_mode = DTHE_AES_XTS; + ctx->keylen = keylen / 2; + memcpy(ctx->key, key, keylen); + + crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(ctx->skcipher_fb, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen); +} + +static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx, + struct dthe_aes_req_ctx *rctx, + u32 *iv_in) +{ + struct dthe_data *dev_data = dthe_get_dev(ctx); + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + u32 ctrl_val = 0; + + writel_relaxed(ctx->key[0], aes_base_reg + DTHE_P_AES_KEY1_0); + writel_relaxed(ctx->key[1], aes_base_reg + DTHE_P_AES_KEY1_1); + writel_relaxed(ctx->key[2], aes_base_reg + DTHE_P_AES_KEY1_2); + writel_relaxed(ctx->key[3], aes_base_reg + DTHE_P_AES_KEY1_3); + + if (ctx->keylen > AES_KEYSIZE_128) { + writel_relaxed(ctx->key[4], aes_base_reg + DTHE_P_AES_KEY1_4); + writel_relaxed(ctx->key[5], aes_base_reg + DTHE_P_AES_KEY1_5); + } + if (ctx->keylen == AES_KEYSIZE_256) { + writel_relaxed(ctx->key[6], aes_base_reg + DTHE_P_AES_KEY1_6); + writel_relaxed(ctx->key[7], aes_base_reg + DTHE_P_AES_KEY1_7); + } + + if (ctx->aes_mode == DTHE_AES_XTS) { + size_t key2_offset = ctx->keylen / sizeof(u32); + + writel_relaxed(ctx->key[key2_offset + 0], aes_base_reg + DTHE_P_AES_KEY2_0); + writel_relaxed(ctx->key[key2_offset + 1], aes_base_reg + DTHE_P_AES_KEY2_1); + writel_relaxed(ctx->key[key2_offset + 2], aes_base_reg + DTHE_P_AES_KEY2_2); + writel_relaxed(ctx->key[key2_offset + 3], aes_base_reg + DTHE_P_AES_KEY2_3); + + if (ctx->keylen > AES_KEYSIZE_128) { + writel_relaxed(ctx->key[key2_offset + 4], aes_base_reg + DTHE_P_AES_KEY2_4); + writel_relaxed(ctx->key[key2_offset + 5], aes_base_reg + DTHE_P_AES_KEY2_5); + } + if (ctx->keylen == AES_KEYSIZE_256) { + writel_relaxed(ctx->key[key2_offset + 6], aes_base_reg + DTHE_P_AES_KEY2_6); + writel_relaxed(ctx->key[key2_offset + 7], aes_base_reg + DTHE_P_AES_KEY2_7); + } + } + + if (rctx->enc) + ctrl_val |= DTHE_AES_CTRL_DIR_ENC; + + if (ctx->keylen == AES_KEYSIZE_128) + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_16B; + else if (ctx->keylen == AES_KEYSIZE_192) + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_24B; + else + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_32B; + + // Write AES mode + ctrl_val &= DTHE_AES_CTRL_MODE_CLEAR_MASK; + switch (ctx->aes_mode) { + case DTHE_AES_ECB: + ctrl_val |= AES_CTRL_ECB_MASK; + break; + case DTHE_AES_CBC: + ctrl_val |= AES_CTRL_CBC_MASK; + break; + case DTHE_AES_XTS: + ctrl_val |= AES_CTRL_XTS_MASK; + break; + } + + if (iv_in) { + ctrl_val |= DTHE_AES_CTRL_SAVE_CTX_SET; + for (int i = 0; i < AES_IV_WORDS; ++i) + writel_relaxed(iv_in[i], + aes_base_reg + DTHE_P_AES_IV_IN_0 + (DTHE_REG_SIZE * i)); + } + + writel_relaxed(ctrl_val, aes_base_reg + DTHE_P_AES_CTRL); +} + +static void dthe_aes_dma_in_callback(void *data) +{ + struct skcipher_request *req = (struct skcipher_request *)data; + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + complete(&rctx->aes_compl); +} + +static int dthe_aes_run(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_data *dev_data = dthe_get_dev(ctx); + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + unsigned int len = req->cryptlen; + struct scatterlist *src = req->src; + struct scatterlist *dst = req->dst; + + int src_nents = sg_nents_for_len(src, len); + int dst_nents; + + int src_mapped_nents; + int dst_mapped_nents; + + bool diff_dst; + enum dma_data_direction src_dir, dst_dir; + + struct device *tx_dev, *rx_dev; + struct dma_async_tx_descriptor *desc_in, *desc_out; + + int ret; + + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + + u32 aes_irqenable_val = readl_relaxed(aes_base_reg + DTHE_P_AES_IRQENABLE); + u32 aes_sysconfig_val = readl_relaxed(aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_sysconfig_val |= DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN; + writel_relaxed(aes_sysconfig_val, aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_irqenable_val |= DTHE_AES_IRQENABLE_EN_ALL; + writel_relaxed(aes_irqenable_val, aes_base_reg + DTHE_P_AES_IRQENABLE); + + if (src == dst) { + diff_dst = false; + src_dir = DMA_BIDIRECTIONAL; + dst_dir = DMA_BIDIRECTIONAL; + } else { + diff_dst = true; + src_dir = DMA_TO_DEVICE; + dst_dir = DMA_FROM_DEVICE; + } + + tx_dev = dmaengine_get_dma_device(dev_data->dma_aes_tx); + rx_dev = dmaengine_get_dma_device(dev_data->dma_aes_rx); + + src_mapped_nents = dma_map_sg(tx_dev, src, src_nents, src_dir); + if (src_mapped_nents == 0) { + ret = -EINVAL; + goto aes_err; + } + + if (!diff_dst) { + dst_nents = src_nents; + dst_mapped_nents = src_mapped_nents; + } else { + dst_nents = sg_nents_for_len(dst, len); + dst_mapped_nents = dma_map_sg(rx_dev, dst, dst_nents, dst_dir); + if (dst_mapped_nents == 0) { + dma_unmap_sg(tx_dev, src, src_nents, src_dir); + ret = -EINVAL; + goto aes_err; + } + } + + desc_in = dmaengine_prep_slave_sg(dev_data->dma_aes_rx, dst, dst_mapped_nents, + DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_in) { + dev_err(dev_data->dev, "IN prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aes_prep_err; + } + + desc_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, src, src_mapped_nents, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_out) { + dev_err(dev_data->dev, "OUT prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aes_prep_err; + } + + desc_in->callback = dthe_aes_dma_in_callback; + desc_in->callback_param = req; + + init_completion(&rctx->aes_compl); + + if (ctx->aes_mode == DTHE_AES_ECB) + dthe_aes_set_ctrl_key(ctx, rctx, NULL); + else + dthe_aes_set_ctrl_key(ctx, rctx, (u32 *)req->iv); + + writel_relaxed(lower_32_bits(req->cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_0); + writel_relaxed(upper_32_bits(req->cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_1); + + dmaengine_submit(desc_in); + dmaengine_submit(desc_out); + + dma_async_issue_pending(dev_data->dma_aes_rx); + dma_async_issue_pending(dev_data->dma_aes_tx); + + // Need to do a timeout to ensure finalise gets called if DMA callback fails for any reason + ret = wait_for_completion_timeout(&rctx->aes_compl, msecs_to_jiffies(DTHE_DMA_TIMEOUT_MS)); + if (!ret) { + ret = -ETIMEDOUT; + dmaengine_terminate_sync(dev_data->dma_aes_rx); + dmaengine_terminate_sync(dev_data->dma_aes_tx); + + for (int i = 0; i < AES_BLOCK_WORDS; ++i) + readl_relaxed(aes_base_reg + DTHE_P_AES_DATA_IN_OUT + (DTHE_REG_SIZE * i)); + } else { + ret = 0; + } + + // For modes other than ECB, read IV_OUT + if (ctx->aes_mode != DTHE_AES_ECB) { + u32 *iv_out = (u32 *)req->iv; + + for (int i = 0; i < AES_IV_WORDS; ++i) + iv_out[i] = readl_relaxed(aes_base_reg + + DTHE_P_AES_IV_IN_0 + + (DTHE_REG_SIZE * i)); + } + +aes_prep_err: + dma_unmap_sg(tx_dev, src, src_nents, src_dir); + if (dst_dir != DMA_BIDIRECTIONAL) + dma_unmap_sg(rx_dev, dst, dst_nents, dst_dir); + +aes_err: + local_bh_disable(); + crypto_finalize_skcipher_request(dev_data->engine, req, ret); + local_bh_enable(); + return 0; +} + +static int dthe_aes_crypt(struct skcipher_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + struct dthe_data *dev_data = dthe_get_dev(ctx); + struct crypto_engine *engine; + + /* + * If data is not a multiple of AES_BLOCK_SIZE: + * - need to return -EINVAL for ECB, CBC as they are block ciphers + * - need to fallback to software as H/W doesn't support Ciphertext Stealing for XTS + */ + if (req->cryptlen % AES_BLOCK_SIZE) { + if (ctx->aes_mode == DTHE_AES_XTS) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->skcipher_fb); + + skcipher_request_set_callback(subreq, skcipher_request_flags(req), + req->base.complete, req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + return rctx->enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + } + return -EINVAL; + } + + /* + * If data length input is zero, no need to do any operation. + * Except for XTS mode, where data length should be non-zero. + */ + if (req->cryptlen == 0) { + if (ctx->aes_mode == DTHE_AES_XTS) + return -EINVAL; + return 0; + } + + engine = dev_data->engine; + return crypto_transfer_skcipher_request_to_engine(engine, req); +} + +static int dthe_aes_encrypt(struct skcipher_request *req) +{ + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + rctx->enc = 1; + return dthe_aes_crypt(req); +} + +static int dthe_aes_decrypt(struct skcipher_request *req) +{ + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + rctx->enc = 0; + return dthe_aes_crypt(req); +} + +static struct skcipher_engine_alg cipher_algs[] = { + { + .base.init = dthe_cipher_init_tfm, + .base.setkey = dthe_aes_ecb_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* ECB AES */ + { + .base.init = dthe_cipher_init_tfm, + .base.setkey = dthe_aes_cbc_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* CBC AES */ + { + .base.init = dthe_cipher_xts_init_tfm, + .base.exit = dthe_cipher_xts_exit_tfm, + .base.setkey = dthe_aes_xts_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE * 2, + .base.max_keysize = AES_MAX_KEY_SIZE * 2, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* XTS AES */ +}; + +int dthe_register_aes_algs(void) +{ + return crypto_engine_register_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); +} + +void dthe_unregister_aes_algs(void) +{ + crypto_engine_unregister_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); +} diff --git a/drivers/crypto/ti/dthev2-common.c b/drivers/crypto/ti/dthev2-common.c new file mode 100644 index 000000000000..c39d37933b9e --- /dev/null +++ b/drivers/crypto/ti/dthev2-common.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham <t-pratham@ti.com> + */ + +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/engine.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/skcipher.h> + +#include "dthev2-common.h" + +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <linux/scatterlist.h> + +#define DRIVER_NAME "dthev2" + +static struct dthe_list dthe_dev_list = { + .dev_list = LIST_HEAD_INIT(dthe_dev_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(dthe_dev_list.lock), +}; + +struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx) +{ + struct dthe_data *dev_data; + + if (ctx->dev_data) + return ctx->dev_data; + + spin_lock_bh(&dthe_dev_list.lock); + dev_data = list_first_entry(&dthe_dev_list.dev_list, struct dthe_data, list); + if (dev_data) + list_move_tail(&dev_data->list, &dthe_dev_list.dev_list); + spin_unlock_bh(&dthe_dev_list.lock); + + return dev_data; +} + +static int dthe_dma_init(struct dthe_data *dev_data) +{ + int ret; + struct dma_slave_config cfg; + + dev_data->dma_aes_rx = NULL; + dev_data->dma_aes_tx = NULL; + dev_data->dma_sha_tx = NULL; + + dev_data->dma_aes_rx = dma_request_chan(dev_data->dev, "rx"); + if (IS_ERR(dev_data->dma_aes_rx)) { + return dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_aes_rx), + "Unable to request rx DMA channel\n"); + } + + dev_data->dma_aes_tx = dma_request_chan(dev_data->dev, "tx1"); + if (IS_ERR(dev_data->dma_aes_tx)) { + ret = dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_aes_tx), + "Unable to request tx1 DMA channel\n"); + goto err_dma_aes_tx; + } + + dev_data->dma_sha_tx = dma_request_chan(dev_data->dev, "tx2"); + if (IS_ERR(dev_data->dma_sha_tx)) { + ret = dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_sha_tx), + "Unable to request tx2 DMA channel\n"); + goto err_dma_sha_tx; + } + + memzero_explicit(&cfg, sizeof(cfg)); + + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = 4; + + ret = dmaengine_slave_config(dev_data->dma_aes_rx, &cfg); + if (ret) { + dev_err(dev_data->dev, "Can't configure IN dmaengine slave: %d\n", ret); + goto err_dma_config; + } + + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_maxburst = 4; + + ret = dmaengine_slave_config(dev_data->dma_aes_tx, &cfg); + if (ret) { + dev_err(dev_data->dev, "Can't configure OUT dmaengine slave: %d\n", ret); + goto err_dma_config; + } + + return 0; + +err_dma_config: + dma_release_channel(dev_data->dma_sha_tx); +err_dma_sha_tx: + dma_release_channel(dev_data->dma_aes_tx); +err_dma_aes_tx: + dma_release_channel(dev_data->dma_aes_rx); + + return ret; +} + +static int dthe_register_algs(void) +{ + return dthe_register_aes_algs(); +} + +static void dthe_unregister_algs(void) +{ + dthe_unregister_aes_algs(); +} + +static int dthe_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dthe_data *dev_data; + int ret; + + dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return -ENOMEM; + + dev_data->dev = dev; + dev_data->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dev_data->regs)) + return PTR_ERR(dev_data->regs); + + platform_set_drvdata(pdev, dev_data); + + spin_lock(&dthe_dev_list.lock); + list_add(&dev_data->list, &dthe_dev_list.dev_list); + spin_unlock(&dthe_dev_list.lock); + + ret = dthe_dma_init(dev_data); + if (ret) + goto probe_dma_err; + + dev_data->engine = crypto_engine_alloc_init(dev, 1); + if (!dev_data->engine) { + ret = -ENOMEM; + goto probe_engine_err; + } + + ret = crypto_engine_start(dev_data->engine); + if (ret) { + dev_err(dev, "Failed to start crypto engine\n"); + goto probe_engine_start_err; + } + + ret = dthe_register_algs(); + if (ret) { + dev_err(dev, "Failed to register algs\n"); + goto probe_engine_start_err; + } + + return 0; + +probe_engine_start_err: + crypto_engine_exit(dev_data->engine); +probe_engine_err: + dma_release_channel(dev_data->dma_aes_rx); + dma_release_channel(dev_data->dma_aes_tx); + dma_release_channel(dev_data->dma_sha_tx); +probe_dma_err: + spin_lock(&dthe_dev_list.lock); + list_del(&dev_data->list); + spin_unlock(&dthe_dev_list.lock); + + return ret; +} + +static void dthe_remove(struct platform_device *pdev) +{ + struct dthe_data *dev_data = platform_get_drvdata(pdev); + + spin_lock(&dthe_dev_list.lock); + list_del(&dev_data->list); + spin_unlock(&dthe_dev_list.lock); + + dthe_unregister_algs(); + + crypto_engine_exit(dev_data->engine); + + dma_release_channel(dev_data->dma_aes_rx); + dma_release_channel(dev_data->dma_aes_tx); + dma_release_channel(dev_data->dma_sha_tx); +} + +static const struct of_device_id dthe_of_match[] = { + { .compatible = "ti,am62l-dthev2", }, + {}, +}; +MODULE_DEVICE_TABLE(of, dthe_of_match); + +static struct platform_driver dthe_driver = { + .probe = dthe_probe, + .remove = dthe_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = dthe_of_match, + }, +}; + +module_platform_driver(dthe_driver); + +MODULE_AUTHOR("T Pratham <t-pratham@ti.com>"); +MODULE_DESCRIPTION("Texas Instruments DTHE V2 driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/ti/dthev2-common.h b/drivers/crypto/ti/dthev2-common.h new file mode 100644 index 000000000000..c7a06a4c353f --- /dev/null +++ b/drivers/crypto/ti/dthev2-common.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham <t-pratham@ti.com> + */ + +#ifndef __TI_DTHEV2_H__ +#define __TI_DTHEV2_H__ + +#include <crypto/aead.h> +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/engine.h> +#include <crypto/hash.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> + +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/scatterlist.h> + +#define DTHE_REG_SIZE 4 +#define DTHE_DMA_TIMEOUT_MS 2000 +/* + * Size of largest possible key (of all algorithms) to be stored in dthe_tfm_ctx + * This is currently the keysize of XTS-AES-256 which is 512 bits (64 bytes) + */ +#define DTHE_MAX_KEYSIZE (AES_MAX_KEY_SIZE * 2) + +enum dthe_aes_mode { + DTHE_AES_ECB = 0, + DTHE_AES_CBC, + DTHE_AES_XTS, +}; + +/* Driver specific struct definitions */ + +/** + * struct dthe_data - DTHE_V2 driver instance data + * @dev: Device pointer + * @regs: Base address of the register space + * @list: list node for dev + * @engine: Crypto engine instance + * @dma_aes_rx: AES Rx DMA Channel + * @dma_aes_tx: AES Tx DMA Channel + * @dma_sha_tx: SHA Tx DMA Channel + */ +struct dthe_data { + struct device *dev; + void __iomem *regs; + struct list_head list; + struct crypto_engine *engine; + + struct dma_chan *dma_aes_rx; + struct dma_chan *dma_aes_tx; + + struct dma_chan *dma_sha_tx; +}; + +/** + * struct dthe_list - device data list head + * @dev_list: linked list head + * @lock: Spinlock protecting accesses to the list + */ +struct dthe_list { + struct list_head dev_list; + spinlock_t lock; +}; + +/** + * struct dthe_tfm_ctx - Transform ctx struct containing ctx for all sub-components of DTHE V2 + * @dev_data: Device data struct pointer + * @keylen: AES key length + * @key: AES key + * @aes_mode: AES mode + * @skcipher_fb: Fallback crypto skcipher handle for AES-XTS mode + */ +struct dthe_tfm_ctx { + struct dthe_data *dev_data; + unsigned int keylen; + u32 key[DTHE_MAX_KEYSIZE / sizeof(u32)]; + enum dthe_aes_mode aes_mode; + struct crypto_sync_skcipher *skcipher_fb; +}; + +/** + * struct dthe_aes_req_ctx - AES engine req ctx struct + * @enc: flag indicating encryption or decryption operation + * @aes_compl: Completion variable for use in manual completion in case of DMA callback failure + */ +struct dthe_aes_req_ctx { + int enc; + struct completion aes_compl; +}; + +/* Struct definitions end */ + +struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx); + +int dthe_register_aes_algs(void); +void dthe_unregister_aes_algs(void); + +#endif diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 48fee07b7e51..2e44915c9f23 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -21,12 +21,11 @@ #include "virtio_crypto_common.h" struct virtio_crypto_rsa_ctx { - MPI n; + unsigned int key_size; }; struct virtio_crypto_akcipher_ctx { struct virtio_crypto *vcrypto; - struct crypto_akcipher *tfm; bool session_valid; __u64 session_id; union { @@ -36,8 +35,6 @@ struct virtio_crypto_akcipher_ctx { struct virtio_crypto_akcipher_request { struct virtio_crypto_request base; - struct virtio_crypto_akcipher_ctx *akcipher_ctx; - struct akcipher_request *akcipher_req; void *src_buf; void *dst_buf; uint32_t opcode; @@ -69,7 +66,9 @@ static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request * { struct virtio_crypto_akcipher_request *vc_akcipher_req = container_of(vc_req, struct virtio_crypto_akcipher_request, base); - struct akcipher_request *akcipher_req; + struct akcipher_request *akcipher_req = + container_of((void *)vc_akcipher_req, struct akcipher_request, + __ctx); int error; switch (vc_req->status) { @@ -88,8 +87,7 @@ static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request * break; } - akcipher_req = vc_akcipher_req->akcipher_req; - /* actual length maybe less than dst buffer */ + /* actual length may be less than dst buffer */ akcipher_req->dst_len = len - sizeof(vc_req->status); sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), vc_akcipher_req->dst_buf, akcipher_req->dst_len); @@ -213,7 +211,8 @@ out: static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req, struct akcipher_request *req, struct data_queue *data_vq) { - struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx; + struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; struct virtio_crypto *vcrypto = ctx->vcrypto; struct virtio_crypto_op_data_req *req_data = vc_req->req_data; @@ -273,7 +272,8 @@ static int virtio_crypto_rsa_do_req(struct crypto_engine *engine, void *vreq) struct akcipher_request *req = container_of(vreq, struct akcipher_request, base); struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req); struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; - struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx; + struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); struct virtio_crypto *vcrypto = ctx->vcrypto; struct data_queue *data_vq = vc_req->dataq; struct virtio_crypto_op_header *header; @@ -319,8 +319,6 @@ static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode) vc_req->dataq = data_vq; vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback; - vc_akcipher_req->akcipher_ctx = ctx; - vc_akcipher_req->akcipher_req = req; vc_akcipher_req->opcode = opcode; return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req); @@ -352,10 +350,7 @@ static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm, int node = virtio_crypto_get_current_node(); uint32_t keytype; int ret; - - /* mpi_free will test n, just free it. */ - mpi_free(rsa_ctx->n); - rsa_ctx->n = NULL; + MPI n; if (private) { keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE; @@ -368,10 +363,13 @@ static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm, if (ret) return ret; - rsa_ctx->n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz); - if (!rsa_ctx->n) + n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz); + if (!n) return -ENOMEM; + rsa_ctx->key_size = mpi_get_size(n); + mpi_free(n); + if (!ctx->vcrypto) { vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER, VIRTIO_CRYPTO_AKCIPHER_RSA); @@ -442,15 +440,11 @@ static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm) struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; - return mpi_get_size(rsa_ctx->n); + return rsa_ctx->key_size; } static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm) { - struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); - - ctx->tfm = tfm; - akcipher_set_reqsize(tfm, sizeof(struct virtio_crypto_akcipher_request)); @@ -460,12 +454,9 @@ static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm) static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm) { struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); - struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; virtio_crypto_alg_akcipher_close_session(ctx); virtcrypto_dev_put(ctx->vcrypto); - mpi_free(rsa_ctx->n); - rsa_ctx->n = NULL; } static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = { diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h index 7059bbe5a2eb..19c934af3df6 100644 --- a/drivers/crypto/virtio/virtio_crypto_common.h +++ b/drivers/crypto/virtio/virtio_crypto_common.h @@ -113,8 +113,6 @@ struct virtio_crypto_request { int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev); struct list_head *virtcrypto_devmgr_get_head(void); void virtcrypto_devmgr_rm_dev(struct virtio_crypto *vcrypto_dev); -struct virtio_crypto *virtcrypto_devmgr_get_first(void); -int virtcrypto_dev_in_use(struct virtio_crypto *vcrypto_dev); int virtcrypto_dev_get(struct virtio_crypto *vcrypto_dev); void virtcrypto_dev_put(struct virtio_crypto *vcrypto_dev); int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev); diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index d0278eb568b9..3d241446099c 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -139,7 +139,7 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) spin_lock_init(&vi->data_vq[i].lock); vi->data_vq[i].vq = vqs[i]; /* Initialize crypto engine */ - vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, NULL, true, + vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, true, virtqueue_get_vring_size(vqs[i])); if (!vi->data_vq[i].engine) { ret = -ENOMEM; @@ -480,10 +480,8 @@ static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto) for (i = 0; i < vcrypto->max_data_queues; i++) { vq = vcrypto->data_vq[i].vq; - while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL) { - kfree(vc_req->req_data); - kfree(vc_req->sgs); - } + while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL) + virtcrypto_clear_request(vc_req); cond_resched(); } } diff --git a/drivers/crypto/virtio/virtio_crypto_mgr.c b/drivers/crypto/virtio/virtio_crypto_mgr.c index 70e778aac0f2..06c74fa132cd 100644 --- a/drivers/crypto/virtio/virtio_crypto_mgr.c +++ b/drivers/crypto/virtio/virtio_crypto_mgr.c @@ -82,42 +82,6 @@ void virtcrypto_devmgr_rm_dev(struct virtio_crypto *vcrypto_dev) } /* - * virtcrypto_devmgr_get_first() - * - * Function returns the first virtio crypto device from the acceleration - * framework. - * - * To be used by virtio crypto device specific drivers. - * - * Return: pointer to vcrypto_dev or NULL if not found. - */ -struct virtio_crypto *virtcrypto_devmgr_get_first(void) -{ - struct virtio_crypto *dev = NULL; - - mutex_lock(&table_lock); - if (!list_empty(&virtio_crypto_table)) - dev = list_first_entry(&virtio_crypto_table, - struct virtio_crypto, - list); - mutex_unlock(&table_lock); - return dev; -} - -/* - * virtcrypto_dev_in_use() - Check whether vcrypto_dev is currently in use - * @vcrypto_dev: Pointer to virtio crypto device. - * - * To be used by virtio crypto device specific drivers. - * - * Return: 1 when device is in use, 0 otherwise. - */ -int virtcrypto_dev_in_use(struct virtio_crypto *vcrypto_dev) -{ - return atomic_read(&vcrypto_dev->ref_count) != 0; -} - -/* * virtcrypto_dev_get() - Increment vcrypto_dev reference count * @vcrypto_dev: Pointer to virtio crypto device. * @@ -256,7 +220,7 @@ int virtcrypto_dev_start(struct virtio_crypto *vcrypto) * @vcrypto: Pointer to virtio crypto device. * * Function notifies all the registered services that the virtio crypto device - * is ready to be used. + * shall no longer be used. * To be used by virtio crypto device specific drivers. * * Return: void diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index 23c41d87d835..1b3fb21a2a7d 100644 --- a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -17,7 +17,6 @@ struct virtio_crypto_skcipher_ctx { struct virtio_crypto *vcrypto; - struct crypto_skcipher *tfm; struct virtio_crypto_sym_session_info enc_sess_info; struct virtio_crypto_sym_session_info dec_sess_info; @@ -28,8 +27,6 @@ struct virtio_crypto_sym_request { /* Cipher or aead */ uint32_t type; - struct virtio_crypto_skcipher_ctx *skcipher_ctx; - struct skcipher_request *skcipher_req; uint8_t *iv; /* Encryption? */ bool encrypt; @@ -57,7 +54,9 @@ static void virtio_crypto_dataq_sym_callback { struct virtio_crypto_sym_request *vc_sym_req = container_of(vc_req, struct virtio_crypto_sym_request, base); - struct skcipher_request *ablk_req; + struct skcipher_request *ablk_req = + container_of((void *)vc_sym_req, struct skcipher_request, + __ctx); int error; /* Finish the encrypt or decrypt process */ @@ -77,7 +76,6 @@ static void virtio_crypto_dataq_sym_callback error = -EIO; break; } - ablk_req = vc_sym_req->skcipher_req; virtio_crypto_skcipher_finalize_req(vc_sym_req, ablk_req, error); } @@ -325,7 +323,7 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, struct data_queue *data_vq) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct virtio_crypto_skcipher_ctx *ctx = vc_sym_req->skcipher_ctx; + struct virtio_crypto_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); struct virtio_crypto_request *vc_req = &vc_sym_req->base; unsigned int ivsize = crypto_skcipher_ivsize(tfm); struct virtio_crypto *vcrypto = ctx->vcrypto; @@ -481,8 +479,6 @@ static int virtio_crypto_skcipher_encrypt(struct skcipher_request *req) vc_req->dataq = data_vq; vc_req->alg_cb = virtio_crypto_dataq_sym_callback; - vc_sym_req->skcipher_ctx = ctx; - vc_sym_req->skcipher_req = req; vc_sym_req->encrypt = true; return crypto_transfer_skcipher_request_to_engine(data_vq->engine, req); @@ -506,8 +502,6 @@ static int virtio_crypto_skcipher_decrypt(struct skcipher_request *req) vc_req->dataq = data_vq; vc_req->alg_cb = virtio_crypto_dataq_sym_callback; - vc_sym_req->skcipher_ctx = ctx; - vc_sym_req->skcipher_req = req; vc_sym_req->encrypt = false; return crypto_transfer_skcipher_request_to_engine(data_vq->engine, req); @@ -515,10 +509,7 @@ static int virtio_crypto_skcipher_decrypt(struct skcipher_request *req) static int virtio_crypto_skcipher_init(struct crypto_skcipher *tfm) { - struct virtio_crypto_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); - crypto_skcipher_set_reqsize(tfm, sizeof(struct virtio_crypto_sym_request)); - ctx->tfm = tfm; return 0; } diff --git a/drivers/crypto/xilinx/Makefile b/drivers/crypto/xilinx/Makefile index 730feff5b5f2..9b51636ef75e 100644 --- a/drivers/crypto/xilinx/Makefile +++ b/drivers/crypto/xilinx/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_XILINX_TRNG) += xilinx-trng.o obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += zynqmp-aes-gcm.o obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_SHA3) += zynqmp-sha.o diff --git a/drivers/crypto/xilinx/xilinx-trng.c b/drivers/crypto/xilinx/xilinx-trng.c new file mode 100644 index 000000000000..db0fbb28ff32 --- /dev/null +++ b/drivers/crypto/xilinx/xilinx-trng.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Versal True Random Number Generator driver + * Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/crypto.h> +#include <linux/delay.h> +#include <linux/firmware/xlnx-zynqmp.h> +#include <linux/hw_random.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <crypto/aes.h> +#include <crypto/df_sp80090a.h> +#include <crypto/internal/drbg.h> +#include <crypto/internal/cipher.h> +#include <crypto/internal/rng.h> + +/* TRNG Registers Offsets */ +#define TRNG_STATUS_OFFSET 0x4U +#define TRNG_CTRL_OFFSET 0x8U +#define TRNG_EXT_SEED_OFFSET 0x40U +#define TRNG_PER_STRNG_OFFSET 0x80U +#define TRNG_CORE_OUTPUT_OFFSET 0xC0U +#define TRNG_RESET_OFFSET 0xD0U +#define TRNG_OSC_EN_OFFSET 0xD4U + +/* Mask values */ +#define TRNG_RESET_VAL_MASK BIT(0) +#define TRNG_OSC_EN_VAL_MASK BIT(0) +#define TRNG_CTRL_PRNGSRST_MASK BIT(0) +#define TRNG_CTRL_EUMODE_MASK BIT(8) +#define TRNG_CTRL_TRSSEN_MASK BIT(2) +#define TRNG_CTRL_PRNGSTART_MASK BIT(5) +#define TRNG_CTRL_PRNGXS_MASK BIT(3) +#define TRNG_CTRL_PRNGMODE_MASK BIT(7) +#define TRNG_STATUS_DONE_MASK BIT(0) +#define TRNG_STATUS_QCNT_MASK GENMASK(11, 9) +#define TRNG_STATUS_QCNT_16_BYTES 0x800 + +/* Sizes in bytes */ +#define TRNG_SEED_LEN_BYTES 48U +#define TRNG_ENTROPY_SEED_LEN_BYTES 64U +#define TRNG_SEC_STRENGTH_SHIFT 5U +#define TRNG_SEC_STRENGTH_BYTES BIT(TRNG_SEC_STRENGTH_SHIFT) +#define TRNG_BYTES_PER_REG 4U +#define TRNG_RESET_DELAY 10 +#define TRNG_NUM_INIT_REGS 12U +#define TRNG_READ_4_WORD 4 +#define TRNG_DATA_READ_DELAY 8000 + +struct xilinx_rng { + void __iomem *rng_base; + struct device *dev; + unsigned char *scratchpadbuf; + struct crypto_aes_ctx *aesctx; + struct mutex lock; /* Protect access to TRNG device */ + struct hwrng trng; +}; + +struct xilinx_rng_ctx { + struct xilinx_rng *rng; +}; + +static struct xilinx_rng *xilinx_rng_dev; + +static void xtrng_readwrite32(void __iomem *addr, u32 mask, u8 value) +{ + u32 val; + + val = ioread32(addr); + val = (val & (~mask)) | (mask & value); + iowrite32(val, addr); +} + +static void xtrng_trng_reset(void __iomem *addr) +{ + xtrng_readwrite32(addr + TRNG_RESET_OFFSET, TRNG_RESET_VAL_MASK, TRNG_RESET_VAL_MASK); + udelay(TRNG_RESET_DELAY); + xtrng_readwrite32(addr + TRNG_RESET_OFFSET, TRNG_RESET_VAL_MASK, 0); +} + +static void xtrng_hold_reset(void __iomem *addr) +{ + xtrng_readwrite32(addr + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, + TRNG_CTRL_PRNGSRST_MASK); + iowrite32(TRNG_RESET_VAL_MASK, addr + TRNG_RESET_OFFSET); + udelay(TRNG_RESET_DELAY); +} + +static void xtrng_softreset(struct xilinx_rng *rng) +{ + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, + TRNG_CTRL_PRNGSRST_MASK); + udelay(TRNG_RESET_DELAY); + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, 0); +} + +/* Return no. of bytes read */ +static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) +{ + int read = 0, ret; + int timeout = 1; + int i, idx; + u32 val; + + if (wait) + timeout = TRNG_DATA_READ_DELAY; + + for (i = 0; i < (blocks32 * 2); i++) { + /* TRNG core generate data in 16 bytes. Read twice to complete 32 bytes read */ + ret = readl_poll_timeout(rng_base + TRNG_STATUS_OFFSET, val, + (val & TRNG_STATUS_QCNT_MASK) == + TRNG_STATUS_QCNT_16_BYTES, !!wait, timeout); + if (ret) + break; + + for (idx = 0; idx < TRNG_READ_4_WORD; idx++) { + *(buf + read) = cpu_to_be32(ioread32(rng_base + TRNG_CORE_OUTPUT_OFFSET)); + read += 1; + } + } + return read * 4; +} + +static int xtrng_collect_random_data(struct xilinx_rng *rng, u8 *rand_gen_buf, + int no_of_random_bytes, bool wait) +{ + u8 randbuf[TRNG_SEC_STRENGTH_BYTES]; + int byteleft, blocks, count = 0; + int ret; + + byteleft = no_of_random_bytes & (TRNG_SEC_STRENGTH_BYTES - 1); + blocks = no_of_random_bytes >> TRNG_SEC_STRENGTH_SHIFT; + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, + TRNG_CTRL_PRNGSTART_MASK); + if (blocks) { + ret = xtrng_readblock32(rng->rng_base, (__be32 *)rand_gen_buf, blocks, wait); + if (!ret) + return 0; + count += ret; + } + + if (byteleft) { + ret = xtrng_readblock32(rng->rng_base, (__be32 *)randbuf, 1, wait); + if (!ret) + return count; + memcpy(rand_gen_buf + (blocks * TRNG_SEC_STRENGTH_BYTES), randbuf, byteleft); + count += byteleft; + } + + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, + TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGSTART_MASK, 0U); + + return count; +} + +static void xtrng_write_multiple_registers(void __iomem *base_addr, u32 *values, size_t n) +{ + void __iomem *reg_addr; + size_t i; + + /* Write seed value into EXTERNAL_SEED Registers in big endian format */ + for (i = 0; i < n; i++) { + reg_addr = (base_addr + ((n - 1 - i) * TRNG_BYTES_PER_REG)); + iowrite32((u32 __force)(cpu_to_be32(values[i])), reg_addr); + } +} + +static void xtrng_enable_entropy(struct xilinx_rng *rng) +{ + iowrite32(TRNG_OSC_EN_VAL_MASK, rng->rng_base + TRNG_OSC_EN_OFFSET); + xtrng_softreset(rng); + iowrite32(TRNG_CTRL_EUMODE_MASK | TRNG_CTRL_TRSSEN_MASK, rng->rng_base + TRNG_CTRL_OFFSET); +} + +static int xtrng_reseed_internal(struct xilinx_rng *rng) +{ + u8 entropy[TRNG_ENTROPY_SEED_LEN_BYTES]; + struct drbg_string data; + LIST_HEAD(seedlist); + u32 val; + int ret; + + drbg_string_fill(&data, entropy, TRNG_SEED_LEN_BYTES); + list_add_tail(&data.list, &seedlist); + memset(entropy, 0, sizeof(entropy)); + xtrng_enable_entropy(rng); + + /* collect random data to use it as entropy (input for DF) */ + ret = xtrng_collect_random_data(rng, entropy, TRNG_SEED_LEN_BYTES, true); + if (ret != TRNG_SEED_LEN_BYTES) + return -EINVAL; + ret = crypto_drbg_ctr_df(rng->aesctx, rng->scratchpadbuf, + TRNG_SEED_LEN_BYTES, &seedlist, AES_BLOCK_SIZE, + TRNG_SEED_LEN_BYTES); + if (ret) + return ret; + + xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, + (u32 *)rng->scratchpadbuf, TRNG_NUM_INIT_REGS); + /* select reseed operation */ + iowrite32(TRNG_CTRL_PRNGXS_MASK, rng->rng_base + TRNG_CTRL_OFFSET); + + /* Start the reseed operation with above configuration and wait for STATUS.Done bit to be + * set. Monitor STATUS.CERTF bit, if set indicates SP800-90B entropy health test has failed. + */ + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, + TRNG_CTRL_PRNGSTART_MASK); + + ret = readl_poll_timeout(rng->rng_base + TRNG_STATUS_OFFSET, val, + (val & TRNG_STATUS_DONE_MASK) == TRNG_STATUS_DONE_MASK, + 1U, 15000U); + if (ret) + return ret; + + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, 0U); + + return 0; +} + +static int xtrng_random_bytes_generate(struct xilinx_rng *rng, u8 *rand_buf_ptr, + u32 rand_buf_size, int wait) +{ + int nbytes; + int ret; + + xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, + TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGXS_MASK, + TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGXS_MASK); + nbytes = xtrng_collect_random_data(rng, rand_buf_ptr, rand_buf_size, wait); + + ret = xtrng_reseed_internal(rng); + if (ret) { + dev_err(rng->dev, "Re-seed fail\n"); + return ret; + } + + return nbytes; +} + +static int xtrng_trng_generate(struct crypto_rng *tfm, const u8 *src, u32 slen, + u8 *dst, u32 dlen) +{ + struct xilinx_rng_ctx *ctx = crypto_rng_ctx(tfm); + int ret; + + mutex_lock(&ctx->rng->lock); + ret = xtrng_random_bytes_generate(ctx->rng, dst, dlen, true); + mutex_unlock(&ctx->rng->lock); + + return ret < 0 ? ret : 0; +} + +static int xtrng_trng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) +{ + return 0; +} + +static int xtrng_trng_init(struct crypto_tfm *rtfm) +{ + struct xilinx_rng_ctx *ctx = crypto_tfm_ctx(rtfm); + + ctx->rng = xilinx_rng_dev; + + return 0; +} + +static struct rng_alg xtrng_trng_alg = { + .generate = xtrng_trng_generate, + .seed = xtrng_trng_seed, + .seedsize = 0, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "xilinx-trng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct xilinx_rng_ctx), + .cra_module = THIS_MODULE, + .cra_init = xtrng_trng_init, + }, +}; + +static int xtrng_hwrng_trng_read(struct hwrng *hwrng, void *data, size_t max, bool wait) +{ + u8 buf[TRNG_SEC_STRENGTH_BYTES]; + struct xilinx_rng *rng; + int ret = -EINVAL, i = 0; + + rng = container_of(hwrng, struct xilinx_rng, trng); + /* Return in case wait not set and lock not available. */ + if (!mutex_trylock(&rng->lock) && !wait) + return 0; + else if (!mutex_is_locked(&rng->lock) && wait) + mutex_lock(&rng->lock); + + while (i < max) { + ret = xtrng_random_bytes_generate(rng, buf, TRNG_SEC_STRENGTH_BYTES, wait); + if (ret < 0) + break; + + memcpy(data + i, buf, min_t(int, ret, (max - i))); + i += min_t(int, ret, (max - i)); + } + mutex_unlock(&rng->lock); + + return ret; +} + +static int xtrng_hwrng_register(struct hwrng *trng) +{ + int ret; + + trng->name = "Xilinx Versal Crypto Engine TRNG"; + trng->read = xtrng_hwrng_trng_read; + + ret = hwrng_register(trng); + if (ret) + pr_err("Fail to register the TRNG\n"); + + return ret; +} + +static void xtrng_hwrng_unregister(struct hwrng *trng) +{ + hwrng_unregister(trng); +} + +static int xtrng_probe(struct platform_device *pdev) +{ + struct xilinx_rng *rng; + size_t sb_size; + int ret; + + rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); + if (!rng) + return -ENOMEM; + + rng->dev = &pdev->dev; + rng->rng_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(rng->rng_base)) { + dev_err(&pdev->dev, "Failed to map resource %pe\n", rng->rng_base); + return PTR_ERR(rng->rng_base); + } + + rng->aesctx = devm_kzalloc(&pdev->dev, sizeof(*rng->aesctx), GFP_KERNEL); + if (!rng->aesctx) + return -ENOMEM; + + sb_size = crypto_drbg_ctr_df_datalen(TRNG_SEED_LEN_BYTES, AES_BLOCK_SIZE); + rng->scratchpadbuf = devm_kzalloc(&pdev->dev, sb_size, GFP_KERNEL); + if (!rng->scratchpadbuf) { + ret = -ENOMEM; + goto end; + } + + xtrng_trng_reset(rng->rng_base); + ret = xtrng_reseed_internal(rng); + if (ret) { + dev_err(&pdev->dev, "TRNG Seed fail\n"); + goto end; + } + + xilinx_rng_dev = rng; + mutex_init(&rng->lock); + ret = crypto_register_rng(&xtrng_trng_alg); + if (ret) { + dev_err(&pdev->dev, "Crypto Random device registration failed: %d\n", ret); + goto end; + } + + ret = xtrng_hwrng_register(&rng->trng); + if (ret) { + dev_err(&pdev->dev, "HWRNG device registration failed: %d\n", ret); + goto crypto_rng_free; + } + platform_set_drvdata(pdev, rng); + + return 0; + +crypto_rng_free: + crypto_unregister_rng(&xtrng_trng_alg); + +end: + return ret; +} + +static void xtrng_remove(struct platform_device *pdev) +{ + struct xilinx_rng *rng; + u32 zero[TRNG_NUM_INIT_REGS] = { }; + + rng = platform_get_drvdata(pdev); + xtrng_hwrng_unregister(&rng->trng); + crypto_unregister_rng(&xtrng_trng_alg); + xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, zero, + TRNG_NUM_INIT_REGS); + xtrng_write_multiple_registers(rng->rng_base + TRNG_PER_STRNG_OFFSET, zero, + TRNG_NUM_INIT_REGS); + xtrng_hold_reset(rng->rng_base); + xilinx_rng_dev = NULL; +} + +static const struct of_device_id xtrng_of_match[] = { + { .compatible = "xlnx,versal-trng", }, + {}, +}; + +MODULE_DEVICE_TABLE(of, xtrng_of_match); + +static struct platform_driver xtrng_driver = { + .driver = { + .name = "xlnx,versal-trng", + .of_match_table = xtrng_of_match, + }, + .probe = xtrng_probe, + .remove = xtrng_remove, +}; + +module_platform_driver(xtrng_driver); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Harsh Jain <h.jain@amd.com>"); +MODULE_AUTHOR("Mounika Botcha <mounika.botcha@amd.com>"); +MODULE_DESCRIPTION("True Random Number Generator Driver"); diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c index 580649f9bff8..5813017b6b79 100644 --- a/drivers/crypto/xilinx/zynqmp-sha.c +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -3,18 +3,18 @@ * Xilinx ZynqMP SHA Driver. * Copyright (c) 2022 Xilinx Inc. */ -#include <linux/cacheflush.h> -#include <crypto/hash.h> #include <crypto/internal/hash.h> #include <crypto/sha3.h> -#include <linux/crypto.h> +#include <linux/cacheflush.h> +#include <linux/cleanup.h> #include <linux/device.h> #include <linux/dma-mapping.h> +#include <linux/err.h> #include <linux/firmware/xlnx-zynqmp.h> -#include <linux/init.h> #include <linux/io.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/spinlock.h> #include <linux/platform_device.h> #define ZYNQMP_DMA_BIT_MASK 32U @@ -36,13 +36,11 @@ struct zynqmp_sha_tfm_ctx { struct crypto_shash *fbk_tfm; }; -struct zynqmp_sha_desc_ctx { - struct shash_desc fbk_req; -}; - static dma_addr_t update_dma_addr, final_dma_addr; static char *ubuf, *fbuf; +static DEFINE_SPINLOCK(zynqmp_sha_lock); + static int zynqmp_sha_init_tfm(struct crypto_shash *hash) { const char *fallback_driver_name = crypto_shash_alg_name(hash); @@ -60,8 +58,13 @@ static int zynqmp_sha_init_tfm(struct crypto_shash *hash) if (IS_ERR(fallback_tfm)) return PTR_ERR(fallback_tfm); + if (crypto_shash_descsize(hash) < + crypto_shash_statesize(tfm_ctx->fbk_tfm)) { + crypto_free_shash(fallback_tfm); + return -EINVAL; + } + tfm_ctx->fbk_tfm = fallback_tfm; - hash->descsize += crypto_shash_descsize(tfm_ctx->fbk_tfm); return 0; } @@ -70,61 +73,55 @@ static void zynqmp_sha_exit_tfm(struct crypto_shash *hash) { struct zynqmp_sha_tfm_ctx *tfm_ctx = crypto_shash_ctx(hash); - if (tfm_ctx->fbk_tfm) { - crypto_free_shash(tfm_ctx->fbk_tfm); - tfm_ctx->fbk_tfm = NULL; - } + crypto_free_shash(tfm_ctx->fbk_tfm); +} - memzero_explicit(tfm_ctx, sizeof(struct zynqmp_sha_tfm_ctx)); +static int zynqmp_sha_continue(struct shash_desc *desc, + struct shash_desc *fbdesc, int err) +{ + err = err ?: crypto_shash_export(fbdesc, shash_desc_ctx(desc)); + shash_desc_zero(fbdesc); + return err; } static int zynqmp_sha_init(struct shash_desc *desc) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); + int err; - dctx->fbk_req.tfm = tctx->fbk_tfm; - return crypto_shash_init(&dctx->fbk_req); + fbdesc->tfm = fbtfm; + err = crypto_shash_init(fbdesc); + return zynqmp_sha_continue(desc, fbdesc, err); } static int zynqmp_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_update(&dctx->fbk_req, data, length); -} - -static int zynqmp_sha_final(struct shash_desc *desc, u8 *out) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); + struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); + int err; - return crypto_shash_final(&dctx->fbk_req, out); + fbdesc->tfm = fbtfm; + err = crypto_shash_import(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_update(fbdesc, data, length); + return zynqmp_sha_continue(desc, fbdesc, err); } static int zynqmp_sha_finup(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) { - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_finup(&dctx->fbk_req, data, length, out); -} - -static int zynqmp_sha_import(struct shash_desc *desc, const void *in) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); - dctx->fbk_req.tfm = tctx->fbk_tfm; - return crypto_shash_import(&dctx->fbk_req, in); + fbdesc->tfm = fbtfm; + return crypto_shash_import(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_finup(fbdesc, data, length, out); } -static int zynqmp_sha_export(struct shash_desc *desc, void *out) -{ - struct zynqmp_sha_desc_ctx *dctx = shash_desc_ctx(desc); - - return crypto_shash_export(&dctx->fbk_req, out); -} - -static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +static int __zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { unsigned int remaining_len = len; int update_size; @@ -159,26 +156,27 @@ static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned i return ret; } +static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +{ + scoped_guard(spinlock_bh, &zynqmp_sha_lock) + return __zynqmp_sha_digest(desc, data, len, out); +} + static struct zynqmp_sha_drv_ctx sha3_drv_ctx = { .sha3_384 = { .init = zynqmp_sha_init, .update = zynqmp_sha_update, - .final = zynqmp_sha_final, .finup = zynqmp_sha_finup, .digest = zynqmp_sha_digest, - .export = zynqmp_sha_export, - .import = zynqmp_sha_import, .init_tfm = zynqmp_sha_init_tfm, .exit_tfm = zynqmp_sha_exit_tfm, - .descsize = sizeof(struct zynqmp_sha_desc_ctx), - .statesize = sizeof(struct sha3_state), + .descsize = SHA3_384_EXPORT_SIZE, .digestsize = SHA3_384_DIGEST_SIZE, .base = { .cra_name = "sha3-384", .cra_driver_name = "zynqmp-sha3-384", .cra_priority = 300, .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ALLOCATES_MEMORY | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct zynqmp_sha_tfm_ctx), |