summaryrefslogtreecommitdiff
path: root/crypto/acompress.c
diff options
context:
space:
mode:
Diffstat (limited to 'crypto/acompress.c')
-rw-r--r--crypto/acompress.c588
1 files changed, 588 insertions, 0 deletions
diff --git a/crypto/acompress.c b/crypto/acompress.c
new file mode 100644
index 000000000000..be28cbfd22e3
--- /dev/null
+++ b/crypto/acompress.c
@@ -0,0 +1,588 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Asynchronous Compression operations
+ *
+ * Copyright (c) 2016, Intel Corporation
+ * Authors: Weigang Li <weigang.li@intel.com>
+ * Giovanni Cabiddu <giovanni.cabiddu@intel.com>
+ */
+
+#include <crypto/internal/acompress.h>
+#include <crypto/scatterwalk.h>
+#include <linux/cryptouser.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/scatterlist.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <linux/smp.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/workqueue.h>
+#include <net/netlink.h>
+
+#include "compress.h"
+
+struct crypto_scomp;
+
+enum {
+ ACOMP_WALK_SLEEP = 1 << 0,
+ ACOMP_WALK_SRC_LINEAR = 1 << 1,
+ ACOMP_WALK_DST_LINEAR = 1 << 2,
+};
+
+static const struct crypto_type crypto_acomp_type;
+
+static void acomp_reqchain_done(void *data, int err);
+
+static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg)
+{
+ return container_of(alg, struct acomp_alg, calg.base);
+}
+
+static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
+{
+ return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg);
+}
+
+static int __maybe_unused crypto_acomp_report(
+ struct sk_buff *skb, struct crypto_alg *alg)
+{
+ struct crypto_report_acomp racomp;
+
+ memset(&racomp, 0, sizeof(racomp));
+
+ strscpy(racomp.type, "acomp", sizeof(racomp.type));
+
+ return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp);
+}
+
+static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
+ __maybe_unused;
+
+static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
+{
+ seq_puts(m, "type : acomp\n");
+}
+
+static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
+ struct acomp_alg *alg = crypto_acomp_alg(acomp);
+
+ if (alg->exit)
+ alg->exit(acomp);
+
+ if (acomp_is_async(acomp))
+ crypto_free_acomp(crypto_acomp_fb(acomp));
+}
+
+static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
+ struct acomp_alg *alg = crypto_acomp_alg(acomp);
+ struct crypto_acomp *fb = NULL;
+ int err;
+
+ if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
+ return crypto_init_scomp_ops_async(tfm);
+
+ if (acomp_is_async(acomp)) {
+ fb = crypto_alloc_acomp(crypto_acomp_alg_name(acomp), 0,
+ CRYPTO_ALG_ASYNC);
+ if (IS_ERR(fb))
+ return PTR_ERR(fb);
+
+ err = -EINVAL;
+ if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE)
+ goto out_free_fb;
+
+ tfm->fb = crypto_acomp_tfm(fb);
+ }
+
+ acomp->compress = alg->compress;
+ acomp->decompress = alg->decompress;
+ acomp->reqsize = alg->base.cra_reqsize;
+
+ acomp->base.exit = crypto_acomp_exit_tfm;
+
+ if (!alg->init)
+ return 0;
+
+ err = alg->init(acomp);
+ if (err)
+ goto out_free_fb;
+
+ return 0;
+
+out_free_fb:
+ crypto_free_acomp(fb);
+ return err;
+}
+
+static unsigned int crypto_acomp_extsize(struct crypto_alg *alg)
+{
+ int extsize = crypto_alg_extsize(alg);
+
+ if (alg->cra_type != &crypto_acomp_type)
+ extsize += sizeof(struct crypto_scomp *);
+
+ return extsize;
+}
+
+static const struct crypto_type crypto_acomp_type = {
+ .extsize = crypto_acomp_extsize,
+ .init_tfm = crypto_acomp_init_tfm,
+#ifdef CONFIG_PROC_FS
+ .show = crypto_acomp_show,
+#endif
+#if IS_ENABLED(CONFIG_CRYPTO_USER)
+ .report = crypto_acomp_report,
+#endif
+ .maskclear = ~CRYPTO_ALG_TYPE_MASK,
+ .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK,
+ .type = CRYPTO_ALG_TYPE_ACOMPRESS,
+ .tfmsize = offsetof(struct crypto_acomp, base),
+ .algsize = offsetof(struct acomp_alg, base),
+};
+
+struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
+ u32 mask)
+{
+ return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask);
+}
+EXPORT_SYMBOL_GPL(crypto_alloc_acomp);
+
+struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type,
+ u32 mask, int node)
+{
+ return crypto_alloc_tfm_node(alg_name, &crypto_acomp_type, type, mask,
+ node);
+}
+EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
+
+static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt)
+{
+ struct acomp_req_chain *state = &req->chain;
+
+ state->compl = req->base.complete;
+ state->data = req->base.data;
+ req->base.complete = cplt;
+ req->base.data = state;
+}
+
+static void acomp_restore_req(struct acomp_req *req)
+{
+ struct acomp_req_chain *state = req->base.data;
+
+ req->base.complete = state->compl;
+ req->base.data = state->data;
+}
+
+static void acomp_reqchain_virt(struct acomp_req *req)
+{
+ struct acomp_req_chain *state = &req->chain;
+ unsigned int slen = req->slen;
+ unsigned int dlen = req->dlen;
+
+ if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT)
+ acomp_request_set_src_dma(req, state->src, slen);
+ if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT)
+ acomp_request_set_dst_dma(req, state->dst, dlen);
+}
+
+static void acomp_virt_to_sg(struct acomp_req *req)
+{
+ struct acomp_req_chain *state = &req->chain;
+
+ state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT |
+ CRYPTO_ACOMP_REQ_DST_VIRT);
+
+ if (acomp_request_src_isvirt(req)) {
+ unsigned int slen = req->slen;
+ const u8 *svirt = req->svirt;
+
+ state->src = svirt;
+ sg_init_one(&state->ssg, svirt, slen);
+ acomp_request_set_src_sg(req, &state->ssg, slen);
+ }
+
+ if (acomp_request_dst_isvirt(req)) {
+ unsigned int dlen = req->dlen;
+ u8 *dvirt = req->dvirt;
+
+ state->dst = dvirt;
+ sg_init_one(&state->dsg, dvirt, dlen);
+ acomp_request_set_dst_sg(req, &state->dsg, dlen);
+ }
+}
+
+static int acomp_do_nondma(struct acomp_req *req, bool comp)
+{
+ ACOMP_FBREQ_ON_STACK(fbreq, req);
+ int err;
+
+ if (comp)
+ err = crypto_acomp_compress(fbreq);
+ else
+ err = crypto_acomp_decompress(fbreq);
+
+ req->dlen = fbreq->dlen;
+ return err;
+}
+
+static int acomp_do_one_req(struct acomp_req *req, bool comp)
+{
+ if (acomp_request_isnondma(req))
+ return acomp_do_nondma(req, comp);
+
+ acomp_virt_to_sg(req);
+ return comp ? crypto_acomp_reqtfm(req)->compress(req) :
+ crypto_acomp_reqtfm(req)->decompress(req);
+}
+
+static int acomp_reqchain_finish(struct acomp_req *req, int err)
+{
+ acomp_reqchain_virt(req);
+ acomp_restore_req(req);
+ return err;
+}
+
+static void acomp_reqchain_done(void *data, int err)
+{
+ struct acomp_req *req = data;
+ crypto_completion_t compl;
+
+ compl = req->chain.compl;
+ data = req->chain.data;
+
+ if (err == -EINPROGRESS)
+ goto notify;
+
+ err = acomp_reqchain_finish(req, err);
+
+notify:
+ compl(data, err);
+}
+
+static int acomp_do_req_chain(struct acomp_req *req, bool comp)
+{
+ int err;
+
+ acomp_save_req(req, acomp_reqchain_done);
+
+ err = acomp_do_one_req(req, comp);
+ if (err == -EBUSY || err == -EINPROGRESS)
+ return err;
+
+ return acomp_reqchain_finish(req, err);
+}
+
+int crypto_acomp_compress(struct acomp_req *req)
+{
+ struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+
+ if (acomp_req_on_stack(req) && acomp_is_async(tfm))
+ return -EAGAIN;
+ if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
+ return crypto_acomp_reqtfm(req)->compress(req);
+ return acomp_do_req_chain(req, true);
+}
+EXPORT_SYMBOL_GPL(crypto_acomp_compress);
+
+int crypto_acomp_decompress(struct acomp_req *req)
+{
+ struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+
+ if (acomp_req_on_stack(req) && acomp_is_async(tfm))
+ return -EAGAIN;
+ if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
+ return crypto_acomp_reqtfm(req)->decompress(req);
+ return acomp_do_req_chain(req, false);
+}
+EXPORT_SYMBOL_GPL(crypto_acomp_decompress);
+
+void comp_prepare_alg(struct comp_alg_common *alg)
+{
+ struct crypto_alg *base = &alg->base;
+
+ base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
+}
+
+int crypto_register_acomp(struct acomp_alg *alg)
+{
+ struct crypto_alg *base = &alg->calg.base;
+
+ comp_prepare_alg(&alg->calg);
+
+ base->cra_type = &crypto_acomp_type;
+ base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS;
+
+ return crypto_register_alg(base);
+}
+EXPORT_SYMBOL_GPL(crypto_register_acomp);
+
+void crypto_unregister_acomp(struct acomp_alg *alg)
+{
+ crypto_unregister_alg(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_acomp);
+
+int crypto_register_acomps(struct acomp_alg *algs, int count)
+{
+ int i, ret;
+
+ for (i = 0; i < count; i++) {
+ ret = crypto_register_acomp(&algs[i]);
+ if (ret)
+ goto err;
+ }
+
+ return 0;
+
+err:
+ for (--i; i >= 0; --i)
+ crypto_unregister_acomp(&algs[i]);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_register_acomps);
+
+void crypto_unregister_acomps(struct acomp_alg *algs, int count)
+{
+ int i;
+
+ for (i = count - 1; i >= 0; --i)
+ crypto_unregister_acomp(&algs[i]);
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_acomps);
+
+static void acomp_stream_workfn(struct work_struct *work)
+{
+ struct crypto_acomp_streams *s =
+ container_of(work, struct crypto_acomp_streams, stream_work);
+ struct crypto_acomp_stream __percpu *streams = s->streams;
+ int cpu;
+
+ for_each_cpu(cpu, &s->stream_want) {
+ struct crypto_acomp_stream *ps;
+ void *ctx;
+
+ ps = per_cpu_ptr(streams, cpu);
+ if (ps->ctx)
+ continue;
+
+ ctx = s->alloc_ctx();
+ if (IS_ERR(ctx))
+ break;
+
+ spin_lock_bh(&ps->lock);
+ ps->ctx = ctx;
+ spin_unlock_bh(&ps->lock);
+
+ cpumask_clear_cpu(cpu, &s->stream_want);
+ }
+}
+
+void crypto_acomp_free_streams(struct crypto_acomp_streams *s)
+{
+ struct crypto_acomp_stream __percpu *streams = s->streams;
+ void (*free_ctx)(void *);
+ int i;
+
+ s->streams = NULL;
+ if (!streams)
+ return;
+
+ cancel_work_sync(&s->stream_work);
+ free_ctx = s->free_ctx;
+
+ for_each_possible_cpu(i) {
+ struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i);
+
+ if (!ps->ctx)
+ continue;
+
+ free_ctx(ps->ctx);
+ }
+
+ free_percpu(streams);
+}
+EXPORT_SYMBOL_GPL(crypto_acomp_free_streams);
+
+int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s)
+{
+ struct crypto_acomp_stream __percpu *streams;
+ struct crypto_acomp_stream *ps;
+ unsigned int i;
+ void *ctx;
+
+ if (s->streams)
+ return 0;
+
+ streams = alloc_percpu(struct crypto_acomp_stream);
+ if (!streams)
+ return -ENOMEM;
+
+ ctx = s->alloc_ctx();
+ if (IS_ERR(ctx)) {
+ free_percpu(streams);
+ return PTR_ERR(ctx);
+ }
+
+ i = cpumask_first(cpu_possible_mask);
+ ps = per_cpu_ptr(streams, i);
+ ps->ctx = ctx;
+
+ for_each_possible_cpu(i) {
+ ps = per_cpu_ptr(streams, i);
+ spin_lock_init(&ps->lock);
+ }
+
+ s->streams = streams;
+
+ INIT_WORK(&s->stream_work, acomp_stream_workfn);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams);
+
+struct crypto_acomp_stream *crypto_acomp_lock_stream_bh(
+ struct crypto_acomp_streams *s) __acquires(stream)
+{
+ struct crypto_acomp_stream __percpu *streams = s->streams;
+ int cpu = raw_smp_processor_id();
+ struct crypto_acomp_stream *ps;
+
+ ps = per_cpu_ptr(streams, cpu);
+ spin_lock_bh(&ps->lock);
+ if (likely(ps->ctx))
+ return ps;
+ spin_unlock(&ps->lock);
+
+ cpumask_set_cpu(cpu, &s->stream_want);
+ schedule_work(&s->stream_work);
+
+ ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask));
+ spin_lock(&ps->lock);
+ return ps;
+}
+EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh);
+
+void acomp_walk_done_src(struct acomp_walk *walk, int used)
+{
+ walk->slen -= used;
+ if ((walk->flags & ACOMP_WALK_SRC_LINEAR))
+ scatterwalk_advance(&walk->in, used);
+ else
+ scatterwalk_done_src(&walk->in, used);
+
+ if ((walk->flags & ACOMP_WALK_SLEEP))
+ cond_resched();
+}
+EXPORT_SYMBOL_GPL(acomp_walk_done_src);
+
+void acomp_walk_done_dst(struct acomp_walk *walk, int used)
+{
+ walk->dlen -= used;
+ if ((walk->flags & ACOMP_WALK_DST_LINEAR))
+ scatterwalk_advance(&walk->out, used);
+ else
+ scatterwalk_done_dst(&walk->out, used);
+
+ if ((walk->flags & ACOMP_WALK_SLEEP))
+ cond_resched();
+}
+EXPORT_SYMBOL_GPL(acomp_walk_done_dst);
+
+int acomp_walk_next_src(struct acomp_walk *walk)
+{
+ unsigned int slen = walk->slen;
+ unsigned int max = UINT_MAX;
+
+ if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
+ max = PAGE_SIZE;
+ if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
+ walk->in.__addr = (void *)(((u8 *)walk->in.sg) +
+ walk->in.offset);
+ return min(slen, max);
+ }
+
+ return slen ? scatterwalk_next(&walk->in, slen) : 0;
+}
+EXPORT_SYMBOL_GPL(acomp_walk_next_src);
+
+int acomp_walk_next_dst(struct acomp_walk *walk)
+{
+ unsigned int dlen = walk->dlen;
+ unsigned int max = UINT_MAX;
+
+ if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
+ max = PAGE_SIZE;
+ if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
+ walk->out.__addr = (void *)(((u8 *)walk->out.sg) +
+ walk->out.offset);
+ return min(dlen, max);
+ }
+
+ return dlen ? scatterwalk_next(&walk->out, dlen) : 0;
+}
+EXPORT_SYMBOL_GPL(acomp_walk_next_dst);
+
+int acomp_walk_virt(struct acomp_walk *__restrict walk,
+ struct acomp_req *__restrict req, bool atomic)
+{
+ struct scatterlist *src = req->src;
+ struct scatterlist *dst = req->dst;
+
+ walk->slen = req->slen;
+ walk->dlen = req->dlen;
+
+ if (!walk->slen || !walk->dlen)
+ return -EINVAL;
+
+ walk->flags = 0;
+ if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic)
+ walk->flags |= ACOMP_WALK_SLEEP;
+ if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT))
+ walk->flags |= ACOMP_WALK_SRC_LINEAR;
+ if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT))
+ walk->flags |= ACOMP_WALK_DST_LINEAR;
+
+ if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
+ walk->in.sg = (void *)req->svirt;
+ walk->in.offset = 0;
+ } else
+ scatterwalk_start(&walk->in, src);
+ if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
+ walk->out.sg = (void *)req->dvirt;
+ walk->out.offset = 0;
+ } else
+ scatterwalk_start(&walk->out, dst);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(acomp_walk_virt);
+
+struct acomp_req *acomp_request_clone(struct acomp_req *req,
+ size_t total, gfp_t gfp)
+{
+ struct acomp_req *nreq;
+
+ nreq = container_of(crypto_request_clone(&req->base, total, gfp),
+ struct acomp_req, base);
+ if (nreq == req)
+ return req;
+
+ if (req->src == &req->chain.ssg)
+ nreq->src = &nreq->chain.ssg;
+ if (req->dst == &req->chain.dsg)
+ nreq->dst = &nreq->chain.dsg;
+ return nreq;
+}
+EXPORT_SYMBOL_GPL(acomp_request_clone);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Asynchronous compression type");
generated by cgit (git 2.34.1) at 2025-12-21 23:49:19 +0000