diff options
Diffstat (limited to 'fs/erofs/decompressor_deflate.c')
-rw-r--r-- | fs/erofs/decompressor_deflate.c | 214 |
1 files changed, 83 insertions, 131 deletions
diff --git a/fs/erofs/decompressor_deflate.c b/fs/erofs/decompressor_deflate.c index 81e65c453ef0..6909b2d529c7 100644 --- a/fs/erofs/decompressor_deflate.c +++ b/fs/erofs/decompressor_deflate.c @@ -15,7 +15,7 @@ static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq); module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444); -void z_erofs_deflate_exit(void) +static void z_erofs_deflate_exit(void) { /* there should be no running fs instance */ while (z_erofs_deflate_avail_strms) { @@ -41,44 +41,20 @@ void z_erofs_deflate_exit(void) } } -int __init z_erofs_deflate_init(void) +static int __init z_erofs_deflate_init(void) { /* by default, use # of possible CPUs instead */ if (!z_erofs_deflate_nstrms) z_erofs_deflate_nstrms = num_possible_cpus(); - - for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; - ++z_erofs_deflate_avail_strms) { - struct z_erofs_deflate *strm; - - strm = kzalloc(sizeof(*strm), GFP_KERNEL); - if (!strm) - goto out_failed; - - /* XXX: in-kernel zlib cannot shrink windowbits currently */ - strm->z.workspace = vmalloc(zlib_inflate_workspacesize()); - if (!strm->z.workspace) { - kfree(strm); - goto out_failed; - } - - spin_lock(&z_erofs_deflate_lock); - strm->next = z_erofs_deflate_head; - z_erofs_deflate_head = strm; - spin_unlock(&z_erofs_deflate_lock); - } return 0; - -out_failed: - erofs_err(NULL, "failed to allocate zlib workspace"); - z_erofs_deflate_exit(); - return -ENOMEM; } -int z_erofs_load_deflate_config(struct super_block *sb, +static int z_erofs_load_deflate_config(struct super_block *sb, struct erofs_super_block *dsb, void *data, int size) { struct z_erofs_deflate_cfgs *dfl = data; + static DEFINE_MUTEX(deflate_resize_mutex); + static bool inited; if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) { erofs_err(sb, "invalid deflate cfgs, size=%u", size); @@ -89,32 +65,52 @@ int z_erofs_load_deflate_config(struct super_block *sb, erofs_err(sb, "unsupported windowbits %u", dfl->windowbits); return -EOPNOTSUPP; } + mutex_lock(&deflate_resize_mutex); + if (!inited) { + for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; + ++z_erofs_deflate_avail_strms) { + struct z_erofs_deflate *strm; + + strm = kzalloc(sizeof(*strm), GFP_KERNEL); + if (!strm) + goto failed; + /* XXX: in-kernel zlib cannot customize windowbits */ + strm->z.workspace = vmalloc(zlib_inflate_workspacesize()); + if (!strm->z.workspace) { + kfree(strm); + goto failed; + } + spin_lock(&z_erofs_deflate_lock); + strm->next = z_erofs_deflate_head; + z_erofs_deflate_head = strm; + spin_unlock(&z_erofs_deflate_lock); + } + inited = true; + } + mutex_unlock(&deflate_resize_mutex); erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!"); return 0; +failed: + mutex_unlock(&deflate_resize_mutex); + z_erofs_deflate_exit(); + return -ENOMEM; } -int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, - struct page **pgpl) +static int __z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl) { - const unsigned int nrpages_out = - PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; - const unsigned int nrpages_in = - PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; struct super_block *sb = rq->sb; - unsigned int insz, outsz, pofs; + struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; struct z_erofs_deflate *strm; - u8 *kin, *kout = NULL; - bool bounced = false; - int no = -1, ni = 0, j = 0, zerr, err; + int zerr, err; /* 1. get the exact DEFLATE compressed size */ - kin = kmap_local_page(*rq->in); - err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, - min_t(unsigned int, rq->inputsize, - sb->s_blocksize - rq->pageofs_in)); + dctx.kin = kmap_local_page(*rq->in); + err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, + min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); if (err) { - kunmap_local(kin); + kunmap_local(dctx.kin); return err; } @@ -131,102 +127,35 @@ again: spin_unlock(&z_erofs_deflate_lock); /* 3. multi-call decompress */ - insz = rq->inputsize; - outsz = rq->outputsize; zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS); if (zerr != Z_OK) { err = -EIO; goto failed_zinit; } - pofs = rq->pageofs_out; - strm->z.avail_in = min_t(u32, insz, PAGE_SIZE - rq->pageofs_in); - insz -= strm->z.avail_in; - strm->z.next_in = kin + rq->pageofs_in; + rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */ + strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); + rq->inputsize -= strm->z.avail_in; + strm->z.next_in = dctx.kin + rq->pageofs_in; strm->z.avail_out = 0; + dctx.bounce = strm->bounce; while (1) { - if (!strm->z.avail_out) { - if (++no >= nrpages_out || !outsz) { - erofs_err(sb, "insufficient space for decompressed data"); - err = -EFSCORRUPTED; - break; - } - - if (kout) - kunmap_local(kout); - strm->z.avail_out = min_t(u32, outsz, PAGE_SIZE - pofs); - outsz -= strm->z.avail_out; - if (!rq->out[no]) { - rq->out[no] = erofs_allocpage(pgpl, rq->gfp); - if (!rq->out[no]) { - kout = NULL; - err = -ENOMEM; - break; - } - set_page_private(rq->out[no], - Z_EROFS_SHORTLIVED_PAGE); - } - kout = kmap_local_page(rq->out[no]); - strm->z.next_out = kout + pofs; - pofs = 0; - } - - if (!strm->z.avail_in && insz) { - if (++ni >= nrpages_in) { - erofs_err(sb, "invalid compressed data"); - err = -EFSCORRUPTED; - break; - } - - if (kout) { /* unlike kmap(), take care of the orders */ - j = strm->z.next_out - kout; - kunmap_local(kout); - } - kunmap_local(kin); - strm->z.avail_in = min_t(u32, insz, PAGE_SIZE); - insz -= strm->z.avail_in; - kin = kmap_local_page(rq->in[ni]); - strm->z.next_in = kin; - bounced = false; - if (kout) { - kout = kmap_local_page(rq->out[no]); - strm->z.next_out = kout + j; - } - } - - /* - * Handle overlapping: Use bounced buffer if the compressed - * data is under processing; Or use short-lived pages from the - * on-stack pagepool where pages share among the same request - * and not _all_ inplace I/O pages are needed to be doubled. - */ - if (!bounced && rq->out[no] == rq->in[ni]) { - memcpy(strm->bounce, strm->z.next_in, strm->z.avail_in); - strm->z.next_in = strm->bounce; - bounced = true; - } - - for (j = ni + 1; j < nrpages_in; ++j) { - struct page *tmppage; - - if (rq->out[no] != rq->in[j]) - continue; - tmppage = erofs_allocpage(pgpl, rq->gfp); - if (!tmppage) { - err = -ENOMEM; - goto failed; - } - set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); - copy_highpage(tmppage, rq->in[j]); - rq->in[j] = tmppage; - } + dctx.avail_out = strm->z.avail_out; + dctx.inbuf_sz = strm->z.avail_in; + err = z_erofs_stream_switch_bufs(&dctx, + (void **)&strm->z.next_out, + (void **)&strm->z.next_in, pgpl); + if (err) + break; + strm->z.avail_out = dctx.avail_out; + strm->z.avail_in = dctx.inbuf_sz; zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH); - if (zerr != Z_OK || !(outsz + strm->z.avail_out)) { + if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) { if (zerr == Z_OK && rq->partial_decoding) break; - if (zerr == Z_STREAM_END && !outsz) + if (zerr == Z_STREAM_END && !rq->outputsize) break; erofs_err(sb, "failed to decompress %d in[%u] out[%u]", zerr, rq->inputsize, rq->outputsize); @@ -234,13 +163,12 @@ again: break; } } -failed: if (zlib_inflateEnd(&strm->z) != Z_OK && !err) err = -EIO; - if (kout) - kunmap_local(kout); + if (dctx.kout) + kunmap_local(dctx.kout); failed_zinit: - kunmap_local(kin); + kunmap_local(dctx.kin); /* 4. push back DEFLATE stream context to the global list */ spin_lock(&z_erofs_deflate_lock); strm->next = z_erofs_deflate_head; @@ -249,3 +177,27 @@ failed_zinit: wake_up(&z_erofs_deflate_wq); return err; } + +static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl) +{ +#ifdef CONFIG_EROFS_FS_ZIP_ACCEL + int err; + + if (!rq->partial_decoding) { + err = z_erofs_crypto_decompress(rq, pgpl); + if (err != -EOPNOTSUPP) + return err; + + } +#endif + return __z_erofs_deflate_decompress(rq, pgpl); +} + +const struct z_erofs_decompressor z_erofs_deflate_decomp = { + .config = z_erofs_load_deflate_config, + .decompress = z_erofs_deflate_decompress, + .init = z_erofs_deflate_init, + .exit = z_erofs_deflate_exit, + .name = "deflate", +}; |