Merge tag 'erofs-for-6.8-rc1-2' of git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs (0507d252) · Commits · git / linux-net

fs/erofs/decompressor.c

+65 −55

Original line number	Diff line number	Diff line
		@@ -121,11 +121,11 @@ static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
		}

		static void z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx ctx,
		void inpage, unsigned int inputmargin, int *maptype,
		bool may_inplace)
		void inpage, void out, unsigned int *inputmargin,
		int *maptype, bool may_inplace)
		{
		struct z_erofs_decompress_req *rq = ctx->rq;
		unsigned int omargin, total, i, j;
		unsigned int omargin, total, i;
		struct page **in;
		void src, tmp;

		@@ -135,12 +135,13 @@ static void z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx ctx,
		omargin < LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize))
		goto docopy;

		for (i = 0; i < ctx->inpages; ++i) {
		DBG_BUGON(rq->in[i] == NULL);
		for (j = 0; j < ctx->outpages - ctx->inpages + i; ++j)
		if (rq->out[j] == rq->in[i])
		for (i = 0; i < ctx->inpages; ++i)
		if (rq->out[ctx->outpages - ctx->inpages + i] !=
		rq->in[i])
		goto docopy;
		}
		kunmap_local(inpage);
		*maptype = 3;
		return out + ((ctx->outpages - ctx->inpages) << PAGE_SHIFT);
		}

		if (ctx->inpages <= 1) {
		@@ -148,7 +149,6 @@ static void z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx ctx,
		return inpage;
		}
		kunmap_local(inpage);
		might_sleep();
		src = erofs_vm_map_ram(rq->in, ctx->inpages);
		if (!src)
		return ERR_PTR(-ENOMEM);
		@@ -204,12 +204,12 @@ int z_erofs_fixup_insize(struct z_erofs_decompress_req rq, const char padbuf,
		}

		static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
		u8 *out)
		u8 *dst)
		{
		struct z_erofs_decompress_req *rq = ctx->rq;
		bool support_0padding = false, may_inplace = false;
		unsigned int inputmargin;
		u8 headpage, src;
		u8 out, headpage, *src;
		int ret, maptype;

		DBG_BUGON(*rq->in == NULL);
		@@ -230,11 +230,12 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
		}

		inputmargin = rq->pageofs_in;
		src = z_erofs_lz4_handle_overlap(ctx, headpage, &inputmargin,
		src = z_erofs_lz4_handle_overlap(ctx, headpage, dst, &inputmargin,
		&maptype, may_inplace);
		if (IS_ERR(src))
		return PTR_ERR(src);

		out = dst + rq->pageofs_out;
		/* legacy format could compress extra data in a pcluster. */
		if (rq->partial_decoding \|\| !support_0padding)
		ret = LZ4_decompress_safe_partial(src + inputmargin, out,
		@@ -246,15 +247,9 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
		if (ret != rq->outputsize) {
		erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
		ret, rq->inputsize, inputmargin, rq->outputsize);

		print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET,
		16, 1, src + inputmargin, rq->inputsize, true);
		print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET,
		16, 1, out, rq->outputsize, true);

		if (ret >= 0)
		memset(out + ret, 0, rq->outputsize - ret);
		ret = -EIO;
		ret = -EFSCORRUPTED;
		} else {
		ret = 0;
		}
		@@ -265,7 +260,7 @@ static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
		vm_unmap_ram(src, ctx->inpages);
		} else if (maptype == 2) {
		erofs_put_pcpubuf(src);
		} else {
		} else if (maptype != 3) {
		DBG_BUGON(1);
		return -EFAULT;
		}
		@@ -308,7 +303,7 @@ static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
		}

		dstmap_out:
		ret = z_erofs_lz4_decompress_mem(&ctx, dst + rq->pageofs_out);
		ret = z_erofs_lz4_decompress_mem(&ctx, dst);
		if (!dst_maptype)
		kunmap_local(dst);
		else if (dst_maptype == 2)
		@@ -319,43 +314,58 @@ static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
		static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq,
		struct page **pagepool)
		{
		const unsigned int inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
		const unsigned int outpages =
		const unsigned int nrpages_in =
		PAGE_ALIGN(rq->pageofs_in + rq->inputsize) >> PAGE_SHIFT;
		const unsigned int nrpages_out =
		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
		const unsigned int righthalf = min_t(unsigned int, rq->outputsize,
		PAGE_SIZE - rq->pageofs_out);
		const unsigned int lefthalf = rq->outputsize - righthalf;
		const unsigned int interlaced_offset =
		rq->alg == Z_EROFS_COMPRESSION_SHIFTED ? 0 : rq->pageofs_out;
		u8 *src;

		if (outpages > 2 && rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {
		DBG_BUGON(1);
		return -EFSCORRUPTED;
		}

		if (rq->out[0] == *rq->in) {
		DBG_BUGON(rq->pageofs_out);
		return 0;
		}

		src = kmap_local_page(rq->in[inpages - 1]) + rq->pageofs_in;
		if (rq->out[0])
		const unsigned int bs = rq->sb->s_blocksize;
		unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt;
		u8 *kin;

		DBG_BUGON(rq->outputsize > rq->inputsize);
		if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) {
		cur = bs - (rq->pageofs_out & (bs - 1));
		pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK;
		cur = min(cur, rq->outputsize);
		if (cur && rq->out[0]) {
		kin = kmap_local_page(rq->in[nrpages_in - 1]);
		if (rq->out[0] == rq->in[nrpages_in - 1]) {
		memmove(kin + rq->pageofs_out, kin + pi, cur);
		flush_dcache_page(rq->out[0]);
		} else {
		memcpy_to_page(rq->out[0], rq->pageofs_out,
		src + interlaced_offset, righthalf);

		if (outpages > inpages) {
		DBG_BUGON(!rq->out[outpages - 1]);
		if (rq->out[outpages - 1] != rq->in[inpages - 1]) {
		memcpy_to_page(rq->out[outpages - 1], 0, src +
		(interlaced_offset ? 0 : righthalf),
		lefthalf);
		} else if (!interlaced_offset) {
		memmove(src, src + righthalf, lefthalf);
		flush_dcache_page(rq->in[inpages - 1]);
		kin + pi, cur);
		}
		kunmap_local(kin);
		}
		rq->outputsize -= cur;
		}
		kunmap_local(src);

		for (; rq->outputsize; rq->pageofs_in = 0, cur += PAGE_SIZE, ni++) {
		insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize);
		rq->outputsize -= insz;
		if (!rq->in[ni])
		continue;
		kin = kmap_local_page(rq->in[ni]);
		pi = 0;
		do {
		no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT;
		po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK;
		DBG_BUGON(no >= nrpages_out);
		cnt = min(insz - pi, PAGE_SIZE - po);
		if (rq->out[no] == rq->in[ni]) {
		memmove(kin + po,
		kin + rq->pageofs_in + pi, cnt);
		flush_dcache_page(rq->out[no]);
		} else if (rq->out[no]) {
		memcpy_to_page(rq->out[no], po,
		kin + rq->pageofs_in + pi, cnt);
		}
		pi += cnt;
		} while (pi < insz);
		kunmap_local(kin);
		}
		DBG_BUGON(ni > nrpages_in);
		return 0;
		}

fs/erofs/decompressor_deflate.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -70,7 +70,7 @@ int __init z_erofs_deflate_init(void)
		return 0;

		out_failed:
		pr_err("failed to allocate zlib workspace\n");
		erofs_err(NULL, "failed to allocate zlib workspace");
		z_erofs_deflate_exit();
		return -ENOMEM;
		}

fs/erofs/inode.c

+4 −2

Original line number	Diff line number	Diff line
		@@ -259,8 +259,10 @@ static int erofs_fill_inode(struct inode *inode)

		if (erofs_inode_is_data_compressed(vi->datalayout)) {
		#ifdef CONFIG_EROFS_FS_ZIP
		if (!erofs_is_fscache_mode(inode->i_sb) &&
		inode->i_sb->s_blocksize_bits == PAGE_SHIFT) {
		if (!erofs_is_fscache_mode(inode->i_sb)) {
		DO_ONCE_LITE_IF(inode->i_sb->s_blocksize != PAGE_SIZE,
		erofs_info, inode->i_sb,
		"EXPERIMENTAL EROFS subpage compressed block support in use. Use at your own risk!");
		inode->i_mapping->a_ops = &z_erofs_aops;
		err = 0;
		goto out_unlock;

fs/erofs/super.c

+8 −2

Original line number	Diff line number	Diff line
		@@ -27,7 +27,10 @@ void _erofs_err(struct super_block sb, const char func, const char *fmt, ...)
		vaf.fmt = fmt;
		vaf.va = &args;

		if (sb)
		pr_err("(device %s): %s: %pV", sb->s_id, func, &vaf);
		else
		pr_err("%s: %pV", func, &vaf);
		va_end(args);
		}

		@@ -41,7 +44,10 @@ void _erofs_info(struct super_block sb, const char func, const char *fmt, ...)
		vaf.fmt = fmt;
		vaf.va = &args;

		if (sb)
		pr_info("(device %s): %pV", sb->s_id, &vaf);
		else
		pr_info("%pV", &vaf);
		va_end(args);
		}

fs/erofs/zdata.c

+119 −148

Original line number	Diff line number	Diff line
		@@ -56,6 +56,9 @@ struct z_erofs_pcluster {
		/* L: total number of bvecs */
		unsigned int vcnt;

		/* I: pcluster size (compressed size) in bytes */
		unsigned int pclustersize;

		/* I: page offset of start position of decompression */
		unsigned short pageofs_out;

		@@ -70,14 +73,6 @@ struct z_erofs_pcluster {
		struct rcu_head rcu;
		};

		union {
		/* I: physical cluster size in pages */
		unsigned short pclusterpages;

		/* I: tailpacking inline compressed size */
		unsigned short tailpacking_size;
		};

		/* I: compression algorithm format */
		unsigned char algorithmformat;

		@@ -115,9 +110,7 @@ static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl)

		static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl)
		{
		if (z_erofs_is_inline_pcluster(pcl))
		return 1;
		return pcl->pclusterpages;
		return PAGE_ALIGN(pcl->pclustersize) >> PAGE_SHIFT;
		}

		/*
		@@ -298,12 +291,12 @@ static int z_erofs_create_pcluster_pool(void)
		return 0;
		}

		static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages)
		static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int size)
		{
		int i;
		unsigned int nrpages = PAGE_ALIGN(size) >> PAGE_SHIFT;
		struct z_erofs_pcluster_slab *pcs = pcluster_pool;

		for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
		struct z_erofs_pcluster_slab *pcs = pcluster_pool + i;
		for (; pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) {
		struct z_erofs_pcluster *pcl;

		if (nrpages > pcs->maxpages)
		@@ -312,7 +305,7 @@ static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages)
		pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS);
		if (!pcl)
		return ERR_PTR(-ENOMEM);
		pcl->pclusterpages = nrpages;
		pcl->pclustersize = size;
		return pcl;
		}
		return ERR_PTR(-EINVAL);
		@@ -559,6 +552,7 @@ static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe)
		{
		struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode));
		struct z_erofs_pcluster *pcl = fe->pcl;
		unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
		bool shouldalloc = z_erofs_should_alloc_cache(fe);
		bool standalone = true;
		/*
		@@ -569,13 +563,14 @@ static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe)
		__GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_NOWARN;
		unsigned int i;

		if (i_blocksize(fe->inode) != PAGE_SIZE)
		return;
		if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED)
		return;

		for (i = 0; i < pcl->pclusterpages; ++i) {
		struct page *page;
		for (i = 0; i < pclusterpages; ++i) {
		struct page page, newpage;
		void t; / mark pages just found for debugging */
		struct page *newpage = NULL;

		/* the compressed page was loaded before */
		if (READ_ONCE(pcl->compressed_bvecs[i].page))
		@@ -585,6 +580,7 @@ static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe)

		if (page) {
		t = (void *)((unsigned long)page \| 1);
		newpage = NULL;
		} else {
		/* I/O is needed, no possible to decompress directly */
		standalone = false;
		@@ -592,9 +588,8 @@ static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe)
		continue;

		/*
		* try to use cached I/O if page allocation
		* succeeds or fallback to in-place I/O instead
		* to avoid any direct reclaim.
		* Try cached I/O if allocation succeeds or fallback to
		* in-place I/O instead to avoid any direct reclaim.
		*/
		newpage = erofs_allocpage(&fe->pagepool, gfp);
		if (!newpage)
		@@ -626,6 +621,7 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
		{
		struct z_erofs_pcluster *const pcl =
		container_of(grp, struct z_erofs_pcluster, obj);
		unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
		int i;

		DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
		@@ -633,7 +629,7 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
		* refcount of workgroup is now freezed as 0,
		* therefore no need to worry about available decompression users.
		*/
		for (i = 0; i < pcl->pclusterpages; ++i) {
		for (i = 0; i < pclusterpages; ++i) {
		struct page *page = pcl->compressed_bvecs[i].page;

		if (!page)
		@@ -657,6 +653,7 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
		static bool z_erofs_cache_release_folio(struct folio *folio, gfp_t gfp)
		{
		struct z_erofs_pcluster *pcl = folio_get_private(folio);
		unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
		bool ret;
		int i;

		@@ -669,7 +666,7 @@ static bool z_erofs_cache_release_folio(struct folio *folio, gfp_t gfp)
		goto out;

		DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
		for (i = 0; i < pcl->pclusterpages; ++i) {
		for (i = 0; i < pclusterpages; ++i) {
		if (pcl->compressed_bvecs[i].page == &folio->page) {
		WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
		ret = true;
		@@ -778,20 +775,20 @@ static void z_erofs_try_to_claim_pcluster(struct z_erofs_decompress_frontend *f)
		static int z_erofs_register_pcluster(struct z_erofs_decompress_frontend *fe)
		{
		struct erofs_map_blocks *map = &fe->map;
		struct super_block *sb = fe->inode->i_sb;
		bool ztailpacking = map->m_flags & EROFS_MAP_META;
		struct z_erofs_pcluster *pcl;
		struct erofs_workgroup *grp;
		int err;

		if (!(map->m_flags & EROFS_MAP_ENCODED) \|\|
		(!ztailpacking && !(map->m_pa >> PAGE_SHIFT))) {
		(!ztailpacking && !erofs_blknr(sb, map->m_pa))) {
		DBG_BUGON(1);
		return -EFSCORRUPTED;
		}

		/* no available pcluster, let's allocate one */
		pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 :
		map->m_plen >> PAGE_SHIFT);
		pcl = z_erofs_alloc_pcluster(map->m_plen);
		if (IS_ERR(pcl))
		return PTR_ERR(pcl);

		@@ -815,10 +812,8 @@ static int z_erofs_register_pcluster(struct z_erofs_decompress_frontend *fe)

		if (ztailpacking) {
		pcl->obj.index = 0; /* which indicates ztailpacking */
		pcl->pageofs_in = erofs_blkoff(fe->inode->i_sb, map->m_pa);
		pcl->tailpacking_size = map->m_plen;
		} else {
		pcl->obj.index = map->m_pa >> PAGE_SHIFT;
		pcl->obj.index = erofs_blknr(sb, map->m_pa);

		grp = erofs_insert_workgroup(fe->inode->i_sb, &pcl->obj);
		if (IS_ERR(grp)) {
		@@ -893,6 +888,7 @@ static int z_erofs_pcluster_begin(struct z_erofs_decompress_frontend *fe)
		}
		get_page(map->buf.page);
		WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page);
		fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK;
		fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE;
		}
		/* file-backed inplace I/O pages are traversed in reverse order */
		@@ -973,12 +969,12 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
		struct inode *const inode = fe->inode;
		struct erofs_map_blocks *const map = &fe->map;
		const loff_t offset = page_offset(page);
		const unsigned int bs = i_blocksize(inode);
		bool tight = true, exclusive;
		unsigned int cur, end, len, split;
		int err = 0;

		z_erofs_onlinepage_init(page);

		split = 0;
		end = PAGE_SIZE;
		repeat:
		@@ -1027,7 +1023,7 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
		* for inplace I/O or bvpage (should be processed in a strict order.)
		*/
		tight &= (fe->mode > Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE);
		exclusive = (!cur && ((split <= 1) \|\| tight));
		exclusive = (!cur && ((split <= 1) \|\| (tight && bs == PAGE_SIZE)));
		if (cur)
		tight &= (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED);

		@@ -1206,34 +1202,27 @@ static int z_erofs_parse_in_bvecs(struct z_erofs_decompress_backend *be,
		struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i];
		struct page *page = bvec->page;

		/* compressed pages ought to be present before decompressing */
		/* compressed data ought to be valid before decompressing */
		if (!page) {
		DBG_BUGON(1);
		err = -EIO;
		continue;
		}
		be->compressed_pages[i] = page;

		if (z_erofs_is_inline_pcluster(pcl)) {
		if (z_erofs_is_inline_pcluster(pcl) \|\|
		erofs_page_is_managed(EROFS_SB(be->sb), page)) {
		if (!PageUptodate(page))
		err = -EIO;
		continue;
		}

		DBG_BUGON(z_erofs_page_is_invalidated(page));
		if (!z_erofs_is_shortlived_page(page)) {
		if (erofs_page_is_managed(EROFS_SB(be->sb), page)) {
		if (!PageUptodate(page))
		err = -EIO;
		if (z_erofs_is_shortlived_page(page))
		continue;
		}
		z_erofs_do_decompressed_bvec(be, bvec);
		*overlapped = true;
		}
		}

		if (err)
		return err;
		return 0;
		}

		static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be,
		@@ -1242,10 +1231,9 @@ static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be,
		struct erofs_sb_info *const sbi = EROFS_SB(be->sb);
		struct z_erofs_pcluster *pcl = be->pcl;
		unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
		const struct z_erofs_decompressor *decompressor =
		const struct z_erofs_decompressor *decomp =
		&erofs_decompressors[pcl->algorithmformat];
		unsigned int i, inputsize;
		int err2;
		int i, err2;
		struct page *page;
		bool overlapped;

		@@ -1279,21 +1267,14 @@ static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be,
		err2 = z_erofs_parse_in_bvecs(be, &overlapped);
		if (err2)
		err = err2;
		if (err)
		goto out;

		if (z_erofs_is_inline_pcluster(pcl))
		inputsize = pcl->tailpacking_size;
		else
		inputsize = pclusterpages * PAGE_SIZE;

		err = decompressor->decompress(&(struct z_erofs_decompress_req) {
		if (!err)
		err = decomp->decompress(&(struct z_erofs_decompress_req) {
		.sb = be->sb,
		.in = be->compressed_pages,
		.out = be->decompressed_pages,
		.pageofs_in = pcl->pageofs_in,
		.pageofs_out = pcl->pageofs_out,
		.inputsize = inputsize,
		.inputsize = pcl->pclustersize,
		.outputsize = pcl->length,
		.alg = pcl->algorithmformat,
		.inplace_io = overlapped,
		@@ -1301,7 +1282,6 @@ static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be,
		.fillgaps = pcl->multibases,
		}, be->pagepool);

		out:
		/* must handle all compressed pages before actual file pages */
		if (z_erofs_is_inline_pcluster(pcl)) {
		page = pcl->compressed_bvecs[0].page;
		@@ -1309,12 +1289,11 @@ static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be,
		put_page(page);
		} else {
		for (i = 0; i < pclusterpages; ++i) {
		page = pcl->compressed_bvecs[i].page;
		/* consider shortlived pages added when decompressing */
		page = be->compressed_pages[i];

		if (erofs_page_is_managed(sbi, page))
		if (!page \|\| erofs_page_is_managed(sbi, page))
		continue;

		/* recycle all individual short-lived pages */
		(void)z_erofs_put_shortlivedpage(be->pagepool, page);
		WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
		}
		@@ -1436,86 +1415,85 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io,
		z_erofs_decompressqueue_work(&io->u.work);
		}

		static struct page pickup_page_for_submission(struct z_erofs_pcluster pcl,
		static void z_erofs_fill_bio_vec(struct bio_vec *bvec,
		struct z_erofs_decompress_frontend *f,
		struct z_erofs_pcluster *pcl,
		unsigned int nr,
		struct page **pagepool,
		struct address_space *mc)
		{
		const pgoff_t index = pcl->obj.index;
		gfp_t gfp = mapping_gfp_mask(mc);
		bool tocache = false;

		struct z_erofs_bvec *zbv = pcl->compressed_bvecs + nr;
		struct address_space *mapping;
		struct page oldpage, page;
		int justfound;
		struct page page, oldpage;
		int justfound, bs = i_blocksize(f->inode);

		/* Except for inplace pages, the entire page can be used for I/Os */
		bvec->bv_offset = 0;
		bvec->bv_len = PAGE_SIZE;
		repeat:
		page = READ_ONCE(pcl->compressed_bvecs[nr].page);
		oldpage = page;

		if (!page)
		oldpage = READ_ONCE(zbv->page);
		if (!oldpage)
		goto out_allocpage;

		justfound = (unsigned long)page & 1UL;
		page = (struct page *)((unsigned long)page & ~1UL);
		justfound = (unsigned long)oldpage & 1UL;
		page = (struct page *)((unsigned long)oldpage & ~1UL);
		bvec->bv_page = page;

		DBG_BUGON(z_erofs_is_shortlived_page(page));
		/*
		* preallocated cached pages, which is used to avoid direct reclaim
		* otherwise, it will go inplace I/O path instead.
		* Handle preallocated cached pages. We tried to allocate such pages
		* without triggering direct reclaim. If allocation failed, inplace
		* file-backed pages will be used instead.
		*/
		if (page->private == Z_EROFS_PREALLOCATED_PAGE) {
		WRITE_ONCE(pcl->compressed_bvecs[nr].page, page);
		set_page_private(page, 0);
		WRITE_ONCE(zbv->page, page);
		tocache = true;
		goto out_tocache;
		}
		mapping = READ_ONCE(page->mapping);

		mapping = READ_ONCE(page->mapping);
		/*
		* file-backed online pages in plcuster are all locked steady,
		* therefore it is impossible for `mapping' to be NULL.
		* File-backed pages for inplace I/Os are all locked steady,
		* therefore it is impossible for `mapping` to be NULL.
		*/
		if (mapping && mapping != mc)
		/* ought to be unmanaged pages */
		goto out;

		/* directly return for shortlived page as well */
		if (z_erofs_is_shortlived_page(page))
		goto out;
		if (mapping && mapping != mc) {
		if (zbv->offset < 0)
		bvec->bv_offset = round_up(-zbv->offset, bs);
		bvec->bv_len = round_up(zbv->end, bs) - bvec->bv_offset;
		return;
		}

		lock_page(page);

		/* only true if page reclaim goes wrong, should never happen */
		DBG_BUGON(justfound && PagePrivate(page));

		/* the page is still in manage cache */
		/* the cached page is still in managed cache */
		if (page->mapping == mc) {
		WRITE_ONCE(pcl->compressed_bvecs[nr].page, page);

		if (!PagePrivate(page)) {
		WRITE_ONCE(zbv->page, page);
		/*
		* impossible to be !PagePrivate(page) for
		* the current restriction as well if
		* the page is already in compressed_bvecs[].
		* The cached page is still available but without a valid
		* `->private` pcluster hint. Let's reconnect them.
		*/
		if (!PagePrivate(page)) {
		DBG_BUGON(!justfound);

		justfound = 0;
		set_page_private(page, (unsigned long)pcl);
		SetPagePrivate(page);
		/* compressed_bvecs[] already takes a ref */
		attach_page_private(page, pcl);
		put_page(page);
		}

		/* no need to submit io if it is already up-to-date */
		/* no need to submit if it is already up-to-date */
		if (PageUptodate(page)) {
		unlock_page(page);
		page = NULL;
		bvec->bv_page = NULL;
		}
		goto out;
		return;
		}

		/*
		* the managed page has been truncated, it's unsafe to
		* reuse this one, let's allocate a new cache-managed page.
		* It has been truncated, so it's unsafe to reuse this one. Let's
		* allocate a new page for compressed data.
		*/
		DBG_BUGON(page->mapping);
		DBG_BUGON(!justfound);
		@@ -1524,25 +1502,23 @@ static struct page pickup_page_for_submission(struct z_erofs_pcluster pcl,
		unlock_page(page);
		put_page(page);
		out_allocpage:
		page = erofs_allocpage(pagepool, gfp \| __GFP_NOFAIL);
		if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page,
		oldpage, page)) {
		erofs_pagepool_add(pagepool, page);
		page = erofs_allocpage(&f->pagepool, gfp \| __GFP_NOFAIL);
		if (oldpage != cmpxchg(&zbv->page, oldpage, page)) {
		erofs_pagepool_add(&f->pagepool, page);
		cond_resched();
		goto repeat;
		}
		bvec->bv_page = page;
		out_tocache:
		if (!tocache \|\| add_to_page_cache_lru(page, mc, index + nr, gfp)) {
		/* turn into temporary page if fails (1 ref) */
		if (!tocache \|\| bs != PAGE_SIZE \|\|
		add_to_page_cache_lru(page, mc, pcl->obj.index + nr, gfp)) {
		/* turn into a temporary shortlived page (1 ref) */
		set_page_private(page, Z_EROFS_SHORTLIVED_PAGE);
		goto out;
		return;
		}
		attach_page_private(page, pcl);
		/* drop a refcount added by allocpage (then we have 2 refs here) */
		/* drop a refcount added by allocpage (then 2 refs in total here) */
		put_page(page);

		out: /* the only exit (for tracing and debugging) */
		return page;
		}

		static struct z_erofs_decompressqueue jobqueue_init(struct super_block sb,
		@@ -1597,7 +1573,7 @@ static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,
		qtail[JQ_BYPASS] = &pcl->next;
		}

		static void z_erofs_decompressqueue_endio(struct bio *bio)
		static void z_erofs_submissionqueue_endio(struct bio *bio)
		{
		struct z_erofs_decompressqueue *q = bio->bi_private;
		blk_status_t err = bio->bi_status;
		@@ -1609,7 +1585,6 @@ static void z_erofs_decompressqueue_endio(struct bio *bio)

		DBG_BUGON(PageUptodate(page));
		DBG_BUGON(z_erofs_page_is_invalidated(page));

		if (erofs_page_is_managed(EROFS_SB(q->sb), page)) {
		if (!err)
		SetPageUptodate(page);
		@@ -1632,17 +1607,14 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f,
		struct z_erofs_decompressqueue *q[NR_JOBQUEUES];
		z_erofs_next_pcluster_t owned_head = f->owned_head;
		/* bio is NULL initially, so no need to initialize last_{index,bdev} */
		pgoff_t last_index;
		erofs_off_t last_pa;
		struct block_device *last_bdev;
		unsigned int nr_bios = 0;
		struct bio *bio = NULL;
		unsigned long pflags;
		int memstall = 0;

		/*
		* if managed cache is enabled, bypass jobqueue is needed,
		* no need to read from device for all pclusters in this queue.
		*/
		/* No need to read from device for pclusters in the bypass queue. */
		q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL);
		q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, force_fg);

		@@ -1655,7 +1627,8 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f,
		do {
		struct erofs_map_dev mdev;
		struct z_erofs_pcluster *pcl;
		pgoff_t cur, end;
		erofs_off_t cur, end;
		struct bio_vec bvec;
		unsigned int i = 0;
		bool bypass = true;

		@@ -1674,18 +1647,14 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f,
		};
		(void)erofs_map_dev(sb, &mdev);

		cur = erofs_blknr(sb, mdev.m_pa);
		end = cur + pcl->pclusterpages;

		cur = mdev.m_pa;
		end = cur + pcl->pclustersize;
		do {
		struct page *page;

		page = pickup_page_for_submission(pcl, i++,
		&f->pagepool, mc);
		if (!page)
		z_erofs_fill_bio_vec(&bvec, f, pcl, i++, mc);
		if (!bvec.bv_page)
		continue;

		if (bio && (cur != last_index + 1 \|\|
		if (bio && (cur != last_pa \|\|
		last_bdev != mdev.m_bdev)) {
		submit_bio_retry:
		submit_bio(bio);
		@@ -1696,7 +1665,8 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f,
		bio = NULL;
		}

		if (unlikely(PageWorkingset(page)) && !memstall) {
		if (unlikely(PageWorkingset(bvec.bv_page)) &&
		!memstall) {
		psi_memstall_enter(&pflags);
		memstall = 1;
		}
		@@ -1704,23 +1674,24 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f,
		if (!bio) {
		bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS,
		REQ_OP_READ, GFP_NOIO);
		bio->bi_end_io = z_erofs_decompressqueue_endio;

		last_bdev = mdev.m_bdev;
		bio->bi_iter.bi_sector = (sector_t)cur <<
		(sb->s_blocksize_bits - 9);
		bio->bi_end_io = z_erofs_submissionqueue_endio;
		bio->bi_iter.bi_sector = cur >> 9;
		bio->bi_private = q[JQ_SUBMIT];
		if (readahead)
		bio->bi_opf \|= REQ_RAHEAD;
		++nr_bios;
		last_bdev = mdev.m_bdev;
		}

		if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
		if (cur + bvec.bv_len > end)
		bvec.bv_len = end - cur;
		if (!bio_add_page(bio, bvec.bv_page, bvec.bv_len,
		bvec.bv_offset))
		goto submit_bio_retry;

		last_index = cur;
		last_pa = cur + bvec.bv_len;
		bypass = false;
		} while (++cur < end);
		} while ((cur += bvec.bv_len) < end);

		if (!bypass)
		qtail[JQ_SUBMIT] = &pcl->next;