Commit 3d6106ae authored by Chenliang Li's avatar Chenliang Li Committed by Jens Axboe
Browse files

io_uring/rsrc: store folio shift and mask into imu 



Store the folio shift and folio mask into imu struct and use it in
iov_iter adjust, as we will have non PAGE_SIZE'd chunks if a
multi-hugepage buffer get coalesced.

Signed-off-by: default avatarChenliang Li <cliang01.li@samsung.com>
Reviewed-by: default avatarAnuj Gupta <anuj20.g@samsung.com>
Reviewed-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/20240731090133.4106-2-cliang01.li@samsung.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent d843634a

io_uring/rsrc.c

+6 −9
Original line number Diff line number Diff line
@@ -915,6 +915,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, 
	imu->ubuf = (unsigned long) iov->iov_base;

	imu->ubuf_end = imu->ubuf + iov->iov_len;

	imu->nr_bvecs = nr_pages;

	imu->folio_shift = PAGE_SHIFT;

	imu->folio_mask = PAGE_MASK;

	*pimu = imu;

	ret = 0;
@@ -1031,23 +1033,18 @@ int io_import_fixed(int ddir, struct iov_iter *iter, 
		 * we know that:

		 *

		 * 1) it's a BVEC iter, we set it up

		 * 2) all bvecs are PAGE_SIZE in size, except potentially the

		 * 2) all bvecs are the same in size, except potentially the

		 *    first and last bvec

		 *

		 * So just find our index, and adjust the iterator afterwards.

		 * If the offset is within the first bvec (or the whole first

		 * bvec, just use iov_iter_advance(). This makes it easier

		 * since we can just skip the first segment, which may not

		 * be PAGE_SIZE aligned.

		 * be folio_size aligned.

		 */

		const struct bio_vec *bvec = imu->bvec;



		if (offset < bvec->bv_len) {

			/*

			 * Note, huge pages buffers consists of one large

			 * bvec entry and should always go this way. The other

			 * branch doesn't expect non PAGE_SIZE'd chunks.

			 */

			iter->bvec = bvec;

			iter->count -= offset;

			iter->iov_offset = offset;
@@ -1056,12 +1053,12 @@ int io_import_fixed(int ddir, struct iov_iter *iter, 


			/* skip first vec */

			offset -= bvec->bv_len;

			seg_skip = 1 + (offset >> PAGE_SHIFT);

			seg_skip = 1 + (offset >> imu->folio_shift);



			iter->bvec = bvec + seg_skip;

			iter->nr_segs -= seg_skip;

			iter->count -= bvec->bv_len + offset;

			iter->iov_offset = offset & ~PAGE_MASK;

			iter->iov_offset = offset & ~imu->folio_mask;

		}

	}

io_uring/rsrc.h

+2 −0
Original line number Diff line number Diff line
@@ -46,7 +46,9 @@ struct io_mapped_ubuf { 
	u64		ubuf;

	u64		ubuf_end;

	unsigned int	nr_bvecs;

	unsigned int    folio_shift;

	unsigned long	acct_pages;

	unsigned long   folio_mask;

	struct bio_vec	bvec[] __counted_by(nr_bvecs);

};