Commit c059f785 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: move io_import_fixed() 



Move io_import_fixed() into rsrc.c where it belongs.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/4d5becb21f332b4fef6a7cedd6a50e65e2371630.1655684496.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent f337a84d

io_uring/rsrc.c

+60 −0
Original line number Diff line number Diff line
@@ -1310,3 +1310,63 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, 
		io_rsrc_node_switch(ctx, NULL);

	return ret;

}



int io_import_fixed(int ddir, struct iov_iter *iter,

			   struct io_mapped_ubuf *imu,

			   u64 buf_addr, size_t len)

{

	u64 buf_end;

	size_t offset;



	if (WARN_ON_ONCE(!imu))

		return -EFAULT;

	if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))

		return -EFAULT;

	/* not inside the mapped region */

	if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))

		return -EFAULT;



	/*

	 * May not be a start of buffer, set size appropriately

	 * and advance us to the beginning.

	 */

	offset = buf_addr - imu->ubuf;

	iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, offset + len);



	if (offset) {

		/*

		 * Don't use iov_iter_advance() here, as it's really slow for

		 * using the latter parts of a big fixed buffer - it iterates

		 * over each segment manually. We can cheat a bit here, because

		 * we know that:

		 *

		 * 1) it's a BVEC iter, we set it up

		 * 2) all bvecs are PAGE_SIZE in size, except potentially the

		 *    first and last bvec

		 *

		 * So just find our index, and adjust the iterator afterwards.

		 * If the offset is within the first bvec (or the whole first

		 * bvec, just use iov_iter_advance(). This makes it easier

		 * since we can just skip the first segment, which may not

		 * be PAGE_SIZE aligned.

		 */

		const struct bio_vec *bvec = imu->bvec;



		if (offset <= bvec->bv_len) {

			iov_iter_advance(iter, offset);

		} else {

			unsigned long seg_skip;



			/* skip first vec */

			offset -= bvec->bv_len;

			seg_skip = 1 + (offset >> PAGE_SHIFT);



			iter->bvec = bvec + seg_skip;

			iter->nr_segs -= seg_skip;

			iter->count -= bvec->bv_len + offset;

			iter->iov_offset = offset & ~PAGE_MASK;

		}

	}



	return 0;

}

io_uring/rsrc.h

+3 −0
Original line number Diff line number Diff line
@@ -64,6 +64,9 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, 
void io_rsrc_node_switch(struct io_ring_ctx *ctx,

			 struct io_rsrc_data *data_to_kill);



int io_import_fixed(int ddir, struct iov_iter *iter,

			   struct io_mapped_ubuf *imu,

			   u64 buf_addr, size_t len);



void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx);

int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);

io_uring/rw.c

+0 −60
Original line number Diff line number Diff line
@@ -273,66 +273,6 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, 
	return IOU_ISSUE_SKIP_COMPLETE;

}



static int io_import_fixed(int ddir, struct iov_iter *iter,

			   struct io_mapped_ubuf *imu,

			   u64 buf_addr, size_t len)

{

	u64 buf_end;

	size_t offset;



	if (WARN_ON_ONCE(!imu))

		return -EFAULT;

	if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))

		return -EFAULT;

	/* not inside the mapped region */

	if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))

		return -EFAULT;



	/*

	 * May not be a start of buffer, set size appropriately

	 * and advance us to the beginning.

	 */

	offset = buf_addr - imu->ubuf;

	iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, offset + len);



	if (offset) {

		/*

		 * Don't use iov_iter_advance() here, as it's really slow for

		 * using the latter parts of a big fixed buffer - it iterates

		 * over each segment manually. We can cheat a bit here, because

		 * we know that:

		 *

		 * 1) it's a BVEC iter, we set it up

		 * 2) all bvecs are PAGE_SIZE in size, except potentially the

		 *    first and last bvec

		 *

		 * So just find our index, and adjust the iterator afterwards.

		 * If the offset is within the first bvec (or the whole first

		 * bvec, just use iov_iter_advance(). This makes it easier

		 * since we can just skip the first segment, which may not

		 * be PAGE_SIZE aligned.

		 */

		const struct bio_vec *bvec = imu->bvec;



		if (offset <= bvec->bv_len) {

			iov_iter_advance(iter, offset);

		} else {

			unsigned long seg_skip;



			/* skip first vec */

			offset -= bvec->bv_len;

			seg_skip = 1 + (offset >> PAGE_SHIFT);



			iter->bvec = bvec + seg_skip;

			iter->nr_segs -= seg_skip;

			iter->count -= bvec->bv_len + offset;

			iter->iov_offset = offset & ~PAGE_MASK;

		}

	}



	return 0;

}



#ifdef CONFIG_COMPAT

static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,

				unsigned int issue_flags)