Commit 94765d71 authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge branch 'for-6.15/io_uring' into for-6.15/io_uring-reg-vec 

* for-6.15/io_uring: (80 commits)
  io_uring: introduce io_cache_free() helper
  io_uring/rsrc: skip NULL file/buffer checks in io_free_rsrc_node()
  io_uring/rsrc: avoid NULL node check on io_sqe_buffer_register() failure
  io_uring/rsrc: call io_free_node() on io_sqe_buffer_register() failure
  io_uring/rsrc: free io_rsrc_node using kfree()
  io_uring/rsrc: split out io_free_node() helper
  io_uring/rsrc: include io_uring_types.h in rsrc.h
  ublk: don't cast registered buffer index to int
  io_uring/nop: use io_find_buf_node()
  io_uring/rsrc: declare io_find_buf_node() in header file
  io_uring/ublk: report error when unregister operation fails
  io_uring: convert cmd_to_io_kiocb() macro to function
  io_uring/uring_cmd: specify io_uring_cmd_import_fixed() pointer type
  io_uring/rsrc: use rq_data_dir() to compute bvec dir
  selftests: ublk: add ublk zero copy test
  selftests: ublk: add file backed ublk
  selftests: ublk: add kernel selftests for ublk
  io_uring: cache nodes and mapped buffers
  ublk: zc register/unregister bvec
  io_uring: add support for kernel registered bvecs
  ...
parents 7eb17214 0d83b8a9

MAINTAINERS

+1 −0
Original line number Diff line number Diff line
@@ -24253,6 +24253,7 @@ S: Maintained 
F:	Documentation/block/ublk.rst

F:	drivers/block/ublk_drv.c

F:	include/uapi/linux/ublk_cmd.h

F:	tools/testing/selftests/ublk/









UBSAN









M:	Kees Cook <kees@kernel.org>

























drivers/block/ublk_drv.c



+49
−7






























Original line number
Diff line number
Diff line







@@ -51,6 +51,9 @@







/* private ioctl command mirror */









#define UBLK_CMD_DEL_DEV_ASYNC	_IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC)



















#define UBLK_IO_REGISTER_IO_BUF		_IOC_NR(UBLK_U_IO_REGISTER_IO_BUF)









#define UBLK_IO_UNREGISTER_IO_BUF	_IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF)



















/* All UBLK_F_* have to be included into UBLK_F_ALL */









#define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \









		| UBLK_F_URING_CMD_COMP_IN_TASK \








@@ -196,12 +199,14 @@ struct ublk_params_header {

















static bool ublk_abort_requests(struct ublk_device *ub, struct ublk_queue *ubq);



















static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,









		struct ublk_queue *ubq, int tag, size_t offset);









static inline unsigned int ublk_req_build_flags(struct request *req);









static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq,









						   int tag);









static inline bool ublk_dev_is_user_copy(const struct ublk_device *ub)









{









	return ub->dev_info.flags & UBLK_F_USER_COPY;









	return ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY);









}



















static inline bool ublk_dev_is_zoned(const struct ublk_device *ub)








@@ -581,7 +586,7 @@ static void ublk_apply_params(struct ublk_device *ub)

















static inline bool ublk_support_user_copy(const struct ublk_queue *ubq)









{









	return ubq->flags & UBLK_F_USER_COPY;









	return ubq->flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY);









}



















static inline bool ublk_need_req_ref(const struct ublk_queue *ubq)








@@ -1747,6 +1752,42 @@ static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,







	io_uring_cmd_mark_cancelable(cmd, issue_flags);









}



















static void ublk_io_release(void *priv)









{









	struct request *rq = priv;









	struct ublk_queue *ubq = rq->mq_hctx->driver_data;



















	ublk_put_req_ref(ubq, rq);









}



















static int ublk_register_io_buf(struct io_uring_cmd *cmd,









				struct ublk_queue *ubq, unsigned int tag,









				unsigned int index, unsigned int issue_flags)









{









	struct ublk_device *ub = cmd->file->private_data;









	struct request *req;









	int ret;



















	req = __ublk_check_and_get_req(ub, ubq, tag, 0);









	if (!req)









		return -EINVAL;



















	ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index,









				      issue_flags);









	if (ret) {









		ublk_put_req_ref(ubq, req);









		return ret;









	}



















	return 0;









}



















static int ublk_unregister_io_buf(struct io_uring_cmd *cmd,









				  unsigned int index, unsigned int issue_flags)









{









	return io_buffer_unregister_bvec(cmd, index, issue_flags);









}



















static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,









			       unsigned int issue_flags,









			       const struct ublksrv_io_cmd *ub_cmd)








@@ -1798,6 +1839,10 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,

















	ret = -EINVAL;









	switch (_IOC_NR(cmd_op)) {









	case UBLK_IO_REGISTER_IO_BUF:









		return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, issue_flags);









	case UBLK_IO_UNREGISTER_IO_BUF:









		return ublk_unregister_io_buf(cmd, ub_cmd->addr, issue_flags);









	case UBLK_IO_FETCH_REQ:









		/* UBLK_IO_FETCH_REQ is only allowed before queue is setup */









		if (ublk_queue_ready(ubq)) {








@@ -2459,7 +2504,7 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)







		 * buffer by pwrite() to ublk char device, which can't be









		 * used for unprivileged device









		 */









		if (info.flags & UBLK_F_USER_COPY)









		if (info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY))









			return -EINVAL;









	}


















@@ -2527,9 +2572,6 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)







		goto out_free_dev_number;









	}



















	/* We are not ready to support zero copy */









	ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY;



















	ub->dev_info.nr_hw_queues = min_t(unsigned int,









			ub->dev_info.nr_hw_queues, nr_cpu_ids);









	ublk_align_max_io_size(ub);








@@ -2860,7 +2902,7 @@ static int ublk_ctrl_get_features(struct io_uring_cmd *cmd)







{









	const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);









	void __user *argp = (void __user *)(unsigned long)header->addr;









	u64 features = UBLK_F_ALL & ~UBLK_F_SUPPORT_ZERO_COPY;









	u64 features = UBLK_F_ALL;



















	if (header->len != UBLK_FEATURES_LEN || !header->addr)









		return -EINVAL;
































drivers/nvme/host/ioctl.c



+7
−5






























Original line number
Diff line number
Diff line







@@ -114,7 +114,8 @@ static struct request *nvme_alloc_user_request(struct request_queue *q,

















static int nvme_map_user_request(struct request *req, u64 ubuffer,









		unsigned bufflen, void __user *meta_buffer, unsigned meta_len,









		struct io_uring_cmd *ioucmd, unsigned int flags)









		struct io_uring_cmd *ioucmd, unsigned int flags,









		unsigned int iou_issue_flags)









{









	struct request_queue *q = req->q;









	struct nvme_ns *ns = q->queuedata;








@@ -142,7 +143,8 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,







		if (WARN_ON_ONCE(flags & NVME_IOCTL_VEC))









			return -EINVAL;









		ret = io_uring_cmd_import_fixed(ubuffer, bufflen,









				rq_data_dir(req), &iter, ioucmd);









				rq_data_dir(req), &iter, ioucmd,









				iou_issue_flags);









		if (ret < 0)









			goto out;









		ret = blk_rq_map_user_iov(q, req, NULL, &iter, GFP_KERNEL);








@@ -194,7 +196,7 @@ static int nvme_submit_user_cmd(struct request_queue *q,







	req->timeout = timeout;









	if (ubuffer && bufflen) {









		ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer,









				meta_len, NULL, flags);









				meta_len, NULL, flags, 0);









		if (ret)









			return ret;









	}








@@ -510,10 +512,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,







		return PTR_ERR(req);









	req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0;



















	if (d.addr && d.data_len) {









	if (d.data_len) {









		ret = nvme_map_user_request(req, d.addr,









			d.data_len, nvme_to_user_ptr(d.metadata),









			d.metadata_len, ioucmd, vec);









			d.metadata_len, ioucmd, vec, issue_flags);









		if (ret)









			return ret;









	}
































include/linux/io_uring/cmd.h



+14
−3






























Original line number
Diff line number
Diff line







@@ -4,6 +4,7 @@

















#include <uapi/linux/io_uring.h>









#include <linux/io_uring_types.h>









#include <linux/blk-mq.h>



















/* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */









#define IORING_URING_CMD_CANCELABLE	(1U << 30)








@@ -39,7 +40,9 @@ static inline void io_uring_cmd_private_sz_check(size_t cmd_sz)

















#if defined(CONFIG_IO_URING)









int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,









			      struct iov_iter *iter, void *ioucmd);









			      struct iov_iter *iter,









			      struct io_uring_cmd *ioucmd,









			      unsigned int issue_flags);



















/*









 * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd








@@ -66,8 +69,10 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,







void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd);



















#else









static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,









			      struct iov_iter *iter, void *ioucmd)









static inline int









io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,









			  struct iov_iter *iter, struct io_uring_cmd *ioucmd,









			  unsigned int issue_flags)









{









	return -EOPNOTSUPP;









}








@@ -123,4 +128,10 @@ static inline struct io_uring_cmd_data *io_uring_cmd_get_async_data(struct io_ur







	return cmd_to_io_kiocb(cmd)->async_data;









}



















int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,









			    void (*release)(void *), unsigned int index,









			    unsigned int issue_flags);









int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,









			      unsigned int issue_flags);



















#endif /* _LINUX_IO_URING_CMD_H */
























include/linux/io_uring_types.h



+15
−5






























Original line number
Diff line number
Diff line







@@ -292,6 +292,8 @@ struct io_ring_ctx {

















		struct io_file_table	file_table;









		struct io_rsrc_data	buf_table;









		struct io_alloc_cache	node_cache;









		struct io_alloc_cache	imu_cache;



















		struct io_submit_state	submit_state;


















@@ -360,7 +362,6 @@ struct io_ring_ctx {

















	spinlock_t		completion_lock;



















	struct list_head	io_buffers_comp;









	struct list_head	cq_overflow_list;



















	struct hlist_head	waitid_list;








@@ -379,8 +380,6 @@ struct io_ring_ctx {







	unsigned int		file_alloc_start;









	unsigned int		file_alloc_end;



















	struct list_head	io_buffers_cache;



















	/* Keep this last, we don't need it for the fast path */









	struct wait_queue_head		poll_wq;









	struct io_restriction		restrictions;








@@ -439,8 +438,15 @@ struct io_ring_ctx {







	struct io_mapped_region		param_region;









};



















/*









 * Token indicating function is called in task work context:









 * ctx->uring_lock is held and any completions generated will be flushed.









 * ONLY core io_uring.c should instantiate this struct.









 */









struct io_tw_state {









};









/* Alias to use in code that doesn't instantiate struct io_tw_state */









typedef struct io_tw_state io_tw_token_t;



















enum {









	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,








@@ -566,7 +572,7 @@ enum {







	REQ_F_HAS_METADATA	= IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),









};



















typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);









typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);



















struct io_task_work {









	struct llist_node		node;








@@ -601,7 +607,11 @@ static inline void io_kiocb_cmd_sz_check(size_t cmd_sz)







	io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \









	((cmd_type *)&(req)->cmd) \









)









#define cmd_to_io_kiocb(ptr)	((struct io_kiocb *) ptr)



















static inline struct io_kiocb *cmd_to_io_kiocb(void *ptr)









{









	return ptr;









}



















struct io_kiocb {









	union {