Commit 394244b2 authored by Joanne Koong's avatar Joanne Koong Committed by Miklos Szeredi
Browse files

fuse: support copying large folios 



Currently, all folios associated with fuse are one page size. As part of
the work to enable large folios, this commit adds support for copying
to/from folios larger than one page size.

Signed-off-by: default avatarJoanne Koong <joannelkoong@gmail.com>
Reviewed-by: default avatarJeff Layton <jlayton@kernel.org>
Reviewed-by: default avatarBernd Schubert <bschubert@ddn.com>
Signed-off-by: default avatarMiklos Szeredi <mszeredi@redhat.com>
parent f0922298

fs/fuse/dev.c

+48 −49
Original line number Diff line number Diff line
@@ -956,10 +956,10 @@ static int fuse_check_folio(struct folio *folio) 
 * folio that was originally in @pagep will lose a reference and the new

 * folio returned in @pagep will carry a reference.

 */

static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)

static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop)

{

	int err;

	struct folio *oldfolio = page_folio(*pagep);

	struct folio *oldfolio = *foliop;

	struct folio *newfolio;

	struct pipe_buffer *buf = cs->pipebufs;
@@ -980,7 +980,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) 
	cs->pipebufs++;

	cs->nr_segs--;



	if (cs->len != PAGE_SIZE)

	if (cs->len != folio_size(oldfolio))

		goto out_fallback;



	if (!pipe_buf_try_steal(cs->pipe, buf))
@@ -1026,7 +1026,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) 
	if (test_bit(FR_ABORTED, &cs->req->flags))

		err = -ENOENT;

	else

		*pagep = &newfolio->page;

		*foliop = newfolio;

	spin_unlock(&cs->req->waitq.lock);



	if (err) {
@@ -1059,7 +1059,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) 
	goto out_put_old;

}



static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,

static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio,

			  unsigned offset, unsigned count)

{

	struct pipe_buffer *buf;
@@ -1068,17 +1068,17 @@ static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page, 
	if (cs->nr_segs >= cs->pipe->max_usage)

		return -EIO;



	get_page(page);

	folio_get(folio);

	err = unlock_request(cs->req);

	if (err) {

		put_page(page);

		folio_put(folio);

		return err;

	}



	fuse_copy_finish(cs);



	buf = cs->pipebufs;

	buf->page = page;

	buf->page = &folio->page;

	buf->offset = offset;

	buf->len = count;
@@ -1090,20 +1090,24 @@ static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page, 
}



/*

 * Copy a page in the request to/from the userspace buffer.  Must be

 * Copy a folio in the request to/from the userspace buffer.  Must be

 * done atomically

 */

static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,

static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,

			   unsigned offset, unsigned count, int zeroing)

{

	int err;

	struct page *page = *pagep;

	struct folio *folio = *foliop;

	size_t size;



	if (page && zeroing && count < PAGE_SIZE)

		clear_highpage(page);

	if (folio) {

		size = folio_size(folio);

		if (zeroing && count < size)

			folio_zero_range(folio, 0, size);

	}



	while (count) {

		if (cs->write && cs->pipebufs && page) {

		if (cs->write && cs->pipebufs && folio) {

			/*

			 * Can't control lifetime of pipe buffers, so always

			 * copy user pages.
@@ -1113,12 +1117,12 @@ static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep, 
				if (err)

					return err;

			} else {

				return fuse_ref_page(cs, page, offset, count);

				return fuse_ref_folio(cs, folio, offset, count);

			}

		} else if (!cs->len) {

			if (cs->move_pages && page &&

			    offset == 0 && count == PAGE_SIZE) {

				err = fuse_try_move_page(cs, pagep);

			if (cs->move_folios && folio &&

			    offset == 0 && count == size) {

				err = fuse_try_move_folio(cs, foliop);

				if (err <= 0)

					return err;

			} else {
@@ -1127,21 +1131,29 @@ static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep, 
					return err;

			}

		}

		if (page) {

			void *mapaddr = kmap_local_page(page);

			void *buf = mapaddr + offset;

			offset += fuse_copy_do(cs, &buf, &count);

		if (folio) {

			void *mapaddr = kmap_local_folio(folio, offset);

			void *buf = mapaddr;

			unsigned int copy = count;

			unsigned int bytes_copied;



			if (folio_test_highmem(folio) && count > PAGE_SIZE - offset_in_page(offset))

				copy = PAGE_SIZE - offset_in_page(offset);



			bytes_copied = fuse_copy_do(cs, &buf, &copy);

			kunmap_local(mapaddr);

			offset += bytes_copied;

			count -= bytes_copied;

		} else

			offset += fuse_copy_do(cs, NULL, &count);

	}

	if (page && !cs->write)

		flush_dcache_page(page);

	if (folio && !cs->write)

		flush_dcache_folio(folio);

	return 0;

}



/* Copy pages in the request to/from userspace buffer */

static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,

/* Copy folios in the request to/from userspace buffer */

static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes,

			    int zeroing)

{

	unsigned i;
@@ -1152,23 +1164,12 @@ static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes, 
		int err;

		unsigned int offset = ap->descs[i].offset;

		unsigned int count = min(nbytes, ap->descs[i].length);

		struct page *orig, *pagep;



		orig = pagep = &ap->folios[i]->page;



		err = fuse_copy_page(cs, &pagep, offset, count, zeroing);

		err = fuse_copy_folio(cs, &ap->folios[i], offset, count, zeroing);

		if (err)

			return err;



		nbytes -= count;



		/*

		 *  fuse_copy_page may have moved a page from a pipe instead of

		 *  copying into our given page, so update the folios if it was

		 *  replaced.

		 */

		if (pagep != orig)

			ap->folios[i] = page_folio(pagep);

	}

	return 0;

}
@@ -1198,7 +1199,7 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, 
	for (i = 0; !err && i < numargs; i++)  {

		struct fuse_arg *arg = &args[i];

		if (i == numargs - 1 && argpages)

			err = fuse_copy_pages(cs, arg->size, zeroing);

			err = fuse_copy_folios(cs, arg->size, zeroing);

		else

			err = fuse_copy_one(cs, arg->value, arg->size);

	}
@@ -1787,7 +1788,6 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, 
	num = outarg.size;

	while (num) {

		struct folio *folio;

		struct page *page;

		unsigned int this_num;



		folio = filemap_grab_folio(mapping, index);
@@ -1795,9 +1795,8 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, 
		if (IS_ERR(folio))

			goto out_iput;



		page = &folio->page;

		this_num = min_t(unsigned, num, folio_size(folio) - offset);

		err = fuse_copy_page(cs, &page, offset, this_num, 0);

		err = fuse_copy_folio(cs, &folio, offset, this_num, 0);

		if (!folio_test_uptodate(folio) && !err && offset == 0 &&

		    (this_num == folio_size(folio) || file_size == end)) {

			folio_zero_segment(folio, this_num, folio_size(folio));
@@ -2038,8 +2037,8 @@ static int fuse_notify_inc_epoch(struct fuse_conn *fc) 
static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,

		       unsigned int size, struct fuse_copy_state *cs)

{

	/* Don't try to move pages (yet) */

	cs->move_pages = false;

	/* Don't try to move folios (yet) */

	cs->move_folios = false;



	switch (code) {

	case FUSE_NOTIFY_POLL:
@@ -2190,7 +2189,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, 
	spin_unlock(&fpq->lock);

	cs->req = req;

	if (!req->args->page_replace)

		cs->move_pages = false;

		cs->move_folios = false;



	if (oh.error)

		err = nbytes != sizeof(oh) ? -EINVAL : 0;
@@ -2308,7 +2307,7 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe, 
	cs.pipe = pipe;



	if (flags & SPLICE_F_MOVE)

		cs.move_pages = true;

		cs.move_folios = true;



	ret = fuse_dev_do_write(fud, &cs, len);

fs/fuse/fuse_dev_i.h

+1 −1
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ struct fuse_copy_state { 
	unsigned int len;

	unsigned int offset;

	bool write:1;

	bool move_pages:1;

	bool move_folios:1;

	bool is_uring:1;

	struct {

		unsigned int copied_sz; /* copied size into the user buffer */