Unverified Commit 06fa229c authored by David Howells's avatar David Howells Committed by Christian Brauner
Browse files

netfs: Abstract out a rolling folio buffer implementation 



A rolling buffer is a series of folios held in a list of folio_queues.  New
folios and folio_queue structs may be inserted at the head simultaneously
with spent ones being removed from the tail without the need for locking.

The rolling buffer includes an iov_iter and it has to be careful managing
this as the list of folio_queues is extended such that an oops doesn't
incurred because the iterator was pointing to the end of a folio_queue
segment that got appended to and then removed.

We need to use the mechanism twice, once for read and once for write, and,
in future patches, we will use a second rolling buffer to handle bounce
buffering for content encryption.

Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
Link: https://lore.kernel.org/r/20241216204124.3752367-6-dhowells@redhat.com


cc: Jeff Layton <jlayton@kernel.org>
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
Signed-off-by: default avatarChristian Brauner <brauner@kernel.org>
parent aabcabf2

fs/netfs/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ netfs-y := \ 
	read_collect.o \

	read_pgpriv2.o \

	read_retry.o \

	rolling_buffer.o \

	write_collect.o \

	write_issue.o

fs/netfs/buffered_read.c

+23 −96
Original line number Diff line number Diff line
@@ -63,37 +63,6 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in 
	return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));

}



/*

 * Decant the list of folios to read into a rolling buffer.

 */

static size_t netfs_load_buffer_from_ra(struct netfs_io_request *rreq,

					struct folio_queue *folioq,

					struct folio_batch *put_batch)

{

	unsigned int order, nr;

	size_t size = 0;



	nr = __readahead_batch(rreq->ractl, (struct page **)folioq->vec.folios,

			       ARRAY_SIZE(folioq->vec.folios));

	folioq->vec.nr = nr;

	for (int i = 0; i < nr; i++) {

		struct folio *folio = folioq_folio(folioq, i);



		trace_netfs_folio(folio, netfs_folio_trace_read);

		order = folio_order(folio);

		folioq->orders[i] = order;

		size += PAGE_SIZE << order;



		if (!folio_batch_add(put_batch, folio))

			folio_batch_release(put_batch);

	}



	for (int i = nr; i < folioq_nr_slots(folioq); i++)

		folioq_clear(folioq, i);



	return size;

}



/*

 * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O

 * @subreq: The subrequest to be set up
@@ -128,18 +97,12 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) 


		folio_batch_init(&put_batch);

		while (rreq->submitted < subreq->start + rsize) {

			struct folio_queue *tail = rreq->buffer_tail, *new;

			size_t added;

			ssize_t added;



			new = netfs_folioq_alloc(rreq->debug_id, GFP_NOFS,

						 netfs_trace_folioq_alloc_read_prep);

			if (!new)

				return -ENOMEM;

			new->prev = tail;

			tail->next = new;

			rreq->buffer_tail = new;

			added = netfs_load_buffer_from_ra(rreq, new, &put_batch);

			rreq->iter.count += added;

			added = rolling_buffer_load_from_ra(&rreq->buffer, rreq->ractl,

							    &put_batch);

			if (added < 0)

				return added;

			rreq->submitted += added;

		}

		folio_batch_release(&put_batch);
@@ -147,7 +110,7 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) 


	subreq->len = rsize;

	if (unlikely(rreq->io_streams[0].sreq_max_segs)) {

		size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize,

		size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize,

						rreq->io_streams[0].sreq_max_segs);



		if (limit < rsize) {
@@ -156,20 +119,16 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) 
		}

	}



	subreq->io_iter	= rreq->iter;

	subreq->io_iter	= rreq->buffer.iter;



	if (iov_iter_is_folioq(&subreq->io_iter)) {

		if (subreq->io_iter.folioq_slot >= folioq_nr_slots(subreq->io_iter.folioq)) {

			subreq->io_iter.folioq = subreq->io_iter.folioq->next;

			subreq->io_iter.folioq_slot = 0;

		}

		subreq->curr_folioq = (struct folio_queue *)subreq->io_iter.folioq;

		subreq->curr_folioq_slot = subreq->io_iter.folioq_slot;

		subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];

	}



	iov_iter_truncate(&subreq->io_iter, subreq->len);

	iov_iter_advance(&rreq->iter, subreq->len);

	rolling_buffer_advance(&rreq->buffer, subreq->len);

	return subreq->len;

}
@@ -352,34 +311,6 @@ static int netfs_wait_for_read(struct netfs_io_request *rreq) 
	return ret;

}



/*

 * Set up the initial folioq of buffer folios in the rolling buffer and set the

 * iterator to refer to it.

 */

static int netfs_prime_buffer(struct netfs_io_request *rreq)

{

	struct folio_queue *folioq;

	struct folio_batch put_batch;

	size_t added;



	folioq = netfs_folioq_alloc(rreq->debug_id, GFP_KERNEL,

				    netfs_trace_folioq_alloc_read_prime);

	if (!folioq)

		return -ENOMEM;



	rreq->buffer = folioq;

	rreq->buffer_tail = folioq;

	rreq->submitted = rreq->start;

	iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, 0);



	folio_batch_init(&put_batch);

	added = netfs_load_buffer_from_ra(rreq, folioq, &put_batch);

	folio_batch_release(&put_batch);

	rreq->iter.count += added;

	rreq->submitted += added;

	return 0;

}



/**

 * netfs_readahead - Helper to manage a read request

 * @ractl: The description of the readahead request
@@ -419,7 +350,8 @@ void netfs_readahead(struct readahead_control *ractl) 
	netfs_rreq_expand(rreq, ractl);



	rreq->ractl = ractl;

	if (netfs_prime_buffer(rreq) < 0)

	rreq->submitted = rreq->start;

	if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST) < 0)

		goto cleanup_free;

	netfs_read_to_pagecache(rreq);
@@ -435,22 +367,18 @@ EXPORT_SYMBOL(netfs_readahead); 
/*

 * Create a rolling buffer with a single occupying folio.

 */

static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio)

static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio,

					unsigned int rollbuf_flags)

{

	struct folio_queue *folioq;

	ssize_t added;



	folioq = netfs_folioq_alloc(rreq->debug_id, GFP_KERNEL,

				    netfs_trace_folioq_alloc_read_sing);

	if (!folioq)

	if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST) < 0)

		return -ENOMEM;



	folioq_append(folioq, folio);

	BUG_ON(folioq_folio(folioq, 0) != folio);

	BUG_ON(folioq_folio_order(folioq, 0) != folio_order(folio));

	rreq->buffer = folioq;

	rreq->buffer_tail = folioq;

	rreq->submitted = rreq->start + rreq->len;

	iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, rreq->len);

	added = rolling_buffer_append(&rreq->buffer, folio, rollbuf_flags);

	if (added < 0)

		return added;

	rreq->submitted = rreq->start + added;

	rreq->ractl = (struct readahead_control *)1UL;

	return 0;

}
@@ -518,7 +446,7 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) 
	}

	if (to < flen)

		bvec_set_folio(&bvec[i++], folio, flen - to, to);

	iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len);

	iov_iter_bvec(&rreq->buffer.iter, ITER_DEST, bvec, i, rreq->len);

	rreq->submitted = rreq->start + flen;



	netfs_read_to_pagecache(rreq);
@@ -586,7 +514,7 @@ int netfs_read_folio(struct file *file, struct folio *folio) 
	trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage);



	/* Set up the output buffer */

	ret = netfs_create_singular_buffer(rreq, folio);

	ret = netfs_create_singular_buffer(rreq, folio, 0);

	if (ret < 0)

		goto discard;
@@ -743,7 +671,7 @@ int netfs_write_begin(struct netfs_inode *ctx, 
	trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin);



	/* Set up the output buffer */

	ret = netfs_create_singular_buffer(rreq, folio);

	ret = netfs_create_singular_buffer(rreq, folio, 0);

	if (ret < 0)

		goto error_put;
@@ -808,11 +736,10 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, 
	trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write);



	/* Set up the output buffer */

	ret = netfs_create_singular_buffer(rreq, folio);

	ret = netfs_create_singular_buffer(rreq, folio, NETFS_ROLLBUF_PAGECACHE_MARK);

	if (ret < 0)

		goto error_put;



	folioq_mark2(rreq->buffer, 0);

	netfs_read_to_pagecache(rreq);

	ret = netfs_wait_for_read(rreq);

	netfs_put_request(rreq, false, netfs_rreq_trace_put_return);

fs/netfs/direct_read.c

+7 −7
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) 
	subreq->len = rsize;



	if (unlikely(rreq->io_streams[0].sreq_max_segs)) {

		size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize,

		size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize,

						rreq->io_streams[0].sreq_max_segs);



		if (limit < rsize) {
@@ -36,9 +36,9 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) 


	trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);



	subreq->io_iter	= rreq->iter;

	subreq->io_iter	= rreq->buffer.iter;

	iov_iter_truncate(&subreq->io_iter, subreq->len);

	iov_iter_advance(&rreq->iter, subreq->len);

	iov_iter_advance(&rreq->buffer.iter, subreq->len);

}



/*
@@ -199,15 +199,15 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i 
	 * the request.

	 */

	if (user_backed_iter(iter)) {

		ret = netfs_extract_user_iter(iter, rreq->len, &rreq->iter, 0);

		ret = netfs_extract_user_iter(iter, rreq->len, &rreq->buffer.iter, 0);

		if (ret < 0)

			goto out;

		rreq->direct_bv = (struct bio_vec *)rreq->iter.bvec;

		rreq->direct_bv = (struct bio_vec *)rreq->buffer.iter.bvec;

		rreq->direct_bv_count = ret;

		rreq->direct_bv_unpin = iov_iter_extract_will_pin(iter);

		rreq->len = iov_iter_count(&rreq->iter);

		rreq->len = iov_iter_count(&rreq->buffer.iter);

	} else {

		rreq->iter = *iter;

		rreq->buffer.iter = *iter;

		rreq->len = orig_count;

		rreq->direct_bv_unpin = false;

		iov_iter_advance(iter, orig_count);

fs/netfs/direct_write.c

+4 −6
Original line number Diff line number Diff line
@@ -68,19 +68,17 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * 
		 * request.

		 */

		if (async || user_backed_iter(iter)) {

			n = netfs_extract_user_iter(iter, len, &wreq->iter, 0);

			n = netfs_extract_user_iter(iter, len, &wreq->buffer.iter, 0);

			if (n < 0) {

				ret = n;

				goto out;

			}

			wreq->direct_bv = (struct bio_vec *)wreq->iter.bvec;

			wreq->direct_bv = (struct bio_vec *)wreq->buffer.iter.bvec;

			wreq->direct_bv_count = n;

			wreq->direct_bv_unpin = iov_iter_extract_will_pin(iter);

		} else {

			wreq->iter = *iter;

			wreq->buffer.iter = *iter;

		}



		wreq->io_iter = wreq->iter;

	}



	__set_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags);
@@ -92,7 +90,7 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * 
	__set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);

	if (async)

		wreq->iocb = iocb;

	wreq->len = iov_iter_count(&wreq->io_iter);

	wreq->len = iov_iter_count(&wreq->buffer.iter);

	wreq->cleanup = netfs_cleanup_dio_write;

	ret = netfs_unbuffered_write(wreq, is_sync_kiocb(iocb), wreq->len);

	if (ret < 0) {

fs/netfs/internal.h

+0 −4
Original line number Diff line number Diff line
@@ -60,10 +60,6 @@ static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {} 
 */

struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq,

					    enum netfs_folioq_trace trace);

int netfs_buffer_append_folio(struct netfs_io_request *rreq, struct folio *folio,

			      bool needs_put);

struct folio_queue *netfs_delete_buffer_head(struct netfs_io_request *wreq);

void netfs_clear_buffer(struct netfs_io_request *rreq);

void netfs_reset_iter(struct netfs_io_subrequest *subreq);



/*