Commit f1a6cf44 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Eric Biggers
Browse files

fsverity: kick off hash readahead at data I/O submission time 



Currently all reads of the fsverity hashes are kicked off from the data
I/O completion handler, leading to needlessly dependent I/O.  This is
worked around a bit by performing readahead on the level 0 nodes, but
still fairly ineffective.

Switch to a model where the ->read_folio and ->readahead methods instead
kick off explicit readahead of the fsverity hashed so they are usually
available at I/O completion time.

For 64k sequential reads on my test VM this improves read performance
from 2.4GB/s - 2.6GB/s to 3.5GB/s - 3.9GB/s.  The improvements for
random reads are likely to be even bigger.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Acked-by: David Sterba <dsterba@suse.com> # btrfs
Link: https://lore.kernel.org/r/20260202060754.270269-5-hch@lst.de


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent 314b652b

fs/btrfs/verity.c

+1 −3
Original line number Diff line number Diff line
@@ -697,7 +697,6 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size) 
 *

 * @inode:         inode to read a merkle tree page for

 * @index:         page index relative to the start of the merkle tree

 * @num_ra_pages:  number of pages to readahead. Optional, we ignore it

 *

 * The Merkle tree is stored in the filesystem btree, but its pages are cached

 * with a logical position past EOF in the inode's mapping.
@@ -705,8 +704,7 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size) 
 * Returns the page we read, or an ERR_PTR on error.

 */

static struct page *btrfs_read_merkle_tree_page(struct inode *inode,

						pgoff_t index,

						unsigned long num_ra_pages)

						pgoff_t index)

{

	struct folio *folio;

	u64 off = (u64)index << PAGE_SHIFT;

fs/ext4/readpage.c

+11 −6
Original line number Diff line number Diff line
@@ -397,18 +397,20 @@ static int ext4_mpage_readpages(struct inode *inode, 


int ext4_read_folio(struct file *file, struct folio *folio)

{

	int ret = -EAGAIN;

	struct inode *inode = folio->mapping->host;

	int ret;



	trace_ext4_read_folio(inode, folio);



	if (ext4_has_inline_data(inode))

	if (ext4_has_inline_data(inode)) {

		ret = ext4_readpage_inline(inode, folio);

		if (ret != -EAGAIN)

			return ret;

	}



	if (ret == -EAGAIN)

	if (ext4_need_verity(inode, folio->index))

		fsverity_readahead(inode, folio->index, folio_nr_pages(folio));

	return ext4_mpage_readpages(inode, NULL, folio);



	return ret;

}



void ext4_readahead(struct readahead_control *rac)
@@ -419,6 +421,9 @@ void ext4_readahead(struct readahead_control *rac) 
	if (ext4_has_inline_data(inode))

		return;



	if (ext4_need_verity(inode, readahead_index(rac)))

		fsverity_readahead(inode, readahead_index(rac),

				   readahead_count(rac));

	ext4_mpage_readpages(inode, rac, NULL);

}

fs/ext4/verity.c

+10 −3
Original line number Diff line number Diff line
@@ -358,11 +358,17 @@ static int ext4_get_verity_descriptor(struct inode *inode, void *buf, 
}



static struct page *ext4_read_merkle_tree_page(struct inode *inode,

					       pgoff_t index,

					       unsigned long num_ra_pages)

					       pgoff_t index)

{

	index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;

	return generic_read_merkle_tree_page(inode, index, num_ra_pages);

	return generic_read_merkle_tree_page(inode, index);

}



static void ext4_readahead_merkle_tree(struct inode *inode, pgoff_t index,

				       unsigned long nr_pages)

{

	index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;

	generic_readahead_merkle_tree(inode, index, nr_pages);

}



static int ext4_write_merkle_tree_block(struct file *file, const void *buf,
@@ -380,5 +386,6 @@ const struct fsverity_operations ext4_verityops = { 
	.end_enable_verity	= ext4_end_enable_verity,

	.get_verity_descriptor	= ext4_get_verity_descriptor,

	.read_merkle_tree_page	= ext4_read_merkle_tree_page,

	.readahead_merkle_tree	= ext4_readahead_merkle_tree,

	.write_merkle_tree_block = ext4_write_merkle_tree_block,

};

fs/f2fs/data.c

+12 −5
Original line number Diff line number Diff line
@@ -2458,7 +2458,7 @@ static int f2fs_mpage_readpages(struct inode *inode, 
static int f2fs_read_data_folio(struct file *file, struct folio *folio)

{

	struct inode *inode = folio->mapping->host;

	int ret = -EAGAIN;

	int ret;



	trace_f2fs_readpage(folio, DATA);
@@ -2468,13 +2468,17 @@ static int f2fs_read_data_folio(struct file *file, struct folio *folio) 
	}



	/* If the file has inline data, try to read it directly */

	if (f2fs_has_inline_data(inode))

	if (f2fs_has_inline_data(inode)) {

		ret = f2fs_read_inline_data(inode, folio);

	if (ret == -EAGAIN)

		ret = f2fs_mpage_readpages(inode, NULL, folio);

		if (ret != -EAGAIN)

			return ret;

	}



	if (f2fs_need_verity(inode, folio->index))

		fsverity_readahead(inode, folio->index, folio_nr_pages(folio));

	return f2fs_mpage_readpages(inode, NULL, folio);

}



static void f2fs_readahead(struct readahead_control *rac)

{

	struct inode *inode = rac->mapping->host;
@@ -2488,6 +2492,9 @@ static void f2fs_readahead(struct readahead_control *rac) 
	if (f2fs_has_inline_data(inode))

		return;



	if (f2fs_need_verity(inode, readahead_index(rac)))

		fsverity_readahead(inode, readahead_index(rac),

				   readahead_count(rac));

	f2fs_mpage_readpages(inode, rac, NULL);

}

fs/f2fs/verity.c

+10 −3
Original line number Diff line number Diff line
@@ -256,11 +256,17 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, 
}



static struct page *f2fs_read_merkle_tree_page(struct inode *inode,

					       pgoff_t index,

					       unsigned long num_ra_pages)

					       pgoff_t index)

{

	index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;

	return generic_read_merkle_tree_page(inode, index, num_ra_pages);

	return generic_read_merkle_tree_page(inode, index);

}



static void f2fs_readahead_merkle_tree(struct inode *inode, pgoff_t index,

				       unsigned long nr_pages)

{

	index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;

	generic_readahead_merkle_tree(inode, index, nr_pages);

}



static int f2fs_write_merkle_tree_block(struct file *file, const void *buf,
@@ -278,5 +284,6 @@ const struct fsverity_operations f2fs_verityops = { 
	.end_enable_verity	= f2fs_end_enable_verity,

	.get_verity_descriptor	= f2fs_get_verity_descriptor,

	.read_merkle_tree_page	= f2fs_read_merkle_tree_page,

	.readahead_merkle_tree	= f2fs_readahead_merkle_tree,

	.write_merkle_tree_block = f2fs_write_merkle_tree_block,

};