Commit 62bcbdca authored by Qu Wenruo's avatar Qu Wenruo Committed by David Sterba
Browse files

btrfs: make btrfs_csum_one_bio() handle bs > ps without large folios 



For bs > ps cases, all folios passed into btrfs_csum_one_bio() are
ensured to be backed by large folios.  But that requirement excludes
features like direct IO and encoded writes.

To support bs > ps without large folios, enhance btrfs_csum_one_bio()
by:

- Split btrfs_calculate_block_csum() into two versions
  * btrfs_calculate_block_csum_folio()
    For call sites where a fs block is always backed by a large folio.

    This will do extra checks on the folio size, build a paddrs[] array,
    and pass it into the newer btrfs_calculate_block_csum_pages()
    helper.

    For now btrfs_check_block_csum() is still using this version.

  * btrfs_calculate_block_csum_pages()
    For call sites that may hit a fs block backed by noncontiguous pages.
    The pages are represented by paddrs[] array, which includes the
    offset inside the page.

    This function will do the proper sub-block handling.

- Make btrfs_csum_one_bio() to use btrfs_calculate_block_csum_pages()
  This means we will need to build a local paddrs[] array, and after
  filling a fs block, do the checksum calculation.

Signed-off-by: default avatarQu Wenruo <wqu@suse.com>
Signed-off-by: default avatarDavid Sterba <dsterba@suse.com>
parent fe1e5003

fs/btrfs/btrfs_inode.h

+4 −2
Original line number Diff line number Diff line
@@ -543,8 +543,10 @@ static inline void btrfs_set_inode_mapping_order(struct btrfs_inode *inode) 
#endif

}



void btrfs_calculate_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr,

				u8 *dest);

void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info,

				      const phys_addr_t paddr, u8 *dest);

void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info,

				      const phys_addr_t paddrs[], u8 *dest);

int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum,

			   const u8 * const csum_expected);

bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,

fs/btrfs/file-item.c

+12 −3
Original line number Diff line number Diff line
@@ -775,15 +775,24 @@ static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src) 
	struct bvec_iter iter = *src;

	phys_addr_t paddr;

	const u32 blocksize = fs_info->sectorsize;

	const u32 step = min(blocksize, PAGE_SIZE);

	const u32 nr_steps = blocksize / step;

	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];

	u32 offset = 0;

	int index = 0;



	shash->tfm = fs_info->csum_shash;



	btrfs_bio_for_each_block(paddr, bio, &iter, blocksize) {

		btrfs_calculate_block_csum(fs_info, paddr, sums->sums + index);

	btrfs_bio_for_each_block(paddr, bio, &iter, step) {

		paddrs[(offset / step) % nr_steps] = paddr;

		offset += step;



		if (IS_ALIGNED(offset, blocksize)) {

			btrfs_calculate_block_csum_pages(fs_info, paddrs, sums->sums + index);

			index += fs_info->csum_size;

		}

	}

}



static void csum_one_bio_work(struct work_struct *work)

{

fs/btrfs/inode.c

+51 −20
Original line number Diff line number Diff line
@@ -3343,36 +3343,67 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered) 
	return btrfs_finish_one_ordered(ordered);

}



void btrfs_calculate_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr,

				u8 *dest)

/*

 * Calculate the checksum of an fs block at physical memory address @paddr,

 * and save the result to @dest.

 *

 * The folio containing @paddr must be large enough to contain a full fs block.

 */

void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info,

				      const phys_addr_t paddr, u8 *dest)

{

	struct folio *folio = page_folio(phys_to_page(paddr));

	const u32 blocksize = fs_info->sectorsize;

	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);

	const u32 step = min(blocksize, PAGE_SIZE);

	const u32 nr_steps = blocksize / step;

	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];



	shash->tfm = fs_info->csum_shash;

	/* The full block must be inside the folio. */

	ASSERT(offset_in_folio(folio, paddr) + blocksize <= folio_size(folio));



	if (folio_test_partial_kmap(folio)) {

		size_t cur = paddr;

	for (int i = 0; i < nr_steps; i++) {

		u32 pindex = offset_in_folio(folio, paddr + i * step) >> PAGE_SHIFT;



		/*

		 * For bs <= ps cases, we will only run the loop once, so the offset

		 * inside the page will only added to paddrs[0].

		 *

		 * For bs > ps cases, the block must be page aligned, thus offset

		 * inside the page will always be 0.

		 */

		paddrs[i] = page_to_phys(folio_page(folio, pindex)) + offset_in_page(paddr);

	}

	return btrfs_calculate_block_csum_pages(fs_info, paddrs, dest);

}



/*

 * Calculate the checksum of a fs block backed by multiple noncontiguous pages

 * at @paddrs[] and save the result to @dest.

 *

 * The folio containing @paddr must be large enough to contain a full fs block.

 */

void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info,

				      const phys_addr_t paddrs[], u8 *dest)

{

	const u32 blocksize = fs_info->sectorsize;

	const u32 step = min(blocksize, PAGE_SIZE);

	const u32 nr_steps = blocksize / step;

	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);



	shash->tfm = fs_info->csum_shash;

	crypto_shash_init(shash);

		while (cur < paddr + blocksize) {

	for (int i = 0; i < nr_steps; i++) {

		const phys_addr_t paddr = paddrs[i];

		void *kaddr;

			size_t len = min(paddr + blocksize - cur,

					 PAGE_SIZE - offset_in_page(cur));



			kaddr = kmap_local_folio(folio, offset_in_folio(folio, cur));

			crypto_shash_update(shash, kaddr, len);

		ASSERT(offset_in_page(paddr) + step <= PAGE_SIZE);

		kaddr = kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr);

		crypto_shash_update(shash, kaddr, step);

		kunmap_local(kaddr);

			cur += len;

	}

	crypto_shash_final(shash, dest);

	} else {

		crypto_shash_digest(shash, phys_to_virt(paddr), blocksize, dest);

	}

}



/*

 * Verify the checksum for a single sector without any extra action that depend

 * on the type of I/O.
@@ -3382,7 +3413,7 @@ void btrfs_calculate_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr 
int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum,

			   const u8 * const csum_expected)

{

	btrfs_calculate_block_csum(fs_info, paddr, csum);

	btrfs_calculate_block_csum_folio(fs_info, paddr, csum);

	if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0))

		return -EIO;

	return 0;