Merge tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux (1896ce8e) · Commits · git / linux-nf

fs/verity/enable.c

+5 −7

Original line number	Diff line number	Diff line
		@@ -19,8 +19,7 @@ struct block_buffer {
		};

		/* Hash a block, writing the result to the next level's pending block buffer. */
		static int hash_one_block(struct inode *inode,
		const struct merkle_tree_params *params,
		static int hash_one_block(const struct merkle_tree_params *params,
		struct block_buffer *cur)
		{
		struct block_buffer *next = cur + 1;
		@@ -36,8 +35,7 @@ static int hash_one_block(struct inode *inode,
		/* Zero-pad the block if it's shorter than the block size. */
		memset(&cur->data[cur->filled], 0, params->block_size - cur->filled);

		fsverity_hash_block(params, inode, cur->data,
		&next->data[next->filled]);
		fsverity_hash_block(params, cur->data, &next->data[next->filled]);
		next->filled += params->digest_size;
		cur->filled = 0;
		return 0;
		@@ -123,7 +121,7 @@ static int build_merkle_tree(struct file *filp,
		fsverity_err(inode, "Short read of file data");
		goto out;
		}
		err = hash_one_block(inode, params, &buffers[-1]);
		err = hash_one_block(params, &buffers[-1]);
		if (err)
		goto out;
		for (level = 0; level < num_levels; level++) {
		@@ -134,7 +132,7 @@ static int build_merkle_tree(struct file *filp,
		}
		/* Next block at @level is full */

		err = hash_one_block(inode, params, &buffers[level]);
		err = hash_one_block(params, &buffers[level]);
		if (err)
		goto out;
		err = write_merkle_tree_block(inode,
		@@ -154,7 +152,7 @@ static int build_merkle_tree(struct file *filp,
		/* Finish all nonempty pending tree blocks. */
		for (level = 0; level < num_levels; level++) {
		if (buffers[level].filled != 0) {
		err = hash_one_block(inode, params, &buffers[level]);
		err = hash_one_block(params, &buffers[level]);
		if (err)
		goto out;
		err = write_merkle_tree_block(inode,

fs/verity/fsverity_private.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -90,7 +90,7 @@ union fsverity_hash_ctx *
		fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg,
		const u8 *salt, size_t salt_size);
		void fsverity_hash_block(const struct merkle_tree_params *params,
		const struct inode inode, const void data, u8 *out);
		const void data, u8 out);
		void fsverity_hash_buffer(const struct fsverity_hash_alg *alg,
		const void data, size_t size, u8 out);
		void __init fsverity_check_hash_algs(void);

fs/verity/hash_algs.c

+1 −2

Original line number	Diff line number	Diff line
		@@ -94,7 +94,6 @@ fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg,
		/**
		* fsverity_hash_block() - hash a single data or hash block
		* @params: the Merkle tree's parameters
		* @inode: inode for which the hashing is being done
		* @data: virtual address of a buffer containing the block to hash
		* @out: output digest, size 'params->digest_size' bytes
		*
		@@ -102,7 +101,7 @@ fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg,
		* in the Merkle tree parameters.
		*/
		void fsverity_hash_block(const struct merkle_tree_params *params,
		const struct inode inode, const void data, u8 *out)
		const void data, u8 out)
		{
		union fsverity_hash_ctx ctx;

fs/verity/verify.c

+140 −35

Original line number	Diff line number	Diff line
		@@ -10,6 +10,31 @@
		#include <linux/bio.h>
		#include <linux/export.h>

		#define FS_VERITY_MAX_PENDING_BLOCKS 2

		struct fsverity_pending_block {
		const void *data;
		u64 pos;
		u8 real_hash[FS_VERITY_MAX_DIGEST_SIZE];
		};

		struct fsverity_verification_context {
		struct inode *inode;
		struct fsverity_info *vi;
		unsigned long max_ra_pages;

		/*
		* This is the queue of data blocks that are pending verification. When
		* the crypto layer supports interleaved hashing, we allow multiple
		* blocks to be queued up in order to utilize it. This can improve
		* performance significantly vs. sequential hashing of each block.
		*/
		int num_pending;
		int max_pending;
		struct fsverity_pending_block
		pending_blocks[FS_VERITY_MAX_PENDING_BLOCKS];
		};

		static struct workqueue_struct *fsverity_read_workqueue;

		/*
		@@ -79,7 +104,7 @@ static bool is_hash_block_verified(struct fsverity_info vi, struct page hpage,
		}

		/*
		* Verify a single data block against the file's Merkle tree.
		* Verify the hash of a single data block against the file's Merkle tree.
		*
		* In principle, we need to verify the entire path to the root node. However,
		* for efficiency the filesystem may cache the hash blocks. Therefore we need
		@@ -88,10 +113,11 @@ static bool is_hash_block_verified(struct fsverity_info vi, struct page hpage,
		*
		* Return: %true if the data block is valid, else %false.
		*/
		static bool
		verify_data_block(struct inode inode, struct fsverity_info vi,
		const void *data, u64 data_pos, unsigned long max_ra_pages)
		static bool verify_data_block(struct inode inode, struct fsverity_info vi,
		const struct fsverity_pending_block *dblock,
		unsigned long max_ra_pages)
		{
		const u64 data_pos = dblock->pos;
		const struct merkle_tree_params *params = &vi->tree_params;
		const unsigned int hsize = params->digest_size;
		int level;
		@@ -115,8 +141,12 @@ verify_data_block(struct inode inode, struct fsverity_info vi,
		*/
		u64 hidx = data_pos >> params->log_blocksize;

		/* Up to 1 + FS_VERITY_MAX_LEVELS pages may be mapped at once */
		BUILD_BUG_ON(1 + FS_VERITY_MAX_LEVELS > KM_MAX_IDX);
		/*
		* Up to FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS pages may
		* be mapped at once.
		*/
		static_assert(FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS <=
		KM_MAX_IDX);

		if (unlikely(data_pos >= inode->i_size)) {
		/*
		@@ -127,7 +157,7 @@ verify_data_block(struct inode inode, struct fsverity_info vi,
		* any part past EOF should be all zeroes. Therefore, we need
		* to verify that any data blocks fully past EOF are all zeroes.
		*/
		if (memchr_inv(data, 0, params->block_size)) {
		if (memchr_inv(dblock->data, 0, params->block_size)) {
		fsverity_err(inode,
		"FILE CORRUPTED! Data past EOF is not zeroed");
		return false;
		@@ -202,7 +232,7 @@ verify_data_block(struct inode inode, struct fsverity_info vi,
		unsigned long hblock_idx = hblocks[level - 1].index;
		unsigned int hoffset = hblocks[level - 1].hoffset;

		fsverity_hash_block(params, inode, haddr, real_hash);
		fsverity_hash_block(params, haddr, real_hash);
		if (memcmp(want_hash, real_hash, hsize) != 0)
		goto corrupted;
		/*
		@@ -220,18 +250,18 @@ verify_data_block(struct inode inode, struct fsverity_info vi,
		put_page(hpage);
		}

		/* Finally, verify the data block. */
		fsverity_hash_block(params, inode, data, real_hash);
		if (memcmp(want_hash, real_hash, hsize) != 0)
		/* Finally, verify the hash of the data block. */
		if (memcmp(want_hash, dblock->real_hash, hsize) != 0)
		goto corrupted;
		return true;

		corrupted:
		fsverity_err(inode,
		fsverity_err(
		inode,
		"FILE CORRUPTED! pos=%llu, level=%d, want_hash=%s:%phN, real_hash=%s:%phN",
		data_pos, level - 1,
		params->hash_alg->name, hsize, want_hash,
		params->hash_alg->name, hsize, real_hash);
		data_pos, level - 1, params->hash_alg->name, hsize, want_hash,
		params->hash_alg->name, hsize,
		level == 0 ? dblock->real_hash : real_hash);
		error:
		for (; level > 0; level--) {
		kunmap_local(hblocks[level - 1].addr);
		@@ -240,13 +270,73 @@ verify_data_block(struct inode inode, struct fsverity_info vi,
		return false;
		}

		static bool
		verify_data_blocks(struct folio *data_folio, size_t len, size_t offset,
		static void
		fsverity_init_verification_context(struct fsverity_verification_context *ctx,
		struct inode *inode,
		unsigned long max_ra_pages)
		{
		struct inode *inode = data_folio->mapping->host;
		struct fsverity_info vi = fsverity_info_addr(inode);
		const unsigned int block_size = vi->tree_params.block_size;

		ctx->inode = inode;
		ctx->vi = vi;
		ctx->max_ra_pages = max_ra_pages;
		ctx->num_pending = 0;
		if (vi->tree_params.hash_alg->algo_id == HASH_ALGO_SHA256 &&
		sha256_finup_2x_is_optimized())
		ctx->max_pending = 2;
		else
		ctx->max_pending = 1;
		}

		static void
		fsverity_clear_pending_blocks(struct fsverity_verification_context *ctx)
		{
		int i;

		for (i = ctx->num_pending - 1; i >= 0; i--) {
		kunmap_local(ctx->pending_blocks[i].data);
		ctx->pending_blocks[i].data = NULL;
		}
		ctx->num_pending = 0;
		}

		static bool
		fsverity_verify_pending_blocks(struct fsverity_verification_context *ctx)
		{
		struct fsverity_info *vi = ctx->vi;
		const struct merkle_tree_params *params = &vi->tree_params;
		int i;

		if (ctx->num_pending == 2) {
		/* num_pending == 2 implies that the algorithm is SHA-256 */
		sha256_finup_2x(params->hashstate ? &params->hashstate->sha256 :
		NULL,
		ctx->pending_blocks[0].data,
		ctx->pending_blocks[1].data, params->block_size,
		ctx->pending_blocks[0].real_hash,
		ctx->pending_blocks[1].real_hash);
		} else {
		for (i = 0; i < ctx->num_pending; i++)
		fsverity_hash_block(params, ctx->pending_blocks[i].data,
		ctx->pending_blocks[i].real_hash);
		}

		for (i = 0; i < ctx->num_pending; i++) {
		if (!verify_data_block(ctx->inode, vi, &ctx->pending_blocks[i],
		ctx->max_ra_pages))
		return false;
		}
		fsverity_clear_pending_blocks(ctx);
		return true;
		}

		static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx,
		struct folio *data_folio, size_t len,
		size_t offset)
		{
		struct fsverity_info *vi = ctx->vi;
		const struct merkle_tree_params *params = &vi->tree_params;
		const unsigned int block_size = params->block_size;
		u64 pos = (u64)data_folio->index << PAGE_SHIFT;

		if (WARN_ON_ONCE(len <= 0 \|\| !IS_ALIGNED(len \| offset, block_size)))
		@@ -255,14 +345,11 @@ verify_data_blocks(struct folio *data_folio, size_t len, size_t offset,
		folio_test_uptodate(data_folio)))
		return false;
		do {
		void *data;
		bool valid;

		data = kmap_local_folio(data_folio, offset);
		valid = verify_data_block(inode, vi, data, pos + offset,
		max_ra_pages);
		kunmap_local(data);
		if (!valid)
		ctx->pending_blocks[ctx->num_pending].data =
		kmap_local_folio(data_folio, offset);
		ctx->pending_blocks[ctx->num_pending].pos = pos + offset;
		if (++ctx->num_pending == ctx->max_pending &&
		!fsverity_verify_pending_blocks(ctx))
		return false;
		offset += block_size;
		len -= block_size;
		@@ -284,7 +371,15 @@ verify_data_blocks(struct folio *data_folio, size_t len, size_t offset,
		*/
		bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset)
		{
		return verify_data_blocks(folio, len, offset, 0);
		struct fsverity_verification_context ctx;

		fsverity_init_verification_context(&ctx, folio->mapping->host, 0);

		if (fsverity_add_data_blocks(&ctx, folio, len, offset) &&
		fsverity_verify_pending_blocks(&ctx))
		return true;
		fsverity_clear_pending_blocks(&ctx);
		return false;
		}
		EXPORT_SYMBOL_GPL(fsverity_verify_blocks);

		@@ -305,6 +400,8 @@ EXPORT_SYMBOL_GPL(fsverity_verify_blocks);
		*/
		void fsverity_verify_bio(struct bio *bio)
		{
		struct inode *inode = bio_first_folio_all(bio)->mapping->host;
		struct fsverity_verification_context ctx;
		struct folio_iter fi;
		unsigned long max_ra_pages = 0;

		@@ -321,13 +418,21 @@ void fsverity_verify_bio(struct bio *bio)
		max_ra_pages = bio->bi_iter.bi_size >> (PAGE_SHIFT + 2);
		}

		fsverity_init_verification_context(&ctx, inode, max_ra_pages);

		bio_for_each_folio_all(fi, bio) {
		if (!verify_data_blocks(fi.folio, fi.length, fi.offset,
		max_ra_pages)) {
		bio->bi_status = BLK_STS_IOERR;
		break;
		}
		if (!fsverity_add_data_blocks(&ctx, fi.folio, fi.length,
		fi.offset))
		goto ioerr;
		}

		if (!fsverity_verify_pending_blocks(&ctx))
		goto ioerr;
		return;

		ioerr:
		fsverity_clear_pending_blocks(&ctx);
		bio->bi_status = BLK_STS_IOERR;
		}
		EXPORT_SYMBOL_GPL(fsverity_verify_bio);
		#endif /* CONFIG_BLOCK */

include/crypto/sha2.h

+28 −0

Original line number	Diff line number	Diff line
		@@ -375,6 +375,34 @@ void sha256_final(struct sha256_ctx *ctx, u8 out[SHA256_DIGEST_SIZE]);
		*/
		void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]);

		/**
		* sha256_finup_2x() - Compute two SHA-256 digests from a common initial
		* context. On some CPUs, this is faster than sequentially
		* computing each digest.
		* @ctx: an optional initial context, which may have already processed data. If
		* NULL, a default initial context is used (equivalent to sha256_init()).
		* @data1: data for the first message
		* @data2: data for the second message
		* @len: the length of each of @data1 and @data2, in bytes
		* @out1: (output) the first SHA-256 message digest
		* @out2: (output) the second SHA-256 message digest
		*
		* Context: Any context.
		*/
		void sha256_finup_2x(const struct sha256_ctx ctx, const u8 data1,
		const u8 *data2, size_t len, u8 out1[SHA256_DIGEST_SIZE],
		u8 out2[SHA256_DIGEST_SIZE]);

		/**
		* sha256_finup_2x_is_optimized() - Check if sha256_finup_2x() is using a real
		* interleaved implementation, as opposed to a
		* sequential fallback
		* @return: true if optimized
		*
		* Context: Any context.
		*/
		bool sha256_finup_2x_is_optimized(void);

		/**
		* struct hmac_sha256_key - Prepared key for HMAC-SHA256
		* @key: private