Commit ba0f428c authored by Eric Biggers's avatar Eric Biggers Committed by Mikulas Patocka
Browse files

dm-verity: use SHA-256 library for SHA-256 

When the hash algorithm is SHA-256 and the verity version is not 0, use
the SHA-256 library instead of crypto_shash.

This is a prerequisite for making dm-verity interleave the computation
of SHA-256 hashes for increased performance.  That optimization is
available in the SHA-256 library but not in crypto_shash.

Even without interleaved hashing, switching to the library also slightly
improves performance by itself because it avoids the overhead of
crypto_shash, including indirect calls and other API overhead.
(Benchmark on x86_64, AMD Zen 5: hashing 4K blocks gets 2.1% faster.)

SHA-256 is by far the most common hash algorithm used with dm-verity.
It makes sense to optimize for the common case and fall back to the
generic crypto layer for uncommon cases, as suggested by Linus:
https://lore.kernel.org/r/CAHk-=wgp-fOSsZsYrbyzqCAfEvrt5jQs1jL-97Wc4seMNTUyng@mail.gmail.com



Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
parent 3ee6c4bc

drivers/md/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -547,6 +547,7 @@ config DM_VERITY 
	depends on BLK_DEV_DM

	select CRYPTO

	select CRYPTO_HASH

	select CRYPTO_LIB_SHA256

	select DM_BUFIO

	help

	  This device-mapper target creates a read-only device that

drivers/md/dm-verity-target.c

+48 −13
Original line number Diff line number Diff line
@@ -117,11 +117,25 @@ static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, 
int verity_hash(struct dm_verity *v, struct dm_verity_io *io,

		const u8 *data, size_t len, u8 *digest)

{

	struct shash_desc *desc = &io->hash_desc;

	struct shash_desc *desc;

	int r;



	if (likely(v->use_sha256_lib)) {

		struct sha256_ctx *ctx = &io->hash_ctx.sha256;



		/*

		 * Fast path using SHA-256 library.  This is enabled only for

		 * verity version 1, where the salt is at the beginning.

		 */

		*ctx = *v->initial_hashstate.sha256;

		sha256_update(ctx, data, len);

		sha256_final(ctx, digest);

		return 0;

	}



	desc = &io->hash_ctx.shash;

	desc->tfm = v->shash_tfm;

	if (unlikely(v->initial_hashstate == NULL)) {

	if (unlikely(v->initial_hashstate.shash == NULL)) {

		/* Version 0: salt at end */

		r = crypto_shash_init(desc) ?:

		    crypto_shash_update(desc, data, len) ?:
@@ -129,7 +143,7 @@ int verity_hash(struct dm_verity *v, struct dm_verity_io *io, 
		    crypto_shash_final(desc, digest);

	} else {

		/* Version 1: salt at beginning */

		r = crypto_shash_import(desc, v->initial_hashstate) ?:

		r = crypto_shash_import(desc, v->initial_hashstate.shash) ?:

		    crypto_shash_finup(desc, data, len, digest);

	}

	if (unlikely(r))
@@ -1004,7 +1018,7 @@ static void verity_dtr(struct dm_target *ti) 


	kvfree(v->validated_blocks);

	kfree(v->salt);

	kfree(v->initial_hashstate);

	kfree(v->initial_hashstate.shash);

	kfree(v->root_digest);

	kfree(v->zero_digest);

	verity_free_sig(v);
@@ -1069,8 +1083,7 @@ static int verity_alloc_zero_digest(struct dm_verity *v) 
	if (!v->zero_digest)

		return r;



	io = kmalloc(sizeof(*io) + crypto_shash_descsize(v->shash_tfm),

		     GFP_KERNEL);

	io = kmalloc(v->ti->per_io_data_size, GFP_KERNEL);



	if (!io)

		return r; /* verity_dtr will free zero_digest */
@@ -1256,6 +1269,20 @@ static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name) 
		ti->error = "Digest size too big";

		return -EINVAL;

	}

	if (likely(v->version && strcmp(alg_name, "sha256") == 0)) {

		/*

		 * Fast path: use the library API for reduced overhead and

		 * interleaved hashing support.

		 */

		v->use_sha256_lib = true;

		ti->per_io_data_size =

			offsetofend(struct dm_verity_io, hash_ctx.sha256);

	} else {

		/* Fallback case: use the generic crypto API. */

		ti->per_io_data_size =

			offsetofend(struct dm_verity_io, hash_ctx.shash) +

			crypto_shash_descsize(shash);

	}

	return 0;

}
@@ -1276,7 +1303,18 @@ static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg) 
			return -EINVAL;

		}

	}

	if (v->version) { /* Version 1: salt at beginning */

	if (likely(v->use_sha256_lib)) {

		/* Implies version 1: salt at beginning */

		v->initial_hashstate.sha256 =

			kmalloc(sizeof(struct sha256_ctx), GFP_KERNEL);

		if (!v->initial_hashstate.sha256) {

			ti->error = "Cannot allocate initial hash state";

			return -ENOMEM;

		}

		sha256_init(v->initial_hashstate.sha256);

		sha256_update(v->initial_hashstate.sha256,

			      v->salt, v->salt_size);

	} else if (v->version) { /* Version 1: salt at beginning */

		SHASH_DESC_ON_STACK(desc, v->shash_tfm);

		int r;
@@ -1284,16 +1322,16 @@ static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg) 
		 * Compute the pre-salted hash state that can be passed to

		 * crypto_shash_import() for each block later.

		 */

		v->initial_hashstate = kmalloc(

		v->initial_hashstate.shash = kmalloc(

			crypto_shash_statesize(v->shash_tfm), GFP_KERNEL);

		if (!v->initial_hashstate) {

		if (!v->initial_hashstate.shash) {

			ti->error = "Cannot allocate initial hash state";

			return -ENOMEM;

		}

		desc->tfm = v->shash_tfm;

		r = crypto_shash_init(desc) ?:

		    crypto_shash_update(desc, v->salt, v->salt_size) ?:

		    crypto_shash_export(desc, v->initial_hashstate);

		    crypto_shash_export(desc, v->initial_hashstate.shash);

		if (r) {

			ti->error = "Cannot set up initial hash state";

			return r;
@@ -1555,9 +1593,6 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) 
		goto bad;

	}



	ti->per_io_data_size = sizeof(struct dm_verity_io) +

			       crypto_shash_descsize(v->shash_tfm);



	r = verity_fec_ctr(v);

	if (r)

		goto bad;

drivers/md/dm-verity.h

+15 −5
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
#include <linux/device-mapper.h>

#include <linux/interrupt.h>

#include <crypto/hash.h>

#include <crypto/sha2.h>



#define DM_VERITY_MAX_LEVELS		63
@@ -42,7 +43,10 @@ struct dm_verity { 
	struct crypto_shash *shash_tfm;

	u8 *root_digest;	/* digest of the root block */

	u8 *salt;		/* salt: its size is salt_size */

	u8 *initial_hashstate;	/* salted initial state, if version >= 1 */

	union {

		struct sha256_ctx *sha256;	/* for use_sha256_lib=1 */

		u8 *shash;			/* for use_sha256_lib=0 */

	} initial_hashstate; /* salted initial state, if version >= 1 */

	u8 *zero_digest;	/* digest for a zero block */

#ifdef CONFIG_SECURITY

	u8 *root_digest_sig;	/* signature of the root digest */
@@ -59,6 +63,7 @@ struct dm_verity { 
	unsigned char version;

	bool hash_failed:1;	/* set if hash of any block failed */

	bool use_bh_wq:1;	/* try to verify in BH wq before normal work-queue */

	bool use_sha256_lib:1;	/* use SHA-256 library instead of generic crypto API */

	unsigned int digest_size;	/* digest size for the current hash algorithm */

	enum verity_mode mode;	/* mode for handling verification errors */

	enum verity_mode error_mode;/* mode for handling I/O errors */
@@ -98,11 +103,16 @@ struct dm_verity_io { 
	u8 want_digest[HASH_MAX_DIGESTSIZE];



	/*

	 * Temporary space for hashing.  This is variable-length and must be at

	 * the end of the struct.  struct shash_desc is just the fixed part;

	 * it's followed by a context of size crypto_shash_descsize(shash_tfm).

	 * Temporary space for hashing.  Either sha256 or shash is used,

	 * depending on the value of use_sha256_lib.  If shash is used,

	 * then this field is variable-length, with total size

	 * sizeof(struct shash_desc) + crypto_shash_descsize(shash_tfm).

	 * For this reason, this field must be the end of the struct.

	 */

	struct shash_desc hash_desc;

	union {

		struct sha256_ctx sha256;

		struct shash_desc shash;

	} hash_ctx;

};



static inline u8 *verity_io_real_digest(struct dm_verity *v,