Commit 28f07f2a authored by Mikulas Patocka's avatar Mikulas Patocka Committed by Mike Snitzer
Browse files

dm-verity: don't use blocking calls from tasklets 



The commit 5721d4e5 enhanced dm-verity, so that it can verify blocks
from tasklets rather than from workqueues. This reportedly improves
performance significantly.

However, dm-verity was using the flag CRYPTO_TFM_REQ_MAY_SLEEP from
tasklets which resulted in warnings about sleeping function being called
from non-sleeping context.

BUG: sleeping function called from invalid context at crypto/internal.h:206
in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 14, name: ksoftirqd/0
preempt_count: 100, expected: 0
RCU nest depth: 0, expected: 0
CPU: 0 PID: 14 Comm: ksoftirqd/0 Tainted: G        W 6.7.0-rc1 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014
Call Trace:
 <TASK>
 dump_stack_lvl+0x32/0x50
 __might_resched+0x110/0x160
 crypto_hash_walk_done+0x54/0xb0
 shash_ahash_update+0x51/0x60
 verity_hash_update.isra.0+0x4a/0x130 [dm_verity]
 verity_verify_io+0x165/0x550 [dm_verity]
 ? free_unref_page+0xdf/0x170
 ? psi_group_change+0x113/0x390
 verity_tasklet+0xd/0x70 [dm_verity]
 tasklet_action_common.isra.0+0xb3/0xc0
 __do_softirq+0xaf/0x1ec
 ? smpboot_thread_fn+0x1d/0x200
 ? sort_range+0x20/0x20
 run_ksoftirqd+0x15/0x30
 smpboot_thread_fn+0xed/0x200
 kthread+0xdc/0x110
 ? kthread_complete_and_exit+0x20/0x20
 ret_from_fork+0x28/0x40
 ? kthread_complete_and_exit+0x20/0x20
 ret_from_fork_asm+0x11/0x20
 </TASK>

This commit fixes dm-verity so that it doesn't use the flags
CRYPTO_TFM_REQ_MAY_SLEEP and CRYPTO_TFM_REQ_MAY_BACKLOG from tasklets. The
crypto API would do GFP_ATOMIC allocation instead, it could return -ENOMEM
and we catch -ENOMEM in verity_tasklet and requeue the request to the
workqueue.

Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
Cc: stable@vger.kernel.org	# v6.0+
Fixes: 5721d4e5 ("dm verity: Add optional "try_verify_in_tasklet" feature")
Signed-off-by: default avatarMike Snitzer <snitzer@kernel.org>
parent 2a695062

drivers/md/dm-verity-fec.c

+2 −2
Original line number Diff line number Diff line
@@ -185,7 +185,7 @@ static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, 
{

	if (unlikely(verity_hash(v, verity_io_hash_req(v, io),

				 data, 1 << v->data_dev_block_bits,

				 verity_io_real_digest(v, io))))

				 verity_io_real_digest(v, io), true)))

		return 0;



	return memcmp(verity_io_real_digest(v, io), want_digest,
@@ -386,7 +386,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, 
	/* Always re-validate the corrected block against the expected hash */

	r = verity_hash(v, verity_io_hash_req(v, io), fio->output,

			1 << v->data_dev_block_bits,

			verity_io_real_digest(v, io));

			verity_io_real_digest(v, io), true);

	if (unlikely(r < 0))

		return r;

drivers/md/dm-verity-target.c

+12 −11
Original line number Diff line number Diff line
@@ -135,19 +135,20 @@ static int verity_hash_update(struct dm_verity *v, struct ahash_request *req, 
 * Wrapper for crypto_ahash_init, which handles verity salting.

 */

static int verity_hash_init(struct dm_verity *v, struct ahash_request *req,

				struct crypto_wait *wait)

				struct crypto_wait *wait, bool may_sleep)

{

	int r;



	ahash_request_set_tfm(req, v->tfm);

	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |

					CRYPTO_TFM_REQ_MAY_BACKLOG,

	ahash_request_set_callback(req,

		may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0,

		crypto_req_done, (void *)wait);

	crypto_init_wait(wait);



	r = crypto_wait_req(crypto_ahash_init(req), wait);



	if (unlikely(r < 0)) {

		if (r != -ENOMEM)

			DMERR("crypto_ahash_init failed: %d", r);

		return r;

	}
@@ -179,12 +180,12 @@ static int verity_hash_final(struct dm_verity *v, struct ahash_request *req, 
}



int verity_hash(struct dm_verity *v, struct ahash_request *req,

		const u8 *data, size_t len, u8 *digest)

		const u8 *data, size_t len, u8 *digest, bool may_sleep)

{

	int r;

	struct crypto_wait wait;



	r = verity_hash_init(v, req, &wait);

	r = verity_hash_init(v, req, &wait, may_sleep);

	if (unlikely(r < 0))

		goto out;
@@ -322,7 +323,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, 


		r = verity_hash(v, verity_io_hash_req(v, io),

				data, 1 << v->hash_dev_block_bits,

				verity_io_real_digest(v, io));

				verity_io_real_digest(v, io), !io->in_tasklet);

		if (unlikely(r < 0))

			goto release_ret_r;
@@ -556,7 +557,7 @@ static int verity_verify_io(struct dm_verity_io *io) 
			continue;

		}



		r = verity_hash_init(v, req, &wait);

		r = verity_hash_init(v, req, &wait, !io->in_tasklet);

		if (unlikely(r < 0))

			return r;
@@ -652,7 +653,7 @@ static void verity_tasklet(unsigned long data) 


	io->in_tasklet = true;

	err = verity_verify_io(io);

	if (err == -EAGAIN) {

	if (err == -EAGAIN || err == -ENOMEM) {

		/* fallback to retrying with work-queue */

		INIT_WORK(&io->work, verity_work);

		queue_work(io->v->verify_wq, &io->work);
@@ -1033,7 +1034,7 @@ static int verity_alloc_zero_digest(struct dm_verity *v) 
		goto out;



	r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits,

			v->zero_digest);

			v->zero_digest, true);



out:

	kfree(req);

drivers/md/dm-verity.h

+1 −1
Original line number Diff line number Diff line
@@ -128,7 +128,7 @@ extern int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, 
					      u8 *data, size_t len));



extern int verity_hash(struct dm_verity *v, struct ahash_request *req,

		       const u8 *data, size_t len, u8 *digest);

		       const u8 *data, size_t len, u8 *digest, bool may_sleep);



extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,

				 sector_t block, u8 *digest, bool *is_zero);