Commit 5c5d0d70 authored by LongPing Wei's avatar LongPing Wei Committed by Mikulas Patocka
Browse files

dm-verity: support block number limits for different ioprio classes 



Calling verity_verify_io in bh for IO of all sizes is not suitable for
embedded devices. From our tests, it can improve the performance of 4K
synchronise random reads.
For example:
./fio --name=rand_read --ioengine=psync --rw=randread --bs=4K \
 --direct=1 --numjobs=8 --runtime=60 --time_based --group_reporting \
 --filename=/dev/block/mapper/xx-verity

But it will degrade the performance of 512K synchronise sequential reads
on our devices.
For example:
./fio --name=read --ioengine=psync --rw=read --bs=512K --direct=1 \
 --numjobs=8 --runtime=60 --time_based --group_reporting \
 --filename=/dev/block/mapper/xx-verity

A parameter array is introduced by this change. And users can modify the
default config by /sys/module/dm_verity/parameters/use_bh_bytes.

The default limits for NONE/RT/BE is set to 8192.
The default limits for IDLE is set to 0.

Call verity_verify_io directly when verity_end_io is not in hardirq.

Signed-off-by: default avatarLongPing Wei <weilongping@oppo.com>
Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
parent d43929ef

Documentation/admin-guide/device-mapper/verity.rst

+9 −2
Original line number Diff line number Diff line
@@ -151,8 +151,15 @@ root_hash_sig_key_desc <key_description> 
    already in the secondary trusted keyring.



try_verify_in_tasklet

    If verity hashes are in cache, verify data blocks in kernel tasklet instead

    of workqueue. This option can reduce IO latency.

    If verity hashes are in cache and the IO size does not exceed the limit,

    verify data blocks in bottom half instead of workqueue. This option can

    reduce IO latency. The size limits can be configured via

    /sys/module/dm_verity/parameters/use_bh_bytes. The four parameters

    correspond to limits for IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT,

    IOPRIO_CLASS_BE and IOPRIO_CLASS_IDLE in turn.

    For example:

    <none>,<rt>,<be>,<idle>

    4096,4096,4096,4096



Theory of operation

===================

drivers/md/dm-verity-target.c

+26 −3
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ 
#define DM_VERITY_ENV_VAR_NAME		"DM_VERITY_ERR_BLOCK_NR"



#define DM_VERITY_DEFAULT_PREFETCH_SIZE	262144

#define DM_VERITY_USE_BH_DEFAULT_BYTES	8192



#define DM_VERITY_MAX_CORRUPTED_ERRS	100
@@ -49,6 +50,15 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE 


module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);



static unsigned int dm_verity_use_bh_bytes[4] = {

	DM_VERITY_USE_BH_DEFAULT_BYTES,	// IOPRIO_CLASS_NONE

	DM_VERITY_USE_BH_DEFAULT_BYTES,	// IOPRIO_CLASS_RT

	DM_VERITY_USE_BH_DEFAULT_BYTES,	// IOPRIO_CLASS_BE

	0				// IOPRIO_CLASS_IDLE

};



module_param_array_named(use_bh_bytes, dm_verity_use_bh_bytes, uint, NULL, 0644);



static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);



/* Is at least one dm-verity instance using ahash_tfm instead of shash_tfm? */
@@ -669,9 +679,17 @@ static void verity_bh_work(struct work_struct *w) 
	verity_finish_io(io, errno_to_blk_status(err));

}



static inline bool verity_use_bh(unsigned int bytes, unsigned short ioprio)

{

	return ioprio <= IOPRIO_CLASS_IDLE &&

		bytes <= READ_ONCE(dm_verity_use_bh_bytes[ioprio]);

}



static void verity_end_io(struct bio *bio)

{

	struct dm_verity_io *io = bio->bi_private;

	unsigned short ioprio = IOPRIO_PRIO_CLASS(bio->bi_ioprio);

	unsigned int bytes = io->n_blocks << io->v->data_dev_block_bits;



	if (bio->bi_status &&

	    (!verity_fec_is_enabled(io->v) ||
@@ -681,9 +699,14 @@ static void verity_end_io(struct bio *bio) 
		return;

	}



	if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) {

	if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq &&

		verity_use_bh(bytes, ioprio)) {

		if (in_hardirq() || irqs_disabled()) {

			INIT_WORK(&io->bh_work, verity_bh_work);

			queue_work(system_bh_wq, &io->bh_work);

		} else {

			verity_bh_work(&io->bh_work);

		}

	} else {

		INIT_WORK(&io->work, verity_work);

		queue_work(io->v->verify_wq, &io->work);