Commit e5167911 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: move the block layer auto-integrity code into a new file 



The code that automatically creates a integrity payload and generates and
verifies the checksums for bios that don't have submitter-provided
integrity payload currently sits right in the middle of the block
integrity metadata infrastructure.  Split it into a separate file to
make the different layers clear.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarAnuj Gupta <anuj20.g@samsung.com>
Reviewed-by: default avatarKanchan Joshi <joshi.k@samsung.com>
Reviewed-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
Reviewed-by: default avatarHannes Reinecke <hare@suse.de>
Reviewed-by: default avatarJohannes Thumshirn <johannes.thumshirn@wdc.com>
Link: https://lore.kernel.org/r/20250225154449.422989-3-hch@lst.de


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5fd0268a

block/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -26,7 +26,8 @@ obj-$(CONFIG_MQ_IOSCHED_KYBER) += kyber-iosched.o 
bfq-y				:= bfq-iosched.o bfq-wf2q.o bfq-cgroup.o

obj-$(CONFIG_IOSCHED_BFQ)	+= bfq.o



obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o

obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o \

				   bio-integrity-auto.o

obj-$(CONFIG_BLK_DEV_ZONED)	+= blk-zoned.o

obj-$(CONFIG_BLK_WBT)		+= blk-wbt.o

obj-$(CONFIG_BLK_DEBUG_FS)	+= blk-mq-debugfs.o

block/bio-integrity-auto.c

0 → 100644
+162 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/*

 * Copyright (C) 2007, 2008, 2009 Oracle Corporation

 * Written by: Martin K. Petersen <martin.petersen@oracle.com>

 *

 * Automatically generate and verify integrity data on PI capable devices if the

 * bio submitter didn't provide PI itself.  This ensures that kernel verifies

 * data integrity even if the file system (or other user of the block device) is

 * not aware of PI.

 */

#include <linux/blk-integrity.h>

#include <linux/workqueue.h>

#include "blk.h"



static struct workqueue_struct *kintegrityd_wq;



static void bio_integrity_verify_fn(struct work_struct *work)

{

	struct bio_integrity_payload *bip =

		container_of(work, struct bio_integrity_payload, bip_work);

	struct bio *bio = bip->bip_bio;



	blk_integrity_verify(bio);



	kfree(bvec_virt(bip->bip_vec));

	bio_integrity_free(bio);

	bio_endio(bio);

}



/**

 * __bio_integrity_endio - Integrity I/O completion function

 * @bio:	Protected bio

 *

 * Normally I/O completion is done in interrupt context.  However, verifying I/O

 * integrity is a time-consuming task which must be run in process context.

 *

 * This function postpones completion accordingly.

 */

bool __bio_integrity_endio(struct bio *bio)

{

	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);

	struct bio_integrity_payload *bip = bio_integrity(bio);



	if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && bi->csum_type) {

		INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);

		queue_work(kintegrityd_wq, &bip->bip_work);

		return false;

	}



	kfree(bvec_virt(bip->bip_vec));

	bio_integrity_free(bio);

	return true;

}



/**

 * bio_integrity_prep - Prepare bio for integrity I/O

 * @bio:	bio to prepare

 *

 * Checks if the bio already has an integrity payload attached.  If it does, the

 * payload has been generated by another kernel subsystem, and we just pass it

 * through.

 * Otherwise allocates integrity payload and for writes the integrity metadata

 * will be generated.  For reads, the completion handler will verify the

 * metadata.

 */

bool bio_integrity_prep(struct bio *bio)

{

	struct bio_integrity_payload *bip;

	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);

	gfp_t gfp = GFP_NOIO;

	unsigned int len;

	void *buf;



	if (!bi)

		return true;



	if (!bio_sectors(bio))

		return true;



	/* Already protected? */

	if (bio_integrity(bio))

		return true;



	switch (bio_op(bio)) {

	case REQ_OP_READ:

		if (bi->flags & BLK_INTEGRITY_NOVERIFY)

			return true;

		break;

	case REQ_OP_WRITE:

		if (bi->flags & BLK_INTEGRITY_NOGENERATE)

			return true;



		/*

		 * Zero the memory allocated to not leak uninitialized kernel

		 * memory to disk for non-integrity metadata where nothing else

		 * initializes the memory.

		 */

		if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE)

			gfp |= __GFP_ZERO;

		break;

	default:

		return true;

	}



	/* Allocate kernel buffer for protection data */

	len = bio_integrity_bytes(bi, bio_sectors(bio));

	buf = kmalloc(len, gfp);

	if (!buf)

		goto err_end_io;



	bip = bio_integrity_alloc(bio, GFP_NOIO, 1);

	if (IS_ERR(bip)) {

		kfree(buf);

		goto err_end_io;

	}



	bip->bip_flags |= BIP_BLOCK_INTEGRITY;

	bip_set_seed(bip, bio->bi_iter.bi_sector);



	if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)

		bip->bip_flags |= BIP_IP_CHECKSUM;

	if (bi->csum_type)

		bip->bip_flags |= BIP_CHECK_GUARD;

	if (bi->flags & BLK_INTEGRITY_REF_TAG)

		bip->bip_flags |= BIP_CHECK_REFTAG;



	if (bio_integrity_add_page(bio, virt_to_page(buf), len,

			offset_in_page(buf)) < len)

		goto err_end_io;



	/* Auto-generate integrity metadata if this is a write */

	if (bio_data_dir(bio) == WRITE)

		blk_integrity_generate(bio);

	else

		bip->bio_iter = bio->bi_iter;

	return true;



err_end_io:

	bio->bi_status = BLK_STS_RESOURCE;

	bio_endio(bio);

	return false;

}

EXPORT_SYMBOL(bio_integrity_prep);



void blk_flush_integrity(void)

{

	flush_workqueue(kintegrityd_wq);

}



static int __init blk_integrity_auto_init(void)

{

	/*

	 * kintegrityd won't block much but may burn a lot of CPU cycles.

	 * Make it highpri CPU intensive wq with max concurrency of 1.

	 */

	kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |

					 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);

	if (!kintegrityd_wq)

		panic("Failed to create kintegrityd\n");

	return 0;

}

subsys_initcall(blk_integrity_auto_init);

block/bio-integrity.c

+0 −159
Original line number Diff line number Diff line
@@ -10,17 +10,10 @@ 
#include <linux/mempool.h>

#include <linux/export.h>

#include <linux/bio.h>

#include <linux/workqueue.h>

#include <linux/slab.h>

#include "blk.h"



static struct kmem_cache *bip_slab;

static struct workqueue_struct *kintegrityd_wq;



void blk_flush_integrity(void)

{

	flush_workqueue(kintegrityd_wq);

}



/**

 * bio_integrity_free - Free bio integrity payload
@@ -413,149 +406,6 @@ int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta) 
	return ret;

}



/**

 * bio_integrity_prep - Prepare bio for integrity I/O

 * @bio:	bio to prepare

 *

 * Description:  Checks if the bio already has an integrity payload attached.

 * If it does, the payload has been generated by another kernel subsystem,

 * and we just pass it through. Otherwise allocates integrity payload.

 * The bio must have data direction, target device and start sector set priot

 * to calling.  In the WRITE case, integrity metadata will be generated using

 * the block device's integrity function.  In the READ case, the buffer

 * will be prepared for DMA and a suitable end_io handler set up.

 */

bool bio_integrity_prep(struct bio *bio)

{

	struct bio_integrity_payload *bip;

	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);

	unsigned int len;

	void *buf;

	gfp_t gfp = GFP_NOIO;



	if (!bi)

		return true;



	if (!bio_sectors(bio))

		return true;



	/* Already protected? */

	if (bio_integrity(bio))

		return true;



	switch (bio_op(bio)) {

	case REQ_OP_READ:

		if (bi->flags & BLK_INTEGRITY_NOVERIFY)

			return true;

		break;

	case REQ_OP_WRITE:

		if (bi->flags & BLK_INTEGRITY_NOGENERATE)

			return true;



		/*

		 * Zero the memory allocated to not leak uninitialized kernel

		 * memory to disk for non-integrity metadata where nothing else

		 * initializes the memory.

		 */

		if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE)

			gfp |= __GFP_ZERO;

		break;

	default:

		return true;

	}



	/* Allocate kernel buffer for protection data */

	len = bio_integrity_bytes(bi, bio_sectors(bio));

	buf = kmalloc(len, gfp);

	if (unlikely(buf == NULL)) {

		goto err_end_io;

	}



	bip = bio_integrity_alloc(bio, GFP_NOIO, 1);

	if (IS_ERR(bip)) {

		kfree(buf);

		goto err_end_io;

	}



	bip->bip_flags |= BIP_BLOCK_INTEGRITY;

	bip_set_seed(bip, bio->bi_iter.bi_sector);



	if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)

		bip->bip_flags |= BIP_IP_CHECKSUM;



	/* describe what tags to check in payload */

	if (bi->csum_type)

		bip->bip_flags |= BIP_CHECK_GUARD;

	if (bi->flags & BLK_INTEGRITY_REF_TAG)

		bip->bip_flags |= BIP_CHECK_REFTAG;

	if (bio_integrity_add_page(bio, virt_to_page(buf), len,

			offset_in_page(buf)) < len) {

		printk(KERN_ERR "could not attach integrity payload\n");

		goto err_end_io;

	}



	/* Auto-generate integrity metadata if this is a write */

	if (bio_data_dir(bio) == WRITE)

		blk_integrity_generate(bio);

	else

		bip->bio_iter = bio->bi_iter;

	return true;



err_end_io:

	bio->bi_status = BLK_STS_RESOURCE;

	bio_endio(bio);

	return false;

}

EXPORT_SYMBOL(bio_integrity_prep);



/**

 * bio_integrity_verify_fn - Integrity I/O completion worker

 * @work:	Work struct stored in bio to be verified

 *

 * Description: This workqueue function is called to complete a READ

 * request.  The function verifies the transferred integrity metadata

 * and then calls the original bio end_io function.

 */

static void bio_integrity_verify_fn(struct work_struct *work)

{

	struct bio_integrity_payload *bip =

		container_of(work, struct bio_integrity_payload, bip_work);

	struct bio *bio = bip->bip_bio;



	blk_integrity_verify(bio);



	kfree(bvec_virt(bip->bip_vec));

	bio_integrity_free(bio);

	bio_endio(bio);

}



/**

 * __bio_integrity_endio - Integrity I/O completion function

 * @bio:	Protected bio

 *

 * Description: Completion for integrity I/O

 *

 * Normally I/O completion is done in interrupt context.  However,

 * verifying I/O integrity is a time-consuming task which must be run

 * in process context.	This function postpones completion

 * accordingly.

 */

bool __bio_integrity_endio(struct bio *bio)

{

	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);

	struct bio_integrity_payload *bip = bio_integrity(bio);



	if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && bi->csum_type) {

		INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);

		queue_work(kintegrityd_wq, &bip->bip_work);

		return false;

	}



	kfree(bvec_virt(bip->bip_vec));

	bio_integrity_free(bio);

	return true;

}



/**

 * bio_integrity_advance - Advance integrity vector

 * @bio:	bio whose integrity vector to update
@@ -644,15 +494,6 @@ void bioset_integrity_free(struct bio_set *bs) 


void __init bio_integrity_init(void)

{

	/*

	 * kintegrityd won't block much but may burn a lot of CPU cycles.

	 * Make it highpri CPU intensive wq with max concurrency of 1.

	 */

	kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |

					 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);

	if (!kintegrityd_wq)

		panic("Failed to create kintegrityd\n");



	bip_slab = kmem_cache_create("bio_integrity_payload",

				     sizeof(struct bio_integrity_payload) +

				     sizeof(struct bio_vec) * BIO_INLINE_VECS,