Commit 2631c946 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

iomap: split out the per-bio logic from iomap_dio_bio_iter



Factor out a separate helper that builds and submits a single bio.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarDamien Le Moal <dlemoal@kernel.org>
Reviewed-by: default avatarDarrick J. Wong <djwong@kernel.org>
Tested-by: default avatarAnuj Gupta <anuj20.g@samsung.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 6e7a6c80
Loading
Loading
Loading
Loading
+59 −52
Original line number Diff line number Diff line
@@ -302,6 +302,56 @@ static int iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,
	return 0;
}

static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
		struct iomap_dio *dio, loff_t pos, unsigned int alignment,
		blk_opf_t op)
{
	struct bio *bio;
	ssize_t ret;

	bio = iomap_dio_alloc_bio(iter, dio,
			bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS),
			op);
	fscrypt_set_bio_crypt_ctx(bio, iter->inode,
			pos >> iter->inode->i_blkbits, GFP_KERNEL);
	bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos);
	bio->bi_write_hint = iter->inode->i_write_hint;
	bio->bi_ioprio = dio->iocb->ki_ioprio;
	bio->bi_private = dio;
	bio->bi_end_io = iomap_dio_bio_end_io;

	ret = bio_iov_iter_get_pages(bio, dio->submit.iter, alignment - 1);
	if (unlikely(ret))
		goto out_put_bio;
	ret = bio->bi_iter.bi_size;

	/*
	 * An atomic write bio must cover the complete length.  If it doesn't,
	 * error out.
	 */
	if ((op & REQ_ATOMIC) && WARN_ON_ONCE(ret != iomap_length(iter))) {
		ret = -EINVAL;
		goto out_put_bio;
	}

	if (dio->flags & IOMAP_DIO_WRITE)
		task_io_account_write(ret);
	else if (dio->flags & IOMAP_DIO_DIRTY)
		bio_set_pages_dirty(bio);

	/*
	 * We can only poll for single bio I/Os.
	 */
	if (iov_iter_count(dio->submit.iter))
		dio->iocb->ki_flags &= ~IOCB_HIPRI;
	iomap_dio_submit_bio(iter, dio, bio, pos);
	return ret;

out_put_bio:
	bio_put(bio);
	return ret;
}

static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
{
	const struct iomap *iomap = &iter->iomap;
@@ -310,12 +360,11 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
	const loff_t length = iomap_length(iter);
	loff_t pos = iter->pos;
	blk_opf_t bio_opf = REQ_SYNC | REQ_IDLE;
	struct bio *bio;
	bool need_zeroout = false;
	int ret = 0;
	u64 copied = 0;
	size_t orig_count;
	unsigned int alignment;
	ssize_t ret = 0;

	/*
	 * File systems that write out of place and always allocate new blocks
@@ -441,68 +490,27 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
	}

	do {
		size_t n;

		/*
		 * If completions already occurred and reported errors, give up now and
		 * don't bother submitting more bios.
		 */
		if (unlikely(data_race(dio->error))) {
			ret = 0;
		if (unlikely(data_race(dio->error)))
			goto out;
		}

		bio = iomap_dio_alloc_bio(iter, dio,
				bio_iov_vecs_to_alloc(dio->submit.iter,
						BIO_MAX_VECS), bio_opf);
		fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
					  GFP_KERNEL);
		bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
		bio->bi_write_hint = inode->i_write_hint;
		bio->bi_ioprio = dio->iocb->ki_ioprio;
		bio->bi_private = dio;
		bio->bi_end_io = iomap_dio_bio_end_io;

		ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
					     alignment - 1);
		if (unlikely(ret)) {
		ret = iomap_dio_bio_iter_one(iter, dio, pos, alignment, bio_opf);
		if (unlikely(ret < 0)) {
			/*
			 * We have to stop part way through an IO. We must fall
			 * through to the sub-block tail zeroing here, otherwise
			 * this short IO may expose stale data in the tail of
			 * the block we haven't written data to.
			 */
			bio_put(bio);
			goto zero_tail;
		}

		n = bio->bi_iter.bi_size;
		if (WARN_ON_ONCE((bio_opf & REQ_ATOMIC) && n != length)) {
			/*
			 * An atomic write bio must cover the complete length,
			 * which it doesn't, so error. We may need to zero out
			 * the tail (complete FS block), similar to when
			 * bio_iov_iter_get_pages() returns an error, above.
			 */
			ret = -EINVAL;
			bio_put(bio);
			goto zero_tail;
			break;
		}
		if (dio->flags & IOMAP_DIO_WRITE)
			task_io_account_write(n);
		else if (dio->flags & IOMAP_DIO_DIRTY)
			bio_set_pages_dirty(bio);

		dio->size += n;
		copied += n;

		/*
		 * We can only poll for single bio I/Os.
		 */
		if (iov_iter_count(dio->submit.iter))
			dio->iocb->ki_flags &= ~IOCB_HIPRI;
		iomap_dio_submit_bio(iter, dio, bio, pos);
		pos += n;
		dio->size += ret;
		copied += ret;
		pos += ret;
		ret = 0;
	} while (iov_iter_count(dio->submit.iter));

	/*
@@ -511,7 +519,6 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
	 * the block tail in the latter case, we can expose stale data via mmap
	 * reads of the EOF block.
	 */
zero_tail:
	if (need_zeroout ||
	    ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
		/* zero out from the end of the write to the end of the block */