Commit 4dffc9bb authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: scatterwalk - Fix memcpy_sglist() to always succeed 



The original implementation of memcpy_sglist() was broken because it
didn't handle scatterlists that describe exactly the same memory, which
is a case that many callers rely on.  The current implementation is
broken too because it calls the skcipher_walk functions which can fail.
It ignores any errors from those functions.

Fix it by replacing it with a new implementation written from scratch.
It always succeeds.  It's also a bit faster, since it avoids the
overhead of skcipher_walk.  skcipher_walk includes a lot of
functionality (such as alignmask handling) that's irrelevant here.

Reported-by: default avatarColin Ian King <coking@nvidia.com>
Closes: https://lore.kernel.org/r/20251114122620.111623-1-coking@nvidia.com


Fixes: 131bdcec ("crypto: scatterwalk - Add memcpy_sglist")
Fixes: 0f8d42bf ("crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist")
Cc: stable@vger.kernel.org
Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 5727a844

crypto/scatterwalk.c

+84 −13
Original line number Diff line number Diff line
@@ -101,26 +101,97 @@ void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, 
}

EXPORT_SYMBOL_GPL(memcpy_to_sglist);



/**

 * memcpy_sglist() - Copy data from one scatterlist to another

 * @dst: The destination scatterlist.  Can be NULL if @nbytes == 0.

 * @src: The source scatterlist.  Can be NULL if @nbytes == 0.

 * @nbytes: Number of bytes to copy

 *

 * The scatterlists can describe exactly the same memory, in which case this

 * function is a no-op.  No other overlaps are supported.

 *

 * Context: Any context

 */

void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src,

		   unsigned int nbytes)

{

	struct skcipher_walk walk = {};

	unsigned int src_offset, dst_offset;



	if (unlikely(nbytes == 0)) /* in case sg == NULL */

	if (unlikely(nbytes == 0)) /* in case src and/or dst is NULL */

		return;



	walk.total = nbytes;



	scatterwalk_start(&walk.in, src);

	scatterwalk_start(&walk.out, dst);

	src_offset = src->offset;

	dst_offset = dst->offset;

	for (;;) {

		/* Compute the length to copy this step. */

		unsigned int len = min3(src->offset + src->length - src_offset,

					dst->offset + dst->length - dst_offset,

					nbytes);

		struct page *src_page = sg_page(src);

		struct page *dst_page = sg_page(dst);

		const void *src_virt;

		void *dst_virt;



		if (IS_ENABLED(CONFIG_HIGHMEM)) {

			/* HIGHMEM: we may have to actually map the pages. */

			const unsigned int src_oip = offset_in_page(src_offset);

			const unsigned int dst_oip = offset_in_page(dst_offset);

			const unsigned int limit = PAGE_SIZE;



			/* Further limit len to not cross a page boundary. */

			len = min3(len, limit - src_oip, limit - dst_oip);



			/* Compute the source and destination pages. */

			src_page += src_offset / PAGE_SIZE;

			dst_page += dst_offset / PAGE_SIZE;



			if (src_page != dst_page) {

				/* Copy between different pages. */

				memcpy_page(dst_page, dst_oip,

					    src_page, src_oip, len);

				flush_dcache_page(dst_page);

			} else if (src_oip != dst_oip) {

				/* Copy between different parts of same page. */

				dst_virt = kmap_local_page(dst_page);

				memcpy(dst_virt + dst_oip, dst_virt + src_oip,

				       len);

				kunmap_local(dst_virt);

				flush_dcache_page(dst_page);

			} /* Else, it's the same memory.  No action needed. */

		} else {

			/*

			 * !HIGHMEM: no mapping needed.  Just work in the linear

			 * buffer of each sg entry.  Note that we can cross page

			 * boundaries, as they are not significant in this case.

			 */

			src_virt = page_address(src_page) + src_offset;

			dst_virt = page_address(dst_page) + dst_offset;

			if (src_virt != dst_virt) {

				memcpy(dst_virt, src_virt, len);

				if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)

					__scatterwalk_flush_dcache_pages(

						dst_page, dst_offset, len);

			} /* Else, it's the same memory.  No action needed. */

		}

		nbytes -= len;

		if (nbytes == 0) /* No more to copy? */

			break;



	skcipher_walk_first(&walk, true);

	do {

		if (walk.src.virt.addr != walk.dst.virt.addr)

			memcpy(walk.dst.virt.addr, walk.src.virt.addr,

			       walk.nbytes);

		skcipher_walk_done(&walk, 0);

	} while (walk.nbytes);

		/*

		 * There's more to copy.  Advance the offsets by the length

		 * copied this step, and advance the sg entries as needed.

		 */

		src_offset += len;

		if (src_offset >= src->offset + src->length) {

			src = sg_next(src);

			src_offset = src->offset;

		}

		dst_offset += len;

		if (dst_offset >= dst->offset + dst->length) {

			dst = sg_next(dst);

			dst_offset = dst->offset;

		}

	}

}

EXPORT_SYMBOL_GPL(memcpy_sglist);

include/crypto/scatterwalk.h

+31 −21
Original line number Diff line number Diff line
@@ -227,6 +227,34 @@ static inline void scatterwalk_done_src(struct scatter_walk *walk, 
	scatterwalk_advance(walk, nbytes);

}



/*

 * Flush the dcache of any pages that overlap the region

 * [offset, offset + nbytes) relative to base_page.

 *

 * This should be called only when ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE, to ensure

 * that all relevant code (including the call to sg_page() in the caller, if

 * applicable) gets fully optimized out when !ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE.

 */

static inline void __scatterwalk_flush_dcache_pages(struct page *base_page,

						    unsigned int offset,

						    unsigned int nbytes)

{

	unsigned int num_pages;



	base_page += offset / PAGE_SIZE;

	offset %= PAGE_SIZE;



	/*

	 * This is an overflow-safe version of

	 * num_pages = DIV_ROUND_UP(offset + nbytes, PAGE_SIZE).

	 */

	num_pages = nbytes / PAGE_SIZE;

	num_pages += DIV_ROUND_UP(offset + (nbytes % PAGE_SIZE), PAGE_SIZE);



	for (unsigned int i = 0; i < num_pages; i++)

		flush_dcache_page(base_page + i);

}



/**

 * scatterwalk_done_dst() - Finish one step of a walk of destination scatterlist

 * @walk: the scatter_walk
@@ -240,27 +268,9 @@ static inline void scatterwalk_done_dst(struct scatter_walk *walk, 
					unsigned int nbytes)

{

	scatterwalk_unmap(walk);

	/*

	 * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just

	 * relying on flush_dcache_page() being a no-op when not implemented,

	 * since otherwise the BUG_ON in sg_page() does not get optimized out.

	 * This also avoids having to consider whether the loop would get

	 * reliably optimized out or not.

	 */

	if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) {

		struct page *base_page;

		unsigned int offset;

		int start, end, i;



		base_page = sg_page(walk->sg);

		offset = walk->offset;

		start = offset >> PAGE_SHIFT;

		end = start + (nbytes >> PAGE_SHIFT);

		end += (offset_in_page(offset) + offset_in_page(nbytes) +

			PAGE_SIZE - 1) >> PAGE_SHIFT;

		for (i = start; i < end; i++)

			flush_dcache_page(base_page + i);

	}

	if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)

		__scatterwalk_flush_dcache_pages(sg_page(walk->sg),

						 walk->offset, nbytes);

	scatterwalk_advance(walk, nbytes);

}