Commit 31b00fe1 authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: scatterwalk - add new functions for iterating through data 



Add scatterwalk_next() which consolidates scatterwalk_clamp() and
scatterwalk_map().  Also add scatterwalk_done_src() and
scatterwalk_done_dst() which consolidate scatterwalk_unmap(),
scatterwalk_advance(), and scatterwalk_done() or scatterwalk_pagedone().
A later patch will remove scatterwalk_done() and scatterwalk_pagedone().

The new code eliminates the error-prone 'more' parameter.  Advancing to
the next sg entry now only happens just-in-time in scatterwalk_next().

The new code also pairs the dcache flush more closely with the actual
write, similar to memcpy_to_page().  Previously it was paired with
advancing to the next page.  This is currently causing bugs where the
dcache flush is incorrectly being skipped, usually due to
scatterwalk_copychunks() being called without a following
scatterwalk_done().  The dcache flush may have been placed where it was
in order to not call flush_dcache_page() redundantly when visiting a
page more than once.  However, that case is rare in practice, and most
architectures either do not implement flush_dcache_page() anyway or
implement it lazily where it just clears a page flag.

Another limitation of the old code was that by the time the flush
happened, there was no way to tell if more than one page needed to be
flushed.  That has been sufficient because the code goes page by page,
but I would like to optimize that on !HIGHMEM platforms.  The new code
makes this possible, and a later patch will implement this optimization.

Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent e21d01a2

include/crypto/scatterwalk.h

+63 −6
Original line number Diff line number Diff line
@@ -64,12 +64,6 @@ static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk, 
	return min(nbytes, scatterwalk_pagelen(walk));

}



static inline void scatterwalk_advance(struct scatter_walk *walk,

				       unsigned int nbytes)

{

	walk->offset += nbytes;

}



static inline struct page *scatterwalk_page(struct scatter_walk *walk)

{

	return sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT);
@@ -86,6 +80,24 @@ static inline void *scatterwalk_map(struct scatter_walk *walk) 
	       offset_in_page(walk->offset);

}



/**

 * scatterwalk_next() - Get the next data buffer in a scatterlist walk

 * @walk: the scatter_walk

 * @total: the total number of bytes remaining, > 0

 * @nbytes_ret: (out) the next number of bytes available, <= @total

 *

 * Return: A virtual address for the next segment of data from the scatterlist.

 *	   The caller must call scatterwalk_done_src() or scatterwalk_done_dst()

 *	   when it is done using this virtual address.

 */

static inline void *scatterwalk_next(struct scatter_walk *walk,

				     unsigned int total,

				     unsigned int *nbytes_ret)

{

	*nbytes_ret = scatterwalk_clamp(walk, total);

	return scatterwalk_map(walk);

}



static inline void scatterwalk_pagedone(struct scatter_walk *walk, int out,

					unsigned int more)

{
@@ -108,6 +120,51 @@ static inline void scatterwalk_done(struct scatter_walk *walk, int out, 
		scatterwalk_pagedone(walk, out, more);

}



static inline void scatterwalk_advance(struct scatter_walk *walk,

				       unsigned int nbytes)

{

	walk->offset += nbytes;

}



/**

 * scatterwalk_done_src() - Finish one step of a walk of source scatterlist

 * @walk: the scatter_walk

 * @vaddr: the address returned by scatterwalk_next()

 * @nbytes: the number of bytes processed this step, less than or equal to the

 *	    number of bytes that scatterwalk_next() returned.

 *

 * Use this if the @vaddr was not written to, i.e. it is source data.

 */

static inline void scatterwalk_done_src(struct scatter_walk *walk,

					const void *vaddr, unsigned int nbytes)

{

	scatterwalk_unmap((void *)vaddr);

	scatterwalk_advance(walk, nbytes);

}



/**

 * scatterwalk_done_dst() - Finish one step of a walk of destination scatterlist

 * @walk: the scatter_walk

 * @vaddr: the address returned by scatterwalk_next()

 * @nbytes: the number of bytes processed this step, less than or equal to the

 *	    number of bytes that scatterwalk_next() returned.

 *

 * Use this if the @vaddr may have been written to, i.e. it is destination data.

 */

static inline void scatterwalk_done_dst(struct scatter_walk *walk,

					void *vaddr, unsigned int nbytes)

{

	scatterwalk_unmap(vaddr);

	/*

	 * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just

	 * relying on flush_dcache_page() being a no-op when not implemented,

	 * since otherwise the BUG_ON in sg_page() does not get optimized out.

	 */

	if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)

		flush_dcache_page(scatterwalk_page(walk));

	scatterwalk_advance(walk, nbytes);

}



void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes);



void scatterwalk_copychunks(void *buf, struct scatter_walk *walk,