Commit bb699e72 authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu
Browse files

crypto: scatterwalk - add new functions for copying data 



Add memcpy_from_sglist() and memcpy_to_sglist() which are more readable
versions of scatterwalk_map_and_copy() with the 'out' argument 0 and 1
respectively.  They follow the same argument order as memcpy_from_page()
and memcpy_to_page() from <linux/highmem.h>.  Note that in the case of
memcpy_from_sglist(), this also happens to be the same argument order
that scatterwalk_map_and_copy() uses.

The new code is also faster, mainly because it builds the scatter_walk
directly without creating a temporary scatterlist.  E.g., a 20%
performance improvement is seen for copying the AES-GCM auth tag.

Make scatterwalk_map_and_copy() be a wrapper around memcpy_from_sglist()
and memcpy_to_sglist().  Callers of scatterwalk_map_and_copy() should be
updated to call memcpy_from_sglist() or memcpy_to_sglist() directly, but
there are a lot of them so they aren't all being updated right away.

Also add functions memcpy_from_scatterwalk() and memcpy_to_scatterwalk()
which are similar but operate on a scatter_walk instead of a
scatterlist.  These will replace scatterwalk_copychunks() with the 'out'
argument 0 and 1 respectively.  Their behavior differs slightly from
scatterwalk_copychunks() in that they automatically take care of
flushing the dcache when needed, making them easier to use.

scatterwalk_copychunks() itself is left unchanged for now.  It will be
removed after its callers are updated to use other functions instead.

Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 31b00fe1

crypto/scatterwalk.c

+50 −9
Original line number Diff line number Diff line
@@ -67,22 +67,63 @@ void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, 
}

EXPORT_SYMBOL_GPL(scatterwalk_copychunks);



void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,

			      unsigned int start, unsigned int nbytes, int out)

inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk,

				    unsigned int nbytes)

{

	do {

		const void *src_addr;

		unsigned int to_copy;



		src_addr = scatterwalk_next(walk, nbytes, &to_copy);

		memcpy(buf, src_addr, to_copy);

		scatterwalk_done_src(walk, src_addr, to_copy);

		buf += to_copy;

		nbytes -= to_copy;

	} while (nbytes);

}

EXPORT_SYMBOL_GPL(memcpy_from_scatterwalk);



inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf,

				  unsigned int nbytes)

{

	do {

		void *dst_addr;

		unsigned int to_copy;



		dst_addr = scatterwalk_next(walk, nbytes, &to_copy);

		memcpy(dst_addr, buf, to_copy);

		scatterwalk_done_dst(walk, dst_addr, to_copy);

		buf += to_copy;

		nbytes -= to_copy;

	} while (nbytes);

}

EXPORT_SYMBOL_GPL(memcpy_to_scatterwalk);



void memcpy_from_sglist(void *buf, struct scatterlist *sg,

			unsigned int start, unsigned int nbytes)

{

	struct scatter_walk walk;

	struct scatterlist tmp[2];



	if (!nbytes)

	if (unlikely(nbytes == 0)) /* in case sg == NULL */

		return;



	sg = scatterwalk_ffwd(tmp, sg, start);

	scatterwalk_start_at_pos(&walk, sg, start);

	memcpy_from_scatterwalk(buf, &walk, nbytes);

}

EXPORT_SYMBOL_GPL(memcpy_from_sglist);



void memcpy_to_sglist(struct scatterlist *sg, unsigned int start,

		      const void *buf, unsigned int nbytes)

{

	struct scatter_walk walk;



	if (unlikely(nbytes == 0)) /* in case sg == NULL */

		return;



	scatterwalk_start(&walk, sg);

	scatterwalk_copychunks(buf, &walk, nbytes, out);

	scatterwalk_done(&walk, out, 0);

	scatterwalk_start_at_pos(&walk, sg, start);

	memcpy_to_scatterwalk(&walk, buf, nbytes);

}

EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy);

EXPORT_SYMBOL_GPL(memcpy_to_sglist);



struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],

				     struct scatterlist *src,

include/crypto/scatterwalk.h

+22 −2
Original line number Diff line number Diff line
@@ -170,8 +170,28 @@ void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes); 
void scatterwalk_copychunks(void *buf, struct scatter_walk *walk,

			    size_t nbytes, int out);



void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,

			      unsigned int start, unsigned int nbytes, int out);

void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk,

			     unsigned int nbytes);



void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf,

			   unsigned int nbytes);



void memcpy_from_sglist(void *buf, struct scatterlist *sg,

			unsigned int start, unsigned int nbytes);



void memcpy_to_sglist(struct scatterlist *sg, unsigned int start,

		      const void *buf, unsigned int nbytes);



/* In new code, please use memcpy_{from,to}_sglist() directly instead. */

static inline void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,

					    unsigned int start,

					    unsigned int nbytes, int out)

{

	if (out)

		memcpy_to_sglist(sg, start, buf, nbytes);

	else

		memcpy_from_sglist(buf, sg, start, nbytes);

}



struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],

				     struct scatterlist *src,