Commit a9a8b1a3 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Eric Biggers
Browse files

crypto/arm64: aes/xts - Use single ksimd scope to reduce stack bloat 



The ciphertext stealing logic in the AES-XTS implementation creates a
separate ksimd scope to call into the FP/SIMD core routines, and in some
cases (CONFIG_KASAN_STACK is one, but there might be others), the 528
byte kernel mode FP/SIMD buffer that is allocated inside this scope is
not shared with the preceding ksimd scope, resulting in unnecessary
stack bloat.

Considering that

a) the XTS ciphertext stealing logic is never called for block
   encryption use cases, and XTS is rarely used for anything else,

b) in the vast majority of cases, the entire input block is processed
   during the first iteration of the loop,

we can combine both ksimd scopes into a single one with no practical
impact on how often/how long FP/SIMD is en/disabled, allowing us to
reuse the same stack slot for both FP/SIMD routine calls.

Fixes: ba3c1b3b ("crypto/arm64: aes-blk - Switch to 'ksimd' scoped guard API")
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Tested-by: default avatarArnd Bergmann <arnd@arndb.de>
Link: https://lore.kernel.org/r/20251203163803.157541-5-ardb@kernel.org


Signed-off-by: default avatarEric Biggers <ebiggers@kernel.org>
parent 68b233b1

arch/arm64/crypto/aes-glue.c

+36 −39
Original line number Diff line number Diff line
@@ -549,13 +549,13 @@ static int __maybe_unused xts_encrypt(struct skcipher_request *req) 
		tail = 0;

	}



	scoped_ksimd() {

		for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {

			int nbytes = walk.nbytes;



			if (walk.nbytes < walk.total)

				nbytes &= ~(AES_BLOCK_SIZE - 1);



		scoped_ksimd()

			aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,

					ctx->key1.key_enc, rounds, nbytes,

					ctx->key2.key_enc, walk.iv, first);
@@ -576,11 +576,10 @@ static int __maybe_unused xts_encrypt(struct skcipher_request *req) 
		if (err)

			return err;



	scoped_ksimd()

		aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,

				ctx->key1.key_enc, rounds, walk.nbytes,

				ctx->key2.key_enc, walk.iv, first);



	}

	return skcipher_walk_done(&walk, 0);

}
@@ -619,13 +618,13 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) 
		tail = 0;

	}



	scoped_ksimd() {

		for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {

			int nbytes = walk.nbytes;



			if (walk.nbytes < walk.total)

				nbytes &= ~(AES_BLOCK_SIZE - 1);



		scoped_ksimd()

			aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,

					ctx->key1.key_dec, rounds, nbytes,

					ctx->key2.key_enc, walk.iv, first);
@@ -646,12 +645,10 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) 
		if (err)

			return err;





	scoped_ksimd()

		aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,

				ctx->key1.key_dec, rounds, walk.nbytes,

				ctx->key2.key_enc, walk.iv, first);



	}

	return skcipher_walk_done(&walk, 0);

}

arch/arm64/crypto/aes-neonbs-glue.c

+21 −23
Original line number Diff line number Diff line
@@ -312,13 +312,13 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, 
	if (err)

		return err;



	scoped_ksimd() {

		while (walk.nbytes >= AES_BLOCK_SIZE) {

			int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;

			out = walk.dst.virt.addr;

			in = walk.src.virt.addr;

			nbytes = walk.nbytes;



		scoped_ksimd() {

			if (blocks >= 8) {

				if (first == 1)

					neon_aes_ecb_encrypt(walk.iv, walk.iv,
@@ -344,7 +344,6 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, 
							     ctx->twkey, walk.iv, first);

				nbytes = first = 0;

			}

		}

			err = skcipher_walk_done(&walk, nbytes);

		}
@@ -367,7 +366,6 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, 
		in = walk.src.virt.addr;

		nbytes = walk.nbytes;



	scoped_ksimd() {

		if (encrypt)

			neon_aes_xts_encrypt(out, in, ctx->cts.key_enc,

					     ctx->key.rounds, nbytes, ctx->twkey,