Commit 88c02b3f authored by Joerg Schmidbauer's avatar Joerg Schmidbauer Committed by Vasily Gorbik
Browse files

s390/sha3: Support sha3 performance enhancements 



On newer machines the SHA3 performance of CPACF instructions KIMD and
KLMD can be enhanced by using additional modifier bits. This allows the
application to omit initializing the ICV, but also affects the internal
processing of the instructions. Performance is mostly gained when
processing short messages.

The new CPACF feature is backwards compatible with older machines, i.e.
the new modifier bits are ignored on older machines. However, to save the
ICV initialization, the application must detect the MSA level and omit
the ICV initialization only if this feature is supported.

Reviewed-by: default avatarHolger Dengler <dengler@linux.ibm.com>
Signed-off-by: default avatarJoerg Schmidbauer <jschmidb@de.ibm.com>
Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
Signed-off-by: default avatarVasily Gorbik <gor@linux.ibm.com>
parent 177b621b

arch/s390/crypto/sha.h

+1 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ struct s390_sha_ctx { 
	u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)];

	u8 buf[SHA_MAX_BLOCK_SIZE];

	int func;		/* KIMD function to use */

	int first_message_part;

};



struct shash_desc;

arch/s390/crypto/sha3_256_s390.c

+6 −2
Original line number Diff line number Diff line
@@ -21,9 +21,11 @@ static int sha3_256_init(struct shash_desc *desc) 
{

	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);



	if (!test_facility(86)) /* msa 12 */

		memset(sctx->state, 0, sizeof(sctx->state));

	sctx->count = 0;

	sctx->func = CPACF_KIMD_SHA3_256;

	sctx->first_message_part = 1;



	return 0;

}
@@ -88,9 +90,11 @@ static int sha3_224_init(struct shash_desc *desc) 
{

	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);



	if (!test_facility(86)) /* msa 12 */

		memset(sctx->state, 0, sizeof(sctx->state));

	sctx->count = 0;

	sctx->func = CPACF_KIMD_SHA3_224;

	sctx->first_message_part = 1;



	return 0;

}

arch/s390/crypto/sha3_512_s390.c

+6 −2
Original line number Diff line number Diff line
@@ -20,9 +20,11 @@ static int sha3_512_init(struct shash_desc *desc) 
{

	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);



	if (!test_facility(86)) /* msa 12 */

		memset(sctx->state, 0, sizeof(sctx->state));

	sctx->count = 0;

	sctx->func = CPACF_KIMD_SHA3_512;

	sctx->first_message_part = 1;



	return 0;

}
@@ -97,9 +99,11 @@ static int sha3_384_init(struct shash_desc *desc) 
{

	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);



	if (!test_facility(86)) /* msa 12 */

		memset(sctx->state, 0, sizeof(sctx->state));

	sctx->count = 0;

	sctx->func = CPACF_KIMD_SHA3_384;

	sctx->first_message_part = 1;



	return 0;

}

arch/s390/crypto/sha_common.c

+15 −4
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) 
	struct s390_sha_ctx *ctx = shash_desc_ctx(desc);

	unsigned int bsize = crypto_shash_blocksize(desc->tfm);

	unsigned int index, n;

	int fc;



	/* how much is already in the buffer? */

	index = ctx->count % bsize;
@@ -26,10 +27,15 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) 
	if ((index + len) < bsize)

		goto store;



	fc = ctx->func;

	if (ctx->first_message_part)

		fc |= test_facility(86) ? CPACF_KIMD_NIP : 0;



	/* process one stored block */

	if (index) {

		memcpy(ctx->buf + index, data, bsize - index);

		cpacf_kimd(ctx->func, ctx->state, ctx->buf, bsize);

		cpacf_kimd(fc, ctx->state, ctx->buf, bsize);

		ctx->first_message_part = 0;

		data += bsize - index;

		len -= bsize - index;

		index = 0;
@@ -38,7 +44,8 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) 
	/* process as many blocks as possible */

	if (len >= bsize) {

		n = (len / bsize) * bsize;

		cpacf_kimd(ctx->func, ctx->state, data, n);

		cpacf_kimd(fc, ctx->state, data, n);

		ctx->first_message_part = 0;

		data += n;

		len -= n;

	}
@@ -75,7 +82,7 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) 
	unsigned int bsize = crypto_shash_blocksize(desc->tfm);

	u64 bits;

	unsigned int n;

	int mbl_offset;

	int mbl_offset, fc;



	n = ctx->count % bsize;

	bits = ctx->count * 8;
@@ -109,7 +116,11 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) 
		return -EINVAL;

	}



	cpacf_klmd(ctx->func, ctx->state, ctx->buf, n);

	fc = ctx->func;

	fc |= test_facility(86) ? CPACF_KLMD_DUFOP : 0;

	if (ctx->first_message_part)

		fc |= CPACF_KLMD_NIP;

	cpacf_klmd(fc, ctx->state, ctx->buf, n);



	/* copy digest to out */

	memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm));

arch/s390/include/asm/cpacf.h

+10 −2
Original line number Diff line number Diff line
@@ -171,6 +171,14 @@ 
#define CPACF_KMA_LAAD	0x200	/* Last-AAD */

#define CPACF_KMA_HS	0x400	/* Hash-subkey Supplied */



/*

 * Flags for the KIMD/KLMD (COMPUTE INTERMEDIATE/LAST MESSAGE DIGEST)

 * instructions

 */

#define CPACF_KIMD_NIP		0x8000

#define CPACF_KLMD_DUFOP	0x4000

#define CPACF_KLMD_NIP		0x8000



typedef struct { unsigned char bytes[16]; } cpacf_mask_t;



/*
@@ -397,7 +405,7 @@ static inline void cpacf_kimd(unsigned long func, void *param, 
	asm volatile(

		"	lgr	0,%[fc]\n"

		"	lgr	1,%[pba]\n"

		"0:	.insn	rre,%[opc] << 16,0,%[src]\n"

		"0:	.insn	rrf,%[opc] << 16,0,%[src],8,0\n"

		"	brc	1,0b\n" /* handle partial completion */

		: [src] "+&d" (s.pair)

		: [fc] "d" (func), [pba] "d" ((unsigned long)(param)),
@@ -422,7 +430,7 @@ static inline void cpacf_klmd(unsigned long func, void *param, 
	asm volatile(

		"	lgr	0,%[fc]\n"

		"	lgr	1,%[pba]\n"

		"0:	.insn	rre,%[opc] << 16,0,%[src]\n"

		"0:	.insn	rrf,%[opc] << 16,0,%[src],8,0\n"

		"	brc	1,0b\n" /* handle partial completion */

		: [src] "+&d" (s.pair)

		: [fc] "d" (func), [pba] "d" ((unsigned long)param),