Commit f796c758 authored by Borislav Petkov (AMD)'s avatar Borislav Petkov (AMD)
Browse files

x86/alternatives: Use a temporary buffer when optimizing NOPs 



Instead of optimizing NOPs in-place, use a temporary buffer like the
usual alternatives patching flow does. This obviates the need to grab
locks when patching, see

  6778977590da ("x86/alternatives: Disable interrupts and sync when optimizing NOPs in place")

While at it, add nomenclature definitions clarifying and simplifying the
naming of function-local variables in the alternatives code.

Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20240130105941.19707-2-bp@alien8.de
parent ee896208

arch/x86/include/asm/text-patching.h

+1 −1
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ 


extern void text_poke_early(void *addr, const void *opcode, size_t len);



extern void apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len);

extern void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len);



/*

 * Clear and restore the kernel write-protection flag on the local CPU.

arch/x86/kernel/alternative.c

+45 −39
Original line number Diff line number Diff line
@@ -124,6 +124,20 @@ const unsigned char * const x86_nops[ASM_NOP_MAX+1] = 
#endif

};



/*

 * Nomenclature for variable names to simplify and clarify this code and ease

 * any potential staring at it:

 *

 * @instr: source address of the original instructions in the kernel text as

 * generated by the compiler.

 *

 * @buf: temporary buffer on which the patching operates. This buffer is

 * eventually text-poked into the kernel image.

 *

 * @replacement/@repl: pointer to the opcodes which are replacing @instr, located

 * in the .altinstr_replacement section.

 */



/*

 * Fill the buffer with a single effective instruction of size @len.

 *
@@ -133,28 +147,28 @@ const unsigned char * const x86_nops[ASM_NOP_MAX+1] = 
 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and

 * *jump* over instead of executing long and daft NOPs.

 */

static void add_nop(u8 *instr, unsigned int len)

static void add_nop(u8 *buf, unsigned int len)

{

	u8 *target = instr + len;

	u8 *target = buf + len;



	if (!len)

		return;



	if (len <= ASM_NOP_MAX) {

		memcpy(instr, x86_nops[len], len);

		memcpy(buf, x86_nops[len], len);

		return;

	}



	if (len < 128) {

		__text_gen_insn(instr, JMP8_INSN_OPCODE, instr, target, JMP8_INSN_SIZE);

		instr += JMP8_INSN_SIZE;

		__text_gen_insn(buf, JMP8_INSN_OPCODE, buf, target, JMP8_INSN_SIZE);

		buf += JMP8_INSN_SIZE;

	} else {

		__text_gen_insn(instr, JMP32_INSN_OPCODE, instr, target, JMP32_INSN_SIZE);

		instr += JMP32_INSN_SIZE;

		__text_gen_insn(buf, JMP32_INSN_OPCODE, buf, target, JMP32_INSN_SIZE);

		buf += JMP32_INSN_SIZE;

	}



	for (;instr < target; instr++)

		*instr = INT3_INSN_OPCODE;

	for (;buf < target; buf++)

		*buf = INT3_INSN_OPCODE;

}



extern s32 __retpoline_sites[], __retpoline_sites_end[];
@@ -187,12 +201,12 @@ static bool insn_is_nop(struct insn *insn) 
 * Find the offset of the first non-NOP instruction starting at @offset

 * but no further than @len.

 */

static int skip_nops(u8 *instr, int offset, int len)

static int skip_nops(u8 *buf, int offset, int len)

{

	struct insn insn;



	for (; offset < len; offset += insn.length) {

		if (insn_decode_kernel(&insn, &instr[offset]))

		if (insn_decode_kernel(&insn, &buf[offset]))

			break;



		if (!insn_is_nop(&insn))
@@ -207,7 +221,7 @@ static int skip_nops(u8 *instr, int offset, int len) 
 * to the end of the NOP sequence into a single NOP.

 */

static bool

__optimize_nops(u8 *instr, size_t len, struct insn *insn, int *next, int *prev, int *target)

__optimize_nops(const u8 * const instr, u8 *buf, size_t len, struct insn *insn, int *next, int *prev, int *target)

{

	int i = *next - insn->length;
@@ -222,12 +236,12 @@ __optimize_nops(u8 *instr, size_t len, struct insn *insn, int *next, int *prev, 
	if (insn_is_nop(insn)) {

		int nop = i;



		*next = skip_nops(instr, *next, len);

		*next = skip_nops(buf, *next, len);

		if (*target && *next == *target)

			nop = *prev;



		add_nop(instr + nop, *next - nop);

		DUMP_BYTES(ALT, instr, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, *next);

		add_nop(buf + nop, *next - nop);

		DUMP_BYTES(ALT, buf, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, *next);

		return true;

	}
@@ -239,32 +253,22 @@ __optimize_nops(u8 *instr, size_t len, struct insn *insn, int *next, int *prev, 
 * "noinline" to cause control flow change and thus invalidate I$ and

 * cause refetch after modification.

 */

static void __init_or_module noinline optimize_nops(u8 *instr, size_t len)

static void __init_or_module noinline optimize_nops(const u8 * const instr, u8 *buf, size_t len)

{

	int prev, target = 0;



	for (int next, i = 0; i < len; i = next) {

		struct insn insn;



		if (insn_decode_kernel(&insn, &instr[i]))

		if (insn_decode_kernel(&insn, &buf[i]))

			return;



		next = i + insn.length;



		__optimize_nops(instr, len, &insn, &next, &prev, &target);

		__optimize_nops(instr, buf, len, &insn, &next, &prev, &target);

	}

}



static void __init_or_module noinline optimize_nops_inplace(u8 *instr, size_t len)

{

	unsigned long flags;



	local_irq_save(flags);

	optimize_nops(instr, len);

	sync_core();

	local_irq_restore(flags);

}



/*

 * In this context, "source" is where the instructions are placed in the

 * section .altinstr_replacement, for example during kernel build by the
@@ -335,11 +339,11 @@ bool need_reloc(unsigned long offset, u8 *src, size_t src_len) 
	return (target < src || target > src + src_len);

}



void apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len)

void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len)

{

	int prev, target = 0;



	for (int next, i = 0; i < len; i = next) {

	for (int next, i = 0; i < instrlen; i = next) {

		struct insn insn;



		if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i])))
@@ -347,7 +351,7 @@ void apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len) 


		next = i + insn.length;



		if (__optimize_nops(buf, len, &insn, &next, &prev, &target))

		if (__optimize_nops(instr, buf, instrlen, &insn, &next, &prev, &target))

			continue;



		switch (insn.opcode.bytes[0]) {
@@ -361,10 +365,10 @@ void apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len) 
		case JMP8_INSN_OPCODE:

		case JMP32_INSN_OPCODE:

		case CALL_INSN_OPCODE:

			if (need_reloc(next + insn.immediate.value, src, src_len)) {

			if (need_reloc(next + insn.immediate.value, repl, repl_len)) {

				apply_reloc(insn.immediate.nbytes,

					    buf + i + insn_offset_immediate(&insn),

					    src - dest);

					    repl - instr);

			}



			/*
@@ -372,7 +376,7 @@ void apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len) 
			 */

			if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) {

				s32 imm = insn.immediate.value;

				imm += src - dest;

				imm += repl - instr;

				imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE;

				if ((imm >> 31) == (imm >> 7)) {

					buf[i+0] = JMP8_INSN_OPCODE;
@@ -385,10 +389,10 @@ void apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len) 
		}



		if (insn_rip_relative(&insn)) {

			if (need_reloc(next + insn.displacement.value, src, src_len)) {

			if (need_reloc(next + insn.displacement.value, repl, repl_len)) {

				apply_reloc(insn.displacement.nbytes,

					    buf + i + insn_offset_displacement(&insn),

					    src - dest);

					    repl - instr);

			}

		}

	}
@@ -504,7 +508,9 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 
		 *   patch if feature is *NOT* present.

		 */

		if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) {

			optimize_nops_inplace(instr, a->instrlen);

			memcpy(insn_buff, instr, a->instrlen);

			optimize_nops(instr, insn_buff, a->instrlen);

			text_poke_early(instr, insn_buff, a->instrlen);

			continue;

		}
@@ -526,7 +532,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 
		for (; insn_buff_sz < a->instrlen; insn_buff_sz++)

			insn_buff[insn_buff_sz] = 0x90;



		apply_relocation(insn_buff, a->instrlen, instr, replacement, a->replacementlen);

		apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen);



		DUMP_BYTES(ALT, instr, a->instrlen, "%px:   old_insn: ", instr);

		DUMP_BYTES(ALT, replacement, a->replacementlen, "%px:   rpl_insn: ", replacement);
@@ -761,7 +767,7 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) 


		len = patch_retpoline(addr, &insn, bytes);

		if (len == insn.length) {

			optimize_nops(bytes, len);

			optimize_nops(addr, bytes, len);

			DUMP_BYTES(RETPOLINE, ((u8*)addr),  len, "%px: orig: ", addr);

			DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr);

			text_poke_early(addr, bytes, len);

arch/x86/kernel/callthunks.c

+3 −6
Original line number Diff line number Diff line
@@ -185,8 +185,7 @@ static void *patch_dest(void *dest, bool direct) 
	u8 *pad = dest - tsize;



	memcpy(insn_buff, skl_call_thunk_template, tsize);

	apply_relocation(insn_buff, tsize, pad,

			 skl_call_thunk_template, tsize);

	apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize);



	/* Already patched? */

	if (!bcmp(pad, insn_buff, tsize))
@@ -308,8 +307,7 @@ static bool is_callthunk(void *addr) 
	pad = (void *)(dest - tmpl_size);



	memcpy(insn_buff, skl_call_thunk_template, tmpl_size);

	apply_relocation(insn_buff, tmpl_size, pad,

			 skl_call_thunk_template, tmpl_size);

	apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size);



	return !bcmp(pad, insn_buff, tmpl_size);

}
@@ -327,8 +325,7 @@ int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip) 
		return 0;



	memcpy(insn_buff, skl_call_thunk_template, tmpl_size);

	apply_relocation(insn_buff, tmpl_size, ip,

			 skl_call_thunk_template, tmpl_size);

	apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size);



	memcpy(*pprog, insn_buff, tmpl_size);

	*pprog += tmpl_size;