Commit 60bc276b authored by Juergen Gross's avatar Juergen Gross Committed by Borislav Petkov (AMD)
Browse files

x86/paravirt: Switch mixed paravirt/alternative calls to alternatives 



Instead of stacking alternative and paravirt patching, use the new
ALT_FLAG_CALL flag to switch those mixed calls to pure alternative
handling.

Eliminate the need to be careful regarding the sequence of alternative
and paravirt patching.

  [ bp: Touch up commit message. ]

Signed-off-by: default avatarJuergen Gross <jgross@suse.com>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20231210062138.2417-5-jgross@suse.com
parent da0fe6e6

arch/x86/include/asm/alternative.h

+3 −2
Original line number Diff line number Diff line
@@ -89,6 +89,8 @@ struct alt_instr { 
	u8  replacementlen;	/* length of new instruction */

} __packed;



extern struct alt_instr __alt_instructions[], __alt_instructions_end[];



/*

 * Debug flag that can be tested to see whether alternative

 * instructions were patched in already:
@@ -104,11 +106,10 @@ extern void apply_fineibt(s32 *start_retpoline, s32 *end_retpoine, 
			  s32 *start_cfi, s32 *end_cfi);



struct module;

struct paravirt_patch_site;



struct callthunk_sites {

	s32				*call_start, *call_end;

	struct paravirt_patch_site	*pv_start, *pv_end;

	struct alt_instr		*alt_start, *alt_end;

};



#ifdef CONFIG_CALL_THUNKS

arch/x86/include/asm/paravirt.h

+6 −6
Original line number Diff line number Diff line
@@ -738,20 +738,20 @@ void native_pv_lock_init(void) __init; 


#ifdef CONFIG_X86_64

#ifdef CONFIG_PARAVIRT_XXL

#ifdef CONFIG_DEBUG_ENTRY



#define PARA_PATCH(off)		((off) / 8)

#define PARA_SITE(ptype, ops)	_PVSITE(ptype, ops)

#define PARA_INDIRECT(addr)	*addr(%rip)



#ifdef CONFIG_DEBUG_ENTRY

.macro PARA_IRQ_save_fl

	PARA_SITE(PARA_PATCH(PV_IRQ_save_fl),

	ANNOTATE_RETPOLINE_SAFE;

		  call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);)

	call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);

.endm



#define SAVE_FLAGS	ALTERNATIVE "PARA_IRQ_save_fl;", "pushf; pop %rax;", \

				    ALT_NOT_XEN

#define SAVE_FLAGS ALTERNATIVE_2 "PARA_IRQ_save_fl;",			\

				 "ALT_CALL_INSTR;", ALT_CALL_ALWAYS,	\

				 "pushf; pop %rax;", ALT_NOT_XEN

#endif

#endif /* CONFIG_PARAVIRT_XXL */

#endif	/* CONFIG_X86_64 */

arch/x86/include/asm/paravirt_types.h

+22 −18
Original line number Diff line number Diff line
@@ -276,15 +276,11 @@ extern struct paravirt_patch_template pv_ops; 
#define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t"



unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, unsigned int len);

#define paravirt_ptr(op)	[paravirt_opptr] "m" (pv_ops.op)



int paravirt_disable_iospace(void);



/*

 * This generates an indirect call based on the operation type number.

 * The type number, computed in PARAVIRT_PATCH, is derived from the

 * offset into the paravirt_patch_template structure, and can therefore be

 * freely converted back into a structure offset.

 */

/* This generates an indirect call based on the operation type number. */

#define PARAVIRT_CALL					\

	ANNOTATE_RETPOLINE_SAFE				\

	"call *%[paravirt_opptr];"
@@ -317,12 +313,6 @@ int paravirt_disable_iospace(void); 
 * However, x86_64 also has to clobber all caller saved registers, which

 * unfortunately, are quite a bit (r8 - r11)

 *

 * The call instruction itself is marked by placing its start address

 * and size into the .parainstructions section, so that

 * apply_paravirt() in arch/i386/kernel/alternative.c can do the

 * appropriate patching under the control of the backend pv_init_ops

 * implementation.

 *

 * Unfortunately there's no way to get gcc to generate the args setup

 * for the call, and then allow the call itself to be generated by an

 * inline asm.  Because of this, we must do the complete arg setup and
@@ -421,14 +411,27 @@ int paravirt_disable_iospace(void); 
		__mask & __eax;						\

	})





/*

 * Use alternative patching for paravirt calls:

 * - For replacing an indirect call with a direct one, use the "normal"

 *   ALTERNATIVE() macro with the indirect call as the initial code sequence,

 *   which will be replaced with the related direct call by using the

 *   ALT_FLAG_DIRECT_CALL special case and the "always on" feature.

 * - In case the replacement is either a direct call or a short code sequence

 *   depending on a feature bit, the ALTERNATIVE_2() macro is being used.

 *   The indirect call is the initial code sequence again, while the special

 *   code sequence is selected with the specified feature bit. In case the

 *   feature is not active, the direct call is used as above via the

 *   ALT_FLAG_DIRECT_CALL special case and the "always on" feature.

 */

#define ____PVOP_CALL(ret, op, call_clbr, extra_clbr, ...)	\

	({								\

		PVOP_CALL_ARGS;						\

		PVOP_TEST_NULL(op);					\

		asm volatile(paravirt_alt(PARAVIRT_CALL)		\

		asm volatile(ALTERNATIVE(PARAVIRT_CALL, ALT_CALL_INSTR,	\

				ALT_CALL_ALWAYS)			\

			     : call_clbr, ASM_CALL_CONSTRAINT		\

			     : paravirt_type(op),			\

			     : paravirt_ptr(op),			\

			       ##__VA_ARGS__				\

			     : "memory", "cc" extra_clbr);		\

		ret;							\
@@ -439,10 +442,11 @@ int paravirt_disable_iospace(void); 
	({								\

		PVOP_CALL_ARGS;						\

		PVOP_TEST_NULL(op);					\

		asm volatile(ALTERNATIVE(paravirt_alt(PARAVIRT_CALL),	\

		asm volatile(ALTERNATIVE_2(PARAVIRT_CALL,		\

				 ALT_CALL_INSTR, ALT_CALL_ALWAYS,	\

				 alt, cond)				\

			     : call_clbr, ASM_CALL_CONSTRAINT		\

			     : paravirt_type(op),			\

			     : paravirt_ptr(op),			\

			       ##__VA_ARGS__				\

			     : "memory", "cc" extra_clbr);		\

		ret;							\

arch/x86/kernel/alternative.c

+0 −1
Original line number Diff line number Diff line
@@ -160,7 +160,6 @@ extern s32 __retpoline_sites[], __retpoline_sites_end[]; 
extern s32 __return_sites[], __return_sites_end[];

extern s32 __cfi_sites[], __cfi_sites_end[];

extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];

extern struct alt_instr __alt_instructions[], __alt_instructions_end[];

extern s32 __smp_locks[], __smp_locks_end[];

void text_poke_early(void *addr, const void *opcode, size_t len);

arch/x86/kernel/callthunks.c

+8 −9
Original line number Diff line number Diff line
@@ -233,14 +233,13 @@ patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 
}



static __init_or_module void

patch_paravirt_call_sites(struct paravirt_patch_site *start,

			  struct paravirt_patch_site *end,

patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end,

		     const struct core_text *ct)

{

	struct paravirt_patch_site *p;

	struct alt_instr *a;



	for (p = start; p < end; p++)

		patch_call((void *)&p->instr_offset + p->instr_offset, ct);

	for (a = start; a < end; a++)

		patch_call((void *)&a->instr_offset + a->instr_offset, ct);

}



static __init_or_module void
@@ -248,7 +247,7 @@ callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 
{

	prdbg("Patching call sites %s\n", ct->name);

	patch_call_sites(cs->call_start, cs->call_end, ct);

	patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);

	patch_alt_call_sites(cs->alt_start, cs->alt_end, ct);

	prdbg("Patching call sites done%s\n", ct->name);

}
@@ -257,8 +256,8 @@ void __init callthunks_patch_builtin_calls(void) 
	struct callthunk_sites cs = {

		.call_start	= __call_sites,

		.call_end	= __call_sites_end,

		.pv_start	= __parainstructions,

		.pv_end		= __parainstructions_end

		.alt_start	= __alt_instructions,

		.alt_end	= __alt_instructions_end

	};



	if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))