Commit ab9fea59 authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

x86/alternative: Simplify callthunk patching 



Now that paravirt call patching is implemented using alternatives, it
is possible to avoid having to patch the alternative sites by
including the altinstr_replacement calls in the call_sites list.

This means we're now stacking relative adjustments like so:

  callthunks_patch_builtin_calls():
    patches all function calls to target: func() -> func()-10
    since the CALL accounting lives in the CALL_PADDING.

    This explicitly includes .altinstr_replacement

  alt_replace_call():
    patches: x86_BUG() -> target()

    this patching is done in a relative manner, and will preserve
    the above adjustment, meaning that with calldepth patching it
    will do: x86_BUG()-10 -> target()-10

  apply_relocation():
    does code relocation, and adjusts all RIP-relative instructions
    to the new location, also in a relative manner.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarSami Tolvanen <samitolvanen@google.com>
Link: https://lore.kernel.org/r/20250207122546.617187089@infradead.org
parent 93f16a1a

arch/x86/include/asm/alternative.h

+0 −1
Original line number Diff line number Diff line
@@ -100,7 +100,6 @@ struct module; 


struct callthunk_sites {

	s32				*call_start, *call_end;

	struct alt_instr		*alt_start, *alt_end;

};



#ifdef CONFIG_CALL_THUNKS

arch/x86/kernel/alternative.c

+4 −4
Original line number Diff line number Diff line
@@ -1705,14 +1705,14 @@ void __init alternative_instructions(void) 
	apply_retpolines(__retpoline_sites, __retpoline_sites_end);

	apply_returns(__return_sites, __return_sites_end);



	apply_alternatives(__alt_instructions, __alt_instructions_end);



	/*

	 * Now all calls are established. Apply the call thunks if

	 * required.

	 * Adjust all CALL instructions to point to func()-10, including

	 * those in .altinstr_replacement.

	 */

	callthunks_patch_builtin_calls();



	apply_alternatives(__alt_instructions, __alt_instructions_end);



	/*

	 * Seal all functions that do not have their address taken.

	 */

arch/x86/kernel/callthunks.c

+0 −13
Original line number Diff line number Diff line
@@ -239,22 +239,11 @@ patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 
		patch_call((void *)s + *s, ct);

}



static __init_or_module void

patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end,

		     const struct core_text *ct)

{

	struct alt_instr *a;



	for (a = start; a < end; a++)

		patch_call((void *)&a->instr_offset + a->instr_offset, ct);

}



static __init_or_module void

callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)

{

	prdbg("Patching call sites %s\n", ct->name);

	patch_call_sites(cs->call_start, cs->call_end, ct);

	patch_alt_call_sites(cs->alt_start, cs->alt_end, ct);

	prdbg("Patching call sites done%s\n", ct->name);

}
@@ -263,8 +252,6 @@ void __init callthunks_patch_builtin_calls(void) 
	struct callthunk_sites cs = {

		.call_start	= __call_sites,

		.call_end	= __call_sites_end,

		.alt_start	= __alt_instructions,

		.alt_end	= __alt_instructions_end

	};



	if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))

arch/x86/kernel/module.c

+6 −11
Original line number Diff line number Diff line
@@ -275,12 +275,7 @@ int module_finalize(const Elf_Ehdr *hdr, 
		void *rseg = (void *)returns->sh_addr;

		apply_returns(rseg, rseg + returns->sh_size);

	}

	if (alt) {

		/* patch .altinstructions */

		void *aseg = (void *)alt->sh_addr;

		apply_alternatives(aseg, aseg + alt->sh_size);

	}

	if (calls || alt) {

	if (calls) {

		struct callthunk_sites cs = {};



		if (calls) {
@@ -288,13 +283,13 @@ int module_finalize(const Elf_Ehdr *hdr, 
			cs.call_end = (void *)calls->sh_addr + calls->sh_size;

		}



		if (alt) {

			cs.alt_start = (void *)alt->sh_addr;

			cs.alt_end = (void *)alt->sh_addr + alt->sh_size;

		}



		callthunks_patch_module_calls(&cs, me);

	}

	if (alt) {

		/* patch .altinstructions */

		void *aseg = (void *)alt->sh_addr;

		apply_alternatives(aseg, aseg + alt->sh_size);

	}

	if (ibt_endbr) {

		void *iseg = (void *)ibt_endbr->sh_addr;

		apply_seal_endbr(iseg, iseg + ibt_endbr->sh_size);

tools/objtool/arch/x86/decode.c

+1 −0
Original line number Diff line number Diff line
@@ -850,5 +850,6 @@ bool arch_is_rethunk(struct symbol *sym) 
bool arch_is_embedded_insn(struct symbol *sym)

{

	return !strcmp(sym->name, "retbleed_return_thunk") ||

	       !strcmp(sym->name, "srso_alias_safe_ret") ||

	       !strcmp(sym->name, "srso_safe_ret");

}