Commit 1d7e707a authored by Mike Rapoport (Microsoft)'s avatar Mike Rapoport (Microsoft) Committed by Peter Zijlstra
Browse files

Revert "x86/module: prepare module loading for ROX allocations of text" 



The module code does not create a writable copy of the executable memory
anymore so there is no need to handle it in module relocation and
alternatives patching.

This reverts commit 9bfc4824.

Signed-off-by: default avatar"Mike Rapoport (Microsoft)" <rppt@kernel.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250126074733.1384926-8-rppt@kernel.org
parent c287c072

arch/um/kernel/um_arch.c

+5 −6
Original line number Diff line number Diff line
@@ -440,25 +440,24 @@ void __init arch_cpu_finalize_init(void) 
	os_check_bugs();

}



void apply_seal_endbr(s32 *start, s32 *end, struct module *mod)

void apply_seal_endbr(s32 *start, s32 *end)

{

}



void apply_retpolines(s32 *start, s32 *end, struct module *mod)

void apply_retpolines(s32 *start, s32 *end)

{

}



void apply_returns(s32 *start, s32 *end, struct module *mod)

void apply_returns(s32 *start, s32 *end)

{

}



void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,

		   s32 *start_cfi, s32 *end_cfi, struct module *mod)

		   s32 *start_cfi, s32 *end_cfi)

{

}



void apply_alternatives(struct alt_instr *start, struct alt_instr *end,

			struct module *mod)

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)

{

}

arch/x86/entry/vdso/vma.c

+1 −2
Original line number Diff line number Diff line
@@ -48,8 +48,7 @@ int __init init_vdso_image(const struct vdso_image *image) 


	apply_alternatives((struct alt_instr *)(image->data + image->alt),

			   (struct alt_instr *)(image->data + image->alt +

						image->alt_len),

			   NULL);

						image->alt_len));



	return 0;

}

arch/x86/include/asm/alternative.h

+7 −7
Original line number Diff line number Diff line
@@ -87,16 +87,16 @@ extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 
 * instructions were patched in already:

 */

extern int alternatives_patched;

struct module;



extern void alternative_instructions(void);

extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end,

			       struct module *mod);

extern void apply_retpolines(s32 *start, s32 *end, struct module *mod);

extern void apply_returns(s32 *start, s32 *end, struct module *mod);

extern void apply_seal_endbr(s32 *start, s32 *end, struct module *mod);

extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);

extern void apply_retpolines(s32 *start, s32 *end);

extern void apply_returns(s32 *start, s32 *end);

extern void apply_seal_endbr(s32 *start, s32 *end);

extern void apply_fineibt(s32 *start_retpoline, s32 *end_retpoine,

			  s32 *start_cfi, s32 *end_cfi, struct module *mod);

			  s32 *start_cfi, s32 *end_cfi);



struct module;



struct callthunk_sites {

	s32				*call_start, *call_end;

arch/x86/kernel/alternative.c

+76 −105
Original line number Diff line number Diff line
@@ -392,10 +392,8 @@ EXPORT_SYMBOL(BUG_func); 
 * Rewrite the "call BUG_func" replacement to point to the target of the

 * indirect pv_ops call "call *disp(%ip)".

 */

static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a,

			    struct module *mod)

static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a)

{

	u8 *wr_instr = module_writable_address(mod, instr);

	void *target, *bug = &BUG_func;

	s32 disp;
@@ -405,14 +403,14 @@ static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a, 
	}



	if (a->instrlen != 6 ||

	    wr_instr[0] != CALL_RIP_REL_OPCODE ||

	    wr_instr[1] != CALL_RIP_REL_MODRM) {

	    instr[0] != CALL_RIP_REL_OPCODE ||

	    instr[1] != CALL_RIP_REL_MODRM) {

		pr_err("ALT_FLAG_DIRECT_CALL set for unrecognized indirect call\n");

		BUG();

	}



	/* Skip CALL_RIP_REL_OPCODE and CALL_RIP_REL_MODRM */

	disp = *(s32 *)(wr_instr + 2);

	disp = *(s32 *)(instr + 2);

#ifdef CONFIG_X86_64

	/* ff 15 00 00 00 00   call   *0x0(%rip) */

	/* target address is stored at "next instruction + disp". */
@@ -450,8 +448,7 @@ static inline u8 * instr_va(struct alt_instr *i) 
 * to refetch changed I$ lines.

 */

void __init_or_module noinline apply_alternatives(struct alt_instr *start,

						  struct alt_instr *end,

						  struct module *mod)

						  struct alt_instr *end)

{

	u8 insn_buff[MAX_PATCH_LEN];

	u8 *instr, *replacement;
@@ -480,7 +477,6 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 
	 */

	for (a = start; a < end; a++) {

		int insn_buff_sz = 0;

		u8 *wr_instr, *wr_replacement;



		/*

		 * In case of nested ALTERNATIVE()s the outer alternative might
@@ -494,11 +490,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 
		}



		instr = instr_va(a);

		wr_instr = module_writable_address(mod, instr);



		replacement = (u8 *)&a->repl_offset + a->repl_offset;

		wr_replacement = module_writable_address(mod, replacement);



		BUG_ON(a->instrlen > sizeof(insn_buff));

		BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
@@ -509,9 +501,9 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 
		 *   patch if feature is *NOT* present.

		 */

		if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) {

			memcpy(insn_buff, wr_instr, a->instrlen);

			memcpy(insn_buff, instr, a->instrlen);

			optimize_nops(instr, insn_buff, a->instrlen);

			text_poke_early(wr_instr, insn_buff, a->instrlen);

			text_poke_early(instr, insn_buff, a->instrlen);

			continue;

		}
@@ -521,12 +513,11 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 
			instr, instr, a->instrlen,

			replacement, a->replacementlen, a->flags);



		memcpy(insn_buff, wr_replacement, a->replacementlen);

		memcpy(insn_buff, replacement, a->replacementlen);

		insn_buff_sz = a->replacementlen;



		if (a->flags & ALT_FLAG_DIRECT_CALL) {

			insn_buff_sz = alt_replace_call(instr, insn_buff, a,

							mod);

			insn_buff_sz = alt_replace_call(instr, insn_buff, a);

			if (insn_buff_sz < 0)

				continue;

		}
@@ -536,11 +527,11 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 


		apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen);



		DUMP_BYTES(ALT, wr_instr, a->instrlen, "%px:   old_insn: ", instr);

		DUMP_BYTES(ALT, instr, a->instrlen, "%px:   old_insn: ", instr);

		DUMP_BYTES(ALT, replacement, a->replacementlen, "%px:   rpl_insn: ", replacement);

		DUMP_BYTES(ALT, insn_buff, insn_buff_sz, "%px: final_insn: ", instr);



		text_poke_early(wr_instr, insn_buff, insn_buff_sz);

		text_poke_early(instr, insn_buff, insn_buff_sz);

	}



	kasan_enable_current();
@@ -731,20 +722,18 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) 
/*

 * Generated by 'objtool --retpoline'.

 */

void __init_or_module noinline apply_retpolines(s32 *start, s32 *end,

						struct module *mod)

void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)

{

	s32 *s;



	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr = module_writable_address(mod, addr);

		struct insn insn;

		int len, ret;

		u8 bytes[16];

		u8 op1, op2;



		ret = insn_decode_kernel(&insn, wr_addr);

		ret = insn_decode_kernel(&insn, addr);

		if (WARN_ON_ONCE(ret < 0))

			continue;
@@ -772,9 +761,9 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end, 
		len = patch_retpoline(addr, &insn, bytes);

		if (len == insn.length) {

			optimize_nops(addr, bytes, len);

			DUMP_BYTES(RETPOLINE, ((u8*)wr_addr),  len, "%px: orig: ", addr);

			DUMP_BYTES(RETPOLINE, ((u8*)addr),  len, "%px: orig: ", addr);

			DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr);

			text_poke_early(wr_addr, bytes, len);

			text_poke_early(addr, bytes, len);

		}

	}

}
@@ -810,8 +799,7 @@ static int patch_return(void *addr, struct insn *insn, u8 *bytes) 
	return i;

}



void __init_or_module noinline apply_returns(s32 *start, s32 *end,

					     struct module *mod)

void __init_or_module noinline apply_returns(s32 *start, s32 *end)

{

	s32 *s;
@@ -820,13 +808,12 @@ void __init_or_module noinline apply_returns(s32 *start, s32 *end, 


	for (s = start; s < end; s++) {

		void *dest = NULL, *addr = (void *)s + *s;

		void *wr_addr = module_writable_address(mod, addr);

		struct insn insn;

		int len, ret;

		u8 bytes[16];

		u8 op;



		ret = insn_decode_kernel(&insn, wr_addr);

		ret = insn_decode_kernel(&insn, addr);

		if (WARN_ON_ONCE(ret < 0))

			continue;
@@ -846,35 +833,32 @@ void __init_or_module noinline apply_returns(s32 *start, s32 *end, 


		len = patch_return(addr, &insn, bytes);

		if (len == insn.length) {

			DUMP_BYTES(RET, ((u8*)wr_addr),  len, "%px: orig: ", addr);

			DUMP_BYTES(RET, ((u8*)addr),  len, "%px: orig: ", addr);

			DUMP_BYTES(RET, ((u8*)bytes), len, "%px: repl: ", addr);

			text_poke_early(wr_addr, bytes, len);

			text_poke_early(addr, bytes, len);

		}

	}

}

#else

void __init_or_module noinline apply_returns(s32 *start, s32 *end,

					     struct module *mod) { }

void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }

#endif /* CONFIG_MITIGATION_RETHUNK */



#else /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */



void __init_or_module noinline apply_retpolines(s32 *start, s32 *end,

						struct module *mod) { }

void __init_or_module noinline apply_returns(s32 *start, s32 *end,

					     struct module *mod) { }

void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }

void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }



#endif /* CONFIG_MITIGATION_RETPOLINE && CONFIG_OBJTOOL */



#ifdef CONFIG_X86_KERNEL_IBT



static void poison_cfi(void *addr, void *wr_addr);

static void poison_cfi(void *addr);



static void __init_or_module poison_endbr(void *addr, void *wr_addr, bool warn)

static void __init_or_module poison_endbr(void *addr, bool warn)

{

	u32 endbr, poison = gen_endbr_poison();



	if (WARN_ON_ONCE(get_kernel_nofault(endbr, wr_addr)))

	if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))

		return;



	if (!is_endbr(endbr)) {
@@ -889,7 +873,7 @@ static void __init_or_module poison_endbr(void *addr, void *wr_addr, bool warn) 
	 */

	DUMP_BYTES(ENDBR, ((u8*)addr), 4, "%px: orig: ", addr);

	DUMP_BYTES(ENDBR, ((u8*)&poison), 4, "%px: repl: ", addr);

	text_poke_early(wr_addr, &poison, 4);

	text_poke_early(addr, &poison, 4);

}



/*
@@ -898,23 +882,22 @@ static void __init_or_module poison_endbr(void *addr, void *wr_addr, bool warn) 
 * Seal the functions for indirect calls by clobbering the ENDBR instructions

 * and the kCFI hash value.

 */

void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end, struct module *mod)

void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end)

{

	s32 *s;



	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr = module_writable_address(mod, addr);



		poison_endbr(addr, wr_addr, true);

		poison_endbr(addr, true);

		if (IS_ENABLED(CONFIG_FINEIBT))

			poison_cfi(addr - 16, wr_addr - 16);

			poison_cfi(addr - 16);

	}

}



#else



void __init_or_module apply_seal_endbr(s32 *start, s32 *end, struct module *mod) { }

void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }



#endif /* CONFIG_X86_KERNEL_IBT */
@@ -1136,7 +1119,7 @@ static u32 decode_caller_hash(void *addr) 
}



/* .retpoline_sites */

static int cfi_disable_callers(s32 *start, s32 *end, struct module *mod)

static int cfi_disable_callers(s32 *start, s32 *end)

{

	/*

	 * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate
@@ -1148,23 +1131,20 @@ static int cfi_disable_callers(s32 *start, s32 *end, struct module *mod) 


	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr;

		u32 hash;



		addr -= fineibt_caller_size;

		wr_addr = module_writable_address(mod, addr);

		hash = decode_caller_hash(wr_addr);



		hash = decode_caller_hash(addr);

		if (!hash) /* nocfi callers */

			continue;



		text_poke_early(wr_addr, jmp, 2);

		text_poke_early(addr, jmp, 2);

	}



	return 0;

}



static int cfi_enable_callers(s32 *start, s32 *end, struct module *mod)

static int cfi_enable_callers(s32 *start, s32 *end)

{

	/*

	 * Re-enable kCFI, undo what cfi_disable_callers() did.
@@ -1174,115 +1154,106 @@ static int cfi_enable_callers(s32 *start, s32 *end, struct module *mod) 


	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr;

		u32 hash;



		addr -= fineibt_caller_size;

		wr_addr = module_writable_address(mod, addr);

		hash = decode_caller_hash(wr_addr);

		hash = decode_caller_hash(addr);

		if (!hash) /* nocfi callers */

			continue;



		text_poke_early(wr_addr, mov, 2);

		text_poke_early(addr, mov, 2);

	}



	return 0;

}



/* .cfi_sites */

static int cfi_rand_preamble(s32 *start, s32 *end, struct module *mod)

static int cfi_rand_preamble(s32 *start, s32 *end)

{

	s32 *s;



	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr = module_writable_address(mod, addr);

		u32 hash;



		hash = decode_preamble_hash(wr_addr);

		hash = decode_preamble_hash(addr);

		if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",

			 addr, addr, 5, addr))

			return -EINVAL;



		hash = cfi_rehash(hash);

		text_poke_early(wr_addr + 1, &hash, 4);

		text_poke_early(addr + 1, &hash, 4);

	}



	return 0;

}



static int cfi_rewrite_preamble(s32 *start, s32 *end, struct module *mod)

static int cfi_rewrite_preamble(s32 *start, s32 *end)

{

	s32 *s;



	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr = module_writable_address(mod, addr);

		u32 hash;



		hash = decode_preamble_hash(wr_addr);

		hash = decode_preamble_hash(addr);

		if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",

			 addr, addr, 5, addr))

			return -EINVAL;



		text_poke_early(wr_addr, fineibt_preamble_start, fineibt_preamble_size);

		WARN_ON(*(u32 *)(wr_addr + fineibt_preamble_hash) != 0x12345678);

		text_poke_early(wr_addr + fineibt_preamble_hash, &hash, 4);

		text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);

		WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678);

		text_poke_early(addr + fineibt_preamble_hash, &hash, 4);

	}



	return 0;

}



static void cfi_rewrite_endbr(s32 *start, s32 *end, struct module *mod)

static void cfi_rewrite_endbr(s32 *start, s32 *end)

{

	s32 *s;



	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr = module_writable_address(mod, addr);



		poison_endbr(addr + 16, wr_addr + 16, false);

		poison_endbr(addr+16, false);

	}

}



/* .retpoline_sites */

static int cfi_rand_callers(s32 *start, s32 *end, struct module *mod)

static int cfi_rand_callers(s32 *start, s32 *end)

{

	s32 *s;



	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr;

		u32 hash;



		addr -= fineibt_caller_size;

		wr_addr = module_writable_address(mod, addr);

		hash = decode_caller_hash(wr_addr);

		hash = decode_caller_hash(addr);

		if (hash) {

			hash = -cfi_rehash(hash);

			text_poke_early(wr_addr + 2, &hash, 4);

			text_poke_early(addr + 2, &hash, 4);

		}

	}



	return 0;

}



static int cfi_rewrite_callers(s32 *start, s32 *end, struct module *mod)

static int cfi_rewrite_callers(s32 *start, s32 *end)

{

	s32 *s;



	for (s = start; s < end; s++) {

		void *addr = (void *)s + *s;

		void *wr_addr;

		u32 hash;



		addr -= fineibt_caller_size;

		wr_addr = module_writable_address(mod, addr);

		hash = decode_caller_hash(wr_addr);

		hash = decode_caller_hash(addr);

		if (hash) {

			text_poke_early(wr_addr, fineibt_caller_start, fineibt_caller_size);

			WARN_ON(*(u32 *)(wr_addr + fineibt_caller_hash) != 0x12345678);

			text_poke_early(wr_addr + fineibt_caller_hash, &hash, 4);

			text_poke_early(addr, fineibt_caller_start, fineibt_caller_size);

			WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678);

			text_poke_early(addr + fineibt_caller_hash, &hash, 4);

		}

		/* rely on apply_retpolines() */

	}
@@ -1291,9 +1262,8 @@ static int cfi_rewrite_callers(s32 *start, s32 *end, struct module *mod) 
}



static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,

			    s32 *start_cfi, s32 *end_cfi, struct module *mod)

			    s32 *start_cfi, s32 *end_cfi, bool builtin)

{

	bool builtin = mod ? false : true;

	int ret;



	if (WARN_ONCE(fineibt_preamble_size != 16,
@@ -1311,7 +1281,7 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 
	 * rewrite them. This disables all CFI. If this succeeds but any of the

	 * later stages fails, we're without CFI.

	 */

	ret = cfi_disable_callers(start_retpoline, end_retpoline, mod);

	ret = cfi_disable_callers(start_retpoline, end_retpoline);

	if (ret)

		goto err;
@@ -1322,11 +1292,11 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 
			cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);

		}



		ret = cfi_rand_preamble(start_cfi, end_cfi, mod);

		ret = cfi_rand_preamble(start_cfi, end_cfi);

		if (ret)

			goto err;



		ret = cfi_rand_callers(start_retpoline, end_retpoline, mod);

		ret = cfi_rand_callers(start_retpoline, end_retpoline);

		if (ret)

			goto err;

	}
@@ -1338,7 +1308,7 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 
		return;



	case CFI_KCFI:

		ret = cfi_enable_callers(start_retpoline, end_retpoline, mod);

		ret = cfi_enable_callers(start_retpoline, end_retpoline);

		if (ret)

			goto err;
@@ -1348,17 +1318,17 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 


	case CFI_FINEIBT:

		/* place the FineIBT preamble at func()-16 */

		ret = cfi_rewrite_preamble(start_cfi, end_cfi, mod);

		ret = cfi_rewrite_preamble(start_cfi, end_cfi);

		if (ret)

			goto err;



		/* rewrite the callers to target func()-16 */

		ret = cfi_rewrite_callers(start_retpoline, end_retpoline, mod);

		ret = cfi_rewrite_callers(start_retpoline, end_retpoline);

		if (ret)

			goto err;



		/* now that nobody targets func()+0, remove ENDBR there */

		cfi_rewrite_endbr(start_cfi, end_cfi, mod);

		cfi_rewrite_endbr(start_cfi, end_cfi);



		if (builtin)

			pr_info("Using FineIBT CFI\n");
@@ -1377,7 +1347,7 @@ static inline void poison_hash(void *addr) 
	*(u32 *)addr = 0;

}



static void poison_cfi(void *addr, void *wr_addr)

static void poison_cfi(void *addr)

{

	switch (cfi_mode) {

	case CFI_FINEIBT:
@@ -1389,8 +1359,8 @@ static void poison_cfi(void *addr, void *wr_addr) 
		 *	ud2

		 * 1:	nop

		 */

		poison_endbr(addr, wr_addr, false);

		poison_hash(wr_addr + fineibt_preamble_hash);

		poison_endbr(addr, false);

		poison_hash(addr + fineibt_preamble_hash);

		break;



	case CFI_KCFI:
@@ -1399,7 +1369,7 @@ static void poison_cfi(void *addr, void *wr_addr) 
		 *	movl	$0, %eax

		 *	.skip	11, 0x90

		 */

		poison_hash(wr_addr + 1);

		poison_hash(addr + 1);

		break;



	default:
@@ -1410,21 +1380,22 @@ static void poison_cfi(void *addr, void *wr_addr) 
#else



static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,

			    s32 *start_cfi, s32 *end_cfi, struct module *mod)

			    s32 *start_cfi, s32 *end_cfi, bool builtin)

{

}



#ifdef CONFIG_X86_KERNEL_IBT

static void poison_cfi(void *addr, void *wr_addr) { }

static void poison_cfi(void *addr) { }

#endif



#endif



void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,

		   s32 *start_cfi, s32 *end_cfi, struct module *mod)

		   s32 *start_cfi, s32 *end_cfi)

{

	return __apply_fineibt(start_retpoline, end_retpoline,

			       start_cfi, end_cfi, mod);

			       start_cfi, end_cfi,

			       /* .builtin = */ false);

}



#ifdef CONFIG_SMP
@@ -1721,16 +1692,16 @@ void __init alternative_instructions(void) 
	paravirt_set_cap();



	__apply_fineibt(__retpoline_sites, __retpoline_sites_end,

			__cfi_sites, __cfi_sites_end, NULL);

			__cfi_sites, __cfi_sites_end, true);



	/*

	 * Rewrite the retpolines, must be done before alternatives since

	 * those can rewrite the retpoline thunks.

	 */

	apply_retpolines(__retpoline_sites, __retpoline_sites_end, NULL);

	apply_returns(__return_sites, __return_sites_end, NULL);

	apply_retpolines(__retpoline_sites, __retpoline_sites_end);

	apply_returns(__return_sites, __return_sites_end);



	apply_alternatives(__alt_instructions, __alt_instructions_end, NULL);

	apply_alternatives(__alt_instructions, __alt_instructions_end);



	/*

	 * Now all calls are established. Apply the call thunks if
@@ -1741,7 +1712,7 @@ void __init alternative_instructions(void) 
	/*

	 * Seal all functions that do not have their address taken.

	 */

	apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end, NULL);

	apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);



#ifdef CONFIG_SMP

	/* Patch to UP if other cpus not imminent. */

arch/x86/kernel/ftrace.c

+14 −16
Original line number Diff line number Diff line
@@ -118,13 +118,10 @@ ftrace_modify_code_direct(unsigned long ip, const char *old_code, 
		return ret;



	/* replace the text with the new text */

	if (ftrace_poke_late) {

	if (ftrace_poke_late)

		text_poke_queue((void *)ip, new_code, MCOUNT_INSN_SIZE, NULL);

	} else {

		mutex_lock(&text_mutex);

		text_poke((void *)ip, new_code, MCOUNT_INSN_SIZE);

		mutex_unlock(&text_mutex);

	}

	else

		text_poke_early((void *)ip, new_code, MCOUNT_INSN_SIZE);

	return 0;

}
@@ -321,7 +318,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) 
	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };

	unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };

	union ftrace_op_code_union op_ptr;

	void *ret;

	int ret;



	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {

		start_offset = (unsigned long)ftrace_regs_caller;
@@ -352,15 +349,15 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) 
	npages = DIV_ROUND_UP(*tramp_size, PAGE_SIZE);



	/* Copy ftrace_caller onto the trampoline memory */

	ret = text_poke_copy(trampoline, (void *)start_offset, size);

	if (WARN_ON(!ret))

	ret = copy_from_kernel_nofault(trampoline, (void *)start_offset, size);

	if (WARN_ON(ret < 0))

		goto fail;



	ip = trampoline + size;

	if (cpu_feature_enabled(X86_FEATURE_RETHUNK))

		__text_gen_insn(ip, JMP32_INSN_OPCODE, ip, x86_return_thunk, JMP32_INSN_SIZE);

	else

		text_poke_copy(ip, retq, sizeof(retq));

		memcpy(ip, retq, sizeof(retq));



	/* No need to test direct calls on created trampolines */

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
@@ -368,7 +365,8 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) 
		ip = trampoline + (jmp_offset - start_offset);

		if (WARN_ON(*(char *)ip != 0x75))

			goto fail;

		if (!text_poke_copy(ip, x86_nops[2], 2))

		ret = copy_from_kernel_nofault(ip, x86_nops[2], 2);

		if (ret < 0)

			goto fail;

	}
@@ -381,7 +379,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) 
	 */



	ptr = (unsigned long *)(trampoline + size + RET_SIZE);

	text_poke_copy(ptr, &ops, sizeof(unsigned long));

	*ptr = (unsigned long)ops;



	op_offset -= start_offset;

	memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);
@@ -397,7 +395,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) 
	op_ptr.offset = offset;



	/* put in the new offset to the ftrace_ops */

	text_poke_copy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);

	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);



	/* put in the call to the function */

	mutex_lock(&text_mutex);
@@ -407,9 +405,9 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) 
	 * the depth accounting before the call already.

	 */

	dest = ftrace_ops_get_func(ops);

	text_poke_copy_locked(trampoline + call_offset,

	memcpy(trampoline + call_offset,

	       text_gen_insn(CALL_INSN_OPCODE, trampoline + call_offset, dest),

	      CALL_INSN_SIZE, false);

	       CALL_INSN_SIZE);

	mutex_unlock(&text_mutex);



	/* ALLOC_TRAMP flags lets us know we created it */