Commit 4461438a authored by Josh Poimboeuf's avatar Josh Poimboeuf Committed by Borislav Petkov (AMD)
Browse files

x86/retpoline: Ensure default return thunk isn't used at runtime 



Make sure the default return thunk is not used after all return
instructions have been patched by the alternatives because the default
return thunk is insufficient when it comes to mitigating Retbleed or
SRSO.

Fix based on an earlier version by David Kaplan <david.kaplan@amd.com>.

  [ bp: Fix the compilation error of warn_thunk_thunk being an invisible
        symbol, hoist thunk macro into calling.h ]

Signed-off-by: default avatarJosh Poimboeuf <jpoimboe@kernel.org>
Co-developed-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20231010171020.462211-4-david.kaplan@amd.com
Link: https://lore.kernel.org/r/20240104132446.GEZZaxnrIgIyat0pqf@fat_crate.local
parent 0911b8c5

arch/x86/entry/calling.h

+60 −0
Original line number Diff line number Diff line
@@ -426,3 +426,63 @@ For 32-bit we have the following conventions - kernel is built with 
.endm



#endif /* CONFIG_SMP */



#ifdef CONFIG_X86_64



/* rdi:	arg1 ... normal C conventions. rax is saved/restored. */

.macro THUNK name, func

SYM_FUNC_START(\name)

	pushq %rbp

	movq %rsp, %rbp



	pushq %rdi

	pushq %rsi

	pushq %rdx

	pushq %rcx

	pushq %rax

	pushq %r8

	pushq %r9

	pushq %r10

	pushq %r11



	call \func



	popq %r11

	popq %r10

	popq %r9

	popq %r8

	popq %rax

	popq %rcx

	popq %rdx

	popq %rsi

	popq %rdi

	popq %rbp

	RET

SYM_FUNC_END(\name)

	_ASM_NOKPROBE(\name)

.endm



#else /* CONFIG_X86_32 */



/* put return address in eax (arg1) */

.macro THUNK name, func, put_ret_addr_in_eax=0

SYM_CODE_START_NOALIGN(\name)

	pushl %eax

	pushl %ecx

	pushl %edx



	.if \put_ret_addr_in_eax

	/* Place EIP in the arg1 */

	movl 3*4(%esp), %eax

	.endif



	call \func

	popl %edx

	popl %ecx

	popl %eax

	RET

	_ASM_NOKPROBE(\name)

SYM_CODE_END(\name)

	.endm



#endif

arch/x86/entry/entry.S

+4 −0
Original line number Diff line number Diff line
@@ -7,6 +7,8 @@ 
#include <linux/linkage.h>

#include <asm/msr-index.h>



#include "calling.h"



.pushsection .noinstr.text, "ax"



SYM_FUNC_START(entry_ibpb)
@@ -20,3 +22,5 @@ SYM_FUNC_END(entry_ibpb) 
EXPORT_SYMBOL_GPL(entry_ibpb);



.popsection



THUNK warn_thunk_thunk, __warn_thunk

arch/x86/entry/thunk_32.S

+8 −26
Original line number Diff line number Diff line
@@ -4,30 +4,12 @@ 
 * Copyright 2008 by Steven Rostedt, Red Hat, Inc

 *  (inspired by Andi Kleen's thunk_64.S)

 */



#include <linux/export.h>

#include <linux/linkage.h>

#include <asm/asm.h>



	/* put return address in eax (arg1) */

	.macro THUNK name, func, put_ret_addr_in_eax=0

SYM_CODE_START_NOALIGN(\name)

	pushl %eax

	pushl %ecx

	pushl %edx



	.if \put_ret_addr_in_eax

	/* Place EIP in the arg1 */

	movl 3*4(%esp), %eax

	.endif



	call \func

	popl %edx

	popl %ecx

	popl %eax

	RET

	_ASM_NOKPROBE(\name)

SYM_CODE_END(\name)

	.endm

#include "calling.h"



THUNK preempt_schedule_thunk, preempt_schedule

THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace

arch/x86/entry/thunk_64.S

+0 −33
Original line number Diff line number Diff line
@@ -9,39 +9,6 @@ 
#include "calling.h"

#include <asm/asm.h>



	/* rdi:	arg1 ... normal C conventions. rax is saved/restored. */

	.macro THUNK name, func

SYM_FUNC_START(\name)

	pushq %rbp

	movq %rsp, %rbp



	pushq %rdi

	pushq %rsi

	pushq %rdx

	pushq %rcx

	pushq %rax

	pushq %r8

	pushq %r9

	pushq %r10

	pushq %r11



	call \func



	popq %r11

	popq %r10

	popq %r9

	popq %r8

	popq %rax

	popq %rcx

	popq %rdx

	popq %rsi

	popq %rdi

	popq %rbp

	RET

SYM_FUNC_END(\name)

	_ASM_NOKPROBE(\name)

	.endm



THUNK preempt_schedule_thunk, preempt_schedule

THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace

EXPORT_SYMBOL(preempt_schedule_thunk)

arch/x86/include/asm/nospec-branch.h

+2 −0
Original line number Diff line number Diff line
@@ -357,6 +357,8 @@ extern void entry_ibpb(void); 


extern void (*x86_return_thunk)(void);



extern void __warn_thunk(void);



#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING

extern void call_depth_return_thunk(void);