Commit 2fdbcf71 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'x86-entry-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 entry updates from Ingo Molnar:

 - Optimize common_interrupt_return()

 - Harden the return-to-user code by making a CONFIG_DEBUG_ENTRY=y check
   unconditional & moving it closer to the IRET.

* tag 'x86-entry-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/entry: Harden return-to-user
  x86/entry: Optimize common_interrupt_return()
parents 33677aef 1e4d3001

arch/x86/entry/calling.h

+9 −3
Original line number Diff line number Diff line
@@ -175,8 +175,7 @@ For 32-bit we have the following conventions - kernel is built with 
#define THIS_CPU_user_pcid_flush_mask   \

	PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask



.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req

	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI

.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req

	mov	%cr3, \scratch_reg



	ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
@@ -206,13 +205,20 @@ For 32-bit we have the following conventions - kernel is built with 
	/* Flip the PGD to the user version */

	orq     $(PTI_USER_PGTABLE_MASK), \scratch_reg

	mov	\scratch_reg, %cr3

.endm



.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req

	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI

	SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2

.Lend_\@:

.endm



.macro SWITCH_TO_USER_CR3_STACK	scratch_reg:req

	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI

	pushq	%rax

	SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax

	SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax

	popq	%rax

.Lend_\@:

.endm



.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req

arch/x86/entry/entry_64.S

+22 −11
Original line number Diff line number Diff line
@@ -559,17 +559,27 @@ __irqentry_text_end: 
SYM_CODE_START_LOCAL(common_interrupt_return)

SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)

	IBRS_EXIT

#ifdef CONFIG_DEBUG_ENTRY

	/* Assert that pt_regs indicates user mode. */

	testb	$3, CS(%rsp)

	jnz	1f

	ud2

1:

#endif

#ifdef CONFIG_XEN_PV

	ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV

#endif

#ifdef CONFIG_PAGE_TABLE_ISOLATION

	ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI

#endif



	STACKLEAK_ERASE

	POP_REGS

	add	$8, %rsp	/* orig_ax */

	UNWIND_HINT_IRET_REGS



.Lswapgs_and_iret:

	swapgs

	/* Assert that the IRET frame indicates user mode. */

	testb	$3, 8(%rsp)

	jnz	.Lnative_iret

	ud2



#ifdef CONFIG_PAGE_TABLE_ISOLATION

.Lpti_restore_regs_and_return_to_usermode:

	POP_REGS pop_rdi=0



	/*
@@ -596,13 +606,14 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) 
	 */

	STACKLEAK_ERASE_NOCLOBBER



	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi

	push	%rax

	SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax

	pop	%rax



	/* Restore RDI. */

	popq	%rdi

	swapgs

	jmp	.Lnative_iret



	jmp	.Lswapgs_and_iret

#endif



SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)

#ifdef CONFIG_DEBUG_ENTRY