Commit c5162137 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

x86/entry: Optimize common_interrupt_return() 



The code in common_interrupt_return() does a bunch of unconditional
work that is really only needed on PTI kernels. Specifically it
unconditionally copies the IRET frame back onto the entry stack,
swizzles onto the entry stack and does IRET from there.

However, without PTI we can simply IRET from whatever stack we're on.

  ivb-ep, mitigations=off, gettid-1m:

  PRE:
       140,118,538      cycles:k                                                      ( +-  0.01% )
       236,692,878      instructions:k            #    1.69  insn per cycle           ( +-  0.00% )

  POST:
       140,026,608      cycles:k                                                      ( +-  0.01% )
       236,696,176      instructions:k            #    1.69  insn per cycle           ( +-  0.00% )

(this is with --repeat 100 and the run-to-run variance is bigger than
the difference shown)

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20231120143626.638107480@infradead.org
parent 98b1cc82

arch/x86/entry/calling.h

+9 −3
Original line number Diff line number Diff line
@@ -175,8 +175,7 @@ For 32-bit we have the following conventions - kernel is built with 
#define THIS_CPU_user_pcid_flush_mask   \

	PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask



.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req

	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI

.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req

	mov	%cr3, \scratch_reg



	ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
@@ -206,13 +205,20 @@ For 32-bit we have the following conventions - kernel is built with 
	/* Flip the PGD to the user version */

	orq     $(PTI_USER_PGTABLE_MASK), \scratch_reg

	mov	\scratch_reg, %cr3

.endm



.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req

	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI

	SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2

.Lend_\@:

.endm



.macro SWITCH_TO_USER_CR3_STACK	scratch_reg:req

	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI

	pushq	%rax

	SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax

	SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax

	popq	%rax

.Lend_\@:

.endm



.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req

arch/x86/entry/entry_64.S

+15 −2
Original line number Diff line number Diff line
@@ -569,7 +569,18 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) 
#ifdef CONFIG_XEN_PV

	ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV

#endif

#ifdef CONFIG_PAGE_TABLE_ISOLATION

	ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI

#endif



	STACKLEAK_ERASE

	POP_REGS

	add	$8, %rsp	/* orig_ax */

	swapgs

	jmp	.Lnative_iret



#ifdef CONFIG_PAGE_TABLE_ISOLATION

.Lpti_restore_regs_and_return_to_usermode:

	POP_REGS pop_rdi=0



	/*
@@ -596,13 +607,15 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) 
	 */

	STACKLEAK_ERASE_NOCLOBBER



	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi

	push	%rax

	SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax

	pop	%rax



	/* Restore RDI. */

	popq	%rdi

	swapgs

	jmp	.Lnative_iret



#endif



SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)

#ifdef CONFIG_DEBUG_ENTRY