Commit d42e6c20 authored by Ada Couprie Diaz's avatar Ada Couprie Diaz Committed by Will Deacon
Browse files

arm64/entry: Mask DAIF in cpu_switch_to(), call_on_irq_stack() 



`cpu_switch_to()` and `call_on_irq_stack()` manipulate SP to change
to different stacks along with the Shadow Call Stack if it is enabled.
Those two stack changes cannot be done atomically and both functions
can be interrupted by SErrors or Debug Exceptions which, though unlikely,
is very much broken : if interrupted, we can end up with mismatched stacks
and Shadow Call Stack leading to clobbered stacks.

In `cpu_switch_to()`, it can happen when SP_EL0 points to the new task,
but x18 stills points to the old task's SCS. When the interrupt handler
tries to save the task's SCS pointer, it will save the old task
SCS pointer (x18) into the new task struct (pointed to by SP_EL0),
clobbering it.

In `call_on_irq_stack()`, it can happen when switching from the task stack
to the IRQ stack and when switching back. In both cases, we can be
interrupted when the SCS pointer points to the IRQ SCS, but SP points to
the task stack. The nested interrupt handler pushes its return addresses
on the IRQ SCS. It then detects that SP points to the task stack,
calls `call_on_irq_stack()` and clobbers the task SCS pointer with
the IRQ SCS pointer, which it will also use !

This leads to tasks returning to addresses on the wrong SCS,
or even on the IRQ SCS, triggering kernel panics via CONFIG_VMAP_STACK
or FPAC if enabled.

This is possible on a default config, but unlikely.
However, when enabling CONFIG_ARM64_PSEUDO_NMI, DAIF is unmasked and
instead the GIC is responsible for filtering what interrupts the CPU
should receive based on priority.
Given the goal of emulating NMIs, pseudo-NMIs can be received by the CPU
even in `cpu_switch_to()` and `call_on_irq_stack()`, possibly *very*
frequently depending on the system configuration and workload, leading
to unpredictable kernel panics.

Completely mask DAIF in `cpu_switch_to()` and restore it when returning.
Do the same in `call_on_irq_stack()`, but restore and mask around
the branch.
Mask DAIF even if CONFIG_SHADOW_CALL_STACK is not enabled for consistency
of behaviour between all configurations.

Introduce and use an assembly macro for saving and masking DAIF,
as the existing one saves but only masks IF.

Cc: <stable@vger.kernel.org>
Signed-off-by: default avatarAda Couprie Diaz <ada.coupriediaz@arm.com>
Reported-by: default avatarCristian Prundeanu <cpru@amazon.com>
Fixes: 59b37fe5 ("arm64: Stash shadow stack pointer in the task struct on interrupt")
Tested-by: default avatarCristian Prundeanu <cpru@amazon.com>
Acked-by: default avatarWill Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20250718142814.133329-1-ada.coupriediaz@arm.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent ab161221

arch/arm64/include/asm/assembler.h

+5 −0
Original line number Diff line number Diff line
@@ -41,6 +41,11 @@ 
/*

 * Save/restore interrupts.

 */

	.macro save_and_disable_daif, flags

	mrs	\flags, daif

	msr	daifset, #0xf

	.endm



	.macro	save_and_disable_irq, flags

	mrs	\flags, daif

	msr	daifset, #3

arch/arm64/kernel/entry.S

+6 −0
Original line number Diff line number Diff line
@@ -825,6 +825,7 @@ SYM_CODE_END(__bp_harden_el1_vectors) 
 *

 */

SYM_FUNC_START(cpu_switch_to)

	save_and_disable_daif x11

	mov	x10, #THREAD_CPU_CONTEXT

	add	x8, x0, x10

	mov	x9, sp
@@ -848,6 +849,7 @@ SYM_FUNC_START(cpu_switch_to) 
	ptrauth_keys_install_kernel x1, x8, x9, x10

	scs_save x0

	scs_load_current

	restore_irq x11

	ret

SYM_FUNC_END(cpu_switch_to)

NOKPROBE(cpu_switch_to)
@@ -874,6 +876,7 @@ NOKPROBE(ret_from_fork) 
 * Calls func(regs) using this CPU's irq stack and shadow irq stack.

 */

SYM_FUNC_START(call_on_irq_stack)

	save_and_disable_daif x9

#ifdef CONFIG_SHADOW_CALL_STACK

	get_current_task x16

	scs_save x16
@@ -888,8 +891,10 @@ SYM_FUNC_START(call_on_irq_stack) 


	/* Move to the new stack and call the function there */

	add	sp, x16, #IRQ_STACK_SIZE

	restore_irq x9

	blr	x1



	save_and_disable_daif x9

	/*

	 * Restore the SP from the FP, and restore the FP and LR from the frame

	 * record.
@@ -897,6 +902,7 @@ SYM_FUNC_START(call_on_irq_stack) 
	mov	sp, x29

	ldp	x29, x30, [sp], #16

	scs_load_current

	restore_irq x9

	ret

SYM_FUNC_END(call_on_irq_stack)

NOKPROBE(call_on_irq_stack)