Commit 64f4b8b1 authored by Jinjie Ruan's avatar Jinjie Ruan Committed by Will Deacon
Browse files

arm64: entry: Refactor preempt_schedule_irq() check code 



To align the structure of the code with irqentry_exit_cond_resched()
from the generic entry code, hoist the need_irq_preemption()
and IS_ENABLED() check earlier. And different preemption check functions
are defined based on whether dynamic preemption is enabled.

Signed-off-by: default avatarJinjie Ruan <ruanjinjie@huawei.com>
Reviewed-by: default avatarAda Couprie Diaz <ada.coupriediaz@arm.com>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 3c973c51

arch/arm64/include/asm/preempt.h

+6 −0
Original line number Diff line number Diff line
@@ -85,6 +85,7 @@ static inline bool should_resched(int preempt_offset) 
void preempt_schedule(void);

void preempt_schedule_notrace(void);



void raw_irqentry_exit_cond_resched(void);

#ifdef CONFIG_PREEMPT_DYNAMIC



DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
@@ -92,13 +93,18 @@ void dynamic_preempt_schedule(void); 
#define __preempt_schedule()		dynamic_preempt_schedule()

void dynamic_preempt_schedule_notrace(void);

#define __preempt_schedule_notrace()	dynamic_preempt_schedule_notrace()

void dynamic_irqentry_exit_cond_resched(void);

#define irqentry_exit_cond_resched()	dynamic_irqentry_exit_cond_resched()



#else /* CONFIG_PREEMPT_DYNAMIC */



#define __preempt_schedule()		preempt_schedule()

#define __preempt_schedule_notrace()	preempt_schedule_notrace()

#define irqentry_exit_cond_resched()	raw_irqentry_exit_cond_resched()



#endif /* CONFIG_PREEMPT_DYNAMIC */

#else /* CONFIG_PREEMPTION */

#define irqentry_exit_cond_resched()	{}

#endif /* CONFIG_PREEMPTION */



#endif /* __ASM_PREEMPT_H */

arch/arm64/kernel/entry-common.c

+22 −15
Original line number Diff line number Diff line
@@ -286,19 +286,8 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, 
		lockdep_hardirqs_on(CALLER_ADDR0);

}



#ifdef CONFIG_PREEMPT_DYNAMIC

DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);

#define need_irq_preemption() \

	(static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))

#else

#define need_irq_preemption()	(IS_ENABLED(CONFIG_PREEMPTION))

#endif



static inline bool arm64_preempt_schedule_irq(void)

{

	if (!need_irq_preemption())

		return false;



	/*

	 * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC

	 * priority masking is used the GIC irqchip driver will clear DAIF.IF
@@ -682,6 +671,26 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, 
	arm64_exit_nmi(regs, state);

}



#ifdef CONFIG_PREEMPTION

void raw_irqentry_exit_cond_resched(void)

{

	if (!preempt_count()) {

		if (need_resched() && arm64_preempt_schedule_irq())

			preempt_schedule_irq();

	}

}

#endif



#ifdef CONFIG_PREEMPT_DYNAMIC

DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);

void dynamic_irqentry_exit_cond_resched(void)

{

	if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))

		return;

	raw_irqentry_exit_cond_resched();

}

#endif



static __always_inline void __el1_irq(struct pt_regs *regs,

				      void (*handler)(struct pt_regs *))

{
@@ -693,10 +702,8 @@ static __always_inline void __el1_irq(struct pt_regs *regs, 
	do_interrupt_handler(regs, handler);

	irq_exit_rcu();



	if (!preempt_count() && need_resched()) {

		if (arm64_preempt_schedule_irq())

			preempt_schedule_irq();

	}

	if (IS_ENABLED(CONFIG_PREEMPTION))

		irqentry_exit_cond_resched();



	exit_to_kernel_mode(regs, state);

}