Commit e3bfa385 authored by Ingo Molnar's avatar Ingo Molnar
Browse files

x86/fpu: Convert task_struct::thread.fpu accesses to use x86_task_fpu() 



This will make the removal of the task_struct::thread.fpu array
easier.

No change in functionality - code generated before and after this
commit is identical on x86-defconfig:

  kepler:~/tip> diff -up vmlinux.before.asm vmlinux.after.asm
  kepler:~/tip>

Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Chang S. Bae <chang.seok.bae@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Link: https://lore.kernel.org/r/20250409211127.3544993-3-mingo@kernel.org
parent 77fbcced

arch/x86/include/asm/fpu/sched.h

+1 −1
Original line number Diff line number Diff line
@@ -41,7 +41,7 @@ static inline void switch_fpu_prepare(struct task_struct *old, int cpu) 
{

	if (cpu_feature_enabled(X86_FEATURE_FPU) &&

	    !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) {

		struct fpu *old_fpu = &old->thread.fpu;

		struct fpu *old_fpu = x86_task_fpu(old);



		save_fpregs_to_fpstate(old_fpu);

		/*

arch/x86/kernel/fpu/context.h

+2 −2
Original line number Diff line number Diff line
@@ -53,7 +53,7 @@ static inline void fpregs_activate(struct fpu *fpu) 
/* Internal helper for switch_fpu_return() and signal frame setup */

static inline void fpregs_restore_userregs(void)

{

	struct fpu *fpu = &current->thread.fpu;

	struct fpu *fpu = x86_task_fpu(current);

	int cpu = smp_processor_id();



	if (WARN_ON_ONCE(current->flags & (PF_KTHREAD | PF_USER_WORKER)))
@@ -67,7 +67,7 @@ static inline void fpregs_restore_userregs(void) 
		 * If PKRU is enabled, then the PKRU value is already

		 * correct because it was either set in switch_to() or in

		 * flush_thread(). So it is excluded because it might be

		 * not up to date in current->thread.fpu.xsave state.

		 * not up to date in current->thread.fpu->xsave state.

		 *

		 * XFD state is handled in restore_fpregs_from_fpstate().

		 */

arch/x86/kernel/fpu/core.c

+15 −15
Original line number Diff line number Diff line
@@ -211,7 +211,7 @@ static void fpu_init_guest_permissions(struct fpu_guest *gfpu) 
		return;



	spin_lock_irq(&current->sighand->siglock);

	fpuperm = &current->group_leader->thread.fpu.guest_perm;

	fpuperm = &x86_task_fpu(current->group_leader)->guest_perm;

	perm = fpuperm->__state_perm;



	/* First fpstate allocation locks down permissions. */
@@ -323,7 +323,7 @@ EXPORT_SYMBOL_GPL(fpu_update_guest_xfd); 
 */

void fpu_sync_guest_vmexit_xfd_state(void)

{

	struct fpstate *fps = current->thread.fpu.fpstate;

	struct fpstate *fps = x86_task_fpu(current)->fpstate;



	lockdep_assert_irqs_disabled();

	if (fpu_state_size_dynamic()) {
@@ -337,7 +337,7 @@ EXPORT_SYMBOL_GPL(fpu_sync_guest_vmexit_xfd_state); 
int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)

{

	struct fpstate *guest_fps = guest_fpu->fpstate;

	struct fpu *fpu = &current->thread.fpu;

	struct fpu *fpu = x86_task_fpu(current);

	struct fpstate *cur_fps = fpu->fpstate;



	fpregs_lock();
@@ -438,7 +438,7 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask) 
	if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) &&

	    !test_thread_flag(TIF_NEED_FPU_LOAD)) {

		set_thread_flag(TIF_NEED_FPU_LOAD);

		save_fpregs_to_fpstate(&current->thread.fpu);

		save_fpregs_to_fpstate(x86_task_fpu(current));

	}

	__cpu_invalidate_fpregs_state();
@@ -467,7 +467,7 @@ EXPORT_SYMBOL_GPL(kernel_fpu_end); 
 */

void fpu_sync_fpstate(struct fpu *fpu)

{

	WARN_ON_FPU(fpu != &current->thread.fpu);

	WARN_ON_FPU(fpu != x86_task_fpu(current));



	fpregs_lock();

	trace_x86_fpu_before_save(fpu);
@@ -552,7 +552,7 @@ void fpstate_reset(struct fpu *fpu) 
static inline void fpu_inherit_perms(struct fpu *dst_fpu)

{

	if (fpu_state_size_dynamic()) {

		struct fpu *src_fpu = &current->group_leader->thread.fpu;

		struct fpu *src_fpu = x86_task_fpu(current->group_leader);



		spin_lock_irq(&current->sighand->siglock);

		/* Fork also inherits the permissions of the parent */
@@ -572,7 +572,7 @@ static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) 
	if (!ssp)

		return 0;



	xstate = get_xsave_addr(&dst->thread.fpu.fpstate->regs.xsave,

	xstate = get_xsave_addr(&x86_task_fpu(dst)->fpstate->regs.xsave,

				XFEATURE_CET_USER);



	/*
@@ -593,8 +593,8 @@ static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) 
int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,

	      unsigned long ssp)

{

	struct fpu *src_fpu = &current->thread.fpu;

	struct fpu *dst_fpu = &dst->thread.fpu;

	struct fpu *src_fpu = x86_task_fpu(current);

	struct fpu *dst_fpu = x86_task_fpu(dst);



	/* The new task's FPU state cannot be valid in the hardware. */

	dst_fpu->last_cpu = -1;
@@ -686,7 +686,7 @@ void fpu__drop(struct fpu *fpu) 
{

	preempt_disable();



	if (fpu == &current->thread.fpu) {

	if (fpu == x86_task_fpu(current)) {

		/* Ignore delayed exceptions from user space */

		asm volatile("1: fwait\n"

			     "2:\n"
@@ -720,7 +720,7 @@ static inline void restore_fpregs_from_init_fpstate(u64 features_mask) 
 */

static void fpu_reset_fpregs(void)

{

	struct fpu *fpu = &current->thread.fpu;

	struct fpu *fpu = x86_task_fpu(current);



	fpregs_lock();

	__fpu_invalidate_fpregs_state(fpu);
@@ -749,7 +749,7 @@ static void fpu_reset_fpregs(void) 
 */

void fpu__clear_user_states(struct fpu *fpu)

{

	WARN_ON_FPU(fpu != &current->thread.fpu);

	WARN_ON_FPU(fpu != x86_task_fpu(current));



	fpregs_lock();

	if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
@@ -782,7 +782,7 @@ void fpu__clear_user_states(struct fpu *fpu) 


void fpu_flush_thread(void)

{

	fpstate_reset(&current->thread.fpu);

	fpstate_reset(x86_task_fpu(current));

	fpu_reset_fpregs();

}

/*
@@ -823,7 +823,7 @@ void fpregs_lock_and_load(void) 
 */

void fpregs_assert_state_consistent(void)

{

	struct fpu *fpu = &current->thread.fpu;

	struct fpu *fpu = x86_task_fpu(current);



	if (test_thread_flag(TIF_NEED_FPU_LOAD))

		return;
@@ -835,7 +835,7 @@ EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent); 


void fpregs_mark_activate(void)

{

	struct fpu *fpu = &current->thread.fpu;

	struct fpu *fpu = x86_task_fpu(current);



	fpregs_activate(fpu);

	fpu->last_cpu = smp_processor_id();

arch/x86/kernel/fpu/init.c

+4 −4
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ static void fpu__init_cpu_generic(void) 
	/* Flush out any pending x87 state: */

#ifdef CONFIG_MATH_EMULATION

	if (!boot_cpu_has(X86_FEATURE_FPU))

		fpstate_init_soft(&current->thread.fpu.fpstate->regs.soft);

		fpstate_init_soft(&x86_task_fpu(current)->fpstate->regs.soft);

	else

#endif

		asm volatile ("fninit");
@@ -154,7 +154,7 @@ static void __init fpu__init_task_struct_size(void) 
	 * Subtract off the static size of the register state.

	 * It potentially has a bunch of padding.

	 */

	task_size -= sizeof(current->thread.fpu.__fpstate.regs);

	task_size -= sizeof(union fpregs_state);



	/*

	 * Add back the dynamically-calculated register state
@@ -204,7 +204,7 @@ static void __init fpu__init_system_xstate_size_legacy(void) 
	fpu_kernel_cfg.default_size = size;

	fpu_user_cfg.max_size = size;

	fpu_user_cfg.default_size = size;

	fpstate_reset(&current->thread.fpu);

	fpstate_reset(x86_task_fpu(current));

}



/*
@@ -213,7 +213,7 @@ static void __init fpu__init_system_xstate_size_legacy(void) 
 */

void __init fpu__init_system(void)

{

	fpstate_reset(&current->thread.fpu);

	fpstate_reset(x86_task_fpu(current));

	fpu__init_system_early_generic();



	/*

arch/x86/kernel/fpu/regset.c

+11 −11
Original line number Diff line number Diff line
@@ -45,7 +45,7 @@ int regset_xregset_fpregs_active(struct task_struct *target, const struct user_r 
 */

static void sync_fpstate(struct fpu *fpu)

{

	if (fpu == &current->thread.fpu)

	if (fpu == x86_task_fpu(current))

		fpu_sync_fpstate(fpu);

}
@@ -63,7 +63,7 @@ static void fpu_force_restore(struct fpu *fpu) 
	 * Only stopped child tasks can be used to modify the FPU

	 * state in the fpstate buffer:

	 */

	WARN_ON_FPU(fpu == &current->thread.fpu);

	WARN_ON_FPU(fpu == x86_task_fpu(current));



	__fpu_invalidate_fpregs_state(fpu);

}
@@ -71,7 +71,7 @@ static void fpu_force_restore(struct fpu *fpu) 
int xfpregs_get(struct task_struct *target, const struct user_regset *regset,

		struct membuf to)

{

	struct fpu *fpu = &target->thread.fpu;

	struct fpu *fpu = x86_task_fpu(target);



	if (!cpu_feature_enabled(X86_FEATURE_FXSR))

		return -ENODEV;
@@ -91,7 +91,7 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, 
		unsigned int pos, unsigned int count,

		const void *kbuf, const void __user *ubuf)

{

	struct fpu *fpu = &target->thread.fpu;

	struct fpu *fpu = x86_task_fpu(target);

	struct fxregs_state newstate;

	int ret;
@@ -133,7 +133,7 @@ int xstateregs_get(struct task_struct *target, const struct user_regset *regset, 
	if (!cpu_feature_enabled(X86_FEATURE_XSAVE))

		return -ENODEV;



	sync_fpstate(&target->thread.fpu);

	sync_fpstate(x86_task_fpu(target));



	copy_xstate_to_uabi_buf(to, target, XSTATE_COPY_XSAVE);

	return 0;
@@ -143,7 +143,7 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset, 
		  unsigned int pos, unsigned int count,

		  const void *kbuf, const void __user *ubuf)

{

	struct fpu *fpu = &target->thread.fpu;

	struct fpu *fpu = x86_task_fpu(target);

	struct xregs_state *tmpbuf = NULL;

	int ret;
@@ -187,7 +187,7 @@ int ssp_active(struct task_struct *target, const struct user_regset *regset) 
int ssp_get(struct task_struct *target, const struct user_regset *regset,

	    struct membuf to)

{

	struct fpu *fpu = &target->thread.fpu;

	struct fpu *fpu = x86_task_fpu(target);

	struct cet_user_state *cetregs;



	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
@@ -214,7 +214,7 @@ int ssp_set(struct task_struct *target, const struct user_regset *regset, 
	    unsigned int pos, unsigned int count,

	    const void *kbuf, const void __user *ubuf)

{

	struct fpu *fpu = &target->thread.fpu;

	struct fpu *fpu = x86_task_fpu(target);

	struct xregs_state *xsave = &fpu->fpstate->regs.xsave;

	struct cet_user_state *cetregs;

	unsigned long user_ssp;
@@ -368,7 +368,7 @@ static void __convert_from_fxsr(struct user_i387_ia32_struct *env, 
void

convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)

{

	__convert_from_fxsr(env, tsk, &tsk->thread.fpu.fpstate->regs.fxsave);

	__convert_from_fxsr(env, tsk, &x86_task_fpu(tsk)->fpstate->regs.fxsave);

}



void convert_to_fxsr(struct fxregs_state *fxsave,
@@ -401,7 +401,7 @@ void convert_to_fxsr(struct fxregs_state *fxsave, 
int fpregs_get(struct task_struct *target, const struct user_regset *regset,

	       struct membuf to)

{

	struct fpu *fpu = &target->thread.fpu;

	struct fpu *fpu = x86_task_fpu(target);

	struct user_i387_ia32_struct env;

	struct fxregs_state fxsave, *fx;
@@ -433,7 +433,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset, 
	       unsigned int pos, unsigned int count,

	       const void *kbuf, const void __user *ubuf)

{

	struct fpu *fpu = &target->thread.fpu;

	struct fpu *fpu = x86_task_fpu(target);

	struct user_i387_ia32_struct env;

	int ret;