Commit cb7ca40a authored by Ingo Molnar's avatar Ingo Molnar
Browse files

x86/fpu: Make task_struct::thread constant size 



Turn thread.fpu into a pointer. Since most FPU code internals work by passing
around the FPU pointer already, the code generation impact is small.

This allows us to remove the old kludge of task_struct being variable size:

  struct task_struct {

       ...
       /*
        * New fields for task_struct should be added above here, so that
        * they are included in the randomized portion of task_struct.
        */
       randomized_struct_fields_end

       /* CPU-specific state of this task: */
       struct thread_struct            thread;

       /*
        * WARNING: on x86, 'thread_struct' contains a variable-sized
        * structure.  It *MUST* be at the end of 'task_struct'.
        *
        * Do not put anything below here!
        */
  };

... which creates a number of problems, such as requiring thread_struct to be
the last member of the struct - not allowing it to be struct-randomized, etc.

But the primary motivation is to allow the decoupling of task_struct from
hardware details (<asm/processor.h> in particular), and to eventually allow
the per-task infrastructure:

   DECLARE_PER_TASK(type, name);
   ...
   per_task(current, name) = val;

... which requires task_struct to be a constant size struct.

The fpu_thread_struct_whitelist() quirk to hardened usercopy can be removed,
now that the FPU structure is not embedded in the task struct anymore, which
reduces text footprint a bit.

Fixed-by: default avatarOleg Nesterov <oleg@redhat.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Chang S. Bae <chang.seok.bae@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20250409211127.3544993-4-mingo@kernel.org
parent e3bfa385

arch/x86/include/asm/processor.h

+9 −11
Original line number Diff line number Diff line
@@ -516,21 +516,19 @@ struct thread_struct { 
#endif



	/* Floating point and extended processor state */

	struct fpu		fpu;

	/*

	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at

	 * the end.

	 */

	struct fpu		*fpu;

};



#define x86_task_fpu(task) (&(task)->thread.fpu)



extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size);

#define x86_task_fpu(task) ((task)->thread.fpu)



static inline void arch_thread_struct_whitelist(unsigned long *offset,

						unsigned long *size)

/*

 * X86 doesn't need any embedded-FPU-struct quirks:

 */

static inline void

arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size)

{

	fpu_thread_struct_whitelist(offset, size);

	*offset = 0;

	*size = 0;

}



static inline void

arch/x86/kernel/fpu/core.c

+12 −11
Original line number Diff line number Diff line
@@ -593,8 +593,19 @@ static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) 
int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,

	      unsigned long ssp)

{

	/*

	 * We allocate the new FPU structure right after the end of the task struct.

	 * task allocation size already took this into account.

	 *

	 * This is safe because task_struct size is a multiple of cacheline size.

	 */

	struct fpu *src_fpu = x86_task_fpu(current);

	struct fpu *dst_fpu = x86_task_fpu(dst);

	struct fpu *dst_fpu = (void *)dst + sizeof(*dst);



	BUILD_BUG_ON(sizeof(*dst) % SMP_CACHE_BYTES != 0);

	BUG_ON(!src_fpu);



	dst->thread.fpu = dst_fpu;



	/* The new task's FPU state cannot be valid in the hardware. */

	dst_fpu->last_cpu = -1;
@@ -663,16 +674,6 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, 
	return 0;

}



/*

 * Whitelist the FPU register state embedded into task_struct for hardened

 * usercopy.

 */

void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size)

{

	*offset = offsetof(struct thread_struct, fpu.__fpstate.regs);

	*size = fpu_kernel_cfg.default_size;

}



/*

 * Drops current FPU state: deactivates the fpregs and

 * the fpstate. NOTE: it still leaves previous contents

arch/x86/kernel/fpu/init.c

+10 −7
Original line number Diff line number Diff line
@@ -71,8 +71,15 @@ static bool __init fpu__probe_without_cpuid(void) 
	return fsw == 0 && (fcw & 0x103f) == 0x003f;

}



static struct fpu x86_init_fpu __attribute__ ((aligned (64))) __read_mostly;



static void __init fpu__init_system_early_generic(void)

{

	fpstate_reset(&x86_init_fpu);

	current->thread.fpu = &x86_init_fpu;

	set_thread_flag(TIF_NEED_FPU_LOAD);

	x86_init_fpu.last_cpu = -1;



	if (!boot_cpu_has(X86_FEATURE_CPUID) &&

	    !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {

		if (fpu__probe_without_cpuid())
@@ -150,6 +157,8 @@ static void __init fpu__init_task_struct_size(void) 
{

	int task_size = sizeof(struct task_struct);



	task_size += sizeof(struct fpu);



	/*

	 * Subtract off the static size of the register state.

	 * It potentially has a bunch of padding.
@@ -164,14 +173,9 @@ static void __init fpu__init_task_struct_size(void) 


	/*

	 * We dynamically size 'struct fpu', so we require that

	 * it be at the end of 'thread_struct' and that

	 * 'thread_struct' be at the end of 'task_struct'.  If

	 * you hit a compile error here, check the structure to

	 * see if something got added to the end.

	 * 'state' be at the end of 'it:

	 */

	CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate);

	CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);

	CHECK_MEMBER_AT_END_OF(struct task_struct, thread);



	arch_task_struct_size = task_size;

}
@@ -213,7 +217,6 @@ static void __init fpu__init_system_xstate_size_legacy(void) 
 */

void __init fpu__init_system(void)

{

	fpstate_reset(x86_task_fpu(current));

	fpu__init_system_early_generic();



	/*

arch/x86/kernel/process.c

+1 −1
Original line number Diff line number Diff line
@@ -103,7 +103,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) 
	dst->thread.vm86 = NULL;

#endif

	/* Drop the copied pointer to current's fpstate */

	x86_task_fpu(dst)->fpstate = NULL;

	dst->thread.fpu = NULL;



	return 0;

}

include/linux/sched.h

+4 −11
Original line number Diff line number Diff line
@@ -1646,22 +1646,15 @@ struct task_struct { 
	struct user_event_mm		*user_event_mm;

#endif



	/*

	 * New fields for task_struct should be added above here, so that

	 * they are included in the randomized portion of task_struct.

	 */

	randomized_struct_fields_end



	/* CPU-specific state of this task: */

	struct thread_struct		thread;



	/*

	 * WARNING: on x86, 'thread_struct' contains a variable-sized

	 * structure.  It *MUST* be at the end of 'task_struct'.

	 *

	 * Do not put anything below here!

	 * New fields for task_struct should be added above here, so that

	 * they are included in the randomized portion of task_struct.

	 */

};

	randomized_struct_fields_end

} __attribute__ ((aligned (64)));



#define TASK_REPORT_IDLE	(TASK_REPORT + 1)

#define TASK_REPORT_MAX		(TASK_REPORT_IDLE << 1)