Commit eec37961 authored by Naveen N Rao's avatar Naveen N Rao Committed by Michael Ellerman
Browse files

powerpc64/ftrace: Move ftrace sequence out of line 



Function profile sequence on powerpc includes two instructions at the
beginning of each function:
	mflr	r0
	bl	ftrace_caller

The call to ftrace_caller() gets nop'ed out during kernel boot and is
patched in when ftrace is enabled.

Given the sequence, we cannot return from ftrace_caller with 'blr' as we
need to keep LR and r0 intact. This results in link stack (return
address predictor) imbalance when ftrace is enabled. To address that, we
would like to use a three instruction sequence:
	mflr	r0
	bl	ftrace_caller
	mtlr	r0

Further more, to support DYNAMIC_FTRACE_WITH_CALL_OPS, we need to
reserve two instruction slots before the function. This results in a
total of five instruction slots to be reserved for ftrace use on each
function that is traced.

Move the function profile sequence out-of-line to minimize its impact.
To do this, we reserve a single nop at function entry using
-fpatchable-function-entry=1 and add a pass on vmlinux.o to determine
the total number of functions that can be traced. This is then used to
generate a .S file reserving the appropriate amount of space for use as
ftrace stubs, which is built and linked into vmlinux.

On bootup, the stub space is split into separate stubs per function and
populated with the proper instruction sequence. A pointer to the
associated stub is maintained in dyn_arch_ftrace.

For modules, space for ftrace stubs is reserved from the generic module
stub space.

This is restricted to and enabled by default only on 64-bit powerpc,
though there are some changes to accommodate 32-bit powerpc. This is
done so that 32-bit powerpc could choose to opt into this based on
further tests and benchmarks.

As an example, after this patch, kernel functions will have a single nop
at function entry:
<kernel_clone>:
	addis	r2,r12,467
	addi	r2,r2,-16028
	nop
	mfocrf	r11,8
	...

When ftrace is enabled, the nop is converted to an unconditional branch
to the stub associated with that function:
<kernel_clone>:
	addis	r2,r12,467
	addi	r2,r2,-16028
	b	ftrace_ool_stub_text_end+0x11b28
	mfocrf	r11,8
	...

The associated stub:
<ftrace_ool_stub_text_end+0x11b28>:
	mflr	r0
	bl	ftrace_caller
	mtlr	r0
	b	kernel_clone+0xc
	...

This change showed an improvement of ~10% in null_syscall benchmark on a
Power 10 system with ftrace enabled.

Signed-off-by: default avatarNaveen N Rao <naveen@kernel.org>
Signed-off-by: default avatarHari Bathini <hbathini@linux.ibm.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://patch.msgid.link/20241030070850.1361304-13-hbathini@linux.ibm.com
parent 1198c9c6

arch/powerpc/Kbuild

+1 −1
Original line number Diff line number Diff line
@@ -19,4 +19,4 @@ obj-$(CONFIG_KEXEC_CORE) += kexec/ 
obj-$(CONFIG_KEXEC_FILE)  += purgatory/



# for cleaning

subdir- += boot
 
subdir- += boot tools

arch/powerpc/Kconfig

+4 −0
Original line number Diff line number Diff line
@@ -569,6 +569,10 @@ config ARCH_USING_PATCHABLE_FUNCTION_ENTRY 
	def_bool $(success,$(srctree)/arch/powerpc/tools/gcc-check-fpatchable-function-entry.sh $(CC) -mlittle-endian) if PPC64 && CPU_LITTLE_ENDIAN

	def_bool $(success,$(srctree)/arch/powerpc/tools/gcc-check-fpatchable-function-entry.sh $(CC) -mbig-endian) if PPC64 && CPU_BIG_ENDIAN



config PPC_FTRACE_OUT_OF_LINE

	def_bool PPC64 && ARCH_USING_PATCHABLE_FUNCTION_ENTRY

	select ARCH_WANTS_PRE_LINK_VMLINUX



config HOTPLUG_CPU

	bool "Support for enabling/disabling CPUs"

	depends on SMP && (PPC_PSERIES || \

arch/powerpc/Makefile

+4 −0
Original line number Diff line number Diff line
@@ -148,7 +148,11 @@ CC_FLAGS_NO_FPU := $(call cc-option,-msoft-float) 
ifdef CONFIG_FUNCTION_TRACER

ifdef CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY

KBUILD_CPPFLAGS	+= -DCC_USING_PATCHABLE_FUNCTION_ENTRY

ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE

CC_FLAGS_FTRACE := -fpatchable-function-entry=1

else

CC_FLAGS_FTRACE := -fpatchable-function-entry=2

endif

else

CC_FLAGS_FTRACE := -pg

ifdef CONFIG_MPROFILE_KERNEL

arch/powerpc/include/asm/ftrace.h

+11 −0
Original line number Diff line number Diff line
@@ -24,6 +24,10 @@ unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, 
struct module;

struct dyn_ftrace;

struct dyn_arch_ftrace {

#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE

	/* pointer to the associated out-of-line stub */

	unsigned long ool_stub;

#endif

};



#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
@@ -130,6 +134,13 @@ static inline u8 this_cpu_get_ftrace_enabled(void) { return 1; } 


#ifdef CONFIG_FUNCTION_TRACER

extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[];

#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE

struct ftrace_ool_stub {

	u32	insn[4];

};

extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_inittext[];

extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_inittext_count;

#endif

void ftrace_free_init_tramp(void);

unsigned long ftrace_call_adjust(unsigned long addr);

#else

arch/powerpc/include/asm/module.h

+5 −0
Original line number Diff line number Diff line
@@ -47,6 +47,11 @@ struct mod_arch_specific { 
#ifdef CONFIG_DYNAMIC_FTRACE

	unsigned long tramp;

	unsigned long tramp_regs;

#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE

	struct ftrace_ool_stub *ool_stubs;

	unsigned int ool_stub_count;

	unsigned int ool_stub_index;

#endif

#endif

};