Commit 2fd65f7a authored by Rong Xu's avatar Rong Xu Committed by Masahiro Yamada
Browse files

AutoFDO: Enable machine function split optimization for AutoFDO 



Enable the machine function split optimization for AutoFDO in Clang.

Machine function split (MFS) is a pass in the Clang compiler that
splits a function into hot and cold parts. The linker groups all
cold blocks across functions together. This decreases hot code
fragmentation and improves iCache and iTLB utilization.

MFS requires a profile so this is enabled only for the AutoFDO builds.

Co-developed-by: default avatarHan Shen <shenhan@google.com>
Signed-off-by: default avatarHan Shen <shenhan@google.com>
Signed-off-by: default avatarRong Xu <xur@google.com>
Suggested-by: default avatarSriraman Tallam <tmsriram@google.com>
Suggested-by: default avatarKrzysztof Pszeniczny <kpszeniczny@google.com>
Tested-by: default avatarYonghong Song <yonghong.song@linux.dev>
Tested-by: default avatarYabin Cui <yabinc@google.com>
Tested-by: default avatarNathan Chancellor <nathan@kernel.org>
Reviewed-by: default avatarKees Cook <kees@kernel.org>
Signed-off-by: default avatarMasahiro Yamada <masahiroy@kernel.org>
parent 0847420f

include/asm-generic/vmlinux.lds.h

+6 −1
Original line number Diff line number Diff line
@@ -556,6 +556,11 @@ defined(CONFIG_AUTOFDO_CLANG) 
		__cpuidle_text_end = .;					\

		__noinstr_text_end = .;



#define TEXT_SPLIT							\

		__split_text_start = .;					\

		*(.text.split .text.split.[0-9a-zA-Z_]*)		\

		__split_text_end = .;



#define TEXT_UNLIKELY							\

		__unlikely_text_start = .;				\

		*(.text.unlikely .text.unlikely.*)			\
@@ -582,6 +587,7 @@ defined(CONFIG_AUTOFDO_CLANG) 
		ALIGN_FUNCTION();					\

		*(.text.asan.* .text.tsan.*)				\

		*(.text.unknown .text.unknown.*)			\

		TEXT_SPLIT						\

		TEXT_UNLIKELY						\

		. = ALIGN(PAGE_SIZE);					\

		TEXT_HOT						\
@@ -589,7 +595,6 @@ defined(CONFIG_AUTOFDO_CLANG) 
		NOINSTR_TEXT						\

		*(.ref.text)





/* sched.text is aling to function alignment to secure we have same

 * address even at second ld pass when generating System.map */

#define SCHED_TEXT							\

scripts/Makefile.autofdo

+2 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ endif 


ifdef CLANG_AUTOFDO_PROFILE

  CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections

  CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions

endif



ifdef CONFIG_LTO_CLANG_THIN
@@ -17,6 +18,7 @@ ifdef CONFIG_LTO_CLANG_THIN 
    KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE)

  endif

  KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true --mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto

  KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions

endif



export CFLAGS_AUTOFDO_CLANG