Commit aa99aad7 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: head: Clear BSS and the kernel page tables in one go 



We will move the CPU feature overrides into BSS in a subsequent patch,
and this requires that BSS is zeroed before the feature override
detection code runs. So let's map BSS read-write in the ID map, and zero
it via this mapping.

Since the kernel page tables are right next to it, and also zeroed via
the ID map, let's drop the separate clear_page_tables() function, and
just zero everything in one go.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-51-ardb+git@google.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 9c4cd2a7

arch/arm64/kernel/head.S

+11 −22
Original line number Diff line number Diff line
@@ -177,17 +177,6 @@ SYM_CODE_START_LOCAL(preserve_boot_args) 
	ret

SYM_CODE_END(preserve_boot_args)



SYM_FUNC_START_LOCAL(clear_page_tables)

	/*

	 * Clear the init page tables.

	 */

	adrp	x0, init_pg_dir

	adrp	x1, init_pg_end

	sub	x2, x1, x0

	mov	x1, xzr

	b	__pi_memset			// tail call

SYM_FUNC_END(clear_page_tables)



/*

 * Macro to populate page table entries, these entries can be pointers to the next level

 * or last level entries pointing to physical memory.
@@ -386,9 +375,9 @@ SYM_FUNC_START_LOCAL(create_idmap) 


	map_memory x0, x1, x3, x6, x7, x3, IDMAP_PGD_ORDER, x10, x11, x12, x13, x14, EXTRA_SHIFT



	/* Remap the kernel page tables r/w in the ID map */

	/* Remap BSS and the kernel page tables r/w in the ID map */

	adrp	x1, _text

	adrp	x2, init_pg_dir

	adrp	x2, __bss_start

	adrp	x3, _end

	bic	x4, x2, #SWAPPER_BLOCK_SIZE - 1

	mov_q	x5, SWAPPER_RW_MMUFLAGS
@@ -489,14 +478,6 @@ SYM_FUNC_START_LOCAL(__primary_switched) 
	mov	x0, x20

	bl	set_cpu_boot_mode_flag



	// Clear BSS

	adr_l	x0, __bss_start

	mov	x1, xzr

	adr_l	x2, __bss_stop

	sub	x2, x2, x0

	bl	__pi_memset

	dsb	ishst				// Make zero page visible to PTW



#if VA_BITS > 48

	adr_l	x8, vabits_actual		// Set this early so KASAN early init

	str	x25, [x8]			// ... observes the correct value
@@ -782,6 +763,15 @@ SYM_FUNC_START_LOCAL(__primary_switch) 
	adrp	x1, reserved_pg_dir

	adrp	x2, init_idmap_pg_dir

	bl	__enable_mmu



	// Clear BSS

	adrp	x0, __bss_start

	mov	x1, xzr

	adrp	x2, init_pg_end

	sub	x2, x2, x0

	bl	__pi_memset

	dsb	ishst				// Make zero page visible to PTW



#ifdef CONFIG_RELOCATABLE

	adrp	x23, KERNEL_START

	and	x23, x23, MIN_KIMG_ALIGN - 1
@@ -796,7 +786,6 @@ SYM_FUNC_START_LOCAL(__primary_switch) 
	orr	x23, x23, x0			// record kernel offset

#endif

#endif

	bl	clear_page_tables

	bl	create_kernel_mapping



	adrp	x1, init_pg_dir

arch/arm64/kernel/vmlinux.lds.S

+3 −0
Original line number Diff line number Diff line
@@ -311,12 +311,15 @@ SECTIONS 
	__pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin);

	_edata = .;



	/* start of zero-init region */

	BSS_SECTION(SBSS_ALIGN, 0, 0)



	. = ALIGN(PAGE_SIZE);

	init_pg_dir = .;

	. += INIT_DIR_SIZE;

	init_pg_end = .;

	/* end of zero-init region */



#ifdef CONFIG_RELOCATABLE

	. += SZ_4K;		/* stack for the early relocation code */

	early_init_stack = .;