arm64: head: move relocation handling to C code (734958ef) · Commits · git / linux-net

arch/arm64/kernel/Makefile

+2 −1

Original line number	Diff line number	Diff line
		@@ -57,7 +57,8 @@ obj-$(CONFIG_ACPI) += acpi.o
		obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o
		obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o
		obj-$(CONFIG_PARAVIRT) += paravirt.o
		obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o pi/
		obj-$(CONFIG_RELOCATABLE) += pi/
		obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
		obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o
		obj-$(CONFIG_ELF_CORE) += elfcore.o
		obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \

arch/arm64/kernel/head.S

+7 −97

Original line number	Diff line number	Diff line
		@@ -81,7 +81,7 @@
		* x20 primary_entry() .. __primary_switch() CPU boot mode
		* x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0
		* x22 create_idmap() .. start_kernel() ID map VA of the DT blob
		* x23 primary_entry() .. start_kernel() physical misalignment/KASLR offset
		* x23 __primary_switch() physical misalignment/KASLR offset
		* x24 __primary_switch() linear map KASLR seed
		* x25 primary_entry() .. start_kernel() supported VA size
		* x28 create_idmap() callee preserved temp register
		@@ -389,7 +389,7 @@ SYM_FUNC_START_LOCAL(create_idmap)
		/* Remap the kernel page tables r/w in the ID map */
		adrp x1, _text
		adrp x2, init_pg_dir
		adrp x3, init_pg_end
		adrp x3, _end
		bic x4, x2, #SWAPPER_BLOCK_SIZE - 1
		mov_q x5, SWAPPER_RW_MMUFLAGS
		mov x6, #SWAPPER_BLOCK_SHIFT
		@@ -779,97 +779,6 @@ SYM_FUNC_START_LOCAL(__no_granule_support)
		b 1b
		SYM_FUNC_END(__no_granule_support)

		#ifdef CONFIG_RELOCATABLE
		SYM_FUNC_START_LOCAL(__relocate_kernel)
		/*
		* Iterate over each entry in the relocation table, and apply the
		* relocations in place.
		*/
		adr_l x9, __rela_start
		adr_l x10, __rela_end
		mov_q x11, KIMAGE_VADDR // default virtual offset
		add x11, x11, x23 // actual virtual offset

		0: cmp x9, x10
		b.hs 1f
		ldp x12, x13, [x9], #24
		ldr x14, [x9, #-8]
		cmp w13, #R_AARCH64_RELATIVE
		b.ne 0b
		add x14, x14, x23 // relocate
		str x14, [x12, x23]
		b 0b

		1:
		#ifdef CONFIG_RELR
		/*
		* Apply RELR relocations.
		*
		* RELR is a compressed format for storing relative relocations. The
		* encoded sequence of entries looks like:
		* [ AAAAAAAA BBBBBBB1 BBBBBBB1 ... AAAAAAAA BBBBBB1 ... ]
		*
		* i.e. start with an address, followed by any number of bitmaps. The
		* address entry encodes 1 relocation. The subsequent bitmap entries
		* encode up to 63 relocations each, at subsequent offsets following
		* the last address entry.
		*
		* The bitmap entries must have 1 in the least significant bit. The
		* assumption here is that an address cannot have 1 in lsb. Odd
		* addresses are not supported. Any odd addresses are stored in the RELA
		* section, which is handled above.
		*
		* Excluding the least significant bit in the bitmap, each non-zero
		* bit in the bitmap represents a relocation to be applied to
		* a corresponding machine word that follows the base address
		* word. The second least significant bit represents the machine
		* word immediately following the initial address, and each bit
		* that follows represents the next word, in linear order. As such,
		* a single bitmap can encode up to 63 relocations in a 64-bit object.
		*
		* In this implementation we store the address of the next RELR table
		* entry in x9, the address being relocated by the current address or
		* bitmap entry in x13 and the address being relocated by the current
		* bit in x14.
		*/
		adr_l x9, __relr_start
		adr_l x10, __relr_end

		2: cmp x9, x10
		b.hs 7f
		ldr x11, [x9], #8
		tbnz x11, #0, 3f // branch to handle bitmaps
		add x13, x11, x23
		ldr x12, [x13] // relocate address entry
		add x12, x12, x23
		str x12, [x13], #8 // adjust to start of bitmap
		b 2b

		3: mov x14, x13
		4: lsr x11, x11, #1
		cbz x11, 6f
		tbz x11, #0, 5f // skip bit if not set
		ldr x12, [x14] // relocate bit
		add x12, x12, x23
		str x12, [x14]

		5: add x14, x14, #8 // move to next bit's address
		b 4b

		6: /*
		* Move to the next bitmap's address. 8 is the word size, and 63 is the
		* number of significant bits in a bitmap entry.
		*/
		add x13, x13, #(8 * 63)
		b 2b

		7:
		#endif
		ret

		SYM_FUNC_END(__relocate_kernel)
		#endif

		SYM_FUNC_START_LOCAL(__primary_switch)
		adrp x1, reserved_pg_dir
		adrp x2, init_idmap_pg_dir
		@@ -877,11 +786,11 @@ SYM_FUNC_START_LOCAL(__primary_switch)
		#ifdef CONFIG_RELOCATABLE
		adrp x23, KERNEL_START
		and x23, x23, MIN_KIMG_ALIGN - 1
		#ifdef CONFIG_RANDOMIZE_BASE
		mov x0, x22
		adrp x1, init_pg_end
		adrp x1, early_init_stack
		mov sp, x1
		mov x29, xzr
		#ifdef CONFIG_RANDOMIZE_BASE
		mov x0, x22
		bl __pi_kaslr_early_init
		and x24, x0, #SZ_2M - 1 // capture memstart offset seed
		bic x0, x0, #SZ_2M - 1
		@@ -894,7 +803,8 @@ SYM_FUNC_START_LOCAL(__primary_switch)
		adrp x1, init_pg_dir
		load_ttbr1 x1, x1, x2
		#ifdef CONFIG_RELOCATABLE
		bl __relocate_kernel
		mov x0, x23
		bl __pi_relocate_kernel
		#endif
		ldr x8, =__primary_switched
		adrp x0, KERNEL_START // __pa(KERNEL_START)

arch/arm64/kernel/pi/Makefile

+3 −2

Original line number	Diff line number	Diff line
		@@ -38,5 +38,6 @@ $(obj)/lib-%.pi.o: OBJCOPYFLAGS += --prefix-alloc-sections=.init
		$(obj)/lib-%.o: $(srctree)/lib/%.c FORCE
		$(call if_changed_rule,cc_o_c)

		obj-y := kaslr_early.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o
		obj-y := relocate.pi.o
		obj-$(CONFIG_RANDOMIZE_BASE) += kaslr_early.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o
		extra-y := $(patsubst %.pi.o,%.o,$(obj-y))

arch/arm64/kernel/pi/relocate.c

0 → 100644

+62 −0

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0-only
		// Copyright 2023 Google LLC
		// Authors: Ard Biesheuvel <ardb@google.com>
		// Peter Collingbourne <pcc@google.com>

		#include <linux/elf.h>
		#include <linux/init.h>
		#include <linux/types.h>

		extern const Elf64_Rela rela_start[], rela_end[];
		extern const u64 relr_start[], relr_end[];

		void __init relocate_kernel(u64 offset)
		{
		u64 *place = NULL;

		for (const Elf64_Rela *rela = rela_start; rela < rela_end; rela++) {
		if (ELF64_R_TYPE(rela->r_info) != R_AARCH64_RELATIVE)
		continue;
		(u64 )(rela->r_offset + offset) = rela->r_addend + offset;
		}

		if (!IS_ENABLED(CONFIG_RELR) \|\| !offset)
		return;

		/*
		* Apply RELR relocations.
		*
		* RELR is a compressed format for storing relative relocations. The
		* encoded sequence of entries looks like:
		* [ AAAAAAAA BBBBBBB1 BBBBBBB1 ... AAAAAAAA BBBBBB1 ... ]
		*
		* i.e. start with an address, followed by any number of bitmaps. The
		* address entry encodes 1 relocation. The subsequent bitmap entries
		* encode up to 63 relocations each, at subsequent offsets following
		* the last address entry.
		*
		* The bitmap entries must have 1 in the least significant bit. The
		* assumption here is that an address cannot have 1 in lsb. Odd
		* addresses are not supported. Any odd addresses are stored in the
		* RELA section, which is handled above.
		*
		* With the exception of the least significant bit, each bit in the
		* bitmap corresponds with a machine word that follows the base address
		* word, and the bit value indicates whether or not a relocation needs
		* to be applied to it. The second least significant bit represents the
		* machine word immediately following the initial address, and each bit
		* that follows represents the next word, in linear order. As such, a
		* single bitmap can encode up to 63 relocations in a 64-bit object.
		*/
		for (const u64 *relr = relr_start; relr < relr_end; relr++) {
		if ((*relr & 1) == 0) {
		place = (u64 )(relr + offset);
		*place++ += offset;
		} else {
		for (u64 p = place, r = relr >> 1; r; p++, r >>= 1)
		if (r & 1)
		*p += offset;
		place += 63;
		}
		}
		}

arch/arm64/kernel/vmlinux.lds.S

+8 −4

Original line number	Diff line number	Diff line
		@@ -270,15 +270,15 @@ SECTIONS
		HYPERVISOR_RELOC_SECTION

		.rela.dyn : ALIGN(8) {
		__rela_start = .;
		__pi_rela_start = .;
		(.rela .rela)
		__rela_end = .;
		__pi_rela_end = .;
		}

		.relr.dyn : ALIGN(8) {
		__relr_start = .;
		__pi_relr_start = .;
		*(.relr.dyn)
		__relr_end = .;
		__pi_relr_end = .;
		}

		. = ALIGN(SEGMENT_ALIGN);
		@@ -317,6 +317,10 @@ SECTIONS
		init_pg_dir = .;
		. += INIT_DIR_SIZE;
		init_pg_end = .;
		#ifdef CONFIG_RELOCATABLE
		. += SZ_4K; /* stack for the early relocation code */
		early_init_stack = .;
		#endif

		. = ALIGN(SEGMENT_ALIGN);
		__pecoff_data_size = ABSOLUTE(. - __initdata_begin);