Commit bcceba3c authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Ingo Molnar
Browse files

x86/asm: Make rip_rel_ptr() usable from fPIC code 



RIP_REL_REF() is used in non-PIC C code that is called very early,
before the kernel virtual mapping is up, which is the mapping that the
linker expects. It is currently used in two different ways:

 - to refer to the value of a global variable, including as an lvalue in
   assignments;

 - to take the address of a global variable via the mapping that the code
   currently executes at.

The former case is only needed in non-PIC code, as PIC code will never
use absolute symbol references when the address of the symbol is not
being used. But taking the address of a variable in PIC code may still
require extra care, as a stack allocated struct assignment may be
emitted as a memcpy() from a statically allocated copy in .rodata.

For instance, this

  void startup_64_setup_gdt_idt(void)
  {
        struct desc_ptr startup_gdt_descr = {
                .address = (__force unsigned long)gdt_page.gdt,
                .size    = GDT_SIZE - 1,
        };

may result in an absolute symbol reference in PIC code, even though the
struct is allocated on the stack and populated at runtime.

To address this case, make rip_rel_ptr() accessible in PIC code, and
update any existing uses where the address of a global variable is
taken using RIP_REL_REF.

Once all code of this nature has been moved into arch/x86/boot/startup
and built with -fPIC, RIP_REL_REF() can be retired, and only
rip_rel_ptr() will remain.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kevin Loughlin <kevinloughlin@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: linux-efi@vger.kernel.org
Link: https://lore.kernel.org/r/20250410134117.3713574-14-ardb+git@google.com
parent 9e8e8794

arch/x86/coco/sev/core.c

+1 −1
Original line number Diff line number Diff line
@@ -2400,7 +2400,7 @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info) 
	 * kernel was loaded (physbase), so the get the CA address using

	 * RIP-relative addressing.

	 */

	pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);

	pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);



	/*

	 * Switch over to the boot SVSM CA while the current CA is still

arch/x86/coco/sev/shared.c

+2 −2
Original line number Diff line number Diff line
@@ -475,7 +475,7 @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid 
 */

static const struct snp_cpuid_table *snp_cpuid_get_table(void)

{

	return &RIP_REL_REF(cpuid_table_copy);

	return rip_rel_ptr(&cpuid_table_copy);

}



/*
@@ -1681,7 +1681,7 @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info) 
	 * routine is running identity mapped when called, both by the decompressor

	 * code and the early kernel code.

	 */

	if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1))

	if (!rmpadjust((unsigned long)rip_rel_ptr(&boot_ghcb_page), RMP_PG_SIZE_4K, 1))

		return false;



	/*

arch/x86/include/asm/asm.h

+1 −1
Original line number Diff line number Diff line
@@ -114,13 +114,13 @@ 
#endif



#ifndef __ASSEMBLER__

#ifndef __pic__

static __always_inline __pure void *rip_rel_ptr(void *p)

{

	asm("leaq %c1(%%rip), %0" : "=r"(p) : "i"(p));



	return p;

}

#ifndef __pic__

#define RIP_REL_REF(var)	(*(typeof(&(var)))rip_rel_ptr(&(var)))

#else

#define RIP_REL_REF(var)	(var)

arch/x86/kernel/head64.c

+12 −12
Original line number Diff line number Diff line
@@ -106,8 +106,8 @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp, 
	 * attribute.

	 */

	if (sme_get_me_mask()) {

		paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);

		paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);

		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);

		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);



		for (; paddr < paddr_end; paddr += PMD_SIZE) {

			/*
@@ -144,8 +144,8 @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp, 
unsigned long __head __startup_64(unsigned long p2v_offset,

				  struct boot_params *bp)

{

	pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);

	unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);

	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);

	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);

	unsigned long va_text, va_end;

	unsigned long pgtable_flags;

	unsigned long load_delta;
@@ -174,18 +174,18 @@ unsigned long __head __startup_64(unsigned long p2v_offset, 
		for (;;);



	va_text = physaddr - p2v_offset;

	va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;

	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;



	/* Include the SME encryption mask in the fixup value */

	load_delta += sme_get_me_mask();



	/* Fixup the physical addresses in the page table */



	pgd = &RIP_REL_REF(early_top_pgt)->pgd;

	pgd = rip_rel_ptr(early_top_pgt);

	pgd[pgd_index(__START_KERNEL_map)] += load_delta;



	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {

		p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);

		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);

		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;



		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
@@ -258,7 +258,7 @@ unsigned long __head __startup_64(unsigned long p2v_offset, 
	 * error, causing the BIOS to halt the system.

	 */



	pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;

	pmd = rip_rel_ptr(level2_kernel_pgt);



	/* invalidate pages before the kernel image */

	for (i = 0; i < pmd_index(va_text); i++)
@@ -531,7 +531,7 @@ static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; 
static void __head startup_64_load_idt(void *vc_handler)

{

	struct desc_ptr desc = {

		.address = (unsigned long)&RIP_REL_REF(bringup_idt_table),

		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),

		.size    = sizeof(bringup_idt_table) - 1,

	};

	struct idt_data data;
@@ -565,11 +565,11 @@ void early_setup_idt(void) 
 */

void __head startup_64_setup_gdt_idt(void)

{

	struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;

	struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);

	void *handler = NULL;



	struct desc_ptr startup_gdt_descr = {

		.address = (unsigned long)&RIP_REL_REF(*gdt),

		.address = (unsigned long)gp->gdt,

		.size    = GDT_SIZE - 1,

	};
@@ -582,7 +582,7 @@ void __head startup_64_setup_gdt_idt(void) 
		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");



	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))

		handler = &RIP_REL_REF(vc_no_ghcb);

		handler = rip_rel_ptr(vc_no_ghcb);



	startup_64_load_idt(handler);

}

arch/x86/mm/mem_encrypt_identity.c

+3 −3
Original line number Diff line number Diff line
@@ -318,8 +318,8 @@ void __head sme_encrypt_kernel(struct boot_params *bp) 
	 *     memory from being cached.

	 */



	kernel_start = (unsigned long)RIP_REL_REF(_text);

	kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);

	kernel_start = (unsigned long)rip_rel_ptr(_text);

	kernel_end = ALIGN((unsigned long)rip_rel_ptr(_end), PMD_SIZE);

	kernel_len = kernel_end - kernel_start;



	initrd_start = 0;
@@ -345,7 +345,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp) 
	 *   pagetable structures for the encryption of the kernel

	 *   pagetable structures for workarea (in case not currently mapped)

	 */

	execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);

	execute_start = workarea_start = (unsigned long)rip_rel_ptr(sme_workarea);

	execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;

	execute_len = execute_end - execute_start;