Commit 82ca151d authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: mmu: Make __cpu_replace_ttbr1() out of line 



__cpu_replace_ttbr1() is a static inline, and so it gets instantiated
wherever it is used. This is not really necessary, as it is never called
on a hot path. It also has the unfortunate side effect that the symbol
idmap_cpu_replace_ttbr1 may never be referenced from kCFI enabled C
code, and this means the type id symbol may not exist either.  This will
result in a build error once we start referring to this symbol from asm
code as well. (Note that this problem only occurs when CnP, KAsan and
suspend/resume are all disabled in the Kconfig but that is a valid
config, if unusual).

So let's just move it out of line so all callers will share the same
implementation, which will reference idmap_cpu_replace_ttbr1
unconditionally.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-62-ardb+git@google.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 293d865f

arch/arm64/include/asm/mmu_context.h

+1 −31
Original line number Diff line number Diff line
@@ -148,37 +148,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) 
	isb();

}



/*

 * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,

 * avoiding the possibility of conflicting TLB entries being allocated.

 */

static inline void __cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap, bool cnp)

{

	typedef void (ttbr_replace_func)(phys_addr_t);

	extern ttbr_replace_func idmap_cpu_replace_ttbr1;

	ttbr_replace_func *replace_phys;

	unsigned long daif;



	/* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */

	phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));



	if (cnp)

		ttbr1 |= TTBR_CNP_BIT;



	replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);



	__cpu_install_idmap(idmap);



	/*

	 * We really don't want to take *any* exceptions while TTBR1 is

	 * in the process of being replaced so mask everything.

	 */

	daif = local_daif_save();

	replace_phys(ttbr1);

	local_daif_restore(daif);



	cpu_uninstall_idmap();

}

void __cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap, bool cnp);



static inline void cpu_enable_swapper_cnp(void)

{

arch/arm64/mm/mmu.c

+32 −0
Original line number Diff line number Diff line
@@ -1486,3 +1486,35 @@ void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte 
{

	set_pte_at(vma->vm_mm, addr, ptep, pte);

}



/*

 * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,

 * avoiding the possibility of conflicting TLB entries being allocated.

 */

void __cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap, bool cnp)

{

	typedef void (ttbr_replace_func)(phys_addr_t);

	extern ttbr_replace_func idmap_cpu_replace_ttbr1;

	ttbr_replace_func *replace_phys;

	unsigned long daif;



	/* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */

	phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));



	if (cnp)

		ttbr1 |= TTBR_CNP_BIT;



	replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);



	__cpu_install_idmap(idmap);



	/*

	 * We really don't want to take *any* exceptions while TTBR1 is

	 * in the process of being replaced so mask everything.

	 */

	daif = local_daif_save();

	replace_phys(ttbr1);

	local_daif_restore(daif);



	cpu_uninstall_idmap();

}