Commit dd82e356 authored by Anup Patel's avatar Anup Patel Committed by Anup Patel
Browse files

RISC-V: KVM: Factor-out g-stage page table management 



The upcoming nested virtualization can share g-stage page table
management with the current host g-stage implementation hence
factor-out g-stage page table management as separate sources
and also use "kvm_riscv_mmu_" prefix for host g-stage functions.

Signed-off-by: default avatarAnup Patel <apatel@ventanamicro.com>
Tested-by: default avatarAtish Patra <atishp@rivosinc.com>
Reviewed-by: default avatarNutty Liu <liujingqi@lanxincomputing.com>
Link: https://lore.kernel.org/r/20250618113532.471448-12-apatel@ventanamicro.com


Signed-off-by: default avatarAnup Patel <anup@brainfault.org>
parent 4c933f3a

arch/riscv/include/asm/kvm_gstage.h

0 → 100644
+72 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * Copyright (C) 2019 Western Digital Corporation or its affiliates.

 * Copyright (c) 2025 Ventana Micro Systems Inc.

 */



#ifndef __RISCV_KVM_GSTAGE_H_

#define __RISCV_KVM_GSTAGE_H_



#include <linux/kvm_types.h>



struct kvm_gstage {

	struct kvm *kvm;

	unsigned long flags;

#define KVM_GSTAGE_FLAGS_LOCAL		BIT(0)

	unsigned long vmid;

	pgd_t *pgd;

};



struct kvm_gstage_mapping {

	gpa_t addr;

	pte_t pte;

	u32 level;

};



#ifdef CONFIG_64BIT

#define kvm_riscv_gstage_index_bits	9

#else

#define kvm_riscv_gstage_index_bits	10

#endif



extern unsigned long kvm_riscv_gstage_mode;

extern unsigned long kvm_riscv_gstage_pgd_levels;



#define kvm_riscv_gstage_pgd_xbits	2

#define kvm_riscv_gstage_pgd_size	(1UL << (HGATP_PAGE_SHIFT + kvm_riscv_gstage_pgd_xbits))

#define kvm_riscv_gstage_gpa_bits	(HGATP_PAGE_SHIFT + \

					 (kvm_riscv_gstage_pgd_levels * \

					  kvm_riscv_gstage_index_bits) + \

					 kvm_riscv_gstage_pgd_xbits)

#define kvm_riscv_gstage_gpa_size	((gpa_t)(1ULL << kvm_riscv_gstage_gpa_bits))



bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr,

			       pte_t **ptepp, u32 *ptep_level);



int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,

			     struct kvm_mmu_memory_cache *pcache,

			     const struct kvm_gstage_mapping *map);



int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage,

			      struct kvm_mmu_memory_cache *pcache,

			      gpa_t gpa, phys_addr_t hpa, unsigned long page_size,

			      bool page_rdonly, bool page_exec,

			      struct kvm_gstage_mapping *out_map);



enum kvm_riscv_gstage_op {

	GSTAGE_OP_NOP = 0,	/* Nothing */

	GSTAGE_OP_CLEAR,	/* Clear/Unmap */

	GSTAGE_OP_WP,		/* Write-protect */

};



void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr,

			     pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op);



void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,

				  gpa_t start, gpa_t size, bool may_block);



void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end);



void kvm_riscv_gstage_mode_detect(void);



#endif

arch/riscv/include/asm/kvm_mmu.h

+10 −22
Original line number Diff line number Diff line
@@ -6,28 +6,16 @@ 
#ifndef __RISCV_KVM_MMU_H_

#define __RISCV_KVM_MMU_H_



#include <linux/kvm_types.h>

#include <asm/kvm_gstage.h>



struct kvm_gstage_mapping {

	gpa_t addr;

	pte_t pte;

	u32 level;

};



int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa,

			     phys_addr_t hpa, unsigned long size,

			     bool writable, bool in_atomic);

void kvm_riscv_gstage_iounmap(struct kvm *kvm, gpa_t gpa,

			      unsigned long size);

int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,

			 struct kvm_memory_slot *memslot,

int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,

			  unsigned long size, bool writable, bool in_atomic);

void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size);

int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,

		      gpa_t gpa, unsigned long hva, bool is_write,

		      struct kvm_gstage_mapping *out_map);

int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm);

void kvm_riscv_gstage_free_pgd(struct kvm *kvm);

void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu);

void kvm_riscv_gstage_mode_detect(void);

unsigned long kvm_riscv_gstage_mode(void);

int kvm_riscv_gstage_gpa_bits(void);

int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm);

void kvm_riscv_mmu_free_pgd(struct kvm *kvm);

void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu);



#endif

arch/riscv/kvm/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ kvm-y += aia.o 
kvm-y += aia_aplic.o

kvm-y += aia_device.o

kvm-y += aia_imsic.o

kvm-y += gstage.o

kvm-y += main.o

kvm-y += mmu.o

kvm-y += nacl.o

arch/riscv/kvm/aia_imsic.c

+5 −6
Original line number Diff line number Diff line
@@ -746,8 +746,7 @@ void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) 
	 */



	/* Purge the G-stage mapping */

	kvm_riscv_gstage_iounmap(vcpu->kvm,

				 vcpu->arch.aia_context.imsic_addr,

	kvm_riscv_mmu_iounmap(vcpu->kvm, vcpu->arch.aia_context.imsic_addr,

			      IMSIC_MMIO_PAGE_SZ);



	/* TODO: Purge the IOMMU mapping ??? */
@@ -831,7 +830,7 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) 
	imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix);



	/* Update G-stage mapping for the new IMSIC VS-file */

	ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,

	ret = kvm_riscv_mmu_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,

				    new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,

				    true, true);

	if (ret)

arch/riscv/kvm/gstage.c

0 → 100644
+337 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/*

 * Copyright (C) 2019 Western Digital Corporation or its affiliates.

 * Copyright (c) 2025 Ventana Micro Systems Inc.

 */



#include <linux/bitops.h>

#include <linux/errno.h>

#include <linux/kvm_host.h>

#include <linux/module.h>

#include <linux/pgtable.h>

#include <asm/kvm_gstage.h>



#ifdef CONFIG_64BIT

unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV39X4;

unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 3;

#else

unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV32X4;

unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 2;

#endif



#define gstage_pte_leaf(__ptep)	\

	(pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC))



static inline unsigned long gstage_pte_index(gpa_t addr, u32 level)

{

	unsigned long mask;

	unsigned long shift = HGATP_PAGE_SHIFT + (kvm_riscv_gstage_index_bits * level);



	if (level == (kvm_riscv_gstage_pgd_levels - 1))

		mask = (PTRS_PER_PTE * (1UL << kvm_riscv_gstage_pgd_xbits)) - 1;

	else

		mask = PTRS_PER_PTE - 1;



	return (addr >> shift) & mask;

}



static inline unsigned long gstage_pte_page_vaddr(pte_t pte)

{

	return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte)));

}



static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level)

{

	u32 i;

	unsigned long psz = 1UL << 12;



	for (i = 0; i < kvm_riscv_gstage_pgd_levels; i++) {

		if (page_size == (psz << (i * kvm_riscv_gstage_index_bits))) {

			*out_level = i;

			return 0;

		}

	}



	return -EINVAL;

}



static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder)

{

	if (kvm_riscv_gstage_pgd_levels < level)

		return -EINVAL;



	*out_pgorder = 12 + (level * kvm_riscv_gstage_index_bits);

	return 0;

}



static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize)

{

	int rc;

	unsigned long page_order = PAGE_SHIFT;



	rc = gstage_level_to_page_order(level, &page_order);

	if (rc)

		return rc;



	*out_pgsize = BIT(page_order);

	return 0;

}



bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr,

			       pte_t **ptepp, u32 *ptep_level)

{

	pte_t *ptep;

	u32 current_level = kvm_riscv_gstage_pgd_levels - 1;



	*ptep_level = current_level;

	ptep = (pte_t *)gstage->pgd;

	ptep = &ptep[gstage_pte_index(addr, current_level)];

	while (ptep && pte_val(ptep_get(ptep))) {

		if (gstage_pte_leaf(ptep)) {

			*ptep_level = current_level;

			*ptepp = ptep;

			return true;

		}



		if (current_level) {

			current_level--;

			*ptep_level = current_level;

			ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));

			ptep = &ptep[gstage_pte_index(addr, current_level)];

		} else {

			ptep = NULL;

		}

	}



	return false;

}



static void gstage_tlb_flush(struct kvm_gstage *gstage, u32 level, gpa_t addr)

{

	unsigned long order = PAGE_SHIFT;



	if (gstage_level_to_page_order(level, &order))

		return;

	addr &= ~(BIT(order) - 1);



	if (gstage->flags & KVM_GSTAGE_FLAGS_LOCAL)

		kvm_riscv_local_hfence_gvma_vmid_gpa(gstage->vmid, addr, BIT(order), order);

	else

		kvm_riscv_hfence_gvma_vmid_gpa(gstage->kvm, -1UL, 0, addr, BIT(order), order);

}



int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,

			     struct kvm_mmu_memory_cache *pcache,

			     const struct kvm_gstage_mapping *map)

{

	u32 current_level = kvm_riscv_gstage_pgd_levels - 1;

	pte_t *next_ptep = (pte_t *)gstage->pgd;

	pte_t *ptep = &next_ptep[gstage_pte_index(map->addr, current_level)];



	if (current_level < map->level)

		return -EINVAL;



	while (current_level != map->level) {

		if (gstage_pte_leaf(ptep))

			return -EEXIST;



		if (!pte_val(ptep_get(ptep))) {

			if (!pcache)

				return -ENOMEM;

			next_ptep = kvm_mmu_memory_cache_alloc(pcache);

			if (!next_ptep)

				return -ENOMEM;

			set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)),

					      __pgprot(_PAGE_TABLE)));

		} else {

			if (gstage_pte_leaf(ptep))

				return -EEXIST;

			next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));

		}



		current_level--;

		ptep = &next_ptep[gstage_pte_index(map->addr, current_level)];

	}



	if (pte_val(*ptep) != pte_val(map->pte)) {

		set_pte(ptep, map->pte);

		if (gstage_pte_leaf(ptep))

			gstage_tlb_flush(gstage, current_level, map->addr);

	}



	return 0;

}



int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage,

			      struct kvm_mmu_memory_cache *pcache,

			      gpa_t gpa, phys_addr_t hpa, unsigned long page_size,

			      bool page_rdonly, bool page_exec,

			      struct kvm_gstage_mapping *out_map)

{

	pgprot_t prot;

	int ret;



	out_map->addr = gpa;

	out_map->level = 0;



	ret = gstage_page_size_to_level(page_size, &out_map->level);

	if (ret)

		return ret;



	/*

	 * A RISC-V implementation can choose to either:

	 * 1) Update 'A' and 'D' PTE bits in hardware

	 * 2) Generate page fault when 'A' and/or 'D' bits are not set

	 *    PTE so that software can update these bits.

	 *

	 * We support both options mentioned above. To achieve this, we

	 * always set 'A' and 'D' PTE bits at time of creating G-stage

	 * mapping. To support KVM dirty page logging with both options

	 * mentioned above, we will write-protect G-stage PTEs to track

	 * dirty pages.

	 */



	if (page_exec) {

		if (page_rdonly)

			prot = PAGE_READ_EXEC;

		else

			prot = PAGE_WRITE_EXEC;

	} else {

		if (page_rdonly)

			prot = PAGE_READ;

		else

			prot = PAGE_WRITE;

	}

	out_map->pte = pfn_pte(PFN_DOWN(hpa), prot);

	out_map->pte = pte_mkdirty(out_map->pte);



	return kvm_riscv_gstage_set_pte(gstage, pcache, out_map);

}



void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr,

			     pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op)

{

	int i, ret;

	pte_t old_pte, *next_ptep;

	u32 next_ptep_level;

	unsigned long next_page_size, page_size;



	ret = gstage_level_to_page_size(ptep_level, &page_size);

	if (ret)

		return;



	WARN_ON(addr & (page_size - 1));



	if (!pte_val(ptep_get(ptep)))

		return;



	if (ptep_level && !gstage_pte_leaf(ptep)) {

		next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));

		next_ptep_level = ptep_level - 1;

		ret = gstage_level_to_page_size(next_ptep_level, &next_page_size);

		if (ret)

			return;



		if (op == GSTAGE_OP_CLEAR)

			set_pte(ptep, __pte(0));

		for (i = 0; i < PTRS_PER_PTE; i++)

			kvm_riscv_gstage_op_pte(gstage, addr + i * next_page_size,

						&next_ptep[i], next_ptep_level, op);

		if (op == GSTAGE_OP_CLEAR)

			put_page(virt_to_page(next_ptep));

	} else {

		old_pte = *ptep;

		if (op == GSTAGE_OP_CLEAR)

			set_pte(ptep, __pte(0));

		else if (op == GSTAGE_OP_WP)

			set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE));

		if (pte_val(*ptep) != pte_val(old_pte))

			gstage_tlb_flush(gstage, ptep_level, addr);

	}

}



void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,

				  gpa_t start, gpa_t size, bool may_block)

{

	int ret;

	pte_t *ptep;

	u32 ptep_level;

	bool found_leaf;

	unsigned long page_size;

	gpa_t addr = start, end = start + size;



	while (addr < end) {

		found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);

		ret = gstage_level_to_page_size(ptep_level, &page_size);

		if (ret)

			break;



		if (!found_leaf)

			goto next;



		if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))

			kvm_riscv_gstage_op_pte(gstage, addr, ptep,

						ptep_level, GSTAGE_OP_CLEAR);



next:

		addr += page_size;



		/*

		 * If the range is too large, release the kvm->mmu_lock

		 * to prevent starvation and lockup detector warnings.

		 */

		if (!(gstage->flags & KVM_GSTAGE_FLAGS_LOCAL) && may_block && addr < end)

			cond_resched_lock(&gstage->kvm->mmu_lock);

	}

}



void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end)

{

	int ret;

	pte_t *ptep;

	u32 ptep_level;

	bool found_leaf;

	gpa_t addr = start;

	unsigned long page_size;



	while (addr < end) {

		found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);

		ret = gstage_level_to_page_size(ptep_level, &page_size);

		if (ret)

			break;



		if (!found_leaf)

			goto next;



		if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))

			kvm_riscv_gstage_op_pte(gstage, addr, ptep,

						ptep_level, GSTAGE_OP_WP);



next:

		addr += page_size;

	}

}



void __init kvm_riscv_gstage_mode_detect(void)

{

#ifdef CONFIG_64BIT

	/* Try Sv57x4 G-stage mode */

	csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT);

	if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) {

		kvm_riscv_gstage_mode = HGATP_MODE_SV57X4;

		kvm_riscv_gstage_pgd_levels = 5;

		goto skip_sv48x4_test;

	}



	/* Try Sv48x4 G-stage mode */

	csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);

	if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) {

		kvm_riscv_gstage_mode = HGATP_MODE_SV48X4;

		kvm_riscv_gstage_pgd_levels = 4;

	}

skip_sv48x4_test:



	csr_write(CSR_HGATP, 0);

	kvm_riscv_local_hfence_gvma_all();

#endif

}