genpt: Generic Page Table base API (7c5b184d) · Commits · git / linux-net

.clang-format

+1 −0

Original line number	Diff line number	Diff line
		@@ -415,6 +415,7 @@ ForEachMacros:
		- 'for_each_prop_dlc_cpus'
		- 'for_each_prop_dlc_platforms'
		- 'for_each_property_of_node'
		- 'for_each_pt_level_entry'
		- 'for_each_rdt_resource'
		- 'for_each_reg'
		- 'for_each_reg_filtered'

drivers/iommu/Kconfig

+2 −0

Original line number	Diff line number	Diff line
		@@ -384,3 +384,5 @@ config SPRD_IOMMU
		Say Y here if you want to use the multimedia devices listed above.

		endif # IOMMU_SUPPORT

		source "drivers/iommu/generic_pt/Kconfig"

drivers/iommu/generic_pt/Kconfig

0 → 100644

+20 −0

Original line number	Diff line number	Diff line
		# SPDX-License-Identifier: GPL-2.0-only

		menuconfig GENERIC_PT
		bool "Generic Radix Page Table"
		help
		Generic library for building radix tree page tables.

		Generic PT provides a set of HW page table formats and a common
		set of APIs to work with them.

		if GENERIC_PT
		config DEBUG_GENERIC_PT
		bool "Extra debugging checks for GENERIC_PT"
		help
		Enable extra run time debugging checks for GENERIC_PT code. This
		incurs a runtime cost and should not be enabled for production
		kernels.

		The kunit tests require this to be enabled to get full coverage.
		endif

drivers/iommu/generic_pt/pt_common.h

0 → 100644

+358 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0-only */
		/*
		* Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
		*
		* This header is included after the format. It contains definitions
		* that build on the format definitions to create the basic format API.
		*
		* The format API is listed here, with kdocs. The functions without bodies are
		* implemented in the format using the pattern:
		* static inline FMTpt_XXX(..) {..}
		* #define pt_XXX FMTpt_XXX
		*
		* If the format doesn't implement a function then pt_fmt_defaults.h can provide
		* a generic version.
		*
		* The routines marked "@pts: Entry to query" operate on the entire contiguous
		* entry and can be called with a pts->index pointing to any sub item that makes
		* up that entry.
		*
		* The header order is:
		* pt_defs.h
		* FMT.h
		* pt_common.h
		*/
		#ifndef __GENERIC_PT_PT_COMMON_H
		#define __GENERIC_PT_PT_COMMON_H

		#include "pt_defs.h"
		#include "pt_fmt_defaults.h"

		/**
		* pt_attr_from_entry() - Convert the permission bits back to attrs
		* @pts: Entry to convert from
		* @attrs: Resulting attrs
		*
		* Fill in the attrs with the permission bits encoded in the current leaf entry.
		* The attrs should be usable with pt_install_leaf_entry() to reconstruct the
		* same entry.
		*/
		static inline void pt_attr_from_entry(const struct pt_state *pts,
		struct pt_write_attrs *attrs);

		/**
		* pt_can_have_leaf() - True if the current level can have an OA entry
		* @pts: The current level
		*
		* True if the current level can support pt_install_leaf_entry(). A leaf
		* entry produce an OA.
		*/
		static inline bool pt_can_have_leaf(const struct pt_state *pts);

		/**
		* pt_can_have_table() - True if the current level can have a lower table
		* @pts: The current level
		*
		* Every level except 0 is allowed to have a lower table.
		*/
		static inline bool pt_can_have_table(const struct pt_state *pts)
		{
		/* No further tables at level 0 */
		return pts->level > 0;
		}

		/**
		* pt_clear_entries() - Make entries empty (non-present)
		* @pts: Starting table index
		* @num_contig_lg2: Number of contiguous items to clear
		*
		* Clear a run of entries. A cleared entry will load back as PT_ENTRY_EMPTY
		* and does not have any effect on table walking. The starting index must be
		* aligned to num_contig_lg2.
		*/
		static inline void pt_clear_entries(struct pt_state *pts,
		unsigned int num_contig_lg2);

		/**
		* pt_entry_make_write_dirty() - Make an entry dirty
		* @pts: Table entry to change
		*
		* Make pt_entry_is_write_dirty() return true for this entry. This can be called
		* asynchronously with any other table manipulation under a RCU lock and must
		* not corrupt the table.
		*/
		static inline bool pt_entry_make_write_dirty(struct pt_state *pts);

		/**
		* pt_entry_make_write_clean() - Make the entry write clean
		* @pts: Table entry to change
		*
		* Modify the entry so that pt_entry_is_write_dirty() == false. The HW will
		* eventually be notified of this change via a TLB flush, which is the point
		* that the HW must become synchronized. Any "write dirty" prior to the TLB
		* flush can be lost, but once the TLB flush completes all writes must make
		* their entries write dirty.
		*
		* The format should alter the entry in a way that is compatible with any
		* concurrent update from HW. The entire contiguous entry is changed.
		*/
		static inline void pt_entry_make_write_clean(struct pt_state *pts);

		/**
		* pt_entry_is_write_dirty() - True if the entry has been written to
		* @pts: Entry to query
		*
		* "write dirty" means that the HW has written to the OA translated
		* by this entry. If the entry is contiguous then the consolidated
		* "write dirty" for all the items must be returned.
		*/
		static inline bool pt_entry_is_write_dirty(const struct pt_state *pts);

		/**
		* pt_dirty_supported() - True if the page table supports dirty tracking
		* @common: Page table to query
		*/
		static inline bool pt_dirty_supported(struct pt_common *common);

		/**
		* pt_entry_num_contig_lg2() - Number of contiguous items for this leaf entry
		* @pts: Entry to query
		*
		* Return the number of contiguous items this leaf entry spans. If the entry
		* is single item it returns ilog2(1).
		*/
		static inline unsigned int pt_entry_num_contig_lg2(const struct pt_state *pts);

		/**
		* pt_entry_oa() - Output Address for this leaf entry
		* @pts: Entry to query
		*
		* Return the output address for the start of the entry. If the entry
		* is contiguous this returns the same value for each sub-item. I.e.::
		*
		* log2_mod(pt_entry_oa(), pt_entry_oa_lg2sz()) == 0
		*
		* See pt_item_oa(). The format should implement one of these two functions
		* depending on how it stores the OAs in the table.
		*/
		static inline pt_oaddr_t pt_entry_oa(const struct pt_state *pts);

		/**
		* pt_entry_oa_lg2sz() - Return the size of an OA entry
		* @pts: Entry to query
		*
		* If the entry is not contiguous this returns pt_table_item_lg2sz(), otherwise
		* it returns the total VA/OA size of the entire contiguous entry.
		*/
		static inline unsigned int pt_entry_oa_lg2sz(const struct pt_state *pts)
		{
		return pt_entry_num_contig_lg2(pts) + pt_table_item_lg2sz(pts);
		}

		/**
		* pt_entry_oa_exact() - Return the complete OA for an entry
		* @pts: Entry to query
		*
		* During iteration the first entry could have a VA with an offset from the
		* natural start of the entry. Return the exact OA including the pts's VA
		* offset.
		*/
		static inline pt_oaddr_t pt_entry_oa_exact(const struct pt_state *pts)
		{
		return _pt_entry_oa_fast(pts) \|
		log2_mod(pts->range->va, pt_entry_oa_lg2sz(pts));
		}

		/**
		* pt_full_va_prefix() - The top bits of the VA
		* @common: Page table to query
		*
		* This is usually 0, but some formats have their VA space going downward from
		* PT_VADDR_MAX, and will return that instead. This value must always be
		* adjusted by struct pt_common max_vasz_lg2.
		*/
		static inline pt_vaddr_t pt_full_va_prefix(const struct pt_common *common);

		/**
		* pt_has_system_page_size() - True if level 0 can install a PAGE_SHIFT entry
		* @common: Page table to query
		*
		* If true the caller can use, at level 0, pt_install_leaf_entry(PAGE_SHIFT).
		* This is useful to create optimized paths for common cases of PAGE_SIZE
		* mappings.
		*/
		static inline bool pt_has_system_page_size(const struct pt_common *common);

		/**
		* pt_install_leaf_entry() - Write a leaf entry to the table
		* @pts: Table index to change
		* @oa: Output Address for this leaf
		* @oasz_lg2: Size in VA/OA for this leaf
		* @attrs: Attributes to modify the entry
		*
		* A leaf OA entry will return PT_ENTRY_OA from pt_load_entry(). It translates
		* the VA indicated by pts to the given OA.
		*
		* For a single item non-contiguous entry oasz_lg2 is pt_table_item_lg2sz().
		* For contiguous it is pt_table_item_lg2sz() + num_contig_lg2.
		*
		* This must not be called if pt_can_have_leaf() == false. Contiguous sizes
		* not indicated by pt_possible_sizes() must not be specified.
		*/
		static inline void pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa,
		unsigned int oasz_lg2,
		const struct pt_write_attrs *attrs);

		/**
		* pt_install_table() - Write a table entry to the table
		* @pts: Table index to change
		* @table_pa: CPU physical address of the lower table's memory
		* @attrs: Attributes to modify the table index
		*
		* A table entry will return PT_ENTRY_TABLE from pt_load_entry(). The table_pa
		* is the table at pts->level - 1. This is done by cmpxchg so pts must have the
		* current entry loaded. The pts is updated with the installed entry.
		*
		* This must not be called if pt_can_have_table() == false.
		*
		* Returns: true if the table was installed successfully.
		*/
		static inline bool pt_install_table(struct pt_state *pts, pt_oaddr_t table_pa,
		const struct pt_write_attrs *attrs);

		/**
		* pt_item_oa() - Output Address for this leaf item
		* @pts: Item to query
		*
		* Return the output address for this item. If the item is part of a contiguous
		* entry it returns the value of the OA for this individual sub item.
		*
		* See pt_entry_oa(). The format should implement one of these two functions
		* depending on how it stores the OA's in the table.
		*/
		static inline pt_oaddr_t pt_item_oa(const struct pt_state *pts);

		/**
		* pt_load_entry_raw() - Read from the location pts points at into the pts
		* @pts: Table index to load
		*
		* Return the type of entry that was loaded. pts->entry will be filled in with
		* the entry's content. See pt_load_entry()
		*/
		static inline enum pt_entry_type pt_load_entry_raw(struct pt_state *pts);

		/**
		* pt_max_oa_lg2() - Return the maximum OA the table format can hold
		* @common: Page table to query
		*
		* The value oalog2_to_max_int(pt_max_oa_lg2()) is the MAX for the
		* OA. This is the absolute maximum address the table can hold. struct pt_common
		* max_oasz_lg2 sets a lower dynamic maximum based on HW capability.
		*/
		static inline unsigned int
		pt_max_oa_lg2(const struct pt_common *common);

		/**
		* pt_num_items_lg2() - Return the number of items in this table level
		* @pts: The current level
		*
		* The number of items in a table level defines the number of bits this level
		* decodes from the VA. This function is not called for the top level,
		* so it does not need to compute a special value for the top case. The
		* result for the top is based on pt_common max_vasz_lg2.
		*
		* The value is used as part of determining the table indexes via the
		* equation::
		*
		* log2_mod(log2_div(VA, pt_table_item_lg2sz()), pt_num_items_lg2())
		*/
		static inline unsigned int pt_num_items_lg2(const struct pt_state *pts);

		/**
		* pt_pgsz_lg2_to_level - Return the level that maps the page size
		* @common: Page table to query
		* @pgsize_lg2: Log2 page size
		*
		* Returns the table level that will map the given page size. The page
		* size must be part of the pt_possible_sizes() for some level.
		*/
		static inline unsigned int pt_pgsz_lg2_to_level(struct pt_common *common,
		unsigned int pgsize_lg2);

		/**
		* pt_possible_sizes() - Return a bitmap of possible output sizes at this level
		* @pts: The current level
		*
		* Each level has a list of possible output sizes that can be installed as
		* leaf entries. If pt_can_have_leaf() is false returns zero.
		*
		* Otherwise the bit in position pt_table_item_lg2sz() should be set indicating
		* that a non-contiguous single item leaf entry is supported. The following
		* pt_num_items_lg2() number of bits can be set indicating contiguous entries
		* are supported. Bit pt_table_item_lg2sz() + pt_num_items_lg2() must not be
		* set, contiguous entries cannot span the entire table.
		*
		* The OR of pt_possible_sizes() of all levels is the typical bitmask of all
		* supported sizes in the entire table.
		*/
		static inline pt_vaddr_t pt_possible_sizes(const struct pt_state *pts);

		/**
		* pt_table_item_lg2sz() - Size of a single item entry in this table level
		* @pts: The current level
		*
		* The size of the item specifies how much VA and OA a single item occupies.
		*
		* See pt_entry_oa_lg2sz() for the same value including the effect of contiguous
		* entries.
		*/
		static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts);

		/**
		* pt_table_oa_lg2sz() - Return the VA/OA size of the entire table
		* @pts: The current level
		*
		* Return the size of VA decoded by the entire table level.
		*/
		static inline unsigned int pt_table_oa_lg2sz(const struct pt_state *pts)
		{
		if (pts->range->top_level == pts->level)
		return pts->range->max_vasz_lg2;
		return min_t(unsigned int, pts->range->common->max_vasz_lg2,
		pt_num_items_lg2(pts) + pt_table_item_lg2sz(pts));
		}

		/**
		* pt_table_pa() - Return the CPU physical address of the table entry
		* @pts: Entry to query
		*
		* This is only ever called on PT_ENTRY_TABLE entries. Must return the same
		* value passed to pt_install_table().
		*/
		static inline pt_oaddr_t pt_table_pa(const struct pt_state *pts);

		/**
		* pt_table_ptr() - Return a CPU pointer for a table item
		* @pts: Entry to query
		*
		* Same as pt_table_pa() but returns a CPU pointer.
		*/
		static inline struct pt_table_p pt_table_ptr(const struct pt_state pts)
		{
		return __va(pt_table_pa(pts));
		}

		/**
		* pt_load_entry() - Read from the location pts points at into the pts
		* @pts: Table index to load
		*
		* Set the type of entry that was loaded. pts->entry and pts->table_lower
		* will be filled in with the entry's content.
		*/
		static inline void pt_load_entry(struct pt_state *pts)
		{
		pts->type = pt_load_entry_raw(pts);
		if (pts->type == PT_ENTRY_TABLE)
		pts->table_lower = pt_table_ptr(pts);
		}
		#endif

drivers/iommu/generic_pt/pt_defs.h

0 → 100644

+329 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0-only */
		/*
		* Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
		*
		* This header is included before the format. It contains definitions
		* that are required to compile the format. The header order is:
		* pt_defs.h
		* fmt_XX.h
		* pt_common.h
		*/
		#ifndef __GENERIC_PT_DEFS_H
		#define __GENERIC_PT_DEFS_H

		#include <linux/generic_pt/common.h>

		#include <linux/types.h>
		#include <linux/atomic.h>
		#include <linux/bits.h>
		#include <linux/limits.h>
		#include <linux/bug.h>
		#include <linux/kconfig.h>
		#include "pt_log2.h"

		/* Header self-compile default defines */
		#ifndef pt_write_attrs
		typedef u64 pt_vaddr_t;
		typedef u64 pt_oaddr_t;
		#endif

		struct pt_table_p;

		enum {
		PT_VADDR_MAX = sizeof(pt_vaddr_t) == 8 ? U64_MAX : U32_MAX,
		PT_VADDR_MAX_LG2 = sizeof(pt_vaddr_t) == 8 ? 64 : 32,
		PT_OADDR_MAX = sizeof(pt_oaddr_t) == 8 ? U64_MAX : U32_MAX,
		PT_OADDR_MAX_LG2 = sizeof(pt_oaddr_t) == 8 ? 64 : 32,
		};

		/*
		* The format instantiation can have features wired off or on to optimize the
		* code gen. Supported features are just a reflection of what the current set of
		* kernel users want to use.
		*/
		#ifndef PT_SUPPORTED_FEATURES
		#define PT_SUPPORTED_FEATURES 0
		#endif

		/*
		* When in debug mode we compile all formats with all features. This allows the
		* kunit to test the full matrix. SIGN_EXTEND can't co-exist with DYNAMIC_TOP or
		* FULL_VA.
		*/
		#if IS_ENABLED(CONFIG_DEBUG_GENERIC_PT)
		enum {
		PT_ORIG_SUPPORTED_FEATURES = PT_SUPPORTED_FEATURES,
		PT_DEBUG_SUPPORTED_FEATURES =
		UINT_MAX &
		~((PT_ORIG_SUPPORTED_FEATURES & BIT(PT_FEAT_SIGN_EXTEND)) ?
		BIT(PT_FEAT_DYNAMIC_TOP) \| BIT(PT_FEAT_FULL_VA) :
		BIT(PT_FEAT_SIGN_EXTEND)),
		};
		#undef PT_SUPPORTED_FEATURES
		#define PT_SUPPORTED_FEATURES PT_DEBUG_SUPPORTED_FEATURES
		#endif

		#ifndef PT_FORCE_ENABLED_FEATURES
		#define PT_FORCE_ENABLED_FEATURES 0
		#endif

		/**
		* DOC: Generic Page Table Language
		*
		* Language used in Generic Page Table
		* VA
		* The input address to the page table, often the virtual address.
		* OA
		* The output address from the page table, often the physical address.
		* leaf
		* An entry that results in an output address.
		* start/end
		* An half-open range, e.g. [0,0) refers to no VA.
		* start/last
		* An inclusive closed range, e.g. [0,0] refers to the VA 0
		* common
		* The generic page table container struct pt_common
		* level
		* Level 0 is always a table of only leaves with no futher table pointers.
		* Increasing levels increase the size of the table items. The least
		* significant VA bits used to index page tables are used to index the Level
		* 0 table. The various labels for table levels used by HW descriptions are
		* not used.
		* top_level
		* The inclusive highest level of the table. A two-level table
		* has a top level of 1.
		* table
		* A linear array of translation items for that level.
		* index
		* The position in a table of an element: item = table[index]
		* item
		* A single index in a table
		* entry
		* A single logical element in a table. If contiguous pages are not
		* supported then item and entry are the same thing, otherwise entry refers
		* to all the items that comprise a single contiguous translation.
		* item/entry_size
		* The number of bytes of VA the table index translates for.
		* If the item is a table entry then the next table covers
		* this size. If the entry translates to an output address then the
		* full OA is: OA \| (VA % entry_size)
		* contig_count
		* The number of consecutive items fused into a single entry.
		* item_size * contig_count is the size of that entry's translation.
		* lg2
		* Indicates the value is encoded as log2, i.e. 1<<x is the actual value.
		* Normally the compiler is fine to optimize divide and mod with log2 values
		* automatically when inlining, however if the values are not constant
		* expressions it can't. So we do it by hand; we want to avoid 64-bit
		* divmod.
		*/

		/* Returned by pt_load_entry() and for_each_pt_level_entry() */
		enum pt_entry_type {
		PT_ENTRY_EMPTY,
		/* Entry is valid and points to a lower table level */
		PT_ENTRY_TABLE,
		/* Entry is valid and returns an output address */
		PT_ENTRY_OA,
		};

		struct pt_range {
		struct pt_common *common;
		struct pt_table_p *top_table;
		pt_vaddr_t va;
		pt_vaddr_t last_va;
		u8 top_level;
		u8 max_vasz_lg2;
		};

		/*
		* Similar to xa_state, this records information about an in-progress parse at a
		* single level.
		*/
		struct pt_state {
		struct pt_range *range;
		struct pt_table_p *table;
		struct pt_table_p *table_lower;
		u64 entry;
		enum pt_entry_type type;
		unsigned short index;
		unsigned short end_index;
		u8 level;
		};

		#define pt_cur_table(pts, type) ((type *)((pts)->table))

		/*
		* Try to install a new table pointer. The locking methodology requires this to
		* be atomic (multiple threads can race to install a pointer). The losing
		* threads will fail the atomic and return false. They should free any memory
		* and reparse the table level again.
		*/
		#if !IS_ENABLED(CONFIG_GENERIC_ATOMIC64)
		static inline bool pt_table_install64(struct pt_state *pts, u64 table_entry)
		{
		u64 *entryp = pt_cur_table(pts, u64) + pts->index;
		u64 old_entry = pts->entry;
		bool ret;

		/*
		* Ensure the zero'd table content itself is visible before its PTE can
		* be. release is a NOP on !SMP, but the HW is still doing an acquire.
		*/
		if (!IS_ENABLED(CONFIG_SMP))
		dma_wmb();
		ret = try_cmpxchg64_release(entryp, &old_entry, table_entry);
		if (ret)
		pts->entry = table_entry;
		return ret;
		}
		#endif

		static inline bool pt_table_install32(struct pt_state *pts, u32 table_entry)
		{
		u32 *entryp = pt_cur_table(pts, u32) + pts->index;
		u32 old_entry = pts->entry;
		bool ret;

		/*
		* Ensure the zero'd table content itself is visible before its PTE can
		* be. release is a NOP on !SMP, but the HW is still doing an acquire.
		*/
		if (!IS_ENABLED(CONFIG_SMP))
		dma_wmb();
		ret = try_cmpxchg_release(entryp, &old_entry, table_entry);
		if (ret)
		pts->entry = table_entry;
		return ret;
		}

		#define PT_SUPPORTED_FEATURE(feature_nr) (PT_SUPPORTED_FEATURES & BIT(feature_nr))

		static inline bool pt_feature(const struct pt_common *common,
		unsigned int feature_nr)
		{
		if (PT_FORCE_ENABLED_FEATURES & BIT(feature_nr))
		return true;
		if (!PT_SUPPORTED_FEATURE(feature_nr))
		return false;
		return common->features & BIT(feature_nr);
		}

		static inline bool pts_feature(const struct pt_state *pts,
		unsigned int feature_nr)
		{
		return pt_feature(pts->range->common, feature_nr);
		}

		/*
		* PT_WARN_ON is used for invariants that the kunit should be checking can't
		* happen.
		*/
		#if IS_ENABLED(CONFIG_DEBUG_GENERIC_PT)
		#define PT_WARN_ON WARN_ON
		#else
		static inline bool PT_WARN_ON(bool condition)
		{
		return false;
		}
		#endif

		/* These all work on the VA type */
		#define log2_to_int(a_lg2) log2_to_int_t(pt_vaddr_t, a_lg2)
		#define log2_to_max_int(a_lg2) log2_to_max_int_t(pt_vaddr_t, a_lg2)
		#define log2_div(a, b_lg2) log2_div_t(pt_vaddr_t, a, b_lg2)
		#define log2_div_eq(a, b, c_lg2) log2_div_eq_t(pt_vaddr_t, a, b, c_lg2)
		#define log2_mod(a, b_lg2) log2_mod_t(pt_vaddr_t, a, b_lg2)
		#define log2_mod_eq_max(a, b_lg2) log2_mod_eq_max_t(pt_vaddr_t, a, b_lg2)
		#define log2_set_mod(a, val, b_lg2) log2_set_mod_t(pt_vaddr_t, a, val, b_lg2)
		#define log2_set_mod_max(a, b_lg2) log2_set_mod_max_t(pt_vaddr_t, a, b_lg2)
		#define log2_mul(a, b_lg2) log2_mul_t(pt_vaddr_t, a, b_lg2)
		#define vaffs(a) ffs_t(pt_vaddr_t, a)
		#define vafls(a) fls_t(pt_vaddr_t, a)
		#define vaffz(a) ffz_t(pt_vaddr_t, a)

		/*
		* The full VA (fva) versions permit the lg2 value to be == PT_VADDR_MAX_LG2 and
		* generate a useful defined result. The non-fva versions will malfunction at
		* this extreme.
		*/
		static inline pt_vaddr_t fvalog2_div(pt_vaddr_t a, unsigned int b_lg2)
		{
		if (PT_SUPPORTED_FEATURE(PT_FEAT_FULL_VA) && b_lg2 == PT_VADDR_MAX_LG2)
		return 0;
		return log2_div_t(pt_vaddr_t, a, b_lg2);
		}

		static inline pt_vaddr_t fvalog2_mod(pt_vaddr_t a, unsigned int b_lg2)
		{
		if (PT_SUPPORTED_FEATURE(PT_FEAT_FULL_VA) && b_lg2 == PT_VADDR_MAX_LG2)
		return a;
		return log2_mod_t(pt_vaddr_t, a, b_lg2);
		}

		static inline bool fvalog2_div_eq(pt_vaddr_t a, pt_vaddr_t b,
		unsigned int c_lg2)
		{
		if (PT_SUPPORTED_FEATURE(PT_FEAT_FULL_VA) && c_lg2 == PT_VADDR_MAX_LG2)
		return true;
		return log2_div_eq_t(pt_vaddr_t, a, b, c_lg2);
		}

		static inline pt_vaddr_t fvalog2_set_mod(pt_vaddr_t a, pt_vaddr_t val,
		unsigned int b_lg2)
		{
		if (PT_SUPPORTED_FEATURE(PT_FEAT_FULL_VA) && b_lg2 == PT_VADDR_MAX_LG2)
		return val;
		return log2_set_mod_t(pt_vaddr_t, a, val, b_lg2);
		}

		static inline pt_vaddr_t fvalog2_set_mod_max(pt_vaddr_t a, unsigned int b_lg2)
		{
		if (PT_SUPPORTED_FEATURE(PT_FEAT_FULL_VA) && b_lg2 == PT_VADDR_MAX_LG2)
		return PT_VADDR_MAX;
		return log2_set_mod_max_t(pt_vaddr_t, a, b_lg2);
		}

		/* These all work on the OA type */
		#define oalog2_to_int(a_lg2) log2_to_int_t(pt_oaddr_t, a_lg2)
		#define oalog2_to_max_int(a_lg2) log2_to_max_int_t(pt_oaddr_t, a_lg2)
		#define oalog2_div(a, b_lg2) log2_div_t(pt_oaddr_t, a, b_lg2)
		#define oalog2_div_eq(a, b, c_lg2) log2_div_eq_t(pt_oaddr_t, a, b, c_lg2)
		#define oalog2_mod(a, b_lg2) log2_mod_t(pt_oaddr_t, a, b_lg2)
		#define oalog2_mod_eq_max(a, b_lg2) log2_mod_eq_max_t(pt_oaddr_t, a, b_lg2)
		#define oalog2_set_mod(a, val, b_lg2) log2_set_mod_t(pt_oaddr_t, a, val, b_lg2)
		#define oalog2_set_mod_max(a, b_lg2) log2_set_mod_max_t(pt_oaddr_t, a, b_lg2)
		#define oalog2_mul(a, b_lg2) log2_mul_t(pt_oaddr_t, a, b_lg2)
		#define oaffs(a) ffs_t(pt_oaddr_t, a)
		#define oafls(a) fls_t(pt_oaddr_t, a)
		#define oaffz(a) ffz_t(pt_oaddr_t, a)

		static inline uintptr_t _pt_top_set(struct pt_table_p *table_mem,
		unsigned int top_level)
		{
		return top_level \| (uintptr_t)table_mem;
		}

		static inline void pt_top_set(struct pt_common *common,
		struct pt_table_p *table_mem,
		unsigned int top_level)
		{
		WRITE_ONCE(common->top_of_table, _pt_top_set(table_mem, top_level));
		}

		static inline void pt_top_set_level(struct pt_common *common,
		unsigned int top_level)
		{
		pt_top_set(common, NULL, top_level);
		}

		static inline unsigned int pt_top_get_level(const struct pt_common *common)
		{
		return READ_ONCE(common->top_of_table) % (1 << PT_TOP_LEVEL_BITS);
		}

		static inline bool pt_check_install_leaf_args(struct pt_state *pts,
		pt_oaddr_t oa,
		unsigned int oasz_lg2);

		#endif