mm: replace pmd_to_swp_entry() with softleaf_from_pmd() (0ac881ef) · Commits · git / linux-net

fs/proc/task_mmu.c

+13 −14

Original line number	Diff line number	Diff line
		@@ -1065,10 +1065,10 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
		page = vm_normal_page_pmd(vma, addr, *pmd);
		present = true;
		} else if (unlikely(thp_migration_supported())) {
		swp_entry_t entry = pmd_to_swp_entry(*pmd);
		const softleaf_t entry = softleaf_from_pmd(*pmd);

		if (is_pfn_swap_entry(entry))
		page = pfn_swap_entry_to_page(entry);
		if (softleaf_has_pfn(entry))
		page = softleaf_to_page(entry);
		}
		if (IS_ERR_OR_NULL(page))
		return;
		@@ -1655,7 +1655,7 @@ static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
		pmd = pmd_clear_soft_dirty(pmd);

		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
		} else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
		} else if (pmd_is_migration_entry(pmd)) {
		pmd = pmd_swp_clear_soft_dirty(pmd);
		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
		}
		@@ -2016,12 +2016,12 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr,
		if (pm->show_pfn)
		frame = pmd_pfn(pmd) + idx;
		} else if (thp_migration_supported()) {
		swp_entry_t entry = pmd_to_swp_entry(pmd);
		const softleaf_t entry = softleaf_from_pmd(pmd);
		unsigned long offset;

		if (pm->show_pfn) {
		if (is_pfn_swap_entry(entry))
		offset = swp_offset_pfn(entry) + idx;
		if (softleaf_has_pfn(entry))
		offset = softleaf_to_pfn(entry) + idx;
		else
		offset = swp_offset(entry) + idx;
		frame = swp_type(entry) \|
		@@ -2032,7 +2032,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr,
		flags \|= PM_SOFT_DIRTY;
		if (pmd_swp_uffd_wp(pmd))
		flags \|= PM_UFFD_WP;
		VM_WARN_ON_ONCE(!is_pmd_migration_entry(pmd));
		VM_WARN_ON_ONCE(!pmd_is_migration_entry(pmd));
		page = pfn_swap_entry_to_page(entry);
		}

		@@ -2426,8 +2426,6 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p,
		if (pmd_soft_dirty(pmd))
		categories \|= PAGE_IS_SOFT_DIRTY;
		} else {
		swp_entry_t swp;

		categories \|= PAGE_IS_SWAPPED;
		if (!pmd_swp_uffd_wp(pmd))
		categories \|= PAGE_IS_WRITTEN;
		@@ -2435,9 +2433,10 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p,
		categories \|= PAGE_IS_SOFT_DIRTY;

		if (p->masks_of_interest & PAGE_IS_FILE) {
		swp = pmd_to_swp_entry(pmd);
		if (is_pfn_swap_entry(swp) &&
		!folio_test_anon(pfn_swap_entry_folio(swp)))
		const softleaf_t entry = softleaf_from_pmd(pmd);

		if (softleaf_has_pfn(entry) &&
		!folio_test_anon(softleaf_to_folio(entry)))
		categories \|= PAGE_IS_FILE;
		}
		}
		@@ -2454,7 +2453,7 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma,
		old = pmdp_invalidate_ad(vma, addr, pmdp);
		pmd = pmd_mkuffd_wp(old);
		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
		} else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
		} else if (pmd_is_migration_entry(pmd)) {
		pmd = pmd_swp_mkuffd_wp(pmd);
		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
		}

include/linux/leafops.h

+216 −2

Original line number	Diff line number	Diff line
		@@ -61,6 +61,57 @@ static inline softleaf_t softleaf_from_pte(pte_t pte)
		return pte_to_swp_entry(pte);
		}

		/**
		* softleaf_to_pte() - Obtain a PTE entry from a leaf entry.
		* @entry: Leaf entry.
		*
		* This generates an architecture-specific PTE entry that can be utilised to
		* encode the metadata the leaf entry encodes.
		*
		* Returns: Architecture-specific PTE entry encoding leaf entry.
		*/
		static inline pte_t softleaf_to_pte(softleaf_t entry)
		{
		/* Temporary until swp_entry_t eliminated. */
		return swp_entry_to_pte(entry);
		}

		#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
		/**
		* softleaf_from_pmd() - Obtain a leaf entry from a PMD entry.
		* @pmd: PMD entry.
		*
		* If @pmd is present (therefore not a leaf entry) the function returns an empty
		* leaf entry. Otherwise, it returns a leaf entry.
		*
		* Returns: Leaf entry.
		*/
		static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
		{
		softleaf_t arch_entry;

		if (pmd_present(pmd) \|\| pmd_none(pmd))
		return softleaf_mk_none();

		if (pmd_swp_soft_dirty(pmd))
		pmd = pmd_swp_clear_soft_dirty(pmd);
		if (pmd_swp_uffd_wp(pmd))
		pmd = pmd_swp_clear_uffd_wp(pmd);
		arch_entry = __pmd_to_swp_entry(pmd);

		/* Temporary until swp_entry_t eliminated. */
		return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
		}

		#else

		static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
		{
		return softleaf_mk_none();
		}

		#endif

		/**
		* softleaf_is_none() - Is the leaf entry empty?
		* @entry: Leaf entry.
		@@ -134,6 +185,43 @@ static inline bool softleaf_is_swap(softleaf_t entry)
		return softleaf_type(entry) == SOFTLEAF_SWAP;
		}

		/**
		* softleaf_is_migration_write() - Is this leaf entry a writable migration entry?
		* @entry: Leaf entry.
		*
		* Returns: true if the leaf entry is a writable migration entry, otherwise
		* false.
		*/
		static inline bool softleaf_is_migration_write(softleaf_t entry)
		{
		return softleaf_type(entry) == SOFTLEAF_MIGRATION_WRITE;
		}

		/**
		* softleaf_is_migration_read() - Is this leaf entry a readable migration entry?
		* @entry: Leaf entry.
		*
		* Returns: true if the leaf entry is a readable migration entry, otherwise
		* false.
		*/
		static inline bool softleaf_is_migration_read(softleaf_t entry)
		{
		return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ;
		}

		/**
		* softleaf_is_migration_read_exclusive() - Is this leaf entry an exclusive
		* readable migration entry?
		* @entry: Leaf entry.
		*
		* Returns: true if the leaf entry is an exclusive readable migration entry,
		* otherwise false.
		*/
		static inline bool softleaf_is_migration_read_exclusive(softleaf_t entry)
		{
		return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
		}

		/**
		* softleaf_is_migration() - Is this leaf entry a migration entry?
		* @entry: Leaf entry.
		@@ -152,6 +240,19 @@ static inline bool softleaf_is_migration(softleaf_t entry)
		}
		}

		/**
		* softleaf_is_device_private_write() - Is this leaf entry a device private
		* writable entry?
		* @entry: Leaf entry.
		*
		* Returns: true if the leaf entry is a device private writable entry, otherwise
		* false.
		*/
		static inline bool softleaf_is_device_private_write(softleaf_t entry)
		{
		return softleaf_type(entry) == SOFTLEAF_DEVICE_PRIVATE_WRITE;
		}

		/**
		* softleaf_is_device_private() - Is this leaf entry a device private entry?
		* @entry: Leaf entry.
		@@ -170,10 +271,10 @@ static inline bool softleaf_is_device_private(softleaf_t entry)
		}

		/**
		* softleaf_is_device_exclusive() - Is this leaf entry a device exclusive entry?
		* softleaf_is_device_exclusive() - Is this leaf entry a device-exclusive entry?
		* @entry: Leaf entry.
		*
		* Returns: true if the leaf entry is a device exclusive entry, otherwise false.
		* Returns: true if the leaf entry is a device-exclusive entry, otherwise false.
		*/
		static inline bool softleaf_is_device_exclusive(softleaf_t entry)
		{
		@@ -332,6 +433,61 @@ static inline bool softleaf_is_uffd_wp_marker(softleaf_t entry)
		return softleaf_to_marker(entry) & PTE_MARKER_UFFD_WP;
		}

		#ifdef CONFIG_MIGRATION

		/**
		* softleaf_is_migration_young() - Does this migration entry contain an accessed
		* bit?
		* @entry: Leaf entry.
		*
		* If the architecture can support storing A/D bits in migration entries, this
		* determines whether the accessed (or 'young') bit was set on the migrated page
		* table entry.
		*
		* Returns: true if the entry contains an accessed bit, otherwise false.
		*/
		static inline bool softleaf_is_migration_young(softleaf_t entry)
		{
		VM_WARN_ON_ONCE(!softleaf_is_migration(entry));

		if (migration_entry_supports_ad())
		return swp_offset(entry) & SWP_MIG_YOUNG;
		/* Keep the old behavior of aging page after migration */
		return false;
		}

		/**
		* softleaf_is_migration_dirty() - Does this migration entry contain a dirty bit?
		* @entry: Leaf entry.
		*
		* If the architecture can support storing A/D bits in migration entries, this
		* determines whether the dirty bit was set on the migrated page table entry.
		*
		* Returns: true if the entry contains a dirty bit, otherwise false.
		*/
		static inline bool softleaf_is_migration_dirty(softleaf_t entry)
		{
		VM_WARN_ON_ONCE(!softleaf_is_migration(entry));

		if (migration_entry_supports_ad())
		return swp_offset(entry) & SWP_MIG_DIRTY;
		/* Keep the old behavior of clean page after migration */
		return false;
		}

		#else /* CONFIG_MIGRATION */

		static inline bool softleaf_is_migration_young(softleaf_t entry)
		{
		return false;
		}

		static inline bool softleaf_is_migration_dirty(softleaf_t entry)
		{
		return false;
		}
		#endif /* CONFIG_MIGRATION */

		/**
		* pte_is_marker() - Does the PTE entry encode a marker leaf entry?
		* @pte: PTE entry.
		@@ -383,5 +539,63 @@ static inline bool pte_is_uffd_marker(pte_t pte)
		return false;
		}

		#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)

		/**
		* pmd_is_device_private_entry() - Check if PMD contains a device private swap
		* entry.
		* @pmd: The PMD to check.
		*
		* Returns true if the PMD contains a swap entry that represents a device private
		* page mapping. This is used for zone device private pages that have been
		* swapped out but still need special handling during various memory management
		* operations.
		*
		* Return: true if PMD contains device private entry, false otherwise
		*/
		static inline bool pmd_is_device_private_entry(pmd_t pmd)
		{
		return softleaf_is_device_private(softleaf_from_pmd(pmd));
		}

		#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */

		static inline bool pmd_is_device_private_entry(pmd_t pmd)
		{
		return false;
		}

		#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */

		/**
		* pmd_is_migration_entry() - Does this PMD entry encode a migration entry?
		* @pmd: PMD entry.
		*
		* Returns: true if the PMD encodes a migration entry, otherwise false.
		*/
		static inline bool pmd_is_migration_entry(pmd_t pmd)
		{
		return softleaf_is_migration(softleaf_from_pmd(pmd));
		}

		/**
		* pmd_is_valid_softleaf() - Is this PMD entry a valid leaf entry?
		* @pmd: PMD entry.
		*
		* PMD leaf entries are valid only if they are device private or migration
		* entries. This function asserts that a PMD leaf entry is valid in this
		* respect.
		*
		* Returns: true if the PMD entry is a valid leaf entry, otherwise false.
		*/
		static inline bool pmd_is_valid_softleaf(pmd_t pmd)
		{
		const softleaf_t entry = softleaf_from_pmd(pmd);

		/* Only device private, migration entries valid for PMD. */
		return softleaf_is_device_private(entry) \|\|
		softleaf_is_migration(entry);
		}

		#endif /* CONFIG_MMU */
		#endif /* _LINUX_LEAFOPS_H */

include/linux/migrate.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -65,7 +65,7 @@ bool isolate_folio_to_list(struct folio folio, struct list_head list);

		int migrate_huge_page_move_mapping(struct address_space *mapping,
		struct folio dst, struct folio src);
		void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl)
		void migration_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl)
		__releases(ptl);
		void folio_migrate_flags(struct folio newfolio, struct folio folio);
		int folio_migrate_mapping(struct address_space *mapping,

include/linux/swapops.h

+0 −100

Original line number	Diff line number	Diff line
		@@ -283,14 +283,6 @@ static inline swp_entry_t make_migration_entry_young(swp_entry_t entry)
		return entry;
		}

		static inline bool is_migration_entry_young(swp_entry_t entry)
		{
		if (migration_entry_supports_ad())
		return swp_offset(entry) & SWP_MIG_YOUNG;
		/* Keep the old behavior of aging page after migration */
		return false;
		}

		static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry)
		{
		if (migration_entry_supports_ad())
		@@ -299,14 +291,6 @@ static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry)
		return entry;
		}

		static inline bool is_migration_entry_dirty(swp_entry_t entry)
		{
		if (migration_entry_supports_ad())
		return swp_offset(entry) & SWP_MIG_DIRTY;
		/* Keep the old behavior of clean page after migration */
		return false;
		}

		extern void migration_entry_wait(struct mm_struct mm, pmd_t pmd,
		unsigned long address);
		extern void migration_entry_wait_huge(struct vm_area_struct vma, unsigned long addr, pte_t pte);
		@@ -349,20 +333,11 @@ static inline swp_entry_t make_migration_entry_young(swp_entry_t entry)
		return entry;
		}

		static inline bool is_migration_entry_young(swp_entry_t entry)
		{
		return false;
		}

		static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry)
		{
		return entry;
		}

		static inline bool is_migration_entry_dirty(swp_entry_t entry)
		{
		return false;
		}
		#endif /* CONFIG_MIGRATION */

		#ifdef CONFIG_MEMORY_FAILURE
		@@ -487,18 +462,6 @@ extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,

		extern void pmd_migration_entry_wait(struct mm_struct mm, pmd_t pmd);

		static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
		{
		swp_entry_t arch_entry;

		if (pmd_swp_soft_dirty(pmd))
		pmd = pmd_swp_clear_soft_dirty(pmd);
		if (pmd_swp_uffd_wp(pmd))
		pmd = pmd_swp_clear_uffd_wp(pmd);
		arch_entry = __pmd_to_swp_entry(pmd);
		return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
		}

		static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
		{
		swp_entry_t arch_entry;
		@@ -507,23 +470,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
		return __swp_entry_to_pmd(arch_entry);
		}

		static inline int is_pmd_migration_entry(pmd_t pmd)
		{
		swp_entry_t entry;

		if (pmd_present(pmd))
		return 0;

		entry = pmd_to_swp_entry(pmd);
		return is_migration_entry(entry);
		}
		#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
		static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
		struct page *page)
		{
		BUILD_BUG();
		}

		static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
		struct page *new)
		{
		@@ -532,64 +479,17 @@ static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,

		static inline void pmd_migration_entry_wait(struct mm_struct m, pmd_t p) { }

		static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
		{
		return swp_entry(0, 0);
		}

		static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
		{
		return __pmd(0);
		}

		static inline int is_pmd_migration_entry(pmd_t pmd)
		{
		return 0;
		}
		#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */

		#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)

		/**
		* is_pmd_device_private_entry() - Check if PMD contains a device private swap entry
		* @pmd: The PMD to check
		*
		* Returns true if the PMD contains a swap entry that represents a device private
		* page mapping. This is used for zone device private pages that have been
		* swapped out but still need special handling during various memory management
		* operations.
		*
		* Return: 1 if PMD contains device private entry, 0 otherwise
		*/
		static inline int is_pmd_device_private_entry(pmd_t pmd)
		{
		swp_entry_t entry;

		if (pmd_present(pmd))
		return 0;

		entry = pmd_to_swp_entry(pmd);
		return is_device_private_entry(entry);
		}

		#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */

		static inline int is_pmd_device_private_entry(pmd_t pmd)
		{
		return 0;
		}

		#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */

		static inline int non_swap_entry(swp_entry_t entry)
		{
		return swp_type(entry) >= MAX_SWAPFILES;
		}

		static inline int is_pmd_non_present_folio_entry(pmd_t pmd)
		{
		return is_pmd_migration_entry(pmd) \|\| is_pmd_device_private_entry(pmd);
		}

		#endif /* CONFIG_MMU */
		#endif /* _LINUX_SWAPOPS_H */

mm/damon/ops-common.c

+3 −3

Original line number	Diff line number	Diff line
		@@ -11,7 +11,7 @@
		#include <linux/pagemap.h>
		#include <linux/rmap.h>
		#include <linux/swap.h>
		#include <linux/swapops.h>
		#include <linux/leafops.h>

		#include "../internal.h"
		#include "ops-common.h"
		@@ -51,7 +51,7 @@ void damon_ptep_mkold(pte_t pte, struct vm_area_struct vma, unsigned long addr
		if (likely(pte_present(pteval)))
		pfn = pte_pfn(pteval);
		else
		pfn = swp_offset_pfn(pte_to_swp_entry(pteval));
		pfn = softleaf_to_pfn(softleaf_from_pte(pteval));

		folio = damon_get_folio(pfn);
		if (!folio)
		@@ -83,7 +83,7 @@ void damon_pmdp_mkold(pmd_t pmd, struct vm_area_struct vma, unsigned long addr
		if (likely(pmd_present(pmdval)))
		pfn = pmd_pfn(pmdval);
		else
		pfn = swp_offset_pfn(pmd_to_swp_entry(pmdval));
		pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval));

		folio = damon_get_folio(pfn);
		if (!folio)