Commit 93976a20 authored by Lorenzo Stoakes's avatar Lorenzo Stoakes Committed by Andrew Morton
Browse files

mm: eliminate further swapops predicates 

Having converted so much of the code base to software leaf entries, we can
mop up some remaining cases.

We replace is_pfn_swap_entry(), pfn_swap_entry_to_page(),
is_writable_device_private_entry(), is_device_exclusive_entry(),
is_migration_entry(), is_writable_migration_entry(),
is_readable_migration_entry(), swp_offset_pfn() and pfn_swap_entry_folio()
with softleaf equivalents.

No functional change intended.

Link: https://lkml.kernel.org/r/956bc9c031604811c0070d2f4bf2f1373f230213.1762812360.git.lorenzo.stoakes@oracle.com


Signed-off-by: default avatarLorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Chris Li <chrisl@kernel.org>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Claudio Imbrenda <imbrenda@linux.ibm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Janosch Frank <frankja@linux.ibm.com>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Leon Romanovsky <leon@kernel.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Naoya Horiguchi <nao.horiguchi@gmail.com>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Nico Pache <npache@redhat.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wei Xu <weixugc@google.com>
Cc: xu xin <xu.xin16@zte.com.cn>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 03bfbc3a

fs/proc/task_mmu.c

+7 −7
Original line number Diff line number Diff line
@@ -1941,13 +1941,13 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, 
		if (pte_uffd_wp(pte))

			flags |= PM_UFFD_WP;

	} else {

		swp_entry_t entry;

		softleaf_t entry;



		if (pte_swp_soft_dirty(pte))

			flags |= PM_SOFT_DIRTY;

		if (pte_swp_uffd_wp(pte))

			flags |= PM_UFFD_WP;

		entry = pte_to_swp_entry(pte);

		entry = softleaf_from_pte(pte);

		if (pm->show_pfn) {

			pgoff_t offset;
@@ -1955,16 +1955,16 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, 
			 * For PFN swap offsets, keeping the offset field

			 * to be PFN only to be compatible with old smaps.

			 */

			if (is_pfn_swap_entry(entry))

				offset = swp_offset_pfn(entry);

			if (softleaf_has_pfn(entry))

				offset = softleaf_to_pfn(entry);

			else

				offset = swp_offset(entry);

			frame = swp_type(entry) |

			    (offset << MAX_SWAPFILES_SHIFT);

		}

		flags |= PM_SWAP;

		if (is_pfn_swap_entry(entry))

			page = pfn_swap_entry_to_page(entry);

		if (softleaf_has_pfn(entry))

			page = softleaf_to_page(entry);

		if (softleaf_is_uffd_wp_marker(entry))

			flags |= PM_UFFD_WP;

		if (softleaf_is_guard_marker(entry))
@@ -2033,7 +2033,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, 
		if (pmd_swp_uffd_wp(pmd))

			flags |= PM_UFFD_WP;

		VM_WARN_ON_ONCE(!pmd_is_migration_entry(pmd));

		page = pfn_swap_entry_to_page(entry);

		page = softleaf_to_page(entry);

	}



	if (page) {

include/linux/leafops.h

+19 −6
Original line number Diff line number Diff line
@@ -355,7 +355,7 @@ static inline unsigned long softleaf_to_pfn(softleaf_t entry) 
	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));



	/* Temporary until swp_entry_t eliminated. */

	return swp_offset_pfn(entry);

	return swp_offset(entry) & SWP_PFN_MASK;

}



/**
@@ -366,10 +366,16 @@ static inline unsigned long softleaf_to_pfn(softleaf_t entry) 
 */

static inline struct page *softleaf_to_page(softleaf_t entry)

{

	struct page *page = pfn_to_page(softleaf_to_pfn(entry));



	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));

	/*

	 * Any use of migration entries may only occur while the

	 * corresponding page is locked

	 */

	VM_WARN_ON_ONCE(softleaf_is_migration(entry) && !PageLocked(page));



	/* Temporary until swp_entry_t eliminated. */

	return pfn_swap_entry_to_page(entry);

	return page;

}



/**
@@ -380,10 +386,17 @@ static inline struct page *softleaf_to_page(softleaf_t entry) 
 */

static inline struct folio *softleaf_to_folio(softleaf_t entry)

{

	struct folio *folio = pfn_folio(softleaf_to_pfn(entry));



	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));

	/*

	 * Any use of migration entries may only occur while the

	 * corresponding folio is locked.

	 */

	VM_WARN_ON_ONCE(softleaf_is_migration(entry) &&

			!folio_test_locked(folio));



	/* Temporary until swp_entry_t eliminated. */

	return pfn_swap_entry_folio(entry);

	return folio;

}



/**

include/linux/swapops.h

+1 −120
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ 
#define SWP_OFFSET_MASK	((1UL << SWP_TYPE_SHIFT) - 1)



/*

 * Definitions only for PFN swap entries (see is_pfn_swap_entry()).  To

 * Definitions only for PFN swap entries (see leafeant_has_pfn()).  To

 * store PFN, we only need SWP_PFN_BITS bits.  Each of the pfn swap entries

 * can use the extra bits to store other information besides PFN.

 */
@@ -66,8 +66,6 @@ 
#define SWP_MIG_YOUNG			BIT(SWP_MIG_YOUNG_BIT)

#define SWP_MIG_DIRTY			BIT(SWP_MIG_DIRTY_BIT)



static inline bool is_pfn_swap_entry(swp_entry_t entry);



/* Clear all flags but only keep swp_entry_t related information */

static inline pte_t pte_swp_clear_flags(pte_t pte)

{
@@ -109,17 +107,6 @@ static inline pgoff_t swp_offset(swp_entry_t entry) 
	return entry.val & SWP_OFFSET_MASK;

}



/*

 * This should only be called upon a pfn swap entry to get the PFN stored

 * in the swap entry.  Please refers to is_pfn_swap_entry() for definition

 * of pfn swap entry.

 */

static inline unsigned long swp_offset_pfn(swp_entry_t entry)

{

	VM_BUG_ON(!is_pfn_swap_entry(entry));

	return swp_offset(entry) & SWP_PFN_MASK;

}



/*

 * Convert the arch-dependent pte representation of a swp_entry_t into an

 * arch-independent swp_entry_t.
@@ -169,27 +156,11 @@ static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 
	return swp_entry(SWP_DEVICE_WRITE, offset);

}



static inline bool is_device_private_entry(swp_entry_t entry)

{

	int type = swp_type(entry);

	return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE;

}



static inline bool is_writable_device_private_entry(swp_entry_t entry)

{

	return unlikely(swp_type(entry) == SWP_DEVICE_WRITE);

}



static inline swp_entry_t make_device_exclusive_entry(pgoff_t offset)

{

	return swp_entry(SWP_DEVICE_EXCLUSIVE, offset);

}



static inline bool is_device_exclusive_entry(swp_entry_t entry)

{

	return swp_type(entry) == SWP_DEVICE_EXCLUSIVE;

}



#else /* CONFIG_DEVICE_PRIVATE */

static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)

{
@@ -201,50 +172,14 @@ static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 
	return swp_entry(0, 0);

}



static inline bool is_device_private_entry(swp_entry_t entry)

{

	return false;

}



static inline bool is_writable_device_private_entry(swp_entry_t entry)

{

	return false;

}



static inline swp_entry_t make_device_exclusive_entry(pgoff_t offset)

{

	return swp_entry(0, 0);

}



static inline bool is_device_exclusive_entry(swp_entry_t entry)

{

	return false;

}



#endif /* CONFIG_DEVICE_PRIVATE */



#ifdef CONFIG_MIGRATION

static inline int is_migration_entry(swp_entry_t entry)

{

	return unlikely(swp_type(entry) == SWP_MIGRATION_READ ||

			swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE ||

			swp_type(entry) == SWP_MIGRATION_WRITE);

}



static inline int is_writable_migration_entry(swp_entry_t entry)

{

	return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE);

}



static inline int is_readable_migration_entry(swp_entry_t entry)

{

	return unlikely(swp_type(entry) == SWP_MIGRATION_READ);

}



static inline int is_readable_exclusive_migration_entry(swp_entry_t entry)

{

	return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE);

}



static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)

{
@@ -310,23 +245,10 @@ static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 
	return swp_entry(0, 0);

}



static inline int is_migration_entry(swp_entry_t swp)

{

	return 0;

}



static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,

					unsigned long address) { }

static inline void migration_entry_wait_huge(struct vm_area_struct *vma,

					     unsigned long addr, pte_t *pte) { }

static inline int is_writable_migration_entry(swp_entry_t entry)

{

	return 0;

}

static inline int is_readable_migration_entry(swp_entry_t entry)

{

	return 0;

}



static inline swp_entry_t make_migration_entry_young(swp_entry_t entry)

{
@@ -410,47 +332,6 @@ static inline swp_entry_t make_guard_swp_entry(void) 
	return make_pte_marker_entry(PTE_MARKER_GUARD);

}



static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)

{

	struct page *p = pfn_to_page(swp_offset_pfn(entry));



	/*

	 * Any use of migration entries may only occur while the

	 * corresponding page is locked

	 */

	BUG_ON(is_migration_entry(entry) && !PageLocked(p));



	return p;

}



static inline struct folio *pfn_swap_entry_folio(swp_entry_t entry)

{

	struct folio *folio = pfn_folio(swp_offset_pfn(entry));



	/*

	 * Any use of migration entries may only occur while the

	 * corresponding folio is locked

	 */

	BUG_ON(is_migration_entry(entry) && !folio_test_locked(folio));



	return folio;

}



/*

 * A pfn swap entry is a special type of swap entry that always has a pfn stored

 * in the swap offset. They can either be used to represent unaddressable device

 * memory, to restrict access to a page undergoing migration or to represent a

 * pfn which has been hwpoisoned and unmapped.

 */

static inline bool is_pfn_swap_entry(swp_entry_t entry)

{

	/* Make sure the swp offset can always store the needed fields */

	BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);



	return is_migration_entry(entry) || is_device_private_entry(entry) ||

	       is_device_exclusive_entry(entry) || is_hwpoison_entry(entry);

}



struct page_vma_mapped_walk;



#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION

mm/debug_vm_pgtable.c

+10 −10
Original line number Diff line number Diff line
@@ -844,7 +844,7 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { } 
static void __init swap_migration_tests(struct pgtable_debug_args *args)

{

	struct page *page;

	swp_entry_t swp;

	softleaf_t entry;



	if (!IS_ENABLED(CONFIG_MIGRATION))

		return;
@@ -867,17 +867,17 @@ static void __init swap_migration_tests(struct pgtable_debug_args *args) 
	 * be locked, otherwise it stumbles upon a BUG_ON().

	 */

	__SetPageLocked(page);

	swp = make_writable_migration_entry(page_to_pfn(page));

	WARN_ON(!is_migration_entry(swp));

	WARN_ON(!is_writable_migration_entry(swp));

	entry = make_writable_migration_entry(page_to_pfn(page));

	WARN_ON(!softleaf_is_migration(entry));

	WARN_ON(!softleaf_is_migration_write(entry));



	swp = make_readable_migration_entry(swp_offset(swp));

	WARN_ON(!is_migration_entry(swp));

	WARN_ON(is_writable_migration_entry(swp));

	entry = make_readable_migration_entry(swp_offset(entry));

	WARN_ON(!softleaf_is_migration(entry));

	WARN_ON(softleaf_is_migration_write(entry));



	swp = make_readable_migration_entry(page_to_pfn(page));

	WARN_ON(!is_migration_entry(swp));

	WARN_ON(is_writable_migration_entry(swp));

	entry = make_readable_migration_entry(page_to_pfn(page));

	WARN_ON(!softleaf_is_migration(entry));

	WARN_ON(softleaf_is_migration_write(entry));

	__ClearPageLocked(page);

}

mm/hmm.c

+1 −1
Original line number Diff line number Diff line
@@ -270,7 +270,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, 
			cpu_flags = HMM_PFN_VALID;

			if (softleaf_is_device_private_write(entry))

				cpu_flags |= HMM_PFN_WRITE;

			new_pfn_flags = swp_offset_pfn(entry) | cpu_flags;

			new_pfn_flags = softleaf_to_pfn(entry) | cpu_flags;

			goto out;

		}