Commit bc617c99 authored by Nhat Pham's avatar Nhat Pham Committed by Andrew Morton
Browse files

mm/shmem, swap: remove SWAP_MAP_SHMEM 

The SWAP_MAP_SHMEM state was introduced in the commit aaa46865
("swap_info: note SWAP_MAP_SHMEM"), to quickly determine if a swap entry
belongs to shmem during swapoff.

However, swapoff has since been rewritten in the commit b56a2d8a ("mm:
rid swapoff of quadratic complexity").  Now having swap count ==
SWAP_MAP_SHMEM value is basically the same as having swap count == 1, and
swap_shmem_alloc() behaves analogously to swap_duplicate().  The only
difference of note is that swap_shmem_alloc() does not check for -ENOMEM
returned from __swap_duplicate(), but it is OK because shmem never
re-duplicates any swap entry it owns.  This will stil be safe if we use
(batched) swap_duplicate() instead.

This commit adds swap_duplicate_nr(), the batched variant of
swap_duplicate(), and removes the SWAP_MAP_SHMEM state and the associated
swap_shmem_alloc() helper to simplify the state machine (both mentally and
in terms of actual code).  We will also have an extra state/special value
that can be repurposed (for swap entries that never gets re-duplicated).

Link: https://lkml.kernel.org/r/20251220-swap-table-p2-v5-8-8862a265a033@tencent.com


Signed-off-by: default avatarKairui Song <kasong@tencent.com>
Signed-off-by: default avatarNhat Pham <nphamcs@gmail.com>
Reviewed-by: default avatarBaolin Wang <baolin.wang@linux.alibaba.com>
Tested-by: default avatarBaolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Chris Li <chrisl@kernel.org>
Cc: Rafael J. Wysocki (Intel) <rafael@kernel.org>
Cc: Yosry Ahmed <yosry.ahmed@linux.dev>
Cc: Deepanshu Kartikey <kartikey406@gmail.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kairui Song <ryncsn@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent c246d236

include/linux/swap.h

+7 −8
Original line number Diff line number Diff line
@@ -230,7 +230,6 @@ enum { 
/* Special value in first swap_map */

#define SWAP_MAP_MAX	0x3e	/* Max count */

#define SWAP_MAP_BAD	0x3f	/* Note page is bad */

#define SWAP_MAP_SHMEM	0xbf	/* Owned by shmem/tmpfs */



/* Special value in each swap_map continuation */

#define SWAP_CONT_MAX	0x7f	/* Max count */
@@ -458,8 +457,7 @@ bool folio_free_swap(struct folio *folio); 
void put_swap_folio(struct folio *folio, swp_entry_t entry);

extern swp_entry_t get_swap_page_of_type(int);

extern int add_swap_count_continuation(swp_entry_t, gfp_t);

extern void swap_shmem_alloc(swp_entry_t, int);

extern int swap_duplicate(swp_entry_t);

extern int swap_duplicate_nr(swp_entry_t entry, int nr);

extern int swapcache_prepare(swp_entry_t entry, int nr);

extern void swap_free_nr(swp_entry_t entry, int nr_pages);

extern void free_swap_and_cache_nr(swp_entry_t entry, int nr);
@@ -514,11 +512,7 @@ static inline int add_swap_count_continuation(swp_entry_t swp, gfp_t gfp_mask) 
	return 0;

}



static inline void swap_shmem_alloc(swp_entry_t swp, int nr)

{

}



static inline int swap_duplicate(swp_entry_t swp)

static inline int swap_duplicate_nr(swp_entry_t swp, int nr_pages)

{

	return 0;

}
@@ -569,6 +563,11 @@ static inline int add_swap_extent(struct swap_info_struct *sis, 
}

#endif /* CONFIG_SWAP */



static inline int swap_duplicate(swp_entry_t entry)

{

	return swap_duplicate_nr(entry, 1);

}



static inline void free_swap_and_cache(swp_entry_t entry)

{

	free_swap_and_cache_nr(entry, 1);

mm/shmem.c

+1 −1
Original line number Diff line number Diff line
@@ -1690,7 +1690,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, 
			spin_unlock(&shmem_swaplist_lock);

		}



		swap_shmem_alloc(folio->swap, nr_pages);

		swap_duplicate_nr(folio->swap, nr_pages);

		shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));



		BUG_ON(folio_mapped(folio));

mm/swapfile.c

+17 −25
Original line number Diff line number Diff line
@@ -201,7 +201,7 @@ static bool swap_is_last_map(struct swap_info_struct *si, 
	unsigned char *map_end = map + nr_pages;

	unsigned char count = *map;



	if (swap_count(count) != 1 && swap_count(count) != SWAP_MAP_SHMEM)

	if (swap_count(count) != 1)

		return false;



	while (++map < map_end) {
@@ -1523,12 +1523,6 @@ static unsigned char swap_entry_put_locked(struct swap_info_struct *si, 
	if (usage == SWAP_HAS_CACHE) {

		VM_BUG_ON(!has_cache);

		has_cache = 0;

	} else if (count == SWAP_MAP_SHMEM) {

		/*

		 * Or we could insist on shmem.c using a special

		 * swap_shmem_free() and free_shmem_swap_and_cache()...

		 */

		count = 0;

	} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {

		if (count == COUNT_CONTINUED) {

			if (swap_count_continued(si, offset, count))
@@ -1626,7 +1620,7 @@ static bool swap_entries_put_map(struct swap_info_struct *si, 
	if (nr <= 1)

		goto fallback;

	count = swap_count(data_race(si->swap_map[offset]));

	if (count != 1 && count != SWAP_MAP_SHMEM)

	if (count != 1)

		goto fallback;



	ci = swap_cluster_lock(si, offset);
@@ -1680,12 +1674,10 @@ static bool swap_entries_put_map_nr(struct swap_info_struct *si, 


/*

 * Check if it's the last ref of swap entry in the freeing path.

 * Qualified value includes 1, SWAP_HAS_CACHE or SWAP_MAP_SHMEM.

 */

static inline bool __maybe_unused swap_is_last_ref(unsigned char count)

{

	return (count == SWAP_HAS_CACHE) || (count == 1) ||

	       (count == SWAP_MAP_SHMEM);

	return (count == SWAP_HAS_CACHE) || (count == 1);

}



/*
@@ -3678,7 +3670,6 @@ static int __swap_duplicate(swp_entry_t entry, unsigned char usage, int nr) 


	offset = swp_offset(entry);

	VM_WARN_ON(nr > SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER);

	VM_WARN_ON(usage == 1 && nr > 1);

	ci = swap_cluster_lock(si, offset);



	err = 0;
@@ -3738,27 +3729,28 @@ static int __swap_duplicate(swp_entry_t entry, unsigned char usage, int nr) 
	return err;

}



/*

 * Help swapoff by noting that swap entry belongs to shmem/tmpfs

 * (in which case its reference count is never incremented).

 */

void swap_shmem_alloc(swp_entry_t entry, int nr)

{

	__swap_duplicate(entry, SWAP_MAP_SHMEM, nr);

}



/*

 * Increase reference count of swap entry by 1.

/**

 * swap_duplicate_nr() - Increase reference count of nr contiguous swap entries

 *                       by 1.

 *

 * @entry: first swap entry from which we want to increase the refcount.

 * @nr: Number of entries in range.

 *

 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required

 * but could not be atomically allocated.  Returns 0, just as if it succeeded,

 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which

 * might occur if a page table entry has got corrupted.

 *

 * Note that we are currently not handling the case where nr > 1 and we need to

 * add swap count continuation. This is OK, because no such user exists - shmem

 * is the only user that can pass nr > 1, and it never re-duplicates any swap

 * entry it owns.

 */

int swap_duplicate(swp_entry_t entry)

int swap_duplicate_nr(swp_entry_t entry, int nr)

{

	int err = 0;



	while (!err && __swap_duplicate(entry, 1, 1) == -ENOMEM)

	while (!err && __swap_duplicate(entry, 1, nr) == -ENOMEM)

		err = add_swap_count_continuation(entry, GFP_ATOMIC);

	return err;

}