Commit 65941f10 authored by Yunsheng Lin's avatar Yunsheng Lin Committed by Jakub Kicinski
Browse files

mm: move the page fragment allocator from page_alloc into its own file 

Inspired by [1], move the page fragment allocator from page_alloc
into its own c file and header file, as we are about to make more
change for it to replace another page_frag implementation in
sock.c

As this patchset is going to replace 'struct page_frag' with
'struct page_frag_cache' in sched.h, including page_frag_cache.h
in sched.h has a compiler error caused by interdependence between
mm_types.h and mm.h for asm-offsets.c, see [2]. So avoid the compiler
error by moving 'struct page_frag_cache' to mm_types_task.h as
suggested by Alexander, see [3].

1. https://lore.kernel.org/all/20230411160902.4134381-3-dhowells@redhat.com/
2. https://lore.kernel.org/all/15623dac-9358-4597-b3ee-3694a5956920@gmail.com/
3. https://lore.kernel.org/all/CAKgT0UdH1yD=LSCXFJ=YM_aiA4OomD-2wXykO42bizaWMt_HOA@mail.gmail.com/


CC: David Howells <dhowells@redhat.com>
CC: Linux-MM <linux-mm@kvack.org>
Signed-off-by: default avatarYunsheng Lin <linyunsheng@huawei.com>
Acked-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Reviewed-by: default avatarAlexander Duyck <alexanderduyck@fb.com>
Link: https://patch.msgid.link/20241028115343.3405838-3-linyunsheng@huawei.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 7fef0dec

include/linux/gfp.h

+0 −22
Original line number Diff line number Diff line
@@ -371,28 +371,6 @@ __meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mas 
extern void __free_pages(struct page *page, unsigned int order);

extern void free_pages(unsigned long addr, unsigned int order);



struct page_frag_cache;

void page_frag_cache_drain(struct page_frag_cache *nc);

extern void __page_frag_cache_drain(struct page *page, unsigned int count);

void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz,

			      gfp_t gfp_mask, unsigned int align_mask);



static inline void *page_frag_alloc_align(struct page_frag_cache *nc,

					  unsigned int fragsz, gfp_t gfp_mask,

					  unsigned int align)

{

	WARN_ON_ONCE(!is_power_of_2(align));

	return __page_frag_alloc_align(nc, fragsz, gfp_mask, -align);

}



static inline void *page_frag_alloc(struct page_frag_cache *nc,

			     unsigned int fragsz, gfp_t gfp_mask)

{

	return __page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u);

}



extern void page_frag_free(void *addr);



#define __free_page(page) __free_pages((page), 0)

#define free_page(addr) free_pages((addr), 0)

include/linux/mm_types.h

+0 −18
Original line number Diff line number Diff line
@@ -521,9 +521,6 @@ static_assert(sizeof(struct ptdesc) <= sizeof(struct page)); 
 */

#define STRUCT_PAGE_MAX_SHIFT	(order_base_2(sizeof(struct page)))



#define PAGE_FRAG_CACHE_MAX_SIZE	__ALIGN_MASK(32768, ~PAGE_MASK)

#define PAGE_FRAG_CACHE_MAX_ORDER	get_order(PAGE_FRAG_CACHE_MAX_SIZE)



/*

 * page_private can be used on tail pages.  However, PagePrivate is only

 * checked by the VM on the head page.  So page_private on the tail pages
@@ -542,21 +539,6 @@ static inline void *folio_get_private(struct folio *folio) 
	return folio->private;

}



struct page_frag_cache {

	void * va;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)

	__u16 offset;

	__u16 size;

#else

	__u32 offset;

#endif

	/* we maintain a pagecount bias, so that we dont dirty cache line

	 * containing page->_refcount every time we allocate a fragment.

	 */

	unsigned int		pagecnt_bias;

	bool pfmemalloc;

};



typedef unsigned long vm_flags_t;



/*

include/linux/mm_types_task.h

+18 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
 * (These are defined separately to decouple sched.h from mm_types.h as much as possible.)

 */



#include <linux/align.h>

#include <linux/types.h>



#include <asm/page.h>
@@ -43,6 +44,23 @@ struct page_frag { 
#endif

};



#define PAGE_FRAG_CACHE_MAX_SIZE	__ALIGN_MASK(32768, ~PAGE_MASK)

#define PAGE_FRAG_CACHE_MAX_ORDER	get_order(PAGE_FRAG_CACHE_MAX_SIZE)

struct page_frag_cache {

	void *va;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)

	__u16 offset;

	__u16 size;

#else

	__u32 offset;

#endif

	/* we maintain a pagecount bias, so that we dont dirty cache line

	 * containing page->_refcount every time we allocate a fragment.

	 */

	unsigned int		pagecnt_bias;

	bool pfmemalloc;

};



/* Track pages that require TLB flushes */

struct tlbflush_unmap_batch {

#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH

include/linux/page_frag_cache.h

0 → 100644
+31 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#ifndef _LINUX_PAGE_FRAG_CACHE_H

#define _LINUX_PAGE_FRAG_CACHE_H



#include <linux/log2.h>

#include <linux/mm_types_task.h>

#include <linux/types.h>



void page_frag_cache_drain(struct page_frag_cache *nc);

void __page_frag_cache_drain(struct page *page, unsigned int count);

void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz,

			      gfp_t gfp_mask, unsigned int align_mask);



static inline void *page_frag_alloc_align(struct page_frag_cache *nc,

					  unsigned int fragsz, gfp_t gfp_mask,

					  unsigned int align)

{

	WARN_ON_ONCE(!is_power_of_2(align));

	return __page_frag_alloc_align(nc, fragsz, gfp_mask, -align);

}



static inline void *page_frag_alloc(struct page_frag_cache *nc,

				    unsigned int fragsz, gfp_t gfp_mask)

{

	return __page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u);

}



void page_frag_free(void *addr);



#endif

include/linux/skbuff.h

+1 −0
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ 
#include <linux/in6.h>

#include <linux/if_packet.h>

#include <linux/llist.h>

#include <linux/page_frag_cache.h>

#include <net/flow.h>

#if IS_ENABLED(CONFIG_NF_CONNTRACK)

#include <linux/netfilter/nf_conntrack_common.h>