Commit 99253de5 authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by Vlastimil Babka
Browse files

mm: Allow GFP_ACCOUNT to be used in alloc_pages_nolock(). 



Change alloc_pages_nolock() to default to __GFP_COMP when allocating
pages, since upcoming reentrant alloc_slab_page() needs __GFP_COMP.
Also allow __GFP_ACCOUNT flag to be specified,
since most of BPF infra needs __GFP_ACCOUNT except BPF streams.

Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Reviewed-by: default avatarShakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: default avatarHarry Yoo <harry.yoo@oracle.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent 4957089a

include/linux/gfp.h

+1 −1
Original line number Diff line number Diff line
@@ -354,7 +354,7 @@ static inline struct page *alloc_page_vma_noprof(gfp_t gfp, 
}

#define alloc_page_vma(...)			alloc_hooks(alloc_page_vma_noprof(__VA_ARGS__))



struct page *alloc_pages_nolock_noprof(int nid, unsigned int order);

struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);

#define alloc_pages_nolock(...)			alloc_hooks(alloc_pages_nolock_noprof(__VA_ARGS__))



extern unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order);

kernel/bpf/stream.c

+1 −1
Original line number Diff line number Diff line
@@ -83,7 +83,7 @@ static struct bpf_stream_page *bpf_stream_page_replace(void) 
	struct bpf_stream_page *stream_page, *old_stream_page;

	struct page *page;



	page = alloc_pages_nolock(NUMA_NO_NODE, 0);

	page = alloc_pages_nolock(/* Don't account */ 0, NUMA_NO_NODE, 0);

	if (!page)

		return NULL;

	stream_page = page_address(page);

kernel/bpf/syscall.c

+1 −1
Original line number Diff line number Diff line
@@ -581,7 +581,7 @@ static bool can_alloc_pages(void) 
static struct page *__bpf_alloc_page(int nid)

{

	if (!can_alloc_pages())

		return alloc_pages_nolock(nid, 0);

		return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);



	return alloc_pages_node(nid,

				GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT

mm/page_alloc.c

+6 −4
Original line number Diff line number Diff line
@@ -7480,6 +7480,7 @@ static bool __free_unaccepted(struct page *page) 


/**

 * alloc_pages_nolock - opportunistic reentrant allocation from any context

 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT allowed.

 * @nid: node to allocate from

 * @order: allocation order size

 *
@@ -7493,7 +7494,7 @@ static bool __free_unaccepted(struct page *page) 
 * Return: allocated page or NULL on failure. NULL does not mean EBUSY or EAGAIN.

 * It means ENOMEM. There is no reason to call it again and expect !NULL.

 */

struct page *alloc_pages_nolock_noprof(int nid, unsigned int order)

struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)

{

	/*

	 * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
@@ -7515,12 +7516,13 @@ struct page *alloc_pages_nolock_noprof(int nid, unsigned int order) 
	 * specify it here to highlight that alloc_pages_nolock()

	 * doesn't want to deplete reserves.

	 */

	gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC

			| __GFP_ACCOUNT;

	gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP

			| gfp_flags;

	unsigned int alloc_flags = ALLOC_TRYLOCK;

	struct alloc_context ac = { };

	struct page *page;



	VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);

	/*

	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is

	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
@@ -7558,7 +7560,7 @@ struct page *alloc_pages_nolock_noprof(int nid, unsigned int order) 
	if (page)

		set_page_refcounted(page);



	if (memcg_kmem_online() && page &&

	if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) &&

	    unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) {

		free_pages_nolock(page, order);

		page = NULL;