Commit 2e73ff23 authored by Gang Li's avatar Gang Li Committed by Andrew Morton
Browse files

hugetlb: pass *next_nid_to_alloc directly to for_each_node_mask_to_alloc 

With parallelization of hugetlb allocation across different threads, each
thread works on a differnet node to allocate pages from, instead of all
allocating from a common node h->next_nid_to_alloc.  To address this, it's
necessary to assign a separate next_nid_to_alloc for each thread.

Consequently, the hstate_next_node_to_alloc and
for_each_node_mask_to_alloc have been modified to directly accept a
*next_nid_to_alloc parameter, ensuring thread-specific allocation and
avoiding concurrent access issues.

Link: https://lkml.kernel.org/r/20240222140422.393911-4-gang.li@linux.dev


Signed-off-by: default avatarGang Li <ligang.bdlg@bytedance.com>
Tested-by: default avatarDavid Rientjes <rientjes@google.com>
Reviewed-by: default avatarTim Chen <tim.c.chen@linux.intel.com>
Reviewed-by: default avatarMuchun Song <muchun.song@linux.dev>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Daniel Jordan <daniel.m.jordan@oracle.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Steffen Klassert <steffen.klassert@secunet.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent d5c3eb3f

mm/hugetlb.c

+12 −10
Original line number Diff line number Diff line
@@ -1464,15 +1464,15 @@ static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 
 * next node from which to allocate, handling wrap at end of node

 * mask.

 */

static int hstate_next_node_to_alloc(struct hstate *h,

static int hstate_next_node_to_alloc(int *next_node,

					nodemask_t *nodes_allowed)

{

	int nid;



	VM_BUG_ON(!nodes_allowed);



	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);

	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);

	nid = get_valid_node_allowed(*next_node, nodes_allowed);

	*next_node = next_node_allowed(nid, nodes_allowed);



	return nid;

}
@@ -1495,10 +1495,10 @@ static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 
	return nid;

}



#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\

#define for_each_node_mask_to_alloc(next_node, nr_nodes, node, mask)		\

	for (nr_nodes = nodes_weight(*mask);				\

		nr_nodes > 0 &&						\

		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\

		((node = hstate_next_node_to_alloc(next_node, mask)) || 1);	\

		nr_nodes--)



#define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
@@ -2350,12 +2350,13 @@ static void prep_and_add_allocated_folios(struct hstate *h, 
 */

static struct folio *alloc_pool_huge_folio(struct hstate *h,

					nodemask_t *nodes_allowed,

					nodemask_t *node_alloc_noretry)

					nodemask_t *node_alloc_noretry,

					int *next_node)

{

	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;

	int nr_nodes, node;



	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {

	for_each_node_mask_to_alloc(next_node, nr_nodes, node, nodes_allowed) {

		struct folio *folio;



		folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, node,
@@ -3310,7 +3311,7 @@ int __alloc_bootmem_huge_page(struct hstate *h, int nid) 
		goto found;

	}

	/* allocate from next node when distributing huge pages */

	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {

	for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, &node_states[N_MEMORY]) {

		m = memblock_alloc_try_nid_raw(

				huge_page_size(h), huge_page_size(h),

				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
@@ -3679,7 +3680,7 @@ static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 
	VM_BUG_ON(delta != -1 && delta != 1);



	if (delta < 0) {

		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {

		for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, nodes_allowed) {

			if (h->surplus_huge_pages_node[node])

				goto found;

		}
@@ -3794,7 +3795,8 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, 
		cond_resched();



		folio = alloc_pool_huge_folio(h, nodes_allowed,

						node_alloc_noretry);

						node_alloc_noretry,

						&h->next_nid_to_alloc);

		if (!folio) {

			prep_and_add_allocated_folios(h, &page_list);

			spin_lock_irq(&hugetlb_lock);