Commit ca74b8ca authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

Merge series "slab: Re-entrant kmalloc_nolock()" 

From the cover letter [1]:

This patch set introduces kmalloc_nolock() which is the next logical
step towards any context allocation necessary to remove bpf_mem_alloc
and get rid of preallocation requirement in BPF infrastructure.

In production BPF maps grew to gigabytes in size. Preallocation wastes
memory. Alloc from any context addresses this issue for BPF and other
subsystems that are forced to preallocate too.

This long task started with introduction of alloc_pages_nolock(), then
memcg and objcg were converted to operate from any context including
NMI, this set completes the task with kmalloc_nolock() that builds on
top of alloc_pages_nolock() and memcg changes.

After that BPF subsystem will gradually adopt it everywhere.

Link: https://lore.kernel.org/all/20250909010007.1660-1-alexei.starovoitov@gmail.com/ [1]
parents b9120619 af92793e

include/linux/gfp.h

+1 −1
Original line number Diff line number Diff line
@@ -354,7 +354,7 @@ static inline struct page *alloc_page_vma_noprof(gfp_t gfp, 
}

#define alloc_page_vma(...)			alloc_hooks(alloc_page_vma_noprof(__VA_ARGS__))



struct page *alloc_pages_nolock_noprof(int nid, unsigned int order);

struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);

#define alloc_pages_nolock(...)			alloc_hooks(alloc_pages_nolock_noprof(__VA_ARGS__))



extern unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order);

include/linux/kasan.h

+8 −5
Original line number Diff line number Diff line
@@ -200,7 +200,7 @@ static __always_inline bool kasan_slab_pre_free(struct kmem_cache *s, 
}



bool __kasan_slab_free(struct kmem_cache *s, void *object, bool init,

		       bool still_accessible);

		       bool still_accessible, bool no_quarantine);

/**

 * kasan_slab_free - Poison, initialize, and quarantine a slab object.

 * @object: Object to be freed.
@@ -227,10 +227,12 @@ bool __kasan_slab_free(struct kmem_cache *s, void *object, bool init, 
 */

static __always_inline bool kasan_slab_free(struct kmem_cache *s,

					    void *object, bool init,

						bool still_accessible)

					    bool still_accessible,

					    bool no_quarantine)

{

	if (kasan_enabled())

		return __kasan_slab_free(s, object, init, still_accessible);

		return __kasan_slab_free(s, object, init, still_accessible,

					 no_quarantine);

	return false;

}
@@ -427,7 +429,8 @@ static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object) 
}



static inline bool kasan_slab_free(struct kmem_cache *s, void *object,

				   bool init, bool still_accessible)

				   bool init, bool still_accessible,

				   bool no_quarantine)

{

	return false;

}

include/linux/local_lock.h

+2 −0
Original line number Diff line number Diff line
@@ -66,6 +66,8 @@ 
 */

#define local_trylock(lock)		__local_trylock(this_cpu_ptr(lock))



#define local_lock_is_locked(lock)	__local_lock_is_locked(lock)



/**

 * local_trylock_irqsave - Try to acquire a per CPU local lock, save and disable

 *			   interrupts if acquired

include/linux/local_lock_internal.h

+7 −0
Original line number Diff line number Diff line
@@ -165,6 +165,9 @@ do { \ 
		!!tl;						\

	})



/* preemption or migration must be disabled before calling __local_lock_is_locked */

#define __local_lock_is_locked(lock) READ_ONCE(this_cpu_ptr(lock)->acquired)



#define __local_lock_release(lock)					\

	do {								\

		local_trylock_t *tl;					\
@@ -285,4 +288,8 @@ do { \ 
		__local_trylock(lock);				\

	})



/* migration must be disabled before calling __local_lock_is_locked */

#define __local_lock_is_locked(__lock)					\

	(rt_mutex_owner(&this_cpu_ptr(__lock)->lock) == current)



#endif /* CONFIG_PREEMPT_RT */

include/linux/memcontrol.h

+10 −2
Original line number Diff line number Diff line
@@ -341,17 +341,25 @@ enum page_memcg_data_flags { 
	__NR_MEMCG_DATA_FLAGS  = (1UL << 2),

};



#define __OBJEXTS_ALLOC_FAIL	MEMCG_DATA_OBJEXTS

#define __FIRST_OBJEXT_FLAG	__NR_MEMCG_DATA_FLAGS



#else /* CONFIG_MEMCG */



#define __OBJEXTS_ALLOC_FAIL	(1UL << 0)

#define __FIRST_OBJEXT_FLAG	(1UL << 0)



#endif /* CONFIG_MEMCG */



enum objext_flags {

	/* slabobj_ext vector failed to allocate */

	OBJEXTS_ALLOC_FAIL = __FIRST_OBJEXT_FLAG,

	/*

	 * Use bit 0 with zero other bits to signal that slabobj_ext vector

	 * failed to allocate. The same bit 0 with valid upper bits means

	 * MEMCG_DATA_OBJEXTS.

	 */

	OBJEXTS_ALLOC_FAIL = __OBJEXTS_ALLOC_FAIL,

	/* slabobj_ext vector allocated with kmalloc_nolock() */

	OBJEXTS_NOSPIN_ALLOC = __FIRST_OBJEXT_FLAG,

	/* the next bit after the last actual flag */

	__NR_OBJEXTS_FLAGS  = (__FIRST_OBJEXT_FLAG << 1),

};