Commit 24d9e8b3 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'slab-for-6.18' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab 

Pull slab updates from Vlastimil Babka:

 - A new layer for caching objects for allocation and free via percpu
   arrays called sheaves.

   The aim is to combine the good parts of SLAB (lower-overhead and
   simpler percpu caching, compared to SLUB) without the past issues
   with arrays for freeing remote NUMA node objects and their flushing.

   It also allows more efficient kfree_rcu(), and cheaper object
   preallocations for cases where the exact number of objects is
   unknown, but an upper bound is.

   Currently VMAs and maple nodes are using this new caching, with a
   plan to enable it for all caches and remove the complex SLUB fastpath
   based on cpu (partial) slabs and this_cpu_cmpxchg_double().
   (Vlastimil Babka, with Liam Howlett and Pedro Falcato for the maple
   tree changes)

 - Re-entrant kmalloc_nolock(), which allows opportunistic allocations
   from NMI and tracing/kprobe contexts.

   Building on prior page allocator and memcg changes, it will result in
   removing BPF-specific caches on top of slab (Alexei Starovoitov)

 - Various fixes and cleanups. (Kuan-Wei Chiu, Matthew Wilcox, Suren
   Baghdasaryan, Ye Liu)

* tag 'slab-for-6.18' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab: (40 commits)
  slab: Introduce kmalloc_nolock() and kfree_nolock().
  slab: Reuse first bit for OBJEXTS_ALLOC_FAIL
  slab: Make slub local_(try)lock more precise for LOCKDEP
  mm: Introduce alloc_frozen_pages_nolock()
  mm: Allow GFP_ACCOUNT to be used in alloc_pages_nolock().
  locking/local_lock: Introduce local_lock_is_locked().
  maple_tree: Convert forking to use the sheaf interface
  maple_tree: Add single node allocation support to maple state
  maple_tree: Prefilled sheaf conversion and testing
  tools/testing: Add support for prefilled slab sheafs
  maple_tree: Replace mt_free_one() with kfree()
  maple_tree: Use kfree_rcu in ma_free_rcu
  testing/radix-tree/maple: Hack around kfree_rcu not existing
  tools/testing: include maple-shim.c in maple.c
  maple_tree: use percpu sheaves for maple_node_cache
  mm, vma: use percpu sheaves for vm_area_struct cache
  tools/testing: Add support for changes to slab for sheaves
  slab: allow NUMA restricted allocations to use percpu sheaves
  tools/testing/vma: Implement vm_refcnt reset
  slab: skip percpu sheaves for remote object freeing
  ...
parents 07fdad3a ca74b8ca

include/linux/gfp.h

+1 −1
Original line number Diff line number Diff line
@@ -354,7 +354,7 @@ static inline struct page *alloc_page_vma_noprof(gfp_t gfp, 
}

#define alloc_page_vma(...)			alloc_hooks(alloc_page_vma_noprof(__VA_ARGS__))



struct page *alloc_pages_nolock_noprof(int nid, unsigned int order);

struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);

#define alloc_pages_nolock(...)			alloc_hooks(alloc_pages_nolock_noprof(__VA_ARGS__))



extern unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order);

include/linux/kasan.h

+8 −5
Original line number Diff line number Diff line
@@ -200,7 +200,7 @@ static __always_inline bool kasan_slab_pre_free(struct kmem_cache *s, 
}



bool __kasan_slab_free(struct kmem_cache *s, void *object, bool init,

		       bool still_accessible);

		       bool still_accessible, bool no_quarantine);

/**

 * kasan_slab_free - Poison, initialize, and quarantine a slab object.

 * @object: Object to be freed.
@@ -227,10 +227,12 @@ bool __kasan_slab_free(struct kmem_cache *s, void *object, bool init, 
 */

static __always_inline bool kasan_slab_free(struct kmem_cache *s,

					    void *object, bool init,

						bool still_accessible)

					    bool still_accessible,

					    bool no_quarantine)

{

	if (kasan_enabled())

		return __kasan_slab_free(s, object, init, still_accessible);

		return __kasan_slab_free(s, object, init, still_accessible,

					 no_quarantine);

	return false;

}
@@ -427,7 +429,8 @@ static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object) 
}



static inline bool kasan_slab_free(struct kmem_cache *s, void *object,

				   bool init, bool still_accessible)

				   bool init, bool still_accessible,

				   bool no_quarantine)

{

	return false;

}

include/linux/local_lock.h

+2 −0
Original line number Diff line number Diff line
@@ -66,6 +66,8 @@ 
 */

#define local_trylock(lock)		__local_trylock(this_cpu_ptr(lock))



#define local_lock_is_locked(lock)	__local_lock_is_locked(lock)



/**

 * local_trylock_irqsave - Try to acquire a per CPU local lock, save and disable

 *			   interrupts if acquired

include/linux/local_lock_internal.h

+13 −3
Original line number Diff line number Diff line
@@ -17,7 +17,10 @@ typedef struct { 


/* local_trylock() and local_trylock_irqsave() only work with local_trylock_t */

typedef struct {

	local_lock_t	llock;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map	dep_map;

	struct task_struct	*owner;

#endif

	u8		acquired;

} local_trylock_t;
@@ -31,7 +34,7 @@ typedef struct { 
	.owner = NULL,



# define LOCAL_TRYLOCK_DEBUG_INIT(lockname)		\

	.llock = { LOCAL_LOCK_DEBUG_INIT((lockname).llock) },

	LOCAL_LOCK_DEBUG_INIT(lockname)



static inline void local_lock_acquire(local_lock_t *l)

{
@@ -81,7 +84,7 @@ do { \ 
	local_lock_debug_init(lock);				\

} while (0)



#define __local_trylock_init(lock) __local_lock_init(lock.llock)

#define __local_trylock_init(lock) __local_lock_init((local_lock_t *)lock)



#define __spinlock_nested_bh_init(lock)				\

do {								\
@@ -162,6 +165,9 @@ do { \ 
		!!tl;						\

	})



/* preemption or migration must be disabled before calling __local_lock_is_locked */

#define __local_lock_is_locked(lock) READ_ONCE(this_cpu_ptr(lock)->acquired)



#define __local_lock_release(lock)					\

	do {								\

		local_trylock_t *tl;					\
@@ -282,4 +288,8 @@ do { \ 
		__local_trylock(lock);				\

	})



/* migration must be disabled before calling __local_lock_is_locked */

#define __local_lock_is_locked(__lock)					\

	(rt_mutex_owner(&this_cpu_ptr(__lock)->lock) == current)



#endif /* CONFIG_PREEMPT_RT */

include/linux/maple_tree.h

+5 −1
Original line number Diff line number Diff line
@@ -442,7 +442,9 @@ struct ma_state { 
	struct maple_enode *node;	/* The node containing this entry */

	unsigned long min;		/* The minimum index of this node - implied pivot min */

	unsigned long max;		/* The maximum index of this node - implied pivot max */

	struct maple_alloc *alloc;	/* Allocated nodes for this operation */

	struct slab_sheaf *sheaf;	/* Allocated nodes for this operation */

	struct maple_node *alloc;	/* A single allocated node for fast path writes */

	unsigned long node_request;	/* The number of nodes to allocate for this operation */

	enum maple_status status;	/* The status of the state (active, start, none, etc) */

	unsigned char depth;		/* depth of tree descent during write */

	unsigned char offset;
@@ -490,7 +492,9 @@ struct ma_wr_state { 
		.status = ma_start,					\

		.min = 0,						\

		.max = ULONG_MAX,					\

		.sheaf = NULL,						\

		.alloc = NULL,						\

		.node_request = 0,					\

		.mas_flags = 0,						\

		.store_type = wr_invalid,				\

	}