Commit 21c690a3 authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

mm: introduce slabobj_ext to support slab object extensions 

Currently slab pages can store only vectors of obj_cgroup pointers in
page->memcg_data.  Introduce slabobj_ext structure to allow more data to
be stored for each slab object.  Wrap obj_cgroup into slabobj_ext to
support current functionality while allowing to extend slabobj_ext in the
future.

Link: https://lkml.kernel.org/r/20240321163705.3067592-7-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Reviewed-by: default avatarPasha Tatashin <pasha.tatashin@soleen.com>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Tested-by: default avatarKees Cook <keescook@chromium.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andreas Hindborg <a.hindborg@samsung.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Gary Guo <gary@garyguo.net>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Miguel Ojeda <ojeda@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent a5674119

include/linux/memcontrol.h

+14 −6
Original line number Diff line number Diff line
@@ -349,8 +349,8 @@ struct mem_cgroup { 
extern struct mem_cgroup *root_mem_cgroup;



enum page_memcg_data_flags {

	/* page->memcg_data is a pointer to an objcgs vector */

	MEMCG_DATA_OBJCGS = (1UL << 0),

	/* page->memcg_data is a pointer to an slabobj_ext vector */

	MEMCG_DATA_OBJEXTS = (1UL << 0),

	/* page has been accounted as a non-slab kernel page */

	MEMCG_DATA_KMEM = (1UL << 1),

	/* the next bit after the last actual flag */
@@ -388,7 +388,7 @@ static inline struct mem_cgroup *__folio_memcg(struct folio *folio) 
	unsigned long memcg_data = folio->memcg_data;



	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);

	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);

	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);

	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);



	return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
@@ -409,7 +409,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) 
	unsigned long memcg_data = folio->memcg_data;



	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);

	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);

	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);

	VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);



	return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
@@ -506,7 +506,7 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) 
	 */

	unsigned long memcg_data = READ_ONCE(folio->memcg_data);



	if (memcg_data & MEMCG_DATA_OBJCGS)

	if (memcg_data & MEMCG_DATA_OBJEXTS)

		return NULL;



	if (memcg_data & MEMCG_DATA_KMEM) {
@@ -552,7 +552,7 @@ static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *ob 
static inline bool folio_memcg_kmem(struct folio *folio)

{

	VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);

	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio);

	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio);

	return folio->memcg_data & MEMCG_DATA_KMEM;

}
@@ -1633,6 +1633,14 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 
}

#endif /* CONFIG_MEMCG */



/*

 * Extended information for slab objects stored as an array in page->memcg_data

 * if MEMCG_DATA_OBJEXTS is set.

 */

struct slabobj_ext {

	struct obj_cgroup *objcg;

} __aligned(8);



static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx)

{

	__mod_lruvec_kmem_state(p, idx, 1);

include/linux/mm_types.h

+2 −2
Original line number Diff line number Diff line
@@ -169,7 +169,7 @@ struct page { 
	/* Usage count. *DO NOT USE DIRECTLY*. See page_ref.h */

	atomic_t _refcount;



#ifdef CONFIG_MEMCG

#ifdef CONFIG_SLAB_OBJ_EXT

	unsigned long memcg_data;

#endif
@@ -331,7 +331,7 @@ struct folio { 
			};

			atomic_t _mapcount;

			atomic_t _refcount;

#ifdef CONFIG_MEMCG

#ifdef CONFIG_SLAB_OBJ_EXT

			unsigned long memcg_data;

#endif

#if defined(WANT_PAGE_VIRTUAL)

init/Kconfig

+4 −0
Original line number Diff line number Diff line
@@ -929,6 +929,9 @@ config NUMA_BALANCING_DEFAULT_ENABLED 
	  If set, automatic NUMA balancing will be enabled if running on a NUMA

	  machine.



config SLAB_OBJ_EXT

	bool



menuconfig CGROUPS

	bool "Control Group support"

	select KERNFS
@@ -962,6 +965,7 @@ config MEMCG 
	bool "Memory controller"

	select PAGE_COUNTER

	select EVENTFD

	select SLAB_OBJ_EXT

	help

	  Provides control over the memory footprint of tasks in a cgroup.

mm/kfence/core.c

+7 −7
Original line number Diff line number Diff line
@@ -595,9 +595,9 @@ static unsigned long kfence_init_pool(void) 
			continue;



		__folio_set_slab(slab_folio(slab));

#ifdef CONFIG_MEMCG

		slab->memcg_data = (unsigned long)&kfence_metadata_init[i / 2 - 1].objcg |

				   MEMCG_DATA_OBJCGS;

#ifdef CONFIG_MEMCG_KMEM

		slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts |

				 MEMCG_DATA_OBJEXTS;

#endif

	}
@@ -645,8 +645,8 @@ static unsigned long kfence_init_pool(void) 


		if (!i || (i % 2))

			continue;

#ifdef CONFIG_MEMCG

		slab->memcg_data = 0;

#ifdef CONFIG_MEMCG_KMEM

		slab->obj_exts = 0;

#endif

		__folio_clear_slab(slab_folio(slab));

	}
@@ -1139,8 +1139,8 @@ void __kfence_free(void *addr) 
{

	struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);



#ifdef CONFIG_MEMCG

	KFENCE_WARN_ON(meta->objcg);

#ifdef CONFIG_MEMCG_KMEM

	KFENCE_WARN_ON(meta->obj_exts.objcg);

#endif

	/*

	 * If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing

mm/kfence/kfence.h

+2 −2
Original line number Diff line number Diff line
@@ -97,8 +97,8 @@ struct kfence_metadata { 
	struct kfence_track free_track;

	/* For updating alloc_covered on frees. */

	u32 alloc_stack_hash;

#ifdef CONFIG_MEMCG

	struct obj_cgroup *objcg;

#ifdef CONFIG_MEMCG_KMEM

	struct slabobj_ext obj_exts;

#endif

};