Commit c7163535 authored by Shakeel Butt's avatar Shakeel Butt Committed by Andrew Morton
Browse files

memcg: move preempt disable to callers of memcg_rstat_updated 

Let's move the explicit preempt disable code to the callers of
memcg_rstat_updated and also remove the memcg_stats_lock and related
functions which ensures the callers of stats update functions have
disabled preemption because now the stats update functions are explicitly
disabling preemption.

Link: https://lkml.kernel.org/r/20250514184158.3471331-3-shakeel.butt@linux.dev


Signed-off-by: default avatarShakeel Butt <shakeel.butt@linux.dev>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 8a4b42b9

mm/memcontrol.c

+19 −55
Original line number Diff line number Diff line
@@ -555,48 +555,22 @@ static u64 flush_last_time; 


#define FLUSH_TIME (2UL*HZ)



/*

 * Accessors to ensure that preemption is disabled on PREEMPT_RT because it can

 * not rely on this as part of an acquired spinlock_t lock. These functions are

 * never used in hardirq context on PREEMPT_RT and therefore disabling preemtion

 * is sufficient.

 */

static void memcg_stats_lock(void)

{

	preempt_disable_nested();

	VM_WARN_ON_IRQS_ENABLED();

}



static void __memcg_stats_lock(void)

{

	preempt_disable_nested();

}



static void memcg_stats_unlock(void)

{

	preempt_enable_nested();

}





static bool memcg_vmstats_needs_flush(struct memcg_vmstats *vmstats)

{

	return atomic64_read(&vmstats->stats_updates) >

		MEMCG_CHARGE_BATCH * num_online_cpus();

}



static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)

static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val,

				       int cpu)

{

	struct memcg_vmstats_percpu __percpu *statc_pcpu;

	struct memcg_vmstats_percpu *statc;

	int cpu;

	unsigned int stats_updates;



	if (!val)

		return;



	/* Don't assume callers have preemption disabled. */

	cpu = get_cpu();



	cgroup_rstat_updated(memcg->css.cgroup, cpu);

	statc_pcpu = memcg->vmstats_percpu;

	for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
@@ -617,7 +591,6 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) 
		stats_updates = this_cpu_xchg(statc_pcpu->stats_updates, 0);

		atomic64_add(stats_updates, &statc->vmstats->stats_updates);

	}

	put_cpu();

}



static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force)
@@ -715,6 +688,7 @@ void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, 
		       int val)

{

	int i = memcg_stats_index(idx);

	int cpu;



	if (mem_cgroup_disabled())

		return;
@@ -722,12 +696,14 @@ void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, 
	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))

		return;



	memcg_stats_lock();

	cpu = get_cpu();



	__this_cpu_add(memcg->vmstats_percpu->state[i], val);

	val = memcg_state_val_in_pages(idx, val);

	memcg_rstat_updated(memcg, val);

	memcg_rstat_updated(memcg, val, cpu);

	trace_mod_memcg_state(memcg, idx, val);

	memcg_stats_unlock();



	put_cpu();

}



#ifdef CONFIG_MEMCG_V1
@@ -756,6 +732,7 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec, 
	struct mem_cgroup_per_node *pn;

	struct mem_cgroup *memcg;

	int i = memcg_stats_index(idx);

	int cpu;



	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))

		return;
@@ -763,24 +740,7 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec, 
	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);

	memcg = pn->memcg;



	/*

	 * The caller from rmap relies on disabled preemption because they never

	 * update their counter from in-interrupt context. For these two

	 * counters we check that the update is never performed from an

	 * interrupt context while other caller need to have disabled interrupt.

	 */

	__memcg_stats_lock();

	if (IS_ENABLED(CONFIG_DEBUG_VM)) {

		switch (idx) {

		case NR_ANON_MAPPED:

		case NR_FILE_MAPPED:

		case NR_ANON_THPS:

			WARN_ON_ONCE(!in_task());

			break;

		default:

			VM_WARN_ON_IRQS_ENABLED();

		}

	}

	cpu = get_cpu();



	/* Update memcg */

	__this_cpu_add(memcg->vmstats_percpu->state[i], val);
@@ -789,9 +749,10 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec, 
	__this_cpu_add(pn->lruvec_stats_percpu->state[i], val);



	val = memcg_state_val_in_pages(idx, val);

	memcg_rstat_updated(memcg, val);

	memcg_rstat_updated(memcg, val, cpu);

	trace_mod_memcg_lruvec_state(memcg, idx, val);

	memcg_stats_unlock();



	put_cpu();

}



/**
@@ -871,6 +832,7 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, 
			  unsigned long count)

{

	int i = memcg_events_index(idx);

	int cpu;



	if (mem_cgroup_disabled())

		return;
@@ -878,11 +840,13 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, 
	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))

		return;



	memcg_stats_lock();

	cpu = get_cpu();



	__this_cpu_add(memcg->vmstats_percpu->events[i], count);

	memcg_rstat_updated(memcg, count);

	memcg_rstat_updated(memcg, count, cpu);

	trace_count_memcg_events(memcg, idx, count);

	memcg_stats_unlock();



	put_cpu();

}



unsigned long memcg_events(struct mem_cgroup *memcg, int event)