Commit 1257b878 authored by Shakeel Butt's avatar Shakeel Butt Committed by Tejun Heo
Browse files

cgroup: support to enable nmi-safe css_rstat_updated 



Add necessary infrastructure to enable the nmi-safe execution of
css_rstat_updated(). Currently css_rstat_updated() takes a per-cpu
per-css raw spinlock to add the given css in the per-cpu per-css update
tree. However the kernel can not spin in nmi context, so we need to
remove the spinning on the raw spinlock in css_rstat_updated().

To support lockless css_rstat_updated(), let's add necessary data
structures in the css and ss structures.

Signed-off-by: default avatarShakeel Butt <shakeel.butt@linux.dev>
Tested-by: default avatarJP Kobryn <inwardvessel@gmail.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 5da4f9db

include/linux/cgroup-defs.h

+4 −0
Original line number Diff line number Diff line
@@ -384,6 +384,9 @@ struct css_rstat_cpu { 
	 */

	struct cgroup_subsys_state *updated_children;

	struct cgroup_subsys_state *updated_next;	/* NULL if not on the list */



	struct llist_node lnode;		/* lockless list for update */

	struct cgroup_subsys_state *owner;	/* back pointer */

};



/*
@@ -822,6 +825,7 @@ struct cgroup_subsys { 


	spinlock_t rstat_ss_lock;

	raw_spinlock_t __percpu *rstat_ss_cpu_lock;

	struct llist_head __percpu *lhead; /* lockless update list head */

};



extern struct percpu_rw_semaphore cgroup_threadgroup_rwsem;

kernel/cgroup/rstat.c

+21 −2
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 


static DEFINE_SPINLOCK(rstat_base_lock);

static DEFINE_PER_CPU(raw_spinlock_t, rstat_base_cpu_lock);

static DEFINE_PER_CPU(struct llist_head, rstat_backlog_list);



static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
@@ -45,6 +46,13 @@ static spinlock_t *ss_rstat_lock(struct cgroup_subsys *ss) 
	return &rstat_base_lock;

}



static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu)

{

	if (ss)

		return per_cpu_ptr(ss->lhead, cpu);

	return per_cpu_ptr(&rstat_backlog_list, cpu);

}



static raw_spinlock_t *ss_rstat_cpu_lock(struct cgroup_subsys *ss, int cpu)

{

	if (ss)
@@ -456,7 +464,8 @@ int css_rstat_init(struct cgroup_subsys_state *css) 
	for_each_possible_cpu(cpu) {

		struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);



		rstatc->updated_children = css;

		rstatc->owner = rstatc->updated_children = css;

		init_llist_node(&rstatc->lnode);



		if (is_self) {

			struct cgroup_rstat_base_cpu *rstatbc;
@@ -525,9 +534,19 @@ int __init ss_rstat_init(struct cgroup_subsys *ss) 
	}

#endif



	if (ss) {

		ss->lhead = alloc_percpu(struct llist_head);

		if (!ss->lhead) {

			free_percpu(ss->rstat_ss_cpu_lock);

			return -ENOMEM;

		}

	}



	spin_lock_init(ss_rstat_lock(ss));

	for_each_possible_cpu(cpu)

	for_each_possible_cpu(cpu) {

		raw_spin_lock_init(ss_rstat_cpu_lock(ss, cpu));

		init_llist_head(ss_lhead_cpu(ss, cpu));

	}



	return 0;

}