Commit 748922dc authored by JP Kobryn's avatar JP Kobryn Committed by Tejun Heo
Browse files

cgroup: use subsystem-specific rstat locks to avoid contention 



It is possible to eliminate contention between subsystems when
updating/flushing stats by using subsystem-specific locks. Let the existing
rstat locks be dedicated to the cgroup base stats and rename them to
reflect that. Add similar locks to the cgroup_subsys struct for use with
individual subsystems.

Lock initialization is done in the new function ss_rstat_init(ss) which
replaces cgroup_rstat_boot(void). If NULL is passed to this function, the
global base stat locks will be initialized. Otherwise, the subsystem locks
will be initialized.

Change the existing lock helper functions to accept a reference to a css.
Then within these functions, conditionally select the appropriate locks
based on the subsystem affiliation of the given css. Add helper functions
for this selection routine to avoid repeated code.

Signed-off-by: default avatarJP Kobryn <inwardvessel@gmail.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 5da3bfa0

block/blk-cgroup.c

+2 −2
Original line number Diff line number Diff line
@@ -1074,8 +1074,8 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) 
	/*

	 * For covering concurrent parent blkg update from blkg_release().

	 *

	 * When flushing from cgroup, cgroup_rstat_lock is always held, so

	 * this lock won't cause contention most of time.

	 * When flushing from cgroup, the subsystem rstat lock is always held,

	 * so this lock won't cause contention most of time.

	 */

	raw_spin_lock_irqsave(&blkg_stat_lock, flags);

include/linux/cgroup-defs.h

+8 −2
Original line number Diff line number Diff line
@@ -222,7 +222,10 @@ struct cgroup_subsys_state { 
	/*

	 * A singly-linked list of css structures to be rstat flushed.

	 * This is a scratch field to be used exclusively by

	 * css_rstat_flush() and protected by cgroup_rstat_lock.

	 * css_rstat_flush().

	 *

	 * Protected by rstat_base_lock when css is cgroup::self.

	 * Protected by css->ss->rstat_ss_lock otherwise.

	 */

	struct cgroup_subsys_state *rstat_flush_next;

};
@@ -362,7 +365,7 @@ struct css_rstat_cpu { 
	 * the css makes it unnecessary for each per-cpu struct to point back

	 * to the associated css.

	 *

	 * Protected by per-cpu cgroup_rstat_cpu_lock.

	 * Protected by per-cpu css->ss->rstat_ss_cpu_lock.

	 */

	struct cgroup_subsys_state *updated_children;

	struct cgroup_subsys_state *updated_next;	/* NULL if not on the list */
@@ -792,6 +795,9 @@ struct cgroup_subsys { 
	 * specifies the mask of subsystems that this one depends on.

	 */

	unsigned int depends_on;



	spinlock_t rstat_ss_lock;

	raw_spinlock_t __percpu *rstat_ss_cpu_lock;

};



extern struct percpu_rw_semaphore cgroup_threadgroup_rwsem;

include/trace/events/cgroup.h

+10 −2
Original line number Diff line number Diff line
@@ -231,7 +231,11 @@ DECLARE_EVENT_CLASS(cgroup_rstat, 
		  __entry->cpu, __entry->contended)

);



/* Related to global: cgroup_rstat_lock */

/*

 * Related to locks:

 * global rstat_base_lock for base stats

 * cgroup_subsys::rstat_ss_lock for subsystem stats

 */

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_lock_contended,



	TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),
@@ -253,7 +257,11 @@ DEFINE_EVENT(cgroup_rstat, cgroup_rstat_unlock, 
	TP_ARGS(cgrp, cpu, contended)

);



/* Related to per CPU: cgroup_rstat_cpu_lock */

/*

 * Related to per CPU locks:

 * global rstat_base_cpu_lock for base stats

 * cgroup_subsys::rstat_ss_cpu_lock for subsystem stats

 */

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_cpu_lock_contended,



	TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

kernel/cgroup/cgroup-internal.h

+1 −1
Original line number Diff line number Diff line
@@ -272,7 +272,7 @@ int cgroup_task_count(const struct cgroup *cgrp); 
 */

int css_rstat_init(struct cgroup_subsys_state *css);

void css_rstat_exit(struct cgroup_subsys_state *css);

void cgroup_rstat_boot(void);

int ss_rstat_init(struct cgroup_subsys *ss);

void cgroup_base_stat_cputime_show(struct seq_file *seq);



/*

kernel/cgroup/cgroup.c

+2 −1
Original line number Diff line number Diff line
@@ -6088,6 +6088,7 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early) 
		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);

		BUG_ON(css->id < 0);



		BUG_ON(ss_rstat_init(ss));

		BUG_ON(css_rstat_init(css));

	}
@@ -6167,7 +6168,7 @@ int __init cgroup_init(void) 
	BUG_ON(cgroup_init_cftypes(NULL, cgroup_psi_files));

	BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files));



	cgroup_rstat_boot();

	BUG_ON(ss_rstat_init(NULL));



	get_user_ns(init_cgroup_ns.user_ns);