Commit 7ffc7ade authored by Jeff Layton's avatar Jeff Layton Committed by Chuck Lever
Browse files

sunrpc: introduce the concept of a minimum number of threads per pool 



Add a new pool->sp_nrthrmin field to track the minimum number of threads
in a pool. Add min_threads parameters to both svc_set_num_threads() and
svc_set_pool_threads(). If min_threads is non-zero and less than the
max, svc_set_num_threads() will ensure that the number of running
threads is between the min and the max.

If the min is 0 or greater than the max, then it is ignored, and the
maximum number of threads will be started, and never spun down.

For now, the min_threads is always 0, but a later patch will pass the
proper value through from nfsd.

Signed-off-by: default avatarJeff Layton <jlayton@kernel.org>
Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
parent 6cd60f42

fs/lockd/svc.c

+2 −2
Original line number Diff line number Diff line
@@ -340,7 +340,7 @@ static int lockd_get(void) 
		return -ENOMEM;

	}



	error = svc_set_num_threads(serv, 1);

	error = svc_set_num_threads(serv, 0, 1);

	if (error < 0) {

		svc_destroy(&serv);

		return error;
@@ -368,7 +368,7 @@ static void lockd_put(void) 
	unregister_inet6addr_notifier(&lockd_inet6addr_notifier);

#endif



	svc_set_num_threads(nlmsvc_serv, 0);

	svc_set_num_threads(nlmsvc_serv, 0, 0);

	timer_delete_sync(&nlmsvc_retry);

	svc_destroy(&nlmsvc_serv);

	dprintk("lockd_down: service destroyed\n");

fs/nfs/callback.c

+4 −4
Original line number Diff line number Diff line
@@ -119,9 +119,9 @@ static int nfs_callback_start_svc(int minorversion, struct rpc_xprt *xprt, 
	if (serv->sv_nrthreads == nrservs)

		return 0;



	ret = svc_set_num_threads(serv, nrservs);

	ret = svc_set_num_threads(serv, 0, nrservs);

	if (ret) {

		svc_set_num_threads(serv, 0);

		svc_set_num_threads(serv, 0, 0);

		return ret;

	}

	dprintk("nfs_callback_up: service started\n");
@@ -242,7 +242,7 @@ int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt) 
	cb_info->users++;

err_net:

	if (!cb_info->users) {

		svc_set_num_threads(cb_info->serv, 0);

		svc_set_num_threads(cb_info->serv, 0, 0);

		svc_destroy(&cb_info->serv);

	}

err_create:
@@ -268,7 +268,7 @@ void nfs_callback_down(int minorversion, struct net *net, struct rpc_xprt *xprt) 
	nfs_callback_down_net(minorversion, serv, net);

	cb_info->users--;

	if (cb_info->users == 0) {

		svc_set_num_threads(serv, 0);

		svc_set_num_threads(serv, 0, 0);

		dprintk("nfs_callback_down: service destroyed\n");

		xprt_svc_destroy_nullify_bc(xprt, &cb_info->serv);

	}

fs/nfsd/nfssvc.c

+4 −4
Original line number Diff line number Diff line
@@ -580,7 +580,7 @@ void nfsd_shutdown_threads(struct net *net) 
	}



	/* Kill outstanding nfsd threads */

	svc_set_num_threads(serv, 0);

	svc_set_num_threads(serv, 0, 0);

	nfsd_destroy_serv(net);

	mutex_unlock(&nfsd_mutex);

}
@@ -690,7 +690,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) 


	/* Special case: When n == 1, distribute threads equally among pools. */

	if (n == 1)

		return svc_set_num_threads(nn->nfsd_serv, nthreads[0]);

		return svc_set_num_threads(nn->nfsd_serv, 0, nthreads[0]);



	if (n > nn->nfsd_serv->sv_nrpools)

		n = nn->nfsd_serv->sv_nrpools;
@@ -718,7 +718,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) 
	for (i = 0; i < n; i++) {

		err = svc_set_pool_threads(nn->nfsd_serv,

					   &nn->nfsd_serv->sv_pools[i],

					   nthreads[i]);

					   0, nthreads[i]);

		if (err)

			goto out;

	}
@@ -727,7 +727,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) 
	for (i = n; i < nn->nfsd_serv->sv_nrpools; ++i) {

		err = svc_set_pool_threads(nn->nfsd_serv,

					   &nn->nfsd_serv->sv_pools[i],

					   0);

					   0, 0);

		if (err)

			goto out;

	}

include/linux/sunrpc/svc.h

+5 −3
Original line number Diff line number Diff line
@@ -36,6 +36,7 @@ 
struct svc_pool {

	unsigned int		sp_id;		/* pool id; also node id on NUMA */

	unsigned int		sp_nrthreads;	/* # of threads currently running in pool */

	unsigned int		sp_nrthrmin;	/* Min number of threads to run per pool */

	unsigned int		sp_nrthrmax;	/* Max requested number of threads in pool */

	struct lwq		sp_xprts;	/* pending transports */

	struct list_head	sp_all_threads;	/* all server threads */
@@ -72,7 +73,7 @@ struct svc_serv { 
	struct svc_stat *	sv_stats;	/* RPC statistics */

	spinlock_t		sv_lock;

	unsigned int		sv_nprogs;	/* Number of sv_programs */

	unsigned int		sv_nrthreads;	/* # of server threads */

	unsigned int		sv_nrthreads;	/* # of running server threads */

	unsigned int		sv_max_payload;	/* datagram payload size */

	unsigned int		sv_max_mesg;	/* max_payload + 1 page for overheads */

	unsigned int		sv_xdrsize;	/* XDR buffer size */
@@ -448,8 +449,9 @@ struct svc_serv * svc_create_pooled(struct svc_program *prog, 
				     unsigned int bufsize,

				     int (*threadfn)(void *data));

int		   svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool,

					unsigned int min_threads, unsigned int max_threads);

int		   svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads,

				       unsigned int nrservs);

int		   svc_set_num_threads(struct svc_serv *serv, unsigned int nrservs);

int		   svc_pool_stats_open(struct svc_info *si, struct file *file);

void		   svc_process(struct svc_rqst *rqstp);

void		   svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp);

net/sunrpc/svc.c

+37 −8
Original line number Diff line number Diff line
@@ -820,9 +820,14 @@ svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) 
 * svc_set_pool_threads - adjust number of threads per pool

 * @serv: RPC service to adjust

 * @pool: Specific pool from which to choose threads

 * @nrservs: New number of threads for @serv (0 means kill all threads)

 * @min_threads: min number of threads to run in @pool

 * @max_threads: max number of threads in @pool (0 means kill all threads)

 *

 * Create or destroy threads in @pool to bring it into an acceptable range

 * between @min_threads and @max_threads.

 *

 * Create or destroy threads in @pool to bring it to @nrservs.

 * If @min_threads is 0 or larger than @max_threads, then it is ignored and

 * the pool will be set to run a static @max_threads number of threads.

 *

 * Caller must ensure mutual exclusion between this and server startup or

 * shutdown.
@@ -832,16 +837,36 @@ svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) 
 */

int

svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool,

		     unsigned int nrservs)

		     unsigned int min_threads, unsigned int max_threads)

{

	int delta = nrservs;

	int delta;



	if (!pool)

		return -EINVAL;



	pool->sp_nrthrmax = nrservs;

	delta -= pool->sp_nrthreads;

	/* clamp min threads to the max */

	if (min_threads > max_threads)

		min_threads = max_threads;



	pool->sp_nrthrmin = min_threads;

	pool->sp_nrthrmax = max_threads;



	/*

	 * When min_threads is set, then only change the number of

	 * threads to bring it within an acceptable range.

	 */

	if (min_threads) {

		if (pool->sp_nrthreads > max_threads)

			delta = max_threads;

		else if (pool->sp_nrthreads < min_threads)

			delta = min_threads;

		else

			return 0;

	} else {

		delta = max_threads;

	}



	delta -= pool->sp_nrthreads;

	if (delta > 0)

		return svc_start_kthreads(serv, pool, delta);

	if (delta < 0)
@@ -853,6 +878,7 @@ EXPORT_SYMBOL_GPL(svc_set_pool_threads); 
/**

 * svc_set_num_threads - adjust number of threads in serv

 * @serv: RPC service to adjust

 * @min_threads: min number of threads to run per pool

 * @nrservs: New number of threads for @serv (0 means kill all threads)

 *

 * Create or destroy threads in @serv to bring it to @nrservs. If there
@@ -867,20 +893,23 @@ EXPORT_SYMBOL_GPL(svc_set_pool_threads); 
 * adjusted; the caller is responsible for recovery.

 */

int

svc_set_num_threads(struct svc_serv *serv, unsigned int nrservs)

svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads,

		    unsigned int nrservs)

{

	unsigned int base = nrservs / serv->sv_nrpools;

	unsigned int remain = nrservs % serv->sv_nrpools;

	int i, err = 0;



	for (i = 0; i < serv->sv_nrpools; ++i) {

		struct svc_pool *pool = &serv->sv_pools[i];

		int threads = base;



		if (remain) {

			++threads;

			--remain;

		}

		err = svc_set_pool_threads(serv, &serv->sv_pools[i], threads);



		err = svc_set_pool_threads(serv, pool, min_threads, threads);

		if (err)

			break;

	}