Skip to content

Commit

Permalink
sunrpc: use better NUMA affinities
Browse files Browse the repository at this point in the history 
Use NUMA aware allocations to reduce latencies and increase throughput.

sunrpc kthreads can use kthread_create_on_node() if pool_mode is
"percpu" or "pernode", and svc_prepare_thread()/svc_init_buffer() can
also take into account NUMA node affinity for memory allocations.

Signed-off-by: Eric Dumazet <[email protected]>
CC: "J. Bruce Fields" <[email protected]>
CC: Neil Brown <[email protected]>
CC: David Miller <[email protected]>
Reviewed-by: Greg Banks <[email protected]>
[[email protected]: fix up caller nfs41_callback_up]
Signed-off-by: J. Bruce Fields <[email protected]>
  • Loading branch information
Eric Dumazet authored and J. Bruce Fields committed Aug 19, 2011
1 parent c1f24ef commit 11fd165
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 13 deletions.
2 changes: 1 addition & 1 deletion fs/lockd/svc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ int lockd_up(void)
/*
* Create the kernel thread and wait for it to start.
*/
nlmsvc_rqst = svc_prepare_thread(serv, &serv->sv_pools[0]);
nlmsvc_rqst = svc_prepare_thread(serv, &serv->sv_pools[0], NUMA_NO_NODE);
if (IS_ERR(nlmsvc_rqst)) {
error = PTR_ERR(nlmsvc_rqst);
nlmsvc_rqst = NULL;
Expand Down
4 changes: 2 additions & 2 deletions fs/nfs/callback.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ nfs4_callback_up(struct svc_serv *serv)
else
goto out_err;

return svc_prepare_thread(serv, &serv->sv_pools[0]);
return svc_prepare_thread(serv, &serv->sv_pools[0], NUMA_NO_NODE);

out_err:
if (ret == 0)
Expand Down Expand Up @@ -199,7 +199,7 @@ nfs41_callback_up(struct svc_serv *serv, struct rpc_xprt *xprt)
INIT_LIST_HEAD(&serv->sv_cb_list);
spin_lock_init(&serv->sv_cb_lock);
init_waitqueue_head(&serv->sv_cb_waitq);
rqstp = svc_prepare_thread(serv, &serv->sv_pools[0]);
rqstp = svc_prepare_thread(serv, &serv->sv_pools[0], NUMA_NO_NODE);
if (IS_ERR(rqstp)) {
svc_xprt_put(serv->sv_bc_xprt);
serv->sv_bc_xprt = NULL;
Expand Down
2 changes: 1 addition & 1 deletion include/linux/sunrpc/svc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ struct svc_procedure {
struct svc_serv *svc_create(struct svc_program *, unsigned int,
void (*shutdown)(struct svc_serv *));
struct svc_rqst *svc_prepare_thread(struct svc_serv *serv,
struct svc_pool *pool);
struct svc_pool *pool, int node);
void svc_exit_thread(struct svc_rqst *);
struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int,
void (*shutdown)(struct svc_serv *),
Expand Down
33 changes: 24 additions & 9 deletions net/sunrpc/svc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,18 @@ svc_pool_map_put(void)
}


static int svc_pool_map_get_node(unsigned int pidx)
{
const struct svc_pool_map *m = &svc_pool_map;

if (m->count) {
if (m->mode == SVC_POOL_PERCPU)
return cpu_to_node(m->pool_to[pidx]);
if (m->mode == SVC_POOL_PERNODE)
return m->pool_to[pidx];
}
return NUMA_NO_NODE;
}
/*
* Set the given thread's cpus_allowed mask so that it
* will only run on cpus in the given pool.
Expand Down Expand Up @@ -499,7 +511,7 @@ EXPORT_SYMBOL_GPL(svc_destroy);
* We allocate pages and place them in rq_argpages.
*/
static int
svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
{
unsigned int pages, arghi;

Expand All @@ -513,7 +525,7 @@ svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
arghi = 0;
BUG_ON(pages > RPCSVC_MAXPAGES);
while (pages) {
struct page *p = alloc_page(GFP_KERNEL);
struct page *p = alloc_pages_node(node, GFP_KERNEL, 0);
if (!p)
break;
rqstp->rq_pages[arghi++] = p;
Expand All @@ -536,11 +548,11 @@ svc_release_buffer(struct svc_rqst *rqstp)
}

struct svc_rqst *
svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool)
svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
{
struct svc_rqst *rqstp;

rqstp = kzalloc(sizeof(*rqstp), GFP_KERNEL);
rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
if (!rqstp)
goto out_enomem;

Expand All @@ -554,15 +566,15 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool)
rqstp->rq_server = serv;
rqstp->rq_pool = pool;

rqstp->rq_argp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
if (!rqstp->rq_argp)
goto out_thread;

rqstp->rq_resp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
if (!rqstp->rq_resp)
goto out_thread;

if (!svc_init_buffer(rqstp, serv->sv_max_mesg))
if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
goto out_thread;

return rqstp;
Expand Down Expand Up @@ -647,6 +659,7 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
struct svc_pool *chosen_pool;
int error = 0;
unsigned int state = serv->sv_nrthreads-1;
int node;

if (pool == NULL) {
/* The -1 assumes caller has done a svc_get() */
Expand All @@ -662,14 +675,16 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
nrservs--;
chosen_pool = choose_pool(serv, pool, &state);

rqstp = svc_prepare_thread(serv, chosen_pool);
node = svc_pool_map_get_node(chosen_pool->sp_id);
rqstp = svc_prepare_thread(serv, chosen_pool, node);
if (IS_ERR(rqstp)) {
error = PTR_ERR(rqstp);
break;
}

__module_get(serv->sv_module);
task = kthread_create(serv->sv_function, rqstp, serv->sv_name);
task = kthread_create_on_node(serv->sv_function, rqstp,
node, serv->sv_name);
if (IS_ERR(task)) {
error = PTR_ERR(task);
module_put(serv->sv_module);
Expand Down

0 comments on commit 11fd165

Please sign in to comment.