Skip to content

Commit

Permalink
SUNRPC: Micro-optimise when the task is known not to be sleeping
Browse files Browse the repository at this point in the history 
In cases where we know the task is not sleeping, try to optimise
away the indirect call to task->tk_action() by replacing it with
a direct call.
Only change tail calls, to allow gcc to perform tail call
elimination.

Signed-off-by: Trond Myklebust <[email protected]>
  • Loading branch information
Trond Myklebust committed Mar 10, 2019
1 parent 03e51d3 commit 009a82f
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 34 deletions.
8 changes: 8 additions & 0 deletions include/linux/sunrpc/sched.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,4 +304,12 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt)
}
#endif /* CONFIG_SUNRPC_SWAP */

static inline bool
rpc_task_need_resched(const struct rpc_task *task)
{
if (RPC_IS_QUEUED(task) || task->tk_callback)
return true;
return false;
}

#endif /* _LINUX_SUNRPC_SCHED_H_ */
99 changes: 65 additions & 34 deletions net/sunrpc/clnt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1540,6 +1540,7 @@ call_start(struct rpc_task *task)
clnt->cl_stats->rpccnt++;
task->tk_action = call_reserve;
rpc_task_set_transport(task, clnt);
call_reserve(task);
}

/*
Expand All @@ -1553,6 +1554,9 @@ call_reserve(struct rpc_task *task)
task->tk_status = 0;
task->tk_action = call_reserveresult;
xprt_reserve(task);
if (rpc_task_need_resched(task))
return;
call_reserveresult(task);
}

static void call_retry_reserve(struct rpc_task *task);
Expand All @@ -1575,6 +1579,7 @@ call_reserveresult(struct rpc_task *task)
if (status >= 0) {
if (task->tk_rqstp) {
task->tk_action = call_refresh;
call_refresh(task);
return;
}

Expand All @@ -1600,6 +1605,7 @@ call_reserveresult(struct rpc_task *task)
/* fall through */
case -EAGAIN: /* woken up; retry */
task->tk_action = call_retry_reserve;
call_retry_reserve(task);
return;
case -EIO: /* probably a shutdown */
break;
Expand All @@ -1622,6 +1628,9 @@ call_retry_reserve(struct rpc_task *task)
task->tk_status = 0;
task->tk_action = call_reserveresult;
xprt_retry_reserve(task);
if (rpc_task_need_resched(task))
return;
call_reserveresult(task);
}

/*
Expand All @@ -1636,6 +1645,9 @@ call_refresh(struct rpc_task *task)
task->tk_status = 0;
task->tk_client->cl_stats->rpcauthrefresh++;
rpcauth_refreshcred(task);
if (rpc_task_need_resched(task))
return;
call_refreshresult(task);
}

/*
Expand All @@ -1654,6 +1666,7 @@ call_refreshresult(struct rpc_task *task)
case 0:
if (rpcauth_uptodatecred(task)) {
task->tk_action = call_allocate;
call_allocate(task);
return;
}
/* Use rate-limiting and a max number of retries if refresh
Expand All @@ -1672,6 +1685,7 @@ call_refreshresult(struct rpc_task *task)
task->tk_cred_retry--;
dprintk("RPC: %5u %s: retry refresh creds\n",
task->tk_pid, __func__);
call_refresh(task);
return;
}
dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
Expand All @@ -1697,8 +1711,10 @@ call_allocate(struct rpc_task *task)
task->tk_status = 0;
task->tk_action = call_encode;

if (req->rq_buffer)
if (req->rq_buffer) {
call_encode(task);
return;
}

if (proc->p_proc != 0) {
BUG_ON(proc->p_arglen == 0);
Expand All @@ -1719,8 +1735,12 @@ call_allocate(struct rpc_task *task)

status = xprt->ops->buf_alloc(task);
xprt_inject_disconnect(xprt);
if (status == 0)
if (status == 0) {
if (rpc_task_need_resched(task))
return;
call_encode(task);
return;
}
if (status != -ENOMEM) {
rpc_exit(task, status);
return;
Expand Down Expand Up @@ -1803,12 +1823,8 @@ call_encode(struct rpc_task *task)
xprt_request_enqueue_receive(task);
xprt_request_enqueue_transmit(task);
out:
task->tk_action = call_transmit;
/* Check that the connection is OK */
if (!xprt_bound(task->tk_xprt))
task->tk_action = call_bind;
else if (!xprt_connected(task->tk_xprt))
task->tk_action = call_connect;
task->tk_action = call_bind;
call_bind(task);
}

/*
Expand Down Expand Up @@ -1842,14 +1858,17 @@ call_bind(struct rpc_task *task)
return;
}

if (xprt_bound(xprt)) {
task->tk_action = call_connect;
call_connect(task);
return;
}

dprint_status(task);

task->tk_action = call_connect;
if (!xprt_bound(xprt)) {
task->tk_action = call_bind_status;
task->tk_timeout = xprt->bind_timeout;
xprt->ops->rpcbind(task);
}
task->tk_action = call_bind_status;
task->tk_timeout = xprt->bind_timeout;
xprt->ops->rpcbind(task);
}

/*
Expand All @@ -1869,6 +1888,7 @@ call_bind_status(struct rpc_task *task)
dprint_status(task);
task->tk_status = 0;
task->tk_action = call_connect;
call_connect(task);
return;
}

Expand Down Expand Up @@ -1949,21 +1969,24 @@ call_connect(struct rpc_task *task)
return;
}

if (xprt_connected(xprt)) {
task->tk_action = call_transmit;
call_transmit(task);
return;
}

dprintk("RPC: %5u call_connect xprt %p %s connected\n",
task->tk_pid, xprt,
(xprt_connected(xprt) ? "is" : "is not"));

task->tk_action = call_transmit;
if (!xprt_connected(xprt)) {
task->tk_action = call_connect_status;
if (task->tk_status < 0)
return;
if (task->tk_flags & RPC_TASK_NOCONNECT) {
rpc_exit(task, -ENOTCONN);
return;
}
xprt_connect(task);
task->tk_action = call_connect_status;
if (task->tk_status < 0)
return;
if (task->tk_flags & RPC_TASK_NOCONNECT) {
rpc_exit(task, -ENOTCONN);
return;
}
xprt_connect(task);
}

/*
Expand Down Expand Up @@ -2016,6 +2039,7 @@ call_connect_status(struct rpc_task *task)
case 0:
clnt->cl_stats->netreconn++;
task->tk_action = call_transmit;
call_transmit(task);
return;
}
rpc_exit(task, status);
Expand All @@ -2040,19 +2064,20 @@ call_transmit(struct rpc_task *task)
dprint_status(task);

task->tk_action = call_transmit_status;
if (!xprt_prepare_transmit(task))
return;
task->tk_status = 0;
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
if (!xprt_prepare_transmit(task))
if (!xprt_connected(task->tk_xprt)) {
task->tk_status = -ENOTCONN;
return;
task->tk_status = 0;
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
if (!xprt_connected(task->tk_xprt)) {
task->tk_status = -ENOTCONN;
return;
}
xprt_transmit(task);
}
xprt_transmit(task);
}
xprt_end_transmit(task);
if (rpc_task_need_resched(task))
return;
call_transmit_status(task);
}

/*
Expand All @@ -2067,8 +2092,12 @@ call_transmit_status(struct rpc_task *task)
* Common case: success. Force the compiler to put this
* test first.
*/
if (task->tk_status == 0) {
xprt_request_wait_receive(task);
if (rpc_task_transmitted(task)) {
if (task->tk_status == 0)
xprt_request_wait_receive(task);
if (rpc_task_need_resched(task))
return;
call_status(task);
return;
}

Expand Down Expand Up @@ -2129,6 +2158,7 @@ call_bc_encode(struct rpc_task *task)
{
xprt_request_enqueue_transmit(task);
task->tk_action = call_bc_transmit;
call_bc_transmit(task);
}

/*
Expand Down Expand Up @@ -2219,6 +2249,7 @@ call_status(struct rpc_task *task)
status = task->tk_status;
if (status >= 0) {
task->tk_action = call_decode;
call_decode(task);
return;
}

Expand Down

0 comments on commit 009a82f

Please sign in to comment.