Skip to content

Commit

Permalink
pnfs: rework LAYOUTGET retry handling
Browse files Browse the repository at this point in the history 
There are several problems in the way a stateid is selected for a
LAYOUTGET operation:

We pick a stateid to use in the RPC prepare op, but that makes
it difficult to serialize LAYOUTGETs that use the open stateid. That
serialization is done in pnfs_update_layout, which occurs well before
the rpc_prepare operation.

Between those two events, the i_lock is dropped and reacquired.
pnfs_update_layout can find that the list has lsegs in it and not do any
serialization, but then later pnfs_choose_layoutget_stateid ends up
choosing the open stateid.

This patch changes the client to select the stateid to use in the
LAYOUTGET earlier, when we're searching for a usable layout segment.
This way we can do it all while holding the i_lock the first time, and
ensure that we serialize any LAYOUTGET call that uses a non-layout
stateid.

This also means a rework of how LAYOUTGET replies are handled, as we
must now get the latest stateid if we want to retransmit in response
to a retryable error.

Most of those errors boil down to the fact that the layout state has
changed in some fashion. Thus, what we really want to do is to re-search
for a layout when it fails with a retryable error, so that we can avoid
reissuing the RPC at all if possible.

While the LAYOUTGET RPC is async, the initiating thread always waits for
it to complete, so it's effectively synchronous anyway. Currently, when
we need to retry a LAYOUTGET because of an error, we drive that retry
via the rpc state machine.

This means that once the call has been submitted, it runs until it
completes. So, we must move the error handling for this RPC out of the
rpc_call_done operation and into the caller.

In order to handle errors like NFS4ERR_DELAY properly, we must also
pass a pointer to the sliding timeout, which is now moved to the stack
in pnfs_update_layout.

The complicating errors are -NFS4ERR_RECALLCONFLICT and
-NFS4ERR_LAYOUTTRYLATER, as those involve a timeout after which we give
up and return NULL back to the caller. So, there is some special
handling for those errors to ensure that the layers driving the retries
can handle that appropriately.

Signed-off-by: Jeff Layton <[email protected]>
Signed-off-by: Anna Schumaker <[email protected]>
  • Loading branch information
@jtlayton @amschuma-ntap
jtlayton authored and amschuma-ntap committed May 17, 2016
1 parent 83026d8 commit 183d9e7
Show file tree
Hide file tree
Showing 7 changed files with 136 additions and 144 deletions.
115 changes: 48 additions & 67 deletions fs/nfs/nfs4proc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,7 @@ static int nfs4_do_handle_exception(struct nfs_server *server,
case -NFS4ERR_DELAY:
nfs_inc_server_stats(server, NFSIOS_DELAY);
case -NFS4ERR_GRACE:
case -NFS4ERR_RECALLCONFLICT:
exception->delay = 1;
return 0;

Expand Down Expand Up @@ -7847,40 +7848,34 @@ nfs4_layoutget_prepare(struct rpc_task *task, void *calldata)
struct nfs4_layoutget *lgp = calldata;
struct nfs_server *server = NFS_SERVER(lgp->args.inode);
struct nfs4_session *session = nfs4_get_session(server);
int ret;

dprintk("--> %s\n", __func__);
/* Note the is a race here, where a CB_LAYOUTRECALL can come in
* right now covering the LAYOUTGET we are about to send.
* However, that is not so catastrophic, and there seems
* to be no way to prevent it completely.
*/
if (nfs41_setup_sequence(session, &lgp->args.seq_args,
&lgp->res.seq_res, task))
return;
ret = pnfs_choose_layoutget_stateid(&lgp->args.stateid,
NFS_I(lgp->args.inode)->layout,
&lgp->args.range,
lgp->args.ctx->state);
if (ret < 0)
rpc_exit(task, ret);
nfs41_setup_sequence(session, &lgp->args.seq_args,
&lgp->res.seq_res, task);
dprintk("<-- %s\n", __func__);
}

static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
{
struct nfs4_layoutget *lgp = calldata;

dprintk("--> %s\n", __func__);
nfs41_sequence_done(task, &lgp->res.seq_res);
dprintk("<-- %s\n", __func__);
}

static int
nfs4_layoutget_handle_exception(struct rpc_task *task,
struct nfs4_layoutget *lgp, struct nfs4_exception *exception)
{
struct inode *inode = lgp->args.inode;
struct nfs_server *server = NFS_SERVER(inode);
struct pnfs_layout_hdr *lo;
struct nfs4_state *state = NULL;
unsigned long timeo, now, giveup;
int status = task->tk_status;

dprintk("--> %s tk_status => %d\n", __func__, -task->tk_status);

if (!nfs41_sequence_done(task, &lgp->res.seq_res))
goto out;

switch (task->tk_status) {
switch (status) {
case 0:
goto out;

Expand All @@ -7890,57 +7885,43 @@ static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
* retry go inband.
*/
case -NFS4ERR_LAYOUTUNAVAILABLE:
task->tk_status = -ENODATA;
status = -ENODATA;
goto out;
/*
* NFS4ERR_BADLAYOUT means the MDS cannot return a layout of
* length lgp->args.minlength != 0 (see RFC5661 section 18.43.3).
*/
case -NFS4ERR_BADLAYOUT:
goto out_overflow;
status = -EOVERFLOW;
goto out;
/*
* NFS4ERR_LAYOUTTRYLATER is a conflict with another client
* (or clients) writing to the same RAID stripe except when
* the minlength argument is 0 (see RFC5661 section 18.43.3).
*
* Treat it like we would RECALLCONFLICT -- we retry for a little
* while, and then eventually give up.
*/
case -NFS4ERR_LAYOUTTRYLATER:
if (lgp->args.minlength == 0)
goto out_overflow;
/*
* NFS4ERR_RECALLCONFLICT is when conflict with self (must recall
* existing layout before getting a new one).
*/
case -NFS4ERR_RECALLCONFLICT:
timeo = rpc_get_timeout(task->tk_client);
giveup = lgp->args.timestamp + timeo;
now = jiffies;
if (time_after(giveup, now)) {
unsigned long delay;

/* Delay for:
* - Not less then NFS4_POLL_RETRY_MIN.
* - One last time a jiffie before we give up
* - exponential backoff (time_now minus start_attempt)
*/
delay = max_t(unsigned long, NFS4_POLL_RETRY_MIN,
min((giveup - now - 1),
now - lgp->args.timestamp));

dprintk("%s: NFS4ERR_RECALLCONFLICT waiting %lu\n",
__func__, delay);
rpc_delay(task, delay);
/* Do not call nfs4_async_handle_error() */
goto out_restart;
if (lgp->args.minlength == 0) {
status = -EOVERFLOW;
goto out;
}
break;
/* Fallthrough */
case -NFS4ERR_RECALLCONFLICT:
nfs4_handle_exception(server, -NFS4ERR_RECALLCONFLICT,
exception);
status = -ERECALLCONFLICT;
goto out;
case -NFS4ERR_EXPIRED:
case -NFS4ERR_BAD_STATEID:
exception->timeout = 0;
spin_lock(&inode->i_lock);
if (nfs4_stateid_match(&lgp->args.stateid,
&lgp->args.ctx->state->stateid)) {
spin_unlock(&inode->i_lock);
/* If the open stateid was bad, then recover it. */
state = lgp->args.ctx->state;
exception->state = lgp->args.ctx->state;
break;
}
lo = NFS_I(inode)->layout;
Expand All @@ -7958,20 +7939,16 @@ static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
pnfs_free_lseg_list(&head);
} else
spin_unlock(&inode->i_lock);
goto out_restart;
status = -EAGAIN;
goto out;
}
if (nfs4_async_handle_error(task, server, state, &lgp->timeout) == -EAGAIN)
goto out_restart;

status = nfs4_handle_exception(server, status, exception);
if (exception->retry)
status = -EAGAIN;
out:
dprintk("<-- %s\n", __func__);
return;
out_restart:
task->tk_status = 0;
rpc_restart_call_prepare(task);
return;
out_overflow:
task->tk_status = -EOVERFLOW;
goto out;
return status;
}

static size_t max_response_pages(struct nfs_server *server)
Expand Down Expand Up @@ -8040,7 +8017,7 @@ static const struct rpc_call_ops nfs4_layoutget_call_ops = {
};

struct pnfs_layout_segment *
nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
nfs4_proc_layoutget(struct nfs4_layoutget *lgp, long *timeout, gfp_t gfp_flags)
{
struct inode *inode = lgp->args.inode;
struct nfs_server *server = NFS_SERVER(inode);
Expand All @@ -8060,6 +8037,7 @@ nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
.flags = RPC_TASK_ASYNC,
};
struct pnfs_layout_segment *lseg = NULL;
struct nfs4_exception exception = { .timeout = *timeout };
int status = 0;

dprintk("--> %s\n", __func__);
Expand All @@ -8073,7 +8051,6 @@ nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
return ERR_PTR(-ENOMEM);
}
lgp->args.layout.pglen = max_pages * PAGE_SIZE;
lgp->args.timestamp = jiffies;

lgp->res.layoutp = &lgp->args.layout;
lgp->res.seq_res.sr_slot = NULL;
Expand All @@ -8083,13 +8060,17 @@ nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags)
if (IS_ERR(task))
return ERR_CAST(task);
status = nfs4_wait_for_completion_rpc_task(task);
if (status == 0)
status = task->tk_status;
if (status == 0) {
status = nfs4_layoutget_handle_exception(task, lgp, &exception);
*timeout = exception.timeout;
}

trace_nfs4_layoutget(lgp->args.ctx,
&lgp->args.range,
&lgp->res.range,
&lgp->res.stateid,
status);

/* if layoutp->len is 0, nfs4_layoutget_prepare called rpc_exit */
if (status == 0 && lgp->res.layoutp->len)
lseg = pnfs_layout_process(lgp);
Expand Down
10 changes: 8 additions & 2 deletions fs/nfs/nfs4trace.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1520,6 +1520,8 @@ DEFINE_NFS4_INODE_EVENT(nfs4_layoutreturn_on_close);
{ PNFS_UPDATE_LAYOUT_FOUND_CACHED, "found cached" }, \
{ PNFS_UPDATE_LAYOUT_RETURN, "layoutreturn" }, \
{ PNFS_UPDATE_LAYOUT_BLOCKED, "layouts blocked" }, \
{ PNFS_UPDATE_LAYOUT_INVALID_OPEN, "invalid open" }, \
{ PNFS_UPDATE_LAYOUT_RETRY, "retrying" }, \
{ PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET, "sent layoutget" })

TRACE_EVENT(pnfs_update_layout,
Expand All @@ -1528,9 +1530,10 @@ TRACE_EVENT(pnfs_update_layout,
u64 count,
enum pnfs_iomode iomode,
struct pnfs_layout_hdr *lo,
struct pnfs_layout_segment *lseg,
enum pnfs_update_layout_reason reason
),
TP_ARGS(inode, pos, count, iomode, lo, reason),
TP_ARGS(inode, pos, count, iomode, lo, lseg, reason),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(u64, fileid)
Expand All @@ -1540,6 +1543,7 @@ TRACE_EVENT(pnfs_update_layout,
__field(enum pnfs_iomode, iomode)
__field(int, layoutstateid_seq)
__field(u32, layoutstateid_hash)
__field(long, lseg)
__field(enum pnfs_update_layout_reason, reason)
),
TP_fast_assign(
Expand All @@ -1559,18 +1563,20 @@ TRACE_EVENT(pnfs_update_layout,
__entry->layoutstateid_seq = 0;
__entry->layoutstateid_hash = 0;
}
__entry->lseg = (long)lseg;
),
TP_printk(
"fileid=%02x:%02x:%llu fhandle=0x%08x "
"iomode=%s pos=%llu count=%llu "
"layoutstateid=%d:0x%08x (%s)",
"layoutstateid=%d:0x%08x lseg=0x%lx (%s)",
MAJOR(__entry->dev), MINOR(__entry->dev),
(unsigned long long)__entry->fileid,
__entry->fhandle,
show_pnfs_iomode(__entry->iomode),
(unsigned long long)__entry->pos,
(unsigned long long)__entry->count,
__entry->layoutstateid_seq, __entry->layoutstateid_hash,
__entry->lseg,
show_pnfs_update_layout_reason(__entry->reason)
)
);
Expand Down
Loading

0 comments on commit 183d9e7

Please sign in to comment.