Commit 4f3c8d8c authored by Chuck Lever's avatar Chuck Lever
Browse files

NFSD: Implement CB_SEQUENCE referring call lists 



We have yet to implement a mechanism in NFSD for resolving races
between a server's reply and a related callback operation. For
example, a CB_OFFLOAD callback can race with the matching COPY
response. The client will not recognize the copy state ID in the
CB_OFFLOAD callback until the COPY response arrives.

Trond adds:
> It is also needed for the same kind of race with delegation
> recalls, layout recalls, CB_NOTIFY_DEVICEID and would also be
> helpful (although not as strongly required) for CB_NOTIFY_LOCK.

RFC 8881 Section 20.9.3 describes referring call lists this way:
> The csa_referring_call_lists array is the list of COMPOUND
> requests, identified by session ID, slot ID, and sequence ID.
> These are requests that the client previously sent to the server.
> These previous requests created state that some operation(s) in
> the same CB_COMPOUND as the csa_referring_call_lists are
> identifying. A session ID is included because leased state is tied
> to a client ID, and a client ID can have multiple sessions. See
> Section 2.10.6.3.

Introduce the XDR infrastructure for populating the
csa_referring_call_lists argument of CB_SEQUENCE. Subsequent patches
will put the referring call list to use.

Note that cb_sequence_enc_sz estimates that only zero or one rcl is
included in each CB_SEQUENCE, but the new infrastructure can
manage any number of referring calls.

Reviewed-by: default avatarJeff Layton <jlayton@kernel.org>
Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
parent 71aeab7b

fs/nfsd/nfs4callback.c

+127 −5
Original line number Diff line number Diff line
@@ -417,6 +417,29 @@ static u32 highest_slotid(struct nfsd4_session *ses) 
	return idx;

}



static void

encode_referring_call4(struct xdr_stream *xdr,

		       const struct nfsd4_referring_call *rc)

{

	encode_uint32(xdr, rc->rc_sequenceid);

	encode_uint32(xdr, rc->rc_slotid);

}



static void

encode_referring_call_list4(struct xdr_stream *xdr,

			    const struct nfsd4_referring_call_list *rcl)

{

	struct nfsd4_referring_call *rc;

	__be32 *p;



	p = xdr_reserve_space(xdr, NFS4_MAX_SESSIONID_LEN);

	xdr_encode_opaque_fixed(p, rcl->rcl_sessionid.data,

					NFS4_MAX_SESSIONID_LEN);

	encode_uint32(xdr, rcl->__nr_referring_calls);

	list_for_each_entry(rc, &rcl->rcl_referring_calls, __list)

		encode_referring_call4(xdr, rc);

}



/*

 * CB_SEQUENCE4args

 *
@@ -434,6 +457,7 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, 
				    struct nfs4_cb_compound_hdr *hdr)

{

	struct nfsd4_session *session = cb->cb_clp->cl_cb_session;

	struct nfsd4_referring_call_list *rcl;

	__be32 *p;



	if (hdr->minorversion == 0)
@@ -442,12 +466,16 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, 
	encode_nfs_cb_opnum4(xdr, OP_CB_SEQUENCE);

	encode_sessionid4(xdr, session);



	p = xdr_reserve_space(xdr, 4 + 4 + 4 + 4 + 4);

	p = xdr_reserve_space(xdr, XDR_UNIT * 4);

	*p++ = cpu_to_be32(session->se_cb_seq_nr[cb->cb_held_slot]);	/* csa_sequenceid */

	*p++ = cpu_to_be32(cb->cb_held_slot);		/* csa_slotid */

	*p++ = cpu_to_be32(highest_slotid(session)); /* csa_highest_slotid */

	*p++ = xdr_zero;			/* csa_cachethis */

	xdr_encode_empty_array(p);		/* csa_referring_call_lists */



	/* csa_referring_call_lists */

	encode_uint32(xdr, cb->cb_nr_referring_call_list);

	list_for_each_entry(rcl, &cb->cb_referring_call_list, __list)

		encode_referring_call_list4(xdr, rcl);



	hdr->nops++;

}
@@ -1320,10 +1348,102 @@ static void nfsd41_destroy_cb(struct nfsd4_callback *cb) 
	nfsd41_cb_inflight_end(clp);

}



/*

 * TODO: cb_sequence should support referring call lists, cachethis,

 * and mark callback channel down on communication errors.

/**

 * nfsd41_cb_referring_call - add a referring call to a callback operation

 * @cb: context of callback to add the rc to

 * @sessionid: referring call's session ID

 * @slotid: referring call's session slot index

 * @seqno: referring call's slot sequence number

 *

 * Caller serializes access to @cb.

 *

 * NB: If memory allocation fails, the referring call is not added.

 */

void nfsd41_cb_referring_call(struct nfsd4_callback *cb,

			      struct nfs4_sessionid *sessionid,

			      u32 slotid, u32 seqno)

{

	struct nfsd4_referring_call_list *rcl;

	struct nfsd4_referring_call *rc;

	bool found;



	might_sleep();



	found = false;

	list_for_each_entry(rcl, &cb->cb_referring_call_list, __list) {

		if (!memcmp(rcl->rcl_sessionid.data, sessionid->data,

			   NFS4_MAX_SESSIONID_LEN)) {

			found = true;

			break;

		}

	}

	if (!found) {

		rcl = kmalloc(sizeof(*rcl), GFP_KERNEL);

		if (!rcl)

			return;

		memcpy(rcl->rcl_sessionid.data, sessionid->data,

		       NFS4_MAX_SESSIONID_LEN);

		rcl->__nr_referring_calls = 0;

		INIT_LIST_HEAD(&rcl->rcl_referring_calls);

		list_add(&rcl->__list, &cb->cb_referring_call_list);

		cb->cb_nr_referring_call_list++;

	}



	found = false;

	list_for_each_entry(rc, &rcl->rcl_referring_calls, __list) {

		if (rc->rc_sequenceid == seqno && rc->rc_slotid == slotid) {

			found = true;

			break;

		}

	}

	if (!found) {

		rc = kmalloc(sizeof(*rc), GFP_KERNEL);

		if (!rc)

			goto out;

		rc->rc_sequenceid = seqno;

		rc->rc_slotid = slotid;

		rcl->__nr_referring_calls++;

		list_add(&rc->__list, &rcl->rcl_referring_calls);

	}



out:

	if (!rcl->__nr_referring_calls) {

		cb->cb_nr_referring_call_list--;

		kfree(rcl);

	}

}



/**

 * nfsd41_cb_destroy_referring_call_list - release referring call info

 * @cb: context of a callback that has completed

 *

 * Callers who allocate referring calls using nfsd41_cb_referring_call() must

 * release those resources by calling nfsd41_cb_destroy_referring_call_list.

 *

 * Caller serializes access to @cb.

 */

void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb)

{

	struct nfsd4_referring_call_list *rcl;

	struct nfsd4_referring_call *rc;



	while (!list_empty(&cb->cb_referring_call_list)) {

		rcl = list_first_entry(&cb->cb_referring_call_list,

				       struct nfsd4_referring_call_list,

				       __list);



		while (!list_empty(&rcl->rcl_referring_calls)) {

			rc = list_first_entry(&rcl->rcl_referring_calls,

					      struct nfsd4_referring_call,

					      __list);

			list_del(&rc->__list);

			kfree(rc);

		}

		list_del(&rcl->__list);

		kfree(rcl);

	}

}



static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata)

{

	struct nfsd4_callback *cb = calldata;
@@ -1643,6 +1763,8 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, 
	INIT_WORK(&cb->cb_work, nfsd4_run_cb_work);

	cb->cb_status = 0;

	cb->cb_held_slot = -1;

	cb->cb_nr_referring_call_list = 0;

	INIT_LIST_HEAD(&cb->cb_referring_call_list);

}



/**

fs/nfsd/state.h

+22 −0
Original line number Diff line number Diff line
@@ -64,6 +64,21 @@ typedef struct { 
	refcount_t		cs_count;

} copy_stateid_t;



struct nfsd4_referring_call {

	struct list_head	__list;



	u32			rc_sequenceid;

	u32			rc_slotid;

};



struct nfsd4_referring_call_list {

	struct list_head	__list;



	struct nfs4_sessionid	rcl_sessionid;

	int			__nr_referring_calls;

	struct list_head	rcl_referring_calls;

};



struct nfsd4_callback {

	struct nfs4_client *cb_clp;

	struct rpc_message cb_msg;
@@ -76,6 +91,9 @@ struct nfsd4_callback { 
	int cb_seq_status;

	int cb_status;

	int cb_held_slot;



	int cb_nr_referring_call_list;

	struct list_head cb_referring_call_list;

};



struct nfsd4_callback_ops {
@@ -774,6 +792,10 @@ extern __be32 nfs4_check_open_reclaim(struct nfs4_client *); 
extern void nfsd4_probe_callback(struct nfs4_client *clp);

extern void nfsd4_probe_callback_sync(struct nfs4_client *clp);

extern void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *);

extern void nfsd41_cb_referring_call(struct nfsd4_callback *cb,

				     struct nfs4_sessionid *sessionid,

				     u32 slotid, u32 seqno);

extern void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb);

extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,

		const struct nfsd4_callback_ops *ops, enum nfsd4_cb_op op);

extern bool nfsd4_run_cb(struct nfsd4_callback *cb);

fs/nfsd/xdr4cb.h

+4 −1
Original line number Diff line number Diff line
@@ -6,8 +6,11 @@ 
#define cb_compound_enc_hdr_sz		4

#define cb_compound_dec_hdr_sz		(3 + (NFS4_MAXTAGLEN >> 2))

#define sessionid_sz			(NFS4_MAX_SESSIONID_LEN >> 2)

#define enc_referring_call4_sz		(1 + 1)

#define enc_referring_call_list4_sz	(sessionid_sz + 1 + \

					enc_referring_call4_sz)

#define cb_sequence_enc_sz		(sessionid_sz + 4 +             \

					1 /* no referring calls list yet */)

					enc_referring_call_list4_sz)

#define cb_sequence_dec_sz		(op_dec_sz + sessionid_sz + 4)



#define op_enc_sz			1