From 3a5019eea2782f6b7feb58451bee2cfe0b5d11b6 Mon Sep 17 00:00:00 2001
From: Dan McDonald <danmcd@joyent.com>
Date: Mon, 16 Sep 2019 17:32:25 -0400
Subject: [PATCH 32/34] curzone reality check and teardown changes to use the
 RIGHT zone

---
 usr/src/uts/common/fs/nfs/nfs3_srv.c    |  3 +++
 usr/src/uts/common/fs/nfs/nfs4_srv_ns.c |  9 +++++++--
 usr/src/uts/common/fs/nfs/nfs4_state.c  |  4 +++-
 usr/src/uts/common/fs/nfs/nfs_auth.c    |  1 +
 usr/src/uts/common/fs/nfs/nfs_export.c  | 18 +++++++++++++++++-
 usr/src/uts/common/fs/nfs/nfs_srv.c     |  4 ++++
 usr/src/uts/common/klm/nlm_impl.c       |  8 +++++++-
 usr/src/uts/common/nfs/export.h         | 18 ++++++++++++++++--
 8 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/usr/src/uts/common/fs/nfs/nfs3_srv.c b/usr/src/uts/common/fs/nfs/nfs3_srv.c
index cededd1790..1efd694178 100644
--- a/usr/src/uts/common/fs/nfs/nfs3_srv.c
+++ b/usr/src/uts/common/fs/nfs/nfs3_srv.c
@@ -436,6 +436,7 @@ rfs3_lookup(LOOKUP3args *args, LOOKUP3res *resp, struct exportinfo *exi,
 	}
 
 	fhp = &args->what.dir;
+	ASSERT3P(curzone, ==, exi->exi_zone); /* exi is guaranteed non-NULL. */
 	if (strcmp(args->what.name, "..") == 0 &&
 	    EQFID(&exi->exi_fid, FH3TOFIDP(fhp))) {
 		if ((exi->exi_export.ex_flags & EX_NOHIDE) &&
@@ -1325,6 +1326,7 @@ rfs3_write(WRITE3args *args, WRITE3res *resp, struct exportinfo *exi,
 		goto err;
 	}
 
+	ASSERT3P(curzone, ==, exi->exi_zone); /* exi is guaranteed non-NULL. */
 	ns = zone_getspecific(rfs3_zone_key, curzone);
 	if (is_system_labeled()) {
 		bslabel_t *clabel = req->rq_label;
@@ -4125,6 +4127,7 @@ rfs3_commit(COMMIT3args *args, COMMIT3res *resp, struct exportinfo *exi,
 		goto out;
 	}
 
+	ASSERT3P(curzone, ==, exi->exi_zone); /* exi is guaranteed non-NULL. */
 	ns = zone_getspecific(rfs3_zone_key, curzone);
 	bva.va_mask = AT_ALL;
 	error = VOP_GETATTR(vp, &bva, 0, cr, NULL);
diff --git a/usr/src/uts/common/fs/nfs/nfs4_srv_ns.c b/usr/src/uts/common/fs/nfs/nfs4_srv_ns.c
index 0781c1e96c..c392e9cf4d 100644
--- a/usr/src/uts/common/fs/nfs/nfs4_srv_ns.c
+++ b/usr/src/uts/common/fs/nfs/nfs4_srv_ns.c
@@ -146,8 +146,8 @@ nfs4_vget_pseudo(struct exportinfo *exi, vnode_t **vpp, fid_t *fidp)
  * b) a descendant of the export root is shared
  */
 struct exportinfo *
-pseudo_exportfs(nfs_export_t *ne, vnode_t *vp, fid_t *fid, struct exp_visible *vis_head,
-    struct exportdata *exdata)
+pseudo_exportfs(nfs_export_t *ne, vnode_t *vp, fid_t *fid,
+    struct exp_visible *vis_head, struct exportdata *exdata)
 {
 	struct exportinfo *exi;
 	struct exportdata *kex;
@@ -165,6 +165,7 @@ pseudo_exportfs(nfs_export_t *ne, vnode_t *vp, fid_t *fid, struct exp_visible *v
 	VN_HOLD(exi->exi_vp);
 	exi->exi_visible = vis_head;
 	exi->exi_count = 1;
+	/* Caller will set exi_zone... */
 	exi->exi_volatile_dev = (vfssw[vp->v_vfsp->vfs_fstype].vsw_flag &
 	    VSW_VOLATILEDEV) ? 1 : 0;
 	mutex_init(&exi->exi_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -642,6 +643,7 @@ treeclimb_export(struct exportinfo *exip)
 	nfs_export_t *ne = nfs_get_export();
 
 	ASSERT(RW_WRITE_HELD(&ne->exported_lock));
+	ASSERT3P(curzone, ==, exip->exi_zone);
 
 	gethrestime(&now);
 
@@ -693,6 +695,7 @@ treeclimb_export(struct exportinfo *exip)
 				 */
 				new_exi = pseudo_exportfs(ne, vp, &fid,
 				    vis_head, NULL);
+				new_exi->exi_zone = exip->exi_zone;
 				vis_head = NULL;
 			}
 
@@ -835,6 +838,7 @@ treeclimb_unexport(nfs_export_t *ne, struct exportinfo *exip)
 	treenode_t *connect_point = NULL;
 
 	ASSERT(RW_WRITE_HELD(&ne->exported_lock));
+	ASSERT(curzone == exip->exi_zone || curzone == global_zone);
 
 	tnode = exip->exi_tree;
 	/*
@@ -983,6 +987,7 @@ has_visible(struct exportinfo *exi, vnode_t *vp)
 	 * list.  i.e. if it does not have a visible list, then there is no
 	 * node in this filesystem leads to any other shared node.
 	 */
+	ASSERT3P(curzone, ==, exi->exi_zone);
 	if (vp_is_exported &&
 	    ((vp->v_flag & VROOT) || VN_IS_CURZONEROOT(vp))) {
 		return (exi->exi_visible ? 1 : 0);
diff --git a/usr/src/uts/common/fs/nfs/nfs4_state.c b/usr/src/uts/common/fs/nfs/nfs4_state.c
index d3bce3d057..94a3426183 100644
--- a/usr/src/uts/common/fs/nfs/nfs4_state.c
+++ b/usr/src/uts/common/fs/nfs/nfs4_state.c
@@ -4107,7 +4107,9 @@ rfs4_clean_state_exi(struct exportinfo *exi)
 {
 	nfs4_srv_t *nsrv4;
 
-	nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
+	/* curzone mightn't be exi_zone, so use exi_zone instead. */
+	ASSERT(exi->exi_zone == curzone || curzone == global_zone);
+	nsrv4 = zone_getspecific(rfs4_zone_key, exi->exi_zone);
 	mutex_enter(&nsrv4->state_lock);
 
 	if (nsrv4->nfs4_server_state == NULL) {
diff --git a/usr/src/uts/common/fs/nfs/nfs_auth.c b/usr/src/uts/common/fs/nfs/nfs_auth.c
index e0412dd713..43a5cd0409 100644
--- a/usr/src/uts/common/fs/nfs/nfs_auth.c
+++ b/usr/src/uts/common/fs/nfs/nfs_auth.c
@@ -880,6 +880,7 @@ nfsauth_cache_get(struct exportinfo *exi, struct svc_req *req, int flavor,
 
 	ASSERT(cr != NULL);
 
+	ASSERT3P(curzone, ==, exi->exi_zone);
 	nag = zone_getspecific(nfsauth_zone_key, curzone);
 
 	/*
diff --git a/usr/src/uts/common/fs/nfs/nfs_export.c b/usr/src/uts/common/fs/nfs/nfs_export.c
index 5ed07d5083..58bd00c8dc 100644
--- a/usr/src/uts/common/fs/nfs/nfs_export.c
+++ b/usr/src/uts/common/fs/nfs/nfs_export.c
@@ -692,9 +692,17 @@ exportinfo_t *
 vis2exi(treenode_t *tnode)
 {
 	exportinfo_t *exi_ret = NULL;
+#ifdef DEBUG
+	zone_t *zone;
+
+	ASSERT(tnode->tree_exi != NULL);
+	zone = tnode->tree_exi->exi_zone;
+#endif
 
 	for (;;) {
 		tnode = tnode->tree_parent;
+		ASSERT(tnode->tree_exi != NULL);
+		ASSERT3P(zone, ==, tnode->tree_exi->exi_zone);
 		if (TREE_ROOT(tnode)) {
 			exi_ret = tnode->tree_exi;
 			break;
@@ -1300,7 +1308,9 @@ exportfs(struct exportfs_args *args, model_t model, cred_t *cr)
 	exi->exi_fid = fid;
 	exi->exi_vp = vp;
 	exi->exi_count = 1;
-	exi->exi_zoneid = crgetzoneid(cr);
+	exi->exi_zone = crgetzone(cr);
+	ASSERT(exi->exi_zone != NULL);		/* XXX KEBE ASKS... */
+	ASSERT3P(exi->exi_zone, ==, curzone);	/* ... are these legit? */
 	exi->exi_volatile_dev = (vfssw[vp->v_vfsp->vfs_fstype].vsw_flag &
 	    VSW_VOLATILEDEV) ? 1 : 0;
 	mutex_init(&exi->exi_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -1814,6 +1824,7 @@ unexport(nfs_export_t *ne, struct exportinfo *exi)
 		exi->exi_visible = NULL;
 
 		/* interconnect the existing treenode with the new exportinfo */
+		newexi->exi_zone = exi->exi_zone;
 		newexi->exi_tree = exi->exi_tree;
 		newexi->exi_tree->tree_exi = newexi;
 
@@ -1843,6 +1854,11 @@ unexport(nfs_export_t *ne, struct exportinfo *exi)
 	 * If this was a public export, restore
 	 * the public filehandle to the root.
 	 */
+
+	/*
+	 * XXX KEBE ASKS --> Should CRED() instead be
+	 * exi->exi_zone->zone_kcred?
+	 */
 	if (exi == ne->exi_public) {
 		ne->exi_public = ne->exi_root;
 
diff --git a/usr/src/uts/common/fs/nfs/nfs_srv.c b/usr/src/uts/common/fs/nfs/nfs_srv.c
index 3074d078d5..131c97456f 100644
--- a/usr/src/uts/common/fs/nfs/nfs_srv.c
+++ b/usr/src/uts/common/fs/nfs/nfs_srv.c
@@ -411,6 +411,7 @@ rfs_climb_crossmnt(vnode_t **dvpp, struct exportinfo **exip, cred_t *cr)
 	struct exportinfo *exi;
 	vnode_t *dvp = *dvpp;
 
+	ASSERT3P((*exip)->exi_zone, ==, curzone);
 	ASSERT((dvp->v_flag & VROOT) || VN_IS_CURZONEROOT(dvp));
 
 	VN_HOLD(dvp);
@@ -421,6 +422,7 @@ rfs_climb_crossmnt(vnode_t **dvpp, struct exportinfo **exip, cred_t *cr)
 		return (-1);
 	}
 
+	ASSERT3P(exi->exi_zone, ==, curzone);
 	exi_rele(*exip);
 	*exip = exi;
 	VN_RELE(*dvpp);
@@ -481,6 +483,7 @@ rfs_lookup(struct nfsdiropargs *da, struct nfsdiropres *dr,
 	}
 
 	exi_hold(exi);
+	ASSERT3P(exi->exi_zone, ==, curzone);
 
 	/*
 	 * Not allow lookup beyond root.
@@ -1312,6 +1315,7 @@ rfs_write(struct nfswriteargs *wa, struct nfsattrstat *ns,
 	caller_context_t ct;
 	nfs_srv_t *nsrv;
 
+	ASSERT3P(curzone, ==, ((exi == NULL) ? curzone : exi->exi_zone));
 	nsrv = zone_getspecific(rfs_zone_key, curzone);
 	if (!nsrv->write_async) {
 		rfs_write_sync(wa, ns, exi, req, cr, ro);
diff --git a/usr/src/uts/common/klm/nlm_impl.c b/usr/src/uts/common/klm/nlm_impl.c
index e787f70ebd..59e5ef84ae 100644
--- a/usr/src/uts/common/klm/nlm_impl.c
+++ b/usr/src/uts/common/klm/nlm_impl.c
@@ -2594,7 +2594,13 @@ nlm_unexport(struct exportinfo *exi)
 	struct nlm_globals *g;
 	struct nlm_host *hostp;
 
-	g = zone_getspecific(nlm_zone_key, curzone);
+	/* This may be called on behalf of global-zone doing shutdown. */
+	ASSERT(exi->exi_zone == curzone || curzone == global_zone);
+	g = zone_getspecific(nlm_zone_key, exi->exi_zone);
+	if (g == NULL) {
+		/* Did zone cleanup get here already? */
+		return;
+	}
 
 	mutex_enter(&g->lock);
 	hostp = avl_first(&g->nlm_hosts_tree);
diff --git a/usr/src/uts/common/nfs/export.h b/usr/src/uts/common/nfs/export.h
index 5dd21aac4c..3a01abfad1 100644
--- a/usr/src/uts/common/nfs/export.h
+++ b/usr/src/uts/common/nfs/export.h
@@ -481,7 +481,7 @@ typedef struct treenode {
 #define	TREE_ROOT(t) \
 	((t)->tree_exi != NULL && \
 	(((t)->tree_exi->exi_vp->v_flag & VROOT) || \
-	VN_IS_CURZONEROOT((t)->tree_exi->exi_vp)))
+	VN_CMP((t)->tree_exi->exi_zone->zone_rootvp, (t)->tree_exi->exi_vp)))
 
 #define	TREE_EXPORTED(t) \
 	((t)->tree_exi && !PSEUDO((t)->tree_exi))
@@ -533,13 +533,27 @@ struct exportinfo {
 	unsigned		exi_moved:1;
 	int			exi_id;
 	avl_node_t		exi_id_link;
-	zoneid_t		exi_zoneid;
+	/*
+	 * Soft-reference/backpointer to the zone.  The ZSD callbacks we have
+	 * invoke cleanup code that crosses into OTHER cleanup functions that
+	 * may assume same-zone context and attempt to find their own ZSD,
+	 * using "curzone" when in fact "curzone" is global when called from
+	 * NFS's ZSD cleanup (see lm_unexport->nlm_unexport for an example).
+	 *
+	 * During ZSD shutdown or destroy callbacks, the zone structure
+	 * does not have its mutex held, and it has just-enough references
+	 * to not free from underneath us.  This field is not a proper
+	 * referenced-held zone pointer, and only ZSD callbacks should use
+	 * it.
+	 */
+	struct zone		*exi_zone;
 #ifdef VOLATILE_FH_TEST
 	uint32_t		exi_volatile_id;
 	struct ex_vol_rename	*exi_vol_rename;
 	kmutex_t		exi_vol_rename_lock;
 #endif /* VOLATILE_FH_TEST */
 };
+#define	exi_zoneid exi_zone->zone_id
 
 typedef struct exportinfo exportinfo_t;
 typedef struct exportdata exportdata_t;
-- 
2.20.1 (Apple Git-117)

