Dan mods to NFS desgin problems re. multiple zone keys
Do rfs4_ss_fini() BEFORE rfs4_servinst_destroy_all().
Go ahead and destroy the NFSv4 database tables

@@ -1494,10 +1494,18 @@
         nfs4_srv_t *nsrv4;
         nsrv4 = nfs4_get_srv();
 
         rfs4_set_deleg_policy(nsrv4, SRV_NEVER_DELEGATE);
 
+        /*
+         * Clean up any dangling stable storage structures BEFORE calling
+         * rfs4_servinst_destroy_all() so there are no dangling structures
+         * (i.e. the srvinsts are all cleared of danglers BEFORE they get
+         * freed).
+         */
+        rfs4_ss_fini(nsrv4);
+
         mutex_enter(&nsrv4->state_lock);
 
         if (nsrv4->nfs4_server_state == NULL) {
                 mutex_exit(&nsrv4->state_lock);
                 return;

@@ -1515,16 +1523,18 @@
         rw_destroy(&nsrv4->rfs4_findclient_lock);
 
         /* First stop all of the reaper threads in the database */
         rfs4_database_shutdown(dbp);
         /*
-         * XXX workaround
-         * Skip destrying the state database yet just in case there
-         * are unfinished operations depending on it.
+         * WARNING: There may be consumers of the rfs4 database still
+         * active as we destroy these.  IF that's the case, consider putting
+         * some of their _zone_fini()-like functions into the zsd key as
+         * ~~SHUTDOWN~~ functions instead of ~~DESTROY~~ functions.  We can
+         * maintain some ordering guarantees better that way.
          */
         /* Now destroy/release the database tables */
-        /* rfs4_database_destroy(dbp); */
+        rfs4_database_destroy(dbp);
 
         /* Reset the cache timers for next time */
         nsrv4->rfs4_client_cache_time = 0;
         nsrv4->rfs4_openowner_cache_time = 0;
         nsrv4->rfs4_state_cache_time = 0;

@@ -1532,13 +1542,10 @@
         nsrv4->rfs4_lockowner_cache_time = 0;
         nsrv4->rfs4_file_cache_time = 0;
         nsrv4->rfs4_deleg_state_cache_time = 0;
 
         mutex_exit(&nsrv4->state_lock);
-
-        /* clean up any dangling stable storage structures */
-        rfs4_ss_fini(nsrv4);
 }
 
 typedef union {
         struct {
                 uint32_t start_time;

@@ -4121,14 +4128,14 @@
         if (nsrv4->nfs4_server_state == NULL) {
                 mutex_exit(&nsrv4->state_lock);
                 return;
         }
 
-        /* CSTYLED */
-        rfs4_dbe_walk(nsrv4->rfs4_lo_state_tab, rfs4_lo_state_walk_callout, exi);
+        rfs4_dbe_walk(nsrv4->rfs4_lo_state_tab,
+            rfs4_lo_state_walk_callout, exi);
         rfs4_dbe_walk(nsrv4->rfs4_state_tab, rfs4_state_walk_callout, exi);
-        /* CSTYLED */
-        rfs4_dbe_walk(nsrv4->rfs4_deleg_state_tab, rfs4_deleg_state_walk_callout, exi);
+        rfs4_dbe_walk(nsrv4->rfs4_deleg_state_tab,
+            rfs4_deleg_state_walk_callout, exi);
         rfs4_dbe_walk(nsrv4->rfs4_file_tab, rfs4_file_walk_callout, exi);
 
         mutex_exit(&nsrv4->state_lock);
 }