From 9bf6e5f740709f470ba350df64cd9f2c93f3f0a7 Mon Sep 17 00:00:00 2001
From: Evan Layton <evan.layton@nexenta.com>
Date: Wed, 10 Apr 2019 15:17:38 -0600
Subject: [PATCH 21/30] NEX-20260 NFS hung in transitional state when RSF marks
 it maintenance NEX-20423 NFSv4 state database entry locking is not always
 used around reference count. Reviewed by: Rick McNeal
 <rick.mcneal@nexenta.com> Reviewed by: Rob Gittins <rob.gittins@nexenta.com>

---
 usr/src/uts/common/fs/nfs/nfs4_db.c    | 57 ++++++++++++++++++++------
 usr/src/uts/common/fs/nfs/nfs4_state.c |  7 +++-
 2 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/usr/src/uts/common/fs/nfs/nfs4_db.c b/usr/src/uts/common/fs/nfs/nfs4_db.c
index 03be0c3c15..01639a78ec 100644
--- a/usr/src/uts/common/fs/nfs/nfs4_db.c
+++ b/usr/src/uts/common/fs/nfs/nfs4_db.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright 2018 Nexenta Systems, Inc.
+ * Copyright 2019 Nexenta Systems, Inc.
  */
 
 #include <sys/systm.h>
@@ -32,7 +32,6 @@
 #include <sys/kmem.h>
 #include <sys/disp.h>
 #include <sys/id_space.h>
-#include <sys/atomic.h>
 #include <rpc/rpc.h>
 #include <nfs/nfs4.h>
 #include <nfs/nfs4_db_impl.h>
@@ -65,7 +64,13 @@ rfs4_dbe_getid(rfs4_dbe_t *entry)
 void
 rfs4_dbe_hold(rfs4_dbe_t *entry)
 {
-	atomic_inc_32(&entry->dbe_refcnt);
+	if (!MUTEX_HELD(entry->dbe_lock)) {
+		mutex_enter(entry->dbe_lock);
+		entry->dbe_refcnt++;
+		mutex_exit(entry->dbe_lock);
+	} else {
+		entry->dbe_refcnt++;
+	}
 }
 
 /*
@@ -74,7 +79,14 @@ rfs4_dbe_hold(rfs4_dbe_t *entry)
 void
 rfs4_dbe_rele_nolock(rfs4_dbe_t *entry)
 {
-	atomic_dec_32(&entry->dbe_refcnt);
+	if (!MUTEX_HELD(entry->dbe_lock)) {
+		ASSERT(entry->dbe_refcnt > 0);
+		mutex_enter(entry->dbe_lock);
+		entry->dbe_refcnt--;
+		mutex_exit(entry->dbe_lock);
+	} else {
+		entry->dbe_refcnt--;
+	}
 }
 
 
@@ -91,8 +103,15 @@ rfs4_dbe_refcnt(rfs4_dbe_t *entry)
 void
 rfs4_dbe_invalidate(rfs4_dbe_t *entry)
 {
-	entry->dbe_invalid = TRUE;
-	entry->dbe_skipsearch = TRUE;
+	if (!MUTEX_HELD(entry->dbe_lock)) {
+		mutex_enter(entry->dbe_lock);
+		entry->dbe_invalid = TRUE;
+		entry->dbe_skipsearch = TRUE;
+		mutex_exit(entry->dbe_lock);
+	} else {
+		entry->dbe_invalid = TRUE;
+		entry->dbe_skipsearch = TRUE;
+	}
 }
 
 /*
@@ -134,7 +153,7 @@ rfs4_dbe_rele(rfs4_dbe_t *entry)
 {
 	mutex_enter(entry->dbe_lock);
 	ASSERT(entry->dbe_refcnt > 1);
-	atomic_dec_32(&entry->dbe_refcnt);
+	entry->dbe_refcnt--;
 	entry->dbe_time_rele = gethrestime_sec();
 	mutex_exit(entry->dbe_lock);
 }
@@ -822,25 +841,32 @@ rfs4_dbe_reap(rfs4_table_t *table, time_t cache_time, uint32_t desired)
 
 	/* Walk the buckets looking for entries to release/destroy */
 	for (i = 0; i < table->dbt_len; i++) {
+		int retries = 0;
 		bp = &buckets[i];
 		do {
 			found = FALSE;
 			rw_enter(bp->dbk_lock, RW_READER);
 			for (l = bp->dbk_head; l; l = l->next) {
 				entry = l->entry;
+				mutex_enter(entry->dbe_lock);
+				ASSERT(entry->dbe_refcnt != 0);
 				/*
 				 * Examine an entry.  Ref count of 1 means
 				 * that the only reference is for the hash
 				 * table reference.
 				 */
-				if (entry->dbe_refcnt != 1)
+				if (entry->dbe_refcnt != 1) {
+#ifdef DEBUG
+					rfs4_dbe_debug(entry);
+#endif
+					mutex_exit(entry->dbe_lock);
 					continue;
-				mutex_enter(entry->dbe_lock);
+				}
 				if ((entry->dbe_refcnt == 1) &&
 				    (table->dbt_reaper_shutdown ||
 				    table->dbt_expiry == NULL ||
 				    (*table->dbt_expiry)(entry->dbe_data))) {
-					entry->dbe_refcnt--;
+					rfs4_dbe_rele_nolock(entry);
 					count++;
 					found = TRUE;
 				}
@@ -857,13 +883,16 @@ rfs4_dbe_reap(rfs4_table_t *table, time_t cache_time, uint32_t desired)
 					t = l;
 					entry = t->entry;
 					l = l->next;
+					mutex_enter(entry->dbe_lock);
 					if (entry->dbe_refcnt == 0) {
 						DEQUEUE(bp->dbk_head, t);
+						mutex_exit(entry->dbe_lock);
 						t->next = NULL;
 						t->prev = NULL;
 						INVALIDATE_ADDR(t->entry);
 						rfs4_dbe_destroy(entry);
-					}
+					} else
+						mutex_exit(entry->dbe_lock);
 				}
 			}
 			rw_exit(bp->dbk_lock);
@@ -876,13 +905,15 @@ rfs4_dbe_reap(rfs4_table_t *table, time_t cache_time, uint32_t desired)
 			 * released.  This is only done in the
 			 * instance that the tables are being shut down.
 			 */
-			if (table->dbt_reaper_shutdown && bp->dbk_head != NULL)
+			if (table->dbt_reaper_shutdown && bp->dbk_head != NULL) {
 				delay(hz/100);
+				retries++;
+			}
 		/*
 		 * If this is a table shutdown, keep going until
 		 * everything is gone
 		 */
-		} while (table->dbt_reaper_shutdown && bp->dbk_head != NULL);
+		} while (table->dbt_reaper_shutdown && bp->dbk_head != NULL && retries < 5);
 
 		if (!table->dbt_reaper_shutdown && desired && count >= desired)
 			break;
diff --git a/usr/src/uts/common/fs/nfs/nfs4_state.c b/usr/src/uts/common/fs/nfs/nfs4_state.c
index fcd8f5b4ad..89678b6318 100644
--- a/usr/src/uts/common/fs/nfs/nfs4_state.c
+++ b/usr/src/uts/common/fs/nfs/nfs4_state.c
@@ -1499,8 +1499,13 @@ rfs4_state_zone_fini()
 
 	/* First stop all of the reaper threads in the database */
 	rfs4_database_shutdown(dbp);
+	/*
+	 * XXX workaround
+	 * Skip destrying the state database yet just in case there
+	 * are unfinished operations depending on it.
+	 */
 	/* Now destroy/release the database tables */
-	rfs4_database_destroy(dbp);
+	/* rfs4_database_destroy(dbp); */
 
 	/* Reset the cache timers for next time */
 	nsrv4->rfs4_client_cache_time = 0;
-- 
2.21.0

