Revert "NEX-20260 NFS hung in transitional state when RSF marks it maintenance"
This reverts commit 9bf6e5f740709f470ba350df64cd9f2c93f3f0a7.

@@ -22,18 +22,19 @@
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*
- * Copyright 2019 Nexenta Systems, Inc.
+ * Copyright 2018 Nexenta Systems, Inc.
  */
 
 #include <sys/systm.h>
 #include <sys/cmn_err.h>
 #include <sys/kmem.h>
 #include <sys/disp.h>
 #include <sys/id_space.h>
+#include <sys/atomic.h>
 #include <rpc/rpc.h>
 #include <nfs/nfs4.h>
 #include <nfs/nfs4_db_impl.h>
 #include <sys/sdt.h>
 

@@ -62,33 +63,20 @@
 }
 
 void
 rfs4_dbe_hold(rfs4_dbe_t *entry)
 {
-        if (!MUTEX_HELD(entry->dbe_lock)) {
-                mutex_enter(entry->dbe_lock);
-                entry->dbe_refcnt++;
-                mutex_exit(entry->dbe_lock);
-        } else {
-                entry->dbe_refcnt++;
-        }
+        atomic_inc_32(&entry->dbe_refcnt);
 }
 
 /*
  * rfs4_dbe_rele_nolock only decrements the reference count of the entry.
  */
 void
 rfs4_dbe_rele_nolock(rfs4_dbe_t *entry)
 {
-        if (!MUTEX_HELD(entry->dbe_lock)) {
-                ASSERT(entry->dbe_refcnt > 0);
-                mutex_enter(entry->dbe_lock);
-                entry->dbe_refcnt--;
-                mutex_exit(entry->dbe_lock);
-        } else {
-                entry->dbe_refcnt--;
-        }
+        atomic_dec_32(&entry->dbe_refcnt);
 }
 
 
 uint32_t
 rfs4_dbe_refcnt(rfs4_dbe_t *entry)

@@ -101,19 +89,12 @@
  * Caller does not want this entry to be found any longer
  */
 void
 rfs4_dbe_invalidate(rfs4_dbe_t *entry)
 {
-        if (!MUTEX_HELD(entry->dbe_lock)) {
-                mutex_enter(entry->dbe_lock);
                 entry->dbe_invalid = TRUE;
                 entry->dbe_skipsearch = TRUE;
-                mutex_exit(entry->dbe_lock);
-        } else {
-                entry->dbe_invalid = TRUE;
-                entry->dbe_skipsearch = TRUE;
-        }
 }
 
 /*
  * Is this entry invalid?
  */

@@ -151,11 +132,11 @@
 void
 rfs4_dbe_rele(rfs4_dbe_t *entry)
 {
         mutex_enter(entry->dbe_lock);
         ASSERT(entry->dbe_refcnt > 1);
-        entry->dbe_refcnt--;
+        atomic_dec_32(&entry->dbe_refcnt);
         entry->dbe_time_rele = gethrestime_sec();
         mutex_exit(entry->dbe_lock);
 }
 
 void

@@ -839,33 +820,29 @@
             (CE_NOTE, "Reaping %d entries older than %ld seconds in table %s",
             desired, cache_time, table->dbt_name));
 
         /* Walk the buckets looking for entries to release/destroy */
         for (i = 0; i < table->dbt_len; i++) {
-                int retries = 0;
                 bp = &buckets[i];
                 do {
                         found = FALSE;
                         rw_enter(bp->dbk_lock, RW_READER);
                         for (l = bp->dbk_head; l; l = l->next) {
                                 entry = l->entry;
-                                mutex_enter(entry->dbe_lock);
-                                ASSERT(entry->dbe_refcnt != 0);
                                 /*
                                  * Examine an entry.  Ref count of 1 means
                                  * that the only reference is for the hash
                                  * table reference.
                                  */
-                                if (entry->dbe_refcnt != 1) {
-                                        mutex_exit(entry->dbe_lock);
+                                if (entry->dbe_refcnt != 1)
                                         continue;
-                                }
+                                mutex_enter(entry->dbe_lock);
                                 if ((entry->dbe_refcnt == 1) &&
                                     (table->dbt_reaper_shutdown ||
                                     table->dbt_expiry == NULL ||
                                     (*table->dbt_expiry)(entry->dbe_data))) {
-                                        rfs4_dbe_rele_nolock(entry);
+                                        entry->dbe_refcnt--;
                                         count++;
                                         found = TRUE;
                                 }
                                 mutex_exit(entry->dbe_lock);
                         }

@@ -878,22 +855,19 @@
                                 l = bp->dbk_head;
                                 while (l) {
                                         t = l;
                                         entry = t->entry;
                                         l = l->next;
-                                        mutex_enter(entry->dbe_lock);
                                         if (entry->dbe_refcnt == 0) {
                                                 DEQUEUE(bp->dbk_head, t);
-                                                mutex_exit(entry->dbe_lock);
                                                 t->next = NULL;
                                                 t->prev = NULL;
                                                 INVALIDATE_ADDR(t->entry);
                                                 rfs4_dbe_destroy(entry);
-                                        } else
-                                                mutex_exit(entry->dbe_lock);
                                 }
                         }
+                        }
                         rw_exit(bp->dbk_lock);
                         /*
                          * delay slightly if there is more work to do
                          * with the expectation that other reaper
                          * threads are freeing data structures as well

@@ -900,19 +874,17 @@
                          * and in turn will reduce ref counts on
                          * entries in this table allowing them to be
                          * released.  This is only done in the
                          * instance that the tables are being shut down.
                          */
-                        if (table->dbt_reaper_shutdown && bp->dbk_head != NULL) {
+                        if (table->dbt_reaper_shutdown && bp->dbk_head != NULL)
                                 delay(hz/100);
-                                retries++;
-                        }
                 /*
                  * If this is a table shutdown, keep going until
                  * everything is gone
                  */
-                } while (table->dbt_reaper_shutdown && bp->dbk_head != NULL && retries < 5);
+                } while (table->dbt_reaper_shutdown && bp->dbk_head != NULL);
 
                 if (!table->dbt_reaper_shutdown && desired && count >= desired)
                         break;
         }