5056 ZFS deadlock on db_mtx and dn_holds
Reviewed by: Will Andrews <willa@spectralogic.com>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

*** 20,29 ****
--- 20,30 ----
   */
  /*
   * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
   * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
   * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
   */
  
  #include <sys/zfs_context.h>
  #include <sys/spa_impl.h>
  #include <sys/spa_boot.h>
*** 549,567 ****
--- 550,570 ----
          spa = kmem_zalloc(sizeof (spa_t), KM_SLEEP);
  
          mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_errlog_lock, NULL, MUTEX_DEFAULT, NULL);
+         mutex_init(&spa->spa_evicting_os_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_history_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_proc_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_props_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_scrub_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_suspend_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_vdev_top_lock, NULL, MUTEX_DEFAULT, NULL);
          mutex_init(&spa->spa_iokstat_lock, NULL, MUTEX_DEFAULT, NULL);
  
          cv_init(&spa->spa_async_cv, NULL, CV_DEFAULT, NULL);
+         cv_init(&spa->spa_evicting_os_cv, NULL, CV_DEFAULT, NULL);
          cv_init(&spa->spa_proc_cv, NULL, CV_DEFAULT, NULL);
          cv_init(&spa->spa_scrub_io_cv, NULL, CV_DEFAULT, NULL);
          cv_init(&spa->spa_suspend_cv, NULL, CV_DEFAULT, NULL);
  
          for (int t = 0; t < TXG_SIZE; t++)
*** 667,676 ****
--- 670,680 ----
  {
          spa_config_dirent_t *dp;
  
          ASSERT(MUTEX_HELD(&spa_namespace_lock));
          ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
+         ASSERT3U(refcount_count(&spa->spa_refcount), ==, 0);
  
          nvlist_free(spa->spa_config_splitting);
  
          avl_remove(&spa_namespace_avl, spa);
          cv_broadcast(&spa_namespace_cv);
*** 708,724 ****
--- 712,730 ----
  
          for (int t = 0; t < TXG_SIZE; t++)
                  bplist_destroy(&spa->spa_free_bplist[t]);
  
          cv_destroy(&spa->spa_async_cv);
+         cv_destroy(&spa->spa_evicting_os_cv);
          cv_destroy(&spa->spa_proc_cv);
          cv_destroy(&spa->spa_scrub_io_cv);
          cv_destroy(&spa->spa_suspend_cv);
  
          mutex_destroy(&spa->spa_async_lock);
          mutex_destroy(&spa->spa_errlist_lock);
          mutex_destroy(&spa->spa_errlog_lock);
+         mutex_destroy(&spa->spa_evicting_os_lock);
          mutex_destroy(&spa->spa_history_lock);
          mutex_destroy(&spa->spa_proc_lock);
          mutex_destroy(&spa->spa_props_lock);
          mutex_destroy(&spa->spa_scrub_lock);
          mutex_destroy(&spa->spa_suspend_lock);
*** 772,781 ****
--- 778,801 ----
              MUTEX_HELD(&spa_namespace_lock));
          (void) refcount_remove(&spa->spa_refcount, tag);
  }
  
  /*
+  * Remove a reference to the given spa_t held by a dsl dir that is
+  * being asynchronously released.  Async releases occur from a taskq
+  * performing eviction of dsl datasets and dirs.  The namespace lock
+  * isn't held and the hold by the object being evicted may contribute to
+  * spa_minref (e.g. dataset or directory released during pool export),
+  * so the asserts in spa_close() do not apply.
+  */
+ void
+ spa_async_close(spa_t *spa, void *tag)
+ {
+         (void) refcount_remove(&spa->spa_refcount, tag);
+ }
+ 
+ /*
   * Check to see if the spa refcount is zero.  Must be called with
   * spa_namespace_lock held.  We really compare against spa_minref, which is the
   * number of references acquired when opening a pool
   */
  boolean_t
*** 1667,1676 ****
--- 1687,1724 ----
  spa_log_class(spa_t *spa)
  {
          return (spa->spa_log_class);
  }
  
+ void
+ spa_evicting_os_register(spa_t *spa, objset_t *os)
+ {
+         mutex_enter(&spa->spa_evicting_os_lock);
+         list_insert_head(&spa->spa_evicting_os_list, os);
+         mutex_exit(&spa->spa_evicting_os_lock);
+ }
+ 
+ void
+ spa_evicting_os_deregister(spa_t *spa, objset_t *os)
+ {
+         mutex_enter(&spa->spa_evicting_os_lock);
+         list_remove(&spa->spa_evicting_os_list, os);
+         cv_broadcast(&spa->spa_evicting_os_cv);
+         mutex_exit(&spa->spa_evicting_os_lock);
+ }
+ 
+ void
+ spa_evicting_os_wait(spa_t *spa)
+ {
+         mutex_enter(&spa->spa_evicting_os_lock);
+         while (!list_is_empty(&spa->spa_evicting_os_list))
+                 cv_wait(&spa->spa_evicting_os_cv, &spa->spa_evicting_os_lock);
+         mutex_exit(&spa->spa_evicting_os_lock);
+ 
+         dmu_buf_user_evict_wait();
+ }
+ 
  int
  spa_max_replication(spa_t *spa)
  {
          /*
           * As of SPA_VERSION == SPA_VERSION_DITTO_BLOCKS, we are able to