5056 ZFS deadlock on db_mtx and dn_holds
Reviewed by: Will Andrews <willa@spectralogic.com>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

@@ -19,10 +19,11 @@
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /*
  * This file contains the top half of the zfs directory structure
  * implementation. The bottom half is in zap_leaf.c.

@@ -50,11 +51,10 @@
 
 int fzap_default_block_shift = 14; /* 16k blocksize */
 
 extern inline zap_phys_t *zap_f_phys(zap_t *zap);
 
-static void zap_leaf_pageout(dmu_buf_t *db, void *vl);
 static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks);
 
 void
 fzap_byteswap(void *vbuf, size_t size)
 {

@@ -79,11 +79,11 @@
         zap_phys_t *zp;
 
         ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
         zap->zap_ismicro = FALSE;
 
-        (void) dmu_buf_update_user(zap->zap_dbuf, zap, zap, zap_evict);
+        zap->zap_dbu.dbu_evict_func = zap_evict;
 
         mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, 0, 0);
         zap->zap_f.zap_block_shift = highbit64(zap->zap_dbuf->db_size) - 1;
 
         zp = zap_f_phys(zap);

@@ -385,15 +385,24 @@
         newblk = zap_f_phys(zap)->zap_freeblk;
         zap_f_phys(zap)->zap_freeblk += nblocks;
         return (newblk);
 }
 
+static void
+zap_leaf_pageout(void *dbu)
+{
+        zap_leaf_t *l = dbu;
+
+        rw_destroy(&l->l_rwlock);
+        kmem_free(l, sizeof (zap_leaf_t));
+}
+
 static zap_leaf_t *
 zap_create_leaf(zap_t *zap, dmu_tx_t *tx)
 {
         void *winner;
-        zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP);
+        zap_leaf_t *l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
 
         ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 
         rw_init(&l->l_rwlock, 0, 0, 0);
         rw_enter(&l->l_rwlock, RW_WRITER);

@@ -401,11 +410,12 @@
         l->l_dbuf = NULL;
 
         VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object,
             l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf,
             DMU_READ_NO_PREFETCH));
-        winner = dmu_buf_set_user(l->l_dbuf, l, zap_leaf_pageout);
+        dmu_buf_init_user(&l->l_dbu, zap_leaf_pageout, &l->l_dbuf);
+        winner = dmu_buf_set_user(l->l_dbuf, &l->l_dbu);
         ASSERT(winner == NULL);
         dmu_buf_will_dirty(l->l_dbuf, tx);
 
         zap_leaf_init(l, zap->zap_normflags != 0);
 

@@ -433,40 +443,31 @@
 {
         rw_exit(&l->l_rwlock);
         dmu_buf_rele(l->l_dbuf, NULL);
 }
 
-_NOTE(ARGSUSED(0))
-static void
-zap_leaf_pageout(dmu_buf_t *db, void *vl)
-{
-        zap_leaf_t *l = vl;
-
-        rw_destroy(&l->l_rwlock);
-        kmem_free(l, sizeof (zap_leaf_t));
-}
-
 static zap_leaf_t *
 zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 {
         zap_leaf_t *l, *winner;
 
         ASSERT(blkid != 0);
 
-        l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP);
+        l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
         rw_init(&l->l_rwlock, 0, 0, 0);
         rw_enter(&l->l_rwlock, RW_WRITER);
         l->l_blkid = blkid;
         l->l_bs = highbit64(db->db_size) - 1;
         l->l_dbuf = db;
 
-        winner = dmu_buf_set_user(db, l, zap_leaf_pageout);
+        dmu_buf_init_user(&l->l_dbu, zap_leaf_pageout, &l->l_dbuf);
+        winner = dmu_buf_set_user(db, &l->l_dbu);
 
         rw_exit(&l->l_rwlock);
         if (winner != NULL) {
                 /* someone else set it first */
-                zap_leaf_pageout(NULL, l);
+                zap_leaf_pageout(&l->l_dbu);
                 l = winner;
         }
 
         /*
          * lhr_pad was previously used for the next leaf in the leaf