NEX-17446 cleanup of hot unplugged disks fails intermittently
Reviewed by: Dan Fields <dan.fields@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>

*** 16,27 ****
--- 16,29 ----
   * fields enclosed by brackets "[]" replaced with your own identifying
   * information: Portions Copyright [yyyy] [name of copyright owner]
   *
   * CDDL HEADER END
   */
+ 
  /*
   * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
+  * Copyright 2018 Nexenta Systems, Inc.
   */
  
  /*
   * miscellaneous routines for the devfs
   */
*** 401,417 ****
          vp->v_type = dmd->ddm_spec_type == S_IFCHR ? VCHR : VBLK;
          vp->v_rdev = dmd->ddm_dev;
          vn_setops(vp, vn_getops(DVTOV(ddv)));
          vn_exists(vp);
  
-         /* increment dev_ref with devi_lock held */
          ASSERT(DEVI_BUSY_OWNED(devi));
!         mutex_enter(&DEVI(devi)->devi_lock);
!         dv->dv_devi = devi;
!         DEVI(devi)->devi_ref++;         /* ndi_hold_devi(dip) */
!         mutex_exit(&DEVI(devi)->devi_lock);
  
          dv->dv_ino = dv_mkino(devi, vp->v_type, vp->v_rdev);
          dv->dv_nlink = 0;               /* updated on insert */
          dv->dv_dotdot = ddv;
          dv->dv_attrvp = NULLVP;
          dv->dv_attr = NULL;
--- 403,416 ----
          vp->v_type = dmd->ddm_spec_type == S_IFCHR ? VCHR : VBLK;
          vp->v_rdev = dmd->ddm_dev;
          vn_setops(vp, vn_getops(DVTOV(ddv)));
          vn_exists(vp);
  
          ASSERT(DEVI_BUSY_OWNED(devi));
!         ndi_hold_devi(devi);
  
+         dv->dv_devi = devi;
          dv->dv_ino = dv_mkino(devi, vp->v_type, vp->v_rdev);
          dv->dv_nlink = 0;               /* updated on insert */
          dv->dv_dotdot = ddv;
          dv->dv_attrvp = NULLVP;
          dv->dv_attr = NULL;
*** 1443,1456 ****
--- 1442,1470 ----
                  } else {
                          ASSERT((vp->v_type == VCHR) || (vp->v_type == VBLK));
                          ASSERT(dv->dv_nlink == 1);      /* no hard links */
                          mutex_enter(&vp->v_lock);
                          if (vp->v_count > 0) {
+                                 /*
+                                  * The file still has references to it.  If
+                                  * DEVI_GONE is *not* set on the devi referenced
+                                  * file is considered busy.
+                                  */
+                                 if (!DEVI_IS_GONE(dv->dv_devi)) {
                                          mutex_exit(&vp->v_lock);
                                          goto set_busy;
                                  }
+ 
+                                 /*
+                                  * Mark referenced file stale so that DR will
+                                  * succeed even if there are userland opens.
+                                  */
+                                 ASSERT(!DV_STALE(dv));
+                                 ndi_rele_devi(dv->dv_devi);
+                                 dv->dv_devi = NULL;
                          }
+                 }
  
                  /* unlink from directory */
                  dv_unlink(ddv, dv);
  
                  /* drop locks */