--- old/usr/src/cmd/dlmgmtd/dlmgmt_db.c	Wed Mar 15 20:39:23 2023
+++ new/usr/src/cmd/dlmgmtd/dlmgmt_db.c	Wed Mar 15 20:39:23 2023
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2016 Joyent, Inc.
+ * Copyright 2023 Oxide Computer Company
  */
 
 #include <assert.h>
@@ -551,9 +552,21 @@
 	assert((flags == DLMGMT_PERSIST) || (flags == DLMGMT_ACTIVE));
 
 	if ((req = dlmgmt_db_req_alloc(op, entryname, linkp->ll_linkid,
-	    linkp->ll_zoneid, flags, &err)) == NULL)
+	    linkp->ll_zoneid, flags, &err)) == NULL) {
 		return (err);
+	}
 
+	/*
+	 * If this is a transient link, then use the global zone cache file.
+	 * This is in order to allow recovery from a dlmgmtd failure that
+	 * leaves a zone in a 'down' state. In that state it is not possible
+	 * to read the zone's cache file (since it is always done from a sub
+	 * process running in the zone's context). As a result, datalinks would
+	 * otherwise remain stuck in the zone.
+	 */
+	if (flags == DLMGMT_ACTIVE && linkp->ll_transient)
+		req->ls_zoneid = GLOBAL_ZONEID;
+
 	/* If transient op and onloan, use the global zone cache file. */
 	if (flags == DLMGMT_ACTIVE && linkp->ll_onloan)
 		req->ls_zoneid = GLOBAL_ZONEID;
@@ -720,16 +733,15 @@
 	char			attr_name[MAXLINKATTRLEN];
 	size_t			attr_buf_len = 0;
 	void			*attr_buf = NULL;
-	boolean_t		rename;
 
 	curr = buf;
 	len = strlen(buf);
 	attr_name[0] = '\0';
 	for (i = 0; i < len; i++) {
-		rename = B_FALSE;
 		char		c = buf[i];
 		boolean_t	match = (c == '=' ||
 		    (c == ',' && !found_type) || c == ';');
+		boolean_t	rename = B_FALSE;
 
 		/*
 		 * Move to the next character if there is no match and
@@ -791,6 +803,10 @@
 					linkp->ll_zoneid = 0;
 					rename = B_TRUE;
 				}
+			} else if (strcmp(attr_name, "transient") == 0) {
+				if (read_boolean(curr, &attr_buf) == 0)
+					goto parse_fail;
+				linkp->ll_transient = *(boolean_t *)attr_buf;
 			} else {
 				attr_buf_len = translators[type].read_func(curr,
 				    &attr_buf);
@@ -840,10 +856,11 @@
 		 * reparenting it to the GZ and renaming it to avoid name
 		 * collisions.
 		 */
-		if (rename == B_TRUE) {
+		if (rename) {
 			(void) snprintf(linkp->ll_link, MAXLINKNAMELEN,
 			    "SUNWorphan%u", (uint16_t)(gethrtime() / 1000));
 		}
+
 		curr = buf + i + 1;
 	}
 
@@ -1157,7 +1174,8 @@
 			continue;
 		}
 
-		link_in_file.ll_zoneid = req->ls_zoneid;
+		assert(req->ls_zoneid == 0 ||
+		    link_in_file.ll_zoneid == req->ls_zoneid);
 		link_in_db = link_by_name(link_in_file.ll_link,
 		    link_in_file.ll_zoneid);
 		if (link_in_db != NULL) {
@@ -1256,6 +1274,7 @@
 	ptr += snprintf(ptr, BUFLEN(lim, ptr), "%s\t", linkp->ll_link);
 	if (!persist) {
 		char zname[ZONENAME_MAX];
+
 		/*
 		 * We store the linkid and the zone name in the active database
 		 * so that dlmgmtd can recover in the event that it is
@@ -1274,6 +1293,11 @@
 	u64 = linkp->ll_media;
 	ptr += write_uint64(ptr, BUFLEN(lim, ptr), "media", &u64);
 
+	if (!persist && linkp->ll_transient) {
+		boolean_t b = B_TRUE;
+		ptr += write_boolean(ptr, BUFLEN(lim, ptr), "transient", &b);
+	}
+
 	/*
 	 * The daemon does not keep any active link attribute. Only store the
 	 * attributes if this request is for persistent configuration,
--- old/usr/src/cmd/dlmgmtd/dlmgmt_door.c	Wed Mar 15 20:39:23 2023
+++ new/usr/src/cmd/dlmgmtd/dlmgmt_door.c	Wed Mar 15 20:39:23 2023
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2019 Joyent, Inc.
+ * Copyright 2023 Oxide Computer Company
  */
 
 /*
@@ -425,8 +426,7 @@
 		retvalp->lr_flags = linkp->ll_flags;
 		retvalp->lr_class = linkp->ll_class;
 		retvalp->lr_media = linkp->ll_media;
-		retvalp->lr_flags |= (linkp->ll_trans == B_TRUE) ?
-		    DLMGMT_TRANSIENT : 0;
+		retvalp->lr_flags |= linkp->ll_transient ? DLMGMT_TRANSIENT : 0;
 	}
 
 
@@ -465,6 +465,7 @@
 	retvalp->lr_flags = linkp->ll_flags;
 	retvalp->lr_class = linkp->ll_class;
 	retvalp->lr_media = linkp->ll_media;
+	retvalp->lr_flags |= linkp->ll_transient ? DLMGMT_TRANSIENT : 0;
 
 done:
 	dlmgmt_table_unlock();
@@ -508,6 +509,7 @@
 		retvalp->lr_class = linkp->ll_class;
 		retvalp->lr_media = linkp->ll_media;
 		retvalp->lr_flags = linkp->ll_flags;
+		retvalp->lr_flags |= linkp->ll_transient ? DLMGMT_TRANSIENT : 0;
 	}
 
 	dlmgmt_table_unlock();
@@ -926,7 +928,7 @@
 dlmgmt_removeconf(void *argp, void *retp, size_t *sz, zoneid_t zoneid,
     ucred_t *cred)
 {
-	dlmgmt_door_removeconf_t 	*removeconf = argp;
+	dlmgmt_door_removeconf_t	*removeconf = argp;
 	dlmgmt_removeconf_retval_t	*retvalp = retp;
 	dlmgmt_link_t			*linkp;
 	int				err;
@@ -1003,7 +1005,7 @@
 {
 	dlmgmt_door_openconf_t	*openconf = argp;
 	dlmgmt_openconf_retval_t *retvalp = retp;
-	dlmgmt_link_t 		*linkp;
+	dlmgmt_link_t		*linkp;
 	datalink_id_t		linkid = openconf->ld_linkid;
 	dlmgmt_dlconf_t		*dlconfp;
 	dlmgmt_linkattr_t	*attrp;
@@ -1072,7 +1074,7 @@
 {
 	dlmgmt_door_getconfsnapshot_t	*snapshot = argp;
 	dlmgmt_getconfsnapshot_retval_t	*retvalp = retp;
-	dlmgmt_link_t 			*linkp;
+	dlmgmt_link_t			*linkp;
 	datalink_id_t			linkid = snapshot->ld_linkid;
 	dlmgmt_linkattr_t		*attrp;
 	char				*buf;
@@ -1254,7 +1256,6 @@
 			    "zone %d: %s", linkid, oldzoneid, strerror(err));
 			goto done;
 		}
-
 		linkp->ll_onloan = B_FALSE;
 	}
 
--- old/usr/src/cmd/dlmgmtd/dlmgmt_impl.h	Wed Mar 15 20:39:23 2023
+++ new/usr/src/cmd/dlmgmtd/dlmgmt_impl.h	Wed Mar 15 20:39:23 2023
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2017 Joyent, Inc.
+ * Copyright 2023 Oxide Computer Company
  */
 
 /*
@@ -70,7 +71,7 @@
 	 * use by an NGZ. If ll_onloan is set then ll_zoneid cannot be 0.
 	 *
 	 * If ll_zoneid is set to the id of an NGZ but ll_onloan is
-	 * not set then the link was created and is owned by the NGZ.
+	 * not set then the link was created by, and is owned by, the NGZ.
 	 */
 	zoneid_t		ll_zoneid;
 	boolean_t		ll_onloan;
@@ -78,7 +79,13 @@
 	avl_node_t		ll_id_node;
 	uint32_t		ll_flags;
 	uint32_t		ll_gen;		/* generation number */
-	boolean_t		ll_trans;	/* transient link */
+	/*
+	 * A transient link is one that is created and destroyed along with the
+	 * lifetime of a zone. It is a non-persistent link that is owned by the
+	 * zone (ll_zoneid is set to the id of the NGZ). It is specifically not
+	 * on-loan from the GZ.
+	 */
+	boolean_t		ll_transient;
 } dlmgmt_link_t;
 
 /*
@@ -149,7 +156,7 @@
 void		dlmgmt_log(int, const char *, ...);
 int		dlmgmt_write_db_entry(const char *, dlmgmt_link_t *, uint32_t);
 int		dlmgmt_delete_db_entry(dlmgmt_link_t *, uint32_t);
-int 		dlmgmt_db_init(zoneid_t, char *);
+int		dlmgmt_db_init(zoneid_t, char *);
 void		dlmgmt_db_fini(zoneid_t);
 
 #ifdef  __cplusplus
--- old/usr/src/cmd/dlmgmtd/dlmgmt_main.c	Wed Mar 15 20:39:24 2023
+++ new/usr/src/cmd/dlmgmtd/dlmgmt_main.c	Wed Mar 15 20:39:24 2023
@@ -262,8 +262,9 @@
 		 * logging errors for any that failed.
 		 */
 		if (zone_getattr(zids[i], ZONE_ATTR_STATUS, &status,
-		    sizeof (status)) < 0)
+		    sizeof (status)) < 0) {
 			continue;
+		}
 		switch (status) {
 			case ZONE_IS_SHUTTING_DOWN:
 			case ZONE_IS_EMPTY:
@@ -270,7 +271,8 @@
 			case ZONE_IS_DOWN:
 			case ZONE_IS_DYING:
 			case ZONE_IS_DEAD:
-				/* FALLTHRU */
+			case ZONE_IS_INITIALIZED:
+			case ZONE_IS_UNINITIALIZED:
 				continue;
 			default:
 				break;
--- old/usr/src/cmd/dlmgmtd/dlmgmt_util.c	Wed Mar 15 20:39:24 2023
+++ new/usr/src/cmd/dlmgmtd/dlmgmt_util.c	Wed Mar 15 20:39:24 2023
@@ -355,7 +355,7 @@
 
 /*
  * Set the DLMGMT_ACTIVE flag on the link to note that it is active.
- * When a link is active and is owned by an NGZ then it is added to
+ * When a link is active and owned by an NGZ then it is added to
  * that zone's datalink list.
  */
 int
@@ -384,6 +384,7 @@
 		if (zoneid != linkp->ll_zoneid) {
 			assert(linkp->ll_zoneid == 0);
 			assert(linkp->ll_onloan == B_FALSE);
+			assert(linkp->ll_transient == 0);
 
 			/*
 			 * If dlmgmtd already has a link with this
@@ -422,11 +423,11 @@
 			 */
 			if (linkp->ll_class == DATALINK_CLASS_VNIC &&
 			    !(linkp->ll_flags & DLMGMT_PERSIST))
-				linkp->ll_trans = B_TRUE;
+				linkp->ll_transient = B_TRUE;
 		}
 	} else if (linkp->ll_zoneid != GLOBAL_ZONEID) {
 		/*
-		 * In this case the link was not found under any NGZs
+		 * In this case the link was not found under any NGZ
 		 * but according to its ll_zoneid member it is owned
 		 * by an NGZ. Add the datalink to the appropriate zone
 		 * datalink list.
@@ -490,8 +491,9 @@
 		return (ENOSPC);
 	if (flags & ~(DLMGMT_ACTIVE | DLMGMT_PERSIST | DLMGMT_TRANSIENT) ||
 	    ((flags & DLMGMT_PERSIST) && (flags & DLMGMT_TRANSIENT)) ||
-	    flags == 0)
+	    flags == 0) {
 		return (EINVAL);
+	}
 
 	if ((linkp = calloc(1, sizeof (dlmgmt_link_t))) == NULL) {
 		err = ENOMEM;
@@ -511,7 +513,7 @@
 	 * DLMGMT_ACTIVE -- it should not survive as a flag beyond
 	 * this point.
 	 */
-	linkp->ll_trans = (flags & DLMGMT_TRANSIENT) ? B_TRUE : B_FALSE;
+	linkp->ll_transient = (flags & DLMGMT_TRANSIENT) ? B_TRUE : B_FALSE;
 	flags &= ~DLMGMT_TRANSIENT;
 
 	if (avl_find(&dlmgmt_name_avl, linkp, &name_where) != NULL ||
--- old/usr/src/cmd/zoneadmd/vplat.c	Wed Mar 15 20:39:24 2023
+++ new/usr/src/cmd/zoneadmd/vplat.c	Wed Mar 15 20:39:24 2023
@@ -25,6 +25,7 @@
  * Copyright (c) 2015, 2016 by Delphix. All rights reserved.
  * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
  * Copyright 2020 RackTop Systems Inc.
+ * Copyright 2023 Oxide Computer Company
  */
 
 /*
@@ -86,6 +87,7 @@
 #include <libdllink.h>
 #include <libdlvlan.h>
 #include <libdlvnic.h>
+#include <libdlaggr.h>
 
 #include <inet/tcp.h>
 #include <arpa/inet.h>
@@ -3028,13 +3030,75 @@
 	return (0);
 }
 
+/*
+ * Retrieve the list of datalink IDs assigned to a zone.
+ *
+ * On return, *count will be updated with the total number of links and, if it
+ * is not NULL, **linksp will be updated to point to allocated memory
+ * containing the link IDs. This should be passed to free() when the caller is
+ * finished with it.
+ */
 static int
+fetch_zone_datalinks(zlog_t *zlogp, zoneid_t zoneid, int *countp,
+    datalink_id_t **linksp)
+{
+	datalink_id_t *links = NULL;
+	int links_size = 0;
+	int num_links;
+
+	if (linksp != NULL)
+		*linksp = NULL;
+	*countp = 0;
+
+	num_links = 0;
+	if (zone_list_datalink(zoneid, &num_links, NULL) != 0) {
+		zerror(zlogp, B_TRUE,
+		    "unable to determine number of network interfaces");
+		return (-1);
+	}
+
+	if (num_links == 0)
+		return (0);
+
+	/* If linkp is NULL, the caller only wants the count. */
+	if (linksp == NULL) {
+		*countp = num_links;
+		return (0);
+	}
+
+	do {
+		datalink_id_t *p;
+
+		links_size = num_links;
+		p = reallocarray(links, links_size, sizeof (datalink_id_t));
+
+		if (p == NULL) {
+			zerror(zlogp, B_TRUE,
+			    "failed to allocate memory for zone links");
+			free(links);
+			return (-1);
+		}
+		links = p;
+
+		if (zone_list_datalink(zoneid, &num_links, links) != 0) {
+			zerror(zlogp, B_TRUE, "failed to list zone links");
+			free(links);
+			return (-1);
+		}
+	} while (links_size < num_links);
+
+	*countp = num_links;
+	*linksp = links;
+
+	return (0);
+}
+
+static int
 remove_datalink_pool(zlog_t *zlogp, zoneid_t zoneid)
 {
 	ushort_t flags;
 	zone_iptype_t iptype;
-	int i, dlnum = 0;
-	datalink_id_t *dllink, *dllinks = NULL;
+	int i;
 	dladm_status_t err;
 
 	if (strlen(pool_name) == 0)
@@ -3054,34 +3118,15 @@
 	}
 
 	if (iptype == ZS_EXCLUSIVE) {
-		/*
-		 * Get the datalink count and for each datalink,
-		 * attempt to clear the pool property and clear
-		 * the pool_name.
-		 */
-		if (zone_list_datalink(zoneid, &dlnum, NULL) != 0) {
-			zerror(zlogp, B_TRUE, "unable to count network "
-			    "interfaces");
-			return (-1);
-		}
+		datalink_id_t *dllinks = NULL;
+		int dlnum = 0;
 
-		if (dlnum == 0)
-			return (0);
-
-		if ((dllinks = malloc(dlnum * sizeof (datalink_id_t)))
-		    == NULL) {
-			zerror(zlogp, B_TRUE, "memory allocation failed");
+		if (fetch_zone_datalinks(zlogp, zoneid, &dlnum, &dllinks) != 0)
 			return (-1);
-		}
-		if (zone_list_datalink(zoneid, &dlnum, dllinks) != 0) {
-			zerror(zlogp, B_TRUE, "unable to list network "
-			    "interfaces");
-			return (-1);
-		}
 
 		bzero(pool_name, sizeof (pool_name));
-		for (i = 0, dllink = dllinks; i < dlnum; i++, dllink++) {
-			err = dladm_set_linkprop(dld_handle, *dllink, "pool",
+		for (i = 0; i < dlnum; i++) {
+			err = dladm_set_linkprop(dld_handle, dllinks[i], "pool",
 			    NULL, 0, DLADM_OPT_ACTIVE);
 			if (err != DLADM_STATUS_OK) {
 				zerror(zlogp, B_TRUE,
@@ -3100,7 +3145,7 @@
 	zone_iptype_t iptype;
 	int i, dlnum = 0;
 	dladm_status_t dlstatus;
-	datalink_id_t *dllink, *dllinks = NULL;
+	datalink_id_t *dllinks = NULL;
 
 	if (zone_getattr(zoneid, ZONE_ATTR_FLAGS, &flags,
 	    sizeof (flags)) < 0) {
@@ -3119,48 +3164,35 @@
 		return (0);
 
 	/*
-	 * Get the datalink count and for each datalink,
-	 * attempt to clear the pool property and clear
-	 * the pool_name.
+	 * Get the datalink count and for each datalink, attempt to clear the
+	 * protection and allowed_ips properties.
 	 */
-	if (zone_list_datalink(zoneid, &dlnum, NULL) != 0) {
-		zerror(zlogp, B_TRUE, "unable to count network interfaces");
-		return (-1);
-	}
 
-	if (dlnum == 0)
-		return (0);
-
-	if ((dllinks = malloc(dlnum * sizeof (datalink_id_t))) == NULL) {
-		zerror(zlogp, B_TRUE, "memory allocation failed");
+	if (fetch_zone_datalinks(zlogp, zoneid, &dlnum, &dllinks) != 0)
 		return (-1);
-	}
-	if (zone_list_datalink(zoneid, &dlnum, dllinks) != 0) {
-		zerror(zlogp, B_TRUE, "unable to list network interfaces");
-		free(dllinks);
-		return (-1);
-	}
 
-	for (i = 0, dllink = dllinks; i < dlnum; i++, dllink++) {
+	for (i = 0; i < dlnum; i++) {
 		char dlerr[DLADM_STRSIZE];
 
-		dlstatus = dladm_set_linkprop(dld_handle, *dllink,
+		dlstatus = dladm_set_linkprop(dld_handle, dllinks[i],
 		    "protection", NULL, 0, DLADM_OPT_ACTIVE);
 		if (dlstatus == DLADM_STATUS_NOTFOUND) {
 			/* datalink does not belong to the GZ */
 			continue;
 		}
-		if (dlstatus != DLADM_STATUS_OK)
+		if (dlstatus != DLADM_STATUS_OK) {
 			zerror(zlogp, B_FALSE,
-			    "clear 'protection' link property: %s",
-			    dladm_status2str(dlstatus, dlerr));
+			    "clear link %d 'protection' link property: %s",
+			    dllinks[i], dladm_status2str(dlstatus, dlerr));
+		}
 
-		dlstatus = dladm_set_linkprop(dld_handle, *dllink,
+		dlstatus = dladm_set_linkprop(dld_handle, dllinks[i],
 		    "allowed-ips", NULL, 0, DLADM_OPT_ACTIVE);
-		if (dlstatus != DLADM_STATUS_OK)
+		if (dlstatus != DLADM_STATUS_OK) {
 			zerror(zlogp, B_FALSE,
-			    "clear 'allowed-ips' link property: %s",
-			    dladm_status2str(dlstatus, dlerr));
+			    "clear link %d 'allowed-ips' link property: %s",
+			    dllinks[i], dladm_status2str(dlstatus, dlerr));
+		}
 	}
 	free(dllinks);
 	return (0);
@@ -3167,6 +3199,79 @@
 }
 
 static int
+unconfigure_exclusive_network_interfaces(zlog_t *zlogp, zoneid_t zoneid)
+{
+	datalink_id_t *dllinks;
+	int dlnum = 0;
+	uint_t i;
+
+	/*
+	 * The kernel shutdown callback for the dls module should have removed
+	 * all datalinks from this zone.  If any remain, then there's a
+	 * problem.
+	 */
+
+	if (fetch_zone_datalinks(zlogp, zoneid, &dlnum, &dllinks) != 0)
+		return (-1);
+
+	if (dlnum == 0)
+		return (0);
+
+	/*
+	 * There are some datalinks left in the zone. The most likely cause of
+	 * this is that the datalink-management daemon (dlmgmtd) was not
+	 * running when the zone was shut down. That prevented the kernel from
+	 * doing the required upcall to move the links back to the GZ. To
+	 * attempt recovery, do that now.
+	 */
+
+	for (i = 0; i < dlnum; i++) {
+		char dlerr[DLADM_STRSIZE];
+		dladm_status_t status;
+		uint32_t link_flags;
+		datalink_id_t link = dllinks[i];
+		char *prop_vals[] = { GLOBAL_ZONENAME };
+
+		status = dladm_datalink_id2info(dld_handle, link,
+		    &link_flags, NULL, NULL, NULL, 0);
+
+		if (status != DLADM_STATUS_OK) {
+			zerror(zlogp, B_FALSE,
+			    "failed to get link info for %u: %s",
+			    link, dladm_status2str(status, dlerr));
+			continue;
+		}
+
+		if (link_flags & DLADM_OPT_TRANSIENT)
+			continue;
+
+		status = dladm_set_linkprop(dld_handle, link, "zone",
+		    prop_vals, 1, DLADM_OPT_ACTIVE);
+
+		if (status != DLADM_STATUS_OK) {
+			zerror(zlogp, B_FALSE,
+			    "failed to move link %u to GZ: %s",
+			    link, dladm_status2str(status, dlerr));
+		}
+	}
+
+	free(dllinks);
+
+	/* Check again and log a message if links remain */
+
+	if (fetch_zone_datalinks(zlogp, zoneid, &dlnum, NULL) != 0)
+		return (-1);
+
+	if (dlnum == 0)
+		return (0);
+
+	zerror(zlogp, B_FALSE, "%d datalink(s) remain in zone after shutdown",
+	    dlnum);
+
+	return (-1);
+}
+
+static int
 tcp_abort_conn(zlog_t *zlogp, zoneid_t zoneid,
     const struct sockaddr_storage *local, const struct sockaddr_storage *remote)
 {
@@ -5215,76 +5320,75 @@
 }
 
 /*
- * Delete all transient VNICs belonging to this zone. A transient VNIC
+ * Delete all transient links belonging to this zone. A transient link
  * is one that is created and destroyed along with the lifetime of the
- * zone. Non-transient VNICs, ones that are assigned from the GZ to a
+ * zone. Non-transient links, ones that are assigned from the GZ to a
  * NGZ, are reassigned to the GZ in zone_shutdown() via the
  * zone-specific data (zsd) callbacks.
  */
 static int
-delete_transient_vnics(zlog_t *zlogp, zoneid_t zoneid)
+delete_transient_links(zlog_t *zlogp, zoneid_t zoneid)
 {
-	dladm_status_t status;
-	int num_links = 0;
-	datalink_id_t *links, link;
-	uint32_t link_flags;
-	datalink_class_t link_class;
-	char link_name[MAXLINKNAMELEN];
+	datalink_id_t *dllinks = NULL;
+	int dlnum = 0;
+	uint_t i;
 
-	if (zone_list_datalink(zoneid, &num_links, NULL) != 0) {
-		zerror(zlogp, B_TRUE, "unable to determine "
-		    "number of network interfaces");
+	if (fetch_zone_datalinks(zlogp, zoneid, &dlnum, &dllinks) != 0)
 		return (-1);
-	}
 
-	if (num_links == 0)
+	if (dlnum == 0)
 		return (0);
 
-	links = malloc(num_links * sizeof (datalink_id_t));
-
-	if (links == NULL) {
-		zerror(zlogp, B_TRUE, "failed to delete "
-		    "network interfaces because of alloc fail");
-		return (-1);
-	}
-
-	if (zone_list_datalink(zoneid, &num_links, links) != 0) {
-		zerror(zlogp, B_TRUE, "failed to delete "
-		    "network interfaces because of failure "
-		    "to list them");
-		return (-1);
-	}
-
-	for (int i = 0; i < num_links; i++) {
+	for (i = 0; i < dlnum; i++) {
+		char link_name[MAXLINKNAMELEN];
 		char dlerr[DLADM_STRSIZE];
-		link = links[i];
+		datalink_id_t link = dllinks[i];
+		datalink_class_t link_class;
+		dladm_status_t status;
+		uint32_t link_flags;
 
 		status = dladm_datalink_id2info(dld_handle, link, &link_flags,
 		    &link_class, NULL, link_name, sizeof (link_name));
 
 		if (status != DLADM_STATUS_OK) {
-			zerror(zlogp, B_FALSE, "failed to "
-			    "delete network interface (%u)"
-			    "due to failure to get link info: %s",
-			    link,
-			    dladm_status2str(status, dlerr));
-			return (-1);
+			zerror(zlogp, B_FALSE,
+			    "failed to get link info for %u: %s",
+			    link, dladm_status2str(status, dlerr));
+			continue;
 		}
 
-		if (link_flags & DLADM_OPT_TRANSIENT) {
-			assert(link_class & DATALINK_CLASS_VNIC);
+		if (!(link_flags & DLADM_OPT_TRANSIENT))
+			continue;
+
+		switch (link_class) {
+		case DATALINK_CLASS_VNIC:
+		case DATALINK_CLASS_ETHERSTUB:
 			status = dladm_vnic_delete(dld_handle, link,
 			    DLADM_OPT_ACTIVE);
+			break;
+		case DATALINK_CLASS_VLAN:
+			status = dladm_vlan_delete(dld_handle, link,
+			    DLADM_OPT_ACTIVE);
+			break;
+		case DATALINK_CLASS_AGGR:
+			status = dladm_aggr_delete(dld_handle, link,
+			    DLADM_OPT_ACTIVE);
+			break;
+		default:
+			zerror(zlogp, B_FALSE,
+			    "unhandled class for transient link %s (%u)",
+			    link_name, link);
+			continue;
+		}
 
-			if (status != DLADM_STATUS_OK) {
-				zerror(zlogp, B_TRUE, "failed to delete link "
-				    "with id %d: %s", link,
-				    dladm_status2str(status, dlerr));
-				return (-1);
-			}
+		if (status != DLADM_STATUS_OK) {
+			zerror(zlogp, B_TRUE,
+			    "failed to delete transient link %s (%u): %s",
+			    link_name, link, dladm_status2str(status, dlerr));
 		}
 	}
 
+	free(dllinks);
 	return (0);
 }
 
@@ -5328,12 +5432,15 @@
 		goto error;
 	}
 
-	if (remove_datalink_pool(zlogp, zoneid) != 0)
-		zerror(zlogp, B_FALSE, "unable clear datalink pool property");
+	if (remove_datalink_pool(zlogp, zoneid) != 0) {
+		zerror(zlogp, B_FALSE,
+		    "unable to clear datalink pool property");
+	}
 
-	if (remove_datalink_protect(zlogp, zoneid) != 0)
+	if (remove_datalink_protect(zlogp, zoneid) != 0) {
 		zerror(zlogp, B_FALSE,
-		    "unable clear datalink protect property");
+		    "unable to clear datalink protect property");
+	}
 
 	/*
 	 * The datalinks assigned to the zone will be removed from the NGZ as
@@ -5399,11 +5506,17 @@
 			}
 			break;
 		case ZS_EXCLUSIVE:
-			if (delete_transient_vnics(zlogp, zoneid) != 0) {
+			if (delete_transient_links(zlogp, zoneid) != 0) {
 				zerror(zlogp, B_FALSE, "unable to delete "
-				    "transient vnics in zone");
+				    "transient links in zone");
 				goto error;
 			}
+			if (unconfigure_exclusive_network_interfaces(zlogp,
+			    zoneid) != 0) {
+				zerror(zlogp, B_FALSE, "unable to unconfigure "
+				    "network interfaces in zone");
+				goto error;
+			}
 
 			status = dladm_zone_halt(dld_handle, zoneid);
 			if (status != DLADM_STATUS_OK) {
--- old/usr/src/lib/libdladm/common/libdlmgmt.c	Wed Mar 15 20:39:24 2023
+++ new/usr/src/lib/libdladm/common/libdlmgmt.c	Wed Mar 15 20:39:24 2023
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2016 Joyent, Inc.
+ * Copyright 2023 Oxide Computer Company
  */
 
 #include <door.h>
@@ -123,6 +124,18 @@
 		return (DLADM_STATUS_BADARG);
 	}
 
+	if (getzoneid() != GLOBAL_ZONEID) {
+		/*
+		 * If we're creating this link in a non-global zone, then do
+		 * not allow it to be persistent, and flag it as transient so
+		 * that it will be automatically cleaned up on zone shutdown,
+		 * rather than being moved to the GZ.
+		 */
+		if (flags & DLADM_OPT_PERSIST)
+			return (DLADM_STATUS_TEMPONLY);
+		flags |= DLADM_OPT_TRANSIENT;
+	}
+
 	dlmgmt_flags = (flags & DLADM_OPT_ACTIVE) ? DLMGMT_ACTIVE : 0;
 	dlmgmt_flags |= (flags & DLADM_OPT_PERSIST) ? DLMGMT_PERSIST : 0;
 	dlmgmt_flags |= (flags & DLADM_OPT_TRANSIENT) ? DLMGMT_TRANSIENT : 0;
@@ -581,9 +594,12 @@
 	if (linkidp != NULL)
 		*linkidp = linkid;
 	if (flagp != NULL) {
-		*flagp = retval.lr_flags & DLMGMT_ACTIVE ? DLADM_OPT_ACTIVE : 0;
+		*flagp = (retval.lr_flags & DLMGMT_ACTIVE) ?
+		    DLADM_OPT_ACTIVE : 0;
 		*flagp |= (retval.lr_flags & DLMGMT_PERSIST) ?
 		    DLADM_OPT_PERSIST : 0;
+		*flagp |= (retval.lr_flags & DLMGMT_TRANSIENT) ?
+		    DLADM_OPT_TRANSIENT : 0;
 	}
 	if (classp != NULL)
 		*classp = retval.lr_class;
--- old/usr/src/uts/common/io/dls/dls_mgmt.c	Wed Mar 15 20:39:24 2023
+++ new/usr/src/uts/common/io/dls/dls_mgmt.c	Wed Mar 15 20:39:24 2023
@@ -928,6 +928,14 @@
 		ddp->dd_tref = 0;
 		ddp->dd_ref++;
 		ddp->dd_owner_zid = zoneid;
+		/*
+		 * If we are creating a new devnet which will be owned by a NGZ
+		 * then mark it as transient. This link has never been in the
+		 * GZ, the GZ will not have a hold on its reference, and we do
+		 * not want to return it to the GZ when the zone halts.
+		 */
+		if (zoneid != GLOBAL_ZONEID)
+			ddp->dd_transient = B_TRUE;
 		(void) strlcpy(ddp->dd_mac, macname, sizeof (ddp->dd_mac));
 		VERIFY(mod_hash_insert(i_dls_devnet_hash,
 		    (mod_hash_key_t)ddp->dd_mac, (mod_hash_val_t)ddp) == 0);
@@ -942,12 +950,6 @@
 		    (mod_hash_val_t)ddp) == 0);
 		devnet_need_rebuild = B_TRUE;
 		stat_create = B_TRUE;
-		mutex_enter(&ddp->dd_mutex);
-		if (!ddp->dd_prop_loaded && (ddp->dd_prop_taskid == 0)) {
-			ddp->dd_prop_taskid = taskq_dispatch(system_taskq,
-			    dls_devnet_prop_task, ddp, TQ_SLEEP);
-		}
-		mutex_exit(&ddp->dd_mutex);
 	}
 	err = 0;
 done:
@@ -959,6 +961,27 @@
 	 * and ensure that *ddp is valid.
 	 */
 	rw_exit(&i_dls_devnet_lock);
+
+	if (err == 0 && zoneid != GLOBAL_ZONEID) {
+		/*
+		 * If this link is being created directly within a non-global
+		 * zone, then flag it as transient so that it will be cleaned
+		 * up when the zone is shut down.
+		 */
+		err = i_dls_devnet_setzid(ddp, zoneid, B_FALSE, B_TRUE);
+		if (err != 0) {
+			/*
+			 * At this point the link is marked as
+			 * DD_INITIALIZING -- there can be no
+			 * outstanding temp refs and therefore no need
+			 * to wait for them.
+			 */
+			ASSERT(ddp->dd_flags & DD_INITIALIZING);
+			(void) dls_devnet_unset(mh, &linkid, B_FALSE);
+			return (err);
+		}
+	}
+
 	if (err == 0) {
 		if (zoneid != GLOBAL_ZONEID &&
 		    (err = i_dls_devnet_setzid(ddp, zoneid, B_FALSE,
@@ -986,8 +1009,8 @@
 			*ddpp = ddp;
 
 		mutex_enter(&ddp->dd_mutex);
-		if (linkid != DATALINK_INVALID_LINKID && !ddp->dd_prop_loaded &&
-		    ddp->dd_prop_taskid == TASKQID_INVALID) {
+		if (linkid != DATALINK_INVALID_LINKID &&
+		    !ddp->dd_prop_loaded && ddp->dd_prop_taskid == 0) {
 			ddp->dd_prop_taskid = taskq_dispatch(system_taskq,
 			    dls_devnet_prop_task, ddp, TQ_SLEEP);
 		}
@@ -1048,7 +1071,7 @@
 		 * zstatus indicates which situation we are dealing with:
 		 *	 0 - means return EBUSY
 		 *	 1 - means case (a), cleanup transient link
-		 *	-1 - means case (b), orphained VNIC
+		 *	-1 - means case (b), orphaned VNIC
 		 */
 		if (ddp->dd_ref > 1 && ddp->dd_zid != GLOBAL_ZONEID) {
 			zone_t	*zp;
@@ -1075,10 +1098,11 @@
 		/*
 		 * We want to delete the link, reset ref to 1;
 		 */
-		if (zstatus == -1)
+		if (zstatus == -1) {
 			/* Log a warning, but continue in this case */
 			cmn_err(CE_WARN, "clear orphaned datalink: %s\n",
 			    ddp->dd_linkname);
+		}
 		ddp->dd_ref = 1;
 	}
 
@@ -1111,7 +1135,21 @@
 	 * safely call dls_devnet_unset() outside the MAC perim.
 	 */
 	if (ddp->dd_zid != GLOBAL_ZONEID) {
+		/*
+		 * We need to release the dd_mutex before we try and destroy the
+		 * stat. When we destroy it, we'll need to grab the lock for the
+		 * kstat but if there's a concurrent reader of the kstat, we'll
+		 * be blocked on it. This will lead to deadlock because these
+		 * kstats employ a ks_update function (dls_devnet_stat_update)
+		 * which needs the dd_mutex that we currently hold.
+		 *
+		 * Because we've already flagged the dls_devnet_t as
+		 * DD_CONDEMNED and we still have a write lock on
+		 * i_dls_devnet_lock, we should be able to release the dd_mutex.
+		 */
+		mutex_exit(&ddp->dd_mutex);
 		dls_devnet_stat_destroy(ddp, ddp->dd_zid);
+		mutex_enter(&ddp->dd_mutex);
 		(void) i_dls_devnet_setzid(ddp, GLOBAL_ZONEID, B_FALSE,
 		    B_FALSE);
 	}
@@ -1128,7 +1166,7 @@
 			cv_wait(&ddp->dd_cv, &ddp->dd_mutex);
 	} else {
 		VERIFY(ddp->dd_tref == 0);
-		VERIFY(ddp->dd_prop_taskid == (taskqid_t)NULL);
+		VERIFY(ddp->dd_prop_taskid == 0);
 	}
 
 	if (ddp->dd_linkid != DATALINK_INVALID_LINKID) {
@@ -1163,8 +1201,8 @@
 	}
 
 	mutex_enter(&ddp->dd_mutex);
-	ASSERT(ddp->dd_ref > 0);
-	if (ddp->dd_flags & DD_CONDEMNED) {
+	VERIFY(ddp->dd_ref > 0);
+	if (DD_NOT_VISIBLE(ddp->dd_flags)) {
 		mutex_exit(&ddp->dd_mutex);
 		rw_exit(&i_dls_devnet_lock);
 		return (ENOENT);
@@ -1935,9 +1973,8 @@
 	 * mac_disable() can succeed. This is because of the implicit
 	 * reference that dls has on the mac_impl_t.
 	 */
-	if (err != 0 && err != ENOENT) {
+	if (err != 0 && err != ENOENT)
 		return (err);
-	}
 
 	mac_perim_enter_by_mh(mh, &mph);
 	err = dls_link_rele_by_name(mac_name(mh));
