--- old/usr/src/uts/common/io/overlay/overlay.c	Mon Oct  5 11:06:56 2020
+++ new/usr/src/uts/common/io/overlay/overlay.c	Mon Oct  5 11:06:56 2020
@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright 2016 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 /*
@@ -1054,6 +1054,7 @@
 	struct msghdr hdr;
 
 	mutex_enter(&odd->odd_lock);
+	ASSERT0(odd->odd_flags & OVERLAY_F_TXSTOPPED);
 	if ((odd->odd_flags & OVERLAY_F_MDDROP) ||
 	    !(odd->odd_flags & OVERLAY_F_IN_MUX)) {
 		mutex_exit(&odd->odd_lock);
@@ -1071,6 +1072,10 @@
 	while (mp != NULL) {
 		socklen_t slen;
 		struct sockaddr_storage storage;
+#ifdef OVERLAY_FC_TEST
+		/* Can deal with it being NULL later... */
+		mblk_t *save_mp = msgpullup(mp, -1);
+#endif
 
 		mp_chain = mp->b_next;
 		mp->b_next = NULL;
@@ -1082,6 +1087,9 @@
 			if (ret == OVERLAY_TARGET_DROP)
 				freemsg(mp);
 			mp = mp_chain;
+#ifdef OVERLAY_FC_TEST
+			freemsg(save_mp);	/* Handles NULL and non-NULL */
+#endif
 			continue;
 		}
 
@@ -1091,6 +1099,9 @@
 		ret = odd->odd_plugin->ovp_ops->ovpo_encap(odd->odd_mh, mp,
 		    &einfo, &ep);
 		if (ret != 0 || ep == NULL) {
+#ifdef OVERLAY_FC_TEST
+			freemsg(save_mp);	/* Handles NULL and non-NULL */
+#endif
 			freemsg(mp);
 			goto out;
 		}
@@ -1097,8 +1108,39 @@
 
 		ASSERT(ep->b_cont == mp || ep == mp);
 		ret = overlay_mux_tx(odd->odd_mux, &hdr, ep);
-		if (ret != 0)
+		if (ret != 0) {
+			if (ret != EWOULDBLOCK) {
+				/*
+				 * Get rid of the packets, something ELSE is
+				 * wrong with the socket, and we really should
+				 * just drop the packets for now.
+				 */
+#ifdef OVERLAY_FC_TEST
+				freemsg(save_mp);
+				save_mp = NULL;
+#endif
+				freemsgchain(mp_chain);
+				mp_chain = NULL;
+			}
+#ifdef OVERLAY_FC_TEST
+			if (save_mp != NULL) {
+				/*
+				 * Return the dropped mp here to see how
+				 * upper-layer MAC reacts to it.
+				 */
+				save_mp->b_next = mp_chain;
+				mp_chain = save_mp;
+			}
+#endif
+			/*
+			 * EWOULDBLOCK is a special case.  Return the rest of
+			 * the mp_chain to MAC and have this instance be
+			 * marked as unable to transmit.  Re-enable this
+			 * instance when the mux's socket is able to send data
+			 * again ("cansend" callback).
+			 */
 			goto out;
+		}
 
 		mp = mp_chain;
 	}
@@ -1106,6 +1148,10 @@
 out:
 	mutex_enter(&odd->odd_lock);
 	overlay_io_done(odd, OVERLAY_F_IN_TX);
+	if (mp_chain != NULL) {
+		/* Note that we're returning an unsent chain to MAC. */
+		odd->odd_flags |= OVERLAY_F_TXSTOPPED;
+	}
 	mutex_exit(&odd->odd_lock);
 	return (mp_chain);
 }
--- old/usr/src/uts/common/io/overlay/overlay_mux.c	Mon Oct  5 11:06:56 2020
+++ new/usr/src/uts/common/io/overlay/overlay_mux.c	Mon Oct  5 11:06:56 2020
@@ -207,6 +207,67 @@
 }
 
 /*
+ * Kernel socket callback to indicate the socket itself is able to send
+ * data again.  Check for devices on this mux that were send-blocked,
+ * and clear them.
+ */
+/* ARGSUSED */
+static void
+overlay_mux_cansend_now(ksocket_t ksock, ksocket_callback_event_t event,
+    void *arg, uintptr_t ignore_me)
+{
+	overlay_mux_t *mux = (overlay_mux_t *)arg;
+	overlay_dev_t *odd;
+	mac_handle_t *mhs_to_update, *current_mh;
+	size_t allocsize;
+
+	ASSERT3P(ksock, ==, mux->omux_ksock);
+	ASSERT3U(event, ==, KSOCKET_EV_CANSEND);
+
+	/* Traverse omux_devices and check for ones marked as send-blocked. */
+	mutex_enter(&mux->omux_lock);
+	if (mux->omux_count == 0) {
+		/* Nothing to wake up. */
+		mutex_exit(&mux->omux_lock);
+		return;
+	}
+	allocsize = sizeof (mac_handle_t) * mux->omux_count;
+	mhs_to_update = kmem_zalloc(allocsize, KM_NOSLEEP);
+	VERIFY(mhs_to_update != NULL);	/* Failure should be rare. */
+	current_mh = mhs_to_update;
+
+	for (odd = avl_first(&mux->omux_devices); odd != NULL;
+	    odd = AVL_NEXT(&mux->omux_devices, odd)) {
+		mac_handle_t odd_mh = NULL;
+
+		mutex_enter(&odd->odd_lock);
+		if ((odd->odd_flags & OVERLAY_F_TXSTOPPED) != 0) {
+			/* Get ready to tell MAC it can transmit again. */
+			odd->odd_flags &= ~OVERLAY_F_TXSTOPPED;
+			odd_mh = odd->odd_mh;
+		}
+		mutex_exit(&odd->odd_lock);
+		if (odd_mh != NULL) {
+			*current_mh = odd_mh;
+			current_mh++;
+		}
+	}
+	mutex_exit(&mux->omux_lock);
+
+	/*
+	 * Yes, I'm using the value-then-decrement.  "current_mh" is
+	 * guaranteed to be at least one ahead of mhs_to_update if there are
+	 * any mac handles that need updating.  I also have to do this outside
+	 * the omux lock because the tx_update may trigger immediate or
+	 * concurrent packet transmission.
+	 */
+	while (current_mh-- != mhs_to_update)
+		mac_tx_update(*current_mh);
+
+	kmem_free(mhs_to_update, allocsize);
+}
+
+/*
  * Register a given device with a socket backend. If no such device socket
  * exists, create a new one.
  */
@@ -217,6 +278,7 @@
 	int err;
 	overlay_mux_t *mux;
 	ksocket_t ksock;
+	ksocket_callbacks_t ks_cb = { 0 };
 
 	if (errp == NULL)
 		errp = &err;
@@ -290,7 +352,36 @@
 	    sizeof (overlay_dev_t), offsetof(overlay_dev_t, odd_muxnode));
 	mutex_init(&mux->omux_lock, NULL, MUTEX_DRIVER, NULL);
 
+#if defined(OVERLAY_PINCH) || defined(OVERLAY_FC_TEST)
+	/* Set the xmit buf to a REALLY SMALL value, say 12k (1-3 packets) */
+	int bufsize = 12 * 1024;
 
+	if (ksocket_setsockopt(ksock, SOL_SOCKET, SO_SNDBUF,
+		(const void *)&bufsize, sizeof (bufsize), CRED()) != 0) {
+		ksocket_close(ksock, kcred);
+		mutex_destroy(&mux->omux_lock);
+		avl_destroy(&mux->omux_devices);
+		kmem_free(mux->omux_addr, len);
+		kmem_free(mux, sizeof (overlay_mux_t));
+		return (NULL);
+	}
+#endif
+	/*
+	 * Set a callback in case we hit socket flow control and need to know
+	 * when it's ready to send again.  See the aforementioned
+	 * ksocket_socket() comments about the use of kcred vs. being
+	 * zone-aware.
+	 */
+	ks_cb.ksock_cb_cansend = overlay_mux_cansend_now;
+	if (ksocket_setcallbacks(ksock, &ks_cb, mux, kcred) != 0) {
+		ksocket_close(ksock, kcred);
+		mutex_destroy(&mux->omux_lock);
+		avl_destroy(&mux->omux_devices);
+		kmem_free(mux->omux_addr, len);
+		kmem_free(mux, sizeof (overlay_mux_t));
+		return (NULL);
+	}
+
 	/* Once this is called, we need to expect to rx data */
 	*errp = ksocket_krecv_set(ksock, overlay_mux_recv, mux);
 	if (*errp != 0) {
@@ -354,8 +445,16 @@
 	/*
 	 * It'd be nice to be able to use MSG_MBLK_QUICKRELE, unfortunately,
 	 * that isn't actually supported by UDP at this time.
+	 *
+	 * Send with MSG_DONTWAIT to indicate clogged UDP sockets upstack.
 	 */
-	ret = ksocket_sendmblk(mux->omux_ksock, hdr, 0, &mp, kcred);
+	ret = ksocket_sendmblk(mux->omux_ksock, hdr, MSG_DONTWAIT, &mp, kcred);
+	/*
+	 * NOTE: ksocket_sendmblk() may send partial packets downstack,
+	 * returning what's not sent in &mp (i.e. mp pre-call might be a
+	 * b_cont of mp post-call).  We can't hold up this message (it's a
+	 * datagram), so we drop, and let the caller cope.
+	 */
 	if (ret != 0)
 		freemsg(mp);
 
--- old/usr/src/uts/common/sys/overlay_impl.h	Mon Oct  5 11:06:56 2020
+++ new/usr/src/uts/common/sys/overlay_impl.h	Mon Oct  5 11:06:56 2020
@@ -97,7 +97,8 @@
 	OVERLAY_F_STOPMASK	= 0x1e,	/* None set when stopping */
 	OVERLAY_F_VARPD		= 0x20,	/* varpd plugin exists */
 	OVERLAY_F_DEGRADED	= 0x40,	/* device is degraded */
-	OVERLAY_F_MASK		= 0x7f	/* mask of everything */
+	OVERLAY_F_TXSTOPPED	= 0x80,	/* device needs max_tx_update */
+	OVERLAY_F_MASK		= 0xff	/* mask of everything */
 } overlay_dev_flag_t;
 
 typedef struct overlay_dev {
