OS-7088 cyclics corked on overlay socket with full queue

@@ -8,11 +8,11 @@
  * source.  A copy of the CDDL is also available via the Internet at
  * http://www.illumos.org/license/CDDL.
  */
 
 /*
- * Copyright 2016 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 /*
  * Overlay Devices
  *

@@ -1052,10 +1052,11 @@
         int ret;
         ovep_encap_info_t einfo;
         struct msghdr hdr;
 
         mutex_enter(&odd->odd_lock);
+        ASSERT0(odd->odd_flags & OVERLAY_F_TXSTOPPED);
         if ((odd->odd_flags & OVERLAY_F_MDDROP) ||
             !(odd->odd_flags & OVERLAY_F_IN_MUX)) {
                 mutex_exit(&odd->odd_lock);
                 freemsgchain(mp_chain);
                 return (NULL);

@@ -1069,10 +1070,14 @@
         einfo.ovdi_id = odd->odd_vid;
         mp = mp_chain;
         while (mp != NULL) {
                 socklen_t slen;
                 struct sockaddr_storage storage;
+#ifdef OVERLAY_FC_TEST
+                /* Can deal with it being NULL later... */
+                mblk_t *save_mp = msgpullup(mp, -1);
+#endif
 
                 mp_chain = mp->b_next;
                 mp->b_next = NULL;
                 ep = NULL;
 

@@ -1080,34 +1085,75 @@
                     (struct sockaddr *)&storage, &slen);
                 if (ret != OVERLAY_TARGET_OK) {
                         if (ret == OVERLAY_TARGET_DROP)
                                 freemsg(mp);
                         mp = mp_chain;
+#ifdef OVERLAY_FC_TEST
+                        freemsg(save_mp);       /* Handles NULL and non-NULL */
+#endif
                         continue;
                 }
 
                 hdr.msg_name = &storage;
                 hdr.msg_namelen = slen;
 
                 ret = odd->odd_plugin->ovp_ops->ovpo_encap(odd->odd_mh, mp,
                     &einfo, &ep);
                 if (ret != 0 || ep == NULL) {
+#ifdef OVERLAY_FC_TEST
+                        freemsg(save_mp);       /* Handles NULL and non-NULL */
+#endif
                         freemsg(mp);
                         goto out;
                 }
 
                 ASSERT(ep->b_cont == mp || ep == mp);
                 ret = overlay_mux_tx(odd->odd_mux, &hdr, ep);
-                if (ret != 0)
+                if (ret != 0) {
+                        if (ret != EWOULDBLOCK) {
+                                /*
+                                 * Get rid of the packets, something ELSE is
+                                 * wrong with the socket, and we really should
+                                 * just drop the packets for now.
+                                 */
+#ifdef OVERLAY_FC_TEST
+                                freemsg(save_mp);
+                                save_mp = NULL;
+#endif
+                                freemsgchain(mp_chain);
+                                mp_chain = NULL;
+                        }
+#ifdef OVERLAY_FC_TEST
+                        if (save_mp != NULL) {
+                                /*
+                                 * Return the dropped mp here to see how
+                                 * upper-layer MAC reacts to it.
+                                 */
+                                save_mp->b_next = mp_chain;
+                                mp_chain = save_mp;
+                        }
+#endif
+                        /*
+                         * EWOULDBLOCK is a special case.  Return the rest of
+                         * the mp_chain to MAC and have this instance be
+                         * marked as unable to transmit.  Re-enable this
+                         * instance when the mux's socket is able to send data
+                         * again ("cansend" callback).
+                         */
                         goto out;
+                }
 
                 mp = mp_chain;
         }
 
 out:
         mutex_enter(&odd->odd_lock);
         overlay_io_done(odd, OVERLAY_F_IN_TX);
+        if (mp_chain != NULL) {
+                /* Note that we're returning an unsent chain to MAC. */
+                odd->odd_flags |= OVERLAY_F_TXSTOPPED;
+        }
         mutex_exit(&odd->odd_lock);
         return (mp_chain);
 }
 
 /* ARGSUSED */