OS-7088 cyclics corked on overlay socket with full queue

*** 8,18 ****
   * source.  A copy of the CDDL is also available via the Internet at
   * http://www.illumos.org/license/CDDL.
   */
  
  /*
!  * Copyright 2016 Joyent, Inc.
   */
  
  /*
   * Overlay Devices
   *
--- 8,18 ----
   * source.  A copy of the CDDL is also available via the Internet at
   * http://www.illumos.org/license/CDDL.
   */
  
  /*
!  * Copyright 2018 Joyent, Inc.
   */
  
  /*
   * Overlay Devices
   *
*** 1052,1061 ****
--- 1052,1062 ----
          int ret;
          ovep_encap_info_t einfo;
          struct msghdr hdr;
  
          mutex_enter(&odd->odd_lock);
+         ASSERT0(odd->odd_flags & OVERLAY_F_TXSTOPPED);
          if ((odd->odd_flags & OVERLAY_F_MDDROP) ||
              !(odd->odd_flags & OVERLAY_F_IN_MUX)) {
                  mutex_exit(&odd->odd_lock);
                  freemsgchain(mp_chain);
                  return (NULL);
*** 1069,1078 ****
--- 1070,1083 ----
          einfo.ovdi_id = odd->odd_vid;
          mp = mp_chain;
          while (mp != NULL) {
                  socklen_t slen;
                  struct sockaddr_storage storage;
+ #ifdef OVERLAY_FC_TEST
+                 /* Can deal with it being NULL later... */
+                 mblk_t *save_mp = msgpullup(mp, -1);
+ #endif
  
                  mp_chain = mp->b_next;
                  mp->b_next = NULL;
                  ep = NULL;
  
*** 1080,1113 ****
                      (struct sockaddr *)&storage, &slen);
                  if (ret != OVERLAY_TARGET_OK) {
                          if (ret == OVERLAY_TARGET_DROP)
                                  freemsg(mp);
                          mp = mp_chain;
                          continue;
                  }
  
                  hdr.msg_name = &storage;
                  hdr.msg_namelen = slen;
  
                  ret = odd->odd_plugin->ovp_ops->ovpo_encap(odd->odd_mh, mp,
                      &einfo, &ep);
                  if (ret != 0 || ep == NULL) {
                          freemsg(mp);
                          goto out;
                  }
  
                  ASSERT(ep->b_cont == mp || ep == mp);
                  ret = overlay_mux_tx(odd->odd_mux, &hdr, ep);
!                 if (ret != 0)
                          goto out;
  
                  mp = mp_chain;
          }
  
  out:
          mutex_enter(&odd->odd_lock);
          overlay_io_done(odd, OVERLAY_F_IN_TX);
          mutex_exit(&odd->odd_lock);
          return (mp_chain);
  }
  
  /* ARGSUSED */
--- 1085,1159 ----
                      (struct sockaddr *)&storage, &slen);
                  if (ret != OVERLAY_TARGET_OK) {
                          if (ret == OVERLAY_TARGET_DROP)
                                  freemsg(mp);
                          mp = mp_chain;
+ #ifdef OVERLAY_FC_TEST
+                         freemsg(save_mp);       /* Handles NULL and non-NULL */
+ #endif
                          continue;
                  }
  
                  hdr.msg_name = &storage;
                  hdr.msg_namelen = slen;
  
                  ret = odd->odd_plugin->ovp_ops->ovpo_encap(odd->odd_mh, mp,
                      &einfo, &ep);
                  if (ret != 0 || ep == NULL) {
+ #ifdef OVERLAY_FC_TEST
+                         freemsg(save_mp);       /* Handles NULL and non-NULL */
+ #endif
                          freemsg(mp);
                          goto out;
                  }
  
                  ASSERT(ep->b_cont == mp || ep == mp);
                  ret = overlay_mux_tx(odd->odd_mux, &hdr, ep);
!                 if (ret != 0) {
!                         if (ret != EWOULDBLOCK) {
!                                 /*
!                                  * Get rid of the packets, something ELSE is
!                                  * wrong with the socket, and we really should
!                                  * just drop the packets for now.
!                                  */
! #ifdef OVERLAY_FC_TEST
!                                 freemsg(save_mp);
!                                 save_mp = NULL;
! #endif
!                                 freemsgchain(mp_chain);
!                                 mp_chain = NULL;
!                         }
! #ifdef OVERLAY_FC_TEST
!                         if (save_mp != NULL) {
!                                 /*
!                                  * Return the dropped mp here to see how
!                                  * upper-layer MAC reacts to it.
!                                  */
!                                 save_mp->b_next = mp_chain;
!                                 mp_chain = save_mp;
!                         }
! #endif
!                         /*
!                          * EWOULDBLOCK is a special case.  Return the rest of
!                          * the mp_chain to MAC and have this instance be
!                          * marked as unable to transmit.  Re-enable this
!                          * instance when the mux's socket is able to send data
!                          * again ("cansend" callback).
!                          */
                          goto out;
+                 }
  
                  mp = mp_chain;
          }
  
  out:
          mutex_enter(&odd->odd_lock);
          overlay_io_done(odd, OVERLAY_F_IN_TX);
+         if (mp_chain != NULL) {
+                 /* Note that we're returning an unsent chain to MAC. */
+                 odd->odd_flags |= OVERLAY_F_TXSTOPPED;
+         }
          mutex_exit(&odd->odd_lock);
          return (mp_chain);
  }
  
  /* ARGSUSED */