6274 MAC tries to use aggr rings from downed links
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

*** 18,27 ****
--- 18,28 ----
   *
   * CDDL HEADER END
   */
  /*
   * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+  * Copyright 2015 Joyent, Inc.
   */
  
  /*
   * IEEE 802.3ad Link Aggregation -- Link Aggregation Groups.
   *
*** 570,579 ****
--- 571,612 ----
  
          return (0);
  }
  
  /*
+  * This is called in response to either our LACP state machine or a MAC
+  * notification that the link has gone down via aggr_send_port_disable(). At
+  * this point, we may need to update our default ring. To that end, we go
+  * through the set of ports (underlying datalinks in an aggregation) that are
+  * currently enabled to transmit data. If all our links have been disabled for
+  * transmit, then we don't do anything.
+  *
+  * Note, because we only have a single TX group, we don't have to worry about
+  * the rings moving between groups and the chance that mac will reassign it
+  * unless someone removes a port, at which point, we play it safe and call this
+  * again.
+  */
+ void
+ aggr_grp_update_default(aggr_grp_t *grp)
+ {
+         aggr_port_t *port;
+         ASSERT(MAC_PERIM_HELD(grp->lg_mh));
+ 
+         rw_enter(&grp->lg_tx_lock, RW_WRITER);
+ 
+         if (grp->lg_ntx_ports == 0) {
+                 rw_exit(&grp->lg_tx_lock);
+                 return;
+         }
+ 
+         port = grp->lg_tx_ports[0];
+         ASSERT(port->lp_tx_ring_cnt > 0);
+         mac_hwring_set_default(grp->lg_mh, port->lp_pseudo_tx_rings[0]);
+         rw_exit(&grp->lg_tx_lock);
+ }
+ 
+ /*
   * Add a pseudo RX ring for the given HW ring handle.
   */
  static int
  aggr_add_pseudo_rx_ring(aggr_port_t *port,
      aggr_pseudo_rx_group_t *rx_grp, mac_ring_handle_t hw_rh)
*** 815,824 ****
--- 848,858 ----
                  if (hw_rh != NULL) {
                          mac_hwring_setup(hw_rh, (mac_resource_handle_t)ring,
                              mac_find_ring(tx_grp->atg_gh, i));
                  }
          }
+ 
          return (err);
  }
  
  /*
   * Remove the pseudo TX ring of the given HW ring handle.
*** 917,926 ****
--- 951,961 ----
                  port->lp_tx_grp_added = B_TRUE;
                  port->lp_tx_notify_mh = mac_client_tx_notify(port->lp_mch,
                      aggr_tx_ring_update, port);
          }
          mac_perim_exit(pmph);
+         aggr_grp_update_default(grp);
          return (err);
  }
  
  /*
   * This function is called by aggr to remove pseudo TX rings over the
*** 950,959 ****
--- 985,995 ----
              (sizeof (mac_ring_handle_t *) * port->lp_tx_ring_cnt));
  
          port->lp_tx_ring_cnt = 0;
          (void) mac_client_tx_notify(port->lp_mch, NULL, port->lp_tx_notify_mh);
          port->lp_tx_grp_added = B_FALSE;
+         aggr_grp_update_default(grp);
  done:
          mac_perim_exit(pmph);
  }
  
  static int