6274 MAC tries to use aggr rings from downed links
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

@@ -18,10 +18,11 @@
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 Joyent, Inc.
  */
 
 /*
  * IEEE 802.3ad Link Aggregation -- Link Aggregation Groups.
  *

@@ -570,10 +571,42 @@
 
         return (0);
 }
 
 /*
+ * This is called in response to either our LACP state machine or a MAC
+ * notification that the link has gone down via aggr_send_port_disable(). At
+ * this point, we may need to update our default ring. To that end, we go
+ * through the set of ports (underlying datalinks in an aggregation) that are
+ * currently enabled to transmit data. If all our links have been disabled for
+ * transmit, then we don't do anything.
+ *
+ * Note, because we only have a single TX group, we don't have to worry about
+ * the rings moving between groups and the chance that mac will reassign it
+ * unless someone removes a port, at which point, we play it safe and call this
+ * again.
+ */
+void
+aggr_grp_update_default(aggr_grp_t *grp)
+{
+        aggr_port_t *port;
+        ASSERT(MAC_PERIM_HELD(grp->lg_mh));
+
+        rw_enter(&grp->lg_tx_lock, RW_WRITER);
+
+        if (grp->lg_ntx_ports == 0) {
+                rw_exit(&grp->lg_tx_lock);
+                return;
+        }
+
+        port = grp->lg_tx_ports[0];
+        ASSERT(port->lp_tx_ring_cnt > 0);
+        mac_hwring_set_default(grp->lg_mh, port->lp_pseudo_tx_rings[0]);
+        rw_exit(&grp->lg_tx_lock);
+}
+
+/*
  * Add a pseudo RX ring for the given HW ring handle.
  */
 static int
 aggr_add_pseudo_rx_ring(aggr_port_t *port,
     aggr_pseudo_rx_group_t *rx_grp, mac_ring_handle_t hw_rh)

@@ -815,10 +848,11 @@
                 if (hw_rh != NULL) {
                         mac_hwring_setup(hw_rh, (mac_resource_handle_t)ring,
                             mac_find_ring(tx_grp->atg_gh, i));
                 }
         }
+
         return (err);
 }
 
 /*
  * Remove the pseudo TX ring of the given HW ring handle.

@@ -917,10 +951,11 @@
                 port->lp_tx_grp_added = B_TRUE;
                 port->lp_tx_notify_mh = mac_client_tx_notify(port->lp_mch,
                     aggr_tx_ring_update, port);
         }
         mac_perim_exit(pmph);
+        aggr_grp_update_default(grp);
         return (err);
 }
 
 /*
  * This function is called by aggr to remove pseudo TX rings over the

@@ -950,10 +985,11 @@
             (sizeof (mac_ring_handle_t *) * port->lp_tx_ring_cnt));
 
         port->lp_tx_ring_cnt = 0;
         (void) mac_client_tx_notify(port->lp_mch, NULL, port->lp_tx_notify_mh);
         port->lp_tx_grp_added = B_FALSE;
+        aggr_grp_update_default(grp);
 done:
         mac_perim_exit(pmph);
 }
 
 static int