1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * IEEE 802.3ad Link Aggregation - Send code.
28 *
29 * Implements the Distributor function.
30 */
31
32 #include <sys/conf.h>
33 #include <sys/modctl.h>
34 #include <sys/sunddi.h>
35 #include <sys/callb.h>
36 #include <sys/vlan.h>
37 #include <sys/strsun.h>
38 #include <sys/strsubr.h>
39 #include <sys/dlpi.h>
40
41 #include <inet/common.h>
42 #include <inet/led.h>
43 #include <inet/ip.h>
44 #include <inet/ip6.h>
45 #include <inet/tcp.h>
46 #include <netinet/udp.h>
47
48 #include <sys/aggr.h>
49 #include <sys/aggr_impl.h>
50
51 /*
52 * Update the TX load balancing policy of the specified group.
53 */
54 void
55 aggr_send_update_policy(aggr_grp_t *grp, uint32_t policy)
56 {
57 uint8_t mac_policy = 0;
58
59 ASSERT(MAC_PERIM_HELD(grp->lg_mh));
60
61 if ((policy & AGGR_POLICY_L2) != 0)
62 mac_policy |= MAC_PKT_HASH_L2;
63 if ((policy & AGGR_POLICY_L3) != 0)
64 mac_policy |= MAC_PKT_HASH_L3;
65 if ((policy & AGGR_POLICY_L4) != 0)
66 mac_policy |= MAC_PKT_HASH_L4;
67
68 grp->lg_tx_policy = policy;
69 grp->lg_mac_tx_policy = mac_policy;
70 }
71
72 #define HASH_HINT(hint) \
73 ((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8))
74
75 /*
76 * Function invoked by mac layer to find a specific TX ring on a port
77 * to send data.
78 */
79 mblk_t *
80 aggr_find_tx_ring(void *arg, mblk_t *mp, uintptr_t hint, mac_ring_handle_t *rh)
81 {
82 aggr_grp_t *grp = arg;
83 aggr_port_t *port;
84 uint64_t hash;
85
86 rw_enter(&grp->lg_tx_lock, RW_READER);
87 if (grp->lg_ntx_ports == 0) {
88 /*
89 * We could have returned from aggr_m_start() before
90 * the ports were actually attached. Drop the chain.
91 */
92 rw_exit(&grp->lg_tx_lock);
93 freemsgchain(mp);
94 return (NULL);
95 }
96 hash = mac_pkt_hash(DL_ETHER, mp, grp->lg_mac_tx_policy, B_TRUE);
97 port = grp->lg_tx_ports[hash % grp->lg_ntx_ports];
98
99 /*
100 * Use hash as the hint so to direct traffic to
101 * different TX rings. Note below bit operation
102 * is needed in case hint is 0 to get the most
103 * benefit from HASH_HINT() algorithm.
104 */
105 if (port->lp_tx_ring_cnt > 1) {
106 if (hint == 0) {
107 hash = (hash << 24 | hash << 16 | hash);
108 hash = (hash << 32 | hash);
109 } else {
110 hash = hint;
111 }
112 hash = HASH_HINT(hash);
113 *rh = port->lp_pseudo_tx_rings[hash % port->lp_tx_ring_cnt];
114 } else {
115 *rh = port->lp_pseudo_tx_rings[0];
116 }
117 rw_exit(&grp->lg_tx_lock);
118
119 return (mp);
120 }
121
122 /*
123 * aggr_tx_notify_thread:
124 *
125 * aggr_tx_ring_update() callback function wakes up this thread when
126 * it gets called. This thread will call mac_tx_ring_update() to
127 * notify upper mac of flow control getting relieved. Note that
128 * aggr_tx_ring_update() cannot call mac_tx_ring_update() directly
129 * because aggr_tx_ring_update() is called from lower mac with
130 * mi_rw_lock held.
131 */
132 void
133 aggr_tx_notify_thread(void *arg)
134 {
135 callb_cpr_t cprinfo;
136 aggr_grp_t *grp = (aggr_grp_t *)arg;
137 mac_ring_handle_t pseudo_mrh;
138
139 CALLB_CPR_INIT(&cprinfo, &grp->lg_tx_flowctl_lock, callb_generic_cpr,
140 "aggr_tx_notify_thread");
141
142 mutex_enter(&grp->lg_tx_flowctl_lock);
143 while (!grp->lg_tx_notify_done) {
144 if ((grp->lg_tx_blocked_cnt) == 0) {
145 CALLB_CPR_SAFE_BEGIN(&cprinfo);
146 cv_wait(&grp->lg_tx_flowctl_cv,
147 &grp->lg_tx_flowctl_lock);
148 CALLB_CPR_SAFE_END(&cprinfo, &grp->lg_tx_flowctl_lock);
149 continue;
150 }
151 while (grp->lg_tx_blocked_cnt != 0) {
152 grp->lg_tx_blocked_cnt--;
153 pseudo_mrh =
154 grp->lg_tx_blocked_rings[grp->lg_tx_blocked_cnt];
155 mutex_exit(&grp->lg_tx_flowctl_lock);
156 mac_tx_ring_update(grp->lg_mh, pseudo_mrh);
157 mutex_enter(&grp->lg_tx_flowctl_lock);
158 }
159 }
160 /*
161 * The grp is being destroyed, exit the thread.
162 */
163 grp->lg_tx_notify_thread = NULL;
164 CALLB_CPR_EXIT(&cprinfo);
165 thread_exit();
166 }
167
168 /*
169 * Callback function registered with lower mac to receive wakeups from
170 * drivers when flow control is relieved (i.e. Tx descriptors are
171 * available).
172 */
173 void
174 aggr_tx_ring_update(void *arg1, uintptr_t arg2)
175 {
176 aggr_port_t *port = (aggr_port_t *)arg1;
177 mac_ring_handle_t mrh = (mac_ring_handle_t)arg2;
178 mac_ring_handle_t pseudo_mrh;
179 aggr_grp_t *grp = port->lp_grp;
180 int i = 0;
181
182 if (mrh == NULL) {
183 /*
184 * If the underlying NIC does not expose TX rings,
185 * still as pseudo TX ring is presented to the
186 * aggr mac.
187 */
188 pseudo_mrh = port->lp_pseudo_tx_rings[0];
189 } else {
190 for (i = 0; i < port->lp_tx_ring_cnt; i++) {
191 if (port->lp_tx_rings[i] == mrh)
192 break;
193 }
194 ASSERT(i < port->lp_tx_ring_cnt);
195 pseudo_mrh = port->lp_pseudo_tx_rings[i];
196 }
197 mutex_enter(&grp->lg_tx_flowctl_lock);
198 /*
199 * It could be possible that some (broken?) device driver
200 * could send more than one wakeup on the same ring. In
201 * such a case, multiple instances of the same pseudo TX
202 * ring should not be saved in lg_tx_blocked_rings[]
203 * array. So first check if woken up ring (pseudo_mrh) is
204 * already in the lg_tx_blocked_rings[] array.
205 */
206 for (i = 0; i < grp->lg_tx_blocked_cnt; i++) {
207 if (grp->lg_tx_blocked_rings[i] == pseudo_mrh) {
208 mutex_exit(&grp->lg_tx_flowctl_lock);
209 return;
210 }
211 }
212 /* A distinct mac_ring_handle. Save and increment count */
213 grp->lg_tx_blocked_rings[grp->lg_tx_blocked_cnt] = pseudo_mrh;
214 grp->lg_tx_blocked_cnt++;
215 cv_signal(&grp->lg_tx_flowctl_cv);
216 mutex_exit(&grp->lg_tx_flowctl_lock);
217 }
218
219 /*
220 * Send function invoked by the MAC service module.
221 */
222 mblk_t *
223 aggr_ring_tx(void *arg, mblk_t *mp)
224 {
225 aggr_pseudo_tx_ring_t *pseudo_ring = (aggr_pseudo_tx_ring_t *)arg;
226 aggr_port_t *port = pseudo_ring->atr_port;
227
228 return (mac_hwring_send_priv(port->lp_mch, pseudo_ring->atr_hw_rh, mp));
229 }
230
231 /*
232 * Enable sending on the specified port.
233 */
234 void
235 aggr_send_port_enable(aggr_port_t *port)
236 {
237 aggr_grp_t *grp = port->lp_grp;
238
239 ASSERT(MAC_PERIM_HELD(grp->lg_mh));
240
241 if (port->lp_tx_enabled || (port->lp_state !=
242 AGGR_PORT_STATE_ATTACHED)) {
243 /* already enabled or port not yet attached */
244 return;
245 }
246
247 /*
248 * Add to group's array of tx ports.
249 */
250 rw_enter(&grp->lg_tx_lock, RW_WRITER);
251 if (grp->lg_tx_ports_size < grp->lg_ntx_ports+1) {
252 /* current array too small */
253 aggr_port_t **new_ports;
254 uint_t new_size;
255
256 new_size = grp->lg_ntx_ports+1;
257 new_ports = kmem_zalloc(new_size * sizeof (aggr_port_t *),
258 KM_SLEEP);
259
260 if (grp->lg_tx_ports_size > 0) {
261 ASSERT(grp->lg_tx_ports != NULL);
262 bcopy(grp->lg_tx_ports, new_ports,
263 grp->lg_ntx_ports * sizeof (aggr_port_t *));
264 kmem_free(grp->lg_tx_ports,
265 grp->lg_tx_ports_size * sizeof (aggr_port_t *));
266 }
267
268 grp->lg_tx_ports = new_ports;
269 grp->lg_tx_ports_size = new_size;
270 }
271
272 grp->lg_tx_ports[grp->lg_ntx_ports++] = port;
273 port->lp_tx_idx = grp->lg_ntx_ports-1;
274 rw_exit(&grp->lg_tx_lock);
275
276 port->lp_tx_enabled = B_TRUE;
277
278 aggr_grp_update_default(grp);
279 }
280
281 /*
282 * Disable sending from the specified port.
283 */
284 void
285 aggr_send_port_disable(aggr_port_t *port)
286 {
287 uint_t idx, ntx;
288 aggr_grp_t *grp = port->lp_grp;
289
290 ASSERT(MAC_PERIM_HELD(grp->lg_mh));
291 ASSERT(MAC_PERIM_HELD(port->lp_mh));
292
293 if (!port->lp_tx_enabled) {
294 /* not yet enabled */
295 return;
296 }
297
298 rw_enter(&grp->lg_tx_lock, RW_WRITER);
299 idx = port->lp_tx_idx;
300 ntx = grp->lg_ntx_ports;
301 ASSERT(idx < ntx);
302
303 /* remove from array of attached ports */
304 if (idx == (ntx - 1)) {
305 grp->lg_tx_ports[idx] = NULL;
306 } else {
307 /* not the last entry, replace with last one */
308 aggr_port_t *victim;
309
310 victim = grp->lg_tx_ports[ntx - 1];
311 grp->lg_tx_ports[ntx - 1] = NULL;
312 victim->lp_tx_idx = idx;
313 grp->lg_tx_ports[idx] = victim;
314 }
315
316 port->lp_tx_idx = 0;
317 grp->lg_ntx_ports--;
318 rw_exit(&grp->lg_tx_lock);
319
320 port->lp_tx_enabled = B_FALSE;
321
322 aggr_grp_update_default(grp);
323 }