Print this page
1925 stack overflow from mac code
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/mac/mac_sched.c
+++ new/usr/src/uts/common/io/mac/mac_sched.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 +/*
26 + * Copyright 2011 Joyent, Inc. All rights reserved.
27 + */
25 28
26 29 #include <sys/types.h>
27 30 #include <sys/callb.h>
28 31 #include <sys/sdt.h>
29 32 #include <sys/strsubr.h>
30 33 #include <sys/strsun.h>
31 34 #include <sys/vlan.h>
35 +#include <sys/stack.h>
36 +#include <sys/archsystm.h>
32 37 #include <inet/ipsec_impl.h>
33 38 #include <inet/ip_impl.h>
34 39 #include <inet/sadb.h>
35 40 #include <inet/ipsecesp.h>
36 41 #include <inet/ipsecah.h>
37 42 #include <inet/ip6.h>
38 43
39 44 #include <sys/mac_impl.h>
40 45 #include <sys/mac_client_impl.h>
41 46 #include <sys/mac_client_priv.h>
42 47 #include <sys/mac_soft_ring.h>
43 48 #include <sys/mac_flow_impl.h>
44 49
45 50 static mac_tx_cookie_t mac_tx_single_ring_mode(mac_soft_ring_set_t *, mblk_t *,
46 51 uintptr_t, uint16_t, mblk_t **);
47 52 static mac_tx_cookie_t mac_tx_serializer_mode(mac_soft_ring_set_t *, mblk_t *,
48 53 uintptr_t, uint16_t, mblk_t **);
49 54 static mac_tx_cookie_t mac_tx_fanout_mode(mac_soft_ring_set_t *, mblk_t *,
50 55 uintptr_t, uint16_t, mblk_t **);
51 56 static mac_tx_cookie_t mac_tx_bw_mode(mac_soft_ring_set_t *, mblk_t *,
52 57 uintptr_t, uint16_t, mblk_t **);
53 58 static mac_tx_cookie_t mac_tx_aggr_mode(mac_soft_ring_set_t *, mblk_t *,
54 59 uintptr_t, uint16_t, mblk_t **);
55 60
56 61 typedef struct mac_tx_mode_s {
57 62 mac_tx_srs_mode_t mac_tx_mode;
58 63 mac_tx_func_t mac_tx_func;
59 64 } mac_tx_mode_t;
60 65
61 66 /*
62 67 * There are seven modes of operation on the Tx side. These modes get set
63 68 * in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode,
64 69 * none of the other modes are user configurable. They get selected by
65 70 * the system depending upon whether the link (or flow) has multiple Tx
66 71 * rings or a bandwidth configured, or if the link is an aggr, etc.
67 72 *
68 73 * When the Tx SRS is operating in aggr mode (st_mode) or if there are
69 74 * multiple Tx rings owned by Tx SRS, then each Tx ring (pseudo or
70 75 * otherwise) will have a soft ring associated with it. These soft rings
71 76 * are stored in srs_tx_soft_rings[] array.
72 77 *
73 78 * Additionally in the case of aggr, there is the st_soft_rings[] array
74 79 * in the mac_srs_tx_t structure. This array is used to store the same
75 80 * set of soft rings that are present in srs_tx_soft_rings[] array but
76 81 * in a different manner. The soft ring associated with the pseudo Tx
77 82 * ring is saved at mr_index (of the pseudo ring) in st_soft_rings[]
78 83 * array. This helps in quickly getting the soft ring associated with the
79 84 * Tx ring when aggr_find_tx_ring() returns the pseudo Tx ring that is to
80 85 * be used for transmit.
81 86 */
82 87 mac_tx_mode_t mac_tx_mode_list[] = {
83 88 {SRS_TX_DEFAULT, mac_tx_single_ring_mode},
84 89 {SRS_TX_SERIALIZE, mac_tx_serializer_mode},
85 90 {SRS_TX_FANOUT, mac_tx_fanout_mode},
86 91 {SRS_TX_BW, mac_tx_bw_mode},
87 92 {SRS_TX_BW_FANOUT, mac_tx_bw_mode},
88 93 {SRS_TX_AGGR, mac_tx_aggr_mode},
89 94 {SRS_TX_BW_AGGR, mac_tx_bw_mode}
90 95 };
91 96
92 97 /*
93 98 * Soft Ring Set (SRS) - The Run time code that deals with
94 99 * dynamic polling from the hardware, bandwidth enforcement,
95 100 * fanout etc.
96 101 *
97 102 * We try to use H/W classification on NIC and assign traffic for
98 103 * a MAC address to a particular Rx ring or ring group. There is a
99 104 * 1-1 mapping between a SRS and a Rx ring. The SRS dynamically
100 105 * switches the underlying Rx ring between interrupt and
101 106 * polling mode and enforces any specified B/W control.
102 107 *
103 108 * There is always a SRS created and tied to each H/W and S/W rule.
104 109 * Whenever we create a H/W rule, we always add the the same rule to
105 110 * S/W classifier and tie a SRS to it.
106 111 *
107 112 * In case a B/W control is specified, it is broken into bytes
108 113 * per ticks and as soon as the quota for a tick is exhausted,
109 114 * the underlying Rx ring is forced into poll mode for remainder of
110 115 * the tick. The SRS poll thread only polls for bytes that are
111 116 * allowed to come in the SRS. We typically let 4x the configured
112 117 * B/W worth of packets to come in the SRS (to prevent unnecessary
113 118 * drops due to bursts) but only process the specified amount.
114 119 *
115 120 * A MAC client (e.g. a VNIC or aggr) can have 1 or more
116 121 * Rx rings (and corresponding SRSs) assigned to it. The SRS
117 122 * in turn can have softrings to do protocol level fanout or
118 123 * softrings to do S/W based fanout or both. In case the NIC
119 124 * has no Rx rings, we do S/W classification to respective SRS.
120 125 * The S/W classification rule is always setup and ready. This
121 126 * allows the MAC layer to reassign Rx rings whenever needed
122 127 * but packets still continue to flow via the default path and
123 128 * getting S/W classified to correct SRS.
124 129 *
125 130 * The SRS's are used on both Tx and Rx side. They use the same
126 131 * data structure but the processing routines have slightly different
127 132 * semantics due to the fact that Rx side needs to do dynamic
128 133 * polling etc.
129 134 *
130 135 * Dynamic Polling Notes
131 136 * =====================
132 137 *
133 138 * Each Soft ring set is capable of switching its Rx ring between
134 139 * interrupt and poll mode and actively 'polls' for packets in
135 140 * poll mode. If the SRS is implementing a B/W limit, it makes
136 141 * sure that only Max allowed packets are pulled in poll mode
137 142 * and goes to poll mode as soon as B/W limit is exceeded. As
138 143 * such, there are no overheads to implement B/W limits.
139 144 *
140 145 * In poll mode, its better to keep the pipeline going where the
141 146 * SRS worker thread keeps processing packets and poll thread
142 147 * keeps bringing more packets (specially if they get to run
143 148 * on different CPUs). This also prevents the overheads associated
144 149 * by excessive signalling (on NUMA machines, this can be
145 150 * pretty devastating). The exception is latency optimized case
146 151 * where worker thread does no work and interrupt and poll thread
147 152 * are allowed to do their own drain.
148 153 *
149 154 * We use the following policy to control Dynamic Polling:
150 155 * 1) We switch to poll mode anytime the processing
151 156 * thread causes a backlog to build up in SRS and
152 157 * its associated Soft Rings (sr_poll_pkt_cnt > 0).
153 158 * 2) As long as the backlog stays under the low water
154 159 * mark (sr_lowat), we poll the H/W for more packets.
155 160 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low
156 161 * water mark, we stay in poll mode but don't poll
157 162 * the H/W for more packets.
158 163 * 4) Anytime in polling mode, if we poll the H/W for
159 164 * packets and find nothing plus we have an existing
160 165 * backlog (sr_poll_pkt_cnt > 0), we stay in polling
161 166 * mode but don't poll the H/W for packets anymore
162 167 * (let the polling thread go to sleep).
163 168 * 5) Once the backlog is relived (packets are processed)
164 169 * we reenable polling (by signalling the poll thread)
165 170 * only when the backlog dips below sr_poll_thres.
166 171 * 6) sr_hiwat is used exclusively when we are not
167 172 * polling capable and is used to decide when to
168 173 * drop packets so the SRS queue length doesn't grow
169 174 * infinitely.
170 175 *
171 176 * NOTE: Also see the block level comment on top of mac_soft_ring.c
172 177 */
173 178
174 179 /*
175 180 * mac_latency_optimize
176 181 *
177 182 * Controls whether the poll thread can process the packets inline
178 183 * or let the SRS worker thread do the processing. This applies if
179 184 * the SRS was not being processed. For latency sensitive traffic,
180 185 * this needs to be true to allow inline processing. For throughput
181 186 * under load, this should be false.
182 187 *
183 188 * This (and other similar) tunable should be rolled into a link
184 189 * or flow specific workload hint that can be set using dladm
185 190 * linkprop (instead of multiple such tunables).
186 191 */
187 192 boolean_t mac_latency_optimize = B_TRUE;
188 193
189 194 /*
190 195 * MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN
191 196 *
192 197 * queue a mp or chain in soft ring set and increment the
193 198 * local count (srs_count) for the SRS and the shared counter
194 199 * (srs_poll_pkt_cnt - shared between SRS and its soft rings
195 200 * to track the total unprocessed packets for polling to work
196 201 * correctly).
197 202 *
198 203 * The size (total bytes queued) counters are incremented only
199 204 * if we are doing B/W control.
200 205 */
201 206 #define MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
202 207 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
203 208 if ((mac_srs)->srs_last != NULL) \
204 209 (mac_srs)->srs_last->b_next = (head); \
205 210 else \
206 211 (mac_srs)->srs_first = (head); \
207 212 (mac_srs)->srs_last = (tail); \
208 213 (mac_srs)->srs_count += count; \
209 214 }
210 215
211 216 #define MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
212 217 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
213 218 \
214 219 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
215 220 srs_rx->sr_poll_pkt_cnt += count; \
216 221 ASSERT(srs_rx->sr_poll_pkt_cnt > 0); \
217 222 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
218 223 (mac_srs)->srs_size += (sz); \
219 224 mutex_enter(&(mac_srs)->srs_bw->mac_bw_lock); \
220 225 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
221 226 mutex_exit(&(mac_srs)->srs_bw->mac_bw_lock); \
222 227 } \
223 228 }
224 229
225 230 #define MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
226 231 mac_srs->srs_state |= SRS_ENQUEUED; \
227 232 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
228 233 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
229 234 (mac_srs)->srs_size += (sz); \
230 235 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
231 236 } \
232 237 }
233 238
234 239 /*
235 240 * Turn polling on routines
236 241 */
237 242 #define MAC_SRS_POLLING_ON(mac_srs) { \
238 243 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
239 244 if (((mac_srs)->srs_state & \
240 245 (SRS_POLLING_CAPAB|SRS_POLLING)) == SRS_POLLING_CAPAB) { \
241 246 (mac_srs)->srs_state |= SRS_POLLING; \
242 247 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
243 248 (mac_srs)->srs_ring); \
244 249 (mac_srs)->srs_rx.sr_poll_on++; \
245 250 } \
246 251 }
247 252
248 253 #define MAC_SRS_WORKER_POLLING_ON(mac_srs) { \
249 254 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
250 255 if (((mac_srs)->srs_state & \
251 256 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_POLLING)) == \
252 257 (SRS_POLLING_CAPAB|SRS_WORKER)) { \
253 258 (mac_srs)->srs_state |= SRS_POLLING; \
254 259 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
255 260 (mac_srs)->srs_ring); \
256 261 (mac_srs)->srs_rx.sr_worker_poll_on++; \
257 262 } \
258 263 }
259 264
260 265 /*
261 266 * MAC_SRS_POLL_RING
262 267 *
263 268 * Signal the SRS poll thread to poll the underlying H/W ring
264 269 * provided it wasn't already polling (SRS_GET_PKTS was set).
265 270 *
266 271 * Poll thread gets to run only from mac_rx_srs_drain() and only
267 272 * if the drain was being done by the worker thread.
268 273 */
269 274 #define MAC_SRS_POLL_RING(mac_srs) { \
270 275 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
271 276 \
272 277 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
273 278 srs_rx->sr_poll_thr_sig++; \
274 279 if (((mac_srs)->srs_state & \
275 280 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_GET_PKTS)) == \
276 281 (SRS_WORKER|SRS_POLLING_CAPAB)) { \
277 282 (mac_srs)->srs_state |= SRS_GET_PKTS; \
278 283 cv_signal(&(mac_srs)->srs_cv); \
279 284 } else { \
280 285 srs_rx->sr_poll_thr_busy++; \
281 286 } \
282 287 }
283 288
284 289 /*
285 290 * MAC_SRS_CHECK_BW_CONTROL
286 291 *
287 292 * Check to see if next tick has started so we can reset the
288 293 * SRS_BW_ENFORCED flag and allow more packets to come in the
289 294 * system.
290 295 */
291 296 #define MAC_SRS_CHECK_BW_CONTROL(mac_srs) { \
292 297 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
293 298 ASSERT(((mac_srs)->srs_type & SRST_TX) || \
294 299 MUTEX_HELD(&(mac_srs)->srs_bw->mac_bw_lock)); \
295 300 clock_t now = ddi_get_lbolt(); \
296 301 if ((mac_srs)->srs_bw->mac_bw_curr_time != now) { \
297 302 (mac_srs)->srs_bw->mac_bw_curr_time = now; \
298 303 (mac_srs)->srs_bw->mac_bw_used = 0; \
299 304 if ((mac_srs)->srs_bw->mac_bw_state & SRS_BW_ENFORCED) \
300 305 (mac_srs)->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; \
301 306 } \
302 307 }
303 308
304 309 /*
305 310 * MAC_SRS_WORKER_WAKEUP
306 311 *
307 312 * Wake up the SRS worker thread to process the queue as long as
308 313 * no one else is processing the queue. If we are optimizing for
309 314 * latency, we wake up the worker thread immediately or else we
310 315 * wait mac_srs_worker_wakeup_ticks before worker thread gets
311 316 * woken up.
312 317 */
313 318 int mac_srs_worker_wakeup_ticks = 0;
314 319 #define MAC_SRS_WORKER_WAKEUP(mac_srs) { \
315 320 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
316 321 if (!((mac_srs)->srs_state & SRS_PROC) && \
317 322 (mac_srs)->srs_tid == NULL) { \
318 323 if (((mac_srs)->srs_state & SRS_LATENCY_OPT) || \
319 324 (mac_srs_worker_wakeup_ticks == 0)) \
320 325 cv_signal(&(mac_srs)->srs_async); \
321 326 else \
322 327 (mac_srs)->srs_tid = \
323 328 timeout(mac_srs_fire, (mac_srs), \
324 329 mac_srs_worker_wakeup_ticks); \
325 330 } \
326 331 }
327 332
328 333 #define TX_BANDWIDTH_MODE(mac_srs) \
329 334 ((mac_srs)->srs_tx.st_mode == SRS_TX_BW || \
330 335 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT || \
331 336 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_AGGR)
332 337
333 338 #define TX_SRS_TO_SOFT_RING(mac_srs, head, hint) { \
334 339 if (tx_mode == SRS_TX_BW_FANOUT) \
335 340 (void) mac_tx_fanout_mode(mac_srs, head, hint, 0, NULL);\
336 341 else \
337 342 (void) mac_tx_aggr_mode(mac_srs, head, hint, 0, NULL); \
338 343 }
339 344
340 345 /*
341 346 * MAC_TX_SRS_BLOCK
342 347 *
343 348 * Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED
344 349 * will be set only if srs_tx_woken_up is FALSE. If
345 350 * srs_tx_woken_up is TRUE, it indicates that the wakeup arrived
346 351 * before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to
347 352 * attempt to transmit again and not setting SRS_TX_BLOCKED does
348 353 * that.
349 354 */
350 355 #define MAC_TX_SRS_BLOCK(srs, mp) { \
351 356 ASSERT(MUTEX_HELD(&(srs)->srs_lock)); \
352 357 if ((srs)->srs_tx.st_woken_up) { \
353 358 (srs)->srs_tx.st_woken_up = B_FALSE; \
354 359 } else { \
355 360 ASSERT(!((srs)->srs_state & SRS_TX_BLOCKED)); \
356 361 (srs)->srs_state |= SRS_TX_BLOCKED; \
357 362 (srs)->srs_tx.st_stat.mts_blockcnt++; \
358 363 } \
359 364 }
360 365
361 366 /*
362 367 * MAC_TX_SRS_TEST_HIWAT
363 368 *
364 369 * Called before queueing a packet onto Tx SRS to test and set
365 370 * SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat.
366 371 */
367 372 #define MAC_TX_SRS_TEST_HIWAT(srs, mp, tail, cnt, sz, cookie) { \
368 373 boolean_t enqueue = 1; \
369 374 \
370 375 if ((srs)->srs_count > (srs)->srs_tx.st_hiwat) { \
371 376 /* \
372 377 * flow-controlled. Store srs in cookie so that it \
373 378 * can be returned as mac_tx_cookie_t to client \
374 379 */ \
375 380 (srs)->srs_state |= SRS_TX_HIWAT; \
376 381 cookie = (mac_tx_cookie_t)srs; \
377 382 (srs)->srs_tx.st_hiwat_cnt++; \
378 383 if ((srs)->srs_count > (srs)->srs_tx.st_max_q_cnt) { \
379 384 /* increment freed stats */ \
380 385 (srs)->srs_tx.st_stat.mts_sdrops += cnt; \
381 386 /* \
382 387 * b_prev may be set to the fanout hint \
383 388 * hence can't use freemsg directly \
384 389 */ \
385 390 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE); \
386 391 DTRACE_PROBE1(tx_queued_hiwat, \
387 392 mac_soft_ring_set_t *, srs); \
388 393 enqueue = 0; \
389 394 } \
390 395 } \
391 396 if (enqueue) \
392 397 MAC_TX_SRS_ENQUEUE_CHAIN(srs, mp, tail, cnt, sz); \
393 398 }
394 399
395 400 /* Some utility macros */
396 401 #define MAC_SRS_BW_LOCK(srs) \
397 402 if (!(srs->srs_type & SRST_TX)) \
398 403 mutex_enter(&srs->srs_bw->mac_bw_lock);
399 404
400 405 #define MAC_SRS_BW_UNLOCK(srs) \
401 406 if (!(srs->srs_type & SRST_TX)) \
402 407 mutex_exit(&srs->srs_bw->mac_bw_lock);
403 408
404 409 #define MAC_TX_SRS_DROP_MESSAGE(srs, mp, cookie) { \
405 410 mac_pkt_drop(NULL, NULL, mp, B_FALSE); \
406 411 /* increment freed stats */ \
407 412 mac_srs->srs_tx.st_stat.mts_sdrops++; \
|
↓ open down ↓ |
366 lines elided |
↑ open up ↑ |
408 413 cookie = (mac_tx_cookie_t)srs; \
409 414 }
410 415
411 416 #define MAC_TX_SET_NO_ENQUEUE(srs, mp_chain, ret_mp, cookie) { \
412 417 mac_srs->srs_state |= SRS_TX_WAKEUP_CLIENT; \
413 418 cookie = (mac_tx_cookie_t)srs; \
414 419 *ret_mp = mp_chain; \
415 420 }
416 421
417 422 /*
423 + * MAC_RX_SRS_TOODEEP
424 + *
425 + * Macro called as part of receive-side processing to determine if handling
426 + * can occur in situ (in the interrupt thread) or if it should be left to a
427 + * worker thread. Note that the constant used to make this determination is
428 + * not entirely made-up, and is a result of some emprical validation. That
429 + * said, the constant is left as a static variable to allow it to be
430 + * dynamically tuned in the field if and as needed.
431 + */
432 +static uintptr_t mac_rx_srs_stack_needed = 10240;
433 +static uint_t mac_rx_srs_stack_toodeep;
434 +
435 +#ifndef STACK_GROWTH_DOWN
436 +#error Downward stack growth assumed.
437 +#endif
438 +
439 +#define MAC_RX_SRS_TOODEEP() (STACK_BIAS + (uintptr_t)getfp() - \
440 + (uintptr_t)curthread->t_stkbase < mac_rx_srs_stack_needed && \
441 + ++mac_rx_srs_stack_toodeep)
442 +
443 +
444 +/*
418 445 * Drop the rx packet and advance to the next one in the chain.
419 446 */
420 447 static void
421 448 mac_rx_drop_pkt(mac_soft_ring_set_t *srs, mblk_t *mp)
422 449 {
423 450 mac_srs_rx_t *srs_rx = &srs->srs_rx;
424 451
425 452 ASSERT(mp->b_next == NULL);
426 453 mutex_enter(&srs->srs_lock);
427 454 MAC_UPDATE_SRS_COUNT_LOCKED(srs, 1);
428 455 MAC_UPDATE_SRS_SIZE_LOCKED(srs, msgdsize(mp));
429 456 mutex_exit(&srs->srs_lock);
430 457
431 458 srs_rx->sr_stat.mrs_sdrops++;
432 459 freemsg(mp);
433 460 }
434 461
435 462 /* DATAPATH RUNTIME ROUTINES */
436 463
437 464 /*
438 465 * mac_srs_fire
439 466 *
440 467 * Timer callback routine for waking up the SRS worker thread.
441 468 */
442 469 static void
443 470 mac_srs_fire(void *arg)
444 471 {
445 472 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)arg;
446 473
447 474 mutex_enter(&mac_srs->srs_lock);
448 475 if (mac_srs->srs_tid == 0) {
449 476 mutex_exit(&mac_srs->srs_lock);
450 477 return;
451 478 }
452 479
453 480 mac_srs->srs_tid = 0;
454 481 if (!(mac_srs->srs_state & SRS_PROC))
455 482 cv_signal(&mac_srs->srs_async);
456 483
457 484 mutex_exit(&mac_srs->srs_lock);
458 485 }
459 486
460 487 /*
461 488 * 'hint' is fanout_hint (type of uint64_t) which is given by the TCP/IP stack,
462 489 * and it is used on the TX path.
463 490 */
464 491 #define HASH_HINT(hint) \
465 492 ((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8))
466 493
467 494
468 495 /*
469 496 * hash based on the src address and the port information.
470 497 */
471 498 #define HASH_ADDR(src, ports) \
472 499 (ntohl((src)) ^ ((ports) >> 24) ^ ((ports) >> 16) ^ \
473 500 ((ports) >> 8) ^ (ports))
474 501
475 502 #define COMPUTE_INDEX(key, sz) (key % sz)
476 503
477 504 #define FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) { \
478 505 if ((tail) != NULL) { \
479 506 ASSERT((tail)->b_next == NULL); \
480 507 (tail)->b_next = (mp); \
481 508 } else { \
482 509 ASSERT((head) == NULL); \
483 510 (head) = (mp); \
484 511 } \
485 512 (tail) = (mp); \
486 513 (cnt)++; \
487 514 if ((bw_ctl)) \
488 515 (sz) += (sz0); \
489 516 }
490 517
491 518 #define MAC_FANOUT_DEFAULT 0
492 519 #define MAC_FANOUT_RND_ROBIN 1
493 520 int mac_fanout_type = MAC_FANOUT_DEFAULT;
494 521
495 522 #define MAX_SR_TYPES 3
496 523 /* fanout types for port based hashing */
497 524 enum pkt_type {
498 525 V4_TCP = 0,
499 526 V4_UDP,
500 527 OTH,
501 528 UNDEF
502 529 };
503 530
504 531 /*
505 532 * In general we do port based hashing to spread traffic over different
506 533 * softrings. The below tunable allows to override that behavior. Setting it
507 534 * to B_TRUE allows to do a fanout based on src ipv6 address. This behavior
508 535 * is also the applicable to ipv6 packets carrying multiple optional headers
509 536 * and other uncommon packet types.
510 537 */
511 538 boolean_t mac_src_ipv6_fanout = B_FALSE;
512 539
513 540 /*
514 541 * Pair of local and remote ports in the transport header
515 542 */
516 543 #define PORTS_SIZE 4
517 544
518 545 /*
519 546 * mac_rx_srs_proto_fanout
520 547 *
521 548 * This routine delivers packets destined to an SRS into one of the
522 549 * protocol soft rings.
523 550 *
524 551 * Given a chain of packets we need to split it up into multiple sub chains
525 552 * destined into TCP, UDP or OTH soft ring. Instead of entering
526 553 * the soft ring one packet at a time, we want to enter it in the form of a
527 554 * chain otherwise we get this start/stop behaviour where the worker thread
528 555 * goes to sleep and then next packets comes in forcing it to wake up etc.
529 556 */
530 557 static void
531 558 mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
532 559 {
533 560 struct ether_header *ehp;
534 561 struct ether_vlan_header *evhp;
535 562 uint32_t sap;
536 563 ipha_t *ipha;
537 564 uint8_t *dstaddr;
538 565 size_t hdrsize;
539 566 mblk_t *mp;
540 567 mblk_t *headmp[MAX_SR_TYPES];
541 568 mblk_t *tailmp[MAX_SR_TYPES];
542 569 int cnt[MAX_SR_TYPES];
543 570 size_t sz[MAX_SR_TYPES];
544 571 size_t sz1;
545 572 boolean_t bw_ctl;
546 573 boolean_t hw_classified;
547 574 boolean_t dls_bypass;
548 575 boolean_t is_ether;
549 576 boolean_t is_unicast;
550 577 enum pkt_type type;
551 578 mac_client_impl_t *mcip = mac_srs->srs_mcip;
552 579
553 580 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
554 581 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
555 582
556 583 /*
557 584 * If we don't have a Rx ring, S/W classification would have done
558 585 * its job and its a packet meant for us. If we were polling on
559 586 * the default ring (i.e. there was a ring assigned to this SRS),
560 587 * then we need to make sure that the mac address really belongs
561 588 * to us.
562 589 */
563 590 hw_classified = mac_srs->srs_ring != NULL &&
564 591 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
565 592
566 593 /*
567 594 * Special clients (eg. VLAN, non ether, etc) need DLS
568 595 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
569 596 * such SRSs. Another way of disabling bypass is to set the
570 597 * MCIS_RX_BYPASS_DISABLE flag.
571 598 */
572 599 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
573 600 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
574 601
575 602 bzero(headmp, MAX_SR_TYPES * sizeof (mblk_t *));
576 603 bzero(tailmp, MAX_SR_TYPES * sizeof (mblk_t *));
577 604 bzero(cnt, MAX_SR_TYPES * sizeof (int));
578 605 bzero(sz, MAX_SR_TYPES * sizeof (size_t));
579 606
580 607 /*
581 608 * We got a chain from SRS that we need to send to the soft rings.
582 609 * Since squeues for TCP & IPv4 sap poll their soft rings (for
583 610 * performance reasons), we need to separate out v4_tcp, v4_udp
584 611 * and the rest goes in other.
585 612 */
586 613 while (head != NULL) {
587 614 mp = head;
588 615 head = head->b_next;
589 616 mp->b_next = NULL;
590 617
591 618 type = OTH;
592 619 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
593 620
594 621 if (is_ether) {
595 622 /*
596 623 * At this point we can be sure the packet at least
597 624 * has an ether header.
598 625 */
599 626 if (sz1 < sizeof (struct ether_header)) {
600 627 mac_rx_drop_pkt(mac_srs, mp);
601 628 continue;
602 629 }
603 630 ehp = (struct ether_header *)mp->b_rptr;
604 631
605 632 /*
606 633 * Determine if this is a VLAN or non-VLAN packet.
607 634 */
608 635 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
609 636 evhp = (struct ether_vlan_header *)mp->b_rptr;
610 637 sap = ntohs(evhp->ether_type);
611 638 hdrsize = sizeof (struct ether_vlan_header);
612 639 /*
613 640 * Check if the VID of the packet, if any,
614 641 * belongs to this client.
615 642 */
616 643 if (!mac_client_check_flow_vid(mcip,
617 644 VLAN_ID(ntohs(evhp->ether_tci)))) {
618 645 mac_rx_drop_pkt(mac_srs, mp);
619 646 continue;
620 647 }
621 648 } else {
622 649 hdrsize = sizeof (struct ether_header);
623 650 }
624 651 is_unicast =
625 652 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
626 653 dstaddr = (uint8_t *)&ehp->ether_dhost;
627 654 } else {
628 655 mac_header_info_t mhi;
629 656
630 657 if (mac_header_info((mac_handle_t)mcip->mci_mip,
631 658 mp, &mhi) != 0) {
632 659 mac_rx_drop_pkt(mac_srs, mp);
633 660 continue;
634 661 }
635 662 hdrsize = mhi.mhi_hdrsize;
636 663 sap = mhi.mhi_bindsap;
637 664 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
638 665 dstaddr = (uint8_t *)mhi.mhi_daddr;
639 666 }
640 667
641 668 if (!dls_bypass) {
642 669 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
643 670 cnt[type], bw_ctl, sz[type], sz1, mp);
644 671 continue;
645 672 }
646 673
647 674 if (sap == ETHERTYPE_IP) {
648 675 /*
649 676 * If we are H/W classified, but we have promisc
650 677 * on, then we need to check for the unicast address.
651 678 */
652 679 if (hw_classified && mcip->mci_promisc_list != NULL) {
653 680 mac_address_t *map;
654 681
655 682 rw_enter(&mcip->mci_rw_lock, RW_READER);
656 683 map = mcip->mci_unicast;
657 684 if (bcmp(dstaddr, map->ma_addr,
658 685 map->ma_len) == 0)
659 686 type = UNDEF;
660 687 rw_exit(&mcip->mci_rw_lock);
661 688 } else if (is_unicast) {
662 689 type = UNDEF;
663 690 }
664 691 }
665 692
666 693 /*
667 694 * This needs to become a contract with the driver for
668 695 * the fast path.
669 696 *
670 697 * In the normal case the packet will have at least the L2
671 698 * header and the IP + Transport header in the same mblk.
672 699 * This is usually the case when the NIC driver sends up
673 700 * the packet. This is also true when the stack generates
674 701 * a packet that is looped back and when the stack uses the
675 702 * fastpath mechanism. The normal case is optimized for
676 703 * performance and may bypass DLS. All other cases go through
677 704 * the 'OTH' type path without DLS bypass.
678 705 */
679 706
680 707 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
681 708 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha))
682 709 type = OTH;
683 710
684 711 if (type == OTH) {
685 712 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
686 713 cnt[type], bw_ctl, sz[type], sz1, mp);
687 714 continue;
688 715 }
689 716
690 717 ASSERT(type == UNDEF);
691 718 /*
692 719 * We look for at least 4 bytes past the IP header to get
693 720 * the port information. If we get an IP fragment, we don't
694 721 * have the port information, and we use just the protocol
695 722 * information.
696 723 */
697 724 switch (ipha->ipha_protocol) {
698 725 case IPPROTO_TCP:
699 726 type = V4_TCP;
700 727 mp->b_rptr += hdrsize;
701 728 break;
702 729 case IPPROTO_UDP:
703 730 type = V4_UDP;
704 731 mp->b_rptr += hdrsize;
705 732 break;
706 733 default:
707 734 type = OTH;
708 735 break;
709 736 }
710 737
711 738 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type],
712 739 bw_ctl, sz[type], sz1, mp);
713 740 }
714 741
715 742 for (type = V4_TCP; type < UNDEF; type++) {
716 743 if (headmp[type] != NULL) {
717 744 mac_soft_ring_t *softring;
718 745
719 746 ASSERT(tailmp[type]->b_next == NULL);
720 747 switch (type) {
721 748 case V4_TCP:
722 749 softring = mac_srs->srs_tcp_soft_rings[0];
723 750 break;
724 751 case V4_UDP:
725 752 softring = mac_srs->srs_udp_soft_rings[0];
726 753 break;
727 754 case OTH:
728 755 softring = mac_srs->srs_oth_soft_rings[0];
729 756 }
730 757 mac_rx_soft_ring_process(mcip, softring,
731 758 headmp[type], tailmp[type], cnt[type], sz[type]);
732 759 }
733 760 }
734 761 }
735 762
736 763 int fanout_unalligned = 0;
737 764
738 765 /*
739 766 * mac_rx_srs_long_fanout
740 767 *
741 768 * The fanout routine for IPv6
742 769 */
743 770 static int
744 771 mac_rx_srs_long_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *mp,
745 772 uint32_t sap, size_t hdrsize, enum pkt_type *type, uint_t *indx)
746 773 {
747 774 ip6_t *ip6h;
748 775 uint8_t *whereptr;
749 776 uint_t hash;
750 777 uint16_t remlen;
751 778 uint8_t nexthdr;
752 779 uint16_t hdr_len;
753 780
754 781 if (sap == ETHERTYPE_IPV6) {
755 782 boolean_t modifiable = B_TRUE;
756 783
757 784 ASSERT(MBLKL(mp) >= hdrsize);
758 785
759 786 ip6h = (ip6_t *)(mp->b_rptr + hdrsize);
760 787 if ((unsigned char *)ip6h == mp->b_wptr) {
761 788 /*
762 789 * The first mblk_t only includes the mac header.
763 790 * Note that it is safe to change the mp pointer here,
764 791 * as the subsequent operation does not assume mp
765 792 * points to the start of the mac header.
766 793 */
767 794 mp = mp->b_cont;
768 795
769 796 /*
770 797 * Make sure ip6h holds the full ip6_t structure.
771 798 */
772 799 if (mp == NULL)
773 800 return (-1);
774 801
775 802 if (MBLKL(mp) < IPV6_HDR_LEN) {
776 803 modifiable = (DB_REF(mp) == 1);
777 804
778 805 if (modifiable &&
779 806 !pullupmsg(mp, IPV6_HDR_LEN)) {
780 807 return (-1);
781 808 }
782 809 }
783 810
784 811 ip6h = (ip6_t *)mp->b_rptr;
785 812 }
786 813
787 814 if (!modifiable || !(OK_32PTR((char *)ip6h)) ||
788 815 ((unsigned char *)ip6h + IPV6_HDR_LEN > mp->b_wptr)) {
789 816 /*
790 817 * If either ip6h is not alligned, or ip6h does not
791 818 * hold the complete ip6_t structure (a pullupmsg()
792 819 * is not an option since it would result in an
793 820 * unalligned ip6h), fanout to the default ring. Note
794 821 * that this may cause packets reordering.
795 822 */
796 823 *indx = 0;
797 824 *type = OTH;
798 825 fanout_unalligned++;
799 826 return (0);
800 827 }
801 828
802 829 remlen = ntohs(ip6h->ip6_plen);
803 830 nexthdr = ip6h->ip6_nxt;
804 831
805 832 if (remlen < MIN_EHDR_LEN)
806 833 return (-1);
807 834 /*
808 835 * Do src based fanout if below tunable is set to B_TRUE or
809 836 * when mac_ip_hdr_length_v6() fails because of malformed
810 837 * packets or because mblk's need to be concatenated using
811 838 * pullupmsg().
812 839 */
813 840 if (mac_src_ipv6_fanout || !mac_ip_hdr_length_v6(ip6h,
814 841 mp->b_wptr, &hdr_len, &nexthdr, NULL)) {
815 842 goto src_based_fanout;
816 843 }
817 844 whereptr = (uint8_t *)ip6h + hdr_len;
818 845
819 846 /* If the transport is one of below, we do port based fanout */
820 847 switch (nexthdr) {
821 848 case IPPROTO_TCP:
822 849 case IPPROTO_UDP:
823 850 case IPPROTO_SCTP:
824 851 case IPPROTO_ESP:
825 852 /*
826 853 * If the ports in the transport header is not part of
827 854 * the mblk, do src_based_fanout, instead of calling
828 855 * pullupmsg().
829 856 */
830 857 if (mp->b_cont != NULL &&
831 858 whereptr + PORTS_SIZE > mp->b_wptr) {
832 859 goto src_based_fanout;
833 860 }
834 861 break;
835 862 default:
836 863 break;
837 864 }
838 865
839 866 switch (nexthdr) {
840 867 case IPPROTO_TCP:
841 868 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src),
842 869 *(uint32_t *)whereptr);
843 870 *indx = COMPUTE_INDEX(hash,
844 871 mac_srs->srs_tcp_ring_count);
845 872 *type = OTH;
846 873 break;
847 874
848 875 case IPPROTO_UDP:
849 876 case IPPROTO_SCTP:
850 877 case IPPROTO_ESP:
851 878 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
852 879 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src),
853 880 *(uint32_t *)whereptr);
854 881 *indx = COMPUTE_INDEX(hash,
855 882 mac_srs->srs_udp_ring_count);
856 883 } else {
857 884 *indx = mac_srs->srs_ind %
858 885 mac_srs->srs_udp_ring_count;
859 886 mac_srs->srs_ind++;
860 887 }
861 888 *type = OTH;
862 889 break;
863 890
864 891 /* For all other protocol, do source based fanout */
865 892 default:
866 893 goto src_based_fanout;
867 894 }
868 895 } else {
869 896 *indx = 0;
870 897 *type = OTH;
871 898 }
872 899 return (0);
873 900
874 901 src_based_fanout:
875 902 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src), (uint32_t)0);
876 903 *indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count);
877 904 *type = OTH;
878 905 return (0);
879 906 }
880 907
881 908 /*
882 909 * mac_rx_srs_fanout
883 910 *
884 911 * This routine delivers packets destined to an SRS into a soft ring member
885 912 * of the set.
886 913 *
887 914 * Given a chain of packets we need to split it up into multiple sub chains
888 915 * destined for one of the TCP, UDP or OTH soft rings. Instead of entering
889 916 * the soft ring one packet at a time, we want to enter it in the form of a
890 917 * chain otherwise we get this start/stop behaviour where the worker thread
891 918 * goes to sleep and then next packets comes in forcing it to wake up etc.
892 919 *
893 920 * Note:
894 921 * Since we know what is the maximum fanout possible, we create a 2D array
895 922 * of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz
896 923 * variables so that we can enter the softrings with chain. We need the
897 924 * MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc
898 925 * for each packet would be expensive). If we ever want to have the
899 926 * ability to have unlimited fanout, we should probably declare a head,
900 927 * tail, cnt, sz with each soft ring (a data struct which contains a softring
901 928 * along with these members) and create an array of this uber struct so we
902 929 * don't have to do kmem_alloc.
903 930 */
904 931 int fanout_oth1 = 0;
905 932 int fanout_oth2 = 0;
906 933 int fanout_oth3 = 0;
907 934 int fanout_oth4 = 0;
908 935 int fanout_oth5 = 0;
909 936
910 937 static void
911 938 mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
912 939 {
913 940 struct ether_header *ehp;
914 941 struct ether_vlan_header *evhp;
915 942 uint32_t sap;
916 943 ipha_t *ipha;
917 944 uint8_t *dstaddr;
918 945 uint_t indx;
919 946 size_t ports_offset;
920 947 size_t ipha_len;
921 948 size_t hdrsize;
922 949 uint_t hash;
923 950 mblk_t *mp;
924 951 mblk_t *headmp[MAX_SR_TYPES][MAX_SR_FANOUT];
925 952 mblk_t *tailmp[MAX_SR_TYPES][MAX_SR_FANOUT];
926 953 int cnt[MAX_SR_TYPES][MAX_SR_FANOUT];
927 954 size_t sz[MAX_SR_TYPES][MAX_SR_FANOUT];
928 955 size_t sz1;
929 956 boolean_t bw_ctl;
930 957 boolean_t hw_classified;
931 958 boolean_t dls_bypass;
932 959 boolean_t is_ether;
933 960 boolean_t is_unicast;
934 961 int fanout_cnt;
935 962 enum pkt_type type;
936 963 mac_client_impl_t *mcip = mac_srs->srs_mcip;
937 964
938 965 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
939 966 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
940 967
941 968 /*
942 969 * If we don't have a Rx ring, S/W classification would have done
943 970 * its job and its a packet meant for us. If we were polling on
944 971 * the default ring (i.e. there was a ring assigned to this SRS),
945 972 * then we need to make sure that the mac address really belongs
946 973 * to us.
947 974 */
948 975 hw_classified = mac_srs->srs_ring != NULL &&
949 976 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
950 977
951 978 /*
952 979 * Special clients (eg. VLAN, non ether, etc) need DLS
953 980 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
954 981 * such SRSs. Another way of disabling bypass is to set the
955 982 * MCIS_RX_BYPASS_DISABLE flag.
956 983 */
957 984 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
958 985 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
959 986
960 987 /*
961 988 * Since the softrings are never destroyed and we always
962 989 * create equal number of softrings for TCP, UDP and rest,
963 990 * its OK to check one of them for count and use it without
964 991 * any lock. In future, if soft rings get destroyed because
965 992 * of reduction in fanout, we will need to ensure that happens
966 993 * behind the SRS_PROC.
967 994 */
968 995 fanout_cnt = mac_srs->srs_tcp_ring_count;
969 996
970 997 bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
971 998 bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
972 999 bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int));
973 1000 bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t));
974 1001
975 1002 /*
976 1003 * We got a chain from SRS that we need to send to the soft rings.
977 1004 * Since squeues for TCP & IPv4 sap poll their soft rings (for
978 1005 * performance reasons), we need to separate out v4_tcp, v4_udp
979 1006 * and the rest goes in other.
980 1007 */
981 1008 while (head != NULL) {
982 1009 mp = head;
983 1010 head = head->b_next;
984 1011 mp->b_next = NULL;
985 1012
986 1013 type = OTH;
987 1014 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
988 1015
989 1016 if (is_ether) {
990 1017 /*
991 1018 * At this point we can be sure the packet at least
992 1019 * has an ether header.
993 1020 */
994 1021 if (sz1 < sizeof (struct ether_header)) {
995 1022 mac_rx_drop_pkt(mac_srs, mp);
996 1023 continue;
997 1024 }
998 1025 ehp = (struct ether_header *)mp->b_rptr;
999 1026
1000 1027 /*
1001 1028 * Determine if this is a VLAN or non-VLAN packet.
1002 1029 */
1003 1030 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
1004 1031 evhp = (struct ether_vlan_header *)mp->b_rptr;
1005 1032 sap = ntohs(evhp->ether_type);
1006 1033 hdrsize = sizeof (struct ether_vlan_header);
1007 1034 /*
1008 1035 * Check if the VID of the packet, if any,
1009 1036 * belongs to this client.
1010 1037 */
1011 1038 if (!mac_client_check_flow_vid(mcip,
1012 1039 VLAN_ID(ntohs(evhp->ether_tci)))) {
1013 1040 mac_rx_drop_pkt(mac_srs, mp);
1014 1041 continue;
1015 1042 }
1016 1043 } else {
1017 1044 hdrsize = sizeof (struct ether_header);
1018 1045 }
1019 1046 is_unicast =
1020 1047 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
1021 1048 dstaddr = (uint8_t *)&ehp->ether_dhost;
1022 1049 } else {
1023 1050 mac_header_info_t mhi;
1024 1051
1025 1052 if (mac_header_info((mac_handle_t)mcip->mci_mip,
1026 1053 mp, &mhi) != 0) {
1027 1054 mac_rx_drop_pkt(mac_srs, mp);
1028 1055 continue;
1029 1056 }
1030 1057 hdrsize = mhi.mhi_hdrsize;
1031 1058 sap = mhi.mhi_bindsap;
1032 1059 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
1033 1060 dstaddr = (uint8_t *)mhi.mhi_daddr;
1034 1061 }
1035 1062
1036 1063 if (!dls_bypass) {
1037 1064 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
1038 1065 hdrsize, &type, &indx) == -1) {
1039 1066 mac_rx_drop_pkt(mac_srs, mp);
1040 1067 continue;
1041 1068 }
1042 1069
1043 1070 FANOUT_ENQUEUE_MP(headmp[type][indx],
1044 1071 tailmp[type][indx], cnt[type][indx], bw_ctl,
1045 1072 sz[type][indx], sz1, mp);
1046 1073 continue;
1047 1074 }
1048 1075
1049 1076
1050 1077 /*
1051 1078 * If we are using the default Rx ring where H/W or S/W
1052 1079 * classification has not happened, we need to verify if
1053 1080 * this unicast packet really belongs to us.
1054 1081 */
1055 1082 if (sap == ETHERTYPE_IP) {
1056 1083 /*
1057 1084 * If we are H/W classified, but we have promisc
1058 1085 * on, then we need to check for the unicast address.
1059 1086 */
1060 1087 if (hw_classified && mcip->mci_promisc_list != NULL) {
1061 1088 mac_address_t *map;
1062 1089
1063 1090 rw_enter(&mcip->mci_rw_lock, RW_READER);
1064 1091 map = mcip->mci_unicast;
1065 1092 if (bcmp(dstaddr, map->ma_addr,
1066 1093 map->ma_len) == 0)
1067 1094 type = UNDEF;
1068 1095 rw_exit(&mcip->mci_rw_lock);
1069 1096 } else if (is_unicast) {
1070 1097 type = UNDEF;
1071 1098 }
1072 1099 }
1073 1100
1074 1101 /*
1075 1102 * This needs to become a contract with the driver for
1076 1103 * the fast path.
1077 1104 */
1078 1105
1079 1106 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
1080 1107 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) {
1081 1108 type = OTH;
1082 1109 fanout_oth1++;
1083 1110 }
1084 1111
1085 1112 if (type != OTH) {
1086 1113 uint16_t frag_offset_flags;
1087 1114
1088 1115 switch (ipha->ipha_protocol) {
1089 1116 case IPPROTO_TCP:
1090 1117 case IPPROTO_UDP:
1091 1118 case IPPROTO_SCTP:
1092 1119 case IPPROTO_ESP:
1093 1120 ipha_len = IPH_HDR_LENGTH(ipha);
1094 1121 if ((uchar_t *)ipha + ipha_len + PORTS_SIZE >
1095 1122 mp->b_wptr) {
1096 1123 type = OTH;
1097 1124 break;
1098 1125 }
1099 1126 frag_offset_flags =
1100 1127 ntohs(ipha->ipha_fragment_offset_and_flags);
1101 1128 if ((frag_offset_flags &
1102 1129 (IPH_MF | IPH_OFFSET)) != 0) {
1103 1130 type = OTH;
1104 1131 fanout_oth3++;
1105 1132 break;
1106 1133 }
1107 1134 ports_offset = hdrsize + ipha_len;
1108 1135 break;
1109 1136 default:
1110 1137 type = OTH;
1111 1138 fanout_oth4++;
1112 1139 break;
1113 1140 }
1114 1141 }
1115 1142
1116 1143 if (type == OTH) {
1117 1144 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
1118 1145 hdrsize, &type, &indx) == -1) {
1119 1146 mac_rx_drop_pkt(mac_srs, mp);
1120 1147 continue;
1121 1148 }
1122 1149
1123 1150 FANOUT_ENQUEUE_MP(headmp[type][indx],
1124 1151 tailmp[type][indx], cnt[type][indx], bw_ctl,
1125 1152 sz[type][indx], sz1, mp);
1126 1153 continue;
1127 1154 }
1128 1155
1129 1156 ASSERT(type == UNDEF);
1130 1157
1131 1158 /*
1132 1159 * XXX-Sunay: We should hold srs_lock since ring_count
1133 1160 * below can change. But if we are always called from
1134 1161 * mac_rx_srs_drain and SRS_PROC is set, then we can
1135 1162 * enforce that ring_count can't be changed i.e.
1136 1163 * to change fanout type or ring count, the calling
1137 1164 * thread needs to be behind SRS_PROC.
1138 1165 */
1139 1166 switch (ipha->ipha_protocol) {
1140 1167 case IPPROTO_TCP:
1141 1168 /*
1142 1169 * Note that for ESP, we fanout on SPI and it is at the
1143 1170 * same offset as the 2x16-bit ports. So it is clumped
1144 1171 * along with TCP, UDP and SCTP.
1145 1172 */
1146 1173 hash = HASH_ADDR(ipha->ipha_src,
1147 1174 *(uint32_t *)(mp->b_rptr + ports_offset));
1148 1175 indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
1149 1176 type = V4_TCP;
1150 1177 mp->b_rptr += hdrsize;
1151 1178 break;
1152 1179 case IPPROTO_UDP:
1153 1180 case IPPROTO_SCTP:
1154 1181 case IPPROTO_ESP:
1155 1182 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
1156 1183 hash = HASH_ADDR(ipha->ipha_src,
1157 1184 *(uint32_t *)(mp->b_rptr + ports_offset));
1158 1185 indx = COMPUTE_INDEX(hash,
1159 1186 mac_srs->srs_udp_ring_count);
1160 1187 } else {
1161 1188 indx = mac_srs->srs_ind %
1162 1189 mac_srs->srs_udp_ring_count;
1163 1190 mac_srs->srs_ind++;
1164 1191 }
1165 1192 type = V4_UDP;
1166 1193 mp->b_rptr += hdrsize;
1167 1194 break;
1168 1195 default:
1169 1196 indx = 0;
1170 1197 type = OTH;
1171 1198 }
1172 1199
1173 1200 FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx],
1174 1201 cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp);
1175 1202 }
1176 1203
1177 1204 for (type = V4_TCP; type < UNDEF; type++) {
1178 1205 int i;
1179 1206
1180 1207 for (i = 0; i < fanout_cnt; i++) {
1181 1208 if (headmp[type][i] != NULL) {
1182 1209 mac_soft_ring_t *softring;
1183 1210
1184 1211 ASSERT(tailmp[type][i]->b_next == NULL);
1185 1212 switch (type) {
1186 1213 case V4_TCP:
1187 1214 softring =
1188 1215 mac_srs->srs_tcp_soft_rings[i];
1189 1216 break;
1190 1217 case V4_UDP:
1191 1218 softring =
1192 1219 mac_srs->srs_udp_soft_rings[i];
1193 1220 break;
1194 1221 case OTH:
1195 1222 softring =
1196 1223 mac_srs->srs_oth_soft_rings[i];
1197 1224 break;
1198 1225 }
1199 1226 mac_rx_soft_ring_process(mcip,
1200 1227 softring, headmp[type][i], tailmp[type][i],
1201 1228 cnt[type][i], sz[type][i]);
1202 1229 }
1203 1230 }
1204 1231 }
1205 1232 }
1206 1233
1207 1234 #define SRS_BYTES_TO_PICKUP 150000
1208 1235 ssize_t max_bytes_to_pickup = SRS_BYTES_TO_PICKUP;
1209 1236
1210 1237 /*
1211 1238 * mac_rx_srs_poll_ring
1212 1239 *
1213 1240 * This SRS Poll thread uses this routine to poll the underlying hardware
1214 1241 * Rx ring to get a chain of packets. It can inline process that chain
1215 1242 * if mac_latency_optimize is set (default) or signal the SRS worker thread
1216 1243 * to do the remaining processing.
1217 1244 *
1218 1245 * Since packets come in the system via interrupt or poll path, we also
1219 1246 * update the stats and deal with promiscous clients here.
1220 1247 */
1221 1248 void
1222 1249 mac_rx_srs_poll_ring(mac_soft_ring_set_t *mac_srs)
1223 1250 {
1224 1251 kmutex_t *lock = &mac_srs->srs_lock;
1225 1252 kcondvar_t *async = &mac_srs->srs_cv;
1226 1253 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
1227 1254 mblk_t *head, *tail, *mp;
1228 1255 callb_cpr_t cprinfo;
1229 1256 ssize_t bytes_to_pickup;
1230 1257 size_t sz;
1231 1258 int count;
1232 1259 mac_client_impl_t *smcip;
1233 1260
1234 1261 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_srs_poll");
1235 1262 mutex_enter(lock);
1236 1263
1237 1264 start:
1238 1265 for (;;) {
1239 1266 if (mac_srs->srs_state & SRS_PAUSE)
1240 1267 goto done;
1241 1268
1242 1269 CALLB_CPR_SAFE_BEGIN(&cprinfo);
1243 1270 cv_wait(async, lock);
1244 1271 CALLB_CPR_SAFE_END(&cprinfo, lock);
1245 1272
1246 1273 if (mac_srs->srs_state & SRS_PAUSE)
1247 1274 goto done;
1248 1275
1249 1276 check_again:
1250 1277 if (mac_srs->srs_type & SRST_BW_CONTROL) {
1251 1278 /*
1252 1279 * We pick as many bytes as we are allowed to queue.
1253 1280 * Its possible that we will exceed the total
1254 1281 * packets queued in case this SRS is part of the
1255 1282 * Rx ring group since > 1 poll thread can be pulling
1256 1283 * upto the max allowed packets at the same time
1257 1284 * but that should be OK.
1258 1285 */
1259 1286 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1260 1287 bytes_to_pickup =
1261 1288 mac_srs->srs_bw->mac_bw_drop_threshold -
1262 1289 mac_srs->srs_bw->mac_bw_sz;
1263 1290 /*
1264 1291 * We shouldn't have been signalled if we
1265 1292 * have 0 or less bytes to pick but since
1266 1293 * some of the bytes accounting is driver
1267 1294 * dependant, we do the safety check.
1268 1295 */
1269 1296 if (bytes_to_pickup < 0)
1270 1297 bytes_to_pickup = 0;
1271 1298 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1272 1299 } else {
1273 1300 /*
1274 1301 * ToDO: Need to change the polling API
1275 1302 * to add a packet count and a flag which
1276 1303 * tells the driver whether we want packets
1277 1304 * based on a count, or bytes, or all the
1278 1305 * packets queued in the driver/HW. This
1279 1306 * way, we never have to check the limits
1280 1307 * on poll path. We truly let only as many
1281 1308 * packets enter the system as we are willing
1282 1309 * to process or queue.
1283 1310 *
1284 1311 * Something along the lines of
1285 1312 * pkts_to_pickup = mac_soft_ring_max_q_cnt -
1286 1313 * mac_srs->srs_poll_pkt_cnt
1287 1314 */
1288 1315
1289 1316 /*
1290 1317 * Since we are not doing B/W control, pick
1291 1318 * as many packets as allowed.
1292 1319 */
1293 1320 bytes_to_pickup = max_bytes_to_pickup;
1294 1321 }
1295 1322
1296 1323 /* Poll the underlying Hardware */
1297 1324 mutex_exit(lock);
1298 1325 head = MAC_HWRING_POLL(mac_srs->srs_ring, (int)bytes_to_pickup);
1299 1326 mutex_enter(lock);
1300 1327
1301 1328 ASSERT((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
1302 1329 SRS_POLL_THR_OWNER);
1303 1330
1304 1331 mp = tail = head;
1305 1332 count = 0;
1306 1333 sz = 0;
1307 1334 while (mp != NULL) {
1308 1335 tail = mp;
1309 1336 sz += msgdsize(mp);
1310 1337 mp = mp->b_next;
1311 1338 count++;
1312 1339 }
1313 1340
1314 1341 if (head != NULL) {
1315 1342 tail->b_next = NULL;
1316 1343 smcip = mac_srs->srs_mcip;
1317 1344
1318 1345 SRS_RX_STAT_UPDATE(mac_srs, pollbytes, sz);
1319 1346 SRS_RX_STAT_UPDATE(mac_srs, pollcnt, count);
1320 1347
1321 1348 /*
1322 1349 * If there are any promiscuous mode callbacks
1323 1350 * defined for this MAC client, pass them a copy
1324 1351 * if appropriate and also update the counters.
1325 1352 */
1326 1353 if (smcip != NULL) {
1327 1354 if (smcip->mci_mip->mi_promisc_list != NULL) {
1328 1355 mutex_exit(lock);
1329 1356 mac_promisc_dispatch(smcip->mci_mip,
1330 1357 head, NULL);
1331 1358 mutex_enter(lock);
1332 1359 }
1333 1360 }
1334 1361 if (mac_srs->srs_type & SRST_BW_CONTROL) {
1335 1362 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1336 1363 mac_srs->srs_bw->mac_bw_polled += sz;
1337 1364 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1338 1365 }
1339 1366 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail,
1340 1367 count, sz);
1341 1368 if (count <= 10)
1342 1369 srs_rx->sr_stat.mrs_chaincntundr10++;
1343 1370 else if (count > 10 && count <= 50)
1344 1371 srs_rx->sr_stat.mrs_chaincnt10to50++;
1345 1372 else
1346 1373 srs_rx->sr_stat.mrs_chaincntover50++;
1347 1374 }
1348 1375
1349 1376 /*
1350 1377 * We are guaranteed that SRS_PROC will be set if we
1351 1378 * are here. Also, poll thread gets to run only if
1352 1379 * the drain was being done by a worker thread although
1353 1380 * its possible that worker thread is still running
1354 1381 * and poll thread was sent down to keep the pipeline
1355 1382 * going instead of doing a complete drain and then
1356 1383 * trying to poll the NIC.
1357 1384 *
1358 1385 * So we need to check SRS_WORKER flag to make sure
1359 1386 * that the worker thread is not processing the queue
1360 1387 * in parallel to us. The flags and conditions are
1361 1388 * protected by the srs_lock to prevent any race. We
1362 1389 * ensure that we don't drop the srs_lock from now
1363 1390 * till the end and similarly we don't drop the srs_lock
1364 1391 * in mac_rx_srs_drain() till similar condition check
1365 1392 * are complete. The mac_rx_srs_drain() needs to ensure
1366 1393 * that SRS_WORKER flag remains set as long as its
1367 1394 * processing the queue.
1368 1395 */
1369 1396 if (!(mac_srs->srs_state & SRS_WORKER) &&
1370 1397 (mac_srs->srs_first != NULL)) {
1371 1398 /*
1372 1399 * We have packets to process and worker thread
1373 1400 * is not running. Check to see if poll thread is
1374 1401 * allowed to process.
1375 1402 */
1376 1403 if (mac_srs->srs_state & SRS_LATENCY_OPT) {
1377 1404 mac_srs->srs_drain_func(mac_srs, SRS_POLL_PROC);
1378 1405 if (!(mac_srs->srs_state & SRS_PAUSE) &&
1379 1406 srs_rx->sr_poll_pkt_cnt <=
1380 1407 srs_rx->sr_lowat) {
1381 1408 srs_rx->sr_poll_again++;
1382 1409 goto check_again;
1383 1410 }
1384 1411 /*
1385 1412 * We are already above low water mark
1386 1413 * so stay in the polling mode but no
1387 1414 * need to poll. Once we dip below
1388 1415 * the polling threshold, the processing
1389 1416 * thread (soft ring) will signal us
1390 1417 * to poll again (MAC_UPDATE_SRS_COUNT)
1391 1418 */
1392 1419 srs_rx->sr_poll_drain_no_poll++;
1393 1420 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
1394 1421 /*
1395 1422 * In B/W control case, its possible
1396 1423 * that the backlog built up due to
1397 1424 * B/W limit being reached and packets
1398 1425 * are queued only in SRS. In this case,
1399 1426 * we should schedule worker thread
1400 1427 * since no one else will wake us up.
1401 1428 */
1402 1429 if ((mac_srs->srs_type & SRST_BW_CONTROL) &&
1403 1430 (mac_srs->srs_tid == NULL)) {
1404 1431 mac_srs->srs_tid =
1405 1432 timeout(mac_srs_fire, mac_srs, 1);
1406 1433 srs_rx->sr_poll_worker_wakeup++;
1407 1434 }
1408 1435 } else {
1409 1436 /*
1410 1437 * Wakeup the worker thread for more processing.
1411 1438 * We optimize for throughput in this case.
1412 1439 */
1413 1440 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
1414 1441 MAC_SRS_WORKER_WAKEUP(mac_srs);
1415 1442 srs_rx->sr_poll_sig_worker++;
1416 1443 }
1417 1444 } else if ((mac_srs->srs_first == NULL) &&
1418 1445 !(mac_srs->srs_state & SRS_WORKER)) {
1419 1446 /*
1420 1447 * There is nothing queued in SRS and
1421 1448 * no worker thread running. Plus we
1422 1449 * didn't get anything from the H/W
1423 1450 * as well (head == NULL);
1424 1451 */
1425 1452 ASSERT(head == NULL);
1426 1453 mac_srs->srs_state &=
1427 1454 ~(SRS_PROC|SRS_GET_PKTS);
1428 1455
1429 1456 /*
1430 1457 * If we have a packets in soft ring, don't allow
1431 1458 * more packets to come into this SRS by keeping the
1432 1459 * interrupts off but not polling the H/W. The
1433 1460 * poll thread will get signaled as soon as
1434 1461 * srs_poll_pkt_cnt dips below poll threshold.
1435 1462 */
1436 1463 if (srs_rx->sr_poll_pkt_cnt == 0) {
1437 1464 srs_rx->sr_poll_intr_enable++;
1438 1465 MAC_SRS_POLLING_OFF(mac_srs);
1439 1466 } else {
1440 1467 /*
1441 1468 * We know nothing is queued in SRS
1442 1469 * since we are here after checking
1443 1470 * srs_first is NULL. The backlog
1444 1471 * is entirely due to packets queued
1445 1472 * in Soft ring which will wake us up
1446 1473 * and get the interface out of polling
1447 1474 * mode once the backlog dips below
1448 1475 * sr_poll_thres.
1449 1476 */
1450 1477 srs_rx->sr_poll_no_poll++;
1451 1478 }
1452 1479 } else {
1453 1480 /*
1454 1481 * Worker thread is already running.
1455 1482 * Nothing much to do. If the polling
1456 1483 * was enabled, worker thread will deal
1457 1484 * with that.
1458 1485 */
1459 1486 mac_srs->srs_state &= ~SRS_GET_PKTS;
1460 1487 srs_rx->sr_poll_goto_sleep++;
1461 1488 }
1462 1489 }
1463 1490 done:
1464 1491 mac_srs->srs_state |= SRS_POLL_THR_QUIESCED;
1465 1492 cv_signal(&mac_srs->srs_async);
1466 1493 /*
1467 1494 * If this is a temporary quiesce then wait for the restart signal
1468 1495 * from the srs worker. Then clear the flags and signal the srs worker
1469 1496 * to ensure a positive handshake and go back to start.
1470 1497 */
1471 1498 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_POLL_THR_RESTART)))
1472 1499 cv_wait(async, lock);
1473 1500 if (mac_srs->srs_state & SRS_POLL_THR_RESTART) {
1474 1501 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
1475 1502 mac_srs->srs_state &=
1476 1503 ~(SRS_POLL_THR_QUIESCED | SRS_POLL_THR_RESTART);
1477 1504 cv_signal(&mac_srs->srs_async);
1478 1505 goto start;
1479 1506 } else {
1480 1507 mac_srs->srs_state |= SRS_POLL_THR_EXITED;
1481 1508 cv_signal(&mac_srs->srs_async);
1482 1509 CALLB_CPR_EXIT(&cprinfo);
1483 1510 thread_exit();
1484 1511 }
1485 1512 }
1486 1513
1487 1514 /*
1488 1515 * mac_srs_pick_chain
1489 1516 *
1490 1517 * In Bandwidth control case, checks how many packets can be processed
1491 1518 * and return them in a sub chain.
1492 1519 */
1493 1520 static mblk_t *
1494 1521 mac_srs_pick_chain(mac_soft_ring_set_t *mac_srs, mblk_t **chain_tail,
1495 1522 size_t *chain_sz, int *chain_cnt)
1496 1523 {
1497 1524 mblk_t *head = NULL;
1498 1525 mblk_t *tail = NULL;
1499 1526 size_t sz;
1500 1527 size_t tsz = 0;
1501 1528 int cnt = 0;
1502 1529 mblk_t *mp;
1503 1530
1504 1531 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1505 1532 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1506 1533 if (((mac_srs->srs_bw->mac_bw_used + mac_srs->srs_size) <=
1507 1534 mac_srs->srs_bw->mac_bw_limit) ||
1508 1535 (mac_srs->srs_bw->mac_bw_limit == 0)) {
1509 1536 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1510 1537 head = mac_srs->srs_first;
1511 1538 mac_srs->srs_first = NULL;
1512 1539 *chain_tail = mac_srs->srs_last;
1513 1540 mac_srs->srs_last = NULL;
1514 1541 *chain_sz = mac_srs->srs_size;
1515 1542 *chain_cnt = mac_srs->srs_count;
1516 1543 mac_srs->srs_count = 0;
1517 1544 mac_srs->srs_size = 0;
1518 1545 return (head);
1519 1546 }
1520 1547
1521 1548 /*
1522 1549 * Can't clear the entire backlog.
1523 1550 * Need to find how many packets to pick
1524 1551 */
1525 1552 ASSERT(MUTEX_HELD(&mac_srs->srs_bw->mac_bw_lock));
1526 1553 while ((mp = mac_srs->srs_first) != NULL) {
1527 1554 sz = msgdsize(mp);
1528 1555 if ((tsz + sz + mac_srs->srs_bw->mac_bw_used) >
1529 1556 mac_srs->srs_bw->mac_bw_limit) {
1530 1557 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED))
1531 1558 mac_srs->srs_bw->mac_bw_state |=
1532 1559 SRS_BW_ENFORCED;
1533 1560 break;
1534 1561 }
1535 1562
1536 1563 /*
1537 1564 * The _size & cnt is decremented from the softrings
1538 1565 * when they send up the packet for polling to work
1539 1566 * properly.
1540 1567 */
1541 1568 tsz += sz;
1542 1569 cnt++;
1543 1570 mac_srs->srs_count--;
1544 1571 mac_srs->srs_size -= sz;
1545 1572 if (tail != NULL)
1546 1573 tail->b_next = mp;
1547 1574 else
1548 1575 head = mp;
1549 1576 tail = mp;
1550 1577 mac_srs->srs_first = mac_srs->srs_first->b_next;
1551 1578 }
1552 1579 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1553 1580 if (mac_srs->srs_first == NULL)
1554 1581 mac_srs->srs_last = NULL;
1555 1582
1556 1583 if (tail != NULL)
1557 1584 tail->b_next = NULL;
1558 1585 *chain_tail = tail;
1559 1586 *chain_cnt = cnt;
1560 1587 *chain_sz = tsz;
1561 1588
1562 1589 return (head);
1563 1590 }
1564 1591
1565 1592 /*
1566 1593 * mac_rx_srs_drain
1567 1594 *
1568 1595 * The SRS drain routine. Gets to run to clear the queue. Any thread
1569 1596 * (worker, interrupt, poll) can call this based on processing model.
1570 1597 * The first thing we do is disable interrupts if possible and then
1571 1598 * drain the queue. we also try to poll the underlying hardware if
1572 1599 * there is a dedicated hardware Rx ring assigned to this SRS.
1573 1600 *
1574 1601 * There is a equivalent drain routine in bandwidth control mode
1575 1602 * mac_rx_srs_drain_bw. There is some code duplication between the two
1576 1603 * routines but they are highly performance sensitive and are easier
1577 1604 * to read/debug if they stay separate. Any code changes here might
1578 1605 * also apply to mac_rx_srs_drain_bw as well.
1579 1606 */
1580 1607 void
1581 1608 mac_rx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
1582 1609 {
1583 1610 mblk_t *head;
1584 1611 mblk_t *tail;
1585 1612 timeout_id_t tid;
1586 1613 int cnt = 0;
1587 1614 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1588 1615 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
1589 1616
1590 1617 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1591 1618 ASSERT(!(mac_srs->srs_type & SRST_BW_CONTROL));
1592 1619
1593 1620 /* If we are blanked i.e. can't do upcalls, then we are done */
1594 1621 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
1595 1622 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
1596 1623 (mac_srs->srs_state & SRS_PAUSE));
1597 1624 goto out;
1598 1625 }
1599 1626
1600 1627 if (mac_srs->srs_first == NULL)
1601 1628 goto out;
1602 1629
1603 1630 if (!(mac_srs->srs_state & SRS_LATENCY_OPT) &&
1604 1631 (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)) {
1605 1632 /*
1606 1633 * In the normal case, the SRS worker thread does no
1607 1634 * work and we wait for a backlog to build up before
1608 1635 * we switch into polling mode. In case we are
1609 1636 * optimizing for throughput, we use the worker thread
1610 1637 * as well. The goal is to let worker thread process
1611 1638 * the queue and poll thread to feed packets into
1612 1639 * the queue. As such, we should signal the poll
1613 1640 * thread to try and get more packets.
1614 1641 *
1615 1642 * We could have pulled this check in the POLL_RING
1616 1643 * macro itself but keeping it explicit here makes
1617 1644 * the architecture more human understandable.
1618 1645 */
1619 1646 MAC_SRS_POLL_RING(mac_srs);
1620 1647 }
1621 1648
1622 1649 again:
1623 1650 head = mac_srs->srs_first;
1624 1651 mac_srs->srs_first = NULL;
1625 1652 tail = mac_srs->srs_last;
1626 1653 mac_srs->srs_last = NULL;
1627 1654 cnt = mac_srs->srs_count;
1628 1655 mac_srs->srs_count = 0;
1629 1656
1630 1657 ASSERT(head != NULL);
1631 1658 ASSERT(tail != NULL);
1632 1659
1633 1660 if ((tid = mac_srs->srs_tid) != 0)
1634 1661 mac_srs->srs_tid = 0;
1635 1662
1636 1663 mac_srs->srs_state |= (SRS_PROC|proc_type);
1637 1664
1638 1665
1639 1666 /*
1640 1667 * mcip is NULL for broadcast and multicast flows. The promisc
1641 1668 * callbacks for broadcast and multicast packets are delivered from
1642 1669 * mac_rx() and we don't need to worry about that case in this path
1643 1670 */
1644 1671 if (mcip != NULL) {
1645 1672 if (mcip->mci_promisc_list != NULL) {
1646 1673 mutex_exit(&mac_srs->srs_lock);
1647 1674 mac_promisc_client_dispatch(mcip, head);
1648 1675 mutex_enter(&mac_srs->srs_lock);
1649 1676 }
1650 1677 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
1651 1678 mutex_exit(&mac_srs->srs_lock);
1652 1679 mac_protect_intercept_dhcp(mcip, head);
1653 1680 mutex_enter(&mac_srs->srs_lock);
1654 1681 }
1655 1682 }
1656 1683
1657 1684 /*
1658 1685 * Check if SRS itself is doing the processing
1659 1686 * This direct path does not apply when subflows are present. In this
1660 1687 * case, packets need to be dispatched to a soft ring according to the
1661 1688 * flow's bandwidth and other resources contraints.
1662 1689 */
1663 1690 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
1664 1691 mac_direct_rx_t proc;
1665 1692 void *arg1;
1666 1693 mac_resource_handle_t arg2;
1667 1694
1668 1695 /*
1669 1696 * This is the case when a Rx is directly
1670 1697 * assigned and we have a fully classified
1671 1698 * protocol chain. We can deal with it in
1672 1699 * one shot.
1673 1700 */
1674 1701 proc = srs_rx->sr_func;
1675 1702 arg1 = srs_rx->sr_arg1;
1676 1703 arg2 = srs_rx->sr_arg2;
1677 1704
1678 1705 mac_srs->srs_state |= SRS_CLIENT_PROC;
1679 1706 mutex_exit(&mac_srs->srs_lock);
1680 1707 if (tid != 0) {
1681 1708 (void) untimeout(tid);
1682 1709 tid = 0;
1683 1710 }
1684 1711
1685 1712 proc(arg1, arg2, head, NULL);
1686 1713 /*
1687 1714 * Decrement the size and count here itelf
1688 1715 * since the packet has been processed.
1689 1716 */
1690 1717 mutex_enter(&mac_srs->srs_lock);
1691 1718 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
1692 1719 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
1693 1720 cv_signal(&mac_srs->srs_client_cv);
1694 1721 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
1695 1722 } else {
1696 1723 /* Some kind of softrings based fanout is required */
1697 1724 mutex_exit(&mac_srs->srs_lock);
1698 1725 if (tid != 0) {
1699 1726 (void) untimeout(tid);
1700 1727 tid = 0;
1701 1728 }
1702 1729
1703 1730 /*
1704 1731 * Since the fanout routines can deal with chains,
1705 1732 * shoot the entire chain up.
1706 1733 */
1707 1734 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
1708 1735 mac_rx_srs_fanout(mac_srs, head);
1709 1736 else
1710 1737 mac_rx_srs_proto_fanout(mac_srs, head);
1711 1738 mutex_enter(&mac_srs->srs_lock);
1712 1739 }
1713 1740
1714 1741 if (!(mac_srs->srs_state & (SRS_BLANK|SRS_PAUSE)) &&
1715 1742 (mac_srs->srs_first != NULL)) {
1716 1743 /*
1717 1744 * More packets arrived while we were clearing the
1718 1745 * SRS. This can be possible because of one of
1719 1746 * three conditions below:
1720 1747 * 1) The driver is using multiple worker threads
1721 1748 * to send the packets to us.
1722 1749 * 2) The driver has a race in switching
1723 1750 * between interrupt and polling mode or
1724 1751 * 3) Packets are arriving in this SRS via the
1725 1752 * S/W classification as well.
1726 1753 *
1727 1754 * We should switch to polling mode and see if we
1728 1755 * need to send the poll thread down. Also, signal
1729 1756 * the worker thread to process whats just arrived.
1730 1757 */
1731 1758 MAC_SRS_POLLING_ON(mac_srs);
1732 1759 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) {
1733 1760 srs_rx->sr_drain_poll_sig++;
1734 1761 MAC_SRS_POLL_RING(mac_srs);
1735 1762 }
1736 1763
1737 1764 /*
1738 1765 * If we didn't signal the poll thread, we need
1739 1766 * to deal with the pending packets ourselves.
1740 1767 */
1741 1768 if (proc_type == SRS_WORKER) {
1742 1769 srs_rx->sr_drain_again++;
1743 1770 goto again;
1744 1771 } else {
1745 1772 srs_rx->sr_drain_worker_sig++;
1746 1773 cv_signal(&mac_srs->srs_async);
1747 1774 }
1748 1775 }
1749 1776
1750 1777 out:
1751 1778 if (mac_srs->srs_state & SRS_GET_PKTS) {
1752 1779 /*
1753 1780 * Poll thread is already running. Leave the
1754 1781 * SRS_RPOC set and hand over the control to
1755 1782 * poll thread.
1756 1783 */
1757 1784 mac_srs->srs_state &= ~proc_type;
1758 1785 srs_rx->sr_drain_poll_running++;
1759 1786 return;
1760 1787 }
1761 1788
1762 1789 /*
1763 1790 * Even if there are no packets queued in SRS, we
1764 1791 * need to make sure that the shared counter is
1765 1792 * clear and any associated softrings have cleared
1766 1793 * all the backlog. Otherwise, leave the interface
1767 1794 * in polling mode and the poll thread will get
1768 1795 * signalled once the count goes down to zero.
1769 1796 *
1770 1797 * If someone is already draining the queue (SRS_PROC is
1771 1798 * set) when the srs_poll_pkt_cnt goes down to zero,
1772 1799 * then it means that drain is already running and we
1773 1800 * will turn off polling at that time if there is
1774 1801 * no backlog.
1775 1802 *
1776 1803 * As long as there are packets queued either
1777 1804 * in soft ring set or its soft rings, we will leave
1778 1805 * the interface in polling mode (even if the drain
1779 1806 * was done being the interrupt thread). We signal
1780 1807 * the poll thread as well if we have dipped below
1781 1808 * low water mark.
1782 1809 *
1783 1810 * NOTE: We can't use the MAC_SRS_POLLING_ON macro
1784 1811 * since that turn polling on only for worker thread.
1785 1812 * Its not worth turning polling on for interrupt
1786 1813 * thread (since NIC will not issue another interrupt)
1787 1814 * unless a backlog builds up.
1788 1815 */
1789 1816 if ((srs_rx->sr_poll_pkt_cnt > 0) &&
1790 1817 (mac_srs->srs_state & SRS_POLLING_CAPAB)) {
1791 1818 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
1792 1819 srs_rx->sr_drain_keep_polling++;
1793 1820 MAC_SRS_POLLING_ON(mac_srs);
1794 1821 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)
1795 1822 MAC_SRS_POLL_RING(mac_srs);
1796 1823 return;
1797 1824 }
1798 1825
1799 1826 /* Nothing else to do. Get out of poll mode */
1800 1827 MAC_SRS_POLLING_OFF(mac_srs);
1801 1828 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
1802 1829 srs_rx->sr_drain_finish_intr++;
1803 1830 }
1804 1831
1805 1832 /*
1806 1833 * mac_rx_srs_drain_bw
1807 1834 *
1808 1835 * The SRS BW drain routine. Gets to run to clear the queue. Any thread
1809 1836 * (worker, interrupt, poll) can call this based on processing model.
1810 1837 * The first thing we do is disable interrupts if possible and then
1811 1838 * drain the queue. we also try to poll the underlying hardware if
1812 1839 * there is a dedicated hardware Rx ring assigned to this SRS.
1813 1840 *
1814 1841 * There is a equivalent drain routine in non bandwidth control mode
1815 1842 * mac_rx_srs_drain. There is some code duplication between the two
1816 1843 * routines but they are highly performance sensitive and are easier
1817 1844 * to read/debug if they stay separate. Any code changes here might
1818 1845 * also apply to mac_rx_srs_drain as well.
1819 1846 */
1820 1847 void
1821 1848 mac_rx_srs_drain_bw(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
1822 1849 {
1823 1850 mblk_t *head;
1824 1851 mblk_t *tail;
1825 1852 timeout_id_t tid;
1826 1853 size_t sz = 0;
1827 1854 int cnt = 0;
1828 1855 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1829 1856 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
1830 1857 clock_t now;
1831 1858
1832 1859 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1833 1860 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
1834 1861 again:
1835 1862 /* Check if we are doing B/W control */
1836 1863 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1837 1864 now = ddi_get_lbolt();
1838 1865 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
1839 1866 mac_srs->srs_bw->mac_bw_curr_time = now;
1840 1867 mac_srs->srs_bw->mac_bw_used = 0;
1841 1868 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
1842 1869 mac_srs->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED;
1843 1870 } else if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) {
1844 1871 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1845 1872 goto done;
1846 1873 } else if (mac_srs->srs_bw->mac_bw_used >
1847 1874 mac_srs->srs_bw->mac_bw_limit) {
1848 1875 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
1849 1876 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1850 1877 goto done;
1851 1878 }
1852 1879 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1853 1880
1854 1881 /* If we are blanked i.e. can't do upcalls, then we are done */
1855 1882 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
1856 1883 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
1857 1884 (mac_srs->srs_state & SRS_PAUSE));
1858 1885 goto done;
1859 1886 }
1860 1887
1861 1888 sz = 0;
1862 1889 cnt = 0;
1863 1890 if ((head = mac_srs_pick_chain(mac_srs, &tail, &sz, &cnt)) == NULL) {
1864 1891 /*
1865 1892 * We couldn't pick up a single packet.
1866 1893 */
1867 1894 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1868 1895 if ((mac_srs->srs_bw->mac_bw_used == 0) &&
1869 1896 (mac_srs->srs_size != 0) &&
1870 1897 !(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
1871 1898 /*
1872 1899 * Seems like configured B/W doesn't
1873 1900 * even allow processing of 1 packet
1874 1901 * per tick.
1875 1902 *
1876 1903 * XXX: raise the limit to processing
1877 1904 * at least 1 packet per tick.
1878 1905 */
1879 1906 mac_srs->srs_bw->mac_bw_limit +=
1880 1907 mac_srs->srs_bw->mac_bw_limit;
1881 1908 mac_srs->srs_bw->mac_bw_drop_threshold +=
1882 1909 mac_srs->srs_bw->mac_bw_drop_threshold;
1883 1910 cmn_err(CE_NOTE, "mac_rx_srs_drain: srs(%p) "
1884 1911 "raised B/W limit to %d since not even a "
1885 1912 "single packet can be processed per "
1886 1913 "tick %d\n", (void *)mac_srs,
1887 1914 (int)mac_srs->srs_bw->mac_bw_limit,
1888 1915 (int)msgdsize(mac_srs->srs_first));
1889 1916 }
1890 1917 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1891 1918 goto done;
1892 1919 }
1893 1920
1894 1921 ASSERT(head != NULL);
1895 1922 ASSERT(tail != NULL);
1896 1923
1897 1924 /* zero bandwidth: drop all and return to interrupt mode */
1898 1925 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1899 1926 if (mac_srs->srs_bw->mac_bw_limit == 0) {
1900 1927 srs_rx->sr_stat.mrs_sdrops += cnt;
1901 1928 ASSERT(mac_srs->srs_bw->mac_bw_sz >= sz);
1902 1929 mac_srs->srs_bw->mac_bw_sz -= sz;
1903 1930 mac_srs->srs_bw->mac_bw_drop_bytes += sz;
1904 1931 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1905 1932 mac_pkt_drop(NULL, NULL, head, B_FALSE);
1906 1933 goto leave_poll;
1907 1934 } else {
1908 1935 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1909 1936 }
1910 1937
1911 1938 if ((tid = mac_srs->srs_tid) != 0)
1912 1939 mac_srs->srs_tid = 0;
1913 1940
1914 1941 mac_srs->srs_state |= (SRS_PROC|proc_type);
1915 1942 MAC_SRS_WORKER_POLLING_ON(mac_srs);
1916 1943
1917 1944 /*
1918 1945 * mcip is NULL for broadcast and multicast flows. The promisc
1919 1946 * callbacks for broadcast and multicast packets are delivered from
1920 1947 * mac_rx() and we don't need to worry about that case in this path
1921 1948 */
1922 1949 if (mcip != NULL) {
1923 1950 if (mcip->mci_promisc_list != NULL) {
1924 1951 mutex_exit(&mac_srs->srs_lock);
1925 1952 mac_promisc_client_dispatch(mcip, head);
1926 1953 mutex_enter(&mac_srs->srs_lock);
1927 1954 }
1928 1955 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
1929 1956 mutex_exit(&mac_srs->srs_lock);
1930 1957 mac_protect_intercept_dhcp(mcip, head);
1931 1958 mutex_enter(&mac_srs->srs_lock);
1932 1959 }
1933 1960 }
1934 1961
1935 1962 /*
1936 1963 * Check if SRS itself is doing the processing
1937 1964 * This direct path does not apply when subflows are present. In this
1938 1965 * case, packets need to be dispatched to a soft ring according to the
1939 1966 * flow's bandwidth and other resources contraints.
1940 1967 */
1941 1968 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
1942 1969 mac_direct_rx_t proc;
1943 1970 void *arg1;
1944 1971 mac_resource_handle_t arg2;
1945 1972
1946 1973 /*
1947 1974 * This is the case when a Rx is directly
1948 1975 * assigned and we have a fully classified
1949 1976 * protocol chain. We can deal with it in
1950 1977 * one shot.
1951 1978 */
1952 1979 proc = srs_rx->sr_func;
1953 1980 arg1 = srs_rx->sr_arg1;
1954 1981 arg2 = srs_rx->sr_arg2;
1955 1982
1956 1983 mac_srs->srs_state |= SRS_CLIENT_PROC;
1957 1984 mutex_exit(&mac_srs->srs_lock);
1958 1985 if (tid != 0) {
1959 1986 (void) untimeout(tid);
1960 1987 tid = 0;
1961 1988 }
1962 1989
1963 1990 proc(arg1, arg2, head, NULL);
1964 1991 /*
1965 1992 * Decrement the size and count here itelf
1966 1993 * since the packet has been processed.
1967 1994 */
1968 1995 mutex_enter(&mac_srs->srs_lock);
1969 1996 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
1970 1997 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
1971 1998
1972 1999 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
1973 2000 cv_signal(&mac_srs->srs_client_cv);
1974 2001 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
1975 2002 } else {
1976 2003 /* Some kind of softrings based fanout is required */
1977 2004 mutex_exit(&mac_srs->srs_lock);
1978 2005 if (tid != 0) {
1979 2006 (void) untimeout(tid);
1980 2007 tid = 0;
1981 2008 }
1982 2009
1983 2010 /*
1984 2011 * Since the fanout routines can deal with chains,
1985 2012 * shoot the entire chain up.
1986 2013 */
1987 2014 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
1988 2015 mac_rx_srs_fanout(mac_srs, head);
1989 2016 else
1990 2017 mac_rx_srs_proto_fanout(mac_srs, head);
1991 2018 mutex_enter(&mac_srs->srs_lock);
1992 2019 }
1993 2020
1994 2021 /*
1995 2022 * Send the poll thread to pick up any packets arrived
1996 2023 * so far. This also serves as the last check in case
1997 2024 * nothing else is queued in the SRS. The poll thread
1998 2025 * is signalled only in the case the drain was done
1999 2026 * by the worker thread and SRS_WORKER is set. The
2000 2027 * worker thread can run in parallel as long as the
2001 2028 * SRS_WORKER flag is set. We we have nothing else to
2002 2029 * process, we can exit while leaving SRS_PROC set
2003 2030 * which gives the poll thread control to process and
2004 2031 * cleanup once it returns from the NIC.
2005 2032 *
2006 2033 * If we have nothing else to process, we need to
2007 2034 * ensure that we keep holding the srs_lock till
2008 2035 * all the checks below are done and control is
2009 2036 * handed to the poll thread if it was running.
2010 2037 */
2011 2038 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2012 2039 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2013 2040 if (mac_srs->srs_first != NULL) {
2014 2041 if (proc_type == SRS_WORKER) {
2015 2042 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2016 2043 if (srs_rx->sr_poll_pkt_cnt <=
2017 2044 srs_rx->sr_lowat)
2018 2045 MAC_SRS_POLL_RING(mac_srs);
2019 2046 goto again;
2020 2047 } else {
2021 2048 cv_signal(&mac_srs->srs_async);
2022 2049 }
2023 2050 }
2024 2051 }
2025 2052 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2026 2053
2027 2054 done:
2028 2055
2029 2056 if (mac_srs->srs_state & SRS_GET_PKTS) {
2030 2057 /*
2031 2058 * Poll thread is already running. Leave the
2032 2059 * SRS_RPOC set and hand over the control to
2033 2060 * poll thread.
2034 2061 */
2035 2062 mac_srs->srs_state &= ~proc_type;
2036 2063 return;
2037 2064 }
2038 2065
2039 2066 /*
2040 2067 * If we can't process packets because we have exceeded
2041 2068 * B/W limit for this tick, just set the timeout
2042 2069 * and leave.
2043 2070 *
2044 2071 * Even if there are no packets queued in SRS, we
2045 2072 * need to make sure that the shared counter is
2046 2073 * clear and any associated softrings have cleared
2047 2074 * all the backlog. Otherwise, leave the interface
2048 2075 * in polling mode and the poll thread will get
2049 2076 * signalled once the count goes down to zero.
2050 2077 *
2051 2078 * If someone is already draining the queue (SRS_PROC is
2052 2079 * set) when the srs_poll_pkt_cnt goes down to zero,
2053 2080 * then it means that drain is already running and we
2054 2081 * will turn off polling at that time if there is
2055 2082 * no backlog. As long as there are packets queued either
2056 2083 * is soft ring set or its soft rings, we will leave
2057 2084 * the interface in polling mode.
2058 2085 */
2059 2086 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2060 2087 if ((mac_srs->srs_state & SRS_POLLING_CAPAB) &&
2061 2088 ((mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) ||
2062 2089 (srs_rx->sr_poll_pkt_cnt > 0))) {
2063 2090 MAC_SRS_POLLING_ON(mac_srs);
2064 2091 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2065 2092 if ((mac_srs->srs_first != NULL) &&
2066 2093 (mac_srs->srs_tid == NULL))
2067 2094 mac_srs->srs_tid = timeout(mac_srs_fire,
2068 2095 mac_srs, 1);
2069 2096 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2070 2097 return;
2071 2098 }
2072 2099 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2073 2100
2074 2101 leave_poll:
2075 2102
2076 2103 /* Nothing else to do. Get out of poll mode */
2077 2104 MAC_SRS_POLLING_OFF(mac_srs);
2078 2105 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2079 2106 }
2080 2107
2081 2108 /*
2082 2109 * mac_srs_worker
2083 2110 *
2084 2111 * The SRS worker routine. Drains the queue when no one else is
2085 2112 * processing it.
2086 2113 */
2087 2114 void
2088 2115 mac_srs_worker(mac_soft_ring_set_t *mac_srs)
2089 2116 {
2090 2117 kmutex_t *lock = &mac_srs->srs_lock;
2091 2118 kcondvar_t *async = &mac_srs->srs_async;
2092 2119 callb_cpr_t cprinfo;
2093 2120 boolean_t bw_ctl_flag;
2094 2121
2095 2122 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "srs_worker");
2096 2123 mutex_enter(lock);
2097 2124
2098 2125 start:
2099 2126 for (;;) {
2100 2127 bw_ctl_flag = B_FALSE;
2101 2128 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2102 2129 MAC_SRS_BW_LOCK(mac_srs);
2103 2130 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
2104 2131 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
2105 2132 bw_ctl_flag = B_TRUE;
2106 2133 MAC_SRS_BW_UNLOCK(mac_srs);
2107 2134 }
2108 2135 /*
2109 2136 * The SRS_BW_ENFORCED flag may change since we have dropped
2110 2137 * the mac_bw_lock. However the drain function can handle both
2111 2138 * a drainable SRS or a bandwidth controlled SRS, and the
2112 2139 * effect of scheduling a timeout is to wakeup the worker
2113 2140 * thread which in turn will call the drain function. Since
2114 2141 * we release the srs_lock atomically only in the cv_wait there
2115 2142 * isn't a fear of waiting for ever.
2116 2143 */
2117 2144 while (((mac_srs->srs_state & SRS_PROC) ||
2118 2145 (mac_srs->srs_first == NULL) || bw_ctl_flag ||
2119 2146 (mac_srs->srs_state & SRS_TX_BLOCKED)) &&
2120 2147 !(mac_srs->srs_state & SRS_PAUSE)) {
2121 2148 /*
2122 2149 * If we have packets queued and we are here
2123 2150 * because B/W control is in place, we better
2124 2151 * schedule the worker wakeup after 1 tick
2125 2152 * to see if bandwidth control can be relaxed.
2126 2153 */
2127 2154 if (bw_ctl_flag && mac_srs->srs_tid == NULL) {
2128 2155 /*
2129 2156 * We need to ensure that a timer is already
2130 2157 * scheduled or we force schedule one for
2131 2158 * later so that we can continue processing
2132 2159 * after this quanta is over.
2133 2160 */
2134 2161 mac_srs->srs_tid = timeout(mac_srs_fire,
2135 2162 mac_srs, 1);
2136 2163 }
2137 2164 wait:
2138 2165 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2139 2166 cv_wait(async, lock);
2140 2167 CALLB_CPR_SAFE_END(&cprinfo, lock);
2141 2168
2142 2169 if (mac_srs->srs_state & SRS_PAUSE)
2143 2170 goto done;
2144 2171 if (mac_srs->srs_state & SRS_PROC)
2145 2172 goto wait;
2146 2173
2147 2174 if (mac_srs->srs_first != NULL &&
2148 2175 mac_srs->srs_type & SRST_BW_CONTROL) {
2149 2176 MAC_SRS_BW_LOCK(mac_srs);
2150 2177 if (mac_srs->srs_bw->mac_bw_state &
2151 2178 SRS_BW_ENFORCED) {
2152 2179 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
2153 2180 }
2154 2181 bw_ctl_flag = mac_srs->srs_bw->mac_bw_state &
2155 2182 SRS_BW_ENFORCED;
2156 2183 MAC_SRS_BW_UNLOCK(mac_srs);
2157 2184 }
2158 2185 }
2159 2186
2160 2187 if (mac_srs->srs_state & SRS_PAUSE)
2161 2188 goto done;
2162 2189 mac_srs->srs_drain_func(mac_srs, SRS_WORKER);
2163 2190 }
2164 2191 done:
2165 2192 /*
2166 2193 * The Rx SRS quiesce logic first cuts off packet supply to the SRS
2167 2194 * from both hard and soft classifications and waits for such threads
2168 2195 * to finish before signaling the worker. So at this point the only
2169 2196 * thread left that could be competing with the worker is the poll
2170 2197 * thread. In the case of Tx, there shouldn't be any thread holding
2171 2198 * SRS_PROC at this point.
2172 2199 */
2173 2200 if (!(mac_srs->srs_state & SRS_PROC)) {
2174 2201 mac_srs->srs_state |= SRS_PROC;
2175 2202 } else {
2176 2203 ASSERT((mac_srs->srs_type & SRST_TX) == 0);
2177 2204 /*
2178 2205 * Poll thread still owns the SRS and is still running
2179 2206 */
2180 2207 ASSERT((mac_srs->srs_poll_thr == NULL) ||
2181 2208 ((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
2182 2209 SRS_POLL_THR_OWNER));
2183 2210 }
2184 2211 mac_srs_worker_quiesce(mac_srs);
2185 2212 /*
2186 2213 * Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator
2187 2214 * of the quiesce operation
2188 2215 */
2189 2216 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_RESTART)))
2190 2217 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
2191 2218
2192 2219 if (mac_srs->srs_state & SRS_RESTART) {
2193 2220 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
2194 2221 mac_srs_worker_restart(mac_srs);
2195 2222 mac_srs->srs_state &= ~SRS_PROC;
2196 2223 goto start;
2197 2224 }
2198 2225
2199 2226 if (!(mac_srs->srs_state & SRS_CONDEMNED_DONE))
2200 2227 mac_srs_worker_quiesce(mac_srs);
2201 2228
2202 2229 mac_srs->srs_state &= ~SRS_PROC;
2203 2230 /* The macro drops the srs_lock */
2204 2231 CALLB_CPR_EXIT(&cprinfo);
2205 2232 thread_exit();
2206 2233 }
2207 2234
2208 2235 /*
2209 2236 * mac_rx_srs_subflow_process
2210 2237 *
2211 2238 * Receive side routine called from interrupt path when there are
2212 2239 * sub flows present on this SRS.
2213 2240 */
2214 2241 /* ARGSUSED */
2215 2242 void
2216 2243 mac_rx_srs_subflow_process(void *arg, mac_resource_handle_t srs,
2217 2244 mblk_t *mp_chain, boolean_t loopback)
2218 2245 {
2219 2246 flow_entry_t *flent = NULL;
2220 2247 flow_entry_t *prev_flent = NULL;
2221 2248 mblk_t *mp = NULL;
2222 2249 mblk_t *tail = NULL;
2223 2250 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
2224 2251 mac_client_impl_t *mcip;
2225 2252
2226 2253 mcip = mac_srs->srs_mcip;
2227 2254 ASSERT(mcip != NULL);
2228 2255
2229 2256 /*
2230 2257 * We need to determine the SRS for every packet
2231 2258 * by walking the flow table, if we don't get any,
2232 2259 * then we proceed using the SRS we came with.
2233 2260 */
2234 2261 mp = tail = mp_chain;
2235 2262 while (mp != NULL) {
2236 2263
2237 2264 /*
2238 2265 * We will increment the stats for the mactching subflow.
2239 2266 * when we get the bytes/pkt count for the classified packets
2240 2267 * later in mac_rx_srs_process.
2241 2268 */
2242 2269 (void) mac_flow_lookup(mcip->mci_subflow_tab, mp,
2243 2270 FLOW_INBOUND, &flent);
2244 2271
2245 2272 if (mp == mp_chain || flent == prev_flent) {
2246 2273 if (prev_flent != NULL)
2247 2274 FLOW_REFRELE(prev_flent);
2248 2275 prev_flent = flent;
2249 2276 flent = NULL;
2250 2277 tail = mp;
2251 2278 mp = mp->b_next;
2252 2279 continue;
2253 2280 }
2254 2281 tail->b_next = NULL;
2255 2282 /*
2256 2283 * A null indicates, this is for the mac_srs itself.
2257 2284 * XXX-venu : probably assert for fe_rx_srs_cnt == 0.
2258 2285 */
2259 2286 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
2260 2287 mac_rx_srs_process(arg,
2261 2288 (mac_resource_handle_t)mac_srs, mp_chain,
2262 2289 loopback);
2263 2290 } else {
2264 2291 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
2265 2292 prev_flent->fe_cb_arg2, mp_chain, loopback);
2266 2293 FLOW_REFRELE(prev_flent);
2267 2294 }
2268 2295 prev_flent = flent;
2269 2296 flent = NULL;
2270 2297 mp_chain = mp;
2271 2298 tail = mp;
2272 2299 mp = mp->b_next;
2273 2300 }
2274 2301 /* Last chain */
2275 2302 ASSERT(mp_chain != NULL);
2276 2303 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
2277 2304 mac_rx_srs_process(arg,
2278 2305 (mac_resource_handle_t)mac_srs, mp_chain, loopback);
2279 2306 } else {
2280 2307 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
2281 2308 prev_flent->fe_cb_arg2, mp_chain, loopback);
2282 2309 FLOW_REFRELE(prev_flent);
2283 2310 }
2284 2311 }
2285 2312
2286 2313 /*
2287 2314 * mac_rx_srs_process
2288 2315 *
2289 2316 * Receive side routine called from the interrupt path.
2290 2317 *
2291 2318 * loopback is set to force a context switch on the loopback
2292 2319 * path between MAC clients.
2293 2320 */
2294 2321 /* ARGSUSED */
2295 2322 void
2296 2323 mac_rx_srs_process(void *arg, mac_resource_handle_t srs, mblk_t *mp_chain,
2297 2324 boolean_t loopback)
2298 2325 {
2299 2326 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
2300 2327 mblk_t *mp, *tail, *head;
2301 2328 int count = 0;
2302 2329 int count1;
2303 2330 size_t sz = 0;
2304 2331 size_t chain_sz, sz1;
2305 2332 mac_bw_ctl_t *mac_bw;
2306 2333 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2307 2334
2308 2335 /*
2309 2336 * Set the tail, count and sz. We set the sz irrespective
2310 2337 * of whether we are doing B/W control or not for the
2311 2338 * purpose of updating the stats.
2312 2339 */
2313 2340 mp = tail = mp_chain;
2314 2341 while (mp != NULL) {
2315 2342 tail = mp;
2316 2343 count++;
2317 2344 sz += msgdsize(mp);
2318 2345 mp = mp->b_next;
2319 2346 }
2320 2347
2321 2348 mutex_enter(&mac_srs->srs_lock);
2322 2349
2323 2350 if (loopback) {
2324 2351 SRS_RX_STAT_UPDATE(mac_srs, lclbytes, sz);
2325 2352 SRS_RX_STAT_UPDATE(mac_srs, lclcnt, count);
2326 2353
2327 2354 } else {
2328 2355 SRS_RX_STAT_UPDATE(mac_srs, intrbytes, sz);
2329 2356 SRS_RX_STAT_UPDATE(mac_srs, intrcnt, count);
2330 2357 }
2331 2358
2332 2359 /*
2333 2360 * If the SRS in already being processed; has been blanked;
2334 2361 * can be processed by worker thread only; or the B/W limit
2335 2362 * has been reached, then queue the chain and check if
2336 2363 * worker thread needs to be awakend.
2337 2364 */
2338 2365 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2339 2366 mac_bw = mac_srs->srs_bw;
2340 2367 ASSERT(mac_bw != NULL);
2341 2368 mutex_enter(&mac_bw->mac_bw_lock);
2342 2369 mac_bw->mac_bw_intr += sz;
2343 2370 if (mac_bw->mac_bw_limit == 0) {
2344 2371 /* zero bandwidth: drop all */
2345 2372 srs_rx->sr_stat.mrs_sdrops += count;
2346 2373 mac_bw->mac_bw_drop_bytes += sz;
2347 2374 mutex_exit(&mac_bw->mac_bw_lock);
2348 2375 mutex_exit(&mac_srs->srs_lock);
2349 2376 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
2350 2377 return;
2351 2378 } else {
2352 2379 if ((mac_bw->mac_bw_sz + sz) <=
2353 2380 mac_bw->mac_bw_drop_threshold) {
2354 2381 mutex_exit(&mac_bw->mac_bw_lock);
2355 2382 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain,
2356 2383 tail, count, sz);
2357 2384 } else {
2358 2385 mp = mp_chain;
2359 2386 chain_sz = 0;
2360 2387 count1 = 0;
2361 2388 tail = NULL;
2362 2389 head = NULL;
2363 2390 while (mp != NULL) {
2364 2391 sz1 = msgdsize(mp);
2365 2392 if (mac_bw->mac_bw_sz + chain_sz + sz1 >
2366 2393 mac_bw->mac_bw_drop_threshold)
2367 2394 break;
2368 2395 chain_sz += sz1;
2369 2396 count1++;
2370 2397 tail = mp;
2371 2398 mp = mp->b_next;
2372 2399 }
2373 2400 mutex_exit(&mac_bw->mac_bw_lock);
2374 2401 if (tail != NULL) {
2375 2402 head = tail->b_next;
2376 2403 tail->b_next = NULL;
2377 2404 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs,
2378 2405 mp_chain, tail, count1, chain_sz);
2379 2406 sz -= chain_sz;
2380 2407 count -= count1;
2381 2408 } else {
2382 2409 /* Can't pick up any */
2383 2410 head = mp_chain;
2384 2411 }
2385 2412 if (head != NULL) {
2386 2413 /* Drop any packet over the threshold */
2387 2414 srs_rx->sr_stat.mrs_sdrops += count;
2388 2415 mutex_enter(&mac_bw->mac_bw_lock);
2389 2416 mac_bw->mac_bw_drop_bytes += sz;
2390 2417 mutex_exit(&mac_bw->mac_bw_lock);
2391 2418 freemsgchain(head);
2392 2419 }
2393 2420 }
2394 2421 MAC_SRS_WORKER_WAKEUP(mac_srs);
2395 2422 mutex_exit(&mac_srs->srs_lock);
2396 2423 return;
2397 2424 }
2398 2425 }
2399 2426
2400 2427 /*
2401 2428 * If the total number of packets queued in the SRS and
2402 2429 * its associated soft rings exceeds the max allowed,
2403 2430 * then drop the chain. If we are polling capable, this
2404 2431 * shouldn't be happening.
2405 2432 */
2406 2433 if (!(mac_srs->srs_type & SRST_BW_CONTROL) &&
2407 2434 (srs_rx->sr_poll_pkt_cnt > srs_rx->sr_hiwat)) {
2408 2435 mac_bw = mac_srs->srs_bw;
2409 2436 srs_rx->sr_stat.mrs_sdrops += count;
2410 2437 mutex_enter(&mac_bw->mac_bw_lock);
2411 2438 mac_bw->mac_bw_drop_bytes += sz;
|
↓ open down ↓ |
1984 lines elided |
↑ open up ↑ |
2412 2439 mutex_exit(&mac_bw->mac_bw_lock);
2413 2440 freemsgchain(mp_chain);
2414 2441 mutex_exit(&mac_srs->srs_lock);
2415 2442 return;
2416 2443 }
2417 2444
2418 2445 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, tail, count, sz);
2419 2446
2420 2447 if (!(mac_srs->srs_state & SRS_PROC)) {
2421 2448 /*
2422 - * If we are coming via loopback or if we are not
2423 - * optimizing for latency, we should signal the
2424 - * worker thread.
2449 + * If we are coming via loopback, if we are not optimizing for
2450 + * latency, or if our stack is running deep, we should signal
2451 + * the worker thread.
2425 2452 */
2426 - if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT)) {
2453 + if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT) ||
2454 + MAC_RX_SRS_TOODEEP()) {
2427 2455 /*
2428 2456 * For loopback, We need to let the worker take
2429 2457 * over as we don't want to continue in the same
2430 2458 * thread even if we can. This could lead to stack
2431 2459 * overflows and may also end up using
2432 2460 * resources (cpu) incorrectly.
2433 2461 */
2434 2462 cv_signal(&mac_srs->srs_async);
2435 2463 } else {
2436 2464 /*
2437 2465 * Seems like no one is processing the SRS and
2438 2466 * there is no backlog. We also inline process
2439 2467 * our packet if its a single packet in non
2440 2468 * latency optimized case (in latency optimized
2441 2469 * case, we inline process chains of any size).
2442 2470 */
2443 2471 mac_srs->srs_drain_func(mac_srs, SRS_PROC_FAST);
2444 2472 }
2445 2473 }
2446 2474 mutex_exit(&mac_srs->srs_lock);
2447 2475 }
2448 2476
2449 2477 /* TX SIDE ROUTINES (RUNTIME) */
2450 2478
2451 2479 /*
2452 2480 * mac_tx_srs_no_desc
2453 2481 *
2454 2482 * This routine is called by Tx single ring default mode
2455 2483 * when Tx ring runs out of descs.
2456 2484 */
2457 2485 mac_tx_cookie_t
2458 2486 mac_tx_srs_no_desc(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2459 2487 uint16_t flag, mblk_t **ret_mp)
2460 2488 {
2461 2489 mac_tx_cookie_t cookie = NULL;
2462 2490 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2463 2491 boolean_t wakeup_worker = B_TRUE;
2464 2492 uint32_t tx_mode = srs_tx->st_mode;
2465 2493 int cnt, sz;
2466 2494 mblk_t *tail;
2467 2495
2468 2496 ASSERT(tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_BW);
2469 2497 if (flag & MAC_DROP_ON_NO_DESC) {
2470 2498 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2471 2499 } else {
2472 2500 if (mac_srs->srs_first != NULL)
2473 2501 wakeup_worker = B_FALSE;
2474 2502 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2475 2503 if (flag & MAC_TX_NO_ENQUEUE) {
2476 2504 /*
2477 2505 * If TX_QUEUED is not set, queue the
2478 2506 * packet and let mac_tx_srs_drain()
2479 2507 * set the TX_BLOCKED bit for the
2480 2508 * reasons explained above. Otherwise,
2481 2509 * return the mblks.
2482 2510 */
2483 2511 if (wakeup_worker) {
2484 2512 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2485 2513 mp_chain, tail, cnt, sz);
2486 2514 } else {
2487 2515 MAC_TX_SET_NO_ENQUEUE(mac_srs,
2488 2516 mp_chain, ret_mp, cookie);
2489 2517 }
2490 2518 } else {
2491 2519 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
2492 2520 tail, cnt, sz, cookie);
2493 2521 }
2494 2522 if (wakeup_worker)
2495 2523 cv_signal(&mac_srs->srs_async);
2496 2524 }
2497 2525 return (cookie);
2498 2526 }
2499 2527
2500 2528 /*
2501 2529 * mac_tx_srs_enqueue
2502 2530 *
2503 2531 * This routine is called when Tx SRS is operating in either serializer
2504 2532 * or bandwidth mode. In serializer mode, a packet will get enqueued
2505 2533 * when a thread cannot enter SRS exclusively. In bandwidth mode,
2506 2534 * packets gets queued if allowed byte-count limit for a tick is
2507 2535 * exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and
2508 2536 * MAC_TX_NO_ENQUEUE is set is different than when operaing in either
2509 2537 * the default mode or fanout mode. Here packets get dropped or
2510 2538 * returned back to the caller only after hi-watermark worth of data
2511 2539 * is queued.
2512 2540 */
2513 2541 static mac_tx_cookie_t
2514 2542 mac_tx_srs_enqueue(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2515 2543 uint16_t flag, uintptr_t fanout_hint, mblk_t **ret_mp)
2516 2544 {
2517 2545 mac_tx_cookie_t cookie = NULL;
2518 2546 int cnt, sz;
2519 2547 mblk_t *tail;
2520 2548 boolean_t wakeup_worker = B_TRUE;
2521 2549
2522 2550 /*
2523 2551 * Ignore fanout hint if we don't have multiple tx rings.
2524 2552 */
2525 2553 if (!MAC_TX_SOFT_RINGS(mac_srs))
2526 2554 fanout_hint = 0;
2527 2555
2528 2556 if (mac_srs->srs_first != NULL)
2529 2557 wakeup_worker = B_FALSE;
2530 2558 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2531 2559 if (flag & MAC_DROP_ON_NO_DESC) {
2532 2560 if (mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) {
2533 2561 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2534 2562 } else {
2535 2563 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2536 2564 mp_chain, tail, cnt, sz);
2537 2565 }
2538 2566 } else if (flag & MAC_TX_NO_ENQUEUE) {
2539 2567 if ((mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) ||
2540 2568 (mac_srs->srs_state & SRS_TX_WAKEUP_CLIENT)) {
2541 2569 MAC_TX_SET_NO_ENQUEUE(mac_srs, mp_chain,
2542 2570 ret_mp, cookie);
2543 2571 } else {
2544 2572 mp_chain->b_prev = (mblk_t *)fanout_hint;
2545 2573 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2546 2574 mp_chain, tail, cnt, sz);
2547 2575 }
2548 2576 } else {
2549 2577 /*
2550 2578 * If you are BW_ENFORCED, just enqueue the
2551 2579 * packet. srs_worker will drain it at the
2552 2580 * prescribed rate. Before enqueueing, save
2553 2581 * the fanout hint.
2554 2582 */
2555 2583 mp_chain->b_prev = (mblk_t *)fanout_hint;
2556 2584 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
2557 2585 tail, cnt, sz, cookie);
2558 2586 }
2559 2587 if (wakeup_worker)
2560 2588 cv_signal(&mac_srs->srs_async);
2561 2589 return (cookie);
2562 2590 }
2563 2591
2564 2592 /*
2565 2593 * There are seven tx modes:
2566 2594 *
2567 2595 * 1) Default mode (SRS_TX_DEFAULT)
2568 2596 * 2) Serialization mode (SRS_TX_SERIALIZE)
2569 2597 * 3) Fanout mode (SRS_TX_FANOUT)
2570 2598 * 4) Bandwdith mode (SRS_TX_BW)
2571 2599 * 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT)
2572 2600 * 6) aggr Tx mode (SRS_TX_AGGR)
2573 2601 * 7) aggr Tx bw mode (SRS_TX_BW_AGGR)
2574 2602 *
2575 2603 * The tx mode in which an SRS operates is decided in mac_tx_srs_setup()
2576 2604 * based on the number of Tx rings requested for an SRS and whether
2577 2605 * bandwidth control is requested or not.
2578 2606 *
2579 2607 * The default mode (i.e., no fanout/no bandwidth) is used when the
2580 2608 * underlying NIC does not have Tx rings or just one Tx ring. In this mode,
2581 2609 * the SRS acts as a pass-thru. Packets will go directly to mac_tx_send().
2582 2610 * When the underlying Tx ring runs out of Tx descs, it starts queueing up
2583 2611 * packets in SRS. When flow-control is relieved, the srs_worker drains
2584 2612 * the queued packets and informs blocked clients to restart sending
2585 2613 * packets.
2586 2614 *
2587 2615 * In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized. This
2588 2616 * mode is used when the link has no Tx rings or only one Tx ring.
2589 2617 *
2590 2618 * In the SRS_TX_FANOUT mode, packets will be fanned out to multiple
2591 2619 * Tx rings. Each Tx ring will have a soft ring associated with it.
2592 2620 * These soft rings will be hung off the Tx SRS. Queueing if it happens
2593 2621 * due to lack of Tx desc will be in individual soft ring (and not srs)
2594 2622 * associated with Tx ring.
2595 2623 *
2596 2624 * In the TX_BW mode, tx srs will allow packets to go down to Tx ring
2597 2625 * only if bw is available. Otherwise the packets will be queued in
2598 2626 * SRS. If fanout to multiple Tx rings is configured, the packets will
2599 2627 * be fanned out among the soft rings associated with the Tx rings.
2600 2628 *
2601 2629 * In SRS_TX_AGGR mode, mac_tx_aggr_mode() routine is called. This routine
2602 2630 * invokes an aggr function, aggr_find_tx_ring(), to find a pseudo Tx ring
2603 2631 * belonging to a port on which the packet has to be sent. Aggr will
2604 2632 * always have a pseudo Tx ring associated with it even when it is an
2605 2633 * aggregation over a single NIC that has no Tx rings. Even in such a
2606 2634 * case, the single pseudo Tx ring will have a soft ring associated with
2607 2635 * it and the soft ring will hang off the SRS.
2608 2636 *
2609 2637 * If a bandwidth is specified for an aggr, SRS_TX_BW_AGGR mode is used.
2610 2638 * In this mode, the bandwidth is first applied on the outgoing packets
2611 2639 * and later mac_tx_addr_mode() function is called to send the packet out
2612 2640 * of one of the pseudo Tx rings.
2613 2641 *
2614 2642 * Four flags are used in srs_state for indicating flow control
2615 2643 * conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT.
2616 2644 * SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the
2617 2645 * driver below.
2618 2646 * SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat
2619 2647 * and flow-control pressure is applied back to clients. The clients expect
2620 2648 * wakeup when flow-control is relieved.
2621 2649 * SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk
2622 2650 * got returned back to client either due to lack of Tx descs or due to bw
2623 2651 * control reasons. The clients expect a wakeup when condition is relieved.
2624 2652 *
2625 2653 * The fourth argument to mac_tx() is the flag. Normally it will be 0 but
2626 2654 * some clients set the following values too: MAC_DROP_ON_NO_DESC,
2627 2655 * MAC_TX_NO_ENQUEUE
2628 2656 * Mac clients that do not want packets to be enqueued in the mac layer set
2629 2657 * MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or
2630 2658 * Tx soft rings but instead get dropped when the NIC runs out of desc. The
2631 2659 * behaviour of this flag is different when the Tx is running in serializer
2632 2660 * or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet
2633 2661 * get dropped when Tx high watermark is reached.
2634 2662 * There are some mac clients like vsw, aggr that want the mblks to be
2635 2663 * returned back to clients instead of being queued in Tx SRS (or Tx soft
2636 2664 * rings) under flow-control (i.e., out of desc or exceeding bw limits)
2637 2665 * conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set.
2638 2666 * In the default and Tx fanout mode, the un-transmitted mblks will be
2639 2667 * returned back to the clients when the driver runs out of Tx descs.
2640 2668 * SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or
2641 2669 * soft ring) so that the clients can be woken up when Tx desc become
2642 2670 * available. When running in serializer or bandwidth mode mode,
2643 2671 * SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached.
2644 2672 */
2645 2673
2646 2674 mac_tx_func_t
2647 2675 mac_tx_get_func(uint32_t mode)
2648 2676 {
2649 2677 return (mac_tx_mode_list[mode].mac_tx_func);
2650 2678 }
2651 2679
2652 2680 /* ARGSUSED */
2653 2681 static mac_tx_cookie_t
2654 2682 mac_tx_single_ring_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2655 2683 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2656 2684 {
2657 2685 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2658 2686 mac_tx_stats_t stats;
2659 2687 mac_tx_cookie_t cookie = NULL;
2660 2688
2661 2689 ASSERT(srs_tx->st_mode == SRS_TX_DEFAULT);
2662 2690
2663 2691 /* Regular case with a single Tx ring */
2664 2692 /*
2665 2693 * SRS_TX_BLOCKED is set when underlying NIC runs
2666 2694 * out of Tx descs and messages start getting
2667 2695 * queued. It won't get reset until
2668 2696 * tx_srs_drain() completely drains out the
2669 2697 * messages.
2670 2698 */
2671 2699 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
2672 2700 /* Tx descs/resources not available */
2673 2701 mutex_enter(&mac_srs->srs_lock);
2674 2702 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
2675 2703 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain,
2676 2704 flag, ret_mp);
2677 2705 mutex_exit(&mac_srs->srs_lock);
2678 2706 return (cookie);
2679 2707 }
2680 2708 /*
2681 2709 * While we were computing mblk count, the
2682 2710 * flow control condition got relieved.
2683 2711 * Continue with the transmission.
2684 2712 */
2685 2713 mutex_exit(&mac_srs->srs_lock);
2686 2714 }
2687 2715
2688 2716 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2689 2717 mp_chain, &stats);
2690 2718
2691 2719 /*
2692 2720 * Multiple threads could be here sending packets.
2693 2721 * Under such conditions, it is not possible to
2694 2722 * automically set SRS_TX_BLOCKED bit to indicate
2695 2723 * out of tx desc condition. To atomically set
2696 2724 * this, we queue the returned packet and do
2697 2725 * the setting of SRS_TX_BLOCKED in
2698 2726 * mac_tx_srs_drain().
2699 2727 */
2700 2728 if (mp_chain != NULL) {
2701 2729 mutex_enter(&mac_srs->srs_lock);
2702 2730 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, flag, ret_mp);
2703 2731 mutex_exit(&mac_srs->srs_lock);
2704 2732 return (cookie);
2705 2733 }
2706 2734 SRS_TX_STATS_UPDATE(mac_srs, &stats);
2707 2735
2708 2736 return (NULL);
2709 2737 }
2710 2738
2711 2739 /*
2712 2740 * mac_tx_serialize_mode
2713 2741 *
2714 2742 * This is an experimental mode implemented as per the request of PAE.
2715 2743 * In this mode, all callers attempting to send a packet to the NIC
2716 2744 * will get serialized. Only one thread at any time will access the
2717 2745 * NIC to send the packet out.
2718 2746 */
2719 2747 /* ARGSUSED */
2720 2748 static mac_tx_cookie_t
2721 2749 mac_tx_serializer_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2722 2750 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2723 2751 {
2724 2752 mac_tx_stats_t stats;
2725 2753 mac_tx_cookie_t cookie = NULL;
2726 2754 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2727 2755
2728 2756 /* Single ring, serialize below */
2729 2757 ASSERT(srs_tx->st_mode == SRS_TX_SERIALIZE);
2730 2758 mutex_enter(&mac_srs->srs_lock);
2731 2759 if ((mac_srs->srs_first != NULL) ||
2732 2760 (mac_srs->srs_state & SRS_PROC)) {
2733 2761 /*
2734 2762 * In serialization mode, queue all packets until
2735 2763 * TX_HIWAT is set.
2736 2764 * If drop bit is set, drop if TX_HIWAT is set.
2737 2765 * If no_enqueue is set, still enqueue until hiwat
2738 2766 * is set and return mblks after TX_HIWAT is set.
2739 2767 */
2740 2768 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain,
2741 2769 flag, NULL, ret_mp);
2742 2770 mutex_exit(&mac_srs->srs_lock);
2743 2771 return (cookie);
2744 2772 }
2745 2773 /*
2746 2774 * No packets queued, nothing on proc and no flow
2747 2775 * control condition. Fast-path, ok. Do inline
2748 2776 * processing.
2749 2777 */
2750 2778 mac_srs->srs_state |= SRS_PROC;
2751 2779 mutex_exit(&mac_srs->srs_lock);
2752 2780
2753 2781 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2754 2782 mp_chain, &stats);
2755 2783
2756 2784 mutex_enter(&mac_srs->srs_lock);
2757 2785 mac_srs->srs_state &= ~SRS_PROC;
2758 2786 if (mp_chain != NULL) {
2759 2787 cookie = mac_tx_srs_enqueue(mac_srs,
2760 2788 mp_chain, flag, NULL, ret_mp);
2761 2789 }
2762 2790 if (mac_srs->srs_first != NULL) {
2763 2791 /*
2764 2792 * We processed inline our packet and a new
2765 2793 * packet/s got queued while we were
2766 2794 * processing. Wakeup srs worker
2767 2795 */
2768 2796 cv_signal(&mac_srs->srs_async);
2769 2797 }
2770 2798 mutex_exit(&mac_srs->srs_lock);
2771 2799
2772 2800 if (cookie == NULL)
2773 2801 SRS_TX_STATS_UPDATE(mac_srs, &stats);
2774 2802
2775 2803 return (cookie);
2776 2804 }
2777 2805
2778 2806 /*
2779 2807 * mac_tx_fanout_mode
2780 2808 *
2781 2809 * In this mode, the SRS will have access to multiple Tx rings to send
2782 2810 * the packet out. The fanout hint that is passed as an argument is
2783 2811 * used to find an appropriate ring to fanout the traffic. Each Tx
2784 2812 * ring, in turn, will have a soft ring associated with it. If a Tx
2785 2813 * ring runs out of Tx desc's the returned packet will be queued in
2786 2814 * the soft ring associated with that Tx ring. The srs itself will not
2787 2815 * queue any packets.
2788 2816 */
2789 2817
2790 2818 #define MAC_TX_SOFT_RING_PROCESS(chain) { \
2791 2819 index = COMPUTE_INDEX(hash, mac_srs->srs_tx_ring_count), \
2792 2820 softring = mac_srs->srs_tx_soft_rings[index]; \
2793 2821 cookie = mac_tx_soft_ring_process(softring, chain, flag, ret_mp); \
2794 2822 DTRACE_PROBE2(tx__fanout, uint64_t, hash, uint_t, index); \
2795 2823 }
2796 2824
2797 2825 static mac_tx_cookie_t
2798 2826 mac_tx_fanout_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2799 2827 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2800 2828 {
2801 2829 mac_soft_ring_t *softring;
2802 2830 uint64_t hash;
2803 2831 uint_t index;
2804 2832 mac_tx_cookie_t cookie = NULL;
2805 2833
2806 2834 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
2807 2835 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT);
2808 2836 if (fanout_hint != 0) {
2809 2837 /*
2810 2838 * The hint is specified by the caller, simply pass the
2811 2839 * whole chain to the soft ring.
2812 2840 */
2813 2841 hash = HASH_HINT(fanout_hint);
2814 2842 MAC_TX_SOFT_RING_PROCESS(mp_chain);
2815 2843 } else {
2816 2844 mblk_t *last_mp, *cur_mp, *sub_chain;
2817 2845 uint64_t last_hash = 0;
2818 2846 uint_t media = mac_srs->srs_mcip->mci_mip->mi_info.mi_media;
2819 2847
2820 2848 /*
2821 2849 * Compute the hash from the contents (headers) of the
2822 2850 * packets of the mblk chain. Split the chains into
2823 2851 * subchains of the same conversation.
2824 2852 *
2825 2853 * Since there may be more than one ring used for
2826 2854 * sub-chains of the same call, and since the caller
2827 2855 * does not maintain per conversation state since it
2828 2856 * passed a zero hint, unsent subchains will be
2829 2857 * dropped.
2830 2858 */
2831 2859
2832 2860 flag |= MAC_DROP_ON_NO_DESC;
2833 2861 ret_mp = NULL;
2834 2862
2835 2863 ASSERT(ret_mp == NULL);
2836 2864
2837 2865 sub_chain = NULL;
2838 2866 last_mp = NULL;
2839 2867
2840 2868 for (cur_mp = mp_chain; cur_mp != NULL;
2841 2869 cur_mp = cur_mp->b_next) {
2842 2870 hash = mac_pkt_hash(media, cur_mp, MAC_PKT_HASH_L4,
2843 2871 B_TRUE);
2844 2872 if (last_hash != 0 && hash != last_hash) {
2845 2873 /*
2846 2874 * Starting a different subchain, send current
2847 2875 * chain out.
2848 2876 */
2849 2877 ASSERT(last_mp != NULL);
2850 2878 last_mp->b_next = NULL;
2851 2879 MAC_TX_SOFT_RING_PROCESS(sub_chain);
2852 2880 sub_chain = NULL;
2853 2881 }
2854 2882
2855 2883 /* add packet to subchain */
2856 2884 if (sub_chain == NULL)
2857 2885 sub_chain = cur_mp;
2858 2886 last_mp = cur_mp;
2859 2887 last_hash = hash;
2860 2888 }
2861 2889
2862 2890 if (sub_chain != NULL) {
2863 2891 /* send last subchain */
2864 2892 ASSERT(last_mp != NULL);
2865 2893 last_mp->b_next = NULL;
2866 2894 MAC_TX_SOFT_RING_PROCESS(sub_chain);
2867 2895 }
2868 2896
2869 2897 cookie = NULL;
2870 2898 }
2871 2899
2872 2900 return (cookie);
2873 2901 }
2874 2902
2875 2903 /*
2876 2904 * mac_tx_bw_mode
2877 2905 *
2878 2906 * In the bandwidth mode, Tx srs will allow packets to go down to Tx ring
2879 2907 * only if bw is available. Otherwise the packets will be queued in
2880 2908 * SRS. If the SRS has multiple Tx rings, then packets will get fanned
2881 2909 * out to a Tx rings.
2882 2910 */
2883 2911 static mac_tx_cookie_t
2884 2912 mac_tx_bw_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2885 2913 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2886 2914 {
2887 2915 int cnt, sz;
2888 2916 mblk_t *tail;
2889 2917 mac_tx_cookie_t cookie = NULL;
2890 2918 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2891 2919 clock_t now;
2892 2920
2893 2921 ASSERT(TX_BANDWIDTH_MODE(mac_srs));
2894 2922 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
2895 2923 mutex_enter(&mac_srs->srs_lock);
2896 2924 if (mac_srs->srs_bw->mac_bw_limit == 0) {
2897 2925 /*
2898 2926 * zero bandwidth, no traffic is sent: drop the packets,
2899 2927 * or return the whole chain if the caller requests all
2900 2928 * unsent packets back.
2901 2929 */
2902 2930 if (flag & MAC_TX_NO_ENQUEUE) {
2903 2931 cookie = (mac_tx_cookie_t)mac_srs;
2904 2932 *ret_mp = mp_chain;
2905 2933 } else {
2906 2934 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2907 2935 }
2908 2936 mutex_exit(&mac_srs->srs_lock);
2909 2937 return (cookie);
2910 2938 } else if ((mac_srs->srs_first != NULL) ||
2911 2939 (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2912 2940 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
2913 2941 fanout_hint, ret_mp);
2914 2942 mutex_exit(&mac_srs->srs_lock);
2915 2943 return (cookie);
2916 2944 }
2917 2945 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2918 2946 now = ddi_get_lbolt();
2919 2947 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
2920 2948 mac_srs->srs_bw->mac_bw_curr_time = now;
2921 2949 mac_srs->srs_bw->mac_bw_used = 0;
2922 2950 } else if (mac_srs->srs_bw->mac_bw_used >
2923 2951 mac_srs->srs_bw->mac_bw_limit) {
2924 2952 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
2925 2953 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2926 2954 mp_chain, tail, cnt, sz);
2927 2955 /*
2928 2956 * Wakeup worker thread. Note that worker
2929 2957 * thread has to be woken up so that it
2930 2958 * can fire up the timer to be woken up
2931 2959 * on the next tick. Also once
2932 2960 * BW_ENFORCED is set, it can only be
2933 2961 * reset by srs_worker thread. Until then
2934 2962 * all packets will get queued up in SRS
2935 2963 * and hence this this code path won't be
2936 2964 * entered until BW_ENFORCED is reset.
2937 2965 */
2938 2966 cv_signal(&mac_srs->srs_async);
2939 2967 mutex_exit(&mac_srs->srs_lock);
2940 2968 return (cookie);
2941 2969 }
2942 2970
2943 2971 mac_srs->srs_bw->mac_bw_used += sz;
2944 2972 mutex_exit(&mac_srs->srs_lock);
2945 2973
2946 2974 if (srs_tx->st_mode == SRS_TX_BW_FANOUT) {
2947 2975 mac_soft_ring_t *softring;
2948 2976 uint_t indx, hash;
2949 2977
2950 2978 hash = HASH_HINT(fanout_hint);
2951 2979 indx = COMPUTE_INDEX(hash,
2952 2980 mac_srs->srs_tx_ring_count);
2953 2981 softring = mac_srs->srs_tx_soft_rings[indx];
2954 2982 return (mac_tx_soft_ring_process(softring, mp_chain, flag,
2955 2983 ret_mp));
2956 2984 } else if (srs_tx->st_mode == SRS_TX_BW_AGGR) {
2957 2985 return (mac_tx_aggr_mode(mac_srs, mp_chain,
2958 2986 fanout_hint, flag, ret_mp));
2959 2987 } else {
2960 2988 mac_tx_stats_t stats;
2961 2989
2962 2990 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2963 2991 mp_chain, &stats);
2964 2992
2965 2993 if (mp_chain != NULL) {
2966 2994 mutex_enter(&mac_srs->srs_lock);
2967 2995 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2968 2996 if (mac_srs->srs_bw->mac_bw_used > sz)
2969 2997 mac_srs->srs_bw->mac_bw_used -= sz;
2970 2998 else
2971 2999 mac_srs->srs_bw->mac_bw_used = 0;
2972 3000 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
2973 3001 fanout_hint, ret_mp);
2974 3002 mutex_exit(&mac_srs->srs_lock);
2975 3003 return (cookie);
2976 3004 }
2977 3005 SRS_TX_STATS_UPDATE(mac_srs, &stats);
2978 3006
2979 3007 return (NULL);
2980 3008 }
2981 3009 }
2982 3010
2983 3011 /*
2984 3012 * mac_tx_aggr_mode
2985 3013 *
2986 3014 * This routine invokes an aggr function, aggr_find_tx_ring(), to find
2987 3015 * a (pseudo) Tx ring belonging to a port on which the packet has to
2988 3016 * be sent. aggr_find_tx_ring() first finds the outgoing port based on
2989 3017 * L2/L3/L4 policy and then uses the fanout_hint passed to it to pick
2990 3018 * a Tx ring from the selected port.
2991 3019 *
2992 3020 * Note that a port can be deleted from the aggregation. In such a case,
2993 3021 * the aggregation layer first separates the port from the rest of the
2994 3022 * ports making sure that port (and thus any Tx rings associated with
2995 3023 * it) won't get selected in the call to aggr_find_tx_ring() function.
2996 3024 * Later calls are made to mac_group_rem_ring() passing pseudo Tx ring
2997 3025 * handles one by one which in turn will quiesce the Tx SRS and remove
2998 3026 * the soft ring associated with the pseudo Tx ring. Unlike Rx side
2999 3027 * where a cookie is used to protect against mac_rx_ring() calls on
3000 3028 * rings that have been removed, no such cookie is needed on the Tx
3001 3029 * side as the pseudo Tx ring won't be available anymore to
3002 3030 * aggr_find_tx_ring() once the port has been removed.
3003 3031 */
3004 3032 static mac_tx_cookie_t
3005 3033 mac_tx_aggr_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3006 3034 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3007 3035 {
3008 3036 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3009 3037 mac_tx_ring_fn_t find_tx_ring_fn;
3010 3038 mac_ring_handle_t ring = NULL;
3011 3039 void *arg;
3012 3040 mac_soft_ring_t *sringp;
3013 3041
3014 3042 find_tx_ring_fn = srs_tx->st_capab_aggr.mca_find_tx_ring_fn;
3015 3043 arg = srs_tx->st_capab_aggr.mca_arg;
3016 3044 if (find_tx_ring_fn(arg, mp_chain, fanout_hint, &ring) == NULL)
3017 3045 return (NULL);
3018 3046 sringp = srs_tx->st_soft_rings[((mac_ring_t *)ring)->mr_index];
3019 3047 return (mac_tx_soft_ring_process(sringp, mp_chain, flag, ret_mp));
3020 3048 }
3021 3049
3022 3050 void
3023 3051 mac_tx_invoke_callbacks(mac_client_impl_t *mcip, mac_tx_cookie_t cookie)
3024 3052 {
3025 3053 mac_cb_t *mcb;
3026 3054 mac_tx_notify_cb_t *mtnfp;
3027 3055
3028 3056 /* Wakeup callback registered clients */
3029 3057 MAC_CALLBACK_WALKER_INC(&mcip->mci_tx_notify_cb_info);
3030 3058 for (mcb = mcip->mci_tx_notify_cb_list; mcb != NULL;
3031 3059 mcb = mcb->mcb_nextp) {
3032 3060 mtnfp = (mac_tx_notify_cb_t *)mcb->mcb_objp;
3033 3061 mtnfp->mtnf_fn(mtnfp->mtnf_arg, cookie);
3034 3062 }
3035 3063 MAC_CALLBACK_WALKER_DCR(&mcip->mci_tx_notify_cb_info,
3036 3064 &mcip->mci_tx_notify_cb_list);
3037 3065 }
3038 3066
3039 3067 /* ARGSUSED */
3040 3068 void
3041 3069 mac_tx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
3042 3070 {
3043 3071 mblk_t *head, *tail;
3044 3072 size_t sz;
3045 3073 uint32_t tx_mode;
3046 3074 uint_t saved_pkt_count;
3047 3075 mac_tx_stats_t stats;
3048 3076 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3049 3077 clock_t now;
3050 3078
3051 3079 saved_pkt_count = 0;
3052 3080 ASSERT(mutex_owned(&mac_srs->srs_lock));
3053 3081 ASSERT(!(mac_srs->srs_state & SRS_PROC));
3054 3082
3055 3083 mac_srs->srs_state |= SRS_PROC;
3056 3084
3057 3085 tx_mode = srs_tx->st_mode;
3058 3086 if (tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_SERIALIZE) {
3059 3087 if (mac_srs->srs_first != NULL) {
3060 3088 head = mac_srs->srs_first;
3061 3089 tail = mac_srs->srs_last;
3062 3090 saved_pkt_count = mac_srs->srs_count;
3063 3091 mac_srs->srs_first = NULL;
3064 3092 mac_srs->srs_last = NULL;
3065 3093 mac_srs->srs_count = 0;
3066 3094 mutex_exit(&mac_srs->srs_lock);
3067 3095
3068 3096 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3069 3097 head, &stats);
3070 3098
3071 3099 mutex_enter(&mac_srs->srs_lock);
3072 3100 if (head != NULL) {
3073 3101 /* Device out of tx desc, set block */
3074 3102 if (head->b_next == NULL)
3075 3103 VERIFY(head == tail);
3076 3104 tail->b_next = mac_srs->srs_first;
3077 3105 mac_srs->srs_first = head;
3078 3106 mac_srs->srs_count +=
3079 3107 (saved_pkt_count - stats.mts_opackets);
3080 3108 if (mac_srs->srs_last == NULL)
3081 3109 mac_srs->srs_last = tail;
3082 3110 MAC_TX_SRS_BLOCK(mac_srs, head);
3083 3111 } else {
3084 3112 srs_tx->st_woken_up = B_FALSE;
3085 3113 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3086 3114 }
3087 3115 }
3088 3116 } else if (tx_mode == SRS_TX_BW) {
3089 3117 /*
3090 3118 * We are here because the timer fired and we have some data
3091 3119 * to tranmit. Also mac_tx_srs_worker should have reset
3092 3120 * SRS_BW_ENFORCED flag
3093 3121 */
3094 3122 ASSERT(!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED));
3095 3123 head = tail = mac_srs->srs_first;
3096 3124 while (mac_srs->srs_first != NULL) {
3097 3125 tail = mac_srs->srs_first;
3098 3126 tail->b_prev = NULL;
3099 3127 mac_srs->srs_first = tail->b_next;
3100 3128 if (mac_srs->srs_first == NULL)
3101 3129 mac_srs->srs_last = NULL;
3102 3130 mac_srs->srs_count--;
3103 3131 sz = msgdsize(tail);
3104 3132 mac_srs->srs_size -= sz;
3105 3133 saved_pkt_count++;
3106 3134 MAC_TX_UPDATE_BW_INFO(mac_srs, sz);
3107 3135
3108 3136 if (mac_srs->srs_bw->mac_bw_used <
3109 3137 mac_srs->srs_bw->mac_bw_limit)
3110 3138 continue;
3111 3139
3112 3140 now = ddi_get_lbolt();
3113 3141 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3114 3142 mac_srs->srs_bw->mac_bw_curr_time = now;
3115 3143 mac_srs->srs_bw->mac_bw_used = sz;
3116 3144 continue;
3117 3145 }
3118 3146 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3119 3147 break;
3120 3148 }
3121 3149
3122 3150 ASSERT((head == NULL && tail == NULL) ||
3123 3151 (head != NULL && tail != NULL));
3124 3152 if (tail != NULL) {
3125 3153 tail->b_next = NULL;
3126 3154 mutex_exit(&mac_srs->srs_lock);
3127 3155
3128 3156 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3129 3157 head, &stats);
3130 3158
3131 3159 mutex_enter(&mac_srs->srs_lock);
3132 3160 if (head != NULL) {
3133 3161 uint_t size_sent;
3134 3162
3135 3163 /* Device out of tx desc, set block */
3136 3164 if (head->b_next == NULL)
3137 3165 VERIFY(head == tail);
3138 3166 tail->b_next = mac_srs->srs_first;
3139 3167 mac_srs->srs_first = head;
3140 3168 mac_srs->srs_count +=
3141 3169 (saved_pkt_count - stats.mts_opackets);
3142 3170 if (mac_srs->srs_last == NULL)
3143 3171 mac_srs->srs_last = tail;
3144 3172 size_sent = sz - stats.mts_obytes;
3145 3173 mac_srs->srs_size += size_sent;
3146 3174 mac_srs->srs_bw->mac_bw_sz += size_sent;
3147 3175 if (mac_srs->srs_bw->mac_bw_used > size_sent) {
3148 3176 mac_srs->srs_bw->mac_bw_used -=
3149 3177 size_sent;
3150 3178 } else {
3151 3179 mac_srs->srs_bw->mac_bw_used = 0;
3152 3180 }
3153 3181 MAC_TX_SRS_BLOCK(mac_srs, head);
3154 3182 } else {
3155 3183 srs_tx->st_woken_up = B_FALSE;
3156 3184 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3157 3185 }
3158 3186 }
3159 3187 } else if (tx_mode == SRS_TX_BW_FANOUT || tx_mode == SRS_TX_BW_AGGR) {
3160 3188 mblk_t *prev;
3161 3189 uint64_t hint;
3162 3190
3163 3191 /*
3164 3192 * We are here because the timer fired and we
3165 3193 * have some quota to tranmit.
3166 3194 */
3167 3195 prev = NULL;
3168 3196 head = tail = mac_srs->srs_first;
3169 3197 while (mac_srs->srs_first != NULL) {
3170 3198 tail = mac_srs->srs_first;
3171 3199 mac_srs->srs_first = tail->b_next;
3172 3200 if (mac_srs->srs_first == NULL)
3173 3201 mac_srs->srs_last = NULL;
3174 3202 mac_srs->srs_count--;
3175 3203 sz = msgdsize(tail);
3176 3204 mac_srs->srs_size -= sz;
3177 3205 mac_srs->srs_bw->mac_bw_used += sz;
3178 3206 if (prev == NULL)
3179 3207 hint = (ulong_t)tail->b_prev;
3180 3208 if (hint != (ulong_t)tail->b_prev) {
3181 3209 prev->b_next = NULL;
3182 3210 mutex_exit(&mac_srs->srs_lock);
3183 3211 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
3184 3212 head = tail;
3185 3213 hint = (ulong_t)tail->b_prev;
3186 3214 mutex_enter(&mac_srs->srs_lock);
3187 3215 }
3188 3216
3189 3217 prev = tail;
3190 3218 tail->b_prev = NULL;
3191 3219 if (mac_srs->srs_bw->mac_bw_used <
3192 3220 mac_srs->srs_bw->mac_bw_limit)
3193 3221 continue;
3194 3222
3195 3223 now = ddi_get_lbolt();
3196 3224 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3197 3225 mac_srs->srs_bw->mac_bw_curr_time = now;
3198 3226 mac_srs->srs_bw->mac_bw_used = 0;
3199 3227 continue;
3200 3228 }
3201 3229 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3202 3230 break;
3203 3231 }
3204 3232 ASSERT((head == NULL && tail == NULL) ||
3205 3233 (head != NULL && tail != NULL));
3206 3234 if (tail != NULL) {
3207 3235 tail->b_next = NULL;
3208 3236 mutex_exit(&mac_srs->srs_lock);
3209 3237 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
3210 3238 mutex_enter(&mac_srs->srs_lock);
3211 3239 }
3212 3240 }
3213 3241 /*
3214 3242 * SRS_TX_FANOUT case not considered here because packets
3215 3243 * won't be queued in the SRS for this case. Packets will
3216 3244 * be sent directly to soft rings underneath and if there
3217 3245 * is any queueing at all, it would be in Tx side soft
3218 3246 * rings.
3219 3247 */
3220 3248
3221 3249 /*
3222 3250 * When srs_count becomes 0, reset SRS_TX_HIWAT and
3223 3251 * SRS_TX_WAKEUP_CLIENT and wakeup registered clients.
3224 3252 */
3225 3253 if (mac_srs->srs_count == 0 && (mac_srs->srs_state &
3226 3254 (SRS_TX_HIWAT | SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED))) {
3227 3255 mac_client_impl_t *mcip = mac_srs->srs_mcip;
3228 3256 boolean_t wakeup_required = B_FALSE;
3229 3257
3230 3258 if (mac_srs->srs_state &
3231 3259 (SRS_TX_HIWAT|SRS_TX_WAKEUP_CLIENT)) {
3232 3260 wakeup_required = B_TRUE;
3233 3261 }
3234 3262 mac_srs->srs_state &= ~(SRS_TX_HIWAT |
3235 3263 SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED);
3236 3264 mutex_exit(&mac_srs->srs_lock);
3237 3265 if (wakeup_required) {
3238 3266 mac_tx_invoke_callbacks(mcip, (mac_tx_cookie_t)mac_srs);
3239 3267 /*
3240 3268 * If the client is not the primary MAC client, then we
3241 3269 * need to send the notification to the clients upper
3242 3270 * MAC, i.e. mci_upper_mip.
3243 3271 */
3244 3272 mac_tx_notify(mcip->mci_upper_mip != NULL ?
3245 3273 mcip->mci_upper_mip : mcip->mci_mip);
3246 3274 }
3247 3275 mutex_enter(&mac_srs->srs_lock);
3248 3276 }
3249 3277 mac_srs->srs_state &= ~SRS_PROC;
3250 3278 }
3251 3279
3252 3280 /*
3253 3281 * Given a packet, get the flow_entry that identifies the flow
3254 3282 * to which that packet belongs. The flow_entry will contain
3255 3283 * the transmit function to be used to send the packet. If the
3256 3284 * function returns NULL, the packet should be sent using the
3257 3285 * underlying NIC.
3258 3286 */
3259 3287 static flow_entry_t *
3260 3288 mac_tx_classify(mac_impl_t *mip, mblk_t *mp)
3261 3289 {
3262 3290 flow_entry_t *flent = NULL;
3263 3291 mac_client_impl_t *mcip;
3264 3292 int err;
3265 3293
3266 3294 /*
3267 3295 * Do classification on the packet.
3268 3296 */
3269 3297 err = mac_flow_lookup(mip->mi_flow_tab, mp, FLOW_OUTBOUND, &flent);
3270 3298 if (err != 0)
3271 3299 return (NULL);
3272 3300
3273 3301 /*
3274 3302 * This flent might just be an additional one on the MAC client,
3275 3303 * i.e. for classification purposes (different fdesc), however
3276 3304 * the resources, SRS et. al., are in the mci_flent, so if
3277 3305 * this isn't the mci_flent, we need to get it.
3278 3306 */
3279 3307 if ((mcip = flent->fe_mcip) != NULL && mcip->mci_flent != flent) {
3280 3308 FLOW_REFRELE(flent);
3281 3309 flent = mcip->mci_flent;
3282 3310 FLOW_TRY_REFHOLD(flent, err);
3283 3311 if (err != 0)
3284 3312 return (NULL);
3285 3313 }
3286 3314
3287 3315 return (flent);
3288 3316 }
3289 3317
3290 3318 /*
3291 3319 * This macro is only meant to be used by mac_tx_send().
3292 3320 */
3293 3321 #define CHECK_VID_AND_ADD_TAG(mp) { \
3294 3322 if (vid_check) { \
3295 3323 int err = 0; \
3296 3324 \
3297 3325 MAC_VID_CHECK(src_mcip, (mp), err); \
3298 3326 if (err != 0) { \
3299 3327 freemsg((mp)); \
3300 3328 (mp) = next; \
3301 3329 oerrors++; \
3302 3330 continue; \
3303 3331 } \
3304 3332 } \
3305 3333 if (add_tag) { \
3306 3334 (mp) = mac_add_vlan_tag((mp), 0, vid); \
3307 3335 if ((mp) == NULL) { \
3308 3336 (mp) = next; \
3309 3337 oerrors++; \
3310 3338 continue; \
3311 3339 } \
3312 3340 } \
3313 3341 }
3314 3342
3315 3343 mblk_t *
3316 3344 mac_tx_send(mac_client_handle_t mch, mac_ring_handle_t ring, mblk_t *mp_chain,
3317 3345 mac_tx_stats_t *stats)
3318 3346 {
3319 3347 mac_client_impl_t *src_mcip = (mac_client_impl_t *)mch;
3320 3348 mac_impl_t *mip = src_mcip->mci_mip;
3321 3349 uint_t obytes = 0, opackets = 0, oerrors = 0;
3322 3350 mblk_t *mp = NULL, *next;
3323 3351 boolean_t vid_check, add_tag;
3324 3352 uint16_t vid = 0;
3325 3353
3326 3354 if (mip->mi_nclients > 1) {
3327 3355 vid_check = MAC_VID_CHECK_NEEDED(src_mcip);
3328 3356 add_tag = MAC_TAG_NEEDED(src_mcip);
3329 3357 if (add_tag)
3330 3358 vid = mac_client_vid(mch);
3331 3359 } else {
3332 3360 ASSERT(mip->mi_nclients == 1);
3333 3361 vid_check = add_tag = B_FALSE;
3334 3362 }
3335 3363
3336 3364 /*
3337 3365 * Fastpath: if there's only one client, we simply send
3338 3366 * the packet down to the underlying NIC.
3339 3367 */
3340 3368 if (mip->mi_nactiveclients == 1) {
3341 3369 DTRACE_PROBE2(fastpath,
3342 3370 mac_client_impl_t *, src_mcip, mblk_t *, mp_chain);
3343 3371
3344 3372 mp = mp_chain;
3345 3373 while (mp != NULL) {
3346 3374 next = mp->b_next;
3347 3375 mp->b_next = NULL;
3348 3376 opackets++;
3349 3377 obytes += (mp->b_cont == NULL ? MBLKL(mp) :
3350 3378 msgdsize(mp));
3351 3379
3352 3380 CHECK_VID_AND_ADD_TAG(mp);
3353 3381 MAC_TX(mip, ring, mp, src_mcip);
3354 3382
3355 3383 /*
3356 3384 * If the driver is out of descriptors and does a
3357 3385 * partial send it will return a chain of unsent
3358 3386 * mblks. Adjust the accounting stats.
3359 3387 */
3360 3388 if (mp != NULL) {
3361 3389 opackets--;
3362 3390 obytes -= msgdsize(mp);
3363 3391 mp->b_next = next;
3364 3392 break;
3365 3393 }
3366 3394 mp = next;
3367 3395 }
3368 3396 goto done;
3369 3397 }
3370 3398
3371 3399 /*
3372 3400 * No fastpath, we either have more than one MAC client
3373 3401 * defined on top of the same MAC, or one or more MAC
3374 3402 * client promiscuous callbacks.
3375 3403 */
3376 3404 DTRACE_PROBE3(slowpath, mac_client_impl_t *,
3377 3405 src_mcip, int, mip->mi_nclients, mblk_t *, mp_chain);
3378 3406
3379 3407 mp = mp_chain;
3380 3408 while (mp != NULL) {
3381 3409 flow_entry_t *dst_flow_ent;
3382 3410 void *flow_cookie;
3383 3411 size_t pkt_size;
3384 3412 mblk_t *mp1;
3385 3413
3386 3414 next = mp->b_next;
3387 3415 mp->b_next = NULL;
3388 3416 opackets++;
3389 3417 pkt_size = (mp->b_cont == NULL ? MBLKL(mp) : msgdsize(mp));
3390 3418 obytes += pkt_size;
3391 3419 CHECK_VID_AND_ADD_TAG(mp);
3392 3420
3393 3421 /*
3394 3422 * Find the destination.
3395 3423 */
3396 3424 dst_flow_ent = mac_tx_classify(mip, mp);
3397 3425
3398 3426 if (dst_flow_ent != NULL) {
3399 3427 size_t hdrsize;
3400 3428 int err = 0;
3401 3429
3402 3430 if (mip->mi_info.mi_nativemedia == DL_ETHER) {
3403 3431 struct ether_vlan_header *evhp =
3404 3432 (struct ether_vlan_header *)mp->b_rptr;
3405 3433
3406 3434 if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN)
3407 3435 hdrsize = sizeof (*evhp);
3408 3436 else
3409 3437 hdrsize = sizeof (struct ether_header);
3410 3438 } else {
3411 3439 mac_header_info_t mhi;
3412 3440
3413 3441 err = mac_header_info((mac_handle_t)mip,
3414 3442 mp, &mhi);
3415 3443 if (err == 0)
3416 3444 hdrsize = mhi.mhi_hdrsize;
3417 3445 }
3418 3446
3419 3447 /*
3420 3448 * Got a matching flow. It's either another
3421 3449 * MAC client, or a broadcast/multicast flow.
3422 3450 * Make sure the packet size is within the
3423 3451 * allowed size. If not drop the packet and
3424 3452 * move to next packet.
3425 3453 */
3426 3454 if (err != 0 ||
3427 3455 (pkt_size - hdrsize) > mip->mi_sdu_max) {
3428 3456 oerrors++;
3429 3457 DTRACE_PROBE2(loopback__drop, size_t, pkt_size,
3430 3458 mblk_t *, mp);
3431 3459 freemsg(mp);
3432 3460 mp = next;
3433 3461 FLOW_REFRELE(dst_flow_ent);
3434 3462 continue;
3435 3463 }
3436 3464 flow_cookie = mac_flow_get_client_cookie(dst_flow_ent);
3437 3465 if (flow_cookie != NULL) {
3438 3466 /*
3439 3467 * The vnic_bcast_send function expects
3440 3468 * to receive the sender MAC client
3441 3469 * as value for arg2.
3442 3470 */
3443 3471 mac_bcast_send(flow_cookie, src_mcip, mp,
3444 3472 B_TRUE);
3445 3473 } else {
3446 3474 /*
3447 3475 * loopback the packet to a local MAC
3448 3476 * client. We force a context switch
3449 3477 * if both source and destination MAC
3450 3478 * clients are used by IP, i.e.
3451 3479 * bypass is set.
3452 3480 */
3453 3481 boolean_t do_switch;
3454 3482 mac_client_impl_t *dst_mcip =
3455 3483 dst_flow_ent->fe_mcip;
3456 3484
3457 3485 /*
3458 3486 * Check if there are promiscuous mode
3459 3487 * callbacks defined. This check is
3460 3488 * done here in the 'else' case and
3461 3489 * not in other cases because this
3462 3490 * path is for local loopback
3463 3491 * communication which does not go
3464 3492 * through MAC_TX(). For paths that go
3465 3493 * through MAC_TX(), the promisc_list
3466 3494 * check is done inside the MAC_TX()
3467 3495 * macro.
3468 3496 */
3469 3497 if (mip->mi_promisc_list != NULL)
3470 3498 mac_promisc_dispatch(mip, mp, src_mcip);
3471 3499
3472 3500 do_switch = ((src_mcip->mci_state_flags &
3473 3501 dst_mcip->mci_state_flags &
3474 3502 MCIS_CLIENT_POLL_CAPABLE) != 0);
3475 3503
3476 3504 if ((mp1 = mac_fix_cksum(mp)) != NULL) {
3477 3505 (dst_flow_ent->fe_cb_fn)(
3478 3506 dst_flow_ent->fe_cb_arg1,
3479 3507 dst_flow_ent->fe_cb_arg2,
3480 3508 mp1, do_switch);
3481 3509 }
3482 3510 }
3483 3511 FLOW_REFRELE(dst_flow_ent);
3484 3512 } else {
3485 3513 /*
3486 3514 * Unknown destination, send via the underlying
3487 3515 * NIC.
3488 3516 */
3489 3517 MAC_TX(mip, ring, mp, src_mcip);
3490 3518 if (mp != NULL) {
3491 3519 /*
3492 3520 * Adjust for the last packet that
3493 3521 * could not be transmitted
3494 3522 */
3495 3523 opackets--;
3496 3524 obytes -= pkt_size;
3497 3525 mp->b_next = next;
3498 3526 break;
3499 3527 }
3500 3528 }
3501 3529 mp = next;
3502 3530 }
3503 3531
3504 3532 done:
3505 3533 stats->mts_obytes = obytes;
3506 3534 stats->mts_opackets = opackets;
3507 3535 stats->mts_oerrors = oerrors;
3508 3536 return (mp);
3509 3537 }
3510 3538
3511 3539 /*
3512 3540 * mac_tx_srs_ring_present
3513 3541 *
3514 3542 * Returns whether the specified ring is part of the specified SRS.
3515 3543 */
3516 3544 boolean_t
3517 3545 mac_tx_srs_ring_present(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
3518 3546 {
3519 3547 int i;
3520 3548 mac_soft_ring_t *soft_ring;
3521 3549
3522 3550 if (srs->srs_tx.st_arg2 == tx_ring)
3523 3551 return (B_TRUE);
3524 3552
3525 3553 for (i = 0; i < srs->srs_tx_ring_count; i++) {
3526 3554 soft_ring = srs->srs_tx_soft_rings[i];
3527 3555 if (soft_ring->s_ring_tx_arg2 == tx_ring)
3528 3556 return (B_TRUE);
3529 3557 }
3530 3558
3531 3559 return (B_FALSE);
3532 3560 }
3533 3561
3534 3562 /*
3535 3563 * mac_tx_srs_get_soft_ring
3536 3564 *
3537 3565 * Returns the TX soft ring associated with the given ring, if present.
3538 3566 */
3539 3567 mac_soft_ring_t *
3540 3568 mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
3541 3569 {
3542 3570 int i;
3543 3571 mac_soft_ring_t *soft_ring;
3544 3572
3545 3573 if (srs->srs_tx.st_arg2 == tx_ring)
3546 3574 return (NULL);
3547 3575
3548 3576 for (i = 0; i < srs->srs_tx_ring_count; i++) {
3549 3577 soft_ring = srs->srs_tx_soft_rings[i];
3550 3578 if (soft_ring->s_ring_tx_arg2 == tx_ring)
3551 3579 return (soft_ring);
3552 3580 }
3553 3581
3554 3582 return (NULL);
3555 3583 }
3556 3584
3557 3585 /*
3558 3586 * mac_tx_srs_wakeup
3559 3587 *
3560 3588 * Called when Tx desc become available. Wakeup the appropriate worker
3561 3589 * thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the
3562 3590 * state field.
3563 3591 */
3564 3592 void
3565 3593 mac_tx_srs_wakeup(mac_soft_ring_set_t *mac_srs, mac_ring_handle_t ring)
3566 3594 {
3567 3595 int i;
3568 3596 mac_soft_ring_t *sringp;
3569 3597 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3570 3598
3571 3599 mutex_enter(&mac_srs->srs_lock);
3572 3600 /*
3573 3601 * srs_tx_ring_count == 0 is the single ring mode case. In
3574 3602 * this mode, there will not be Tx soft rings associated
3575 3603 * with the SRS.
3576 3604 */
3577 3605 if (!MAC_TX_SOFT_RINGS(mac_srs)) {
3578 3606 if (srs_tx->st_arg2 == ring &&
3579 3607 mac_srs->srs_state & SRS_TX_BLOCKED) {
3580 3608 mac_srs->srs_state &= ~SRS_TX_BLOCKED;
3581 3609 srs_tx->st_stat.mts_unblockcnt++;
3582 3610 cv_signal(&mac_srs->srs_async);
3583 3611 }
3584 3612 /*
3585 3613 * A wakeup can come before tx_srs_drain() could
3586 3614 * grab srs lock and set SRS_TX_BLOCKED. So
3587 3615 * always set woken_up flag when we come here.
3588 3616 */
3589 3617 srs_tx->st_woken_up = B_TRUE;
3590 3618 mutex_exit(&mac_srs->srs_lock);
3591 3619 return;
3592 3620 }
3593 3621
3594 3622 /*
3595 3623 * If you are here, it is for FANOUT, BW_FANOUT,
3596 3624 * AGGR_MODE or AGGR_BW_MODE case
3597 3625 */
3598 3626 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
3599 3627 sringp = mac_srs->srs_tx_soft_rings[i];
3600 3628 mutex_enter(&sringp->s_ring_lock);
3601 3629 if (sringp->s_ring_tx_arg2 == ring) {
3602 3630 if (sringp->s_ring_state & S_RING_BLOCK) {
3603 3631 sringp->s_ring_state &= ~S_RING_BLOCK;
3604 3632 sringp->s_st_stat.mts_unblockcnt++;
3605 3633 cv_signal(&sringp->s_ring_async);
3606 3634 }
3607 3635 sringp->s_ring_tx_woken_up = B_TRUE;
3608 3636 }
3609 3637 mutex_exit(&sringp->s_ring_lock);
3610 3638 }
3611 3639 mutex_exit(&mac_srs->srs_lock);
3612 3640 }
3613 3641
3614 3642 /*
3615 3643 * Once the driver is done draining, send a MAC_NOTE_TX notification to unleash
3616 3644 * the blocked clients again.
3617 3645 */
3618 3646 void
3619 3647 mac_tx_notify(mac_impl_t *mip)
3620 3648 {
3621 3649 i_mac_notify(mip, MAC_NOTE_TX);
3622 3650 }
3623 3651
3624 3652 /*
3625 3653 * RX SOFTRING RELATED FUNCTIONS
3626 3654 *
3627 3655 * These functions really belong in mac_soft_ring.c and here for
3628 3656 * a short period.
3629 3657 */
3630 3658
3631 3659 #define SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
3632 3660 /* \
3633 3661 * Enqueue our mblk chain. \
3634 3662 */ \
3635 3663 ASSERT(MUTEX_HELD(&(ringp)->s_ring_lock)); \
3636 3664 \
3637 3665 if ((ringp)->s_ring_last != NULL) \
3638 3666 (ringp)->s_ring_last->b_next = (mp); \
3639 3667 else \
3640 3668 (ringp)->s_ring_first = (mp); \
3641 3669 (ringp)->s_ring_last = (tail); \
3642 3670 (ringp)->s_ring_count += (cnt); \
3643 3671 ASSERT((ringp)->s_ring_count > 0); \
3644 3672 if ((ringp)->s_ring_type & ST_RING_BW_CTL) { \
3645 3673 (ringp)->s_ring_size += sz; \
3646 3674 } \
3647 3675 }
3648 3676
3649 3677 /*
3650 3678 * Default entry point to deliver a packet chain to a MAC client.
3651 3679 * If the MAC client has flows, do the classification with these
3652 3680 * flows as well.
3653 3681 */
3654 3682 /* ARGSUSED */
3655 3683 void
3656 3684 mac_rx_deliver(void *arg1, mac_resource_handle_t mrh, mblk_t *mp_chain,
3657 3685 mac_header_info_t *arg3)
3658 3686 {
3659 3687 mac_client_impl_t *mcip = arg1;
3660 3688
3661 3689 if (mcip->mci_nvids == 1 &&
3662 3690 !(mcip->mci_state_flags & MCIS_STRIP_DISABLE)) {
3663 3691 /*
3664 3692 * If the client has exactly one VID associated with it
3665 3693 * and striping of VLAN header is not disabled,
3666 3694 * remove the VLAN tag from the packet before
3667 3695 * passing it on to the client's receive callback.
3668 3696 * Note that this needs to be done after we dispatch
3669 3697 * the packet to the promiscuous listeners of the
3670 3698 * client, since they expect to see the whole
3671 3699 * frame including the VLAN headers.
3672 3700 */
3673 3701 mp_chain = mac_strip_vlan_tag_chain(mp_chain);
3674 3702 }
3675 3703
3676 3704 mcip->mci_rx_fn(mcip->mci_rx_arg, mrh, mp_chain, B_FALSE);
3677 3705 }
3678 3706
3679 3707 /*
3680 3708 * mac_rx_soft_ring_process
3681 3709 *
3682 3710 * process a chain for a given soft ring. The number of packets queued
3683 3711 * in the SRS and its associated soft rings (including this one) is
3684 3712 * very small (tracked by srs_poll_pkt_cnt), then allow the entering
3685 3713 * thread (interrupt or poll thread) to do inline processing. This
3686 3714 * helps keep the latency down under low load.
3687 3715 *
3688 3716 * The proc and arg for each mblk is already stored in the mblk in
3689 3717 * appropriate places.
3690 3718 */
3691 3719 /* ARGSUSED */
3692 3720 void
3693 3721 mac_rx_soft_ring_process(mac_client_impl_t *mcip, mac_soft_ring_t *ringp,
3694 3722 mblk_t *mp_chain, mblk_t *tail, int cnt, size_t sz)
3695 3723 {
3696 3724 mac_direct_rx_t proc;
3697 3725 void *arg1;
3698 3726 mac_resource_handle_t arg2;
3699 3727 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3700 3728
3701 3729 ASSERT(ringp != NULL);
3702 3730 ASSERT(mp_chain != NULL);
3703 3731 ASSERT(tail != NULL);
3704 3732 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3705 3733
3706 3734 mutex_enter(&ringp->s_ring_lock);
3707 3735 ringp->s_ring_total_inpkt += cnt;
3708 3736 ringp->s_ring_total_rbytes += sz;
3709 3737 if ((mac_srs->srs_rx.sr_poll_pkt_cnt <= 1) &&
3710 3738 !(ringp->s_ring_type & ST_RING_WORKER_ONLY)) {
3711 3739 /* If on processor or blanking on, then enqueue and return */
3712 3740 if (ringp->s_ring_state & S_RING_BLANK ||
3713 3741 ringp->s_ring_state & S_RING_PROC) {
3714 3742 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3715 3743 mutex_exit(&ringp->s_ring_lock);
3716 3744 return;
3717 3745 }
3718 3746 proc = ringp->s_ring_rx_func;
3719 3747 arg1 = ringp->s_ring_rx_arg1;
3720 3748 arg2 = ringp->s_ring_rx_arg2;
3721 3749 /*
3722 3750 * See if anything is already queued. If we are the
3723 3751 * first packet, do inline processing else queue the
3724 3752 * packet and do the drain.
3725 3753 */
3726 3754 if (ringp->s_ring_first == NULL) {
3727 3755 /*
3728 3756 * Fast-path, ok to process and nothing queued.
3729 3757 */
3730 3758 ringp->s_ring_run = curthread;
3731 3759 ringp->s_ring_state |= (S_RING_PROC);
3732 3760
3733 3761 mutex_exit(&ringp->s_ring_lock);
3734 3762
3735 3763 /*
3736 3764 * We are the chain of 1 packet so
3737 3765 * go through this fast path.
3738 3766 */
3739 3767 ASSERT(mp_chain->b_next == NULL);
3740 3768
3741 3769 (*proc)(arg1, arg2, mp_chain, NULL);
3742 3770
3743 3771 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3744 3772 /*
3745 3773 * If we have a soft ring set which is doing
3746 3774 * bandwidth control, we need to decrement
3747 3775 * srs_size and count so it the SRS can have a
3748 3776 * accurate idea of what is the real data
3749 3777 * queued between SRS and its soft rings. We
3750 3778 * decrement the counters only when the packet
3751 3779 * gets processed by both SRS and the soft ring.
3752 3780 */
3753 3781 mutex_enter(&mac_srs->srs_lock);
3754 3782 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
3755 3783 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
3756 3784 mutex_exit(&mac_srs->srs_lock);
3757 3785
3758 3786 mutex_enter(&ringp->s_ring_lock);
3759 3787 ringp->s_ring_run = NULL;
3760 3788 ringp->s_ring_state &= ~S_RING_PROC;
3761 3789 if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
3762 3790 cv_signal(&ringp->s_ring_client_cv);
3763 3791
3764 3792 if ((ringp->s_ring_first == NULL) ||
3765 3793 (ringp->s_ring_state & S_RING_BLANK)) {
3766 3794 /*
3767 3795 * We processed inline our packet and
3768 3796 * nothing new has arrived or our
3769 3797 * receiver doesn't want to receive
3770 3798 * any packets. We are done.
3771 3799 */
3772 3800 mutex_exit(&ringp->s_ring_lock);
3773 3801 return;
3774 3802 }
3775 3803 } else {
3776 3804 SOFT_RING_ENQUEUE_CHAIN(ringp,
3777 3805 mp_chain, tail, cnt, sz);
3778 3806 }
3779 3807
3780 3808 /*
3781 3809 * We are here because either we couldn't do inline
3782 3810 * processing (because something was already
3783 3811 * queued), or we had a chain of more than one
3784 3812 * packet, or something else arrived after we were
3785 3813 * done with inline processing.
3786 3814 */
3787 3815 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
3788 3816 ASSERT(ringp->s_ring_first != NULL);
3789 3817
3790 3818 ringp->s_ring_drain_func(ringp);
3791 3819 mutex_exit(&ringp->s_ring_lock);
3792 3820 return;
3793 3821 } else {
3794 3822 /* ST_RING_WORKER_ONLY case */
3795 3823 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3796 3824 mac_soft_ring_worker_wakeup(ringp);
3797 3825 mutex_exit(&ringp->s_ring_lock);
3798 3826 }
3799 3827 }
3800 3828
3801 3829 /*
3802 3830 * TX SOFTRING RELATED FUNCTIONS
3803 3831 *
3804 3832 * These functions really belong in mac_soft_ring.c and here for
3805 3833 * a short period.
3806 3834 */
3807 3835
3808 3836 #define TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
3809 3837 ASSERT(MUTEX_HELD(&ringp->s_ring_lock)); \
3810 3838 ringp->s_ring_state |= S_RING_ENQUEUED; \
3811 3839 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); \
3812 3840 }
3813 3841
3814 3842 /*
3815 3843 * mac_tx_sring_queued
3816 3844 *
3817 3845 * When we are out of transmit descriptors and we already have a
3818 3846 * queue that exceeds hiwat (or the client called us with
3819 3847 * MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the
3820 3848 * soft ring pointer as the opaque cookie for the client enable
3821 3849 * flow control.
3822 3850 */
3823 3851 static mac_tx_cookie_t
3824 3852 mac_tx_sring_enqueue(mac_soft_ring_t *ringp, mblk_t *mp_chain, uint16_t flag,
3825 3853 mblk_t **ret_mp)
3826 3854 {
3827 3855 int cnt;
3828 3856 size_t sz;
3829 3857 mblk_t *tail;
3830 3858 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3831 3859 mac_tx_cookie_t cookie = NULL;
3832 3860 boolean_t wakeup_worker = B_TRUE;
3833 3861
3834 3862 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
3835 3863 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3836 3864 if (flag & MAC_DROP_ON_NO_DESC) {
3837 3865 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
3838 3866 /* increment freed stats */
3839 3867 ringp->s_ring_drops += cnt;
3840 3868 cookie = (mac_tx_cookie_t)ringp;
3841 3869 } else {
3842 3870 if (ringp->s_ring_first != NULL)
3843 3871 wakeup_worker = B_FALSE;
3844 3872
3845 3873 if (flag & MAC_TX_NO_ENQUEUE) {
3846 3874 /*
3847 3875 * If QUEUED is not set, queue the packet
3848 3876 * and let mac_tx_soft_ring_drain() set
3849 3877 * the TX_BLOCKED bit for the reasons
3850 3878 * explained above. Otherwise, return the
3851 3879 * mblks.
3852 3880 */
3853 3881 if (wakeup_worker) {
3854 3882 TX_SOFT_RING_ENQUEUE_CHAIN(ringp,
3855 3883 mp_chain, tail, cnt, sz);
3856 3884 } else {
3857 3885 ringp->s_ring_state |= S_RING_WAKEUP_CLIENT;
3858 3886 cookie = (mac_tx_cookie_t)ringp;
3859 3887 *ret_mp = mp_chain;
3860 3888 }
3861 3889 } else {
3862 3890 boolean_t enqueue = B_TRUE;
3863 3891
3864 3892 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
3865 3893 /*
3866 3894 * flow-controlled. Store ringp in cookie
3867 3895 * so that it can be returned as
3868 3896 * mac_tx_cookie_t to client
3869 3897 */
3870 3898 ringp->s_ring_state |= S_RING_TX_HIWAT;
3871 3899 cookie = (mac_tx_cookie_t)ringp;
3872 3900 ringp->s_ring_hiwat_cnt++;
3873 3901 if (ringp->s_ring_count >
3874 3902 ringp->s_ring_tx_max_q_cnt) {
3875 3903 /* increment freed stats */
3876 3904 ringp->s_ring_drops += cnt;
3877 3905 /*
3878 3906 * b_prev may be set to the fanout hint
3879 3907 * hence can't use freemsg directly
3880 3908 */
3881 3909 mac_pkt_drop(NULL, NULL,
3882 3910 mp_chain, B_FALSE);
3883 3911 DTRACE_PROBE1(tx_queued_hiwat,
3884 3912 mac_soft_ring_t *, ringp);
3885 3913 enqueue = B_FALSE;
3886 3914 }
3887 3915 }
3888 3916 if (enqueue) {
3889 3917 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain,
3890 3918 tail, cnt, sz);
3891 3919 }
3892 3920 }
3893 3921 if (wakeup_worker)
3894 3922 cv_signal(&ringp->s_ring_async);
3895 3923 }
3896 3924 return (cookie);
3897 3925 }
3898 3926
3899 3927
3900 3928 /*
3901 3929 * mac_tx_soft_ring_process
3902 3930 *
3903 3931 * This routine is called when fanning out outgoing traffic among
3904 3932 * multipe Tx rings.
3905 3933 * Note that a soft ring is associated with a h/w Tx ring.
3906 3934 */
3907 3935 mac_tx_cookie_t
3908 3936 mac_tx_soft_ring_process(mac_soft_ring_t *ringp, mblk_t *mp_chain,
3909 3937 uint16_t flag, mblk_t **ret_mp)
3910 3938 {
3911 3939 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3912 3940 int cnt;
3913 3941 size_t sz;
3914 3942 mblk_t *tail;
3915 3943 mac_tx_cookie_t cookie = NULL;
3916 3944
3917 3945 ASSERT(ringp != NULL);
3918 3946 ASSERT(mp_chain != NULL);
3919 3947 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3920 3948 /*
3921 3949 * The following modes can come here: SRS_TX_BW_FANOUT,
3922 3950 * SRS_TX_FANOUT, SRS_TX_AGGR, SRS_TX_BW_AGGR.
3923 3951 */
3924 3952 ASSERT(MAC_TX_SOFT_RINGS(mac_srs));
3925 3953 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
3926 3954 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT ||
3927 3955 mac_srs->srs_tx.st_mode == SRS_TX_AGGR ||
3928 3956 mac_srs->srs_tx.st_mode == SRS_TX_BW_AGGR);
3929 3957
3930 3958 if (ringp->s_ring_type & ST_RING_WORKER_ONLY) {
3931 3959 /* Serialization mode */
3932 3960
3933 3961 mutex_enter(&ringp->s_ring_lock);
3934 3962 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
3935 3963 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
3936 3964 flag, ret_mp);
3937 3965 mutex_exit(&ringp->s_ring_lock);
3938 3966 return (cookie);
3939 3967 }
3940 3968 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3941 3969 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3942 3970 if (ringp->s_ring_state & (S_RING_BLOCK | S_RING_PROC)) {
3943 3971 /*
3944 3972 * If ring is blocked due to lack of Tx
3945 3973 * descs, just return. Worker thread
3946 3974 * will get scheduled when Tx desc's
3947 3975 * become available.
3948 3976 */
3949 3977 mutex_exit(&ringp->s_ring_lock);
3950 3978 return (cookie);
3951 3979 }
3952 3980 mac_soft_ring_worker_wakeup(ringp);
3953 3981 mutex_exit(&ringp->s_ring_lock);
3954 3982 return (cookie);
3955 3983 } else {
3956 3984 /* Default fanout mode */
3957 3985 /*
3958 3986 * S_RING_BLOCKED is set when underlying NIC runs
3959 3987 * out of Tx descs and messages start getting
3960 3988 * queued. It won't get reset until
3961 3989 * tx_srs_drain() completely drains out the
3962 3990 * messages.
3963 3991 */
3964 3992 mac_tx_stats_t stats;
3965 3993
3966 3994 if (ringp->s_ring_state & S_RING_ENQUEUED) {
3967 3995 /* Tx descs/resources not available */
3968 3996 mutex_enter(&ringp->s_ring_lock);
3969 3997 if (ringp->s_ring_state & S_RING_ENQUEUED) {
3970 3998 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
3971 3999 flag, ret_mp);
3972 4000 mutex_exit(&ringp->s_ring_lock);
3973 4001 return (cookie);
3974 4002 }
3975 4003 /*
3976 4004 * While we were computing mblk count, the
3977 4005 * flow control condition got relieved.
3978 4006 * Continue with the transmission.
3979 4007 */
3980 4008 mutex_exit(&ringp->s_ring_lock);
3981 4009 }
3982 4010
3983 4011 mp_chain = mac_tx_send(ringp->s_ring_tx_arg1,
3984 4012 ringp->s_ring_tx_arg2, mp_chain, &stats);
3985 4013
3986 4014 /*
3987 4015 * Multiple threads could be here sending packets.
3988 4016 * Under such conditions, it is not possible to
3989 4017 * automically set S_RING_BLOCKED bit to indicate
3990 4018 * out of tx desc condition. To atomically set
3991 4019 * this, we queue the returned packet and do
3992 4020 * the setting of S_RING_BLOCKED in
3993 4021 * mac_tx_soft_ring_drain().
3994 4022 */
3995 4023 if (mp_chain != NULL) {
3996 4024 mutex_enter(&ringp->s_ring_lock);
3997 4025 cookie =
3998 4026 mac_tx_sring_enqueue(ringp, mp_chain, flag, ret_mp);
3999 4027 mutex_exit(&ringp->s_ring_lock);
4000 4028 return (cookie);
4001 4029 }
4002 4030 SRS_TX_STATS_UPDATE(mac_srs, &stats);
4003 4031 SOFTRING_TX_STATS_UPDATE(ringp, &stats);
4004 4032
4005 4033 return (NULL);
4006 4034 }
4007 4035 }
|
↓ open down ↓ |
1571 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX