1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright 2011 Joyent, Inc. All rights reserved.
27 */
28
29 #include <sys/types.h>
30 #include <sys/callb.h>
31 #include <sys/sdt.h>
32 #include <sys/strsubr.h>
33 #include <sys/strsun.h>
34 #include <sys/vlan.h>
35 #include <sys/stack.h>
36 #include <sys/archsystm.h>
37 #include <inet/ipsec_impl.h>
38 #include <inet/ip_impl.h>
39 #include <inet/sadb.h>
40 #include <inet/ipsecesp.h>
41 #include <inet/ipsecah.h>
42 #include <inet/ip6.h>
43
44 #include <sys/mac_impl.h>
45 #include <sys/mac_client_impl.h>
46 #include <sys/mac_client_priv.h>
47 #include <sys/mac_soft_ring.h>
48 #include <sys/mac_flow_impl.h>
49
50 static mac_tx_cookie_t mac_tx_single_ring_mode(mac_soft_ring_set_t *, mblk_t *,
51 uintptr_t, uint16_t, mblk_t **);
52 static mac_tx_cookie_t mac_tx_serializer_mode(mac_soft_ring_set_t *, mblk_t *,
53 uintptr_t, uint16_t, mblk_t **);
54 static mac_tx_cookie_t mac_tx_fanout_mode(mac_soft_ring_set_t *, mblk_t *,
55 uintptr_t, uint16_t, mblk_t **);
56 static mac_tx_cookie_t mac_tx_bw_mode(mac_soft_ring_set_t *, mblk_t *,
57 uintptr_t, uint16_t, mblk_t **);
58 static mac_tx_cookie_t mac_tx_aggr_mode(mac_soft_ring_set_t *, mblk_t *,
59 uintptr_t, uint16_t, mblk_t **);
60
61 typedef struct mac_tx_mode_s {
62 mac_tx_srs_mode_t mac_tx_mode;
63 mac_tx_func_t mac_tx_func;
64 } mac_tx_mode_t;
65
66 /*
67 * There are seven modes of operation on the Tx side. These modes get set
68 * in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode,
69 * none of the other modes are user configurable. They get selected by
70 * the system depending upon whether the link (or flow) has multiple Tx
71 * rings or a bandwidth configured, or if the link is an aggr, etc.
72 *
73 * When the Tx SRS is operating in aggr mode (st_mode) or if there are
74 * multiple Tx rings owned by Tx SRS, then each Tx ring (pseudo or
75 * otherwise) will have a soft ring associated with it. These soft rings
76 * are stored in srs_tx_soft_rings[] array.
77 *
78 * Additionally in the case of aggr, there is the st_soft_rings[] array
79 * in the mac_srs_tx_t structure. This array is used to store the same
80 * set of soft rings that are present in srs_tx_soft_rings[] array but
81 * in a different manner. The soft ring associated with the pseudo Tx
82 * ring is saved at mr_index (of the pseudo ring) in st_soft_rings[]
83 * array. This helps in quickly getting the soft ring associated with the
84 * Tx ring when aggr_find_tx_ring() returns the pseudo Tx ring that is to
85 * be used for transmit.
86 */
87 mac_tx_mode_t mac_tx_mode_list[] = {
88 {SRS_TX_DEFAULT, mac_tx_single_ring_mode},
89 {SRS_TX_SERIALIZE, mac_tx_serializer_mode},
90 {SRS_TX_FANOUT, mac_tx_fanout_mode},
91 {SRS_TX_BW, mac_tx_bw_mode},
92 {SRS_TX_BW_FANOUT, mac_tx_bw_mode},
93 {SRS_TX_AGGR, mac_tx_aggr_mode},
94 {SRS_TX_BW_AGGR, mac_tx_bw_mode}
95 };
96
97 /*
98 * Soft Ring Set (SRS) - The Run time code that deals with
99 * dynamic polling from the hardware, bandwidth enforcement,
100 * fanout etc.
101 *
102 * We try to use H/W classification on NIC and assign traffic for
103 * a MAC address to a particular Rx ring or ring group. There is a
104 * 1-1 mapping between a SRS and a Rx ring. The SRS dynamically
105 * switches the underlying Rx ring between interrupt and
106 * polling mode and enforces any specified B/W control.
107 *
108 * There is always a SRS created and tied to each H/W and S/W rule.
109 * Whenever we create a H/W rule, we always add the the same rule to
110 * S/W classifier and tie a SRS to it.
111 *
112 * In case a B/W control is specified, it is broken into bytes
113 * per ticks and as soon as the quota for a tick is exhausted,
114 * the underlying Rx ring is forced into poll mode for remainder of
115 * the tick. The SRS poll thread only polls for bytes that are
116 * allowed to come in the SRS. We typically let 4x the configured
117 * B/W worth of packets to come in the SRS (to prevent unnecessary
118 * drops due to bursts) but only process the specified amount.
119 *
120 * A MAC client (e.g. a VNIC or aggr) can have 1 or more
121 * Rx rings (and corresponding SRSs) assigned to it. The SRS
122 * in turn can have softrings to do protocol level fanout or
123 * softrings to do S/W based fanout or both. In case the NIC
124 * has no Rx rings, we do S/W classification to respective SRS.
125 * The S/W classification rule is always setup and ready. This
126 * allows the MAC layer to reassign Rx rings whenever needed
127 * but packets still continue to flow via the default path and
128 * getting S/W classified to correct SRS.
129 *
130 * The SRS's are used on both Tx and Rx side. They use the same
131 * data structure but the processing routines have slightly different
132 * semantics due to the fact that Rx side needs to do dynamic
133 * polling etc.
134 *
135 * Dynamic Polling Notes
136 * =====================
137 *
138 * Each Soft ring set is capable of switching its Rx ring between
139 * interrupt and poll mode and actively 'polls' for packets in
140 * poll mode. If the SRS is implementing a B/W limit, it makes
141 * sure that only Max allowed packets are pulled in poll mode
142 * and goes to poll mode as soon as B/W limit is exceeded. As
143 * such, there are no overheads to implement B/W limits.
144 *
145 * In poll mode, its better to keep the pipeline going where the
146 * SRS worker thread keeps processing packets and poll thread
147 * keeps bringing more packets (specially if they get to run
148 * on different CPUs). This also prevents the overheads associated
149 * by excessive signalling (on NUMA machines, this can be
150 * pretty devastating). The exception is latency optimized case
151 * where worker thread does no work and interrupt and poll thread
152 * are allowed to do their own drain.
153 *
154 * We use the following policy to control Dynamic Polling:
155 * 1) We switch to poll mode anytime the processing
156 * thread causes a backlog to build up in SRS and
157 * its associated Soft Rings (sr_poll_pkt_cnt > 0).
158 * 2) As long as the backlog stays under the low water
159 * mark (sr_lowat), we poll the H/W for more packets.
160 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low
161 * water mark, we stay in poll mode but don't poll
162 * the H/W for more packets.
163 * 4) Anytime in polling mode, if we poll the H/W for
164 * packets and find nothing plus we have an existing
165 * backlog (sr_poll_pkt_cnt > 0), we stay in polling
166 * mode but don't poll the H/W for packets anymore
167 * (let the polling thread go to sleep).
168 * 5) Once the backlog is relived (packets are processed)
169 * we reenable polling (by signalling the poll thread)
170 * only when the backlog dips below sr_poll_thres.
171 * 6) sr_hiwat is used exclusively when we are not
172 * polling capable and is used to decide when to
173 * drop packets so the SRS queue length doesn't grow
174 * infinitely.
175 *
176 * NOTE: Also see the block level comment on top of mac_soft_ring.c
177 */
178
179 /*
180 * mac_latency_optimize
181 *
182 * Controls whether the poll thread can process the packets inline
183 * or let the SRS worker thread do the processing. This applies if
184 * the SRS was not being processed. For latency sensitive traffic,
185 * this needs to be true to allow inline processing. For throughput
186 * under load, this should be false.
187 *
188 * This (and other similar) tunable should be rolled into a link
189 * or flow specific workload hint that can be set using dladm
190 * linkprop (instead of multiple such tunables).
191 */
192 boolean_t mac_latency_optimize = B_TRUE;
193
194 /*
195 * MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN
196 *
197 * queue a mp or chain in soft ring set and increment the
198 * local count (srs_count) for the SRS and the shared counter
199 * (srs_poll_pkt_cnt - shared between SRS and its soft rings
200 * to track the total unprocessed packets for polling to work
201 * correctly).
202 *
203 * The size (total bytes queued) counters are incremented only
204 * if we are doing B/W control.
205 */
206 #define MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
207 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
208 if ((mac_srs)->srs_last != NULL) \
209 (mac_srs)->srs_last->b_next = (head); \
210 else \
211 (mac_srs)->srs_first = (head); \
212 (mac_srs)->srs_last = (tail); \
213 (mac_srs)->srs_count += count; \
214 }
215
216 #define MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
217 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
218 \
219 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
220 srs_rx->sr_poll_pkt_cnt += count; \
221 ASSERT(srs_rx->sr_poll_pkt_cnt > 0); \
222 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
223 (mac_srs)->srs_size += (sz); \
224 mutex_enter(&(mac_srs)->srs_bw->mac_bw_lock); \
225 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
226 mutex_exit(&(mac_srs)->srs_bw->mac_bw_lock); \
227 } \
228 }
229
230 #define MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
231 mac_srs->srs_state |= SRS_ENQUEUED; \
232 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
233 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
234 (mac_srs)->srs_size += (sz); \
235 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
236 } \
237 }
238
239 /*
240 * Turn polling on routines
241 */
242 #define MAC_SRS_POLLING_ON(mac_srs) { \
243 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
244 if (((mac_srs)->srs_state & \
245 (SRS_POLLING_CAPAB|SRS_POLLING)) == SRS_POLLING_CAPAB) { \
246 (mac_srs)->srs_state |= SRS_POLLING; \
247 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
248 (mac_srs)->srs_ring); \
249 (mac_srs)->srs_rx.sr_poll_on++; \
250 } \
251 }
252
253 #define MAC_SRS_WORKER_POLLING_ON(mac_srs) { \
254 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
255 if (((mac_srs)->srs_state & \
256 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_POLLING)) == \
257 (SRS_POLLING_CAPAB|SRS_WORKER)) { \
258 (mac_srs)->srs_state |= SRS_POLLING; \
259 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
260 (mac_srs)->srs_ring); \
261 (mac_srs)->srs_rx.sr_worker_poll_on++; \
262 } \
263 }
264
265 /*
266 * MAC_SRS_POLL_RING
267 *
268 * Signal the SRS poll thread to poll the underlying H/W ring
269 * provided it wasn't already polling (SRS_GET_PKTS was set).
270 *
271 * Poll thread gets to run only from mac_rx_srs_drain() and only
272 * if the drain was being done by the worker thread.
273 */
274 #define MAC_SRS_POLL_RING(mac_srs) { \
275 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
276 \
277 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
278 srs_rx->sr_poll_thr_sig++; \
279 if (((mac_srs)->srs_state & \
280 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_GET_PKTS)) == \
281 (SRS_WORKER|SRS_POLLING_CAPAB)) { \
282 (mac_srs)->srs_state |= SRS_GET_PKTS; \
283 cv_signal(&(mac_srs)->srs_cv); \
284 } else { \
285 srs_rx->sr_poll_thr_busy++; \
286 } \
287 }
288
289 /*
290 * MAC_SRS_CHECK_BW_CONTROL
291 *
292 * Check to see if next tick has started so we can reset the
293 * SRS_BW_ENFORCED flag and allow more packets to come in the
294 * system.
295 */
296 #define MAC_SRS_CHECK_BW_CONTROL(mac_srs) { \
297 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
298 ASSERT(((mac_srs)->srs_type & SRST_TX) || \
299 MUTEX_HELD(&(mac_srs)->srs_bw->mac_bw_lock)); \
300 clock_t now = ddi_get_lbolt(); \
301 if ((mac_srs)->srs_bw->mac_bw_curr_time != now) { \
302 (mac_srs)->srs_bw->mac_bw_curr_time = now; \
303 (mac_srs)->srs_bw->mac_bw_used = 0; \
304 if ((mac_srs)->srs_bw->mac_bw_state & SRS_BW_ENFORCED) \
305 (mac_srs)->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; \
306 } \
307 }
308
309 /*
310 * MAC_SRS_WORKER_WAKEUP
311 *
312 * Wake up the SRS worker thread to process the queue as long as
313 * no one else is processing the queue. If we are optimizing for
314 * latency, we wake up the worker thread immediately or else we
315 * wait mac_srs_worker_wakeup_ticks before worker thread gets
316 * woken up.
317 */
318 int mac_srs_worker_wakeup_ticks = 0;
319 #define MAC_SRS_WORKER_WAKEUP(mac_srs) { \
320 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
321 if (!((mac_srs)->srs_state & SRS_PROC) && \
322 (mac_srs)->srs_tid == NULL) { \
323 if (((mac_srs)->srs_state & SRS_LATENCY_OPT) || \
324 (mac_srs_worker_wakeup_ticks == 0)) \
325 cv_signal(&(mac_srs)->srs_async); \
326 else \
327 (mac_srs)->srs_tid = \
328 timeout(mac_srs_fire, (mac_srs), \
329 mac_srs_worker_wakeup_ticks); \
330 } \
331 }
332
333 #define TX_BANDWIDTH_MODE(mac_srs) \
334 ((mac_srs)->srs_tx.st_mode == SRS_TX_BW || \
335 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT || \
336 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_AGGR)
337
338 #define TX_SRS_TO_SOFT_RING(mac_srs, head, hint) { \
339 if (tx_mode == SRS_TX_BW_FANOUT) \
340 (void) mac_tx_fanout_mode(mac_srs, head, hint, 0, NULL);\
341 else \
342 (void) mac_tx_aggr_mode(mac_srs, head, hint, 0, NULL); \
343 }
344
345 /*
346 * MAC_TX_SRS_BLOCK
347 *
348 * Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED
349 * will be set only if srs_tx_woken_up is FALSE. If
350 * srs_tx_woken_up is TRUE, it indicates that the wakeup arrived
351 * before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to
352 * attempt to transmit again and not setting SRS_TX_BLOCKED does
353 * that.
354 */
355 #define MAC_TX_SRS_BLOCK(srs, mp) { \
356 ASSERT(MUTEX_HELD(&(srs)->srs_lock)); \
357 if ((srs)->srs_tx.st_woken_up) { \
358 (srs)->srs_tx.st_woken_up = B_FALSE; \
359 } else { \
360 ASSERT(!((srs)->srs_state & SRS_TX_BLOCKED)); \
361 (srs)->srs_state |= SRS_TX_BLOCKED; \
362 (srs)->srs_tx.st_stat.mts_blockcnt++; \
363 } \
364 }
365
366 /*
367 * MAC_TX_SRS_TEST_HIWAT
368 *
369 * Called before queueing a packet onto Tx SRS to test and set
370 * SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat.
371 */
372 #define MAC_TX_SRS_TEST_HIWAT(srs, mp, tail, cnt, sz, cookie) { \
373 boolean_t enqueue = 1; \
374 \
375 if ((srs)->srs_count > (srs)->srs_tx.st_hiwat) { \
376 /* \
377 * flow-controlled. Store srs in cookie so that it \
378 * can be returned as mac_tx_cookie_t to client \
379 */ \
380 (srs)->srs_state |= SRS_TX_HIWAT; \
381 cookie = (mac_tx_cookie_t)srs; \
382 (srs)->srs_tx.st_hiwat_cnt++; \
383 if ((srs)->srs_count > (srs)->srs_tx.st_max_q_cnt) { \
384 /* increment freed stats */ \
385 (srs)->srs_tx.st_stat.mts_sdrops += cnt; \
386 /* \
387 * b_prev may be set to the fanout hint \
388 * hence can't use freemsg directly \
389 */ \
390 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE); \
391 DTRACE_PROBE1(tx_queued_hiwat, \
392 mac_soft_ring_set_t *, srs); \
393 enqueue = 0; \
394 } \
395 } \
396 if (enqueue) \
397 MAC_TX_SRS_ENQUEUE_CHAIN(srs, mp, tail, cnt, sz); \
398 }
399
400 /* Some utility macros */
401 #define MAC_SRS_BW_LOCK(srs) \
402 if (!(srs->srs_type & SRST_TX)) \
403 mutex_enter(&srs->srs_bw->mac_bw_lock);
404
405 #define MAC_SRS_BW_UNLOCK(srs) \
406 if (!(srs->srs_type & SRST_TX)) \
407 mutex_exit(&srs->srs_bw->mac_bw_lock);
408
409 #define MAC_TX_SRS_DROP_MESSAGE(srs, mp, cookie) { \
410 mac_pkt_drop(NULL, NULL, mp, B_FALSE); \
411 /* increment freed stats */ \
412 mac_srs->srs_tx.st_stat.mts_sdrops++; \
413 cookie = (mac_tx_cookie_t)srs; \
414 }
415
416 #define MAC_TX_SET_NO_ENQUEUE(srs, mp_chain, ret_mp, cookie) { \
417 mac_srs->srs_state |= SRS_TX_WAKEUP_CLIENT; \
418 cookie = (mac_tx_cookie_t)srs; \
419 *ret_mp = mp_chain; \
420 }
421
422 /*
423 * MAC_RX_SRS_TOODEEP
424 *
425 * Macro called as part of receive-side processing to determine if handling
426 * can occur in situ (in the interrupt thread) or if it should be left to a
427 * worker thread. Note that the constant used to make this determination is
428 * not entirely made-up, and is a result of some emprical validation. That
429 * said, the constant is left as a static variable to allow it to be
430 * dynamically tuned in the field if and as needed.
431 */
432 static uintptr_t mac_rx_srs_stack_needed = 10240;
433 static uint_t mac_rx_srs_stack_toodeep;
434
435 #ifndef STACK_GROWTH_DOWN
436 #error Downward stack growth assumed.
437 #endif
438
439 #define MAC_RX_SRS_TOODEEP() (STACK_BIAS + (uintptr_t)getfp() - \
440 (uintptr_t)curthread->t_stkbase < mac_rx_srs_stack_needed && \
441 ++mac_rx_srs_stack_toodeep)
442
443
444 /*
445 * Drop the rx packet and advance to the next one in the chain.
446 */
447 static void
448 mac_rx_drop_pkt(mac_soft_ring_set_t *srs, mblk_t *mp)
449 {
450 mac_srs_rx_t *srs_rx = &srs->srs_rx;
451
452 ASSERT(mp->b_next == NULL);
453 mutex_enter(&srs->srs_lock);
454 MAC_UPDATE_SRS_COUNT_LOCKED(srs, 1);
455 MAC_UPDATE_SRS_SIZE_LOCKED(srs, msgdsize(mp));
456 mutex_exit(&srs->srs_lock);
457
458 srs_rx->sr_stat.mrs_sdrops++;
459 freemsg(mp);
460 }
461
462 /* DATAPATH RUNTIME ROUTINES */
463
464 /*
465 * mac_srs_fire
466 *
467 * Timer callback routine for waking up the SRS worker thread.
468 */
469 static void
470 mac_srs_fire(void *arg)
471 {
472 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)arg;
473
474 mutex_enter(&mac_srs->srs_lock);
475 if (mac_srs->srs_tid == 0) {
476 mutex_exit(&mac_srs->srs_lock);
477 return;
478 }
479
480 mac_srs->srs_tid = 0;
481 if (!(mac_srs->srs_state & SRS_PROC))
482 cv_signal(&mac_srs->srs_async);
483
484 mutex_exit(&mac_srs->srs_lock);
485 }
486
487 /*
488 * 'hint' is fanout_hint (type of uint64_t) which is given by the TCP/IP stack,
489 * and it is used on the TX path.
490 */
491 #define HASH_HINT(hint) \
492 ((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8))
493
494
495 /*
496 * hash based on the src address and the port information.
497 */
498 #define HASH_ADDR(src, ports) \
499 (ntohl((src)) ^ ((ports) >> 24) ^ ((ports) >> 16) ^ \
500 ((ports) >> 8) ^ (ports))
501
502 #define COMPUTE_INDEX(key, sz) (key % sz)
503
504 #define FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) { \
505 if ((tail) != NULL) { \
506 ASSERT((tail)->b_next == NULL); \
507 (tail)->b_next = (mp); \
508 } else { \
509 ASSERT((head) == NULL); \
510 (head) = (mp); \
511 } \
512 (tail) = (mp); \
513 (cnt)++; \
514 if ((bw_ctl)) \
515 (sz) += (sz0); \
516 }
517
518 #define MAC_FANOUT_DEFAULT 0
519 #define MAC_FANOUT_RND_ROBIN 1
520 int mac_fanout_type = MAC_FANOUT_DEFAULT;
521
522 #define MAX_SR_TYPES 3
523 /* fanout types for port based hashing */
524 enum pkt_type {
525 V4_TCP = 0,
526 V4_UDP,
527 OTH,
528 UNDEF
529 };
530
531 /*
532 * In general we do port based hashing to spread traffic over different
533 * softrings. The below tunable allows to override that behavior. Setting it
534 * to B_TRUE allows to do a fanout based on src ipv6 address. This behavior
535 * is also the applicable to ipv6 packets carrying multiple optional headers
536 * and other uncommon packet types.
537 */
538 boolean_t mac_src_ipv6_fanout = B_FALSE;
539
540 /*
541 * Pair of local and remote ports in the transport header
542 */
543 #define PORTS_SIZE 4
544
545 /*
546 * mac_rx_srs_proto_fanout
547 *
548 * This routine delivers packets destined to an SRS into one of the
549 * protocol soft rings.
550 *
551 * Given a chain of packets we need to split it up into multiple sub chains
552 * destined into TCP, UDP or OTH soft ring. Instead of entering
553 * the soft ring one packet at a time, we want to enter it in the form of a
554 * chain otherwise we get this start/stop behaviour where the worker thread
555 * goes to sleep and then next packets comes in forcing it to wake up etc.
556 */
557 static void
558 mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
559 {
560 struct ether_header *ehp;
561 struct ether_vlan_header *evhp;
562 uint32_t sap;
563 ipha_t *ipha;
564 uint8_t *dstaddr;
565 size_t hdrsize;
566 mblk_t *mp;
567 mblk_t *headmp[MAX_SR_TYPES];
568 mblk_t *tailmp[MAX_SR_TYPES];
569 int cnt[MAX_SR_TYPES];
570 size_t sz[MAX_SR_TYPES];
571 size_t sz1;
572 boolean_t bw_ctl;
573 boolean_t hw_classified;
574 boolean_t dls_bypass;
575 boolean_t is_ether;
576 boolean_t is_unicast;
577 enum pkt_type type;
578 mac_client_impl_t *mcip = mac_srs->srs_mcip;
579
580 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
581 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
582
583 /*
584 * If we don't have a Rx ring, S/W classification would have done
585 * its job and its a packet meant for us. If we were polling on
586 * the default ring (i.e. there was a ring assigned to this SRS),
587 * then we need to make sure that the mac address really belongs
588 * to us.
589 */
590 hw_classified = mac_srs->srs_ring != NULL &&
591 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
592
593 /*
594 * Special clients (eg. VLAN, non ether, etc) need DLS
595 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
596 * such SRSs. Another way of disabling bypass is to set the
597 * MCIS_RX_BYPASS_DISABLE flag.
598 */
599 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
600 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
601
602 bzero(headmp, MAX_SR_TYPES * sizeof (mblk_t *));
603 bzero(tailmp, MAX_SR_TYPES * sizeof (mblk_t *));
604 bzero(cnt, MAX_SR_TYPES * sizeof (int));
605 bzero(sz, MAX_SR_TYPES * sizeof (size_t));
606
607 /*
608 * We got a chain from SRS that we need to send to the soft rings.
609 * Since squeues for TCP & IPv4 sap poll their soft rings (for
610 * performance reasons), we need to separate out v4_tcp, v4_udp
611 * and the rest goes in other.
612 */
613 while (head != NULL) {
614 mp = head;
615 head = head->b_next;
616 mp->b_next = NULL;
617
618 type = OTH;
619 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
620
621 if (is_ether) {
622 /*
623 * At this point we can be sure the packet at least
624 * has an ether header.
625 */
626 if (sz1 < sizeof (struct ether_header)) {
627 mac_rx_drop_pkt(mac_srs, mp);
628 continue;
629 }
630 ehp = (struct ether_header *)mp->b_rptr;
631
632 /*
633 * Determine if this is a VLAN or non-VLAN packet.
634 */
635 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
636 evhp = (struct ether_vlan_header *)mp->b_rptr;
637 sap = ntohs(evhp->ether_type);
638 hdrsize = sizeof (struct ether_vlan_header);
639 /*
640 * Check if the VID of the packet, if any,
641 * belongs to this client.
642 */
643 if (!mac_client_check_flow_vid(mcip,
644 VLAN_ID(ntohs(evhp->ether_tci)))) {
645 mac_rx_drop_pkt(mac_srs, mp);
646 continue;
647 }
648 } else {
649 hdrsize = sizeof (struct ether_header);
650 }
651 is_unicast =
652 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
653 dstaddr = (uint8_t *)&ehp->ether_dhost;
654 } else {
655 mac_header_info_t mhi;
656
657 if (mac_header_info((mac_handle_t)mcip->mci_mip,
658 mp, &mhi) != 0) {
659 mac_rx_drop_pkt(mac_srs, mp);
660 continue;
661 }
662 hdrsize = mhi.mhi_hdrsize;
663 sap = mhi.mhi_bindsap;
664 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
665 dstaddr = (uint8_t *)mhi.mhi_daddr;
666 }
667
668 if (!dls_bypass) {
669 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
670 cnt[type], bw_ctl, sz[type], sz1, mp);
671 continue;
672 }
673
674 if (sap == ETHERTYPE_IP) {
675 /*
676 * If we are H/W classified, but we have promisc
677 * on, then we need to check for the unicast address.
678 */
679 if (hw_classified && mcip->mci_promisc_list != NULL) {
680 mac_address_t *map;
681
682 rw_enter(&mcip->mci_rw_lock, RW_READER);
683 map = mcip->mci_unicast;
684 if (bcmp(dstaddr, map->ma_addr,
685 map->ma_len) == 0)
686 type = UNDEF;
687 rw_exit(&mcip->mci_rw_lock);
688 } else if (is_unicast) {
689 type = UNDEF;
690 }
691 }
692
693 /*
694 * This needs to become a contract with the driver for
695 * the fast path.
696 *
697 * In the normal case the packet will have at least the L2
698 * header and the IP + Transport header in the same mblk.
699 * This is usually the case when the NIC driver sends up
700 * the packet. This is also true when the stack generates
701 * a packet that is looped back and when the stack uses the
702 * fastpath mechanism. The normal case is optimized for
703 * performance and may bypass DLS. All other cases go through
704 * the 'OTH' type path without DLS bypass.
705 */
706
707 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
708 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha))
709 type = OTH;
710
711 if (type == OTH) {
712 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
713 cnt[type], bw_ctl, sz[type], sz1, mp);
714 continue;
715 }
716
717 ASSERT(type == UNDEF);
718 /*
719 * We look for at least 4 bytes past the IP header to get
720 * the port information. If we get an IP fragment, we don't
721 * have the port information, and we use just the protocol
722 * information.
723 */
724 switch (ipha->ipha_protocol) {
725 case IPPROTO_TCP:
726 type = V4_TCP;
727 mp->b_rptr += hdrsize;
728 break;
729 case IPPROTO_UDP:
730 type = V4_UDP;
731 mp->b_rptr += hdrsize;
732 break;
733 default:
734 type = OTH;
735 break;
736 }
737
738 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type],
739 bw_ctl, sz[type], sz1, mp);
740 }
741
742 for (type = V4_TCP; type < UNDEF; type++) {
743 if (headmp[type] != NULL) {
744 mac_soft_ring_t *softring;
745
746 ASSERT(tailmp[type]->b_next == NULL);
747 switch (type) {
748 case V4_TCP:
749 softring = mac_srs->srs_tcp_soft_rings[0];
750 break;
751 case V4_UDP:
752 softring = mac_srs->srs_udp_soft_rings[0];
753 break;
754 case OTH:
755 softring = mac_srs->srs_oth_soft_rings[0];
756 }
757 mac_rx_soft_ring_process(mcip, softring,
758 headmp[type], tailmp[type], cnt[type], sz[type]);
759 }
760 }
761 }
762
763 int fanout_unalligned = 0;
764
765 /*
766 * mac_rx_srs_long_fanout
767 *
768 * The fanout routine for IPv6
769 */
770 static int
771 mac_rx_srs_long_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *mp,
772 uint32_t sap, size_t hdrsize, enum pkt_type *type, uint_t *indx)
773 {
774 ip6_t *ip6h;
775 uint8_t *whereptr;
776 uint_t hash;
777 uint16_t remlen;
778 uint8_t nexthdr;
779 uint16_t hdr_len;
780
781 if (sap == ETHERTYPE_IPV6) {
782 boolean_t modifiable = B_TRUE;
783
784 ASSERT(MBLKL(mp) >= hdrsize);
785
786 ip6h = (ip6_t *)(mp->b_rptr + hdrsize);
787 if ((unsigned char *)ip6h == mp->b_wptr) {
788 /*
789 * The first mblk_t only includes the mac header.
790 * Note that it is safe to change the mp pointer here,
791 * as the subsequent operation does not assume mp
792 * points to the start of the mac header.
793 */
794 mp = mp->b_cont;
795
796 /*
797 * Make sure ip6h holds the full ip6_t structure.
798 */
799 if (mp == NULL)
800 return (-1);
801
802 if (MBLKL(mp) < IPV6_HDR_LEN) {
803 modifiable = (DB_REF(mp) == 1);
804
805 if (modifiable &&
806 !pullupmsg(mp, IPV6_HDR_LEN)) {
807 return (-1);
808 }
809 }
810
811 ip6h = (ip6_t *)mp->b_rptr;
812 }
813
814 if (!modifiable || !(OK_32PTR((char *)ip6h)) ||
815 ((unsigned char *)ip6h + IPV6_HDR_LEN > mp->b_wptr)) {
816 /*
817 * If either ip6h is not alligned, or ip6h does not
818 * hold the complete ip6_t structure (a pullupmsg()
819 * is not an option since it would result in an
820 * unalligned ip6h), fanout to the default ring. Note
821 * that this may cause packets reordering.
822 */
823 *indx = 0;
824 *type = OTH;
825 fanout_unalligned++;
826 return (0);
827 }
828
829 remlen = ntohs(ip6h->ip6_plen);
830 nexthdr = ip6h->ip6_nxt;
831
832 if (remlen < MIN_EHDR_LEN)
833 return (-1);
834 /*
835 * Do src based fanout if below tunable is set to B_TRUE or
836 * when mac_ip_hdr_length_v6() fails because of malformed
837 * packets or because mblk's need to be concatenated using
838 * pullupmsg().
839 */
840 if (mac_src_ipv6_fanout || !mac_ip_hdr_length_v6(ip6h,
841 mp->b_wptr, &hdr_len, &nexthdr, NULL)) {
842 goto src_based_fanout;
843 }
844 whereptr = (uint8_t *)ip6h + hdr_len;
845
846 /* If the transport is one of below, we do port based fanout */
847 switch (nexthdr) {
848 case IPPROTO_TCP:
849 case IPPROTO_UDP:
850 case IPPROTO_SCTP:
851 case IPPROTO_ESP:
852 /*
853 * If the ports in the transport header is not part of
854 * the mblk, do src_based_fanout, instead of calling
855 * pullupmsg().
856 */
857 if (mp->b_cont != NULL &&
858 whereptr + PORTS_SIZE > mp->b_wptr) {
859 goto src_based_fanout;
860 }
861 break;
862 default:
863 break;
864 }
865
866 switch (nexthdr) {
867 case IPPROTO_TCP:
868 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src),
869 *(uint32_t *)whereptr);
870 *indx = COMPUTE_INDEX(hash,
871 mac_srs->srs_tcp_ring_count);
872 *type = OTH;
873 break;
874
875 case IPPROTO_UDP:
876 case IPPROTO_SCTP:
877 case IPPROTO_ESP:
878 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
879 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src),
880 *(uint32_t *)whereptr);
881 *indx = COMPUTE_INDEX(hash,
882 mac_srs->srs_udp_ring_count);
883 } else {
884 *indx = mac_srs->srs_ind %
885 mac_srs->srs_udp_ring_count;
886 mac_srs->srs_ind++;
887 }
888 *type = OTH;
889 break;
890
891 /* For all other protocol, do source based fanout */
892 default:
893 goto src_based_fanout;
894 }
895 } else {
896 *indx = 0;
897 *type = OTH;
898 }
899 return (0);
900
901 src_based_fanout:
902 hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src), (uint32_t)0);
903 *indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count);
904 *type = OTH;
905 return (0);
906 }
907
908 /*
909 * mac_rx_srs_fanout
910 *
911 * This routine delivers packets destined to an SRS into a soft ring member
912 * of the set.
913 *
914 * Given a chain of packets we need to split it up into multiple sub chains
915 * destined for one of the TCP, UDP or OTH soft rings. Instead of entering
916 * the soft ring one packet at a time, we want to enter it in the form of a
917 * chain otherwise we get this start/stop behaviour where the worker thread
918 * goes to sleep and then next packets comes in forcing it to wake up etc.
919 *
920 * Note:
921 * Since we know what is the maximum fanout possible, we create a 2D array
922 * of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz
923 * variables so that we can enter the softrings with chain. We need the
924 * MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc
925 * for each packet would be expensive). If we ever want to have the
926 * ability to have unlimited fanout, we should probably declare a head,
927 * tail, cnt, sz with each soft ring (a data struct which contains a softring
928 * along with these members) and create an array of this uber struct so we
929 * don't have to do kmem_alloc.
930 */
931 int fanout_oth1 = 0;
932 int fanout_oth2 = 0;
933 int fanout_oth3 = 0;
934 int fanout_oth4 = 0;
935 int fanout_oth5 = 0;
936
937 static void
938 mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
939 {
940 struct ether_header *ehp;
941 struct ether_vlan_header *evhp;
942 uint32_t sap;
943 ipha_t *ipha;
944 uint8_t *dstaddr;
945 uint_t indx;
946 size_t ports_offset;
947 size_t ipha_len;
948 size_t hdrsize;
949 uint_t hash;
950 mblk_t *mp;
951 mblk_t *headmp[MAX_SR_TYPES][MAX_SR_FANOUT];
952 mblk_t *tailmp[MAX_SR_TYPES][MAX_SR_FANOUT];
953 int cnt[MAX_SR_TYPES][MAX_SR_FANOUT];
954 size_t sz[MAX_SR_TYPES][MAX_SR_FANOUT];
955 size_t sz1;
956 boolean_t bw_ctl;
957 boolean_t hw_classified;
958 boolean_t dls_bypass;
959 boolean_t is_ether;
960 boolean_t is_unicast;
961 int fanout_cnt;
962 enum pkt_type type;
963 mac_client_impl_t *mcip = mac_srs->srs_mcip;
964
965 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
966 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
967
968 /*
969 * If we don't have a Rx ring, S/W classification would have done
970 * its job and its a packet meant for us. If we were polling on
971 * the default ring (i.e. there was a ring assigned to this SRS),
972 * then we need to make sure that the mac address really belongs
973 * to us.
974 */
975 hw_classified = mac_srs->srs_ring != NULL &&
976 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
977
978 /*
979 * Special clients (eg. VLAN, non ether, etc) need DLS
980 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
981 * such SRSs. Another way of disabling bypass is to set the
982 * MCIS_RX_BYPASS_DISABLE flag.
983 */
984 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
985 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
986
987 /*
988 * Since the softrings are never destroyed and we always
989 * create equal number of softrings for TCP, UDP and rest,
990 * its OK to check one of them for count and use it without
991 * any lock. In future, if soft rings get destroyed because
992 * of reduction in fanout, we will need to ensure that happens
993 * behind the SRS_PROC.
994 */
995 fanout_cnt = mac_srs->srs_tcp_ring_count;
996
997 bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
998 bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
999 bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int));
1000 bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t));
1001
1002 /*
1003 * We got a chain from SRS that we need to send to the soft rings.
1004 * Since squeues for TCP & IPv4 sap poll their soft rings (for
1005 * performance reasons), we need to separate out v4_tcp, v4_udp
1006 * and the rest goes in other.
1007 */
1008 while (head != NULL) {
1009 mp = head;
1010 head = head->b_next;
1011 mp->b_next = NULL;
1012
1013 type = OTH;
1014 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1015
1016 if (is_ether) {
1017 /*
1018 * At this point we can be sure the packet at least
1019 * has an ether header.
1020 */
1021 if (sz1 < sizeof (struct ether_header)) {
1022 mac_rx_drop_pkt(mac_srs, mp);
1023 continue;
1024 }
1025 ehp = (struct ether_header *)mp->b_rptr;
1026
1027 /*
1028 * Determine if this is a VLAN or non-VLAN packet.
1029 */
1030 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
1031 evhp = (struct ether_vlan_header *)mp->b_rptr;
1032 sap = ntohs(evhp->ether_type);
1033 hdrsize = sizeof (struct ether_vlan_header);
1034 /*
1035 * Check if the VID of the packet, if any,
1036 * belongs to this client.
1037 */
1038 if (!mac_client_check_flow_vid(mcip,
1039 VLAN_ID(ntohs(evhp->ether_tci)))) {
1040 mac_rx_drop_pkt(mac_srs, mp);
1041 continue;
1042 }
1043 } else {
1044 hdrsize = sizeof (struct ether_header);
1045 }
1046 is_unicast =
1047 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
1048 dstaddr = (uint8_t *)&ehp->ether_dhost;
1049 } else {
1050 mac_header_info_t mhi;
1051
1052 if (mac_header_info((mac_handle_t)mcip->mci_mip,
1053 mp, &mhi) != 0) {
1054 mac_rx_drop_pkt(mac_srs, mp);
1055 continue;
1056 }
1057 hdrsize = mhi.mhi_hdrsize;
1058 sap = mhi.mhi_bindsap;
1059 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
1060 dstaddr = (uint8_t *)mhi.mhi_daddr;
1061 }
1062
1063 if (!dls_bypass) {
1064 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
1065 hdrsize, &type, &indx) == -1) {
1066 mac_rx_drop_pkt(mac_srs, mp);
1067 continue;
1068 }
1069
1070 FANOUT_ENQUEUE_MP(headmp[type][indx],
1071 tailmp[type][indx], cnt[type][indx], bw_ctl,
1072 sz[type][indx], sz1, mp);
1073 continue;
1074 }
1075
1076
1077 /*
1078 * If we are using the default Rx ring where H/W or S/W
1079 * classification has not happened, we need to verify if
1080 * this unicast packet really belongs to us.
1081 */
1082 if (sap == ETHERTYPE_IP) {
1083 /*
1084 * If we are H/W classified, but we have promisc
1085 * on, then we need to check for the unicast address.
1086 */
1087 if (hw_classified && mcip->mci_promisc_list != NULL) {
1088 mac_address_t *map;
1089
1090 rw_enter(&mcip->mci_rw_lock, RW_READER);
1091 map = mcip->mci_unicast;
1092 if (bcmp(dstaddr, map->ma_addr,
1093 map->ma_len) == 0)
1094 type = UNDEF;
1095 rw_exit(&mcip->mci_rw_lock);
1096 } else if (is_unicast) {
1097 type = UNDEF;
1098 }
1099 }
1100
1101 /*
1102 * This needs to become a contract with the driver for
1103 * the fast path.
1104 */
1105
1106 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
1107 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) {
1108 type = OTH;
1109 fanout_oth1++;
1110 }
1111
1112 if (type != OTH) {
1113 uint16_t frag_offset_flags;
1114
1115 switch (ipha->ipha_protocol) {
1116 case IPPROTO_TCP:
1117 case IPPROTO_UDP:
1118 case IPPROTO_SCTP:
1119 case IPPROTO_ESP:
1120 ipha_len = IPH_HDR_LENGTH(ipha);
1121 if ((uchar_t *)ipha + ipha_len + PORTS_SIZE >
1122 mp->b_wptr) {
1123 type = OTH;
1124 break;
1125 }
1126 frag_offset_flags =
1127 ntohs(ipha->ipha_fragment_offset_and_flags);
1128 if ((frag_offset_flags &
1129 (IPH_MF | IPH_OFFSET)) != 0) {
1130 type = OTH;
1131 fanout_oth3++;
1132 break;
1133 }
1134 ports_offset = hdrsize + ipha_len;
1135 break;
1136 default:
1137 type = OTH;
1138 fanout_oth4++;
1139 break;
1140 }
1141 }
1142
1143 if (type == OTH) {
1144 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
1145 hdrsize, &type, &indx) == -1) {
1146 mac_rx_drop_pkt(mac_srs, mp);
1147 continue;
1148 }
1149
1150 FANOUT_ENQUEUE_MP(headmp[type][indx],
1151 tailmp[type][indx], cnt[type][indx], bw_ctl,
1152 sz[type][indx], sz1, mp);
1153 continue;
1154 }
1155
1156 ASSERT(type == UNDEF);
1157
1158 /*
1159 * XXX-Sunay: We should hold srs_lock since ring_count
1160 * below can change. But if we are always called from
1161 * mac_rx_srs_drain and SRS_PROC is set, then we can
1162 * enforce that ring_count can't be changed i.e.
1163 * to change fanout type or ring count, the calling
1164 * thread needs to be behind SRS_PROC.
1165 */
1166 switch (ipha->ipha_protocol) {
1167 case IPPROTO_TCP:
1168 /*
1169 * Note that for ESP, we fanout on SPI and it is at the
1170 * same offset as the 2x16-bit ports. So it is clumped
1171 * along with TCP, UDP and SCTP.
1172 */
1173 hash = HASH_ADDR(ipha->ipha_src,
1174 *(uint32_t *)(mp->b_rptr + ports_offset));
1175 indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
1176 type = V4_TCP;
1177 mp->b_rptr += hdrsize;
1178 break;
1179 case IPPROTO_UDP:
1180 case IPPROTO_SCTP:
1181 case IPPROTO_ESP:
1182 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
1183 hash = HASH_ADDR(ipha->ipha_src,
1184 *(uint32_t *)(mp->b_rptr + ports_offset));
1185 indx = COMPUTE_INDEX(hash,
1186 mac_srs->srs_udp_ring_count);
1187 } else {
1188 indx = mac_srs->srs_ind %
1189 mac_srs->srs_udp_ring_count;
1190 mac_srs->srs_ind++;
1191 }
1192 type = V4_UDP;
1193 mp->b_rptr += hdrsize;
1194 break;
1195 default:
1196 indx = 0;
1197 type = OTH;
1198 }
1199
1200 FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx],
1201 cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp);
1202 }
1203
1204 for (type = V4_TCP; type < UNDEF; type++) {
1205 int i;
1206
1207 for (i = 0; i < fanout_cnt; i++) {
1208 if (headmp[type][i] != NULL) {
1209 mac_soft_ring_t *softring;
1210
1211 ASSERT(tailmp[type][i]->b_next == NULL);
1212 switch (type) {
1213 case V4_TCP:
1214 softring =
1215 mac_srs->srs_tcp_soft_rings[i];
1216 break;
1217 case V4_UDP:
1218 softring =
1219 mac_srs->srs_udp_soft_rings[i];
1220 break;
1221 case OTH:
1222 softring =
1223 mac_srs->srs_oth_soft_rings[i];
1224 break;
1225 }
1226 mac_rx_soft_ring_process(mcip,
1227 softring, headmp[type][i], tailmp[type][i],
1228 cnt[type][i], sz[type][i]);
1229 }
1230 }
1231 }
1232 }
1233
1234 #define SRS_BYTES_TO_PICKUP 150000
1235 ssize_t max_bytes_to_pickup = SRS_BYTES_TO_PICKUP;
1236
1237 /*
1238 * mac_rx_srs_poll_ring
1239 *
1240 * This SRS Poll thread uses this routine to poll the underlying hardware
1241 * Rx ring to get a chain of packets. It can inline process that chain
1242 * if mac_latency_optimize is set (default) or signal the SRS worker thread
1243 * to do the remaining processing.
1244 *
1245 * Since packets come in the system via interrupt or poll path, we also
1246 * update the stats and deal with promiscous clients here.
1247 */
1248 void
1249 mac_rx_srs_poll_ring(mac_soft_ring_set_t *mac_srs)
1250 {
1251 kmutex_t *lock = &mac_srs->srs_lock;
1252 kcondvar_t *async = &mac_srs->srs_cv;
1253 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
1254 mblk_t *head, *tail, *mp;
1255 callb_cpr_t cprinfo;
1256 ssize_t bytes_to_pickup;
1257 size_t sz;
1258 int count;
1259 mac_client_impl_t *smcip;
1260
1261 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_srs_poll");
1262 mutex_enter(lock);
1263
1264 start:
1265 for (;;) {
1266 if (mac_srs->srs_state & SRS_PAUSE)
1267 goto done;
1268
1269 CALLB_CPR_SAFE_BEGIN(&cprinfo);
1270 cv_wait(async, lock);
1271 CALLB_CPR_SAFE_END(&cprinfo, lock);
1272
1273 if (mac_srs->srs_state & SRS_PAUSE)
1274 goto done;
1275
1276 check_again:
1277 if (mac_srs->srs_type & SRST_BW_CONTROL) {
1278 /*
1279 * We pick as many bytes as we are allowed to queue.
1280 * Its possible that we will exceed the total
1281 * packets queued in case this SRS is part of the
1282 * Rx ring group since > 1 poll thread can be pulling
1283 * upto the max allowed packets at the same time
1284 * but that should be OK.
1285 */
1286 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1287 bytes_to_pickup =
1288 mac_srs->srs_bw->mac_bw_drop_threshold -
1289 mac_srs->srs_bw->mac_bw_sz;
1290 /*
1291 * We shouldn't have been signalled if we
1292 * have 0 or less bytes to pick but since
1293 * some of the bytes accounting is driver
1294 * dependant, we do the safety check.
1295 */
1296 if (bytes_to_pickup < 0)
1297 bytes_to_pickup = 0;
1298 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1299 } else {
1300 /*
1301 * ToDO: Need to change the polling API
1302 * to add a packet count and a flag which
1303 * tells the driver whether we want packets
1304 * based on a count, or bytes, or all the
1305 * packets queued in the driver/HW. This
1306 * way, we never have to check the limits
1307 * on poll path. We truly let only as many
1308 * packets enter the system as we are willing
1309 * to process or queue.
1310 *
1311 * Something along the lines of
1312 * pkts_to_pickup = mac_soft_ring_max_q_cnt -
1313 * mac_srs->srs_poll_pkt_cnt
1314 */
1315
1316 /*
1317 * Since we are not doing B/W control, pick
1318 * as many packets as allowed.
1319 */
1320 bytes_to_pickup = max_bytes_to_pickup;
1321 }
1322
1323 /* Poll the underlying Hardware */
1324 mutex_exit(lock);
1325 head = MAC_HWRING_POLL(mac_srs->srs_ring, (int)bytes_to_pickup);
1326 mutex_enter(lock);
1327
1328 ASSERT((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
1329 SRS_POLL_THR_OWNER);
1330
1331 mp = tail = head;
1332 count = 0;
1333 sz = 0;
1334 while (mp != NULL) {
1335 tail = mp;
1336 sz += msgdsize(mp);
1337 mp = mp->b_next;
1338 count++;
1339 }
1340
1341 if (head != NULL) {
1342 tail->b_next = NULL;
1343 smcip = mac_srs->srs_mcip;
1344
1345 SRS_RX_STAT_UPDATE(mac_srs, pollbytes, sz);
1346 SRS_RX_STAT_UPDATE(mac_srs, pollcnt, count);
1347
1348 /*
1349 * If there are any promiscuous mode callbacks
1350 * defined for this MAC client, pass them a copy
1351 * if appropriate and also update the counters.
1352 */
1353 if (smcip != NULL) {
1354 if (smcip->mci_mip->mi_promisc_list != NULL) {
1355 mutex_exit(lock);
1356 mac_promisc_dispatch(smcip->mci_mip,
1357 head, NULL);
1358 mutex_enter(lock);
1359 }
1360 }
1361 if (mac_srs->srs_type & SRST_BW_CONTROL) {
1362 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1363 mac_srs->srs_bw->mac_bw_polled += sz;
1364 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1365 }
1366 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail,
1367 count, sz);
1368 if (count <= 10)
1369 srs_rx->sr_stat.mrs_chaincntundr10++;
1370 else if (count > 10 && count <= 50)
1371 srs_rx->sr_stat.mrs_chaincnt10to50++;
1372 else
1373 srs_rx->sr_stat.mrs_chaincntover50++;
1374 }
1375
1376 /*
1377 * We are guaranteed that SRS_PROC will be set if we
1378 * are here. Also, poll thread gets to run only if
1379 * the drain was being done by a worker thread although
1380 * its possible that worker thread is still running
1381 * and poll thread was sent down to keep the pipeline
1382 * going instead of doing a complete drain and then
1383 * trying to poll the NIC.
1384 *
1385 * So we need to check SRS_WORKER flag to make sure
1386 * that the worker thread is not processing the queue
1387 * in parallel to us. The flags and conditions are
1388 * protected by the srs_lock to prevent any race. We
1389 * ensure that we don't drop the srs_lock from now
1390 * till the end and similarly we don't drop the srs_lock
1391 * in mac_rx_srs_drain() till similar condition check
1392 * are complete. The mac_rx_srs_drain() needs to ensure
1393 * that SRS_WORKER flag remains set as long as its
1394 * processing the queue.
1395 */
1396 if (!(mac_srs->srs_state & SRS_WORKER) &&
1397 (mac_srs->srs_first != NULL)) {
1398 /*
1399 * We have packets to process and worker thread
1400 * is not running. Check to see if poll thread is
1401 * allowed to process.
1402 */
1403 if (mac_srs->srs_state & SRS_LATENCY_OPT) {
1404 mac_srs->srs_drain_func(mac_srs, SRS_POLL_PROC);
1405 if (!(mac_srs->srs_state & SRS_PAUSE) &&
1406 srs_rx->sr_poll_pkt_cnt <=
1407 srs_rx->sr_lowat) {
1408 srs_rx->sr_poll_again++;
1409 goto check_again;
1410 }
1411 /*
1412 * We are already above low water mark
1413 * so stay in the polling mode but no
1414 * need to poll. Once we dip below
1415 * the polling threshold, the processing
1416 * thread (soft ring) will signal us
1417 * to poll again (MAC_UPDATE_SRS_COUNT)
1418 */
1419 srs_rx->sr_poll_drain_no_poll++;
1420 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
1421 /*
1422 * In B/W control case, its possible
1423 * that the backlog built up due to
1424 * B/W limit being reached and packets
1425 * are queued only in SRS. In this case,
1426 * we should schedule worker thread
1427 * since no one else will wake us up.
1428 */
1429 if ((mac_srs->srs_type & SRST_BW_CONTROL) &&
1430 (mac_srs->srs_tid == NULL)) {
1431 mac_srs->srs_tid =
1432 timeout(mac_srs_fire, mac_srs, 1);
1433 srs_rx->sr_poll_worker_wakeup++;
1434 }
1435 } else {
1436 /*
1437 * Wakeup the worker thread for more processing.
1438 * We optimize for throughput in this case.
1439 */
1440 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
1441 MAC_SRS_WORKER_WAKEUP(mac_srs);
1442 srs_rx->sr_poll_sig_worker++;
1443 }
1444 } else if ((mac_srs->srs_first == NULL) &&
1445 !(mac_srs->srs_state & SRS_WORKER)) {
1446 /*
1447 * There is nothing queued in SRS and
1448 * no worker thread running. Plus we
1449 * didn't get anything from the H/W
1450 * as well (head == NULL);
1451 */
1452 ASSERT(head == NULL);
1453 mac_srs->srs_state &=
1454 ~(SRS_PROC|SRS_GET_PKTS);
1455
1456 /*
1457 * If we have a packets in soft ring, don't allow
1458 * more packets to come into this SRS by keeping the
1459 * interrupts off but not polling the H/W. The
1460 * poll thread will get signaled as soon as
1461 * srs_poll_pkt_cnt dips below poll threshold.
1462 */
1463 if (srs_rx->sr_poll_pkt_cnt == 0) {
1464 srs_rx->sr_poll_intr_enable++;
1465 MAC_SRS_POLLING_OFF(mac_srs);
1466 } else {
1467 /*
1468 * We know nothing is queued in SRS
1469 * since we are here after checking
1470 * srs_first is NULL. The backlog
1471 * is entirely due to packets queued
1472 * in Soft ring which will wake us up
1473 * and get the interface out of polling
1474 * mode once the backlog dips below
1475 * sr_poll_thres.
1476 */
1477 srs_rx->sr_poll_no_poll++;
1478 }
1479 } else {
1480 /*
1481 * Worker thread is already running.
1482 * Nothing much to do. If the polling
1483 * was enabled, worker thread will deal
1484 * with that.
1485 */
1486 mac_srs->srs_state &= ~SRS_GET_PKTS;
1487 srs_rx->sr_poll_goto_sleep++;
1488 }
1489 }
1490 done:
1491 mac_srs->srs_state |= SRS_POLL_THR_QUIESCED;
1492 cv_signal(&mac_srs->srs_async);
1493 /*
1494 * If this is a temporary quiesce then wait for the restart signal
1495 * from the srs worker. Then clear the flags and signal the srs worker
1496 * to ensure a positive handshake and go back to start.
1497 */
1498 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_POLL_THR_RESTART)))
1499 cv_wait(async, lock);
1500 if (mac_srs->srs_state & SRS_POLL_THR_RESTART) {
1501 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
1502 mac_srs->srs_state &=
1503 ~(SRS_POLL_THR_QUIESCED | SRS_POLL_THR_RESTART);
1504 cv_signal(&mac_srs->srs_async);
1505 goto start;
1506 } else {
1507 mac_srs->srs_state |= SRS_POLL_THR_EXITED;
1508 cv_signal(&mac_srs->srs_async);
1509 CALLB_CPR_EXIT(&cprinfo);
1510 thread_exit();
1511 }
1512 }
1513
1514 /*
1515 * mac_srs_pick_chain
1516 *
1517 * In Bandwidth control case, checks how many packets can be processed
1518 * and return them in a sub chain.
1519 */
1520 static mblk_t *
1521 mac_srs_pick_chain(mac_soft_ring_set_t *mac_srs, mblk_t **chain_tail,
1522 size_t *chain_sz, int *chain_cnt)
1523 {
1524 mblk_t *head = NULL;
1525 mblk_t *tail = NULL;
1526 size_t sz;
1527 size_t tsz = 0;
1528 int cnt = 0;
1529 mblk_t *mp;
1530
1531 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1532 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1533 if (((mac_srs->srs_bw->mac_bw_used + mac_srs->srs_size) <=
1534 mac_srs->srs_bw->mac_bw_limit) ||
1535 (mac_srs->srs_bw->mac_bw_limit == 0)) {
1536 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1537 head = mac_srs->srs_first;
1538 mac_srs->srs_first = NULL;
1539 *chain_tail = mac_srs->srs_last;
1540 mac_srs->srs_last = NULL;
1541 *chain_sz = mac_srs->srs_size;
1542 *chain_cnt = mac_srs->srs_count;
1543 mac_srs->srs_count = 0;
1544 mac_srs->srs_size = 0;
1545 return (head);
1546 }
1547
1548 /*
1549 * Can't clear the entire backlog.
1550 * Need to find how many packets to pick
1551 */
1552 ASSERT(MUTEX_HELD(&mac_srs->srs_bw->mac_bw_lock));
1553 while ((mp = mac_srs->srs_first) != NULL) {
1554 sz = msgdsize(mp);
1555 if ((tsz + sz + mac_srs->srs_bw->mac_bw_used) >
1556 mac_srs->srs_bw->mac_bw_limit) {
1557 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED))
1558 mac_srs->srs_bw->mac_bw_state |=
1559 SRS_BW_ENFORCED;
1560 break;
1561 }
1562
1563 /*
1564 * The _size & cnt is decremented from the softrings
1565 * when they send up the packet for polling to work
1566 * properly.
1567 */
1568 tsz += sz;
1569 cnt++;
1570 mac_srs->srs_count--;
1571 mac_srs->srs_size -= sz;
1572 if (tail != NULL)
1573 tail->b_next = mp;
1574 else
1575 head = mp;
1576 tail = mp;
1577 mac_srs->srs_first = mac_srs->srs_first->b_next;
1578 }
1579 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1580 if (mac_srs->srs_first == NULL)
1581 mac_srs->srs_last = NULL;
1582
1583 if (tail != NULL)
1584 tail->b_next = NULL;
1585 *chain_tail = tail;
1586 *chain_cnt = cnt;
1587 *chain_sz = tsz;
1588
1589 return (head);
1590 }
1591
1592 /*
1593 * mac_rx_srs_drain
1594 *
1595 * The SRS drain routine. Gets to run to clear the queue. Any thread
1596 * (worker, interrupt, poll) can call this based on processing model.
1597 * The first thing we do is disable interrupts if possible and then
1598 * drain the queue. we also try to poll the underlying hardware if
1599 * there is a dedicated hardware Rx ring assigned to this SRS.
1600 *
1601 * There is a equivalent drain routine in bandwidth control mode
1602 * mac_rx_srs_drain_bw. There is some code duplication between the two
1603 * routines but they are highly performance sensitive and are easier
1604 * to read/debug if they stay separate. Any code changes here might
1605 * also apply to mac_rx_srs_drain_bw as well.
1606 */
1607 void
1608 mac_rx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
1609 {
1610 mblk_t *head;
1611 mblk_t *tail;
1612 timeout_id_t tid;
1613 int cnt = 0;
1614 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1615 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
1616
1617 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1618 ASSERT(!(mac_srs->srs_type & SRST_BW_CONTROL));
1619
1620 /* If we are blanked i.e. can't do upcalls, then we are done */
1621 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
1622 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
1623 (mac_srs->srs_state & SRS_PAUSE));
1624 goto out;
1625 }
1626
1627 if (mac_srs->srs_first == NULL)
1628 goto out;
1629
1630 if (!(mac_srs->srs_state & SRS_LATENCY_OPT) &&
1631 (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)) {
1632 /*
1633 * In the normal case, the SRS worker thread does no
1634 * work and we wait for a backlog to build up before
1635 * we switch into polling mode. In case we are
1636 * optimizing for throughput, we use the worker thread
1637 * as well. The goal is to let worker thread process
1638 * the queue and poll thread to feed packets into
1639 * the queue. As such, we should signal the poll
1640 * thread to try and get more packets.
1641 *
1642 * We could have pulled this check in the POLL_RING
1643 * macro itself but keeping it explicit here makes
1644 * the architecture more human understandable.
1645 */
1646 MAC_SRS_POLL_RING(mac_srs);
1647 }
1648
1649 again:
1650 head = mac_srs->srs_first;
1651 mac_srs->srs_first = NULL;
1652 tail = mac_srs->srs_last;
1653 mac_srs->srs_last = NULL;
1654 cnt = mac_srs->srs_count;
1655 mac_srs->srs_count = 0;
1656
1657 ASSERT(head != NULL);
1658 ASSERT(tail != NULL);
1659
1660 if ((tid = mac_srs->srs_tid) != 0)
1661 mac_srs->srs_tid = 0;
1662
1663 mac_srs->srs_state |= (SRS_PROC|proc_type);
1664
1665
1666 /*
1667 * mcip is NULL for broadcast and multicast flows. The promisc
1668 * callbacks for broadcast and multicast packets are delivered from
1669 * mac_rx() and we don't need to worry about that case in this path
1670 */
1671 if (mcip != NULL) {
1672 if (mcip->mci_promisc_list != NULL) {
1673 mutex_exit(&mac_srs->srs_lock);
1674 mac_promisc_client_dispatch(mcip, head);
1675 mutex_enter(&mac_srs->srs_lock);
1676 }
1677 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
1678 mutex_exit(&mac_srs->srs_lock);
1679 mac_protect_intercept_dhcp(mcip, head);
1680 mutex_enter(&mac_srs->srs_lock);
1681 }
1682 }
1683
1684 /*
1685 * Check if SRS itself is doing the processing
1686 * This direct path does not apply when subflows are present. In this
1687 * case, packets need to be dispatched to a soft ring according to the
1688 * flow's bandwidth and other resources contraints.
1689 */
1690 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
1691 mac_direct_rx_t proc;
1692 void *arg1;
1693 mac_resource_handle_t arg2;
1694
1695 /*
1696 * This is the case when a Rx is directly
1697 * assigned and we have a fully classified
1698 * protocol chain. We can deal with it in
1699 * one shot.
1700 */
1701 proc = srs_rx->sr_func;
1702 arg1 = srs_rx->sr_arg1;
1703 arg2 = srs_rx->sr_arg2;
1704
1705 mac_srs->srs_state |= SRS_CLIENT_PROC;
1706 mutex_exit(&mac_srs->srs_lock);
1707 if (tid != 0) {
1708 (void) untimeout(tid);
1709 tid = 0;
1710 }
1711
1712 proc(arg1, arg2, head, NULL);
1713 /*
1714 * Decrement the size and count here itelf
1715 * since the packet has been processed.
1716 */
1717 mutex_enter(&mac_srs->srs_lock);
1718 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
1719 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
1720 cv_signal(&mac_srs->srs_client_cv);
1721 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
1722 } else {
1723 /* Some kind of softrings based fanout is required */
1724 mutex_exit(&mac_srs->srs_lock);
1725 if (tid != 0) {
1726 (void) untimeout(tid);
1727 tid = 0;
1728 }
1729
1730 /*
1731 * Since the fanout routines can deal with chains,
1732 * shoot the entire chain up.
1733 */
1734 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
1735 mac_rx_srs_fanout(mac_srs, head);
1736 else
1737 mac_rx_srs_proto_fanout(mac_srs, head);
1738 mutex_enter(&mac_srs->srs_lock);
1739 }
1740
1741 if (!(mac_srs->srs_state & (SRS_BLANK|SRS_PAUSE)) &&
1742 (mac_srs->srs_first != NULL)) {
1743 /*
1744 * More packets arrived while we were clearing the
1745 * SRS. This can be possible because of one of
1746 * three conditions below:
1747 * 1) The driver is using multiple worker threads
1748 * to send the packets to us.
1749 * 2) The driver has a race in switching
1750 * between interrupt and polling mode or
1751 * 3) Packets are arriving in this SRS via the
1752 * S/W classification as well.
1753 *
1754 * We should switch to polling mode and see if we
1755 * need to send the poll thread down. Also, signal
1756 * the worker thread to process whats just arrived.
1757 */
1758 MAC_SRS_POLLING_ON(mac_srs);
1759 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) {
1760 srs_rx->sr_drain_poll_sig++;
1761 MAC_SRS_POLL_RING(mac_srs);
1762 }
1763
1764 /*
1765 * If we didn't signal the poll thread, we need
1766 * to deal with the pending packets ourselves.
1767 */
1768 if (proc_type == SRS_WORKER) {
1769 srs_rx->sr_drain_again++;
1770 goto again;
1771 } else {
1772 srs_rx->sr_drain_worker_sig++;
1773 cv_signal(&mac_srs->srs_async);
1774 }
1775 }
1776
1777 out:
1778 if (mac_srs->srs_state & SRS_GET_PKTS) {
1779 /*
1780 * Poll thread is already running. Leave the
1781 * SRS_RPOC set and hand over the control to
1782 * poll thread.
1783 */
1784 mac_srs->srs_state &= ~proc_type;
1785 srs_rx->sr_drain_poll_running++;
1786 return;
1787 }
1788
1789 /*
1790 * Even if there are no packets queued in SRS, we
1791 * need to make sure that the shared counter is
1792 * clear and any associated softrings have cleared
1793 * all the backlog. Otherwise, leave the interface
1794 * in polling mode and the poll thread will get
1795 * signalled once the count goes down to zero.
1796 *
1797 * If someone is already draining the queue (SRS_PROC is
1798 * set) when the srs_poll_pkt_cnt goes down to zero,
1799 * then it means that drain is already running and we
1800 * will turn off polling at that time if there is
1801 * no backlog.
1802 *
1803 * As long as there are packets queued either
1804 * in soft ring set or its soft rings, we will leave
1805 * the interface in polling mode (even if the drain
1806 * was done being the interrupt thread). We signal
1807 * the poll thread as well if we have dipped below
1808 * low water mark.
1809 *
1810 * NOTE: We can't use the MAC_SRS_POLLING_ON macro
1811 * since that turn polling on only for worker thread.
1812 * Its not worth turning polling on for interrupt
1813 * thread (since NIC will not issue another interrupt)
1814 * unless a backlog builds up.
1815 */
1816 if ((srs_rx->sr_poll_pkt_cnt > 0) &&
1817 (mac_srs->srs_state & SRS_POLLING_CAPAB)) {
1818 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
1819 srs_rx->sr_drain_keep_polling++;
1820 MAC_SRS_POLLING_ON(mac_srs);
1821 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)
1822 MAC_SRS_POLL_RING(mac_srs);
1823 return;
1824 }
1825
1826 /* Nothing else to do. Get out of poll mode */
1827 MAC_SRS_POLLING_OFF(mac_srs);
1828 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
1829 srs_rx->sr_drain_finish_intr++;
1830 }
1831
1832 /*
1833 * mac_rx_srs_drain_bw
1834 *
1835 * The SRS BW drain routine. Gets to run to clear the queue. Any thread
1836 * (worker, interrupt, poll) can call this based on processing model.
1837 * The first thing we do is disable interrupts if possible and then
1838 * drain the queue. we also try to poll the underlying hardware if
1839 * there is a dedicated hardware Rx ring assigned to this SRS.
1840 *
1841 * There is a equivalent drain routine in non bandwidth control mode
1842 * mac_rx_srs_drain. There is some code duplication between the two
1843 * routines but they are highly performance sensitive and are easier
1844 * to read/debug if they stay separate. Any code changes here might
1845 * also apply to mac_rx_srs_drain as well.
1846 */
1847 void
1848 mac_rx_srs_drain_bw(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
1849 {
1850 mblk_t *head;
1851 mblk_t *tail;
1852 timeout_id_t tid;
1853 size_t sz = 0;
1854 int cnt = 0;
1855 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1856 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
1857 clock_t now;
1858
1859 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1860 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
1861 again:
1862 /* Check if we are doing B/W control */
1863 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1864 now = ddi_get_lbolt();
1865 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
1866 mac_srs->srs_bw->mac_bw_curr_time = now;
1867 mac_srs->srs_bw->mac_bw_used = 0;
1868 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
1869 mac_srs->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED;
1870 } else if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) {
1871 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1872 goto done;
1873 } else if (mac_srs->srs_bw->mac_bw_used >
1874 mac_srs->srs_bw->mac_bw_limit) {
1875 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
1876 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1877 goto done;
1878 }
1879 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1880
1881 /* If we are blanked i.e. can't do upcalls, then we are done */
1882 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
1883 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
1884 (mac_srs->srs_state & SRS_PAUSE));
1885 goto done;
1886 }
1887
1888 sz = 0;
1889 cnt = 0;
1890 if ((head = mac_srs_pick_chain(mac_srs, &tail, &sz, &cnt)) == NULL) {
1891 /*
1892 * We couldn't pick up a single packet.
1893 */
1894 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1895 if ((mac_srs->srs_bw->mac_bw_used == 0) &&
1896 (mac_srs->srs_size != 0) &&
1897 !(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
1898 /*
1899 * Seems like configured B/W doesn't
1900 * even allow processing of 1 packet
1901 * per tick.
1902 *
1903 * XXX: raise the limit to processing
1904 * at least 1 packet per tick.
1905 */
1906 mac_srs->srs_bw->mac_bw_limit +=
1907 mac_srs->srs_bw->mac_bw_limit;
1908 mac_srs->srs_bw->mac_bw_drop_threshold +=
1909 mac_srs->srs_bw->mac_bw_drop_threshold;
1910 cmn_err(CE_NOTE, "mac_rx_srs_drain: srs(%p) "
1911 "raised B/W limit to %d since not even a "
1912 "single packet can be processed per "
1913 "tick %d\n", (void *)mac_srs,
1914 (int)mac_srs->srs_bw->mac_bw_limit,
1915 (int)msgdsize(mac_srs->srs_first));
1916 }
1917 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1918 goto done;
1919 }
1920
1921 ASSERT(head != NULL);
1922 ASSERT(tail != NULL);
1923
1924 /* zero bandwidth: drop all and return to interrupt mode */
1925 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1926 if (mac_srs->srs_bw->mac_bw_limit == 0) {
1927 srs_rx->sr_stat.mrs_sdrops += cnt;
1928 ASSERT(mac_srs->srs_bw->mac_bw_sz >= sz);
1929 mac_srs->srs_bw->mac_bw_sz -= sz;
1930 mac_srs->srs_bw->mac_bw_drop_bytes += sz;
1931 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1932 mac_pkt_drop(NULL, NULL, head, B_FALSE);
1933 goto leave_poll;
1934 } else {
1935 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1936 }
1937
1938 if ((tid = mac_srs->srs_tid) != 0)
1939 mac_srs->srs_tid = 0;
1940
1941 mac_srs->srs_state |= (SRS_PROC|proc_type);
1942 MAC_SRS_WORKER_POLLING_ON(mac_srs);
1943
1944 /*
1945 * mcip is NULL for broadcast and multicast flows. The promisc
1946 * callbacks for broadcast and multicast packets are delivered from
1947 * mac_rx() and we don't need to worry about that case in this path
1948 */
1949 if (mcip != NULL) {
1950 if (mcip->mci_promisc_list != NULL) {
1951 mutex_exit(&mac_srs->srs_lock);
1952 mac_promisc_client_dispatch(mcip, head);
1953 mutex_enter(&mac_srs->srs_lock);
1954 }
1955 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
1956 mutex_exit(&mac_srs->srs_lock);
1957 mac_protect_intercept_dhcp(mcip, head);
1958 mutex_enter(&mac_srs->srs_lock);
1959 }
1960 }
1961
1962 /*
1963 * Check if SRS itself is doing the processing
1964 * This direct path does not apply when subflows are present. In this
1965 * case, packets need to be dispatched to a soft ring according to the
1966 * flow's bandwidth and other resources contraints.
1967 */
1968 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
1969 mac_direct_rx_t proc;
1970 void *arg1;
1971 mac_resource_handle_t arg2;
1972
1973 /*
1974 * This is the case when a Rx is directly
1975 * assigned and we have a fully classified
1976 * protocol chain. We can deal with it in
1977 * one shot.
1978 */
1979 proc = srs_rx->sr_func;
1980 arg1 = srs_rx->sr_arg1;
1981 arg2 = srs_rx->sr_arg2;
1982
1983 mac_srs->srs_state |= SRS_CLIENT_PROC;
1984 mutex_exit(&mac_srs->srs_lock);
1985 if (tid != 0) {
1986 (void) untimeout(tid);
1987 tid = 0;
1988 }
1989
1990 proc(arg1, arg2, head, NULL);
1991 /*
1992 * Decrement the size and count here itelf
1993 * since the packet has been processed.
1994 */
1995 mutex_enter(&mac_srs->srs_lock);
1996 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
1997 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
1998
1999 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
2000 cv_signal(&mac_srs->srs_client_cv);
2001 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
2002 } else {
2003 /* Some kind of softrings based fanout is required */
2004 mutex_exit(&mac_srs->srs_lock);
2005 if (tid != 0) {
2006 (void) untimeout(tid);
2007 tid = 0;
2008 }
2009
2010 /*
2011 * Since the fanout routines can deal with chains,
2012 * shoot the entire chain up.
2013 */
2014 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
2015 mac_rx_srs_fanout(mac_srs, head);
2016 else
2017 mac_rx_srs_proto_fanout(mac_srs, head);
2018 mutex_enter(&mac_srs->srs_lock);
2019 }
2020
2021 /*
2022 * Send the poll thread to pick up any packets arrived
2023 * so far. This also serves as the last check in case
2024 * nothing else is queued in the SRS. The poll thread
2025 * is signalled only in the case the drain was done
2026 * by the worker thread and SRS_WORKER is set. The
2027 * worker thread can run in parallel as long as the
2028 * SRS_WORKER flag is set. We we have nothing else to
2029 * process, we can exit while leaving SRS_PROC set
2030 * which gives the poll thread control to process and
2031 * cleanup once it returns from the NIC.
2032 *
2033 * If we have nothing else to process, we need to
2034 * ensure that we keep holding the srs_lock till
2035 * all the checks below are done and control is
2036 * handed to the poll thread if it was running.
2037 */
2038 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2039 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2040 if (mac_srs->srs_first != NULL) {
2041 if (proc_type == SRS_WORKER) {
2042 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2043 if (srs_rx->sr_poll_pkt_cnt <=
2044 srs_rx->sr_lowat)
2045 MAC_SRS_POLL_RING(mac_srs);
2046 goto again;
2047 } else {
2048 cv_signal(&mac_srs->srs_async);
2049 }
2050 }
2051 }
2052 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2053
2054 done:
2055
2056 if (mac_srs->srs_state & SRS_GET_PKTS) {
2057 /*
2058 * Poll thread is already running. Leave the
2059 * SRS_RPOC set and hand over the control to
2060 * poll thread.
2061 */
2062 mac_srs->srs_state &= ~proc_type;
2063 return;
2064 }
2065
2066 /*
2067 * If we can't process packets because we have exceeded
2068 * B/W limit for this tick, just set the timeout
2069 * and leave.
2070 *
2071 * Even if there are no packets queued in SRS, we
2072 * need to make sure that the shared counter is
2073 * clear and any associated softrings have cleared
2074 * all the backlog. Otherwise, leave the interface
2075 * in polling mode and the poll thread will get
2076 * signalled once the count goes down to zero.
2077 *
2078 * If someone is already draining the queue (SRS_PROC is
2079 * set) when the srs_poll_pkt_cnt goes down to zero,
2080 * then it means that drain is already running and we
2081 * will turn off polling at that time if there is
2082 * no backlog. As long as there are packets queued either
2083 * is soft ring set or its soft rings, we will leave
2084 * the interface in polling mode.
2085 */
2086 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2087 if ((mac_srs->srs_state & SRS_POLLING_CAPAB) &&
2088 ((mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) ||
2089 (srs_rx->sr_poll_pkt_cnt > 0))) {
2090 MAC_SRS_POLLING_ON(mac_srs);
2091 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2092 if ((mac_srs->srs_first != NULL) &&
2093 (mac_srs->srs_tid == NULL))
2094 mac_srs->srs_tid = timeout(mac_srs_fire,
2095 mac_srs, 1);
2096 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2097 return;
2098 }
2099 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2100
2101 leave_poll:
2102
2103 /* Nothing else to do. Get out of poll mode */
2104 MAC_SRS_POLLING_OFF(mac_srs);
2105 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2106 }
2107
2108 /*
2109 * mac_srs_worker
2110 *
2111 * The SRS worker routine. Drains the queue when no one else is
2112 * processing it.
2113 */
2114 void
2115 mac_srs_worker(mac_soft_ring_set_t *mac_srs)
2116 {
2117 kmutex_t *lock = &mac_srs->srs_lock;
2118 kcondvar_t *async = &mac_srs->srs_async;
2119 callb_cpr_t cprinfo;
2120 boolean_t bw_ctl_flag;
2121
2122 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "srs_worker");
2123 mutex_enter(lock);
2124
2125 start:
2126 for (;;) {
2127 bw_ctl_flag = B_FALSE;
2128 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2129 MAC_SRS_BW_LOCK(mac_srs);
2130 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
2131 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
2132 bw_ctl_flag = B_TRUE;
2133 MAC_SRS_BW_UNLOCK(mac_srs);
2134 }
2135 /*
2136 * The SRS_BW_ENFORCED flag may change since we have dropped
2137 * the mac_bw_lock. However the drain function can handle both
2138 * a drainable SRS or a bandwidth controlled SRS, and the
2139 * effect of scheduling a timeout is to wakeup the worker
2140 * thread which in turn will call the drain function. Since
2141 * we release the srs_lock atomically only in the cv_wait there
2142 * isn't a fear of waiting for ever.
2143 */
2144 while (((mac_srs->srs_state & SRS_PROC) ||
2145 (mac_srs->srs_first == NULL) || bw_ctl_flag ||
2146 (mac_srs->srs_state & SRS_TX_BLOCKED)) &&
2147 !(mac_srs->srs_state & SRS_PAUSE)) {
2148 /*
2149 * If we have packets queued and we are here
2150 * because B/W control is in place, we better
2151 * schedule the worker wakeup after 1 tick
2152 * to see if bandwidth control can be relaxed.
2153 */
2154 if (bw_ctl_flag && mac_srs->srs_tid == NULL) {
2155 /*
2156 * We need to ensure that a timer is already
2157 * scheduled or we force schedule one for
2158 * later so that we can continue processing
2159 * after this quanta is over.
2160 */
2161 mac_srs->srs_tid = timeout(mac_srs_fire,
2162 mac_srs, 1);
2163 }
2164 wait:
2165 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2166 cv_wait(async, lock);
2167 CALLB_CPR_SAFE_END(&cprinfo, lock);
2168
2169 if (mac_srs->srs_state & SRS_PAUSE)
2170 goto done;
2171 if (mac_srs->srs_state & SRS_PROC)
2172 goto wait;
2173
2174 if (mac_srs->srs_first != NULL &&
2175 mac_srs->srs_type & SRST_BW_CONTROL) {
2176 MAC_SRS_BW_LOCK(mac_srs);
2177 if (mac_srs->srs_bw->mac_bw_state &
2178 SRS_BW_ENFORCED) {
2179 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
2180 }
2181 bw_ctl_flag = mac_srs->srs_bw->mac_bw_state &
2182 SRS_BW_ENFORCED;
2183 MAC_SRS_BW_UNLOCK(mac_srs);
2184 }
2185 }
2186
2187 if (mac_srs->srs_state & SRS_PAUSE)
2188 goto done;
2189 mac_srs->srs_drain_func(mac_srs, SRS_WORKER);
2190 }
2191 done:
2192 /*
2193 * The Rx SRS quiesce logic first cuts off packet supply to the SRS
2194 * from both hard and soft classifications and waits for such threads
2195 * to finish before signaling the worker. So at this point the only
2196 * thread left that could be competing with the worker is the poll
2197 * thread. In the case of Tx, there shouldn't be any thread holding
2198 * SRS_PROC at this point.
2199 */
2200 if (!(mac_srs->srs_state & SRS_PROC)) {
2201 mac_srs->srs_state |= SRS_PROC;
2202 } else {
2203 ASSERT((mac_srs->srs_type & SRST_TX) == 0);
2204 /*
2205 * Poll thread still owns the SRS and is still running
2206 */
2207 ASSERT((mac_srs->srs_poll_thr == NULL) ||
2208 ((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
2209 SRS_POLL_THR_OWNER));
2210 }
2211 mac_srs_worker_quiesce(mac_srs);
2212 /*
2213 * Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator
2214 * of the quiesce operation
2215 */
2216 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_RESTART)))
2217 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
2218
2219 if (mac_srs->srs_state & SRS_RESTART) {
2220 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
2221 mac_srs_worker_restart(mac_srs);
2222 mac_srs->srs_state &= ~SRS_PROC;
2223 goto start;
2224 }
2225
2226 if (!(mac_srs->srs_state & SRS_CONDEMNED_DONE))
2227 mac_srs_worker_quiesce(mac_srs);
2228
2229 mac_srs->srs_state &= ~SRS_PROC;
2230 /* The macro drops the srs_lock */
2231 CALLB_CPR_EXIT(&cprinfo);
2232 thread_exit();
2233 }
2234
2235 /*
2236 * mac_rx_srs_subflow_process
2237 *
2238 * Receive side routine called from interrupt path when there are
2239 * sub flows present on this SRS.
2240 */
2241 /* ARGSUSED */
2242 void
2243 mac_rx_srs_subflow_process(void *arg, mac_resource_handle_t srs,
2244 mblk_t *mp_chain, boolean_t loopback)
2245 {
2246 flow_entry_t *flent = NULL;
2247 flow_entry_t *prev_flent = NULL;
2248 mblk_t *mp = NULL;
2249 mblk_t *tail = NULL;
2250 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
2251 mac_client_impl_t *mcip;
2252
2253 mcip = mac_srs->srs_mcip;
2254 ASSERT(mcip != NULL);
2255
2256 /*
2257 * We need to determine the SRS for every packet
2258 * by walking the flow table, if we don't get any,
2259 * then we proceed using the SRS we came with.
2260 */
2261 mp = tail = mp_chain;
2262 while (mp != NULL) {
2263
2264 /*
2265 * We will increment the stats for the mactching subflow.
2266 * when we get the bytes/pkt count for the classified packets
2267 * later in mac_rx_srs_process.
2268 */
2269 (void) mac_flow_lookup(mcip->mci_subflow_tab, mp,
2270 FLOW_INBOUND, &flent);
2271
2272 if (mp == mp_chain || flent == prev_flent) {
2273 if (prev_flent != NULL)
2274 FLOW_REFRELE(prev_flent);
2275 prev_flent = flent;
2276 flent = NULL;
2277 tail = mp;
2278 mp = mp->b_next;
2279 continue;
2280 }
2281 tail->b_next = NULL;
2282 /*
2283 * A null indicates, this is for the mac_srs itself.
2284 * XXX-venu : probably assert for fe_rx_srs_cnt == 0.
2285 */
2286 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
2287 mac_rx_srs_process(arg,
2288 (mac_resource_handle_t)mac_srs, mp_chain,
2289 loopback);
2290 } else {
2291 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
2292 prev_flent->fe_cb_arg2, mp_chain, loopback);
2293 FLOW_REFRELE(prev_flent);
2294 }
2295 prev_flent = flent;
2296 flent = NULL;
2297 mp_chain = mp;
2298 tail = mp;
2299 mp = mp->b_next;
2300 }
2301 /* Last chain */
2302 ASSERT(mp_chain != NULL);
2303 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
2304 mac_rx_srs_process(arg,
2305 (mac_resource_handle_t)mac_srs, mp_chain, loopback);
2306 } else {
2307 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
2308 prev_flent->fe_cb_arg2, mp_chain, loopback);
2309 FLOW_REFRELE(prev_flent);
2310 }
2311 }
2312
2313 /*
2314 * mac_rx_srs_process
2315 *
2316 * Receive side routine called from the interrupt path.
2317 *
2318 * loopback is set to force a context switch on the loopback
2319 * path between MAC clients.
2320 */
2321 /* ARGSUSED */
2322 void
2323 mac_rx_srs_process(void *arg, mac_resource_handle_t srs, mblk_t *mp_chain,
2324 boolean_t loopback)
2325 {
2326 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
2327 mblk_t *mp, *tail, *head;
2328 int count = 0;
2329 int count1;
2330 size_t sz = 0;
2331 size_t chain_sz, sz1;
2332 mac_bw_ctl_t *mac_bw;
2333 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2334
2335 /*
2336 * Set the tail, count and sz. We set the sz irrespective
2337 * of whether we are doing B/W control or not for the
2338 * purpose of updating the stats.
2339 */
2340 mp = tail = mp_chain;
2341 while (mp != NULL) {
2342 tail = mp;
2343 count++;
2344 sz += msgdsize(mp);
2345 mp = mp->b_next;
2346 }
2347
2348 mutex_enter(&mac_srs->srs_lock);
2349
2350 if (loopback) {
2351 SRS_RX_STAT_UPDATE(mac_srs, lclbytes, sz);
2352 SRS_RX_STAT_UPDATE(mac_srs, lclcnt, count);
2353
2354 } else {
2355 SRS_RX_STAT_UPDATE(mac_srs, intrbytes, sz);
2356 SRS_RX_STAT_UPDATE(mac_srs, intrcnt, count);
2357 }
2358
2359 /*
2360 * If the SRS in already being processed; has been blanked;
2361 * can be processed by worker thread only; or the B/W limit
2362 * has been reached, then queue the chain and check if
2363 * worker thread needs to be awakend.
2364 */
2365 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2366 mac_bw = mac_srs->srs_bw;
2367 ASSERT(mac_bw != NULL);
2368 mutex_enter(&mac_bw->mac_bw_lock);
2369 mac_bw->mac_bw_intr += sz;
2370 if (mac_bw->mac_bw_limit == 0) {
2371 /* zero bandwidth: drop all */
2372 srs_rx->sr_stat.mrs_sdrops += count;
2373 mac_bw->mac_bw_drop_bytes += sz;
2374 mutex_exit(&mac_bw->mac_bw_lock);
2375 mutex_exit(&mac_srs->srs_lock);
2376 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
2377 return;
2378 } else {
2379 if ((mac_bw->mac_bw_sz + sz) <=
2380 mac_bw->mac_bw_drop_threshold) {
2381 mutex_exit(&mac_bw->mac_bw_lock);
2382 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain,
2383 tail, count, sz);
2384 } else {
2385 mp = mp_chain;
2386 chain_sz = 0;
2387 count1 = 0;
2388 tail = NULL;
2389 head = NULL;
2390 while (mp != NULL) {
2391 sz1 = msgdsize(mp);
2392 if (mac_bw->mac_bw_sz + chain_sz + sz1 >
2393 mac_bw->mac_bw_drop_threshold)
2394 break;
2395 chain_sz += sz1;
2396 count1++;
2397 tail = mp;
2398 mp = mp->b_next;
2399 }
2400 mutex_exit(&mac_bw->mac_bw_lock);
2401 if (tail != NULL) {
2402 head = tail->b_next;
2403 tail->b_next = NULL;
2404 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs,
2405 mp_chain, tail, count1, chain_sz);
2406 sz -= chain_sz;
2407 count -= count1;
2408 } else {
2409 /* Can't pick up any */
2410 head = mp_chain;
2411 }
2412 if (head != NULL) {
2413 /* Drop any packet over the threshold */
2414 srs_rx->sr_stat.mrs_sdrops += count;
2415 mutex_enter(&mac_bw->mac_bw_lock);
2416 mac_bw->mac_bw_drop_bytes += sz;
2417 mutex_exit(&mac_bw->mac_bw_lock);
2418 freemsgchain(head);
2419 }
2420 }
2421 MAC_SRS_WORKER_WAKEUP(mac_srs);
2422 mutex_exit(&mac_srs->srs_lock);
2423 return;
2424 }
2425 }
2426
2427 /*
2428 * If the total number of packets queued in the SRS and
2429 * its associated soft rings exceeds the max allowed,
2430 * then drop the chain. If we are polling capable, this
2431 * shouldn't be happening.
2432 */
2433 if (!(mac_srs->srs_type & SRST_BW_CONTROL) &&
2434 (srs_rx->sr_poll_pkt_cnt > srs_rx->sr_hiwat)) {
2435 mac_bw = mac_srs->srs_bw;
2436 srs_rx->sr_stat.mrs_sdrops += count;
2437 mutex_enter(&mac_bw->mac_bw_lock);
2438 mac_bw->mac_bw_drop_bytes += sz;
2439 mutex_exit(&mac_bw->mac_bw_lock);
2440 freemsgchain(mp_chain);
2441 mutex_exit(&mac_srs->srs_lock);
2442 return;
2443 }
2444
2445 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, tail, count, sz);
2446
2447 if (!(mac_srs->srs_state & SRS_PROC)) {
2448 /*
2449 * If we are coming via loopback, if we are not optimizing for
2450 * latency, or if our stack is running deep, we should signal
2451 * the worker thread.
2452 */
2453 if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT) ||
2454 MAC_RX_SRS_TOODEEP()) {
2455 /*
2456 * For loopback, We need to let the worker take
2457 * over as we don't want to continue in the same
2458 * thread even if we can. This could lead to stack
2459 * overflows and may also end up using
2460 * resources (cpu) incorrectly.
2461 */
2462 cv_signal(&mac_srs->srs_async);
2463 } else {
2464 /*
2465 * Seems like no one is processing the SRS and
2466 * there is no backlog. We also inline process
2467 * our packet if its a single packet in non
2468 * latency optimized case (in latency optimized
2469 * case, we inline process chains of any size).
2470 */
2471 mac_srs->srs_drain_func(mac_srs, SRS_PROC_FAST);
2472 }
2473 }
2474 mutex_exit(&mac_srs->srs_lock);
2475 }
2476
2477 /* TX SIDE ROUTINES (RUNTIME) */
2478
2479 /*
2480 * mac_tx_srs_no_desc
2481 *
2482 * This routine is called by Tx single ring default mode
2483 * when Tx ring runs out of descs.
2484 */
2485 mac_tx_cookie_t
2486 mac_tx_srs_no_desc(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2487 uint16_t flag, mblk_t **ret_mp)
2488 {
2489 mac_tx_cookie_t cookie = NULL;
2490 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2491 boolean_t wakeup_worker = B_TRUE;
2492 uint32_t tx_mode = srs_tx->st_mode;
2493 int cnt, sz;
2494 mblk_t *tail;
2495
2496 ASSERT(tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_BW);
2497 if (flag & MAC_DROP_ON_NO_DESC) {
2498 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2499 } else {
2500 if (mac_srs->srs_first != NULL)
2501 wakeup_worker = B_FALSE;
2502 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2503 if (flag & MAC_TX_NO_ENQUEUE) {
2504 /*
2505 * If TX_QUEUED is not set, queue the
2506 * packet and let mac_tx_srs_drain()
2507 * set the TX_BLOCKED bit for the
2508 * reasons explained above. Otherwise,
2509 * return the mblks.
2510 */
2511 if (wakeup_worker) {
2512 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2513 mp_chain, tail, cnt, sz);
2514 } else {
2515 MAC_TX_SET_NO_ENQUEUE(mac_srs,
2516 mp_chain, ret_mp, cookie);
2517 }
2518 } else {
2519 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
2520 tail, cnt, sz, cookie);
2521 }
2522 if (wakeup_worker)
2523 cv_signal(&mac_srs->srs_async);
2524 }
2525 return (cookie);
2526 }
2527
2528 /*
2529 * mac_tx_srs_enqueue
2530 *
2531 * This routine is called when Tx SRS is operating in either serializer
2532 * or bandwidth mode. In serializer mode, a packet will get enqueued
2533 * when a thread cannot enter SRS exclusively. In bandwidth mode,
2534 * packets gets queued if allowed byte-count limit for a tick is
2535 * exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and
2536 * MAC_TX_NO_ENQUEUE is set is different than when operaing in either
2537 * the default mode or fanout mode. Here packets get dropped or
2538 * returned back to the caller only after hi-watermark worth of data
2539 * is queued.
2540 */
2541 static mac_tx_cookie_t
2542 mac_tx_srs_enqueue(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2543 uint16_t flag, uintptr_t fanout_hint, mblk_t **ret_mp)
2544 {
2545 mac_tx_cookie_t cookie = NULL;
2546 int cnt, sz;
2547 mblk_t *tail;
2548 boolean_t wakeup_worker = B_TRUE;
2549
2550 /*
2551 * Ignore fanout hint if we don't have multiple tx rings.
2552 */
2553 if (!MAC_TX_SOFT_RINGS(mac_srs))
2554 fanout_hint = 0;
2555
2556 if (mac_srs->srs_first != NULL)
2557 wakeup_worker = B_FALSE;
2558 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2559 if (flag & MAC_DROP_ON_NO_DESC) {
2560 if (mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) {
2561 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2562 } else {
2563 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2564 mp_chain, tail, cnt, sz);
2565 }
2566 } else if (flag & MAC_TX_NO_ENQUEUE) {
2567 if ((mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) ||
2568 (mac_srs->srs_state & SRS_TX_WAKEUP_CLIENT)) {
2569 MAC_TX_SET_NO_ENQUEUE(mac_srs, mp_chain,
2570 ret_mp, cookie);
2571 } else {
2572 mp_chain->b_prev = (mblk_t *)fanout_hint;
2573 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2574 mp_chain, tail, cnt, sz);
2575 }
2576 } else {
2577 /*
2578 * If you are BW_ENFORCED, just enqueue the
2579 * packet. srs_worker will drain it at the
2580 * prescribed rate. Before enqueueing, save
2581 * the fanout hint.
2582 */
2583 mp_chain->b_prev = (mblk_t *)fanout_hint;
2584 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
2585 tail, cnt, sz, cookie);
2586 }
2587 if (wakeup_worker)
2588 cv_signal(&mac_srs->srs_async);
2589 return (cookie);
2590 }
2591
2592 /*
2593 * There are seven tx modes:
2594 *
2595 * 1) Default mode (SRS_TX_DEFAULT)
2596 * 2) Serialization mode (SRS_TX_SERIALIZE)
2597 * 3) Fanout mode (SRS_TX_FANOUT)
2598 * 4) Bandwdith mode (SRS_TX_BW)
2599 * 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT)
2600 * 6) aggr Tx mode (SRS_TX_AGGR)
2601 * 7) aggr Tx bw mode (SRS_TX_BW_AGGR)
2602 *
2603 * The tx mode in which an SRS operates is decided in mac_tx_srs_setup()
2604 * based on the number of Tx rings requested for an SRS and whether
2605 * bandwidth control is requested or not.
2606 *
2607 * The default mode (i.e., no fanout/no bandwidth) is used when the
2608 * underlying NIC does not have Tx rings or just one Tx ring. In this mode,
2609 * the SRS acts as a pass-thru. Packets will go directly to mac_tx_send().
2610 * When the underlying Tx ring runs out of Tx descs, it starts queueing up
2611 * packets in SRS. When flow-control is relieved, the srs_worker drains
2612 * the queued packets and informs blocked clients to restart sending
2613 * packets.
2614 *
2615 * In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized. This
2616 * mode is used when the link has no Tx rings or only one Tx ring.
2617 *
2618 * In the SRS_TX_FANOUT mode, packets will be fanned out to multiple
2619 * Tx rings. Each Tx ring will have a soft ring associated with it.
2620 * These soft rings will be hung off the Tx SRS. Queueing if it happens
2621 * due to lack of Tx desc will be in individual soft ring (and not srs)
2622 * associated with Tx ring.
2623 *
2624 * In the TX_BW mode, tx srs will allow packets to go down to Tx ring
2625 * only if bw is available. Otherwise the packets will be queued in
2626 * SRS. If fanout to multiple Tx rings is configured, the packets will
2627 * be fanned out among the soft rings associated with the Tx rings.
2628 *
2629 * In SRS_TX_AGGR mode, mac_tx_aggr_mode() routine is called. This routine
2630 * invokes an aggr function, aggr_find_tx_ring(), to find a pseudo Tx ring
2631 * belonging to a port on which the packet has to be sent. Aggr will
2632 * always have a pseudo Tx ring associated with it even when it is an
2633 * aggregation over a single NIC that has no Tx rings. Even in such a
2634 * case, the single pseudo Tx ring will have a soft ring associated with
2635 * it and the soft ring will hang off the SRS.
2636 *
2637 * If a bandwidth is specified for an aggr, SRS_TX_BW_AGGR mode is used.
2638 * In this mode, the bandwidth is first applied on the outgoing packets
2639 * and later mac_tx_addr_mode() function is called to send the packet out
2640 * of one of the pseudo Tx rings.
2641 *
2642 * Four flags are used in srs_state for indicating flow control
2643 * conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT.
2644 * SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the
2645 * driver below.
2646 * SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat
2647 * and flow-control pressure is applied back to clients. The clients expect
2648 * wakeup when flow-control is relieved.
2649 * SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk
2650 * got returned back to client either due to lack of Tx descs or due to bw
2651 * control reasons. The clients expect a wakeup when condition is relieved.
2652 *
2653 * The fourth argument to mac_tx() is the flag. Normally it will be 0 but
2654 * some clients set the following values too: MAC_DROP_ON_NO_DESC,
2655 * MAC_TX_NO_ENQUEUE
2656 * Mac clients that do not want packets to be enqueued in the mac layer set
2657 * MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or
2658 * Tx soft rings but instead get dropped when the NIC runs out of desc. The
2659 * behaviour of this flag is different when the Tx is running in serializer
2660 * or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet
2661 * get dropped when Tx high watermark is reached.
2662 * There are some mac clients like vsw, aggr that want the mblks to be
2663 * returned back to clients instead of being queued in Tx SRS (or Tx soft
2664 * rings) under flow-control (i.e., out of desc or exceeding bw limits)
2665 * conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set.
2666 * In the default and Tx fanout mode, the un-transmitted mblks will be
2667 * returned back to the clients when the driver runs out of Tx descs.
2668 * SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or
2669 * soft ring) so that the clients can be woken up when Tx desc become
2670 * available. When running in serializer or bandwidth mode mode,
2671 * SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached.
2672 */
2673
2674 mac_tx_func_t
2675 mac_tx_get_func(uint32_t mode)
2676 {
2677 return (mac_tx_mode_list[mode].mac_tx_func);
2678 }
2679
2680 /* ARGSUSED */
2681 static mac_tx_cookie_t
2682 mac_tx_single_ring_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2683 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2684 {
2685 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2686 mac_tx_stats_t stats;
2687 mac_tx_cookie_t cookie = NULL;
2688
2689 ASSERT(srs_tx->st_mode == SRS_TX_DEFAULT);
2690
2691 /* Regular case with a single Tx ring */
2692 /*
2693 * SRS_TX_BLOCKED is set when underlying NIC runs
2694 * out of Tx descs and messages start getting
2695 * queued. It won't get reset until
2696 * tx_srs_drain() completely drains out the
2697 * messages.
2698 */
2699 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
2700 /* Tx descs/resources not available */
2701 mutex_enter(&mac_srs->srs_lock);
2702 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
2703 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain,
2704 flag, ret_mp);
2705 mutex_exit(&mac_srs->srs_lock);
2706 return (cookie);
2707 }
2708 /*
2709 * While we were computing mblk count, the
2710 * flow control condition got relieved.
2711 * Continue with the transmission.
2712 */
2713 mutex_exit(&mac_srs->srs_lock);
2714 }
2715
2716 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2717 mp_chain, &stats);
2718
2719 /*
2720 * Multiple threads could be here sending packets.
2721 * Under such conditions, it is not possible to
2722 * automically set SRS_TX_BLOCKED bit to indicate
2723 * out of tx desc condition. To atomically set
2724 * this, we queue the returned packet and do
2725 * the setting of SRS_TX_BLOCKED in
2726 * mac_tx_srs_drain().
2727 */
2728 if (mp_chain != NULL) {
2729 mutex_enter(&mac_srs->srs_lock);
2730 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, flag, ret_mp);
2731 mutex_exit(&mac_srs->srs_lock);
2732 return (cookie);
2733 }
2734 SRS_TX_STATS_UPDATE(mac_srs, &stats);
2735
2736 return (NULL);
2737 }
2738
2739 /*
2740 * mac_tx_serialize_mode
2741 *
2742 * This is an experimental mode implemented as per the request of PAE.
2743 * In this mode, all callers attempting to send a packet to the NIC
2744 * will get serialized. Only one thread at any time will access the
2745 * NIC to send the packet out.
2746 */
2747 /* ARGSUSED */
2748 static mac_tx_cookie_t
2749 mac_tx_serializer_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2750 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2751 {
2752 mac_tx_stats_t stats;
2753 mac_tx_cookie_t cookie = NULL;
2754 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2755
2756 /* Single ring, serialize below */
2757 ASSERT(srs_tx->st_mode == SRS_TX_SERIALIZE);
2758 mutex_enter(&mac_srs->srs_lock);
2759 if ((mac_srs->srs_first != NULL) ||
2760 (mac_srs->srs_state & SRS_PROC)) {
2761 /*
2762 * In serialization mode, queue all packets until
2763 * TX_HIWAT is set.
2764 * If drop bit is set, drop if TX_HIWAT is set.
2765 * If no_enqueue is set, still enqueue until hiwat
2766 * is set and return mblks after TX_HIWAT is set.
2767 */
2768 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain,
2769 flag, NULL, ret_mp);
2770 mutex_exit(&mac_srs->srs_lock);
2771 return (cookie);
2772 }
2773 /*
2774 * No packets queued, nothing on proc and no flow
2775 * control condition. Fast-path, ok. Do inline
2776 * processing.
2777 */
2778 mac_srs->srs_state |= SRS_PROC;
2779 mutex_exit(&mac_srs->srs_lock);
2780
2781 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2782 mp_chain, &stats);
2783
2784 mutex_enter(&mac_srs->srs_lock);
2785 mac_srs->srs_state &= ~SRS_PROC;
2786 if (mp_chain != NULL) {
2787 cookie = mac_tx_srs_enqueue(mac_srs,
2788 mp_chain, flag, NULL, ret_mp);
2789 }
2790 if (mac_srs->srs_first != NULL) {
2791 /*
2792 * We processed inline our packet and a new
2793 * packet/s got queued while we were
2794 * processing. Wakeup srs worker
2795 */
2796 cv_signal(&mac_srs->srs_async);
2797 }
2798 mutex_exit(&mac_srs->srs_lock);
2799
2800 if (cookie == NULL)
2801 SRS_TX_STATS_UPDATE(mac_srs, &stats);
2802
2803 return (cookie);
2804 }
2805
2806 /*
2807 * mac_tx_fanout_mode
2808 *
2809 * In this mode, the SRS will have access to multiple Tx rings to send
2810 * the packet out. The fanout hint that is passed as an argument is
2811 * used to find an appropriate ring to fanout the traffic. Each Tx
2812 * ring, in turn, will have a soft ring associated with it. If a Tx
2813 * ring runs out of Tx desc's the returned packet will be queued in
2814 * the soft ring associated with that Tx ring. The srs itself will not
2815 * queue any packets.
2816 */
2817
2818 #define MAC_TX_SOFT_RING_PROCESS(chain) { \
2819 index = COMPUTE_INDEX(hash, mac_srs->srs_tx_ring_count), \
2820 softring = mac_srs->srs_tx_soft_rings[index]; \
2821 cookie = mac_tx_soft_ring_process(softring, chain, flag, ret_mp); \
2822 DTRACE_PROBE2(tx__fanout, uint64_t, hash, uint_t, index); \
2823 }
2824
2825 static mac_tx_cookie_t
2826 mac_tx_fanout_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2827 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2828 {
2829 mac_soft_ring_t *softring;
2830 uint64_t hash;
2831 uint_t index;
2832 mac_tx_cookie_t cookie = NULL;
2833
2834 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
2835 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT);
2836 if (fanout_hint != 0) {
2837 /*
2838 * The hint is specified by the caller, simply pass the
2839 * whole chain to the soft ring.
2840 */
2841 hash = HASH_HINT(fanout_hint);
2842 MAC_TX_SOFT_RING_PROCESS(mp_chain);
2843 } else {
2844 mblk_t *last_mp, *cur_mp, *sub_chain;
2845 uint64_t last_hash = 0;
2846 uint_t media = mac_srs->srs_mcip->mci_mip->mi_info.mi_media;
2847
2848 /*
2849 * Compute the hash from the contents (headers) of the
2850 * packets of the mblk chain. Split the chains into
2851 * subchains of the same conversation.
2852 *
2853 * Since there may be more than one ring used for
2854 * sub-chains of the same call, and since the caller
2855 * does not maintain per conversation state since it
2856 * passed a zero hint, unsent subchains will be
2857 * dropped.
2858 */
2859
2860 flag |= MAC_DROP_ON_NO_DESC;
2861 ret_mp = NULL;
2862
2863 ASSERT(ret_mp == NULL);
2864
2865 sub_chain = NULL;
2866 last_mp = NULL;
2867
2868 for (cur_mp = mp_chain; cur_mp != NULL;
2869 cur_mp = cur_mp->b_next) {
2870 hash = mac_pkt_hash(media, cur_mp, MAC_PKT_HASH_L4,
2871 B_TRUE);
2872 if (last_hash != 0 && hash != last_hash) {
2873 /*
2874 * Starting a different subchain, send current
2875 * chain out.
2876 */
2877 ASSERT(last_mp != NULL);
2878 last_mp->b_next = NULL;
2879 MAC_TX_SOFT_RING_PROCESS(sub_chain);
2880 sub_chain = NULL;
2881 }
2882
2883 /* add packet to subchain */
2884 if (sub_chain == NULL)
2885 sub_chain = cur_mp;
2886 last_mp = cur_mp;
2887 last_hash = hash;
2888 }
2889
2890 if (sub_chain != NULL) {
2891 /* send last subchain */
2892 ASSERT(last_mp != NULL);
2893 last_mp->b_next = NULL;
2894 MAC_TX_SOFT_RING_PROCESS(sub_chain);
2895 }
2896
2897 cookie = NULL;
2898 }
2899
2900 return (cookie);
2901 }
2902
2903 /*
2904 * mac_tx_bw_mode
2905 *
2906 * In the bandwidth mode, Tx srs will allow packets to go down to Tx ring
2907 * only if bw is available. Otherwise the packets will be queued in
2908 * SRS. If the SRS has multiple Tx rings, then packets will get fanned
2909 * out to a Tx rings.
2910 */
2911 static mac_tx_cookie_t
2912 mac_tx_bw_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2913 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2914 {
2915 int cnt, sz;
2916 mblk_t *tail;
2917 mac_tx_cookie_t cookie = NULL;
2918 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2919 clock_t now;
2920
2921 ASSERT(TX_BANDWIDTH_MODE(mac_srs));
2922 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
2923 mutex_enter(&mac_srs->srs_lock);
2924 if (mac_srs->srs_bw->mac_bw_limit == 0) {
2925 /*
2926 * zero bandwidth, no traffic is sent: drop the packets,
2927 * or return the whole chain if the caller requests all
2928 * unsent packets back.
2929 */
2930 if (flag & MAC_TX_NO_ENQUEUE) {
2931 cookie = (mac_tx_cookie_t)mac_srs;
2932 *ret_mp = mp_chain;
2933 } else {
2934 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2935 }
2936 mutex_exit(&mac_srs->srs_lock);
2937 return (cookie);
2938 } else if ((mac_srs->srs_first != NULL) ||
2939 (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2940 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
2941 fanout_hint, ret_mp);
2942 mutex_exit(&mac_srs->srs_lock);
2943 return (cookie);
2944 }
2945 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2946 now = ddi_get_lbolt();
2947 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
2948 mac_srs->srs_bw->mac_bw_curr_time = now;
2949 mac_srs->srs_bw->mac_bw_used = 0;
2950 } else if (mac_srs->srs_bw->mac_bw_used >
2951 mac_srs->srs_bw->mac_bw_limit) {
2952 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
2953 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2954 mp_chain, tail, cnt, sz);
2955 /*
2956 * Wakeup worker thread. Note that worker
2957 * thread has to be woken up so that it
2958 * can fire up the timer to be woken up
2959 * on the next tick. Also once
2960 * BW_ENFORCED is set, it can only be
2961 * reset by srs_worker thread. Until then
2962 * all packets will get queued up in SRS
2963 * and hence this this code path won't be
2964 * entered until BW_ENFORCED is reset.
2965 */
2966 cv_signal(&mac_srs->srs_async);
2967 mutex_exit(&mac_srs->srs_lock);
2968 return (cookie);
2969 }
2970
2971 mac_srs->srs_bw->mac_bw_used += sz;
2972 mutex_exit(&mac_srs->srs_lock);
2973
2974 if (srs_tx->st_mode == SRS_TX_BW_FANOUT) {
2975 mac_soft_ring_t *softring;
2976 uint_t indx, hash;
2977
2978 hash = HASH_HINT(fanout_hint);
2979 indx = COMPUTE_INDEX(hash,
2980 mac_srs->srs_tx_ring_count);
2981 softring = mac_srs->srs_tx_soft_rings[indx];
2982 return (mac_tx_soft_ring_process(softring, mp_chain, flag,
2983 ret_mp));
2984 } else if (srs_tx->st_mode == SRS_TX_BW_AGGR) {
2985 return (mac_tx_aggr_mode(mac_srs, mp_chain,
2986 fanout_hint, flag, ret_mp));
2987 } else {
2988 mac_tx_stats_t stats;
2989
2990 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2991 mp_chain, &stats);
2992
2993 if (mp_chain != NULL) {
2994 mutex_enter(&mac_srs->srs_lock);
2995 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2996 if (mac_srs->srs_bw->mac_bw_used > sz)
2997 mac_srs->srs_bw->mac_bw_used -= sz;
2998 else
2999 mac_srs->srs_bw->mac_bw_used = 0;
3000 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
3001 fanout_hint, ret_mp);
3002 mutex_exit(&mac_srs->srs_lock);
3003 return (cookie);
3004 }
3005 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3006
3007 return (NULL);
3008 }
3009 }
3010
3011 /*
3012 * mac_tx_aggr_mode
3013 *
3014 * This routine invokes an aggr function, aggr_find_tx_ring(), to find
3015 * a (pseudo) Tx ring belonging to a port on which the packet has to
3016 * be sent. aggr_find_tx_ring() first finds the outgoing port based on
3017 * L2/L3/L4 policy and then uses the fanout_hint passed to it to pick
3018 * a Tx ring from the selected port.
3019 *
3020 * Note that a port can be deleted from the aggregation. In such a case,
3021 * the aggregation layer first separates the port from the rest of the
3022 * ports making sure that port (and thus any Tx rings associated with
3023 * it) won't get selected in the call to aggr_find_tx_ring() function.
3024 * Later calls are made to mac_group_rem_ring() passing pseudo Tx ring
3025 * handles one by one which in turn will quiesce the Tx SRS and remove
3026 * the soft ring associated with the pseudo Tx ring. Unlike Rx side
3027 * where a cookie is used to protect against mac_rx_ring() calls on
3028 * rings that have been removed, no such cookie is needed on the Tx
3029 * side as the pseudo Tx ring won't be available anymore to
3030 * aggr_find_tx_ring() once the port has been removed.
3031 */
3032 static mac_tx_cookie_t
3033 mac_tx_aggr_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3034 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3035 {
3036 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3037 mac_tx_ring_fn_t find_tx_ring_fn;
3038 mac_ring_handle_t ring = NULL;
3039 void *arg;
3040 mac_soft_ring_t *sringp;
3041
3042 find_tx_ring_fn = srs_tx->st_capab_aggr.mca_find_tx_ring_fn;
3043 arg = srs_tx->st_capab_aggr.mca_arg;
3044 if (find_tx_ring_fn(arg, mp_chain, fanout_hint, &ring) == NULL)
3045 return (NULL);
3046 sringp = srs_tx->st_soft_rings[((mac_ring_t *)ring)->mr_index];
3047 return (mac_tx_soft_ring_process(sringp, mp_chain, flag, ret_mp));
3048 }
3049
3050 void
3051 mac_tx_invoke_callbacks(mac_client_impl_t *mcip, mac_tx_cookie_t cookie)
3052 {
3053 mac_cb_t *mcb;
3054 mac_tx_notify_cb_t *mtnfp;
3055
3056 /* Wakeup callback registered clients */
3057 MAC_CALLBACK_WALKER_INC(&mcip->mci_tx_notify_cb_info);
3058 for (mcb = mcip->mci_tx_notify_cb_list; mcb != NULL;
3059 mcb = mcb->mcb_nextp) {
3060 mtnfp = (mac_tx_notify_cb_t *)mcb->mcb_objp;
3061 mtnfp->mtnf_fn(mtnfp->mtnf_arg, cookie);
3062 }
3063 MAC_CALLBACK_WALKER_DCR(&mcip->mci_tx_notify_cb_info,
3064 &mcip->mci_tx_notify_cb_list);
3065 }
3066
3067 /* ARGSUSED */
3068 void
3069 mac_tx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
3070 {
3071 mblk_t *head, *tail;
3072 size_t sz;
3073 uint32_t tx_mode;
3074 uint_t saved_pkt_count;
3075 mac_tx_stats_t stats;
3076 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3077 clock_t now;
3078
3079 saved_pkt_count = 0;
3080 ASSERT(mutex_owned(&mac_srs->srs_lock));
3081 ASSERT(!(mac_srs->srs_state & SRS_PROC));
3082
3083 mac_srs->srs_state |= SRS_PROC;
3084
3085 tx_mode = srs_tx->st_mode;
3086 if (tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_SERIALIZE) {
3087 if (mac_srs->srs_first != NULL) {
3088 head = mac_srs->srs_first;
3089 tail = mac_srs->srs_last;
3090 saved_pkt_count = mac_srs->srs_count;
3091 mac_srs->srs_first = NULL;
3092 mac_srs->srs_last = NULL;
3093 mac_srs->srs_count = 0;
3094 mutex_exit(&mac_srs->srs_lock);
3095
3096 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3097 head, &stats);
3098
3099 mutex_enter(&mac_srs->srs_lock);
3100 if (head != NULL) {
3101 /* Device out of tx desc, set block */
3102 if (head->b_next == NULL)
3103 VERIFY(head == tail);
3104 tail->b_next = mac_srs->srs_first;
3105 mac_srs->srs_first = head;
3106 mac_srs->srs_count +=
3107 (saved_pkt_count - stats.mts_opackets);
3108 if (mac_srs->srs_last == NULL)
3109 mac_srs->srs_last = tail;
3110 MAC_TX_SRS_BLOCK(mac_srs, head);
3111 } else {
3112 srs_tx->st_woken_up = B_FALSE;
3113 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3114 }
3115 }
3116 } else if (tx_mode == SRS_TX_BW) {
3117 /*
3118 * We are here because the timer fired and we have some data
3119 * to tranmit. Also mac_tx_srs_worker should have reset
3120 * SRS_BW_ENFORCED flag
3121 */
3122 ASSERT(!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED));
3123 head = tail = mac_srs->srs_first;
3124 while (mac_srs->srs_first != NULL) {
3125 tail = mac_srs->srs_first;
3126 tail->b_prev = NULL;
3127 mac_srs->srs_first = tail->b_next;
3128 if (mac_srs->srs_first == NULL)
3129 mac_srs->srs_last = NULL;
3130 mac_srs->srs_count--;
3131 sz = msgdsize(tail);
3132 mac_srs->srs_size -= sz;
3133 saved_pkt_count++;
3134 MAC_TX_UPDATE_BW_INFO(mac_srs, sz);
3135
3136 if (mac_srs->srs_bw->mac_bw_used <
3137 mac_srs->srs_bw->mac_bw_limit)
3138 continue;
3139
3140 now = ddi_get_lbolt();
3141 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3142 mac_srs->srs_bw->mac_bw_curr_time = now;
3143 mac_srs->srs_bw->mac_bw_used = sz;
3144 continue;
3145 }
3146 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3147 break;
3148 }
3149
3150 ASSERT((head == NULL && tail == NULL) ||
3151 (head != NULL && tail != NULL));
3152 if (tail != NULL) {
3153 tail->b_next = NULL;
3154 mutex_exit(&mac_srs->srs_lock);
3155
3156 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3157 head, &stats);
3158
3159 mutex_enter(&mac_srs->srs_lock);
3160 if (head != NULL) {
3161 uint_t size_sent;
3162
3163 /* Device out of tx desc, set block */
3164 if (head->b_next == NULL)
3165 VERIFY(head == tail);
3166 tail->b_next = mac_srs->srs_first;
3167 mac_srs->srs_first = head;
3168 mac_srs->srs_count +=
3169 (saved_pkt_count - stats.mts_opackets);
3170 if (mac_srs->srs_last == NULL)
3171 mac_srs->srs_last = tail;
3172 size_sent = sz - stats.mts_obytes;
3173 mac_srs->srs_size += size_sent;
3174 mac_srs->srs_bw->mac_bw_sz += size_sent;
3175 if (mac_srs->srs_bw->mac_bw_used > size_sent) {
3176 mac_srs->srs_bw->mac_bw_used -=
3177 size_sent;
3178 } else {
3179 mac_srs->srs_bw->mac_bw_used = 0;
3180 }
3181 MAC_TX_SRS_BLOCK(mac_srs, head);
3182 } else {
3183 srs_tx->st_woken_up = B_FALSE;
3184 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3185 }
3186 }
3187 } else if (tx_mode == SRS_TX_BW_FANOUT || tx_mode == SRS_TX_BW_AGGR) {
3188 mblk_t *prev;
3189 uint64_t hint;
3190
3191 /*
3192 * We are here because the timer fired and we
3193 * have some quota to tranmit.
3194 */
3195 prev = NULL;
3196 head = tail = mac_srs->srs_first;
3197 while (mac_srs->srs_first != NULL) {
3198 tail = mac_srs->srs_first;
3199 mac_srs->srs_first = tail->b_next;
3200 if (mac_srs->srs_first == NULL)
3201 mac_srs->srs_last = NULL;
3202 mac_srs->srs_count--;
3203 sz = msgdsize(tail);
3204 mac_srs->srs_size -= sz;
3205 mac_srs->srs_bw->mac_bw_used += sz;
3206 if (prev == NULL)
3207 hint = (ulong_t)tail->b_prev;
3208 if (hint != (ulong_t)tail->b_prev) {
3209 prev->b_next = NULL;
3210 mutex_exit(&mac_srs->srs_lock);
3211 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
3212 head = tail;
3213 hint = (ulong_t)tail->b_prev;
3214 mutex_enter(&mac_srs->srs_lock);
3215 }
3216
3217 prev = tail;
3218 tail->b_prev = NULL;
3219 if (mac_srs->srs_bw->mac_bw_used <
3220 mac_srs->srs_bw->mac_bw_limit)
3221 continue;
3222
3223 now = ddi_get_lbolt();
3224 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3225 mac_srs->srs_bw->mac_bw_curr_time = now;
3226 mac_srs->srs_bw->mac_bw_used = 0;
3227 continue;
3228 }
3229 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3230 break;
3231 }
3232 ASSERT((head == NULL && tail == NULL) ||
3233 (head != NULL && tail != NULL));
3234 if (tail != NULL) {
3235 tail->b_next = NULL;
3236 mutex_exit(&mac_srs->srs_lock);
3237 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
3238 mutex_enter(&mac_srs->srs_lock);
3239 }
3240 }
3241 /*
3242 * SRS_TX_FANOUT case not considered here because packets
3243 * won't be queued in the SRS for this case. Packets will
3244 * be sent directly to soft rings underneath and if there
3245 * is any queueing at all, it would be in Tx side soft
3246 * rings.
3247 */
3248
3249 /*
3250 * When srs_count becomes 0, reset SRS_TX_HIWAT and
3251 * SRS_TX_WAKEUP_CLIENT and wakeup registered clients.
3252 */
3253 if (mac_srs->srs_count == 0 && (mac_srs->srs_state &
3254 (SRS_TX_HIWAT | SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED))) {
3255 mac_client_impl_t *mcip = mac_srs->srs_mcip;
3256 boolean_t wakeup_required = B_FALSE;
3257
3258 if (mac_srs->srs_state &
3259 (SRS_TX_HIWAT|SRS_TX_WAKEUP_CLIENT)) {
3260 wakeup_required = B_TRUE;
3261 }
3262 mac_srs->srs_state &= ~(SRS_TX_HIWAT |
3263 SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED);
3264 mutex_exit(&mac_srs->srs_lock);
3265 if (wakeup_required) {
3266 mac_tx_invoke_callbacks(mcip, (mac_tx_cookie_t)mac_srs);
3267 /*
3268 * If the client is not the primary MAC client, then we
3269 * need to send the notification to the clients upper
3270 * MAC, i.e. mci_upper_mip.
3271 */
3272 mac_tx_notify(mcip->mci_upper_mip != NULL ?
3273 mcip->mci_upper_mip : mcip->mci_mip);
3274 }
3275 mutex_enter(&mac_srs->srs_lock);
3276 }
3277 mac_srs->srs_state &= ~SRS_PROC;
3278 }
3279
3280 /*
3281 * Given a packet, get the flow_entry that identifies the flow
3282 * to which that packet belongs. The flow_entry will contain
3283 * the transmit function to be used to send the packet. If the
3284 * function returns NULL, the packet should be sent using the
3285 * underlying NIC.
3286 */
3287 static flow_entry_t *
3288 mac_tx_classify(mac_impl_t *mip, mblk_t *mp)
3289 {
3290 flow_entry_t *flent = NULL;
3291 mac_client_impl_t *mcip;
3292 int err;
3293
3294 /*
3295 * Do classification on the packet.
3296 */
3297 err = mac_flow_lookup(mip->mi_flow_tab, mp, FLOW_OUTBOUND, &flent);
3298 if (err != 0)
3299 return (NULL);
3300
3301 /*
3302 * This flent might just be an additional one on the MAC client,
3303 * i.e. for classification purposes (different fdesc), however
3304 * the resources, SRS et. al., are in the mci_flent, so if
3305 * this isn't the mci_flent, we need to get it.
3306 */
3307 if ((mcip = flent->fe_mcip) != NULL && mcip->mci_flent != flent) {
3308 FLOW_REFRELE(flent);
3309 flent = mcip->mci_flent;
3310 FLOW_TRY_REFHOLD(flent, err);
3311 if (err != 0)
3312 return (NULL);
3313 }
3314
3315 return (flent);
3316 }
3317
3318 /*
3319 * This macro is only meant to be used by mac_tx_send().
3320 */
3321 #define CHECK_VID_AND_ADD_TAG(mp) { \
3322 if (vid_check) { \
3323 int err = 0; \
3324 \
3325 MAC_VID_CHECK(src_mcip, (mp), err); \
3326 if (err != 0) { \
3327 freemsg((mp)); \
3328 (mp) = next; \
3329 oerrors++; \
3330 continue; \
3331 } \
3332 } \
3333 if (add_tag) { \
3334 (mp) = mac_add_vlan_tag((mp), 0, vid); \
3335 if ((mp) == NULL) { \
3336 (mp) = next; \
3337 oerrors++; \
3338 continue; \
3339 } \
3340 } \
3341 }
3342
3343 mblk_t *
3344 mac_tx_send(mac_client_handle_t mch, mac_ring_handle_t ring, mblk_t *mp_chain,
3345 mac_tx_stats_t *stats)
3346 {
3347 mac_client_impl_t *src_mcip = (mac_client_impl_t *)mch;
3348 mac_impl_t *mip = src_mcip->mci_mip;
3349 uint_t obytes = 0, opackets = 0, oerrors = 0;
3350 mblk_t *mp = NULL, *next;
3351 boolean_t vid_check, add_tag;
3352 uint16_t vid = 0;
3353
3354 if (mip->mi_nclients > 1) {
3355 vid_check = MAC_VID_CHECK_NEEDED(src_mcip);
3356 add_tag = MAC_TAG_NEEDED(src_mcip);
3357 if (add_tag)
3358 vid = mac_client_vid(mch);
3359 } else {
3360 ASSERT(mip->mi_nclients == 1);
3361 vid_check = add_tag = B_FALSE;
3362 }
3363
3364 /*
3365 * Fastpath: if there's only one client, we simply send
3366 * the packet down to the underlying NIC.
3367 */
3368 if (mip->mi_nactiveclients == 1) {
3369 DTRACE_PROBE2(fastpath,
3370 mac_client_impl_t *, src_mcip, mblk_t *, mp_chain);
3371
3372 mp = mp_chain;
3373 while (mp != NULL) {
3374 next = mp->b_next;
3375 mp->b_next = NULL;
3376 opackets++;
3377 obytes += (mp->b_cont == NULL ? MBLKL(mp) :
3378 msgdsize(mp));
3379
3380 CHECK_VID_AND_ADD_TAG(mp);
3381 MAC_TX(mip, ring, mp, src_mcip);
3382
3383 /*
3384 * If the driver is out of descriptors and does a
3385 * partial send it will return a chain of unsent
3386 * mblks. Adjust the accounting stats.
3387 */
3388 if (mp != NULL) {
3389 opackets--;
3390 obytes -= msgdsize(mp);
3391 mp->b_next = next;
3392 break;
3393 }
3394 mp = next;
3395 }
3396 goto done;
3397 }
3398
3399 /*
3400 * No fastpath, we either have more than one MAC client
3401 * defined on top of the same MAC, or one or more MAC
3402 * client promiscuous callbacks.
3403 */
3404 DTRACE_PROBE3(slowpath, mac_client_impl_t *,
3405 src_mcip, int, mip->mi_nclients, mblk_t *, mp_chain);
3406
3407 mp = mp_chain;
3408 while (mp != NULL) {
3409 flow_entry_t *dst_flow_ent;
3410 void *flow_cookie;
3411 size_t pkt_size;
3412 mblk_t *mp1;
3413
3414 next = mp->b_next;
3415 mp->b_next = NULL;
3416 opackets++;
3417 pkt_size = (mp->b_cont == NULL ? MBLKL(mp) : msgdsize(mp));
3418 obytes += pkt_size;
3419 CHECK_VID_AND_ADD_TAG(mp);
3420
3421 /*
3422 * Find the destination.
3423 */
3424 dst_flow_ent = mac_tx_classify(mip, mp);
3425
3426 if (dst_flow_ent != NULL) {
3427 size_t hdrsize;
3428 int err = 0;
3429
3430 if (mip->mi_info.mi_nativemedia == DL_ETHER) {
3431 struct ether_vlan_header *evhp =
3432 (struct ether_vlan_header *)mp->b_rptr;
3433
3434 if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN)
3435 hdrsize = sizeof (*evhp);
3436 else
3437 hdrsize = sizeof (struct ether_header);
3438 } else {
3439 mac_header_info_t mhi;
3440
3441 err = mac_header_info((mac_handle_t)mip,
3442 mp, &mhi);
3443 if (err == 0)
3444 hdrsize = mhi.mhi_hdrsize;
3445 }
3446
3447 /*
3448 * Got a matching flow. It's either another
3449 * MAC client, or a broadcast/multicast flow.
3450 * Make sure the packet size is within the
3451 * allowed size. If not drop the packet and
3452 * move to next packet.
3453 */
3454 if (err != 0 ||
3455 (pkt_size - hdrsize) > mip->mi_sdu_max) {
3456 oerrors++;
3457 DTRACE_PROBE2(loopback__drop, size_t, pkt_size,
3458 mblk_t *, mp);
3459 freemsg(mp);
3460 mp = next;
3461 FLOW_REFRELE(dst_flow_ent);
3462 continue;
3463 }
3464 flow_cookie = mac_flow_get_client_cookie(dst_flow_ent);
3465 if (flow_cookie != NULL) {
3466 /*
3467 * The vnic_bcast_send function expects
3468 * to receive the sender MAC client
3469 * as value for arg2.
3470 */
3471 mac_bcast_send(flow_cookie, src_mcip, mp,
3472 B_TRUE);
3473 } else {
3474 /*
3475 * loopback the packet to a local MAC
3476 * client. We force a context switch
3477 * if both source and destination MAC
3478 * clients are used by IP, i.e.
3479 * bypass is set.
3480 */
3481 boolean_t do_switch;
3482 mac_client_impl_t *dst_mcip =
3483 dst_flow_ent->fe_mcip;
3484
3485 /*
3486 * Check if there are promiscuous mode
3487 * callbacks defined. This check is
3488 * done here in the 'else' case and
3489 * not in other cases because this
3490 * path is for local loopback
3491 * communication which does not go
3492 * through MAC_TX(). For paths that go
3493 * through MAC_TX(), the promisc_list
3494 * check is done inside the MAC_TX()
3495 * macro.
3496 */
3497 if (mip->mi_promisc_list != NULL)
3498 mac_promisc_dispatch(mip, mp, src_mcip);
3499
3500 do_switch = ((src_mcip->mci_state_flags &
3501 dst_mcip->mci_state_flags &
3502 MCIS_CLIENT_POLL_CAPABLE) != 0);
3503
3504 if ((mp1 = mac_fix_cksum(mp)) != NULL) {
3505 (dst_flow_ent->fe_cb_fn)(
3506 dst_flow_ent->fe_cb_arg1,
3507 dst_flow_ent->fe_cb_arg2,
3508 mp1, do_switch);
3509 }
3510 }
3511 FLOW_REFRELE(dst_flow_ent);
3512 } else {
3513 /*
3514 * Unknown destination, send via the underlying
3515 * NIC.
3516 */
3517 MAC_TX(mip, ring, mp, src_mcip);
3518 if (mp != NULL) {
3519 /*
3520 * Adjust for the last packet that
3521 * could not be transmitted
3522 */
3523 opackets--;
3524 obytes -= pkt_size;
3525 mp->b_next = next;
3526 break;
3527 }
3528 }
3529 mp = next;
3530 }
3531
3532 done:
3533 stats->mts_obytes = obytes;
3534 stats->mts_opackets = opackets;
3535 stats->mts_oerrors = oerrors;
3536 return (mp);
3537 }
3538
3539 /*
3540 * mac_tx_srs_ring_present
3541 *
3542 * Returns whether the specified ring is part of the specified SRS.
3543 */
3544 boolean_t
3545 mac_tx_srs_ring_present(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
3546 {
3547 int i;
3548 mac_soft_ring_t *soft_ring;
3549
3550 if (srs->srs_tx.st_arg2 == tx_ring)
3551 return (B_TRUE);
3552
3553 for (i = 0; i < srs->srs_tx_ring_count; i++) {
3554 soft_ring = srs->srs_tx_soft_rings[i];
3555 if (soft_ring->s_ring_tx_arg2 == tx_ring)
3556 return (B_TRUE);
3557 }
3558
3559 return (B_FALSE);
3560 }
3561
3562 /*
3563 * mac_tx_srs_get_soft_ring
3564 *
3565 * Returns the TX soft ring associated with the given ring, if present.
3566 */
3567 mac_soft_ring_t *
3568 mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
3569 {
3570 int i;
3571 mac_soft_ring_t *soft_ring;
3572
3573 if (srs->srs_tx.st_arg2 == tx_ring)
3574 return (NULL);
3575
3576 for (i = 0; i < srs->srs_tx_ring_count; i++) {
3577 soft_ring = srs->srs_tx_soft_rings[i];
3578 if (soft_ring->s_ring_tx_arg2 == tx_ring)
3579 return (soft_ring);
3580 }
3581
3582 return (NULL);
3583 }
3584
3585 /*
3586 * mac_tx_srs_wakeup
3587 *
3588 * Called when Tx desc become available. Wakeup the appropriate worker
3589 * thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the
3590 * state field.
3591 */
3592 void
3593 mac_tx_srs_wakeup(mac_soft_ring_set_t *mac_srs, mac_ring_handle_t ring)
3594 {
3595 int i;
3596 mac_soft_ring_t *sringp;
3597 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3598
3599 mutex_enter(&mac_srs->srs_lock);
3600 /*
3601 * srs_tx_ring_count == 0 is the single ring mode case. In
3602 * this mode, there will not be Tx soft rings associated
3603 * with the SRS.
3604 */
3605 if (!MAC_TX_SOFT_RINGS(mac_srs)) {
3606 if (srs_tx->st_arg2 == ring &&
3607 mac_srs->srs_state & SRS_TX_BLOCKED) {
3608 mac_srs->srs_state &= ~SRS_TX_BLOCKED;
3609 srs_tx->st_stat.mts_unblockcnt++;
3610 cv_signal(&mac_srs->srs_async);
3611 }
3612 /*
3613 * A wakeup can come before tx_srs_drain() could
3614 * grab srs lock and set SRS_TX_BLOCKED. So
3615 * always set woken_up flag when we come here.
3616 */
3617 srs_tx->st_woken_up = B_TRUE;
3618 mutex_exit(&mac_srs->srs_lock);
3619 return;
3620 }
3621
3622 /*
3623 * If you are here, it is for FANOUT, BW_FANOUT,
3624 * AGGR_MODE or AGGR_BW_MODE case
3625 */
3626 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
3627 sringp = mac_srs->srs_tx_soft_rings[i];
3628 mutex_enter(&sringp->s_ring_lock);
3629 if (sringp->s_ring_tx_arg2 == ring) {
3630 if (sringp->s_ring_state & S_RING_BLOCK) {
3631 sringp->s_ring_state &= ~S_RING_BLOCK;
3632 sringp->s_st_stat.mts_unblockcnt++;
3633 cv_signal(&sringp->s_ring_async);
3634 }
3635 sringp->s_ring_tx_woken_up = B_TRUE;
3636 }
3637 mutex_exit(&sringp->s_ring_lock);
3638 }
3639 mutex_exit(&mac_srs->srs_lock);
3640 }
3641
3642 /*
3643 * Once the driver is done draining, send a MAC_NOTE_TX notification to unleash
3644 * the blocked clients again.
3645 */
3646 void
3647 mac_tx_notify(mac_impl_t *mip)
3648 {
3649 i_mac_notify(mip, MAC_NOTE_TX);
3650 }
3651
3652 /*
3653 * RX SOFTRING RELATED FUNCTIONS
3654 *
3655 * These functions really belong in mac_soft_ring.c and here for
3656 * a short period.
3657 */
3658
3659 #define SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
3660 /* \
3661 * Enqueue our mblk chain. \
3662 */ \
3663 ASSERT(MUTEX_HELD(&(ringp)->s_ring_lock)); \
3664 \
3665 if ((ringp)->s_ring_last != NULL) \
3666 (ringp)->s_ring_last->b_next = (mp); \
3667 else \
3668 (ringp)->s_ring_first = (mp); \
3669 (ringp)->s_ring_last = (tail); \
3670 (ringp)->s_ring_count += (cnt); \
3671 ASSERT((ringp)->s_ring_count > 0); \
3672 if ((ringp)->s_ring_type & ST_RING_BW_CTL) { \
3673 (ringp)->s_ring_size += sz; \
3674 } \
3675 }
3676
3677 /*
3678 * Default entry point to deliver a packet chain to a MAC client.
3679 * If the MAC client has flows, do the classification with these
3680 * flows as well.
3681 */
3682 /* ARGSUSED */
3683 void
3684 mac_rx_deliver(void *arg1, mac_resource_handle_t mrh, mblk_t *mp_chain,
3685 mac_header_info_t *arg3)
3686 {
3687 mac_client_impl_t *mcip = arg1;
3688
3689 if (mcip->mci_nvids == 1 &&
3690 !(mcip->mci_state_flags & MCIS_STRIP_DISABLE)) {
3691 /*
3692 * If the client has exactly one VID associated with it
3693 * and striping of VLAN header is not disabled,
3694 * remove the VLAN tag from the packet before
3695 * passing it on to the client's receive callback.
3696 * Note that this needs to be done after we dispatch
3697 * the packet to the promiscuous listeners of the
3698 * client, since they expect to see the whole
3699 * frame including the VLAN headers.
3700 */
3701 mp_chain = mac_strip_vlan_tag_chain(mp_chain);
3702 }
3703
3704 mcip->mci_rx_fn(mcip->mci_rx_arg, mrh, mp_chain, B_FALSE);
3705 }
3706
3707 /*
3708 * mac_rx_soft_ring_process
3709 *
3710 * process a chain for a given soft ring. The number of packets queued
3711 * in the SRS and its associated soft rings (including this one) is
3712 * very small (tracked by srs_poll_pkt_cnt), then allow the entering
3713 * thread (interrupt or poll thread) to do inline processing. This
3714 * helps keep the latency down under low load.
3715 *
3716 * The proc and arg for each mblk is already stored in the mblk in
3717 * appropriate places.
3718 */
3719 /* ARGSUSED */
3720 void
3721 mac_rx_soft_ring_process(mac_client_impl_t *mcip, mac_soft_ring_t *ringp,
3722 mblk_t *mp_chain, mblk_t *tail, int cnt, size_t sz)
3723 {
3724 mac_direct_rx_t proc;
3725 void *arg1;
3726 mac_resource_handle_t arg2;
3727 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3728
3729 ASSERT(ringp != NULL);
3730 ASSERT(mp_chain != NULL);
3731 ASSERT(tail != NULL);
3732 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3733
3734 mutex_enter(&ringp->s_ring_lock);
3735 ringp->s_ring_total_inpkt += cnt;
3736 ringp->s_ring_total_rbytes += sz;
3737 if ((mac_srs->srs_rx.sr_poll_pkt_cnt <= 1) &&
3738 !(ringp->s_ring_type & ST_RING_WORKER_ONLY)) {
3739 /* If on processor or blanking on, then enqueue and return */
3740 if (ringp->s_ring_state & S_RING_BLANK ||
3741 ringp->s_ring_state & S_RING_PROC) {
3742 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3743 mutex_exit(&ringp->s_ring_lock);
3744 return;
3745 }
3746 proc = ringp->s_ring_rx_func;
3747 arg1 = ringp->s_ring_rx_arg1;
3748 arg2 = ringp->s_ring_rx_arg2;
3749 /*
3750 * See if anything is already queued. If we are the
3751 * first packet, do inline processing else queue the
3752 * packet and do the drain.
3753 */
3754 if (ringp->s_ring_first == NULL) {
3755 /*
3756 * Fast-path, ok to process and nothing queued.
3757 */
3758 ringp->s_ring_run = curthread;
3759 ringp->s_ring_state |= (S_RING_PROC);
3760
3761 mutex_exit(&ringp->s_ring_lock);
3762
3763 /*
3764 * We are the chain of 1 packet so
3765 * go through this fast path.
3766 */
3767 ASSERT(mp_chain->b_next == NULL);
3768
3769 (*proc)(arg1, arg2, mp_chain, NULL);
3770
3771 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3772 /*
3773 * If we have a soft ring set which is doing
3774 * bandwidth control, we need to decrement
3775 * srs_size and count so it the SRS can have a
3776 * accurate idea of what is the real data
3777 * queued between SRS and its soft rings. We
3778 * decrement the counters only when the packet
3779 * gets processed by both SRS and the soft ring.
3780 */
3781 mutex_enter(&mac_srs->srs_lock);
3782 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
3783 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
3784 mutex_exit(&mac_srs->srs_lock);
3785
3786 mutex_enter(&ringp->s_ring_lock);
3787 ringp->s_ring_run = NULL;
3788 ringp->s_ring_state &= ~S_RING_PROC;
3789 if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
3790 cv_signal(&ringp->s_ring_client_cv);
3791
3792 if ((ringp->s_ring_first == NULL) ||
3793 (ringp->s_ring_state & S_RING_BLANK)) {
3794 /*
3795 * We processed inline our packet and
3796 * nothing new has arrived or our
3797 * receiver doesn't want to receive
3798 * any packets. We are done.
3799 */
3800 mutex_exit(&ringp->s_ring_lock);
3801 return;
3802 }
3803 } else {
3804 SOFT_RING_ENQUEUE_CHAIN(ringp,
3805 mp_chain, tail, cnt, sz);
3806 }
3807
3808 /*
3809 * We are here because either we couldn't do inline
3810 * processing (because something was already
3811 * queued), or we had a chain of more than one
3812 * packet, or something else arrived after we were
3813 * done with inline processing.
3814 */
3815 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
3816 ASSERT(ringp->s_ring_first != NULL);
3817
3818 ringp->s_ring_drain_func(ringp);
3819 mutex_exit(&ringp->s_ring_lock);
3820 return;
3821 } else {
3822 /* ST_RING_WORKER_ONLY case */
3823 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3824 mac_soft_ring_worker_wakeup(ringp);
3825 mutex_exit(&ringp->s_ring_lock);
3826 }
3827 }
3828
3829 /*
3830 * TX SOFTRING RELATED FUNCTIONS
3831 *
3832 * These functions really belong in mac_soft_ring.c and here for
3833 * a short period.
3834 */
3835
3836 #define TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
3837 ASSERT(MUTEX_HELD(&ringp->s_ring_lock)); \
3838 ringp->s_ring_state |= S_RING_ENQUEUED; \
3839 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); \
3840 }
3841
3842 /*
3843 * mac_tx_sring_queued
3844 *
3845 * When we are out of transmit descriptors and we already have a
3846 * queue that exceeds hiwat (or the client called us with
3847 * MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the
3848 * soft ring pointer as the opaque cookie for the client enable
3849 * flow control.
3850 */
3851 static mac_tx_cookie_t
3852 mac_tx_sring_enqueue(mac_soft_ring_t *ringp, mblk_t *mp_chain, uint16_t flag,
3853 mblk_t **ret_mp)
3854 {
3855 int cnt;
3856 size_t sz;
3857 mblk_t *tail;
3858 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3859 mac_tx_cookie_t cookie = NULL;
3860 boolean_t wakeup_worker = B_TRUE;
3861
3862 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
3863 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3864 if (flag & MAC_DROP_ON_NO_DESC) {
3865 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
3866 /* increment freed stats */
3867 ringp->s_ring_drops += cnt;
3868 cookie = (mac_tx_cookie_t)ringp;
3869 } else {
3870 if (ringp->s_ring_first != NULL)
3871 wakeup_worker = B_FALSE;
3872
3873 if (flag & MAC_TX_NO_ENQUEUE) {
3874 /*
3875 * If QUEUED is not set, queue the packet
3876 * and let mac_tx_soft_ring_drain() set
3877 * the TX_BLOCKED bit for the reasons
3878 * explained above. Otherwise, return the
3879 * mblks.
3880 */
3881 if (wakeup_worker) {
3882 TX_SOFT_RING_ENQUEUE_CHAIN(ringp,
3883 mp_chain, tail, cnt, sz);
3884 } else {
3885 ringp->s_ring_state |= S_RING_WAKEUP_CLIENT;
3886 cookie = (mac_tx_cookie_t)ringp;
3887 *ret_mp = mp_chain;
3888 }
3889 } else {
3890 boolean_t enqueue = B_TRUE;
3891
3892 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
3893 /*
3894 * flow-controlled. Store ringp in cookie
3895 * so that it can be returned as
3896 * mac_tx_cookie_t to client
3897 */
3898 ringp->s_ring_state |= S_RING_TX_HIWAT;
3899 cookie = (mac_tx_cookie_t)ringp;
3900 ringp->s_ring_hiwat_cnt++;
3901 if (ringp->s_ring_count >
3902 ringp->s_ring_tx_max_q_cnt) {
3903 /* increment freed stats */
3904 ringp->s_ring_drops += cnt;
3905 /*
3906 * b_prev may be set to the fanout hint
3907 * hence can't use freemsg directly
3908 */
3909 mac_pkt_drop(NULL, NULL,
3910 mp_chain, B_FALSE);
3911 DTRACE_PROBE1(tx_queued_hiwat,
3912 mac_soft_ring_t *, ringp);
3913 enqueue = B_FALSE;
3914 }
3915 }
3916 if (enqueue) {
3917 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain,
3918 tail, cnt, sz);
3919 }
3920 }
3921 if (wakeup_worker)
3922 cv_signal(&ringp->s_ring_async);
3923 }
3924 return (cookie);
3925 }
3926
3927
3928 /*
3929 * mac_tx_soft_ring_process
3930 *
3931 * This routine is called when fanning out outgoing traffic among
3932 * multipe Tx rings.
3933 * Note that a soft ring is associated with a h/w Tx ring.
3934 */
3935 mac_tx_cookie_t
3936 mac_tx_soft_ring_process(mac_soft_ring_t *ringp, mblk_t *mp_chain,
3937 uint16_t flag, mblk_t **ret_mp)
3938 {
3939 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3940 int cnt;
3941 size_t sz;
3942 mblk_t *tail;
3943 mac_tx_cookie_t cookie = NULL;
3944
3945 ASSERT(ringp != NULL);
3946 ASSERT(mp_chain != NULL);
3947 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3948 /*
3949 * The following modes can come here: SRS_TX_BW_FANOUT,
3950 * SRS_TX_FANOUT, SRS_TX_AGGR, SRS_TX_BW_AGGR.
3951 */
3952 ASSERT(MAC_TX_SOFT_RINGS(mac_srs));
3953 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
3954 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT ||
3955 mac_srs->srs_tx.st_mode == SRS_TX_AGGR ||
3956 mac_srs->srs_tx.st_mode == SRS_TX_BW_AGGR);
3957
3958 if (ringp->s_ring_type & ST_RING_WORKER_ONLY) {
3959 /* Serialization mode */
3960
3961 mutex_enter(&ringp->s_ring_lock);
3962 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
3963 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
3964 flag, ret_mp);
3965 mutex_exit(&ringp->s_ring_lock);
3966 return (cookie);
3967 }
3968 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3969 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3970 if (ringp->s_ring_state & (S_RING_BLOCK | S_RING_PROC)) {
3971 /*
3972 * If ring is blocked due to lack of Tx
3973 * descs, just return. Worker thread
3974 * will get scheduled when Tx desc's
3975 * become available.
3976 */
3977 mutex_exit(&ringp->s_ring_lock);
3978 return (cookie);
3979 }
3980 mac_soft_ring_worker_wakeup(ringp);
3981 mutex_exit(&ringp->s_ring_lock);
3982 return (cookie);
3983 } else {
3984 /* Default fanout mode */
3985 /*
3986 * S_RING_BLOCKED is set when underlying NIC runs
3987 * out of Tx descs and messages start getting
3988 * queued. It won't get reset until
3989 * tx_srs_drain() completely drains out the
3990 * messages.
3991 */
3992 mac_tx_stats_t stats;
3993
3994 if (ringp->s_ring_state & S_RING_ENQUEUED) {
3995 /* Tx descs/resources not available */
3996 mutex_enter(&ringp->s_ring_lock);
3997 if (ringp->s_ring_state & S_RING_ENQUEUED) {
3998 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
3999 flag, ret_mp);
4000 mutex_exit(&ringp->s_ring_lock);
4001 return (cookie);
4002 }
4003 /*
4004 * While we were computing mblk count, the
4005 * flow control condition got relieved.
4006 * Continue with the transmission.
4007 */
4008 mutex_exit(&ringp->s_ring_lock);
4009 }
4010
4011 mp_chain = mac_tx_send(ringp->s_ring_tx_arg1,
4012 ringp->s_ring_tx_arg2, mp_chain, &stats);
4013
4014 /*
4015 * Multiple threads could be here sending packets.
4016 * Under such conditions, it is not possible to
4017 * automically set S_RING_BLOCKED bit to indicate
4018 * out of tx desc condition. To atomically set
4019 * this, we queue the returned packet and do
4020 * the setting of S_RING_BLOCKED in
4021 * mac_tx_soft_ring_drain().
4022 */
4023 if (mp_chain != NULL) {
4024 mutex_enter(&ringp->s_ring_lock);
4025 cookie =
4026 mac_tx_sring_enqueue(ringp, mp_chain, flag, ret_mp);
4027 mutex_exit(&ringp->s_ring_lock);
4028 return (cookie);
4029 }
4030 SRS_TX_STATS_UPDATE(mac_srs, &stats);
4031 SOFTRING_TX_STATS_UPDATE(ringp, &stats);
4032
4033 return (NULL);
4034 }
4035 }