1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2017 Joyent, Inc.
26 */
27
28 #ifndef _SYS_MAC_SOFT_RING_H
29 #define _SYS_MAC_SOFT_RING_H
30
31 #ifdef __cplusplus
32 extern "C" {
33 #endif
34
35 #include <sys/types.h>
36 #include <sys/cpuvar.h>
37 #include <sys/cpupart.h>
38 #include <sys/processor.h>
39 #include <sys/stream.h>
40 #include <sys/squeue.h>
41 #include <sys/dlpi.h>
42 #include <sys/mac_impl.h>
43 #include <sys/mac_stat.h>
44
45 #define S_RING_NAMELEN 64
46
47 #define MAX_SR_FANOUT 24
48
49 extern boolean_t mac_soft_ring_enable;
50 extern boolean_t mac_latency_optimize;
51
52 typedef struct mac_soft_ring_s mac_soft_ring_t;
53 typedef struct mac_soft_ring_set_s mac_soft_ring_set_t;
54
55 typedef void (*mac_soft_ring_drain_func_t)(mac_soft_ring_t *);
56 typedef mac_tx_cookie_t (*mac_tx_func_t)(mac_soft_ring_set_t *, mblk_t *,
57 uintptr_t, uint16_t, mblk_t **);
58
59
60 /* Tx notify callback */
61 typedef struct mac_tx_notify_cb_s {
62 mac_cb_t mtnf_link; /* Linked list of callbacks */
63 mac_tx_notify_t mtnf_fn; /* The callback function */
64 void *mtnf_arg; /* Callback function argument */
65 } mac_tx_notify_cb_t;
66
67 struct mac_soft_ring_s {
68 /* Keep the most used members 64bytes cache aligned */
69 kmutex_t s_ring_lock; /* lock before using any member */
70 uint16_t s_ring_type; /* processing model of the sq */
71 uint16_t s_ring_state; /* state flags and message count */
72 int s_ring_count; /* # of mblocks in mac_soft_ring */
73 size_t s_ring_size; /* Size of data queued */
74 mblk_t *s_ring_first; /* first mblk chain or NULL */
75 mblk_t *s_ring_last; /* last mblk chain or NULL */
76
77 mac_direct_rx_t s_ring_rx_func;
78 void *s_ring_rx_arg1;
79 mac_resource_handle_t s_ring_rx_arg2;
80
81 /*
82 * Threshold after which packets get dropped.
83 * Is always greater than s_ring_tx_hiwat
84 */
85 int s_ring_tx_max_q_cnt;
86 /* # of mblocks after which to apply flow control */
87 int s_ring_tx_hiwat;
88 /* # of mblocks after which to relieve flow control */
89 int s_ring_tx_lowat;
90 boolean_t s_ring_tx_woken_up;
91 uint32_t s_ring_hiwat_cnt; /* times blocked for Tx descs */
92
93 void *s_ring_tx_arg1;
94 void *s_ring_tx_arg2;
95
96 /* Tx notify callback */
97 mac_cb_info_t s_ring_notify_cb_info; /* cb list info */
98 mac_cb_t *s_ring_notify_cb_list; /* The cb list */
99
100 clock_t s_ring_awaken; /* time async thread was awakened */
101
102 kthread_t *s_ring_run; /* Current thread processing sq */
103 processorid_t s_ring_cpuid; /* processor to bind to */
104 processorid_t s_ring_cpuid_save; /* saved cpuid during offline */
105 kcondvar_t s_ring_async; /* async thread blocks on */
106 clock_t s_ring_wait; /* lbolts to wait after a fill() */
107 timeout_id_t s_ring_tid; /* timer id of pending timeout() */
108 kthread_t *s_ring_worker; /* kernel thread id */
109 char s_ring_name[S_RING_NAMELEN + 1];
110 uint32_t s_ring_total_inpkt;
111 uint32_t s_ring_total_rbytes;
112 uint32_t s_ring_drops;
113 struct mac_client_impl_s *s_ring_mcip;
114 kstat_t *s_ring_ksp;
115
116 /* Teardown, poll disable control ops */
117 kcondvar_t s_ring_client_cv; /* Client wait for control op */
118
119 mac_soft_ring_set_t *s_ring_set; /* The SRS this ring belongs to */
120 mac_soft_ring_t *s_ring_next;
121 mac_soft_ring_t *s_ring_prev;
122 mac_soft_ring_drain_func_t s_ring_drain_func;
123
124 mac_tx_stats_t s_st_stat;
125 };
126
127 typedef void (*mac_srs_drain_proc_t)(mac_soft_ring_set_t *, uint_t);
128
129 /* Transmit side Soft Ring Set */
130 typedef struct mac_srs_tx_s {
131 /* Members for Tx size processing */
132 uint32_t st_mode;
133 mac_tx_func_t st_func;
134 void *st_arg1;
135 void *st_arg2;
136 mac_group_t *st_group; /* TX group for share */
137 boolean_t st_woken_up;
138
139 /*
140 * st_max_q_cnt is the queue depth threshold to limit
141 * outstanding packets on the Tx SRS. Once the limit
142 * is reached, Tx SRS will drop packets until the
143 * limit goes below the threshold.
144 */
145 uint32_t st_max_q_cnt; /* max. outstanding packets */
146 /*
147 * st_hiwat is used Tx serializer and bandwidth mode.
148 * This is the queue depth threshold upto which
149 * packets will get buffered with no flow-control
150 * back pressure applied to the caller. Once this
151 * threshold is reached, back pressure will be
152 * applied to the caller of mac_tx() (mac_tx() starts
153 * returning a cookie to indicate a blocked SRS).
154 * st_hiwat should always be lesser than or equal to
155 * st_max_q_cnt.
156 */
157 uint32_t st_hiwat; /* mblk cnt to apply flow control */
158 uint32_t st_lowat; /* mblk cnt to relieve flow control */
159 uint32_t st_hiwat_cnt; /* times blocked for Tx descs */
160 mac_tx_stats_t st_stat;
161 mac_capab_aggr_t st_capab_aggr;
162 /*
163 * st_soft_rings is used as an array to store aggr Tx soft
164 * rings. When aggr_find_tx_ring() returns a pseudo ring,
165 * the associated soft ring has to be found. st_soft_rings
166 * array stores the soft ring associated with a pseudo Tx
167 * ring and it can be accessed using the pseudo ring
168 * index (mr_index). Note that the ring index is unique
169 * for each ring in a group.
170 */
171 mac_soft_ring_t **st_soft_rings;
172 } mac_srs_tx_t;
173
174 /* Receive side Soft Ring Set */
175 typedef struct mac_srs_rx_s {
176 /*
177 * Upcall Function for fanout, Rx processing etc. Perhaps
178 * the same 3 members below can be used for Tx
179 * processing, but looking around, mac_rx_func_t has
180 * proliferated too much into various files at different
181 * places. I am leaving the consolidation battle for
182 * another day.
183 */
184 mac_direct_rx_t sr_func; /* srs_lock */
185 void *sr_arg1; /* srs_lock */
186 mac_resource_handle_t sr_arg2; /* srs_lock */
187 mac_rx_func_t sr_lower_proc; /* Atomically changed */
188 uint32_t sr_poll_pkt_cnt;
189 uint32_t sr_poll_thres;
190
191 /* mblk cnt to apply flow control */
192 uint32_t sr_hiwat;
193 /* mblk cnt to relieve flow control */
194 uint32_t sr_lowat;
195 mac_rx_stats_t sr_stat;
196
197 /* Times polling was enabled */
198 uint32_t sr_poll_on;
199 /* Times polling was enabled by worker thread */
200 uint32_t sr_worker_poll_on;
201 /* Times polling was disabled */
202 uint32_t sr_poll_off;
203 /* Poll thread signalled count */
204 uint32_t sr_poll_thr_sig;
205 /* Poll thread busy */
206 uint32_t sr_poll_thr_busy;
207 /* SRS drains, stays in poll mode but doesn't poll */
208 uint32_t sr_poll_drain_no_poll;
209 /*
210 * SRS has nothing to do and no packets in H/W but
211 * there is a backlog in softrings. SRS stays in
212 * poll mode but doesn't do polling.
213 */
214 uint32_t sr_poll_no_poll;
215 /* Active polling restarted */
216 uint32_t sr_below_hiwat;
217 /* Found packets in last poll so try and poll again */
218 uint32_t sr_poll_again;
219 /*
220 * Packets in queue but poll thread not allowed to process so
221 * signal the worker thread.
222 */
223 uint32_t sr_poll_sig_worker;
224 /*
225 * Poll thread has nothing to do and H/W has nothing so
226 * reenable the interrupts.
227 */
228 uint32_t sr_poll_intr_enable;
229 /*
230 * Poll thread has nothing to do and worker thread was already
231 * running so it can decide to reenable interrupt or poll again.
232 */
233 uint32_t sr_poll_goto_sleep;
234 /* Worker thread goes back to draining the queue */
235 uint32_t sr_drain_again;
236 /* More Packets in queue so signal the poll thread to drain */
237 uint32_t sr_drain_poll_sig;
238 /* More Packets in queue so signal the worker thread to drain */
239 uint32_t sr_drain_worker_sig;
240 /* Poll thread is already running so worker has nothing to do */
241 uint32_t sr_drain_poll_running;
242 /* We have packets already queued so keep polling */
243 uint32_t sr_drain_keep_polling;
244 /* Drain is done and interrupts are reenabled */
245 uint32_t sr_drain_finish_intr;
246 /* Polling thread needs to schedule worker wakeup */
247 uint32_t sr_poll_worker_wakeup;
248 } mac_srs_rx_t;
249
250 /*
251 * mac_soft_ring_set_s:
252 * This is used both for Tx and Rx side. The srs_type identifies Rx or
253 * Tx type.
254 *
255 * Note that the structure is carefully crafted, with Rx elements coming
256 * first followed by Tx specific members. Future additions to this
257 * structure should follow the same guidelines.
258 *
259 * Rx-side notes:
260 * mac_rx_classify_flow_add() always creates a mac_soft_ring_set_t and fn_flow
261 * points to info from it (func = srs_lower_proc, arg = soft_ring_set). On
262 * interrupt path, srs_lower_proc does B/W adjustment and switch to polling mode
263 * (if poll capable) and feeds the packets to soft_ring_list via choosen
264 * fanout type (specified by srs_type). In poll mode, the poll thread which is
265 * also a pointer can pick up the packets and feed them to various
266 * soft_ring_list.
267 *
268 * The srs_type can either be protocol based or fanout based where fanout itelf
269 * can be various types
270 *
271 * The polling works by turning off interrupts as soon as a packets
272 * are queued on the soft ring set. Once the backlog is clear and poll
273 * thread return empty handed i.e. Rx ring doesn't have anything, the
274 * interrupt is turned back on. For this purpose we keep a separate
275 * srs_poll_pkt_cnt counter which tracks the packets queued between SRS
276 * and the soft rings as well. The counter is incremented when packets
277 * are queued and decremented when SRS processes them (in case it has
278 * no soft rings) or the soft ring process them. Its important that
279 * in case SRS has softrings, the decrement doesn't happen till the
280 * packet is processed by the soft rings since it takes very little time
281 * for SRS to queue packet from SRS to soft rings and it will keep
282 * bringing more packets in the system faster than soft rings can
283 * process them.
284 *
285 * Tx side notes:
286 * The srs structure acts as a serializer with a worker thread. The
287 * default behavior of srs though is to act as a pass-thru. The queues
288 * (srs_first, srs_last, srs_count) get used when Tx ring runs out of Tx
289 * descriptors or to enforce bandwidth limits.
290 *
291 * When multiple Tx rings are present, the SRS state will be set to
292 * SRS_FANOUT_OTH. Outgoing packets coming into mac_tx_srs_process()
293 * function will be fanned out to one of the Tx side soft rings based on
294 * a hint passed in mac_tx_srs_process(). Each soft ring, in turn, will
295 * be associated with a distinct h/w Tx ring.
296 */
297
298 struct mac_soft_ring_set_s {
299 /*
300 * Common elements, common to both Rx and Tx SRS type.
301 * The following block of fields are protected by srs_lock
302 */
303 kmutex_t srs_lock;
304 uint32_t srs_type;
305 uint32_t srs_state; /* state flags */
306 uint32_t srs_count;
307 mblk_t *srs_first; /* first mblk chain or NULL */
308 mblk_t *srs_last; /* last mblk chain or NULL */
309 kcondvar_t srs_async; /* cv for worker thread */
310 kcondvar_t srs_cv; /* cv for poll thread */
311 kcondvar_t srs_quiesce_done_cv; /* cv for removal */
312 timeout_id_t srs_tid; /* timeout id for pending timeout */
313
314 /*
315 * List of soft rings & processing function.
316 * The following block is protected by Rx quiescence.
317 * i.e. they can be changed only after quiescing the SRS
318 * Protected by srs_lock.
319 */
320 mac_soft_ring_t *srs_soft_ring_head;
321 mac_soft_ring_t *srs_soft_ring_tail;
322 int srs_soft_ring_count;
323 int srs_soft_ring_quiesced_count;
324 int srs_soft_ring_condemned_count;
325 mac_soft_ring_t **srs_tcp_soft_rings;
326 int srs_tcp_ring_count;
327 mac_soft_ring_t **srs_udp_soft_rings;
328 int srs_udp_ring_count;
329 mac_soft_ring_t **srs_oth_soft_rings;
330 int srs_oth_ring_count;
331 /*
332 * srs_tx_soft_rings is used by tx_srs in
333 * when operating in multi tx ring mode.
334 */
335 mac_soft_ring_t **srs_tx_soft_rings;
336 int srs_tx_ring_count;
337
338 /*
339 * Bandwidth control related members.
340 * They are common to both Rx- and Tx-side.
341 * Following protected by srs_lock
342 */
343 mac_bw_ctl_t *srs_bw;
344 size_t srs_size; /* Size of packets queued in bytes */
345 pri_t srs_pri;
346
347 mac_soft_ring_set_t *srs_next; /* mac_srs_g_lock */
348 mac_soft_ring_set_t *srs_prev; /* mac_srs_g_lock */
349
350 /* Attribute specific drain func (BW ctl vs non-BW ctl) */
351 mac_srs_drain_proc_t srs_drain_func; /* Write once (WO) */
352
353 /*
354 * If the associated ring is exclusively used by a mac client, e.g.,
355 * an aggregation, this fields is used to keep a reference to the
356 * MAC client's pseudo ring.
357 */
358 mac_resource_handle_t srs_mrh;
359 /*
360 * The following blocks are write once (WO) and valid for the life
361 * of the SRS
362 */
363 struct mac_client_impl_s *srs_mcip; /* back ptr to mac client */
364 void *srs_flent; /* back ptr to flent */
365 mac_ring_t *srs_ring; /* Ring Descriptor */
366
367 /* Teardown, disable control ops */
368 kcondvar_t srs_client_cv; /* Client wait for the control op */
369
370 kthread_t *srs_worker; /* WO, worker thread */
371 kthread_t *srs_poll_thr; /* WO, poll thread */
372
373 uint_t srs_ind; /* Round Robin indx for picking up SR */
374 processorid_t srs_worker_cpuid; /* processor to bind to */
375 processorid_t srs_worker_cpuid_save; /* saved cpuid during offline */
376 processorid_t srs_poll_cpuid; /* processor to bind to */
377 processorid_t srs_poll_cpuid_save; /* saved cpuid during offline */
378 uint_t srs_fanout_state;
379 mac_cpus_t srs_cpu;
380
381 mac_srs_rx_t srs_rx;
382 mac_srs_tx_t srs_tx;
383 kstat_t *srs_ksp;
384 };
385
386 /*
387 * type flags - combination allowed to process and drain the queue
388 */
389 #define ST_RING_WORKER_ONLY 0x0001 /* Worker thread only */
390 #define ST_RING_ANY 0x0002 /* Any thread can process the queue */
391 #define ST_RING_TCP 0x0004
392 #define ST_RING_UDP 0x0008
393 #define ST_RING_OTH 0x0010
394
395 #define ST_RING_BW_CTL 0x0020
396 #define ST_RING_TX 0x0040
397
398 /*
399 * State flags.
400 */
401 #define S_RING_PROC 0x0001 /* being processed */
402 #define S_RING_BOUND 0x0002 /* Worker thread is bound to a cpu */
403 #define S_RING_BLOCK 0x0004 /* No Tx descs */
404 #define S_RING_TX_HIWAT 0x0008 /* Tx high watermark reached */
405
406 #define S_RING_WAKEUP_CLIENT 0x0010 /* flow ctrl, client wakeup needed */
407 #define S_RING_BLANK 0x0020 /* Has been put into polling mode */
408 #define S_RING_CLIENT_WAIT 0x0040 /* Client waiting for control op */
409
410 #define S_RING_CONDEMNED 0x0100 /* Being torn down */
411 #define S_RING_CONDEMNED_DONE 0x0200 /* Being torn down */
412 #define S_RING_QUIESCE 0x0400 /* No traffic flow, transient flag */
413 #define S_RING_QUIESCE_DONE 0x0800 /* No traffic flow, transient flag */
414
415 #define S_RING_RESTART 0x1000 /* Go back to normal traffic flow */
416 #define S_RING_ENQUEUED 0x2000 /* Pkts enqueued in Tx soft ring */
417
418 /*
419 * arguments for processors to bind to
420 */
421 #define S_RING_BIND_NONE -1
422
423 /*
424 * defines for srs_type - identifies a link or a sub-flow
425 * and other static characteristics of a SRS like a tx
426 * srs, tcp only srs, etc.
427 */
428 #define SRST_LINK 0x00000001
429 #define SRST_FLOW 0x00000002
430 #define SRST_NO_SOFT_RINGS 0x00000004
431 #define SRST_TCP_ONLY 0x00000008
432
433 #define SRST_FANOUT_PROTO 0x00000010
434 #define SRST_FANOUT_SRC_IP 0x00000020
435 #define SRST_FANOUT_OTH 0x00000040
436 #define SRST_DEFAULT_GRP 0x00000080
437
438 #define SRST_TX 0x00000100
439 #define SRST_BW_CONTROL 0x00000200
440 #define SRST_DIRECT_POLL 0x00000400
441
442 #define SRST_DLS_BYPASS 0x00001000
443 #define SRST_CLIENT_POLL_ENABLED 0x00002000
444
445 /*
446 * soft ring set flags. These bits are dynamic in nature and get
447 * applied to srs_state. They reflect the state of SRS at any
448 * point of time
449 */
450 #define SRS_BLANK 0x00000001
451 #define SRS_WORKER_BOUND 0x00000002
452 #define SRS_POLL_BOUND 0x00000004
453 #define SRS_POLLING_CAPAB 0x00000008
454
455 #define SRS_PROC 0x00000010
456 #define SRS_GET_PKTS 0x00000020
457 #define SRS_POLLING 0x00000040
458 #define SRS_BW_ENFORCED 0x00000080
459
460 #define SRS_WORKER 0x00000100
461 #define SRS_ENQUEUED 0x00000200
462 #define SRS_ANY_PROCESS 0x00000400
463 #define SRS_PROC_FAST 0x00000800
464
465 #define SRS_POLL_PROC 0x00001000
466 #define SRS_TX_BLOCKED 0x00002000 /* out of Tx descs */
467 #define SRS_TX_HIWAT 0x00004000 /* Tx count exceeds hiwat */
468 #define SRS_TX_WAKEUP_CLIENT 0x00008000 /* Flow-ctl: wakeup client */
469
470 #define SRS_CLIENT_PROC 0x00010000
471 #define SRS_CLIENT_WAIT 0x00020000
472 #define SRS_QUIESCE 0x00040000
473 #define SRS_QUIESCE_DONE 0x00080000
474
475 #define SRS_CONDEMNED 0x00100000
476 #define SRS_CONDEMNED_DONE 0x00200000
477 #define SRS_POLL_THR_QUIESCED 0x00400000
478 #define SRS_RESTART 0x00800000
479
480 #define SRS_RESTART_DONE 0x01000000
481 #define SRS_POLL_THR_RESTART 0x02000000
482 #define SRS_IN_GLIST 0x04000000
483 #define SRS_POLL_THR_EXITED 0x08000000
484
485 #define SRS_QUIESCE_PERM 0x10000000
486 #define SRS_LATENCY_OPT 0x20000000
487 #define SRS_SOFTRING_QUEUE 0x40000000
488
489 #define SRS_QUIESCED(srs) (srs->srs_state & SRS_QUIESCE_DONE)
490
491 /*
492 * If the SRS_QUIESCE_PERM flag is set, the SRS worker thread will not be
493 * able to be restarted.
494 */
495 #define SRS_QUIESCED_PERMANENT(srs) (srs->srs_state & SRS_QUIESCE_PERM)
496
497 /*
498 * soft ring set (SRS) Tx modes
499 */
500 typedef enum {
501 SRS_TX_DEFAULT = 0,
502 SRS_TX_SERIALIZE,
503 SRS_TX_FANOUT,
504 SRS_TX_BW,
505 SRS_TX_BW_FANOUT,
506 SRS_TX_AGGR,
507 SRS_TX_BW_AGGR
508 } mac_tx_srs_mode_t;
509
510 /*
511 * SRS fanout states
512 */
513 typedef enum {
514 SRS_FANOUT_UNINIT = 0,
515 SRS_FANOUT_INIT,
516 SRS_FANOUT_REINIT
517 } mac_srs_fanout_state_t;
518
519 /*
520 * Structure for dls statistics
521 */
522 struct dls_kstats {
523 kstat_named_t dlss_soft_ring_pkt_drop;
524 };
525
526 extern struct dls_kstats dls_kstat;
527
528 #define DLS_BUMP_STAT(x, y) (dls_kstat.x.value.ui32 += y)
529
530 /* Turn dynamic polling off */
531 #define MAC_SRS_POLLING_OFF(mac_srs) { \
532 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
533 if (((mac_srs)->srs_state & (SRS_POLLING_CAPAB|SRS_POLLING)) == \
534 (SRS_POLLING_CAPAB|SRS_POLLING)) { \
535 (mac_srs)->srs_state &= ~SRS_POLLING; \
536 (void) mac_hwring_enable_intr((mac_ring_handle_t) \
537 (mac_srs)->srs_ring); \
538 (mac_srs)->srs_rx.sr_poll_off++; \
539 } \
540 }
541
542 #define MAC_COUNT_CHAIN(mac_srs, head, tail, cnt, sz) { \
543 mblk_t *tmp; \
544 boolean_t bw_ctl = B_FALSE; \
545 \
546 ASSERT((head) != NULL); \
547 cnt = 0; \
548 sz = 0; \
549 if ((mac_srs)->srs_type & SRST_BW_CONTROL) \
550 bw_ctl = B_TRUE; \
551 tmp = tail = (head); \
552 if ((head)->b_next == NULL) { \
553 cnt = 1; \
554 if (bw_ctl) \
555 sz += msgdsize(head); \
556 } else { \
557 while (tmp != NULL) { \
558 tail = tmp; \
559 cnt++; \
560 if (bw_ctl) \
561 sz += msgdsize(tmp); \
562 tmp = tmp->b_next; \
563 } \
564 } \
565 }
566
567 /*
568 * Decrement the cumulative packet count in SRS and its
569 * soft rings. If the srs_poll_pkt_cnt goes below lowat, then check
570 * if if the interface was left in a polling mode and no one
571 * is really processing the queue (to get the interface out
572 * of poll mode). If no one is processing the queue, then
573 * acquire the PROC and signal the poll thread to check the
574 * interface for packets and get the interface back to interrupt
575 * mode if nothing is found.
576 */
577 #define MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt) { \
578 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
579 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
580 \
581 srs_rx->sr_poll_pkt_cnt -= cnt; \
582 if ((srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_poll_thres) && \
583 (((mac_srs)->srs_state & \
584 (SRS_POLLING|SRS_PROC|SRS_GET_PKTS)) == SRS_POLLING)) \
585 { \
586 (mac_srs)->srs_state |= (SRS_PROC|SRS_GET_PKTS); \
587 cv_signal(&(mac_srs)->srs_cv); \
588 srs_rx->sr_below_hiwat++; \
589 } \
590 }
591
592 /*
593 * The following two macros are used to update the inbound packet and byte.
594 * count. The packet and byte count reflect the packets and bytes that are
595 * taken out of the SRS's queue, i.e. indicating they are being delivered.
596 * The srs_count and srs_size are updated in different locations as the
597 * srs_size is also used to take into account any bandwidth limits. The
598 * srs_size is updated only when a soft ring, if any, sends a packet up,
599 * as opposed to updating it when the SRS sends a packet to the SR, i.e.
600 * the srs_size reflects the packets in the SRS and SRs. These
601 * macros decrement the srs_size and srs_count and also increment the
602 * ipackets and ibytes stats resp.
603 *
604 * xxx-venu These are done under srs_lock, for now we still update
605 * mci_stat_ibytes/mci_stat_ipackets atomically, need to check if
606 * just updating them would be accurate enough.
607 *
608 * If we are updating these for a sub-flow SRS, then we need to also
609 * updated it's MAC client bandwidth info, if the MAC client is also
610 * bandwidth regulated.
611 */
612 #define MAC_UPDATE_SRS_SIZE_LOCKED(srs, sz) { \
613 if ((srs)->srs_type & SRST_BW_CONTROL) { \
614 mutex_enter(&(srs)->srs_bw->mac_bw_lock); \
615 (srs)->srs_bw->mac_bw_sz -= (sz); \
616 (srs)->srs_bw->mac_bw_used += (sz); \
617 mutex_exit(&(srs)->srs_bw->mac_bw_lock); \
618 } \
619 }
620
621 #define MAC_TX_UPDATE_BW_INFO(srs, sz) { \
622 (srs)->srs_bw->mac_bw_sz -= (sz); \
623 (srs)->srs_bw->mac_bw_used += (sz); \
624 }
625
626 #define MAC_TX_SOFT_RINGS(mac_srs) ((mac_srs)->srs_tx_ring_count >= 1)
627
628 /* Soft ring flags for teardown */
629 #define SRS_POLL_THR_OWNER (SRS_PROC | SRS_POLLING | SRS_GET_PKTS)
630 #define SRS_PAUSE (SRS_CONDEMNED | SRS_QUIESCE)
631 #define S_RING_PAUSE (S_RING_CONDEMNED | S_RING_QUIESCE)
632
633 /* Soft rings */
634 extern void mac_soft_ring_init(void);
635 extern void mac_soft_ring_finish(void);
636 extern void mac_fanout_setup(mac_client_impl_t *, flow_entry_t *,
637 mac_resource_props_t *, mac_direct_rx_t, void *, mac_resource_handle_t,
638 cpupart_t *);
639
640 extern void mac_soft_ring_worker_wakeup(mac_soft_ring_t *);
641 extern void mac_soft_ring_blank(void *, time_t, uint_t, int);
642 extern mblk_t *mac_soft_ring_poll(mac_soft_ring_t *, int);
643 extern void mac_soft_ring_destroy(mac_soft_ring_t *);
644 extern void mac_soft_ring_dls_bypass(void *, mac_direct_rx_t, void *);
645
646 /* Rx SRS */
647 extern mac_soft_ring_set_t *mac_srs_create(struct mac_client_impl_s *,
648 flow_entry_t *, uint32_t, mac_direct_rx_t, void *, mac_resource_handle_t,
649 mac_ring_t *);
650 extern void mac_srs_free(mac_soft_ring_set_t *);
651 extern void mac_srs_signal(mac_soft_ring_set_t *, uint_t);
652 extern cpu_t *mac_srs_bind(mac_soft_ring_set_t *, processorid_t);
653 extern void mac_rx_srs_retarget_intr(mac_soft_ring_set_t *, processorid_t);
654 extern void mac_tx_srs_retarget_intr(mac_soft_ring_set_t *);
655
656 extern void mac_srs_quiesce_initiate(mac_soft_ring_set_t *);
657 extern void mac_srs_client_poll_enable(struct mac_client_impl_s *,
658 mac_soft_ring_set_t *);
659 extern void mac_srs_client_poll_disable(struct mac_client_impl_s *,
660 mac_soft_ring_set_t *);
661 extern void mac_srs_client_poll_quiesce(struct mac_client_impl_s *,
662 mac_soft_ring_set_t *);
663 extern void mac_srs_client_poll_restart(struct mac_client_impl_s *,
664 mac_soft_ring_set_t *);
665 extern void mac_rx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
666 extern void mac_rx_srs_restart(mac_soft_ring_set_t *);
667 extern void mac_rx_srs_subflow_process(void *, mac_resource_handle_t, mblk_t *,
668 boolean_t);
669 extern void mac_tx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
670
671 /* Tx SRS, Tx softring */
672 extern void mac_tx_srs_wakeup(mac_soft_ring_set_t *, mac_ring_handle_t);
673 extern void mac_tx_srs_setup(struct mac_client_impl_s *, flow_entry_t *);
674 extern mac_tx_func_t mac_tx_get_func(uint32_t);
675 extern mblk_t *mac_tx_send(mac_client_handle_t, mac_ring_handle_t, mblk_t *,
676 mac_tx_stats_t *);
677 extern boolean_t mac_tx_srs_ring_present(mac_soft_ring_set_t *, mac_ring_t *);
678 extern mac_soft_ring_t *mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *,
679 mac_ring_t *);
680 extern void mac_tx_srs_add_ring(mac_soft_ring_set_t *, mac_ring_t *);
681 extern void mac_tx_srs_del_ring(mac_soft_ring_set_t *, mac_ring_t *);
682 extern mac_tx_cookie_t mac_tx_srs_no_desc(mac_soft_ring_set_t *, mblk_t *,
683 uint16_t, mblk_t **);
684
685 /* Subflow specific stuff */
686 extern int mac_srs_flow_create(struct mac_client_impl_s *, flow_entry_t *,
687 mac_resource_props_t *, int, int, mac_direct_rx_t);
688 extern void mac_srs_update_bwlimit(flow_entry_t *, mac_resource_props_t *);
689 extern void mac_srs_adjust_subflow_bwlimit(struct mac_client_impl_s *);
690 extern void mac_srs_update_drv(struct mac_client_impl_s *);
691 extern void mac_update_srs_priority(mac_soft_ring_set_t *, pri_t);
692 extern void mac_client_update_classifier(mac_client_impl_t *, boolean_t);
693
694 extern void mac_soft_ring_intr_enable(void *);
695 extern boolean_t mac_soft_ring_intr_disable(void *);
696 extern mac_soft_ring_t *mac_soft_ring_create(int, clock_t, uint16_t,
697 pri_t, mac_client_impl_t *, mac_soft_ring_set_t *,
698 processorid_t, mac_direct_rx_t, void *, mac_resource_handle_t);
699 extern cpu_t *mac_soft_ring_bind(mac_soft_ring_t *, processorid_t);
700 extern void mac_soft_ring_unbind(mac_soft_ring_t *);
701 extern void mac_soft_ring_free(mac_soft_ring_t *);
702 extern void mac_soft_ring_signal(mac_soft_ring_t *, uint_t);
703 extern void mac_rx_soft_ring_process(mac_client_impl_t *, mac_soft_ring_t *,
704 mblk_t *, mblk_t *, int, size_t);
705 extern mac_tx_cookie_t mac_tx_soft_ring_process(mac_soft_ring_t *,
706 mblk_t *, uint16_t, mblk_t **);
707 extern void mac_srs_worker_quiesce(mac_soft_ring_set_t *);
708 extern void mac_srs_worker_restart(mac_soft_ring_set_t *);
709 extern void mac_rx_attach_flow_srs(mac_impl_t *, flow_entry_t *,
710 mac_soft_ring_set_t *, mac_ring_t *, mac_classify_type_t);
711
712 extern void mac_rx_srs_drain_bw(mac_soft_ring_set_t *, uint_t);
713 extern void mac_rx_srs_drain(mac_soft_ring_set_t *, uint_t);
714 extern void mac_rx_srs_process(void *, mac_resource_handle_t, mblk_t *,
715 boolean_t);
716 extern void mac_srs_worker(mac_soft_ring_set_t *);
717 extern void mac_rx_srs_poll_ring(mac_soft_ring_set_t *);
718 extern void mac_tx_srs_drain(mac_soft_ring_set_t *, uint_t);
719
720 extern void mac_tx_srs_restart(mac_soft_ring_set_t *);
721 extern void mac_rx_srs_remove(mac_soft_ring_set_t *);
722
723 #ifdef __cplusplus
724 }
725 #endif
726
727 #endif /* _SYS_MAC_SOFT_RING_H */