1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2019 Joyent, Inc.
26 */
27
28 /*
29 * MAC data path
30 *
31 * The MAC data path is concerned with the flow of traffic from mac clients --
32 * DLS, IP, etc. -- to various GLDv3 device drivers -- e1000g, vnic, aggr,
33 * ixgbe, etc. -- and from the GLDv3 device drivers back to clients.
34 *
35 * -----------
36 * Terminology
37 * -----------
38 *
39 * MAC uses a lot of different, but related terms that are associated with the
40 * design and structure of the data path. Before we cover other aspects, first
41 * let's review the terminology that MAC uses.
42 *
43 * MAC
44 *
45 * This driver. It interfaces with device drivers and provides abstractions
46 * that the rest of the system consumes. All data links -- things managed
47 * with dladm(1M), are accessed through MAC.
48 *
49 * GLDv3 DEVICE DRIVER
50 *
51 * A GLDv3 device driver refers to a driver, both for pseudo-devices and
52 * real devices, which implement the GLDv3 driver API. Common examples of
53 * these are igb and ixgbe, which are drivers for various Intel networking
54 * cards. These devices may or may not have various features, such as
55 * hardware rings and checksum offloading. For MAC, a GLDv3 device is the
56 * final point for the transmission of a packet and the starting point for
57 * the receipt of a packet.
58 *
59 * FLOWS
60 *
61 * At a high level, a flow refers to a series of packets that are related.
62 * Often times the term is used in the context of TCP to indicate a unique
63 * TCP connection and the traffic over it. However, a flow can exist at
64 * other levels of the system as well. MAC has a notion of a default flow
65 * which is used for all unicast traffic addressed to the address of a MAC
66 * device. For example, when a VNIC is created, a default flow is created
67 * for the VNIC's MAC address. In addition, flows are created for broadcast
68 * groups and a user may create a flow with flowadm(1M).
69 *
70 * CLASSIFICATION
71 *
72 * Classification refers to the notion of identifying an incoming frame
73 * based on its destination address and optionally its source addresses and
74 * doing different processing based on that information. Classification can
75 * be done in both hardware and software. In general, we usually only
76 * classify based on the layer two destination, eg. for Ethernet, the
77 * destination MAC address.
78 *
79 * The system also will do classification based on layer three and layer
80 * four properties. This is used to support things like flowadm(1M), which
81 * allows setting QoS and other properties on a per-flow basis.
82 *
83 * RING
84 *
85 * Conceptually, a ring represents a series of framed messages, often in a
86 * contiguous chunk of memory that acts as a circular buffer. Rings come in
87 * a couple of forms. Generally they are either a hardware construct (hw
88 * ring) or they are a software construct (sw ring) maintained by MAC.
89 *
90 * HW RING
91 *
92 * A hardware ring is a set of resources provided by a GLDv3 device driver
93 * (even if it is a pseudo-device). A hardware ring comes in two different
94 * forms: receive (rx) rings and transmit (tx) rings. An rx hw ring is
95 * something that has a unique DMA (direct memory access) region and
96 * generally supports some form of classification (though it isn't always
97 * used), as well as a means of generating an interrupt specific to that
98 * ring. For example, the device may generate a specific MSI-X for a PCI
99 * express device. A tx ring is similar, except that it is dedicated to
100 * transmission. It may also be a vector for enabling features such as VLAN
101 * tagging and large transmit offloading. It usually has its own dedicated
102 * interrupts for transmit being completed.
103 *
104 * SW RING
105 *
106 * A software ring is a construction of MAC. It represents the same thing
107 * that a hardware ring generally does, a collection of frames. However,
108 * instead of being in a contiguous ring of memory, they're instead linked
109 * by using the mblk_t's b_next pointer. Each frame may itself be multiple
110 * mblk_t's linked together by the b_cont pointer. A software ring always
111 * represents a collection of classified packets; however, it varies as to
112 * whether it uses only layer two information, or a combination of that and
113 * additional layer three and layer four data.
114 *
115 * FANOUT
116 *
117 * Fanout is the idea of spreading out the load of processing frames based
118 * on the source and destination information contained in the layer two,
119 * three, and four headers, such that the data can then be processed in
120 * parallel using multiple hardware threads.
121 *
122 * A fanout algorithm hashes the headers and uses that to place different
123 * flows into a bucket. The most important thing is that packets that are
124 * in the same flow end up in the same bucket. If they do not, performance
125 * can be adversely affected. Consider the case of TCP. TCP severely
126 * penalizes a connection if the data arrives out of order. If a given flow
127 * is processed on different CPUs, then the data will appear out of order,
128 * hence the invariant that fanout always hash a given flow to the same
129 * bucket and thus get processed on the same CPU.
130 *
131 * RECEIVE SIDE SCALING (RSS)
132 *
133 *
134 * Receive side scaling is a term that isn't common in illumos, but is used
135 * by vendors and was popularized by Microsoft. It refers to the idea of
136 * spreading the incoming receive load out across multiple interrupts which
137 * can be directed to different CPUs. This allows a device to leverage
138 * hardware rings even when it doesn't support hardware classification. The
139 * hardware uses an algorithm to perform fanout that ensures the flow
140 * invariant is maintained.
141 *
142 * SOFT RING SET
143 *
144 * A soft ring set, commonly abbreviated SRS, is a collection of rings and
145 * is used for both transmitting and receiving. It is maintained in the
146 * structure mac_soft_ring_set_t. A soft ring set is usually associated
147 * with flows, and coordinates both the use of hardware and software rings.
148 * Because the use of hardware rings can change as devices such as VNICs
149 * come and go, we always ensure that the set has software classification
150 * rules that correspond to the hardware classification rules from rings.
151 *
152 * Soft ring sets are also used for the enforcement of various QoS
153 * properties. For example, if a bandwidth limit has been placed on a
154 * specific flow or device, then that will be enforced by the soft ring
155 * set.
156 *
157 * SERVICE ATTACHMENT POINT (SAP)
158 *
159 * The service attachment point is a DLPI (Data Link Provider Interface)
160 * concept; however, it comes up quite often in MAC. Most MAC devices speak
161 * a protocol that has some notion of different channels or message type
162 * identifiers. For example, Ethernet defines an EtherType which is a part
163 * of the Ethernet header and defines the particular protocol of the data
164 * payload. If the EtherType is set to 0x0800, then it defines that the
165 * contents of that Ethernet frame is IPv4 traffic. For Ethernet, the
166 * EtherType is the SAP.
167 *
168 * In DLPI, a given consumer attaches to a specific SAP. In illumos, the ip
169 * and arp drivers attach to the EtherTypes for IPv4, IPv6, and ARP. Using
170 * libdlpi(3LIB) user software can attach to arbitrary SAPs. With the
171 * exception of 802.1Q VLAN tagged traffic, MAC itself does not directly
172 * consume the SAP; however, it uses that information as part of hashing
173 * and it may be used as part of the construction of flows.
174 *
175 * PRIMARY MAC CLIENT
176 *
177 * The primary mac client refers to a mac client whose unicast address
178 * matches the address of the device itself. For example, if the system has
179 * instance of the e1000g driver such as e1000g0, e1000g1, etc., the
180 * primary mac client is the one named after the device itself. VNICs that
181 * are created on top of such devices are not the primary client.
182 *
183 * TRANSMIT DESCRIPTORS
184 *
185 * Transmit descriptors are a resource that most GLDv3 device drivers have.
186 * Generally, a GLDv3 device driver takes a frame that's meant to be output
187 * and puts a copy of it into a region of memory. Each region of memory
188 * usually has an associated descriptor that the device uses to manage
189 * properties of the frames. Devices have a limited number of such
190 * descriptors. They get reclaimed once the device finishes putting the
191 * frame on the wire.
192 *
193 * If the driver runs out of transmit descriptors, for example, the OS is
194 * generating more frames than it can put on the wire, then it will return
195 * them back to the MAC layer.
196 *
197 * ---------------------------------
198 * Rings, Classification, and Fanout
199 * ---------------------------------
200 *
201 * The heart of MAC is made up of rings, and not those that Elven-kings wear.
202 * When receiving a packet, MAC breaks the work into two different, though
203 * interrelated phases. The first phase is generally classification and then the
204 * second phase is generally fanout. When a frame comes in from a GLDv3 Device,
205 * MAC needs to determine where that frame should be delivered. If it's a
206 * unicast frame (say a normal TCP/IP packet), then it will be delivered to a
207 * single MAC client; however, if it's a broadcast or multicast frame, then MAC
208 * may need to deliver it to multiple MAC clients.
209 *
210 * On transmit, classification isn't quite as important, but may still be used.
211 * Unlike with the receive path, the classification is not used to determine
212 * devices that should transmit something, but rather is used for special
213 * properties of a flow, eg. bandwidth limits for a given IP address, device, or
214 * connection.
215 *
216 * MAC employs a software classifier and leverages hardware classification as
217 * well. The software classifier can leverage the full layer two information,
218 * source, destination, VLAN, and SAP. If the SAP indicates that IP traffic is
219 * being sent, it can classify based on the IP header, and finally, it also
220 * knows how to classify based on the local and remote ports of TCP, UDP, and
221 * SCTP.
222 *
223 * Hardware classifiers vary in capability. Generally all hardware classifiers
224 * provide the capability to classify based on the destination MAC address. Some
225 * hardware has additional filters built in for performing more in-depth
226 * classification; however, it often has much more limited resources for these
227 * activities as compared to the layer two destination address classification.
228 *
229 * The modus operandi in MAC is to always ensure that we have software-based
230 * capabilities and rules in place and then to supplement that with hardware
231 * resources when available. In general, simple layer two classification is
232 * sufficient and nothing else is used, unless a specific flow is created with
233 * tools such as flowadm(1M) or bandwidth limits are set on a device with
234 * dladm(1M).
235 *
236 * RINGS AND GROUPS
237 *
238 * To get into how rings and classification play together, it's first important
239 * to understand how hardware devices commonly associate rings and allow them to
240 * be programmed. Recall that a hardware ring should be thought of as a DMA
241 * buffer and an interrupt resource. Rings are then collected into groups. A
242 * group itself has a series of classification rules. One or more MAC addresses
243 * are assigned to a group.
244 *
245 * Hardware devices vary in terms of what capabilities they provide. Sometimes
246 * they allow for a dynamic assignment of rings to a group and sometimes they
247 * have a static assignment of rings to a group. For example, the ixgbe driver
248 * has a static assignment of rings to groups such that every group has exactly
249 * one ring and the number of groups is equal to the number of rings.
250 *
251 * Classification and receive side scaling both come into play with how a device
252 * advertises itself to MAC and how MAC uses it. If a device supports layer two
253 * classification of frames, then MAC will assign MAC addresses to a group as a
254 * form of primary classification. If a single MAC address is assigned to a
255 * group, a common case, then MAC will consider packets that come in from rings
256 * on that group to be fully classified and will not need to do any software
257 * classification unless a specific flow has been created.
258 *
259 * If a device supports receive side scaling, then it may advertise or support
260 * groups with multiple rings. In those cases, then receive side scaling will
261 * come into play and MAC will use that as a means of fanning out received
262 * frames across multiple CPUs. This can also be combined with groups that
263 * support layer two classification.
264 *
265 * If a device supports dynamic assignments of rings to groups, then MAC will
266 * change around the way that rings are assigned to various groups as devices
267 * come and go from the system. For example, when a VNIC is created, a new flow
268 * will be created for the VNIC's MAC address. If a hardware ring is available,
269 * MAC may opt to reassign it from one group to another.
270 *
271 * ASSIGNMENT OF HARDWARE RINGS
272 *
273 * This is a bit of a complicated subject that varies depending on the device,
274 * the use of aggregations, the special nature of the primary mac client. This
275 * section deserves being fleshed out.
276 *
277 * FANOUT
278 *
279 * illumos uses fanout to help spread out the incoming processing load of chains
280 * of frames away from a single CPU. If a device supports receive side scaling,
281 * then that provides an initial form of fanout; however, what we're concerned
282 * with all happens after the context of a given set of frames being classified
283 * to a soft ring set.
284 *
285 * After frames reach a soft ring set and account for any potential bandwidth
286 * related accounting, they may be fanned out based on one of the following
287 * three modes:
288 *
289 * o No Fanout
290 * o Protocol level fanout
291 * o Full software ring protocol fanout
292 *
293 * MAC makes the determination as to which of these modes a given soft ring set
294 * obtains based on parameters such as whether or not it's the primary mac
295 * client, whether it's on a 10 GbE or faster device, user controlled dladm(1M)
296 * properties, and the nature of the hardware and the resources that it has.
297 *
298 * When there is no fanout, MAC does not create any soft rings for a device and
299 * the device has frames delivered directly to the MAC client.
300 *
301 * Otherwise, all fanout is performed by software. MAC divides incoming frames
302 * into one of three buckets -- IPv4 TCP traffic, IPv4 UDP traffic, and
303 * everything else. Note, VLAN tagged traffic is considered other, regardless of
304 * the interior EtherType. Regardless of the type of fanout, these three
305 * categories or buckets are always used.
306 *
307 * The difference between protocol level fanout and full software ring protocol
308 * fanout is the number of software rings that end up getting created. The
309 * system always uses the same number of software rings per protocol bucket. So
310 * in the first case when we're just doing protocol level fanout, we just create
311 * one software ring each for IPv4 TCP traffic, IPv4 UDP traffic, and everything
312 * else.
313 *
314 * In the case where we do full software ring protocol fanout, we generally use
315 * mac_compute_soft_ring_count() to determine the number of rings. There are
316 * other combinations of properties and devices that may send us down other
317 * paths, but this is a common starting point. If it's a non-bandwidth enforced
318 * device and we're on at least a 10 GbE link, then we'll use eight soft rings
319 * per protocol bucket as a starting point. See mac_compute_soft_ring_count()
320 * for more information on the total number.
321 *
322 * For each of these rings, we create a mac_soft_ring_t and an associated worker
323 * thread. Particularly when doing full software ring protocol fanout, we bind
324 * each of the worker threads to individual CPUs.
325 *
326 * The other advantage of these software rings is that it allows upper layers to
327 * optionally poll on them. For example, TCP can leverage an squeue to poll on
328 * the software ring, see squeue.c for more information.
329 *
330 * DLS BYPASS
331 *
332 * DLS is the data link services module. It interfaces with DLPI, which is the
333 * primary way that other parts of the system such as IP interface with the MAC
334 * layer. While DLS is traditionally a STREAMS-based interface, it allows for
335 * certain modules such as IP to negotiate various more modern interfaces to be
336 * used, which are useful for higher performance and allow it to use direct
337 * function calls to DLS instead of using STREAMS.
338 *
339 * When we have IPv4 TCP or UDP software rings, then traffic on those rings is
340 * eligible for what we call the dls bypass. In those cases, rather than going
341 * out mac_rx_deliver() to DLS, DLS instead registers them to go directly via
342 * the direct callback registered with DLS, generally ip_input().
343 *
344 * HARDWARE RING POLLING
345 *
346 * GLDv3 devices with hardware rings generally deliver chains of messages
347 * (mblk_t chain) during the context of a single interrupt. However, interrupts
348 * are not the only way that these devices may be used. As part of implementing
349 * ring support, a GLDv3 device driver must have a way to disable the generation
350 * of that interrupt and allow for the operating system to poll on that ring.
351 *
352 * To implement this, every soft ring set has a worker thread and a polling
353 * thread. If a sufficient packet rate comes into the system, MAC will 'blank'
354 * (disable) interrupts on that specific ring and the polling thread will start
355 * consuming packets from the hardware device and deliver them to the soft ring
356 * set, where the worker thread will take over.
357 *
358 * Once the rate of packet intake drops down below a certain threshold, then
359 * polling on the hardware ring will be quiesced and interrupts will be
360 * re-enabled for the given ring. This effectively allows the system to shift
361 * how it handles a ring based on its load. At high packet rates, polling on the
362 * device as opposed to relying on interrupts can actually reduce overall system
363 * load due to the minimization of interrupt activity.
364 *
365 * Note the importance of each ring having its own interrupt source. The whole
366 * idea here is that we do not disable interrupts on the device as a whole, but
367 * rather each ring can be independently toggled.
368 *
369 * USE OF WORKER THREADS
370 *
371 * Both the soft ring set and individual soft rings have a worker thread
372 * associated with them that may be bound to a specific CPU in the system. Any
373 * such assignment will get reassessed as part of dynamic reconfiguration events
374 * in the system such as the onlining and offlining of CPUs and the creation of
375 * CPU partitions.
376 *
377 * In many cases, while in an interrupt, we try to deliver a frame all the way
378 * through the stack in the context of the interrupt itself. However, if the
379 * amount of queued frames has exceeded a threshold, then we instead defer to
380 * the worker thread to do this work and signal it. This is particularly useful
381 * when you have the soft ring set delivering frames into multiple software
382 * rings. If it was only delivering frames into a single software ring then
383 * there'd be no need to have another thread take over. However, if it's
384 * delivering chains of frames to multiple rings, then it's worthwhile to have
385 * the worker for the software ring take over so that the different software
386 * rings can be processed in parallel.
387 *
388 * In a similar fashion to the hardware polling thread, if we don't have a
389 * backlog or there's nothing to do, then the worker thread will go back to
390 * sleep and frames can be delivered all the way from an interrupt. This
391 * behavior is useful as it's designed to minimize latency and the default
392 * disposition of MAC is to optimize for latency.
393 *
394 * MAINTAINING CHAINS
395 *
396 * Another useful idea that MAC uses is to try and maintain frames in chains for
397 * as long as possible. The idea is that all of MAC can handle chains of frames
398 * structured as a series of mblk_t structures linked with the b_next pointer.
399 * When performing software classification and software fanout, MAC does not
400 * simply determine the destination and send the frame along. Instead, in the
401 * case of classification, it tries to maintain a chain for as long as possible
402 * before passing it along and performing additional processing.
403 *
404 * In the case of fanout, MAC first determines what the target software ring is
405 * for every frame in the original chain and constructs a new chain for each
406 * target. MAC then delivers the new chain to each software ring in succession.
407 *
408 * The whole rationale for doing this is that we want to try and maintain the
409 * pipe as much as possible and deliver as many frames through the stack at once
410 * that we can, rather than just pushing a single frame through. This can often
411 * help bring down latency and allows MAC to get a better sense of the overall
412 * activity in the system and properly engage worker threads.
413 *
414 * --------------------
415 * Bandwidth Management
416 * --------------------
417 *
418 * Bandwidth management is something that's built into the soft ring set itself.
419 * When bandwidth limits are placed on a flow, a corresponding soft ring set is
420 * toggled into bandwidth mode. This changes how we transmit and receive the
421 * frames in question.
422 *
423 * Bandwidth management is done on a per-tick basis. We translate the user's
424 * requested bandwidth from a quantity per-second into a quantity per-tick. MAC
425 * cannot process a frame across more than one tick, thus it sets a lower bound
426 * for the bandwidth cap to be a single MTU. This also means that when
427 * hires ticks are enabled (hz is set to 1000), that the minimum amount of
428 * bandwidth is higher, because the number of ticks has increased and MAC has to
429 * go from accepting 100 packets / sec to 1000 / sec.
430 *
431 * The bandwidth counter is reset by either the soft ring set's worker thread or
432 * a thread that is doing an inline transmit or receive if they discover that
433 * the current tick is in the future from the recorded tick.
434 *
435 * Whenever we're receiving or transmitting data, we end up leaving most of the
436 * work to the soft ring set's worker thread. This forces data inserted into the
437 * soft ring set to be effectively serialized and allows us to exhume bandwidth
438 * at a reasonable rate. If there is nothing in the soft ring set at the moment
439 * and the set has available bandwidth, then it may processed inline.
440 * Otherwise, the worker is responsible for taking care of the soft ring set.
441 *
442 * ---------------------
443 * The Receive Data Path
444 * ---------------------
445 *
446 * The following series of ASCII art images breaks apart the way that a frame
447 * comes in and is processed in MAC.
448 *
449 * Part 1 -- Initial frame receipt, SRS classification
450 *
451 * Here, a frame is received by a GLDv3 driver, generally in the context of an
452 * interrupt, and it ends up in mac_rx_common(). A driver calls either mac_rx or
453 * mac_rx_ring, depending on whether or not it supports rings and can identify
454 * the interrupt as having come from a specific ring. Here we determine whether
455 * or not it's fully classified and perform software classification as
456 * appropriate. From here, everything always ends up going to either entry [A]
457 * or entry [B] based on whether or not they have subflow processing needed. We
458 * leave via fanout or delivery.
459 *
460 * +===========+
461 * v hardware v
462 * v interrupt v
463 * +===========+
464 * |
465 * * . . appropriate
466 * | upcall made
467 * | by GLDv3 driver . . always
468 * | .
469 * +--------+ | +----------+ . +---------------+
470 * | GLDv3 | +---->| mac_rx |-----*--->| mac_rx_common |
471 * | Driver |-->--+ +----------+ +---------------+
472 * +--------+ | ^ |
473 * | | ^ v
474 * ^ | * . . always +----------------------+
475 * | | | | mac_promisc_dispatch |
476 * | | +-------------+ +----------------------+
477 * | +--->| mac_rx_ring | |
478 * | +-------------+ * . . hw classified
479 * | v or single flow?
480 * | |
481 * | +--------++--------------+
482 * | | | * hw class,
483 * | | * hw classified | subflows
484 * | no hw class and . * | or single | exist
485 * | subflows | | flow |
486 * | | v v
487 * | | +-----------+ +-----------+
488 * | | | goto | | goto |
489 * | | | entry [A] | | entry [B] |
490 * | | +-----------+ +-----------+
491 * | v ^
492 * | +-------------+ |
493 * | | mac_rx_flow | * SRS and flow found,
494 * | +-------------+ | call flow cb
495 * | | +------+
496 * | v |
497 * v +==========+ +-----------------+
498 * | v For each v--->| mac_rx_classify |
499 * +----------+ v mblk_t v +-----------------+
500 * | srs | +==========+
501 * | pollling |
502 * | thread |->------------------------------------------+
503 * +----------+ |
504 * v . inline
505 * +--------------------+ +----------+ +---------+ .
506 * [A]---->| mac_rx_srs_process |-->| check bw |-->| enqueue |--*---------+
507 * +--------------------+ | limits | | frames | |
508 * ^ +----------+ | to SRS | |
509 * | +---------+ |
510 * | send chain +--------+ | |
511 * * when clasified | signal | * BW limits, |
512 * | flow changes | srs |<---+ loopback, |
513 * | | worker | stack too |
514 * | +--------+ deep |
515 * +-----------------+ +--------+ |
516 * | mac_flow_lookup | | srs | +---------------------+ |
517 * +-----------------+ | worker |---->| mac_rx_srs_drain |<---+
518 * ^ | thread | | mac_rx_srs_drain_bw |
519 * | +--------+ +---------------------+
520 * | |
521 * +----------------------------+ * software rings
522 * [B]-->| mac_rx_srs_subflow_process | | for fanout?
523 * +----------------------------+ |
524 * +----------+-----------+
525 * | |
526 * v v
527 * +--------+ +--------+
528 * | goto | | goto |
529 * | Part 2 | | Part 3 |
530 * +--------+ +--------+
531 *
532 * Part 2 -- Fanout
533 *
534 * This part is concerned with using software fanout to assign frames to
535 * software rings and then deliver them to MAC clients or allow those rings to
536 * be polled upon. While there are two different primary fanout entry points,
537 * mac_rx_fanout and mac_rx_proto_fanout, they behave in similar ways, and aside
538 * from some of the individual hashing techniques used, most of the general
539 * flow is the same.
540 *
541 * +--------+ +-------------------+
542 * | From |---+--------->| mac_rx_srs_fanout |----+
543 * | Part 1 | | +-------------------+ | +=================+
544 * +--------+ | | v for each mblk_t v
545 * * . . protocol only +--->v assign to new v
546 * | fanout | v chain based on v
547 * | | v hash % nrings v
548 * | +-------------------------+ | +=================+
549 * +--->| mac_rx_srs_proto_fanout |----+ |
550 * +-------------------------+ |
551 * v
552 * +------------+ +--------------------------+ +================+
553 * | enqueue in |<---| mac_rx_soft_ring_process |<------v for each chain v
554 * | soft ring | +--------------------------+ +================+
555 * +------------+
556 * | +-----------+
557 * * soft ring set | soft ring |
558 * | empty and no | worker |
559 * | worker? | thread |
560 * | +-----------+
561 * +------*----------------+ |
562 * | . | v
563 * No . * . Yes | +------------------------+
564 * | +----<--| mac_rx_soft_ring_drain |
565 * | | +------------------------+
566 * v |
567 * +-----------+ v
568 * | signal | +---------------+
569 * | soft ring | | Deliver chain |
570 * | worker | | goto Part 3 |
571 * +-----------+ +---------------+
572 *
573 *
574 * Part 3 -- Packet Delivery
575 *
576 * Here, we go through and deliver the mblk_t chain directly to a given
577 * processing function. In a lot of cases this is mac_rx_deliver(). In the case
578 * of DLS bypass being used, then instead we end up going ahead and deliver it
579 * to the direct callback registered with DLS, generally ip_input.
580 *
581 *
582 * +---------+ +----------------+ +------------------+
583 * | From |---+------->| mac_rx_deliver |--->| Off to DLS, or |
584 * | Parts 1 | | +----------------+ | other MAC client |
585 * | and 2 | * DLS bypass +------------------+
586 * +---------+ | enabled +----------+ +-------------+
587 * +---------->| ip_input |--->| To IP |
588 * +----------+ | and beyond! |
589 * +-------------+
590 *
591 * ----------------------
592 * The Transmit Data Path
593 * ----------------------
594 *
595 * Before we go into the images, it's worth talking about a problem that is a
596 * bit different from the receive data path. GLDv3 device drivers have a finite
597 * amount of transmit descriptors. When they run out, they return unused frames
598 * back to MAC. MAC, at this point has several options about what it will do,
599 * which vary based upon the settings that the client uses.
600 *
601 * When a device runs out of descriptors, the next thing that MAC does is
602 * enqueue them off of the soft ring set or a software ring, depending on the
603 * configuration of the soft ring set. MAC will enqueue up to a high watermark
604 * of mblk_t chains, at which point it will indicate flow control back to the
605 * client. Once this condition is reached, any mblk_t chains that were not
606 * enqueued will be returned to the caller and they will have to decide what to
607 * do with them. There are various flags that control this behavior that a
608 * client may pass, which are discussed below.
609 *
610 * When this condition is hit, MAC also returns a cookie to the client in
611 * addition to unconsumed frames. Clients can poll on that cookie and register a
612 * callback with MAC to be notified when they are no longer subject to flow
613 * control, at which point they may continue to call mac_tx(). This flow control
614 * actually manages to work itself all the way up the stack, back through dls,
615 * to ip, through the various protocols, and to sockfs.
616 *
617 * While the behavior described above is the default, this behavior can be
618 * modified. There are two alternate modes, described below, which are
619 * controlled with flags.
620 *
621 * DROP MODE
622 *
623 * This mode is controlled by having the client pass the MAC_DROP_ON_NO_DESC
624 * flag. When this is passed, if a device driver runs out of transmit
625 * descriptors, then the MAC layer will drop any unsent traffic. The client in
626 * this case will never have any frames returned to it.
627 *
628 * DON'T ENQUEUE
629 *
630 * This mode is controlled by having the client pass the MAC_TX_NO_ENQUEUE flag.
631 * If the MAC_DROP_ON_NO_DESC flag is also passed, it takes precedence. In this
632 * mode, when we hit a case where a driver runs out of transmit descriptors,
633 * then instead of enqueuing packets in a soft ring set or software ring, we
634 * instead return the mblk_t chain back to the caller and immediately put the
635 * soft ring set into flow control mode.
636 *
637 * The following series of ASCII art images describe the transmit data path that
638 * MAC clients enter into based on calling into mac_tx(). A soft ring set has a
639 * transmission function associated with it. There are seven possible
640 * transmission modes, some of which share function entry points. The one that a
641 * soft ring set gets depends on properties such as whether there are
642 * transmission rings for fanout, whether the device involves aggregations,
643 * whether any bandwidth limits exist, etc.
644 *
645 *
646 * Part 1 -- Initial checks
647 *
648 * * . called by
649 * | MAC clients
650 * v . . No
651 * +--------+ +-----------+ . +-------------------+ +====================+
652 * | mac_tx |->| device |-*-->| mac_protect_check |->v Is this the simple v
653 * +--------+ | quiesced? | +-------------------+ v case? See [1] v
654 * +-----------+ | +====================+
655 * * . Yes * failed |
656 * v | frames |
657 * +--------------+ | +-------+---------+
658 * | freemsgchain |<---------+ Yes . * No . *
659 * +--------------+ v v
660 * +-----------+ +--------+
661 * | goto | | goto |
662 * | Part 2 | | SRS TX |
663 * | Entry [A] | | func |
664 * +-----------+ +--------+
665 * | |
666 * | v
667 * | +--------+
668 * +---------->| return |
669 * | cookie |
670 * +--------+
671 *
672 * [1] The simple case refers to the SRS being configured with the
673 * SRS_TX_DEFAULT transmission mode, having a single mblk_t (not a chain), their
674 * being only a single active client, and not having a backlog in the srs.
675 *
676 *
677 * Part 2 -- The SRS transmission functions
678 *
679 * This part is a bit more complicated. The different transmission paths often
680 * leverage one another. In this case, we'll draw out the more common ones
681 * before the parts that depend upon them. Here, we're going to start with the
682 * workings of mac_tx_send() a common function that most of the others end up
683 * calling.
684 *
685 * +-------------+
686 * | mac_tx_send |
687 * +-------------+
688 * |
689 * v
690 * +=============+ +==============+
691 * v more than v--->v check v
692 * v one client? v v VLAN and add v
693 * +=============+ v VLAN tags v
694 * | +==============+
695 * | |
696 * +------------------+
697 * |
698 * | [A]
699 * v |
700 * +============+ . No v
701 * v more than v . +==========+ +--------------------------+
702 * v one active v-*---->v for each v---->| mac_promisc_dispatch_one |---+
703 * v client? v v mblk_t v +--------------------------+ |
704 * +============+ +==========+ ^ |
705 * | | +==========+ |
706 * * . Yes | v hardware v<-------+
707 * v +------------+ v rings? v
708 * +==========+ | +==========+
709 * v for each v No . . . * |
710 * v mblk_t v specific | |
711 * +==========+ flow | +-----+-----+
712 * | | | |
713 * v | v v
714 * +-----------------+ | +-------+ +---------+
715 * | mac_tx_classify |------------+ | GLDv3 | | GLDv3 |
716 * +-----------------+ |TX func| | ring tx |
717 * | +-------+ | func |
718 * * Specific flow, generally | +---------+
719 * | bcast, mcast, loopback | |
720 * v +-----+-----+
721 * +==========+ +---------+ |
722 * v valid L2 v--*--->| freemsg | v
723 * v header v . No +---------+ +-------------------+
724 * +==========+ | return unconsumed |
725 * * . Yes | frames to the |
726 * v | caller |
727 * +===========+ +-------------------+
728 * v braodcast v +----------------+ ^
729 * v flow? v--*-->| mac_bcast_send |------------------+
730 * +===========+ . +----------------+ |
731 * | . . Yes |
732 * No . * v
733 * | +---------------------+ +---------------+ +----------+
734 * +->|mac_promisc_dispatch |->| mac_fix_cksum |->| flow |
735 * +---------------------+ +---------------+ | callback |
736 * +----------+
737 *
738 *
739 * In addition, many but not all of the routines, all rely on
740 * mac_tx_softring_process as an entry point.
741 *
742 *
743 * . No . No
744 * +--------------------------+ +========+ . +===========+ . +-------------+
745 * | mac_tx_soft_ring_process |-->v worker v-*->v out of tx v-*->| goto |
746 * +--------------------------+ v only? v v descr.? v | mac_tx_send |
747 * +========+ +===========+ +-------------+
748 * Yes . * * . Yes |
749 * . No v | v
750 * v=========+ . +===========+ . Yes | Yes . +==========+
751 * v apppend v<--*----------v out of tx v-*-------+---------*--v returned v
752 * v mblk_t v v descr.? v | v frames? v
753 * v chain v +===========+ | +==========+
754 * +=========+ | *. No
755 * | | v
756 * v v +------------+
757 * +===================+ +----------------------+ | done |
758 * v worker scheduled? v | mac_tx_sring_enqueue | | processing |
759 * v Out of tx descr? v +----------------------+ +------------+
760 * +===================+ |
761 * | | . Yes v
762 * * Yes * No . +============+
763 * | v +-*---------v drop on no v
764 * | +========+ v v TX desc? v
765 * | v wake v +----------+ +============+
766 * | v worker v | mac_pkt_ | * . No
767 * | +========+ | drop | | . Yes . No
768 * | | +----------+ v . .
769 * | | v ^ +===============+ . +========+ .
770 * +--+--------+---------+ | v Don't enqueue v-*->v ring v-*----+
771 * | | v Set? v v empty? v |
772 * | +---------------+ +===============+ +========+ |
773 * | | | | |
774 * | | +-------------------+ | |
775 * | *. Yes | +---------+ |
776 * | | v v v
777 * | | +===========+ +========+ +--------------+
778 * | +<-v At hiwat? v v append v | return |
779 * | +===========+ v mblk_t v | mblk_t chain |
780 * | * No v chain v | and flow |
781 * | v +========+ | control |
782 * | +=========+ | | cookie |
783 * | v append v v +--------------+
784 * | v mblk_t v +========+
785 * | v chain v v wake v +------------+
786 * | +=========+ v worker v-->| done |
787 * | | +========+ | processing |
788 * | v .. Yes +------------+
789 * | +=========+ . +========+
790 * | v first v--*-->v wake v
791 * | v append? v v worker v
792 * | +=========+ +========+
793 * | | |
794 * | No . * |
795 * | v |
796 * | +--------------+ |
797 * +------>| Return | |
798 * | flow control |<------------+
799 * | cookie |
800 * +--------------+
801 *
802 *
803 * The remaining images are all specific to each of the different transmission
804 * modes.
805 *
806 * SRS TX DEFAULT
807 *
808 * [ From Part 1 ]
809 * |
810 * v
811 * +-------------------------+
812 * | mac_tx_single_ring_mode |
813 * +-------------------------+
814 * |
815 * | . Yes
816 * v .
817 * +==========+ . +============+
818 * v SRS v-*->v Try to v---->---------------------+
819 * v backlog? v v enqueue in v |
820 * +==========+ v SRS v-->------+ * . . Queue too
821 * | +============+ * don't enqueue | deep or
822 * * . No ^ | | flag or at | drop flag
823 * | | v | hiwat, |
824 * v | | | return +---------+
825 * +-------------+ | | | cookie | freemsg |
826 * | goto |-*-----+ | | +---------+
827 * | mac_tx_send | . returned | | |
828 * +-------------+ mblk_t | | |
829 * | | | |
830 * | | | |
831 * * . . all mblk_t * queued, | |
832 * v consumed | may return | |
833 * +-------------+ | tx cookie | |
834 * | SRS TX func |<------------+------------+----------------+
835 * | completed |
836 * +-------------+
837 *
838 * SRS_TX_SERIALIZE
839 *
840 * +------------------------+
841 * | mac_tx_serializer_mode |
842 * +------------------------+
843 * |
844 * | . No
845 * v .
846 * +============+ . +============+ +-------------+ +============+
847 * v srs being v-*->v set SRS v--->| goto |-->v remove SRS v
848 * v processed? v v proc flags v | mac_tx_send | v proc flag v
849 * +============+ +============+ +-------------+ +============+
850 * | |
851 * * Yes |
852 * v . No v
853 * +--------------------+ . +==========+
854 * | mac_tx_srs_enqueue | +------------------------*-----<--v returned v
855 * +--------------------+ | v frames? v
856 * | | . Yes +==========+
857 * | | . |
858 * | | . +=========+ v
859 * v +-<-*-v queued v +--------------------+
860 * +-------------+ | v frames? v<----| mac_tx_srs_enqueue |
861 * | SRS TX func | | +=========+ +--------------------+
862 * | completed, |<------+ * . Yes
863 * | may return | | v
864 * | cookie | | +========+
865 * +-------------+ +-<---v wake v
866 * v worker v
867 * +========+
868 *
869 *
870 * SRS_TX_FANOUT
871 *
872 * . Yes
873 * +--------------------+ +=============+ . +--------------------------+
874 * | mac_tx_fanout_mode |--->v Have fanout v-*-->| goto |
875 * +--------------------+ v hint? v | mac_rx_soft_ring_process |
876 * +=============+ +--------------------------+
877 * * . No |
878 * v ^
879 * +===========+ |
880 * +--->v for each v +===============+
881 * | v mblk_t v v pick softring v
882 * same * +===========+ v from hash v
883 * hash | | +===============+
884 * | v |
885 * | +--------------+ |
886 * +---| mac_pkt_hash |--->*------------+
887 * +--------------+ . different
888 * hash or
889 * done proc.
890 * SRS_TX_AGGR chain
891 *
892 * +------------------+ +================================+
893 * | mac_tx_aggr_mode |--->v Use aggr capab function to v
894 * +------------------+ v find appropriate tx ring. v
895 * v Applies hash based on aggr v
896 * v policy, see mac_tx_aggr_mode() v
897 * +================================+
898 * |
899 * v
900 * +-------------------------------+
901 * | goto |
902 * | mac_rx_srs_soft_ring_process |
903 * +-------------------------------+
904 *
905 *
906 * SRS_TX_BW, SRS_TX_BW_FANOUT, SRS_TX_BW_AGGR
907 *
908 * Note, all three of these tx functions start from the same place --
909 * mac_tx_bw_mode().
910 *
911 * +----------------+
912 * | mac_tx_bw_mode |
913 * +----------------+
914 * |
915 * v . No . No . Yes
916 * +==============+ . +============+ . +=============+ . +=========+
917 * v Out of BW? v--*->v SRS empty? v--*->v reset BW v-*->v Bump BW v
918 * +==============+ +============+ v tick count? v v Usage v
919 * | | +=============+ +=========+
920 * | +---------+ | |
921 * | | +--------------------+ |
922 * | | | +----------------------+
923 * v | v v
924 * +===============+ | +==========+ +==========+ +------------------+
925 * v Don't enqueue v | v set bw v v Is aggr? v--*-->| goto |
926 * v flag set? v | v enforced v +==========+ . | mac_tx_aggr_mode |-+
927 * +===============+ | +==========+ | . +------------------+ |
928 * | Yes .* | | No . * . |
929 * | | | | | . Yes |
930 * * . No | | v | |
931 * | +---------+ | +========+ v +======+ |
932 * | | freemsg | | v append v +============+ . Yes v pick v |
933 * | +---------+ | v mblk_t v v Is fanout? v--*---->v ring v |
934 * | | | v chain v +============+ +======+ |
935 * +------+ | +========+ | | |
936 * v | | v v |
937 * +---------+ | v +-------------+ +--------------------+ |
938 * | return | | +========+ | goto | | goto | |
939 * | flow | | v wakeup v | mac_tx_send | | mac_tx_fanout_mode | |
940 * | control | | v worker v +-------------+ +--------------------+ |
941 * | cookie | | +========+ | | |
942 * +---------+ | | | +------+------+
943 * | v | |
944 * | +---------+ | v
945 * | | return | +============+ +------------+
946 * | | flow | v unconsumed v-------+ | done |
947 * | | control | v frames? v | | processing |
948 * | | cookie | +============+ | +------------+
949 * | +---------+ | |
950 * | Yes * |
951 * | | |
952 * | +===========+ |
953 * | v subtract v |
954 * | v unused bw v |
955 * | +===========+ |
956 * | | |
957 * | v |
958 * | +--------------------+ |
959 * +------------->| mac_tx_srs_enqueue | |
960 * +--------------------+ |
961 * | |
962 * | |
963 * +------------+ |
964 * | return fc | |
965 * | cookie and |<------+
966 * | mblk_t |
967 * +------------+
968 */
969
970 #include <sys/types.h>
971 #include <sys/callb.h>
972 #include <sys/sdt.h>
973 #include <sys/strsubr.h>
974 #include <sys/strsun.h>
975 #include <sys/vlan.h>
976 #include <sys/stack.h>
977 #include <sys/archsystm.h>
978 #include <inet/ipsec_impl.h>
979 #include <inet/ip_impl.h>
980 #include <inet/sadb.h>
981 #include <inet/ipsecesp.h>
982 #include <inet/ipsecah.h>
983 #include <inet/ip6.h>
984
985 #include <sys/mac_impl.h>
986 #include <sys/mac_client_impl.h>
987 #include <sys/mac_client_priv.h>
988 #include <sys/mac_soft_ring.h>
989 #include <sys/mac_flow_impl.h>
990
991 static mac_tx_cookie_t mac_tx_single_ring_mode(mac_soft_ring_set_t *, mblk_t *,
992 uintptr_t, uint16_t, mblk_t **);
993 static mac_tx_cookie_t mac_tx_serializer_mode(mac_soft_ring_set_t *, mblk_t *,
994 uintptr_t, uint16_t, mblk_t **);
995 static mac_tx_cookie_t mac_tx_fanout_mode(mac_soft_ring_set_t *, mblk_t *,
996 uintptr_t, uint16_t, mblk_t **);
997 static mac_tx_cookie_t mac_tx_bw_mode(mac_soft_ring_set_t *, mblk_t *,
998 uintptr_t, uint16_t, mblk_t **);
999 static mac_tx_cookie_t mac_tx_aggr_mode(mac_soft_ring_set_t *, mblk_t *,
1000 uintptr_t, uint16_t, mblk_t **);
1001
1002 typedef struct mac_tx_mode_s {
1003 mac_tx_srs_mode_t mac_tx_mode;
1004 mac_tx_func_t mac_tx_func;
1005 } mac_tx_mode_t;
1006
1007 /*
1008 * There are seven modes of operation on the Tx side. These modes get set
1009 * in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode,
1010 * none of the other modes are user configurable. They get selected by
1011 * the system depending upon whether the link (or flow) has multiple Tx
1012 * rings or a bandwidth configured, or if the link is an aggr, etc.
1013 *
1014 * When the Tx SRS is operating in aggr mode (st_mode) or if there are
1015 * multiple Tx rings owned by Tx SRS, then each Tx ring (pseudo or
1016 * otherwise) will have a soft ring associated with it. These soft rings
1017 * are stored in srs_tx_soft_rings[] array.
1018 *
1019 * Additionally in the case of aggr, there is the st_soft_rings[] array
1020 * in the mac_srs_tx_t structure. This array is used to store the same
1021 * set of soft rings that are present in srs_tx_soft_rings[] array but
1022 * in a different manner. The soft ring associated with the pseudo Tx
1023 * ring is saved at mr_index (of the pseudo ring) in st_soft_rings[]
1024 * array. This helps in quickly getting the soft ring associated with the
1025 * Tx ring when aggr_find_tx_ring() returns the pseudo Tx ring that is to
1026 * be used for transmit.
1027 */
1028 mac_tx_mode_t mac_tx_mode_list[] = {
1029 {SRS_TX_DEFAULT, mac_tx_single_ring_mode},
1030 {SRS_TX_SERIALIZE, mac_tx_serializer_mode},
1031 {SRS_TX_FANOUT, mac_tx_fanout_mode},
1032 {SRS_TX_BW, mac_tx_bw_mode},
1033 {SRS_TX_BW_FANOUT, mac_tx_bw_mode},
1034 {SRS_TX_AGGR, mac_tx_aggr_mode},
1035 {SRS_TX_BW_AGGR, mac_tx_bw_mode}
1036 };
1037
1038 /*
1039 * Soft Ring Set (SRS) - The Run time code that deals with
1040 * dynamic polling from the hardware, bandwidth enforcement,
1041 * fanout etc.
1042 *
1043 * We try to use H/W classification on NIC and assign traffic for
1044 * a MAC address to a particular Rx ring or ring group. There is a
1045 * 1-1 mapping between a SRS and a Rx ring. The SRS dynamically
1046 * switches the underlying Rx ring between interrupt and
1047 * polling mode and enforces any specified B/W control.
1048 *
1049 * There is always a SRS created and tied to each H/W and S/W rule.
1050 * Whenever we create a H/W rule, we always add the the same rule to
1051 * S/W classifier and tie a SRS to it.
1052 *
1053 * In case a B/W control is specified, it is broken into bytes
1054 * per ticks and as soon as the quota for a tick is exhausted,
1055 * the underlying Rx ring is forced into poll mode for remainder of
1056 * the tick. The SRS poll thread only polls for bytes that are
1057 * allowed to come in the SRS. We typically let 4x the configured
1058 * B/W worth of packets to come in the SRS (to prevent unnecessary
1059 * drops due to bursts) but only process the specified amount.
1060 *
1061 * A MAC client (e.g. a VNIC or aggr) can have 1 or more
1062 * Rx rings (and corresponding SRSs) assigned to it. The SRS
1063 * in turn can have softrings to do protocol level fanout or
1064 * softrings to do S/W based fanout or both. In case the NIC
1065 * has no Rx rings, we do S/W classification to respective SRS.
1066 * The S/W classification rule is always setup and ready. This
1067 * allows the MAC layer to reassign Rx rings whenever needed
1068 * but packets still continue to flow via the default path and
1069 * getting S/W classified to correct SRS.
1070 *
1071 * The SRS's are used on both Tx and Rx side. They use the same
1072 * data structure but the processing routines have slightly different
1073 * semantics due to the fact that Rx side needs to do dynamic
1074 * polling etc.
1075 *
1076 * Dynamic Polling Notes
1077 * =====================
1078 *
1079 * Each Soft ring set is capable of switching its Rx ring between
1080 * interrupt and poll mode and actively 'polls' for packets in
1081 * poll mode. If the SRS is implementing a B/W limit, it makes
1082 * sure that only Max allowed packets are pulled in poll mode
1083 * and goes to poll mode as soon as B/W limit is exceeded. As
1084 * such, there are no overheads to implement B/W limits.
1085 *
1086 * In poll mode, its better to keep the pipeline going where the
1087 * SRS worker thread keeps processing packets and poll thread
1088 * keeps bringing more packets (specially if they get to run
1089 * on different CPUs). This also prevents the overheads associated
1090 * by excessive signalling (on NUMA machines, this can be
1091 * pretty devastating). The exception is latency optimized case
1092 * where worker thread does no work and interrupt and poll thread
1093 * are allowed to do their own drain.
1094 *
1095 * We use the following policy to control Dynamic Polling:
1096 * 1) We switch to poll mode anytime the processing
1097 * thread causes a backlog to build up in SRS and
1098 * its associated Soft Rings (sr_poll_pkt_cnt > 0).
1099 * 2) As long as the backlog stays under the low water
1100 * mark (sr_lowat), we poll the H/W for more packets.
1101 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low
1102 * water mark, we stay in poll mode but don't poll
1103 * the H/W for more packets.
1104 * 4) Anytime in polling mode, if we poll the H/W for
1105 * packets and find nothing plus we have an existing
1106 * backlog (sr_poll_pkt_cnt > 0), we stay in polling
1107 * mode but don't poll the H/W for packets anymore
1108 * (let the polling thread go to sleep).
1109 * 5) Once the backlog is relived (packets are processed)
1110 * we reenable polling (by signalling the poll thread)
1111 * only when the backlog dips below sr_poll_thres.
1112 * 6) sr_hiwat is used exclusively when we are not
1113 * polling capable and is used to decide when to
1114 * drop packets so the SRS queue length doesn't grow
1115 * infinitely.
1116 *
1117 * NOTE: Also see the block level comment on top of mac_soft_ring.c
1118 */
1119
1120 /*
1121 * mac_latency_optimize
1122 *
1123 * Controls whether the poll thread can process the packets inline
1124 * or let the SRS worker thread do the processing. This applies if
1125 * the SRS was not being processed. For latency sensitive traffic,
1126 * this needs to be true to allow inline processing. For throughput
1127 * under load, this should be false.
1128 *
1129 * This (and other similar) tunable should be rolled into a link
1130 * or flow specific workload hint that can be set using dladm
1131 * linkprop (instead of multiple such tunables).
1132 */
1133 boolean_t mac_latency_optimize = B_TRUE;
1134
1135 /*
1136 * MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN
1137 *
1138 * queue a mp or chain in soft ring set and increment the
1139 * local count (srs_count) for the SRS and the shared counter
1140 * (srs_poll_pkt_cnt - shared between SRS and its soft rings
1141 * to track the total unprocessed packets for polling to work
1142 * correctly).
1143 *
1144 * The size (total bytes queued) counters are incremented only
1145 * if we are doing B/W control.
1146 */
1147 #define MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
1148 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1149 if ((mac_srs)->srs_last != NULL) \
1150 (mac_srs)->srs_last->b_next = (head); \
1151 else \
1152 (mac_srs)->srs_first = (head); \
1153 (mac_srs)->srs_last = (tail); \
1154 (mac_srs)->srs_count += count; \
1155 }
1156
1157 #define MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
1158 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
1159 \
1160 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
1161 srs_rx->sr_poll_pkt_cnt += count; \
1162 ASSERT(srs_rx->sr_poll_pkt_cnt > 0); \
1163 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
1164 (mac_srs)->srs_size += (sz); \
1165 mutex_enter(&(mac_srs)->srs_bw->mac_bw_lock); \
1166 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
1167 mutex_exit(&(mac_srs)->srs_bw->mac_bw_lock); \
1168 } \
1169 }
1170
1171 #define MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
1172 mac_srs->srs_state |= SRS_ENQUEUED; \
1173 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
1174 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
1175 (mac_srs)->srs_size += (sz); \
1176 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
1177 } \
1178 }
1179
1180 /*
1181 * Turn polling on routines
1182 */
1183 #define MAC_SRS_POLLING_ON(mac_srs) { \
1184 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1185 if (((mac_srs)->srs_state & \
1186 (SRS_POLLING_CAPAB|SRS_POLLING)) == SRS_POLLING_CAPAB) { \
1187 (mac_srs)->srs_state |= SRS_POLLING; \
1188 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
1189 (mac_srs)->srs_ring); \
1190 (mac_srs)->srs_rx.sr_poll_on++; \
1191 } \
1192 }
1193
1194 #define MAC_SRS_WORKER_POLLING_ON(mac_srs) { \
1195 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1196 if (((mac_srs)->srs_state & \
1197 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_POLLING)) == \
1198 (SRS_POLLING_CAPAB|SRS_WORKER)) { \
1199 (mac_srs)->srs_state |= SRS_POLLING; \
1200 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
1201 (mac_srs)->srs_ring); \
1202 (mac_srs)->srs_rx.sr_worker_poll_on++; \
1203 } \
1204 }
1205
1206 /*
1207 * MAC_SRS_POLL_RING
1208 *
1209 * Signal the SRS poll thread to poll the underlying H/W ring
1210 * provided it wasn't already polling (SRS_GET_PKTS was set).
1211 *
1212 * Poll thread gets to run only from mac_rx_srs_drain() and only
1213 * if the drain was being done by the worker thread.
1214 */
1215 #define MAC_SRS_POLL_RING(mac_srs) { \
1216 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
1217 \
1218 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1219 srs_rx->sr_poll_thr_sig++; \
1220 if (((mac_srs)->srs_state & \
1221 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_GET_PKTS)) == \
1222 (SRS_WORKER|SRS_POLLING_CAPAB)) { \
1223 (mac_srs)->srs_state |= SRS_GET_PKTS; \
1224 cv_signal(&(mac_srs)->srs_cv); \
1225 } else { \
1226 srs_rx->sr_poll_thr_busy++; \
1227 } \
1228 }
1229
1230 /*
1231 * MAC_SRS_CHECK_BW_CONTROL
1232 *
1233 * Check to see if next tick has started so we can reset the
1234 * SRS_BW_ENFORCED flag and allow more packets to come in the
1235 * system.
1236 */
1237 #define MAC_SRS_CHECK_BW_CONTROL(mac_srs) { \
1238 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1239 ASSERT(((mac_srs)->srs_type & SRST_TX) || \
1240 MUTEX_HELD(&(mac_srs)->srs_bw->mac_bw_lock)); \
1241 clock_t now = ddi_get_lbolt(); \
1242 if ((mac_srs)->srs_bw->mac_bw_curr_time != now) { \
1243 (mac_srs)->srs_bw->mac_bw_curr_time = now; \
1244 (mac_srs)->srs_bw->mac_bw_used = 0; \
1245 if ((mac_srs)->srs_bw->mac_bw_state & SRS_BW_ENFORCED) \
1246 (mac_srs)->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; \
1247 } \
1248 }
1249
1250 /*
1251 * MAC_SRS_WORKER_WAKEUP
1252 *
1253 * Wake up the SRS worker thread to process the queue as long as
1254 * no one else is processing the queue. If we are optimizing for
1255 * latency, we wake up the worker thread immediately or else we
1256 * wait mac_srs_worker_wakeup_ticks before worker thread gets
1257 * woken up.
1258 */
1259 int mac_srs_worker_wakeup_ticks = 0;
1260 #define MAC_SRS_WORKER_WAKEUP(mac_srs) { \
1261 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1262 if (!((mac_srs)->srs_state & SRS_PROC) && \
1263 (mac_srs)->srs_tid == NULL) { \
1264 if (((mac_srs)->srs_state & SRS_LATENCY_OPT) || \
1265 (mac_srs_worker_wakeup_ticks == 0)) \
1266 cv_signal(&(mac_srs)->srs_async); \
1267 else \
1268 (mac_srs)->srs_tid = \
1269 timeout(mac_srs_fire, (mac_srs), \
1270 mac_srs_worker_wakeup_ticks); \
1271 } \
1272 }
1273
1274 #define TX_BANDWIDTH_MODE(mac_srs) \
1275 ((mac_srs)->srs_tx.st_mode == SRS_TX_BW || \
1276 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT || \
1277 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_AGGR)
1278
1279 #define TX_SRS_TO_SOFT_RING(mac_srs, head, hint) { \
1280 if (tx_mode == SRS_TX_BW_FANOUT) \
1281 (void) mac_tx_fanout_mode(mac_srs, head, hint, 0, NULL);\
1282 else \
1283 (void) mac_tx_aggr_mode(mac_srs, head, hint, 0, NULL); \
1284 }
1285
1286 /*
1287 * MAC_TX_SRS_BLOCK
1288 *
1289 * Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED
1290 * will be set only if srs_tx_woken_up is FALSE. If
1291 * srs_tx_woken_up is TRUE, it indicates that the wakeup arrived
1292 * before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to
1293 * attempt to transmit again and not setting SRS_TX_BLOCKED does
1294 * that.
1295 */
1296 #define MAC_TX_SRS_BLOCK(srs, mp) { \
1297 ASSERT(MUTEX_HELD(&(srs)->srs_lock)); \
1298 if ((srs)->srs_tx.st_woken_up) { \
1299 (srs)->srs_tx.st_woken_up = B_FALSE; \
1300 } else { \
1301 ASSERT(!((srs)->srs_state & SRS_TX_BLOCKED)); \
1302 (srs)->srs_state |= SRS_TX_BLOCKED; \
1303 (srs)->srs_tx.st_stat.mts_blockcnt++; \
1304 } \
1305 }
1306
1307 /*
1308 * MAC_TX_SRS_TEST_HIWAT
1309 *
1310 * Called before queueing a packet onto Tx SRS to test and set
1311 * SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat.
1312 */
1313 #define MAC_TX_SRS_TEST_HIWAT(srs, mp, tail, cnt, sz, cookie) { \
1314 boolean_t enqueue = 1; \
1315 \
1316 if ((srs)->srs_count > (srs)->srs_tx.st_hiwat) { \
1317 /* \
1318 * flow-controlled. Store srs in cookie so that it \
1319 * can be returned as mac_tx_cookie_t to client \
1320 */ \
1321 (srs)->srs_state |= SRS_TX_HIWAT; \
1322 cookie = (mac_tx_cookie_t)srs; \
1323 (srs)->srs_tx.st_hiwat_cnt++; \
1324 if ((srs)->srs_count > (srs)->srs_tx.st_max_q_cnt) { \
1325 /* increment freed stats */ \
1326 (srs)->srs_tx.st_stat.mts_sdrops += cnt; \
1327 /* \
1328 * b_prev may be set to the fanout hint \
1329 * hence can't use freemsg directly \
1330 */ \
1331 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE); \
1332 DTRACE_PROBE1(tx_queued_hiwat, \
1333 mac_soft_ring_set_t *, srs); \
1334 enqueue = 0; \
1335 } \
1336 } \
1337 if (enqueue) \
1338 MAC_TX_SRS_ENQUEUE_CHAIN(srs, mp, tail, cnt, sz); \
1339 }
1340
1341 /* Some utility macros */
1342 #define MAC_SRS_BW_LOCK(srs) \
1343 if (!(srs->srs_type & SRST_TX)) \
1344 mutex_enter(&srs->srs_bw->mac_bw_lock);
1345
1346 #define MAC_SRS_BW_UNLOCK(srs) \
1347 if (!(srs->srs_type & SRST_TX)) \
1348 mutex_exit(&srs->srs_bw->mac_bw_lock);
1349
1350 #define MAC_TX_SRS_DROP_MESSAGE(srs, mp, cookie) { \
1351 mac_pkt_drop(NULL, NULL, mp, B_FALSE); \
1352 /* increment freed stats */ \
1353 mac_srs->srs_tx.st_stat.mts_sdrops++; \
1354 cookie = (mac_tx_cookie_t)srs; \
1355 }
1356
1357 #define MAC_TX_SET_NO_ENQUEUE(srs, mp_chain, ret_mp, cookie) { \
1358 mac_srs->srs_state |= SRS_TX_WAKEUP_CLIENT; \
1359 cookie = (mac_tx_cookie_t)srs; \
1360 *ret_mp = mp_chain; \
1361 }
1362
1363 /*
1364 * MAC_RX_SRS_TOODEEP
1365 *
1366 * Macro called as part of receive-side processing to determine if handling
1367 * can occur in situ (in the interrupt thread) or if it should be left to a
1368 * worker thread. Note that the constant used to make this determination is
1369 * not entirely made-up, and is a result of some emprical validation. That
1370 * said, the constant is left as a static variable to allow it to be
1371 * dynamically tuned in the field if and as needed.
1372 */
1373 static uintptr_t mac_rx_srs_stack_needed = 10240;
1374 static uint_t mac_rx_srs_stack_toodeep;
1375
1376 #ifndef STACK_GROWTH_DOWN
1377 #error Downward stack growth assumed.
1378 #endif
1379
1380 #define MAC_RX_SRS_TOODEEP() (STACK_BIAS + (uintptr_t)getfp() - \
1381 (uintptr_t)curthread->t_stkbase < mac_rx_srs_stack_needed && \
1382 ++mac_rx_srs_stack_toodeep)
1383
1384
1385 /*
1386 * Drop the rx packet and advance to the next one in the chain.
1387 */
1388 static void
1389 mac_rx_drop_pkt(mac_soft_ring_set_t *srs, mblk_t *mp)
1390 {
1391 mac_srs_rx_t *srs_rx = &srs->srs_rx;
1392
1393 ASSERT(mp->b_next == NULL);
1394 mutex_enter(&srs->srs_lock);
1395 MAC_UPDATE_SRS_COUNT_LOCKED(srs, 1);
1396 MAC_UPDATE_SRS_SIZE_LOCKED(srs, msgdsize(mp));
1397 mutex_exit(&srs->srs_lock);
1398
1399 srs_rx->sr_stat.mrs_sdrops++;
1400 freemsg(mp);
1401 }
1402
1403 /* DATAPATH RUNTIME ROUTINES */
1404
1405 /*
1406 * mac_srs_fire
1407 *
1408 * Timer callback routine for waking up the SRS worker thread.
1409 */
1410 static void
1411 mac_srs_fire(void *arg)
1412 {
1413 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)arg;
1414
1415 mutex_enter(&mac_srs->srs_lock);
1416 if (mac_srs->srs_tid == NULL) {
1417 mutex_exit(&mac_srs->srs_lock);
1418 return;
1419 }
1420
1421 mac_srs->srs_tid = NULL;
1422 if (!(mac_srs->srs_state & SRS_PROC))
1423 cv_signal(&mac_srs->srs_async);
1424
1425 mutex_exit(&mac_srs->srs_lock);
1426 }
1427
1428 /*
1429 * 'hint' is fanout_hint (type of uint64_t) which is given by the TCP/IP stack,
1430 * and it is used on the TX path.
1431 */
1432 #define HASH_HINT(hint) \
1433 ((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8))
1434
1435
1436 /*
1437 * hash based on the src address, dst address and the port information.
1438 */
1439 #define HASH_ADDR(src, dst, ports) \
1440 (ntohl((src) + (dst)) ^ ((ports) >> 24) ^ ((ports) >> 16) ^ \
1441 ((ports) >> 8) ^ (ports))
1442
1443 #define COMPUTE_INDEX(key, sz) (key % sz)
1444
1445 #define FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) { \
1446 if ((tail) != NULL) { \
1447 ASSERT((tail)->b_next == NULL); \
1448 (tail)->b_next = (mp); \
1449 } else { \
1450 ASSERT((head) == NULL); \
1451 (head) = (mp); \
1452 } \
1453 (tail) = (mp); \
1454 (cnt)++; \
1455 if ((bw_ctl)) \
1456 (sz) += (sz0); \
1457 }
1458
1459 #define MAC_FANOUT_DEFAULT 0
1460 #define MAC_FANOUT_RND_ROBIN 1
1461 int mac_fanout_type = MAC_FANOUT_DEFAULT;
1462
1463 #define MAX_SR_TYPES 3
1464 /* fanout types for port based hashing */
1465 enum pkt_type {
1466 V4_TCP = 0,
1467 V4_UDP,
1468 OTH,
1469 UNDEF
1470 };
1471
1472 /*
1473 * Pair of local and remote ports in the transport header
1474 */
1475 #define PORTS_SIZE 4
1476
1477 /*
1478 * mac_rx_srs_proto_fanout
1479 *
1480 * This routine delivers packets destined to an SRS into one of the
1481 * protocol soft rings.
1482 *
1483 * Given a chain of packets we need to split it up into multiple sub chains
1484 * destined into TCP, UDP or OTH soft ring. Instead of entering
1485 * the soft ring one packet at a time, we want to enter it in the form of a
1486 * chain otherwise we get this start/stop behaviour where the worker thread
1487 * goes to sleep and then next packets comes in forcing it to wake up etc.
1488 */
1489 static void
1490 mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
1491 {
1492 struct ether_header *ehp;
1493 struct ether_vlan_header *evhp;
1494 uint32_t sap;
1495 ipha_t *ipha;
1496 uint8_t *dstaddr;
1497 size_t hdrsize;
1498 mblk_t *mp;
1499 mblk_t *headmp[MAX_SR_TYPES];
1500 mblk_t *tailmp[MAX_SR_TYPES];
1501 int cnt[MAX_SR_TYPES];
1502 size_t sz[MAX_SR_TYPES];
1503 size_t sz1;
1504 boolean_t bw_ctl;
1505 boolean_t hw_classified;
1506 boolean_t dls_bypass;
1507 boolean_t is_ether;
1508 boolean_t is_unicast;
1509 enum pkt_type type;
1510 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1511
1512 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
1513 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
1514
1515 /*
1516 * If we don't have a Rx ring, S/W classification would have done
1517 * its job and its a packet meant for us. If we were polling on
1518 * the default ring (i.e. there was a ring assigned to this SRS),
1519 * then we need to make sure that the mac address really belongs
1520 * to us.
1521 */
1522 hw_classified = mac_srs->srs_ring != NULL &&
1523 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
1524
1525 /*
1526 * Special clients (eg. VLAN, non ether, etc) need DLS
1527 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
1528 * such SRSs. Another way of disabling bypass is to set the
1529 * MCIS_RX_BYPASS_DISABLE flag.
1530 */
1531 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
1532 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
1533
1534 bzero(headmp, MAX_SR_TYPES * sizeof (mblk_t *));
1535 bzero(tailmp, MAX_SR_TYPES * sizeof (mblk_t *));
1536 bzero(cnt, MAX_SR_TYPES * sizeof (int));
1537 bzero(sz, MAX_SR_TYPES * sizeof (size_t));
1538
1539 /*
1540 * We got a chain from SRS that we need to send to the soft rings.
1541 * Since squeues for TCP & IPv4 sap poll their soft rings (for
1542 * performance reasons), we need to separate out v4_tcp, v4_udp
1543 * and the rest goes in other.
1544 */
1545 while (head != NULL) {
1546 mp = head;
1547 head = head->b_next;
1548 mp->b_next = NULL;
1549
1550 type = OTH;
1551 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1552
1553 if (is_ether) {
1554 /*
1555 * At this point we can be sure the packet at least
1556 * has an ether header.
1557 */
1558 if (sz1 < sizeof (struct ether_header)) {
1559 mac_rx_drop_pkt(mac_srs, mp);
1560 continue;
1561 }
1562 ehp = (struct ether_header *)mp->b_rptr;
1563
1564 /*
1565 * Determine if this is a VLAN or non-VLAN packet.
1566 */
1567 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
1568 evhp = (struct ether_vlan_header *)mp->b_rptr;
1569 sap = ntohs(evhp->ether_type);
1570 hdrsize = sizeof (struct ether_vlan_header);
1571 /*
1572 * Check if the VID of the packet, if any,
1573 * belongs to this client.
1574 */
1575 if (!mac_client_check_flow_vid(mcip,
1576 VLAN_ID(ntohs(evhp->ether_tci)))) {
1577 mac_rx_drop_pkt(mac_srs, mp);
1578 continue;
1579 }
1580 } else {
1581 hdrsize = sizeof (struct ether_header);
1582 }
1583 is_unicast =
1584 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
1585 dstaddr = (uint8_t *)&ehp->ether_dhost;
1586 } else {
1587 mac_header_info_t mhi;
1588
1589 if (mac_header_info((mac_handle_t)mcip->mci_mip,
1590 mp, &mhi) != 0) {
1591 mac_rx_drop_pkt(mac_srs, mp);
1592 continue;
1593 }
1594 hdrsize = mhi.mhi_hdrsize;
1595 sap = mhi.mhi_bindsap;
1596 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
1597 dstaddr = (uint8_t *)mhi.mhi_daddr;
1598 }
1599
1600 if (!dls_bypass) {
1601 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
1602 cnt[type], bw_ctl, sz[type], sz1, mp);
1603 continue;
1604 }
1605
1606 if (sap == ETHERTYPE_IP) {
1607 /*
1608 * If we are H/W classified, but we have promisc
1609 * on, then we need to check for the unicast address.
1610 */
1611 if (hw_classified && mcip->mci_promisc_list != NULL) {
1612 mac_address_t *map;
1613
1614 rw_enter(&mcip->mci_rw_lock, RW_READER);
1615 map = mcip->mci_unicast;
1616 if (bcmp(dstaddr, map->ma_addr,
1617 map->ma_len) == 0)
1618 type = UNDEF;
1619 rw_exit(&mcip->mci_rw_lock);
1620 } else if (is_unicast) {
1621 type = UNDEF;
1622 }
1623 }
1624
1625 /*
1626 * This needs to become a contract with the driver for
1627 * the fast path.
1628 *
1629 * In the normal case the packet will have at least the L2
1630 * header and the IP + Transport header in the same mblk.
1631 * This is usually the case when the NIC driver sends up
1632 * the packet. This is also true when the stack generates
1633 * a packet that is looped back and when the stack uses the
1634 * fastpath mechanism. The normal case is optimized for
1635 * performance and may bypass DLS. All other cases go through
1636 * the 'OTH' type path without DLS bypass.
1637 */
1638
1639 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
1640 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha))
1641 type = OTH;
1642
1643 if (type == OTH) {
1644 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
1645 cnt[type], bw_ctl, sz[type], sz1, mp);
1646 continue;
1647 }
1648
1649 ASSERT(type == UNDEF);
1650 /*
1651 * We look for at least 4 bytes past the IP header to get
1652 * the port information. If we get an IP fragment, we don't
1653 * have the port information, and we use just the protocol
1654 * information.
1655 */
1656 switch (ipha->ipha_protocol) {
1657 case IPPROTO_TCP:
1658 type = V4_TCP;
1659 mp->b_rptr += hdrsize;
1660 break;
1661 case IPPROTO_UDP:
1662 type = V4_UDP;
1663 mp->b_rptr += hdrsize;
1664 break;
1665 default:
1666 type = OTH;
1667 break;
1668 }
1669
1670 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type],
1671 bw_ctl, sz[type], sz1, mp);
1672 }
1673
1674 for (type = V4_TCP; type < UNDEF; type++) {
1675 if (headmp[type] != NULL) {
1676 mac_soft_ring_t *softring;
1677
1678 ASSERT(tailmp[type]->b_next == NULL);
1679 switch (type) {
1680 case V4_TCP:
1681 softring = mac_srs->srs_tcp_soft_rings[0];
1682 break;
1683 case V4_UDP:
1684 softring = mac_srs->srs_udp_soft_rings[0];
1685 break;
1686 case OTH:
1687 softring = mac_srs->srs_oth_soft_rings[0];
1688 }
1689 mac_rx_soft_ring_process(mcip, softring,
1690 headmp[type], tailmp[type], cnt[type], sz[type]);
1691 }
1692 }
1693 }
1694
1695 int fanout_unaligned = 0;
1696
1697 /*
1698 * mac_rx_srs_long_fanout
1699 *
1700 * The fanout routine for VLANs, and for anything else that isn't performing
1701 * explicit dls bypass. Returns -1 on an error (drop the packet due to a
1702 * malformed packet), 0 on success, with values written in *indx and *type.
1703 */
1704 static int
1705 mac_rx_srs_long_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *mp,
1706 uint32_t sap, size_t hdrsize, enum pkt_type *type, uint_t *indx)
1707 {
1708 ip6_t *ip6h;
1709 ipha_t *ipha;
1710 uint8_t *whereptr;
1711 uint_t hash;
1712 uint16_t remlen;
1713 uint8_t nexthdr;
1714 uint16_t hdr_len;
1715 uint32_t src_val, dst_val;
1716 boolean_t modifiable = B_TRUE;
1717 boolean_t v6;
1718 int errno;
1719
1720 ASSERT(MBLKL(mp) >= hdrsize);
1721
1722 if (sap == ETHERTYPE_IPV6) {
1723 v6 = B_TRUE;
1724 hdr_len = IPV6_HDR_LEN;
1725 } else if (sap == ETHERTYPE_IP) {
1726 v6 = B_FALSE;
1727 hdr_len = IP_SIMPLE_HDR_LENGTH;
1728 } else {
1729 *indx = 0;
1730 *type = OTH;
1731 return (0);
1732 }
1733
1734 ip6h = (ip6_t *)(mp->b_rptr + hdrsize);
1735 ipha = (ipha_t *)ip6h;
1736
1737 if ((uint8_t *)ip6h == mp->b_wptr) {
1738 /*
1739 * The first mblk_t only includes the mac header.
1740 * Note that it is safe to change the mp pointer here,
1741 * as the subsequent operation does not assume mp
1742 * points to the start of the mac header.
1743 */
1744 mp = mp->b_cont;
1745
1746 /*
1747 * Make sure the IP header points to an entire one.
1748 */
1749 if (mp == NULL)
1750 return (-1);
1751
1752 if (MBLKL(mp) < hdr_len) {
1753 modifiable = (DB_REF(mp) == 1);
1754
1755 if (modifiable && !pullupmsg(mp, hdr_len))
1756 return (-1);
1757 }
1758
1759 ip6h = (ip6_t *)mp->b_rptr;
1760 ipha = (ipha_t *)ip6h;
1761 }
1762
1763 if (!modifiable || !(OK_32PTR((char *)ip6h)) ||
1764 ((uint8_t *)ip6h + hdr_len > mp->b_wptr)) {
1765 /*
1766 * If either the IP header is not aligned, or it does not hold
1767 * the complete simple structure (a pullupmsg() is not an
1768 * option since it would result in an unaligned IP header),
1769 * fanout to the default ring.
1770 *
1771 * Note that this may cause packet reordering.
1772 */
1773 *indx = 0;
1774 *type = OTH;
1775 fanout_unaligned++;
1776 return (0);
1777 }
1778
1779 /*
1780 * Extract next-header, full header length, and source-hash value
1781 * using v4/v6 specific fields.
1782 */
1783 if (v6) {
1784 remlen = ntohs(ip6h->ip6_plen);
1785 nexthdr = ip6h->ip6_nxt;
1786 src_val = V4_PART_OF_V6(ip6h->ip6_src);
1787 dst_val = V4_PART_OF_V6(ip6h->ip6_dst);
1788 /*
1789 * Do src based fanout if below tunable is set to B_TRUE or
1790 * when mac_ip_hdr_length_v6() fails because of malformed
1791 * packets or because mblks need to be concatenated using
1792 * pullupmsg().
1793 */
1794 errno = mac_ip_hdr_length_v6(ip6h, mp->b_wptr, &hdr_len,
1795 &nexthdr, NULL);
1796 switch (errno) {
1797 case EINVAL:
1798 /* Bad version. */
1799 *indx = 0;
1800 *type = OTH;
1801 return (0);
1802 case 0:
1803 break;
1804 default:
1805 goto src_dst_based_fanout;
1806 }
1807 } else {
1808 if (IPH_HDR_VERSION(ipha) != IPV4_VERSION) {
1809 /* Bad version. */
1810 *indx = 0;
1811 *type = OTH;
1812 return (0);
1813 }
1814 hdr_len = IPH_HDR_LENGTH(ipha);
1815 remlen = ntohs(ipha->ipha_length) - hdr_len;
1816 nexthdr = ipha->ipha_protocol;
1817 src_val = (uint32_t)ipha->ipha_src;
1818 dst_val = (uint32_t)ipha->ipha_dst;
1819 /*
1820 * Catch IPv4 fragment case here. IPv6 has nexthdr == FRAG
1821 * for its equivalent case.
1822 */
1823 if ((ntohs(ipha->ipha_fragment_offset_and_flags) &
1824 (IPH_MF | IPH_OFFSET)) != 0) {
1825 goto src_dst_based_fanout;
1826 }
1827 }
1828 if (remlen < MIN_EHDR_LEN)
1829 return (-1);
1830 whereptr = (uint8_t *)ip6h + hdr_len;
1831
1832 /* If the transport is one of below, we do port/SPI based fanout */
1833 switch (nexthdr) {
1834 case IPPROTO_TCP:
1835 case IPPROTO_UDP:
1836 case IPPROTO_SCTP:
1837 case IPPROTO_ESP:
1838 /*
1839 * If the ports or SPI in the transport header is not part of
1840 * the mblk, do src_based_fanout, instead of calling
1841 * pullupmsg().
1842 */
1843 if (mp->b_cont == NULL || whereptr + PORTS_SIZE <= mp->b_wptr)
1844 break; /* out of switch... */
1845 /* FALLTHRU */
1846 default:
1847 goto src_dst_based_fanout;
1848 }
1849
1850 switch (nexthdr) {
1851 case IPPROTO_TCP:
1852 hash = HASH_ADDR(src_val, dst_val, *(uint32_t *)whereptr);
1853 *indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
1854 *type = OTH;
1855 break;
1856 case IPPROTO_UDP:
1857 case IPPROTO_SCTP:
1858 case IPPROTO_ESP:
1859 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
1860 hash = HASH_ADDR(src_val, dst_val,
1861 *(uint32_t *)whereptr);
1862 *indx = COMPUTE_INDEX(hash,
1863 mac_srs->srs_udp_ring_count);
1864 } else {
1865 *indx = mac_srs->srs_ind % mac_srs->srs_udp_ring_count;
1866 mac_srs->srs_ind++;
1867 }
1868 *type = OTH;
1869 break;
1870 }
1871 return (0);
1872
1873 src_dst_based_fanout:
1874 hash = HASH_ADDR(src_val, dst_val, (uint32_t)0);
1875 *indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count);
1876 *type = OTH;
1877 return (0);
1878 }
1879
1880 /*
1881 * mac_rx_srs_fanout
1882 *
1883 * This routine delivers packets destined to an SRS into a soft ring member
1884 * of the set.
1885 *
1886 * Given a chain of packets we need to split it up into multiple sub chains
1887 * destined for one of the TCP, UDP or OTH soft rings. Instead of entering
1888 * the soft ring one packet at a time, we want to enter it in the form of a
1889 * chain otherwise we get this start/stop behaviour where the worker thread
1890 * goes to sleep and then next packets comes in forcing it to wake up etc.
1891 *
1892 * Note:
1893 * Since we know what is the maximum fanout possible, we create a 2D array
1894 * of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz
1895 * variables so that we can enter the softrings with chain. We need the
1896 * MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc
1897 * for each packet would be expensive). If we ever want to have the
1898 * ability to have unlimited fanout, we should probably declare a head,
1899 * tail, cnt, sz with each soft ring (a data struct which contains a softring
1900 * along with these members) and create an array of this uber struct so we
1901 * don't have to do kmem_alloc.
1902 */
1903 int fanout_oth1 = 0;
1904 int fanout_oth2 = 0;
1905 int fanout_oth3 = 0;
1906 int fanout_oth4 = 0;
1907 int fanout_oth5 = 0;
1908
1909 static void
1910 mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
1911 {
1912 struct ether_header *ehp;
1913 struct ether_vlan_header *evhp;
1914 uint32_t sap;
1915 ipha_t *ipha;
1916 uint8_t *dstaddr;
1917 uint_t indx;
1918 size_t ports_offset;
1919 size_t ipha_len;
1920 size_t hdrsize;
1921 uint_t hash;
1922 mblk_t *mp;
1923 mblk_t *headmp[MAX_SR_TYPES][MAX_SR_FANOUT];
1924 mblk_t *tailmp[MAX_SR_TYPES][MAX_SR_FANOUT];
1925 int cnt[MAX_SR_TYPES][MAX_SR_FANOUT];
1926 size_t sz[MAX_SR_TYPES][MAX_SR_FANOUT];
1927 size_t sz1;
1928 boolean_t bw_ctl;
1929 boolean_t hw_classified;
1930 boolean_t dls_bypass;
1931 boolean_t is_ether;
1932 boolean_t is_unicast;
1933 int fanout_cnt;
1934 enum pkt_type type;
1935 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1936
1937 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
1938 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
1939
1940 /*
1941 * If we don't have a Rx ring, S/W classification would have done
1942 * its job and its a packet meant for us. If we were polling on
1943 * the default ring (i.e. there was a ring assigned to this SRS),
1944 * then we need to make sure that the mac address really belongs
1945 * to us.
1946 */
1947 hw_classified = mac_srs->srs_ring != NULL &&
1948 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
1949
1950 /*
1951 * Special clients (eg. VLAN, non ether, etc) need DLS
1952 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
1953 * such SRSs. Another way of disabling bypass is to set the
1954 * MCIS_RX_BYPASS_DISABLE flag.
1955 */
1956 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
1957 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
1958
1959 /*
1960 * Since the softrings are never destroyed and we always
1961 * create equal number of softrings for TCP, UDP and rest,
1962 * its OK to check one of them for count and use it without
1963 * any lock. In future, if soft rings get destroyed because
1964 * of reduction in fanout, we will need to ensure that happens
1965 * behind the SRS_PROC.
1966 */
1967 fanout_cnt = mac_srs->srs_tcp_ring_count;
1968
1969 bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
1970 bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
1971 bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int));
1972 bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t));
1973
1974 /*
1975 * We got a chain from SRS that we need to send to the soft rings.
1976 * Since squeues for TCP & IPv4 sap poll their soft rings (for
1977 * performance reasons), we need to separate out v4_tcp, v4_udp
1978 * and the rest goes in other.
1979 */
1980 while (head != NULL) {
1981 mp = head;
1982 head = head->b_next;
1983 mp->b_next = NULL;
1984
1985 type = OTH;
1986 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1987
1988 if (is_ether) {
1989 /*
1990 * At this point we can be sure the packet at least
1991 * has an ether header.
1992 */
1993 if (sz1 < sizeof (struct ether_header)) {
1994 mac_rx_drop_pkt(mac_srs, mp);
1995 continue;
1996 }
1997 ehp = (struct ether_header *)mp->b_rptr;
1998
1999 /*
2000 * Determine if this is a VLAN or non-VLAN packet.
2001 */
2002 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
2003 evhp = (struct ether_vlan_header *)mp->b_rptr;
2004 sap = ntohs(evhp->ether_type);
2005 hdrsize = sizeof (struct ether_vlan_header);
2006 /*
2007 * Check if the VID of the packet, if any,
2008 * belongs to this client.
2009 */
2010 if (!mac_client_check_flow_vid(mcip,
2011 VLAN_ID(ntohs(evhp->ether_tci)))) {
2012 mac_rx_drop_pkt(mac_srs, mp);
2013 continue;
2014 }
2015 } else {
2016 hdrsize = sizeof (struct ether_header);
2017 }
2018 is_unicast =
2019 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
2020 dstaddr = (uint8_t *)&ehp->ether_dhost;
2021 } else {
2022 mac_header_info_t mhi;
2023
2024 if (mac_header_info((mac_handle_t)mcip->mci_mip,
2025 mp, &mhi) != 0) {
2026 mac_rx_drop_pkt(mac_srs, mp);
2027 continue;
2028 }
2029 hdrsize = mhi.mhi_hdrsize;
2030 sap = mhi.mhi_bindsap;
2031 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
2032 dstaddr = (uint8_t *)mhi.mhi_daddr;
2033 }
2034
2035 if (!dls_bypass) {
2036 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
2037 hdrsize, &type, &indx) == -1) {
2038 mac_rx_drop_pkt(mac_srs, mp);
2039 continue;
2040 }
2041
2042 FANOUT_ENQUEUE_MP(headmp[type][indx],
2043 tailmp[type][indx], cnt[type][indx], bw_ctl,
2044 sz[type][indx], sz1, mp);
2045 continue;
2046 }
2047
2048
2049 /*
2050 * If we are using the default Rx ring where H/W or S/W
2051 * classification has not happened, we need to verify if
2052 * this unicast packet really belongs to us.
2053 */
2054 if (sap == ETHERTYPE_IP) {
2055 /*
2056 * If we are H/W classified, but we have promisc
2057 * on, then we need to check for the unicast address.
2058 */
2059 if (hw_classified && mcip->mci_promisc_list != NULL) {
2060 mac_address_t *map;
2061
2062 rw_enter(&mcip->mci_rw_lock, RW_READER);
2063 map = mcip->mci_unicast;
2064 if (bcmp(dstaddr, map->ma_addr,
2065 map->ma_len) == 0)
2066 type = UNDEF;
2067 rw_exit(&mcip->mci_rw_lock);
2068 } else if (is_unicast) {
2069 type = UNDEF;
2070 }
2071 }
2072
2073 /*
2074 * This needs to become a contract with the driver for
2075 * the fast path.
2076 */
2077
2078 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
2079 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) {
2080 type = OTH;
2081 fanout_oth1++;
2082 }
2083
2084 if (type != OTH) {
2085 uint16_t frag_offset_flags;
2086
2087 switch (ipha->ipha_protocol) {
2088 case IPPROTO_TCP:
2089 case IPPROTO_UDP:
2090 case IPPROTO_SCTP:
2091 case IPPROTO_ESP:
2092 ipha_len = IPH_HDR_LENGTH(ipha);
2093 if ((uchar_t *)ipha + ipha_len + PORTS_SIZE >
2094 mp->b_wptr) {
2095 type = OTH;
2096 break;
2097 }
2098 frag_offset_flags =
2099 ntohs(ipha->ipha_fragment_offset_and_flags);
2100 if ((frag_offset_flags &
2101 (IPH_MF | IPH_OFFSET)) != 0) {
2102 type = OTH;
2103 fanout_oth3++;
2104 break;
2105 }
2106 ports_offset = hdrsize + ipha_len;
2107 break;
2108 default:
2109 type = OTH;
2110 fanout_oth4++;
2111 break;
2112 }
2113 }
2114
2115 if (type == OTH) {
2116 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
2117 hdrsize, &type, &indx) == -1) {
2118 mac_rx_drop_pkt(mac_srs, mp);
2119 continue;
2120 }
2121
2122 FANOUT_ENQUEUE_MP(headmp[type][indx],
2123 tailmp[type][indx], cnt[type][indx], bw_ctl,
2124 sz[type][indx], sz1, mp);
2125 continue;
2126 }
2127
2128 ASSERT(type == UNDEF);
2129
2130 /*
2131 * XXX-Sunay: We should hold srs_lock since ring_count
2132 * below can change. But if we are always called from
2133 * mac_rx_srs_drain and SRS_PROC is set, then we can
2134 * enforce that ring_count can't be changed i.e.
2135 * to change fanout type or ring count, the calling
2136 * thread needs to be behind SRS_PROC.
2137 */
2138 switch (ipha->ipha_protocol) {
2139 case IPPROTO_TCP:
2140 /*
2141 * Note that for ESP, we fanout on SPI and it is at the
2142 * same offset as the 2x16-bit ports. So it is clumped
2143 * along with TCP, UDP and SCTP.
2144 */
2145 hash = HASH_ADDR(ipha->ipha_src, ipha->ipha_dst,
2146 *(uint32_t *)(mp->b_rptr + ports_offset));
2147 indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
2148 type = V4_TCP;
2149 mp->b_rptr += hdrsize;
2150 break;
2151 case IPPROTO_UDP:
2152 case IPPROTO_SCTP:
2153 case IPPROTO_ESP:
2154 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
2155 hash = HASH_ADDR(ipha->ipha_src, ipha->ipha_dst,
2156 *(uint32_t *)(mp->b_rptr + ports_offset));
2157 indx = COMPUTE_INDEX(hash,
2158 mac_srs->srs_udp_ring_count);
2159 } else {
2160 indx = mac_srs->srs_ind %
2161 mac_srs->srs_udp_ring_count;
2162 mac_srs->srs_ind++;
2163 }
2164 type = V4_UDP;
2165 mp->b_rptr += hdrsize;
2166 break;
2167 default:
2168 indx = 0;
2169 type = OTH;
2170 }
2171
2172 FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx],
2173 cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp);
2174 }
2175
2176 for (type = V4_TCP; type < UNDEF; type++) {
2177 int i;
2178
2179 for (i = 0; i < fanout_cnt; i++) {
2180 if (headmp[type][i] != NULL) {
2181 mac_soft_ring_t *softring;
2182
2183 ASSERT(tailmp[type][i]->b_next == NULL);
2184 switch (type) {
2185 case V4_TCP:
2186 softring =
2187 mac_srs->srs_tcp_soft_rings[i];
2188 break;
2189 case V4_UDP:
2190 softring =
2191 mac_srs->srs_udp_soft_rings[i];
2192 break;
2193 case OTH:
2194 softring =
2195 mac_srs->srs_oth_soft_rings[i];
2196 break;
2197 }
2198 mac_rx_soft_ring_process(mcip,
2199 softring, headmp[type][i], tailmp[type][i],
2200 cnt[type][i], sz[type][i]);
2201 }
2202 }
2203 }
2204 }
2205
2206 #define SRS_BYTES_TO_PICKUP 150000
2207 ssize_t max_bytes_to_pickup = SRS_BYTES_TO_PICKUP;
2208
2209 /*
2210 * mac_rx_srs_poll_ring
2211 *
2212 * This SRS Poll thread uses this routine to poll the underlying hardware
2213 * Rx ring to get a chain of packets. It can inline process that chain
2214 * if mac_latency_optimize is set (default) or signal the SRS worker thread
2215 * to do the remaining processing.
2216 *
2217 * Since packets come in the system via interrupt or poll path, we also
2218 * update the stats and deal with promiscous clients here.
2219 */
2220 void
2221 mac_rx_srs_poll_ring(mac_soft_ring_set_t *mac_srs)
2222 {
2223 kmutex_t *lock = &mac_srs->srs_lock;
2224 kcondvar_t *async = &mac_srs->srs_cv;
2225 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2226 mblk_t *head, *tail, *mp;
2227 callb_cpr_t cprinfo;
2228 ssize_t bytes_to_pickup;
2229 size_t sz;
2230 int count;
2231 mac_client_impl_t *smcip;
2232
2233 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_srs_poll");
2234 mutex_enter(lock);
2235
2236 start:
2237 for (;;) {
2238 if (mac_srs->srs_state & SRS_PAUSE)
2239 goto done;
2240
2241 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2242 cv_wait(async, lock);
2243 CALLB_CPR_SAFE_END(&cprinfo, lock);
2244
2245 if (mac_srs->srs_state & SRS_PAUSE)
2246 goto done;
2247
2248 check_again:
2249 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2250 /*
2251 * We pick as many bytes as we are allowed to queue.
2252 * Its possible that we will exceed the total
2253 * packets queued in case this SRS is part of the
2254 * Rx ring group since > 1 poll thread can be pulling
2255 * upto the max allowed packets at the same time
2256 * but that should be OK.
2257 */
2258 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2259 bytes_to_pickup =
2260 mac_srs->srs_bw->mac_bw_drop_threshold -
2261 mac_srs->srs_bw->mac_bw_sz;
2262 /*
2263 * We shouldn't have been signalled if we
2264 * have 0 or less bytes to pick but since
2265 * some of the bytes accounting is driver
2266 * dependant, we do the safety check.
2267 */
2268 if (bytes_to_pickup < 0)
2269 bytes_to_pickup = 0;
2270 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2271 } else {
2272 /*
2273 * ToDO: Need to change the polling API
2274 * to add a packet count and a flag which
2275 * tells the driver whether we want packets
2276 * based on a count, or bytes, or all the
2277 * packets queued in the driver/HW. This
2278 * way, we never have to check the limits
2279 * on poll path. We truly let only as many
2280 * packets enter the system as we are willing
2281 * to process or queue.
2282 *
2283 * Something along the lines of
2284 * pkts_to_pickup = mac_soft_ring_max_q_cnt -
2285 * mac_srs->srs_poll_pkt_cnt
2286 */
2287
2288 /*
2289 * Since we are not doing B/W control, pick
2290 * as many packets as allowed.
2291 */
2292 bytes_to_pickup = max_bytes_to_pickup;
2293 }
2294
2295 /* Poll the underlying Hardware */
2296 mutex_exit(lock);
2297 head = MAC_HWRING_POLL(mac_srs->srs_ring, (int)bytes_to_pickup);
2298 mutex_enter(lock);
2299
2300 ASSERT((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
2301 SRS_POLL_THR_OWNER);
2302
2303 mp = tail = head;
2304 count = 0;
2305 sz = 0;
2306 while (mp != NULL) {
2307 tail = mp;
2308 sz += msgdsize(mp);
2309 mp = mp->b_next;
2310 count++;
2311 }
2312
2313 if (head != NULL) {
2314 tail->b_next = NULL;
2315 smcip = mac_srs->srs_mcip;
2316
2317 SRS_RX_STAT_UPDATE(mac_srs, pollbytes, sz);
2318 SRS_RX_STAT_UPDATE(mac_srs, pollcnt, count);
2319
2320 /*
2321 * If there are any promiscuous mode callbacks
2322 * defined for this MAC client, pass them a copy
2323 * if appropriate and also update the counters.
2324 */
2325 if (smcip != NULL) {
2326 if (smcip->mci_mip->mi_promisc_list != NULL) {
2327 mutex_exit(lock);
2328 mac_promisc_dispatch(smcip->mci_mip,
2329 head, NULL);
2330 mutex_enter(lock);
2331 }
2332 }
2333 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2334 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2335 mac_srs->srs_bw->mac_bw_polled += sz;
2336 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2337 }
2338 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail,
2339 count, sz);
2340 if (count <= 10)
2341 srs_rx->sr_stat.mrs_chaincntundr10++;
2342 else if (count > 10 && count <= 50)
2343 srs_rx->sr_stat.mrs_chaincnt10to50++;
2344 else
2345 srs_rx->sr_stat.mrs_chaincntover50++;
2346 }
2347
2348 /*
2349 * We are guaranteed that SRS_PROC will be set if we
2350 * are here. Also, poll thread gets to run only if
2351 * the drain was being done by a worker thread although
2352 * its possible that worker thread is still running
2353 * and poll thread was sent down to keep the pipeline
2354 * going instead of doing a complete drain and then
2355 * trying to poll the NIC.
2356 *
2357 * So we need to check SRS_WORKER flag to make sure
2358 * that the worker thread is not processing the queue
2359 * in parallel to us. The flags and conditions are
2360 * protected by the srs_lock to prevent any race. We
2361 * ensure that we don't drop the srs_lock from now
2362 * till the end and similarly we don't drop the srs_lock
2363 * in mac_rx_srs_drain() till similar condition check
2364 * are complete. The mac_rx_srs_drain() needs to ensure
2365 * that SRS_WORKER flag remains set as long as its
2366 * processing the queue.
2367 */
2368 if (!(mac_srs->srs_state & SRS_WORKER) &&
2369 (mac_srs->srs_first != NULL)) {
2370 /*
2371 * We have packets to process and worker thread
2372 * is not running. Check to see if poll thread is
2373 * allowed to process.
2374 */
2375 if (mac_srs->srs_state & SRS_LATENCY_OPT) {
2376 mac_srs->srs_drain_func(mac_srs, SRS_POLL_PROC);
2377 if (!(mac_srs->srs_state & SRS_PAUSE) &&
2378 srs_rx->sr_poll_pkt_cnt <=
2379 srs_rx->sr_lowat) {
2380 srs_rx->sr_poll_again++;
2381 goto check_again;
2382 }
2383 /*
2384 * We are already above low water mark
2385 * so stay in the polling mode but no
2386 * need to poll. Once we dip below
2387 * the polling threshold, the processing
2388 * thread (soft ring) will signal us
2389 * to poll again (MAC_UPDATE_SRS_COUNT)
2390 */
2391 srs_rx->sr_poll_drain_no_poll++;
2392 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
2393 /*
2394 * In B/W control case, its possible
2395 * that the backlog built up due to
2396 * B/W limit being reached and packets
2397 * are queued only in SRS. In this case,
2398 * we should schedule worker thread
2399 * since no one else will wake us up.
2400 */
2401 if ((mac_srs->srs_type & SRST_BW_CONTROL) &&
2402 (mac_srs->srs_tid == NULL)) {
2403 mac_srs->srs_tid =
2404 timeout(mac_srs_fire, mac_srs, 1);
2405 srs_rx->sr_poll_worker_wakeup++;
2406 }
2407 } else {
2408 /*
2409 * Wakeup the worker thread for more processing.
2410 * We optimize for throughput in this case.
2411 */
2412 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
2413 MAC_SRS_WORKER_WAKEUP(mac_srs);
2414 srs_rx->sr_poll_sig_worker++;
2415 }
2416 } else if ((mac_srs->srs_first == NULL) &&
2417 !(mac_srs->srs_state & SRS_WORKER)) {
2418 /*
2419 * There is nothing queued in SRS and
2420 * no worker thread running. Plus we
2421 * didn't get anything from the H/W
2422 * as well (head == NULL);
2423 */
2424 ASSERT(head == NULL);
2425 mac_srs->srs_state &=
2426 ~(SRS_PROC|SRS_GET_PKTS);
2427
2428 /*
2429 * If we have a packets in soft ring, don't allow
2430 * more packets to come into this SRS by keeping the
2431 * interrupts off but not polling the H/W. The
2432 * poll thread will get signaled as soon as
2433 * srs_poll_pkt_cnt dips below poll threshold.
2434 */
2435 if (srs_rx->sr_poll_pkt_cnt == 0) {
2436 srs_rx->sr_poll_intr_enable++;
2437 MAC_SRS_POLLING_OFF(mac_srs);
2438 } else {
2439 /*
2440 * We know nothing is queued in SRS
2441 * since we are here after checking
2442 * srs_first is NULL. The backlog
2443 * is entirely due to packets queued
2444 * in Soft ring which will wake us up
2445 * and get the interface out of polling
2446 * mode once the backlog dips below
2447 * sr_poll_thres.
2448 */
2449 srs_rx->sr_poll_no_poll++;
2450 }
2451 } else {
2452 /*
2453 * Worker thread is already running.
2454 * Nothing much to do. If the polling
2455 * was enabled, worker thread will deal
2456 * with that.
2457 */
2458 mac_srs->srs_state &= ~SRS_GET_PKTS;
2459 srs_rx->sr_poll_goto_sleep++;
2460 }
2461 }
2462 done:
2463 mac_srs->srs_state |= SRS_POLL_THR_QUIESCED;
2464 cv_signal(&mac_srs->srs_async);
2465 /*
2466 * If this is a temporary quiesce then wait for the restart signal
2467 * from the srs worker. Then clear the flags and signal the srs worker
2468 * to ensure a positive handshake and go back to start.
2469 */
2470 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_POLL_THR_RESTART)))
2471 cv_wait(async, lock);
2472 if (mac_srs->srs_state & SRS_POLL_THR_RESTART) {
2473 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
2474 mac_srs->srs_state &=
2475 ~(SRS_POLL_THR_QUIESCED | SRS_POLL_THR_RESTART);
2476 cv_signal(&mac_srs->srs_async);
2477 goto start;
2478 } else {
2479 mac_srs->srs_state |= SRS_POLL_THR_EXITED;
2480 cv_signal(&mac_srs->srs_async);
2481 CALLB_CPR_EXIT(&cprinfo);
2482 thread_exit();
2483 }
2484 }
2485
2486 /*
2487 * mac_srs_pick_chain
2488 *
2489 * In Bandwidth control case, checks how many packets can be processed
2490 * and return them in a sub chain.
2491 */
2492 static mblk_t *
2493 mac_srs_pick_chain(mac_soft_ring_set_t *mac_srs, mblk_t **chain_tail,
2494 size_t *chain_sz, int *chain_cnt)
2495 {
2496 mblk_t *head = NULL;
2497 mblk_t *tail = NULL;
2498 size_t sz;
2499 size_t tsz = 0;
2500 int cnt = 0;
2501 mblk_t *mp;
2502
2503 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2504 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2505 if (((mac_srs->srs_bw->mac_bw_used + mac_srs->srs_size) <=
2506 mac_srs->srs_bw->mac_bw_limit) ||
2507 (mac_srs->srs_bw->mac_bw_limit == 0)) {
2508 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2509 head = mac_srs->srs_first;
2510 mac_srs->srs_first = NULL;
2511 *chain_tail = mac_srs->srs_last;
2512 mac_srs->srs_last = NULL;
2513 *chain_sz = mac_srs->srs_size;
2514 *chain_cnt = mac_srs->srs_count;
2515 mac_srs->srs_count = 0;
2516 mac_srs->srs_size = 0;
2517 return (head);
2518 }
2519
2520 /*
2521 * Can't clear the entire backlog.
2522 * Need to find how many packets to pick
2523 */
2524 ASSERT(MUTEX_HELD(&mac_srs->srs_bw->mac_bw_lock));
2525 while ((mp = mac_srs->srs_first) != NULL) {
2526 sz = msgdsize(mp);
2527 if ((tsz + sz + mac_srs->srs_bw->mac_bw_used) >
2528 mac_srs->srs_bw->mac_bw_limit) {
2529 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED))
2530 mac_srs->srs_bw->mac_bw_state |=
2531 SRS_BW_ENFORCED;
2532 break;
2533 }
2534
2535 /*
2536 * The _size & cnt is decremented from the softrings
2537 * when they send up the packet for polling to work
2538 * properly.
2539 */
2540 tsz += sz;
2541 cnt++;
2542 mac_srs->srs_count--;
2543 mac_srs->srs_size -= sz;
2544 if (tail != NULL)
2545 tail->b_next = mp;
2546 else
2547 head = mp;
2548 tail = mp;
2549 mac_srs->srs_first = mac_srs->srs_first->b_next;
2550 }
2551 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2552 if (mac_srs->srs_first == NULL)
2553 mac_srs->srs_last = NULL;
2554
2555 if (tail != NULL)
2556 tail->b_next = NULL;
2557 *chain_tail = tail;
2558 *chain_cnt = cnt;
2559 *chain_sz = tsz;
2560
2561 return (head);
2562 }
2563
2564 /*
2565 * mac_rx_srs_drain
2566 *
2567 * The SRS drain routine. Gets to run to clear the queue. Any thread
2568 * (worker, interrupt, poll) can call this based on processing model.
2569 * The first thing we do is disable interrupts if possible and then
2570 * drain the queue. we also try to poll the underlying hardware if
2571 * there is a dedicated hardware Rx ring assigned to this SRS.
2572 *
2573 * There is a equivalent drain routine in bandwidth control mode
2574 * mac_rx_srs_drain_bw. There is some code duplication between the two
2575 * routines but they are highly performance sensitive and are easier
2576 * to read/debug if they stay separate. Any code changes here might
2577 * also apply to mac_rx_srs_drain_bw as well.
2578 */
2579 void
2580 mac_rx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
2581 {
2582 mblk_t *head;
2583 mblk_t *tail;
2584 timeout_id_t tid;
2585 int cnt = 0;
2586 mac_client_impl_t *mcip = mac_srs->srs_mcip;
2587 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2588
2589 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2590 ASSERT(!(mac_srs->srs_type & SRST_BW_CONTROL));
2591
2592 /* If we are blanked i.e. can't do upcalls, then we are done */
2593 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
2594 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
2595 (mac_srs->srs_state & SRS_PAUSE));
2596 goto out;
2597 }
2598
2599 if (mac_srs->srs_first == NULL)
2600 goto out;
2601
2602 if (!(mac_srs->srs_state & SRS_LATENCY_OPT) &&
2603 (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)) {
2604 /*
2605 * In the normal case, the SRS worker thread does no
2606 * work and we wait for a backlog to build up before
2607 * we switch into polling mode. In case we are
2608 * optimizing for throughput, we use the worker thread
2609 * as well. The goal is to let worker thread process
2610 * the queue and poll thread to feed packets into
2611 * the queue. As such, we should signal the poll
2612 * thread to try and get more packets.
2613 *
2614 * We could have pulled this check in the POLL_RING
2615 * macro itself but keeping it explicit here makes
2616 * the architecture more human understandable.
2617 */
2618 MAC_SRS_POLL_RING(mac_srs);
2619 }
2620
2621 again:
2622 head = mac_srs->srs_first;
2623 mac_srs->srs_first = NULL;
2624 tail = mac_srs->srs_last;
2625 mac_srs->srs_last = NULL;
2626 cnt = mac_srs->srs_count;
2627 mac_srs->srs_count = 0;
2628
2629 ASSERT(head != NULL);
2630 ASSERT(tail != NULL);
2631
2632 if ((tid = mac_srs->srs_tid) != NULL)
2633 mac_srs->srs_tid = NULL;
2634
2635 mac_srs->srs_state |= (SRS_PROC|proc_type);
2636
2637
2638 /*
2639 * mcip is NULL for broadcast and multicast flows. The promisc
2640 * callbacks for broadcast and multicast packets are delivered from
2641 * mac_rx() and we don't need to worry about that case in this path
2642 */
2643 if (mcip != NULL) {
2644 if (mcip->mci_promisc_list != NULL) {
2645 mutex_exit(&mac_srs->srs_lock);
2646 mac_promisc_client_dispatch(mcip, head);
2647 mutex_enter(&mac_srs->srs_lock);
2648 }
2649 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
2650 mutex_exit(&mac_srs->srs_lock);
2651 mac_protect_intercept_dynamic(mcip, head);
2652 mutex_enter(&mac_srs->srs_lock);
2653 }
2654 }
2655
2656 /*
2657 * Check if SRS itself is doing the processing
2658 * This direct path does not apply when subflows are present. In this
2659 * case, packets need to be dispatched to a soft ring according to the
2660 * flow's bandwidth and other resources contraints.
2661 */
2662 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
2663 mac_direct_rx_t proc;
2664 void *arg1;
2665 mac_resource_handle_t arg2;
2666
2667 /*
2668 * This is the case when a Rx is directly
2669 * assigned and we have a fully classified
2670 * protocol chain. We can deal with it in
2671 * one shot.
2672 */
2673 proc = srs_rx->sr_func;
2674 arg1 = srs_rx->sr_arg1;
2675 arg2 = srs_rx->sr_arg2;
2676
2677 mac_srs->srs_state |= SRS_CLIENT_PROC;
2678 mutex_exit(&mac_srs->srs_lock);
2679 if (tid != NULL) {
2680 (void) untimeout(tid);
2681 tid = NULL;
2682 }
2683
2684 proc(arg1, arg2, head, NULL);
2685 /*
2686 * Decrement the size and count here itelf
2687 * since the packet has been processed.
2688 */
2689 mutex_enter(&mac_srs->srs_lock);
2690 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
2691 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
2692 cv_signal(&mac_srs->srs_client_cv);
2693 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
2694 } else {
2695 /* Some kind of softrings based fanout is required */
2696 mutex_exit(&mac_srs->srs_lock);
2697 if (tid != NULL) {
2698 (void) untimeout(tid);
2699 tid = NULL;
2700 }
2701
2702 /*
2703 * Since the fanout routines can deal with chains,
2704 * shoot the entire chain up.
2705 */
2706 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
2707 mac_rx_srs_fanout(mac_srs, head);
2708 else
2709 mac_rx_srs_proto_fanout(mac_srs, head);
2710 mutex_enter(&mac_srs->srs_lock);
2711 }
2712
2713 if (!(mac_srs->srs_state & (SRS_BLANK|SRS_PAUSE)) &&
2714 (mac_srs->srs_first != NULL)) {
2715 /*
2716 * More packets arrived while we were clearing the
2717 * SRS. This can be possible because of one of
2718 * three conditions below:
2719 * 1) The driver is using multiple worker threads
2720 * to send the packets to us.
2721 * 2) The driver has a race in switching
2722 * between interrupt and polling mode or
2723 * 3) Packets are arriving in this SRS via the
2724 * S/W classification as well.
2725 *
2726 * We should switch to polling mode and see if we
2727 * need to send the poll thread down. Also, signal
2728 * the worker thread to process whats just arrived.
2729 */
2730 MAC_SRS_POLLING_ON(mac_srs);
2731 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) {
2732 srs_rx->sr_drain_poll_sig++;
2733 MAC_SRS_POLL_RING(mac_srs);
2734 }
2735
2736 /*
2737 * If we didn't signal the poll thread, we need
2738 * to deal with the pending packets ourselves.
2739 */
2740 if (proc_type == SRS_WORKER) {
2741 srs_rx->sr_drain_again++;
2742 goto again;
2743 } else {
2744 srs_rx->sr_drain_worker_sig++;
2745 cv_signal(&mac_srs->srs_async);
2746 }
2747 }
2748
2749 out:
2750 if (mac_srs->srs_state & SRS_GET_PKTS) {
2751 /*
2752 * Poll thread is already running. Leave the
2753 * SRS_RPOC set and hand over the control to
2754 * poll thread.
2755 */
2756 mac_srs->srs_state &= ~proc_type;
2757 srs_rx->sr_drain_poll_running++;
2758 return;
2759 }
2760
2761 /*
2762 * Even if there are no packets queued in SRS, we
2763 * need to make sure that the shared counter is
2764 * clear and any associated softrings have cleared
2765 * all the backlog. Otherwise, leave the interface
2766 * in polling mode and the poll thread will get
2767 * signalled once the count goes down to zero.
2768 *
2769 * If someone is already draining the queue (SRS_PROC is
2770 * set) when the srs_poll_pkt_cnt goes down to zero,
2771 * then it means that drain is already running and we
2772 * will turn off polling at that time if there is
2773 * no backlog.
2774 *
2775 * As long as there are packets queued either
2776 * in soft ring set or its soft rings, we will leave
2777 * the interface in polling mode (even if the drain
2778 * was done being the interrupt thread). We signal
2779 * the poll thread as well if we have dipped below
2780 * low water mark.
2781 *
2782 * NOTE: We can't use the MAC_SRS_POLLING_ON macro
2783 * since that turn polling on only for worker thread.
2784 * Its not worth turning polling on for interrupt
2785 * thread (since NIC will not issue another interrupt)
2786 * unless a backlog builds up.
2787 */
2788 if ((srs_rx->sr_poll_pkt_cnt > 0) &&
2789 (mac_srs->srs_state & SRS_POLLING_CAPAB)) {
2790 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2791 srs_rx->sr_drain_keep_polling++;
2792 MAC_SRS_POLLING_ON(mac_srs);
2793 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)
2794 MAC_SRS_POLL_RING(mac_srs);
2795 return;
2796 }
2797
2798 /* Nothing else to do. Get out of poll mode */
2799 MAC_SRS_POLLING_OFF(mac_srs);
2800 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2801 srs_rx->sr_drain_finish_intr++;
2802 }
2803
2804 /*
2805 * mac_rx_srs_drain_bw
2806 *
2807 * The SRS BW drain routine. Gets to run to clear the queue. Any thread
2808 * (worker, interrupt, poll) can call this based on processing model.
2809 * The first thing we do is disable interrupts if possible and then
2810 * drain the queue. we also try to poll the underlying hardware if
2811 * there is a dedicated hardware Rx ring assigned to this SRS.
2812 *
2813 * There is a equivalent drain routine in non bandwidth control mode
2814 * mac_rx_srs_drain. There is some code duplication between the two
2815 * routines but they are highly performance sensitive and are easier
2816 * to read/debug if they stay separate. Any code changes here might
2817 * also apply to mac_rx_srs_drain as well.
2818 */
2819 void
2820 mac_rx_srs_drain_bw(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
2821 {
2822 mblk_t *head;
2823 mblk_t *tail;
2824 timeout_id_t tid;
2825 size_t sz = 0;
2826 int cnt = 0;
2827 mac_client_impl_t *mcip = mac_srs->srs_mcip;
2828 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2829 clock_t now;
2830
2831 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2832 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
2833 again:
2834 /* Check if we are doing B/W control */
2835 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2836 now = ddi_get_lbolt();
2837 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
2838 mac_srs->srs_bw->mac_bw_curr_time = now;
2839 mac_srs->srs_bw->mac_bw_used = 0;
2840 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
2841 mac_srs->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED;
2842 } else if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) {
2843 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2844 goto done;
2845 } else if (mac_srs->srs_bw->mac_bw_used >
2846 mac_srs->srs_bw->mac_bw_limit) {
2847 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
2848 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2849 goto done;
2850 }
2851 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2852
2853 /* If we are blanked i.e. can't do upcalls, then we are done */
2854 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
2855 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
2856 (mac_srs->srs_state & SRS_PAUSE));
2857 goto done;
2858 }
2859
2860 sz = 0;
2861 cnt = 0;
2862 if ((head = mac_srs_pick_chain(mac_srs, &tail, &sz, &cnt)) == NULL) {
2863 /*
2864 * We couldn't pick up a single packet.
2865 */
2866 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2867 if ((mac_srs->srs_bw->mac_bw_used == 0) &&
2868 (mac_srs->srs_size != 0) &&
2869 !(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2870 /*
2871 * Seems like configured B/W doesn't
2872 * even allow processing of 1 packet
2873 * per tick.
2874 *
2875 * XXX: raise the limit to processing
2876 * at least 1 packet per tick.
2877 */
2878 mac_srs->srs_bw->mac_bw_limit +=
2879 mac_srs->srs_bw->mac_bw_limit;
2880 mac_srs->srs_bw->mac_bw_drop_threshold +=
2881 mac_srs->srs_bw->mac_bw_drop_threshold;
2882 cmn_err(CE_NOTE, "mac_rx_srs_drain: srs(%p) "
2883 "raised B/W limit to %d since not even a "
2884 "single packet can be processed per "
2885 "tick %d\n", (void *)mac_srs,
2886 (int)mac_srs->srs_bw->mac_bw_limit,
2887 (int)msgdsize(mac_srs->srs_first));
2888 }
2889 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2890 goto done;
2891 }
2892
2893 ASSERT(head != NULL);
2894 ASSERT(tail != NULL);
2895
2896 /* zero bandwidth: drop all and return to interrupt mode */
2897 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2898 if (mac_srs->srs_bw->mac_bw_limit == 0) {
2899 srs_rx->sr_stat.mrs_sdrops += cnt;
2900 ASSERT(mac_srs->srs_bw->mac_bw_sz >= sz);
2901 mac_srs->srs_bw->mac_bw_sz -= sz;
2902 mac_srs->srs_bw->mac_bw_drop_bytes += sz;
2903 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2904 mac_pkt_drop(NULL, NULL, head, B_FALSE);
2905 goto leave_poll;
2906 } else {
2907 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2908 }
2909
2910 if ((tid = mac_srs->srs_tid) != NULL)
2911 mac_srs->srs_tid = NULL;
2912
2913 mac_srs->srs_state |= (SRS_PROC|proc_type);
2914 MAC_SRS_WORKER_POLLING_ON(mac_srs);
2915
2916 /*
2917 * mcip is NULL for broadcast and multicast flows. The promisc
2918 * callbacks for broadcast and multicast packets are delivered from
2919 * mac_rx() and we don't need to worry about that case in this path
2920 */
2921 if (mcip != NULL) {
2922 if (mcip->mci_promisc_list != NULL) {
2923 mutex_exit(&mac_srs->srs_lock);
2924 mac_promisc_client_dispatch(mcip, head);
2925 mutex_enter(&mac_srs->srs_lock);
2926 }
2927 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
2928 mutex_exit(&mac_srs->srs_lock);
2929 mac_protect_intercept_dynamic(mcip, head);
2930 mutex_enter(&mac_srs->srs_lock);
2931 }
2932 }
2933
2934 /*
2935 * Check if SRS itself is doing the processing
2936 * This direct path does not apply when subflows are present. In this
2937 * case, packets need to be dispatched to a soft ring according to the
2938 * flow's bandwidth and other resources contraints.
2939 */
2940 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
2941 mac_direct_rx_t proc;
2942 void *arg1;
2943 mac_resource_handle_t arg2;
2944
2945 /*
2946 * This is the case when a Rx is directly
2947 * assigned and we have a fully classified
2948 * protocol chain. We can deal with it in
2949 * one shot.
2950 */
2951 proc = srs_rx->sr_func;
2952 arg1 = srs_rx->sr_arg1;
2953 arg2 = srs_rx->sr_arg2;
2954
2955 mac_srs->srs_state |= SRS_CLIENT_PROC;
2956 mutex_exit(&mac_srs->srs_lock);
2957 if (tid != NULL) {
2958 (void) untimeout(tid);
2959 tid = NULL;
2960 }
2961
2962 proc(arg1, arg2, head, NULL);
2963 /*
2964 * Decrement the size and count here itelf
2965 * since the packet has been processed.
2966 */
2967 mutex_enter(&mac_srs->srs_lock);
2968 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
2969 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
2970
2971 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
2972 cv_signal(&mac_srs->srs_client_cv);
2973 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
2974 } else {
2975 /* Some kind of softrings based fanout is required */
2976 mutex_exit(&mac_srs->srs_lock);
2977 if (tid != NULL) {
2978 (void) untimeout(tid);
2979 tid = NULL;
2980 }
2981
2982 /*
2983 * Since the fanout routines can deal with chains,
2984 * shoot the entire chain up.
2985 */
2986 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
2987 mac_rx_srs_fanout(mac_srs, head);
2988 else
2989 mac_rx_srs_proto_fanout(mac_srs, head);
2990 mutex_enter(&mac_srs->srs_lock);
2991 }
2992
2993 /*
2994 * Send the poll thread to pick up any packets arrived
2995 * so far. This also serves as the last check in case
2996 * nothing else is queued in the SRS. The poll thread
2997 * is signalled only in the case the drain was done
2998 * by the worker thread and SRS_WORKER is set. The
2999 * worker thread can run in parallel as long as the
3000 * SRS_WORKER flag is set. We we have nothing else to
3001 * process, we can exit while leaving SRS_PROC set
3002 * which gives the poll thread control to process and
3003 * cleanup once it returns from the NIC.
3004 *
3005 * If we have nothing else to process, we need to
3006 * ensure that we keep holding the srs_lock till
3007 * all the checks below are done and control is
3008 * handed to the poll thread if it was running.
3009 */
3010 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
3011 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
3012 if (mac_srs->srs_first != NULL) {
3013 if (proc_type == SRS_WORKER) {
3014 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3015 if (srs_rx->sr_poll_pkt_cnt <=
3016 srs_rx->sr_lowat)
3017 MAC_SRS_POLL_RING(mac_srs);
3018 goto again;
3019 } else {
3020 cv_signal(&mac_srs->srs_async);
3021 }
3022 }
3023 }
3024 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3025
3026 done:
3027
3028 if (mac_srs->srs_state & SRS_GET_PKTS) {
3029 /*
3030 * Poll thread is already running. Leave the
3031 * SRS_RPOC set and hand over the control to
3032 * poll thread.
3033 */
3034 mac_srs->srs_state &= ~proc_type;
3035 return;
3036 }
3037
3038 /*
3039 * If we can't process packets because we have exceeded
3040 * B/W limit for this tick, just set the timeout
3041 * and leave.
3042 *
3043 * Even if there are no packets queued in SRS, we
3044 * need to make sure that the shared counter is
3045 * clear and any associated softrings have cleared
3046 * all the backlog. Otherwise, leave the interface
3047 * in polling mode and the poll thread will get
3048 * signalled once the count goes down to zero.
3049 *
3050 * If someone is already draining the queue (SRS_PROC is
3051 * set) when the srs_poll_pkt_cnt goes down to zero,
3052 * then it means that drain is already running and we
3053 * will turn off polling at that time if there is
3054 * no backlog. As long as there are packets queued either
3055 * is soft ring set or its soft rings, we will leave
3056 * the interface in polling mode.
3057 */
3058 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
3059 if ((mac_srs->srs_state & SRS_POLLING_CAPAB) &&
3060 ((mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) ||
3061 (srs_rx->sr_poll_pkt_cnt > 0))) {
3062 MAC_SRS_POLLING_ON(mac_srs);
3063 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
3064 if ((mac_srs->srs_first != NULL) &&
3065 (mac_srs->srs_tid == NULL))
3066 mac_srs->srs_tid = timeout(mac_srs_fire,
3067 mac_srs, 1);
3068 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3069 return;
3070 }
3071 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3072
3073 leave_poll:
3074
3075 /* Nothing else to do. Get out of poll mode */
3076 MAC_SRS_POLLING_OFF(mac_srs);
3077 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
3078 }
3079
3080 /*
3081 * mac_srs_worker
3082 *
3083 * The SRS worker routine. Drains the queue when no one else is
3084 * processing it.
3085 */
3086 void
3087 mac_srs_worker(mac_soft_ring_set_t *mac_srs)
3088 {
3089 kmutex_t *lock = &mac_srs->srs_lock;
3090 kcondvar_t *async = &mac_srs->srs_async;
3091 callb_cpr_t cprinfo;
3092 boolean_t bw_ctl_flag;
3093
3094 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "srs_worker");
3095 mutex_enter(lock);
3096
3097 start:
3098 for (;;) {
3099 bw_ctl_flag = B_FALSE;
3100 if (mac_srs->srs_type & SRST_BW_CONTROL) {
3101 MAC_SRS_BW_LOCK(mac_srs);
3102 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
3103 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
3104 bw_ctl_flag = B_TRUE;
3105 MAC_SRS_BW_UNLOCK(mac_srs);
3106 }
3107 /*
3108 * The SRS_BW_ENFORCED flag may change since we have dropped
3109 * the mac_bw_lock. However the drain function can handle both
3110 * a drainable SRS or a bandwidth controlled SRS, and the
3111 * effect of scheduling a timeout is to wakeup the worker
3112 * thread which in turn will call the drain function. Since
3113 * we release the srs_lock atomically only in the cv_wait there
3114 * isn't a fear of waiting for ever.
3115 */
3116 while (((mac_srs->srs_state & SRS_PROC) ||
3117 (mac_srs->srs_first == NULL) || bw_ctl_flag ||
3118 (mac_srs->srs_state & SRS_TX_BLOCKED)) &&
3119 !(mac_srs->srs_state & SRS_PAUSE)) {
3120 /*
3121 * If we have packets queued and we are here
3122 * because B/W control is in place, we better
3123 * schedule the worker wakeup after 1 tick
3124 * to see if bandwidth control can be relaxed.
3125 */
3126 if (bw_ctl_flag && mac_srs->srs_tid == NULL) {
3127 /*
3128 * We need to ensure that a timer is already
3129 * scheduled or we force schedule one for
3130 * later so that we can continue processing
3131 * after this quanta is over.
3132 */
3133 mac_srs->srs_tid = timeout(mac_srs_fire,
3134 mac_srs, 1);
3135 }
3136 wait:
3137 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3138 cv_wait(async, lock);
3139 CALLB_CPR_SAFE_END(&cprinfo, lock);
3140
3141 if (mac_srs->srs_state & SRS_PAUSE)
3142 goto done;
3143 if (mac_srs->srs_state & SRS_PROC)
3144 goto wait;
3145
3146 if (mac_srs->srs_first != NULL &&
3147 mac_srs->srs_type & SRST_BW_CONTROL) {
3148 MAC_SRS_BW_LOCK(mac_srs);
3149 if (mac_srs->srs_bw->mac_bw_state &
3150 SRS_BW_ENFORCED) {
3151 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
3152 }
3153 bw_ctl_flag = mac_srs->srs_bw->mac_bw_state &
3154 SRS_BW_ENFORCED;
3155 MAC_SRS_BW_UNLOCK(mac_srs);
3156 }
3157 }
3158
3159 if (mac_srs->srs_state & SRS_PAUSE)
3160 goto done;
3161 mac_srs->srs_drain_func(mac_srs, SRS_WORKER);
3162 }
3163 done:
3164 /*
3165 * The Rx SRS quiesce logic first cuts off packet supply to the SRS
3166 * from both hard and soft classifications and waits for such threads
3167 * to finish before signaling the worker. So at this point the only
3168 * thread left that could be competing with the worker is the poll
3169 * thread. In the case of Tx, there shouldn't be any thread holding
3170 * SRS_PROC at this point.
3171 */
3172 if (!(mac_srs->srs_state & SRS_PROC)) {
3173 mac_srs->srs_state |= SRS_PROC;
3174 } else {
3175 ASSERT((mac_srs->srs_type & SRST_TX) == 0);
3176 /*
3177 * Poll thread still owns the SRS and is still running
3178 */
3179 ASSERT((mac_srs->srs_poll_thr == NULL) ||
3180 ((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
3181 SRS_POLL_THR_OWNER));
3182 }
3183 mac_srs_worker_quiesce(mac_srs);
3184 /*
3185 * Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator
3186 * of the quiesce operation
3187 */
3188 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_RESTART)))
3189 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3190
3191 if (mac_srs->srs_state & SRS_RESTART) {
3192 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
3193 mac_srs_worker_restart(mac_srs);
3194 mac_srs->srs_state &= ~SRS_PROC;
3195 goto start;
3196 }
3197
3198 if (!(mac_srs->srs_state & SRS_CONDEMNED_DONE))
3199 mac_srs_worker_quiesce(mac_srs);
3200
3201 mac_srs->srs_state &= ~SRS_PROC;
3202 /* The macro drops the srs_lock */
3203 CALLB_CPR_EXIT(&cprinfo);
3204 thread_exit();
3205 }
3206
3207 /*
3208 * mac_rx_srs_subflow_process
3209 *
3210 * Receive side routine called from interrupt path when there are
3211 * sub flows present on this SRS.
3212 */
3213 /* ARGSUSED */
3214 void
3215 mac_rx_srs_subflow_process(void *arg, mac_resource_handle_t srs,
3216 mblk_t *mp_chain, boolean_t loopback)
3217 {
3218 flow_entry_t *flent = NULL;
3219 flow_entry_t *prev_flent = NULL;
3220 mblk_t *mp = NULL;
3221 mblk_t *tail = NULL;
3222 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
3223 mac_client_impl_t *mcip;
3224
3225 mcip = mac_srs->srs_mcip;
3226 ASSERT(mcip != NULL);
3227
3228 /*
3229 * We need to determine the SRS for every packet
3230 * by walking the flow table, if we don't get any,
3231 * then we proceed using the SRS we came with.
3232 */
3233 mp = tail = mp_chain;
3234 while (mp != NULL) {
3235
3236 /*
3237 * We will increment the stats for the mactching subflow.
3238 * when we get the bytes/pkt count for the classified packets
3239 * later in mac_rx_srs_process.
3240 */
3241 (void) mac_flow_lookup(mcip->mci_subflow_tab, mp,
3242 FLOW_INBOUND, &flent);
3243
3244 if (mp == mp_chain || flent == prev_flent) {
3245 if (prev_flent != NULL)
3246 FLOW_REFRELE(prev_flent);
3247 prev_flent = flent;
3248 flent = NULL;
3249 tail = mp;
3250 mp = mp->b_next;
3251 continue;
3252 }
3253 tail->b_next = NULL;
3254 /*
3255 * A null indicates, this is for the mac_srs itself.
3256 * XXX-venu : probably assert for fe_rx_srs_cnt == 0.
3257 */
3258 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
3259 mac_rx_srs_process(arg,
3260 (mac_resource_handle_t)mac_srs, mp_chain,
3261 loopback);
3262 } else {
3263 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
3264 prev_flent->fe_cb_arg2, mp_chain, loopback);
3265 FLOW_REFRELE(prev_flent);
3266 }
3267 prev_flent = flent;
3268 flent = NULL;
3269 mp_chain = mp;
3270 tail = mp;
3271 mp = mp->b_next;
3272 }
3273 /* Last chain */
3274 ASSERT(mp_chain != NULL);
3275 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
3276 mac_rx_srs_process(arg,
3277 (mac_resource_handle_t)mac_srs, mp_chain, loopback);
3278 } else {
3279 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
3280 prev_flent->fe_cb_arg2, mp_chain, loopback);
3281 FLOW_REFRELE(prev_flent);
3282 }
3283 }
3284
3285 /*
3286 * mac_rx_srs_process
3287 *
3288 * Receive side routine called from the interrupt path.
3289 *
3290 * loopback is set to force a context switch on the loopback
3291 * path between MAC clients.
3292 */
3293 /* ARGSUSED */
3294 void
3295 mac_rx_srs_process(void *arg, mac_resource_handle_t srs, mblk_t *mp_chain,
3296 boolean_t loopback)
3297 {
3298 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
3299 mblk_t *mp, *tail, *head;
3300 int count = 0;
3301 int count1;
3302 size_t sz = 0;
3303 size_t chain_sz, sz1;
3304 mac_bw_ctl_t *mac_bw;
3305 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
3306
3307 /*
3308 * Set the tail, count and sz. We set the sz irrespective
3309 * of whether we are doing B/W control or not for the
3310 * purpose of updating the stats.
3311 */
3312 mp = tail = mp_chain;
3313 while (mp != NULL) {
3314 tail = mp;
3315 count++;
3316 sz += msgdsize(mp);
3317 mp = mp->b_next;
3318 }
3319
3320 mutex_enter(&mac_srs->srs_lock);
3321
3322 if (loopback) {
3323 SRS_RX_STAT_UPDATE(mac_srs, lclbytes, sz);
3324 SRS_RX_STAT_UPDATE(mac_srs, lclcnt, count);
3325
3326 } else {
3327 SRS_RX_STAT_UPDATE(mac_srs, intrbytes, sz);
3328 SRS_RX_STAT_UPDATE(mac_srs, intrcnt, count);
3329 }
3330
3331 /*
3332 * If the SRS in already being processed; has been blanked;
3333 * can be processed by worker thread only; or the B/W limit
3334 * has been reached, then queue the chain and check if
3335 * worker thread needs to be awakend.
3336 */
3337 if (mac_srs->srs_type & SRST_BW_CONTROL) {
3338 mac_bw = mac_srs->srs_bw;
3339 ASSERT(mac_bw != NULL);
3340 mutex_enter(&mac_bw->mac_bw_lock);
3341 mac_bw->mac_bw_intr += sz;
3342 if (mac_bw->mac_bw_limit == 0) {
3343 /* zero bandwidth: drop all */
3344 srs_rx->sr_stat.mrs_sdrops += count;
3345 mac_bw->mac_bw_drop_bytes += sz;
3346 mutex_exit(&mac_bw->mac_bw_lock);
3347 mutex_exit(&mac_srs->srs_lock);
3348 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
3349 return;
3350 } else {
3351 if ((mac_bw->mac_bw_sz + sz) <=
3352 mac_bw->mac_bw_drop_threshold) {
3353 mutex_exit(&mac_bw->mac_bw_lock);
3354 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain,
3355 tail, count, sz);
3356 } else {
3357 mp = mp_chain;
3358 chain_sz = 0;
3359 count1 = 0;
3360 tail = NULL;
3361 head = NULL;
3362 while (mp != NULL) {
3363 sz1 = msgdsize(mp);
3364 if (mac_bw->mac_bw_sz + chain_sz + sz1 >
3365 mac_bw->mac_bw_drop_threshold)
3366 break;
3367 chain_sz += sz1;
3368 count1++;
3369 tail = mp;
3370 mp = mp->b_next;
3371 }
3372 mutex_exit(&mac_bw->mac_bw_lock);
3373 if (tail != NULL) {
3374 head = tail->b_next;
3375 tail->b_next = NULL;
3376 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs,
3377 mp_chain, tail, count1, chain_sz);
3378 sz -= chain_sz;
3379 count -= count1;
3380 } else {
3381 /* Can't pick up any */
3382 head = mp_chain;
3383 }
3384 if (head != NULL) {
3385 /* Drop any packet over the threshold */
3386 srs_rx->sr_stat.mrs_sdrops += count;
3387 mutex_enter(&mac_bw->mac_bw_lock);
3388 mac_bw->mac_bw_drop_bytes += sz;
3389 mutex_exit(&mac_bw->mac_bw_lock);
3390 freemsgchain(head);
3391 }
3392 }
3393 MAC_SRS_WORKER_WAKEUP(mac_srs);
3394 mutex_exit(&mac_srs->srs_lock);
3395 return;
3396 }
3397 }
3398
3399 /*
3400 * If the total number of packets queued in the SRS and
3401 * its associated soft rings exceeds the max allowed,
3402 * then drop the chain. If we are polling capable, this
3403 * shouldn't be happening.
3404 */
3405 if (!(mac_srs->srs_type & SRST_BW_CONTROL) &&
3406 (srs_rx->sr_poll_pkt_cnt > srs_rx->sr_hiwat)) {
3407 mac_bw = mac_srs->srs_bw;
3408 srs_rx->sr_stat.mrs_sdrops += count;
3409 mutex_enter(&mac_bw->mac_bw_lock);
3410 mac_bw->mac_bw_drop_bytes += sz;
3411 mutex_exit(&mac_bw->mac_bw_lock);
3412 freemsgchain(mp_chain);
3413 mutex_exit(&mac_srs->srs_lock);
3414 return;
3415 }
3416
3417 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, tail, count, sz);
3418
3419 if (!(mac_srs->srs_state & SRS_PROC)) {
3420 /*
3421 * If we are coming via loopback, if we are not optimizing for
3422 * latency, or if our stack is running deep, we should signal
3423 * the worker thread.
3424 */
3425 if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT) ||
3426 MAC_RX_SRS_TOODEEP()) {
3427 /*
3428 * For loopback, We need to let the worker take
3429 * over as we don't want to continue in the same
3430 * thread even if we can. This could lead to stack
3431 * overflows and may also end up using
3432 * resources (cpu) incorrectly.
3433 */
3434 cv_signal(&mac_srs->srs_async);
3435 } else {
3436 /*
3437 * Seems like no one is processing the SRS and
3438 * there is no backlog. We also inline process
3439 * our packet if its a single packet in non
3440 * latency optimized case (in latency optimized
3441 * case, we inline process chains of any size).
3442 */
3443 mac_srs->srs_drain_func(mac_srs, SRS_PROC_FAST);
3444 }
3445 }
3446 mutex_exit(&mac_srs->srs_lock);
3447 }
3448
3449 /* TX SIDE ROUTINES (RUNTIME) */
3450
3451 /*
3452 * mac_tx_srs_no_desc
3453 *
3454 * This routine is called by Tx single ring default mode
3455 * when Tx ring runs out of descs.
3456 */
3457 mac_tx_cookie_t
3458 mac_tx_srs_no_desc(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3459 uint16_t flag, mblk_t **ret_mp)
3460 {
3461 mac_tx_cookie_t cookie = NULL;
3462 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3463 boolean_t wakeup_worker = B_TRUE;
3464 uint32_t tx_mode = srs_tx->st_mode;
3465 int cnt, sz;
3466 mblk_t *tail;
3467
3468 ASSERT(tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_BW);
3469 if (flag & MAC_DROP_ON_NO_DESC) {
3470 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3471 } else {
3472 if (mac_srs->srs_first != NULL)
3473 wakeup_worker = B_FALSE;
3474 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3475 if (flag & MAC_TX_NO_ENQUEUE) {
3476 /*
3477 * If TX_QUEUED is not set, queue the
3478 * packet and let mac_tx_srs_drain()
3479 * set the TX_BLOCKED bit for the
3480 * reasons explained above. Otherwise,
3481 * return the mblks.
3482 */
3483 if (wakeup_worker) {
3484 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3485 mp_chain, tail, cnt, sz);
3486 } else {
3487 MAC_TX_SET_NO_ENQUEUE(mac_srs,
3488 mp_chain, ret_mp, cookie);
3489 }
3490 } else {
3491 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
3492 tail, cnt, sz, cookie);
3493 }
3494 if (wakeup_worker)
3495 cv_signal(&mac_srs->srs_async);
3496 }
3497 return (cookie);
3498 }
3499
3500 /*
3501 * mac_tx_srs_enqueue
3502 *
3503 * This routine is called when Tx SRS is operating in either serializer
3504 * or bandwidth mode. In serializer mode, a packet will get enqueued
3505 * when a thread cannot enter SRS exclusively. In bandwidth mode,
3506 * packets gets queued if allowed byte-count limit for a tick is
3507 * exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and
3508 * MAC_TX_NO_ENQUEUE is set is different than when operaing in either
3509 * the default mode or fanout mode. Here packets get dropped or
3510 * returned back to the caller only after hi-watermark worth of data
3511 * is queued.
3512 */
3513 static mac_tx_cookie_t
3514 mac_tx_srs_enqueue(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3515 uint16_t flag, uintptr_t fanout_hint, mblk_t **ret_mp)
3516 {
3517 mac_tx_cookie_t cookie = NULL;
3518 int cnt, sz;
3519 mblk_t *tail;
3520 boolean_t wakeup_worker = B_TRUE;
3521
3522 /*
3523 * Ignore fanout hint if we don't have multiple tx rings.
3524 */
3525 if (!MAC_TX_SOFT_RINGS(mac_srs))
3526 fanout_hint = 0;
3527
3528 if (mac_srs->srs_first != NULL)
3529 wakeup_worker = B_FALSE;
3530 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3531 if (flag & MAC_DROP_ON_NO_DESC) {
3532 if (mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) {
3533 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3534 } else {
3535 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3536 mp_chain, tail, cnt, sz);
3537 }
3538 } else if (flag & MAC_TX_NO_ENQUEUE) {
3539 if ((mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) ||
3540 (mac_srs->srs_state & SRS_TX_WAKEUP_CLIENT)) {
3541 MAC_TX_SET_NO_ENQUEUE(mac_srs, mp_chain,
3542 ret_mp, cookie);
3543 } else {
3544 mp_chain->b_prev = (mblk_t *)fanout_hint;
3545 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3546 mp_chain, tail, cnt, sz);
3547 }
3548 } else {
3549 /*
3550 * If you are BW_ENFORCED, just enqueue the
3551 * packet. srs_worker will drain it at the
3552 * prescribed rate. Before enqueueing, save
3553 * the fanout hint.
3554 */
3555 mp_chain->b_prev = (mblk_t *)fanout_hint;
3556 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
3557 tail, cnt, sz, cookie);
3558 }
3559 if (wakeup_worker)
3560 cv_signal(&mac_srs->srs_async);
3561 return (cookie);
3562 }
3563
3564 /*
3565 * There are seven tx modes:
3566 *
3567 * 1) Default mode (SRS_TX_DEFAULT)
3568 * 2) Serialization mode (SRS_TX_SERIALIZE)
3569 * 3) Fanout mode (SRS_TX_FANOUT)
3570 * 4) Bandwdith mode (SRS_TX_BW)
3571 * 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT)
3572 * 6) aggr Tx mode (SRS_TX_AGGR)
3573 * 7) aggr Tx bw mode (SRS_TX_BW_AGGR)
3574 *
3575 * The tx mode in which an SRS operates is decided in mac_tx_srs_setup()
3576 * based on the number of Tx rings requested for an SRS and whether
3577 * bandwidth control is requested or not.
3578 *
3579 * The default mode (i.e., no fanout/no bandwidth) is used when the
3580 * underlying NIC does not have Tx rings or just one Tx ring. In this mode,
3581 * the SRS acts as a pass-thru. Packets will go directly to mac_tx_send().
3582 * When the underlying Tx ring runs out of Tx descs, it starts queueing up
3583 * packets in SRS. When flow-control is relieved, the srs_worker drains
3584 * the queued packets and informs blocked clients to restart sending
3585 * packets.
3586 *
3587 * In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized. This
3588 * mode is used when the link has no Tx rings or only one Tx ring.
3589 *
3590 * In the SRS_TX_FANOUT mode, packets will be fanned out to multiple
3591 * Tx rings. Each Tx ring will have a soft ring associated with it.
3592 * These soft rings will be hung off the Tx SRS. Queueing if it happens
3593 * due to lack of Tx desc will be in individual soft ring (and not srs)
3594 * associated with Tx ring.
3595 *
3596 * In the TX_BW mode, tx srs will allow packets to go down to Tx ring
3597 * only if bw is available. Otherwise the packets will be queued in
3598 * SRS. If fanout to multiple Tx rings is configured, the packets will
3599 * be fanned out among the soft rings associated with the Tx rings.
3600 *
3601 * In SRS_TX_AGGR mode, mac_tx_aggr_mode() routine is called. This routine
3602 * invokes an aggr function, aggr_find_tx_ring(), to find a pseudo Tx ring
3603 * belonging to a port on which the packet has to be sent. Aggr will
3604 * always have a pseudo Tx ring associated with it even when it is an
3605 * aggregation over a single NIC that has no Tx rings. Even in such a
3606 * case, the single pseudo Tx ring will have a soft ring associated with
3607 * it and the soft ring will hang off the SRS.
3608 *
3609 * If a bandwidth is specified for an aggr, SRS_TX_BW_AGGR mode is used.
3610 * In this mode, the bandwidth is first applied on the outgoing packets
3611 * and later mac_tx_addr_mode() function is called to send the packet out
3612 * of one of the pseudo Tx rings.
3613 *
3614 * Four flags are used in srs_state for indicating flow control
3615 * conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT.
3616 * SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the
3617 * driver below.
3618 * SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat
3619 * and flow-control pressure is applied back to clients. The clients expect
3620 * wakeup when flow-control is relieved.
3621 * SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk
3622 * got returned back to client either due to lack of Tx descs or due to bw
3623 * control reasons. The clients expect a wakeup when condition is relieved.
3624 *
3625 * The fourth argument to mac_tx() is the flag. Normally it will be 0 but
3626 * some clients set the following values too: MAC_DROP_ON_NO_DESC,
3627 * MAC_TX_NO_ENQUEUE
3628 * Mac clients that do not want packets to be enqueued in the mac layer set
3629 * MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or
3630 * Tx soft rings but instead get dropped when the NIC runs out of desc. The
3631 * behaviour of this flag is different when the Tx is running in serializer
3632 * or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet
3633 * get dropped when Tx high watermark is reached.
3634 * There are some mac clients like vsw, aggr that want the mblks to be
3635 * returned back to clients instead of being queued in Tx SRS (or Tx soft
3636 * rings) under flow-control (i.e., out of desc or exceeding bw limits)
3637 * conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set.
3638 * In the default and Tx fanout mode, the un-transmitted mblks will be
3639 * returned back to the clients when the driver runs out of Tx descs.
3640 * SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or
3641 * soft ring) so that the clients can be woken up when Tx desc become
3642 * available. When running in serializer or bandwidth mode mode,
3643 * SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached.
3644 */
3645
3646 mac_tx_func_t
3647 mac_tx_get_func(uint32_t mode)
3648 {
3649 return (mac_tx_mode_list[mode].mac_tx_func);
3650 }
3651
3652 /* ARGSUSED */
3653 static mac_tx_cookie_t
3654 mac_tx_single_ring_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3655 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3656 {
3657 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3658 mac_tx_stats_t stats;
3659 mac_tx_cookie_t cookie = NULL;
3660
3661 ASSERT(srs_tx->st_mode == SRS_TX_DEFAULT);
3662
3663 /* Regular case with a single Tx ring */
3664 /*
3665 * SRS_TX_BLOCKED is set when underlying NIC runs
3666 * out of Tx descs and messages start getting
3667 * queued. It won't get reset until
3668 * tx_srs_drain() completely drains out the
3669 * messages.
3670 */
3671 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
3672 /* Tx descs/resources not available */
3673 mutex_enter(&mac_srs->srs_lock);
3674 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
3675 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain,
3676 flag, ret_mp);
3677 mutex_exit(&mac_srs->srs_lock);
3678 return (cookie);
3679 }
3680 /*
3681 * While we were computing mblk count, the
3682 * flow control condition got relieved.
3683 * Continue with the transmission.
3684 */
3685 mutex_exit(&mac_srs->srs_lock);
3686 }
3687
3688 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3689 mp_chain, &stats);
3690
3691 /*
3692 * Multiple threads could be here sending packets.
3693 * Under such conditions, it is not possible to
3694 * automically set SRS_TX_BLOCKED bit to indicate
3695 * out of tx desc condition. To atomically set
3696 * this, we queue the returned packet and do
3697 * the setting of SRS_TX_BLOCKED in
3698 * mac_tx_srs_drain().
3699 */
3700 if (mp_chain != NULL) {
3701 mutex_enter(&mac_srs->srs_lock);
3702 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, flag, ret_mp);
3703 mutex_exit(&mac_srs->srs_lock);
3704 return (cookie);
3705 }
3706 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3707
3708 return (NULL);
3709 }
3710
3711 /*
3712 * mac_tx_serialize_mode
3713 *
3714 * This is an experimental mode implemented as per the request of PAE.
3715 * In this mode, all callers attempting to send a packet to the NIC
3716 * will get serialized. Only one thread at any time will access the
3717 * NIC to send the packet out.
3718 */
3719 /* ARGSUSED */
3720 static mac_tx_cookie_t
3721 mac_tx_serializer_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3722 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3723 {
3724 mac_tx_stats_t stats;
3725 mac_tx_cookie_t cookie = NULL;
3726 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3727
3728 /* Single ring, serialize below */
3729 ASSERT(srs_tx->st_mode == SRS_TX_SERIALIZE);
3730 mutex_enter(&mac_srs->srs_lock);
3731 if ((mac_srs->srs_first != NULL) ||
3732 (mac_srs->srs_state & SRS_PROC)) {
3733 /*
3734 * In serialization mode, queue all packets until
3735 * TX_HIWAT is set.
3736 * If drop bit is set, drop if TX_HIWAT is set.
3737 * If no_enqueue is set, still enqueue until hiwat
3738 * is set and return mblks after TX_HIWAT is set.
3739 */
3740 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain,
3741 flag, NULL, ret_mp);
3742 mutex_exit(&mac_srs->srs_lock);
3743 return (cookie);
3744 }
3745 /*
3746 * No packets queued, nothing on proc and no flow
3747 * control condition. Fast-path, ok. Do inline
3748 * processing.
3749 */
3750 mac_srs->srs_state |= SRS_PROC;
3751 mutex_exit(&mac_srs->srs_lock);
3752
3753 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3754 mp_chain, &stats);
3755
3756 mutex_enter(&mac_srs->srs_lock);
3757 mac_srs->srs_state &= ~SRS_PROC;
3758 if (mp_chain != NULL) {
3759 cookie = mac_tx_srs_enqueue(mac_srs,
3760 mp_chain, flag, NULL, ret_mp);
3761 }
3762 if (mac_srs->srs_first != NULL) {
3763 /*
3764 * We processed inline our packet and a new
3765 * packet/s got queued while we were
3766 * processing. Wakeup srs worker
3767 */
3768 cv_signal(&mac_srs->srs_async);
3769 }
3770 mutex_exit(&mac_srs->srs_lock);
3771
3772 if (cookie == NULL)
3773 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3774
3775 return (cookie);
3776 }
3777
3778 /*
3779 * mac_tx_fanout_mode
3780 *
3781 * In this mode, the SRS will have access to multiple Tx rings to send
3782 * the packet out. The fanout hint that is passed as an argument is
3783 * used to find an appropriate ring to fanout the traffic. Each Tx
3784 * ring, in turn, will have a soft ring associated with it. If a Tx
3785 * ring runs out of Tx desc's the returned packet will be queued in
3786 * the soft ring associated with that Tx ring. The srs itself will not
3787 * queue any packets.
3788 */
3789
3790 #define MAC_TX_SOFT_RING_PROCESS(chain) { \
3791 index = COMPUTE_INDEX(hash, mac_srs->srs_tx_ring_count), \
3792 softring = mac_srs->srs_tx_soft_rings[index]; \
3793 cookie = mac_tx_soft_ring_process(softring, chain, flag, ret_mp); \
3794 DTRACE_PROBE2(tx__fanout, uint64_t, hash, uint_t, index); \
3795 }
3796
3797 static mac_tx_cookie_t
3798 mac_tx_fanout_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3799 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3800 {
3801 mac_soft_ring_t *softring;
3802 uint64_t hash;
3803 uint_t index;
3804 mac_tx_cookie_t cookie = NULL;
3805
3806 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
3807 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT);
3808 if (fanout_hint != 0) {
3809 /*
3810 * The hint is specified by the caller, simply pass the
3811 * whole chain to the soft ring.
3812 */
3813 hash = HASH_HINT(fanout_hint);
3814 MAC_TX_SOFT_RING_PROCESS(mp_chain);
3815 } else {
3816 mblk_t *last_mp, *cur_mp, *sub_chain;
3817 uint64_t last_hash = 0;
3818 uint_t media = mac_srs->srs_mcip->mci_mip->mi_info.mi_media;
3819
3820 /*
3821 * Compute the hash from the contents (headers) of the
3822 * packets of the mblk chain. Split the chains into
3823 * subchains of the same conversation.
3824 *
3825 * Since there may be more than one ring used for
3826 * sub-chains of the same call, and since the caller
3827 * does not maintain per conversation state since it
3828 * passed a zero hint, unsent subchains will be
3829 * dropped.
3830 */
3831
3832 flag |= MAC_DROP_ON_NO_DESC;
3833 ret_mp = NULL;
3834
3835 ASSERT(ret_mp == NULL);
3836
3837 sub_chain = NULL;
3838 last_mp = NULL;
3839
3840 for (cur_mp = mp_chain; cur_mp != NULL;
3841 cur_mp = cur_mp->b_next) {
3842 hash = mac_pkt_hash(media, cur_mp, MAC_PKT_HASH_L4,
3843 B_TRUE);
3844 if (last_hash != 0 && hash != last_hash) {
3845 /*
3846 * Starting a different subchain, send current
3847 * chain out.
3848 */
3849 ASSERT(last_mp != NULL);
3850 last_mp->b_next = NULL;
3851 MAC_TX_SOFT_RING_PROCESS(sub_chain);
3852 sub_chain = NULL;
3853 }
3854
3855 /* add packet to subchain */
3856 if (sub_chain == NULL)
3857 sub_chain = cur_mp;
3858 last_mp = cur_mp;
3859 last_hash = hash;
3860 }
3861
3862 if (sub_chain != NULL) {
3863 /* send last subchain */
3864 ASSERT(last_mp != NULL);
3865 last_mp->b_next = NULL;
3866 MAC_TX_SOFT_RING_PROCESS(sub_chain);
3867 }
3868
3869 cookie = NULL;
3870 }
3871
3872 return (cookie);
3873 }
3874
3875 /*
3876 * mac_tx_bw_mode
3877 *
3878 * In the bandwidth mode, Tx srs will allow packets to go down to Tx ring
3879 * only if bw is available. Otherwise the packets will be queued in
3880 * SRS. If the SRS has multiple Tx rings, then packets will get fanned
3881 * out to a Tx rings.
3882 */
3883 static mac_tx_cookie_t
3884 mac_tx_bw_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3885 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3886 {
3887 int cnt, sz;
3888 mblk_t *tail;
3889 mac_tx_cookie_t cookie = NULL;
3890 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3891 clock_t now;
3892
3893 ASSERT(TX_BANDWIDTH_MODE(mac_srs));
3894 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
3895 mutex_enter(&mac_srs->srs_lock);
3896 if (mac_srs->srs_bw->mac_bw_limit == 0) {
3897 /*
3898 * zero bandwidth, no traffic is sent: drop the packets,
3899 * or return the whole chain if the caller requests all
3900 * unsent packets back.
3901 */
3902 if (flag & MAC_TX_NO_ENQUEUE) {
3903 cookie = (mac_tx_cookie_t)mac_srs;
3904 *ret_mp = mp_chain;
3905 } else {
3906 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3907 }
3908 mutex_exit(&mac_srs->srs_lock);
3909 return (cookie);
3910 } else if ((mac_srs->srs_first != NULL) ||
3911 (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
3912 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
3913 fanout_hint, ret_mp);
3914 mutex_exit(&mac_srs->srs_lock);
3915 return (cookie);
3916 }
3917 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3918 now = ddi_get_lbolt();
3919 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3920 mac_srs->srs_bw->mac_bw_curr_time = now;
3921 mac_srs->srs_bw->mac_bw_used = 0;
3922 } else if (mac_srs->srs_bw->mac_bw_used >
3923 mac_srs->srs_bw->mac_bw_limit) {
3924 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3925 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3926 mp_chain, tail, cnt, sz);
3927 /*
3928 * Wakeup worker thread. Note that worker
3929 * thread has to be woken up so that it
3930 * can fire up the timer to be woken up
3931 * on the next tick. Also once
3932 * BW_ENFORCED is set, it can only be
3933 * reset by srs_worker thread. Until then
3934 * all packets will get queued up in SRS
3935 * and hence this this code path won't be
3936 * entered until BW_ENFORCED is reset.
3937 */
3938 cv_signal(&mac_srs->srs_async);
3939 mutex_exit(&mac_srs->srs_lock);
3940 return (cookie);
3941 }
3942
3943 mac_srs->srs_bw->mac_bw_used += sz;
3944 mutex_exit(&mac_srs->srs_lock);
3945
3946 if (srs_tx->st_mode == SRS_TX_BW_FANOUT) {
3947 mac_soft_ring_t *softring;
3948 uint_t indx, hash;
3949
3950 hash = HASH_HINT(fanout_hint);
3951 indx = COMPUTE_INDEX(hash,
3952 mac_srs->srs_tx_ring_count);
3953 softring = mac_srs->srs_tx_soft_rings[indx];
3954 return (mac_tx_soft_ring_process(softring, mp_chain, flag,
3955 ret_mp));
3956 } else if (srs_tx->st_mode == SRS_TX_BW_AGGR) {
3957 return (mac_tx_aggr_mode(mac_srs, mp_chain,
3958 fanout_hint, flag, ret_mp));
3959 } else {
3960 mac_tx_stats_t stats;
3961
3962 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3963 mp_chain, &stats);
3964
3965 if (mp_chain != NULL) {
3966 mutex_enter(&mac_srs->srs_lock);
3967 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3968 if (mac_srs->srs_bw->mac_bw_used > sz)
3969 mac_srs->srs_bw->mac_bw_used -= sz;
3970 else
3971 mac_srs->srs_bw->mac_bw_used = 0;
3972 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
3973 fanout_hint, ret_mp);
3974 mutex_exit(&mac_srs->srs_lock);
3975 return (cookie);
3976 }
3977 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3978
3979 return (NULL);
3980 }
3981 }
3982
3983 /*
3984 * mac_tx_aggr_mode
3985 *
3986 * This routine invokes an aggr function, aggr_find_tx_ring(), to find
3987 * a (pseudo) Tx ring belonging to a port on which the packet has to
3988 * be sent. aggr_find_tx_ring() first finds the outgoing port based on
3989 * L2/L3/L4 policy and then uses the fanout_hint passed to it to pick
3990 * a Tx ring from the selected port.
3991 *
3992 * Note that a port can be deleted from the aggregation. In such a case,
3993 * the aggregation layer first separates the port from the rest of the
3994 * ports making sure that port (and thus any Tx rings associated with
3995 * it) won't get selected in the call to aggr_find_tx_ring() function.
3996 * Later calls are made to mac_group_rem_ring() passing pseudo Tx ring
3997 * handles one by one which in turn will quiesce the Tx SRS and remove
3998 * the soft ring associated with the pseudo Tx ring. Unlike Rx side
3999 * where a cookie is used to protect against mac_rx_ring() calls on
4000 * rings that have been removed, no such cookie is needed on the Tx
4001 * side as the pseudo Tx ring won't be available anymore to
4002 * aggr_find_tx_ring() once the port has been removed.
4003 */
4004 static mac_tx_cookie_t
4005 mac_tx_aggr_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
4006 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
4007 {
4008 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
4009 mac_tx_ring_fn_t find_tx_ring_fn;
4010 mac_ring_handle_t ring = NULL;
4011 void *arg;
4012 mac_soft_ring_t *sringp;
4013
4014 find_tx_ring_fn = srs_tx->st_capab_aggr.mca_find_tx_ring_fn;
4015 arg = srs_tx->st_capab_aggr.mca_arg;
4016 if (find_tx_ring_fn(arg, mp_chain, fanout_hint, &ring) == NULL)
4017 return (NULL);
4018 sringp = srs_tx->st_soft_rings[((mac_ring_t *)ring)->mr_index];
4019 return (mac_tx_soft_ring_process(sringp, mp_chain, flag, ret_mp));
4020 }
4021
4022 void
4023 mac_tx_invoke_callbacks(mac_client_impl_t *mcip, mac_tx_cookie_t cookie)
4024 {
4025 mac_cb_t *mcb;
4026 mac_tx_notify_cb_t *mtnfp;
4027
4028 /* Wakeup callback registered clients */
4029 MAC_CALLBACK_WALKER_INC(&mcip->mci_tx_notify_cb_info);
4030 for (mcb = mcip->mci_tx_notify_cb_list; mcb != NULL;
4031 mcb = mcb->mcb_nextp) {
4032 mtnfp = (mac_tx_notify_cb_t *)mcb->mcb_objp;
4033 mtnfp->mtnf_fn(mtnfp->mtnf_arg, cookie);
4034 }
4035 MAC_CALLBACK_WALKER_DCR(&mcip->mci_tx_notify_cb_info,
4036 &mcip->mci_tx_notify_cb_list);
4037 }
4038
4039 /* ARGSUSED */
4040 void
4041 mac_tx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
4042 {
4043 mblk_t *head, *tail;
4044 size_t sz;
4045 uint32_t tx_mode;
4046 uint_t saved_pkt_count;
4047 mac_tx_stats_t stats;
4048 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
4049 clock_t now;
4050
4051 saved_pkt_count = 0;
4052 ASSERT(mutex_owned(&mac_srs->srs_lock));
4053 ASSERT(!(mac_srs->srs_state & SRS_PROC));
4054
4055 mac_srs->srs_state |= SRS_PROC;
4056
4057 tx_mode = srs_tx->st_mode;
4058 if (tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_SERIALIZE) {
4059 if (mac_srs->srs_first != NULL) {
4060 head = mac_srs->srs_first;
4061 tail = mac_srs->srs_last;
4062 saved_pkt_count = mac_srs->srs_count;
4063 mac_srs->srs_first = NULL;
4064 mac_srs->srs_last = NULL;
4065 mac_srs->srs_count = 0;
4066 mutex_exit(&mac_srs->srs_lock);
4067
4068 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
4069 head, &stats);
4070
4071 mutex_enter(&mac_srs->srs_lock);
4072 if (head != NULL) {
4073 /* Device out of tx desc, set block */
4074 if (head->b_next == NULL)
4075 VERIFY(head == tail);
4076 tail->b_next = mac_srs->srs_first;
4077 mac_srs->srs_first = head;
4078 mac_srs->srs_count +=
4079 (saved_pkt_count - stats.mts_opackets);
4080 if (mac_srs->srs_last == NULL)
4081 mac_srs->srs_last = tail;
4082 MAC_TX_SRS_BLOCK(mac_srs, head);
4083 } else {
4084 srs_tx->st_woken_up = B_FALSE;
4085 SRS_TX_STATS_UPDATE(mac_srs, &stats);
4086 }
4087 }
4088 } else if (tx_mode == SRS_TX_BW) {
4089 /*
4090 * We are here because the timer fired and we have some data
4091 * to tranmit. Also mac_tx_srs_worker should have reset
4092 * SRS_BW_ENFORCED flag
4093 */
4094 ASSERT(!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED));
4095 head = tail = mac_srs->srs_first;
4096 while (mac_srs->srs_first != NULL) {
4097 tail = mac_srs->srs_first;
4098 tail->b_prev = NULL;
4099 mac_srs->srs_first = tail->b_next;
4100 if (mac_srs->srs_first == NULL)
4101 mac_srs->srs_last = NULL;
4102 mac_srs->srs_count--;
4103 sz = msgdsize(tail);
4104 mac_srs->srs_size -= sz;
4105 saved_pkt_count++;
4106 MAC_TX_UPDATE_BW_INFO(mac_srs, sz);
4107
4108 if (mac_srs->srs_bw->mac_bw_used <
4109 mac_srs->srs_bw->mac_bw_limit)
4110 continue;
4111
4112 now = ddi_get_lbolt();
4113 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
4114 mac_srs->srs_bw->mac_bw_curr_time = now;
4115 mac_srs->srs_bw->mac_bw_used = sz;
4116 continue;
4117 }
4118 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
4119 break;
4120 }
4121
4122 ASSERT((head == NULL && tail == NULL) ||
4123 (head != NULL && tail != NULL));
4124 if (tail != NULL) {
4125 tail->b_next = NULL;
4126 mutex_exit(&mac_srs->srs_lock);
4127
4128 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
4129 head, &stats);
4130
4131 mutex_enter(&mac_srs->srs_lock);
4132 if (head != NULL) {
4133 uint_t size_sent;
4134
4135 /* Device out of tx desc, set block */
4136 if (head->b_next == NULL)
4137 VERIFY(head == tail);
4138 tail->b_next = mac_srs->srs_first;
4139 mac_srs->srs_first = head;
4140 mac_srs->srs_count +=
4141 (saved_pkt_count - stats.mts_opackets);
4142 if (mac_srs->srs_last == NULL)
4143 mac_srs->srs_last = tail;
4144 size_sent = sz - stats.mts_obytes;
4145 mac_srs->srs_size += size_sent;
4146 mac_srs->srs_bw->mac_bw_sz += size_sent;
4147 if (mac_srs->srs_bw->mac_bw_used > size_sent) {
4148 mac_srs->srs_bw->mac_bw_used -=
4149 size_sent;
4150 } else {
4151 mac_srs->srs_bw->mac_bw_used = 0;
4152 }
4153 MAC_TX_SRS_BLOCK(mac_srs, head);
4154 } else {
4155 srs_tx->st_woken_up = B_FALSE;
4156 SRS_TX_STATS_UPDATE(mac_srs, &stats);
4157 }
4158 }
4159 } else if (tx_mode == SRS_TX_BW_FANOUT || tx_mode == SRS_TX_BW_AGGR) {
4160 mblk_t *prev;
4161 uint64_t hint;
4162
4163 /*
4164 * We are here because the timer fired and we
4165 * have some quota to tranmit.
4166 */
4167 prev = NULL;
4168 head = tail = mac_srs->srs_first;
4169 while (mac_srs->srs_first != NULL) {
4170 tail = mac_srs->srs_first;
4171 mac_srs->srs_first = tail->b_next;
4172 if (mac_srs->srs_first == NULL)
4173 mac_srs->srs_last = NULL;
4174 mac_srs->srs_count--;
4175 sz = msgdsize(tail);
4176 mac_srs->srs_size -= sz;
4177 mac_srs->srs_bw->mac_bw_used += sz;
4178 if (prev == NULL)
4179 hint = (ulong_t)tail->b_prev;
4180 if (hint != (ulong_t)tail->b_prev) {
4181 prev->b_next = NULL;
4182 mutex_exit(&mac_srs->srs_lock);
4183 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
4184 head = tail;
4185 hint = (ulong_t)tail->b_prev;
4186 mutex_enter(&mac_srs->srs_lock);
4187 }
4188
4189 prev = tail;
4190 tail->b_prev = NULL;
4191 if (mac_srs->srs_bw->mac_bw_used <
4192 mac_srs->srs_bw->mac_bw_limit)
4193 continue;
4194
4195 now = ddi_get_lbolt();
4196 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
4197 mac_srs->srs_bw->mac_bw_curr_time = now;
4198 mac_srs->srs_bw->mac_bw_used = 0;
4199 continue;
4200 }
4201 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
4202 break;
4203 }
4204 ASSERT((head == NULL && tail == NULL) ||
4205 (head != NULL && tail != NULL));
4206 if (tail != NULL) {
4207 tail->b_next = NULL;
4208 mutex_exit(&mac_srs->srs_lock);
4209 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
4210 mutex_enter(&mac_srs->srs_lock);
4211 }
4212 }
4213 /*
4214 * SRS_TX_FANOUT case not considered here because packets
4215 * won't be queued in the SRS for this case. Packets will
4216 * be sent directly to soft rings underneath and if there
4217 * is any queueing at all, it would be in Tx side soft
4218 * rings.
4219 */
4220
4221 /*
4222 * When srs_count becomes 0, reset SRS_TX_HIWAT and
4223 * SRS_TX_WAKEUP_CLIENT and wakeup registered clients.
4224 */
4225 if (mac_srs->srs_count == 0 && (mac_srs->srs_state &
4226 (SRS_TX_HIWAT | SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED))) {
4227 mac_client_impl_t *mcip = mac_srs->srs_mcip;
4228 boolean_t wakeup_required = B_FALSE;
4229
4230 if (mac_srs->srs_state &
4231 (SRS_TX_HIWAT|SRS_TX_WAKEUP_CLIENT)) {
4232 wakeup_required = B_TRUE;
4233 }
4234 mac_srs->srs_state &= ~(SRS_TX_HIWAT |
4235 SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED);
4236 mutex_exit(&mac_srs->srs_lock);
4237 if (wakeup_required) {
4238 mac_tx_invoke_callbacks(mcip, (mac_tx_cookie_t)mac_srs);
4239 /*
4240 * If the client is not the primary MAC client, then we
4241 * need to send the notification to the clients upper
4242 * MAC, i.e. mci_upper_mip.
4243 */
4244 mac_tx_notify(mcip->mci_upper_mip != NULL ?
4245 mcip->mci_upper_mip : mcip->mci_mip);
4246 }
4247 mutex_enter(&mac_srs->srs_lock);
4248 }
4249 mac_srs->srs_state &= ~SRS_PROC;
4250 }
4251
4252 /*
4253 * Given a packet, get the flow_entry that identifies the flow
4254 * to which that packet belongs. The flow_entry will contain
4255 * the transmit function to be used to send the packet. If the
4256 * function returns NULL, the packet should be sent using the
4257 * underlying NIC.
4258 */
4259 static flow_entry_t *
4260 mac_tx_classify(mac_impl_t *mip, mblk_t *mp)
4261 {
4262 flow_entry_t *flent = NULL;
4263 mac_client_impl_t *mcip;
4264 int err;
4265
4266 /*
4267 * Do classification on the packet.
4268 */
4269 err = mac_flow_lookup(mip->mi_flow_tab, mp, FLOW_OUTBOUND, &flent);
4270 if (err != 0)
4271 return (NULL);
4272
4273 /*
4274 * This flent might just be an additional one on the MAC client,
4275 * i.e. for classification purposes (different fdesc), however
4276 * the resources, SRS et. al., are in the mci_flent, so if
4277 * this isn't the mci_flent, we need to get it.
4278 */
4279 if ((mcip = flent->fe_mcip) != NULL && mcip->mci_flent != flent) {
4280 FLOW_REFRELE(flent);
4281 flent = mcip->mci_flent;
4282 FLOW_TRY_REFHOLD(flent, err);
4283 if (err != 0)
4284 return (NULL);
4285 }
4286
4287 return (flent);
4288 }
4289
4290 /*
4291 * This macro is only meant to be used by mac_tx_send().
4292 */
4293 #define CHECK_VID_AND_ADD_TAG(mp) { \
4294 if (vid_check) { \
4295 int err = 0; \
4296 \
4297 MAC_VID_CHECK(src_mcip, (mp), err); \
4298 if (err != 0) { \
4299 freemsg((mp)); \
4300 (mp) = next; \
4301 oerrors++; \
4302 continue; \
4303 } \
4304 } \
4305 if (add_tag) { \
4306 (mp) = mac_add_vlan_tag((mp), 0, vid); \
4307 if ((mp) == NULL) { \
4308 (mp) = next; \
4309 oerrors++; \
4310 continue; \
4311 } \
4312 } \
4313 }
4314
4315 mblk_t *
4316 mac_tx_send(mac_client_handle_t mch, mac_ring_handle_t ring, mblk_t *mp_chain,
4317 mac_tx_stats_t *stats)
4318 {
4319 mac_client_impl_t *src_mcip = (mac_client_impl_t *)mch;
4320 mac_impl_t *mip = src_mcip->mci_mip;
4321 uint_t obytes = 0, opackets = 0, oerrors = 0;
4322 mblk_t *mp = NULL, *next;
4323 boolean_t vid_check, add_tag;
4324 uint16_t vid = 0;
4325
4326 if (mip->mi_nclients > 1) {
4327 vid_check = MAC_VID_CHECK_NEEDED(src_mcip);
4328 add_tag = MAC_TAG_NEEDED(src_mcip);
4329 if (add_tag)
4330 vid = mac_client_vid(mch);
4331 } else {
4332 ASSERT(mip->mi_nclients == 1);
4333 vid_check = add_tag = B_FALSE;
4334 }
4335
4336 /*
4337 * Fastpath: if there's only one client, we simply send
4338 * the packet down to the underlying NIC.
4339 */
4340 if (mip->mi_nactiveclients == 1) {
4341 DTRACE_PROBE2(fastpath,
4342 mac_client_impl_t *, src_mcip, mblk_t *, mp_chain);
4343
4344 mp = mp_chain;
4345 while (mp != NULL) {
4346 next = mp->b_next;
4347 mp->b_next = NULL;
4348 opackets++;
4349 obytes += (mp->b_cont == NULL ? MBLKL(mp) :
4350 msgdsize(mp));
4351
4352 CHECK_VID_AND_ADD_TAG(mp);
4353 MAC_TX(mip, ring, mp, src_mcip);
4354
4355 /*
4356 * If the driver is out of descriptors and does a
4357 * partial send it will return a chain of unsent
4358 * mblks. Adjust the accounting stats.
4359 */
4360 if (mp != NULL) {
4361 opackets--;
4362 obytes -= msgdsize(mp);
4363 mp->b_next = next;
4364 break;
4365 }
4366 mp = next;
4367 }
4368 goto done;
4369 }
4370
4371 /*
4372 * No fastpath, we either have more than one MAC client
4373 * defined on top of the same MAC, or one or more MAC
4374 * client promiscuous callbacks.
4375 */
4376 DTRACE_PROBE3(slowpath, mac_client_impl_t *,
4377 src_mcip, int, mip->mi_nclients, mblk_t *, mp_chain);
4378
4379 mp = mp_chain;
4380 while (mp != NULL) {
4381 flow_entry_t *dst_flow_ent;
4382 void *flow_cookie;
4383 size_t pkt_size;
4384 mblk_t *mp1;
4385
4386 next = mp->b_next;
4387 mp->b_next = NULL;
4388 opackets++;
4389 pkt_size = (mp->b_cont == NULL ? MBLKL(mp) : msgdsize(mp));
4390 obytes += pkt_size;
4391 CHECK_VID_AND_ADD_TAG(mp);
4392
4393 /*
4394 * Find the destination.
4395 */
4396 dst_flow_ent = mac_tx_classify(mip, mp);
4397
4398 if (dst_flow_ent != NULL) {
4399 size_t hdrsize;
4400 int err = 0;
4401
4402 if (mip->mi_info.mi_nativemedia == DL_ETHER) {
4403 struct ether_vlan_header *evhp =
4404 (struct ether_vlan_header *)mp->b_rptr;
4405
4406 if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN)
4407 hdrsize = sizeof (*evhp);
4408 else
4409 hdrsize = sizeof (struct ether_header);
4410 } else {
4411 mac_header_info_t mhi;
4412
4413 err = mac_header_info((mac_handle_t)mip,
4414 mp, &mhi);
4415 if (err == 0)
4416 hdrsize = mhi.mhi_hdrsize;
4417 }
4418
4419 /*
4420 * Got a matching flow. It's either another
4421 * MAC client, or a broadcast/multicast flow.
4422 * Make sure the packet size is within the
4423 * allowed size. If not drop the packet and
4424 * move to next packet.
4425 */
4426 if (err != 0 ||
4427 (pkt_size - hdrsize) > mip->mi_sdu_max) {
4428 oerrors++;
4429 DTRACE_PROBE2(loopback__drop, size_t, pkt_size,
4430 mblk_t *, mp);
4431 freemsg(mp);
4432 mp = next;
4433 FLOW_REFRELE(dst_flow_ent);
4434 continue;
4435 }
4436 flow_cookie = mac_flow_get_client_cookie(dst_flow_ent);
4437 if (flow_cookie != NULL) {
4438 /*
4439 * The vnic_bcast_send function expects
4440 * to receive the sender MAC client
4441 * as value for arg2.
4442 */
4443 mac_bcast_send(flow_cookie, src_mcip, mp,
4444 B_TRUE);
4445 } else {
4446 /*
4447 * loopback the packet to a local MAC
4448 * client. We force a context switch
4449 * if both source and destination MAC
4450 * clients are used by IP, i.e.
4451 * bypass is set.
4452 */
4453 boolean_t do_switch;
4454 mac_client_impl_t *dst_mcip =
4455 dst_flow_ent->fe_mcip;
4456
4457 /*
4458 * Check if there are promiscuous mode
4459 * callbacks defined. This check is
4460 * done here in the 'else' case and
4461 * not in other cases because this
4462 * path is for local loopback
4463 * communication which does not go
4464 * through MAC_TX(). For paths that go
4465 * through MAC_TX(), the promisc_list
4466 * check is done inside the MAC_TX()
4467 * macro.
4468 */
4469 if (mip->mi_promisc_list != NULL)
4470 mac_promisc_dispatch(mip, mp, src_mcip);
4471
4472 do_switch = ((src_mcip->mci_state_flags &
4473 dst_mcip->mci_state_flags &
4474 MCIS_CLIENT_POLL_CAPABLE) != 0);
4475
4476 if ((mp1 = mac_fix_cksum(mp)) != NULL) {
4477 (dst_flow_ent->fe_cb_fn)(
4478 dst_flow_ent->fe_cb_arg1,
4479 dst_flow_ent->fe_cb_arg2,
4480 mp1, do_switch);
4481 }
4482 }
4483 FLOW_REFRELE(dst_flow_ent);
4484 } else {
4485 /*
4486 * Unknown destination, send via the underlying
4487 * NIC.
4488 */
4489 MAC_TX(mip, ring, mp, src_mcip);
4490 if (mp != NULL) {
4491 /*
4492 * Adjust for the last packet that
4493 * could not be transmitted
4494 */
4495 opackets--;
4496 obytes -= pkt_size;
4497 mp->b_next = next;
4498 break;
4499 }
4500 }
4501 mp = next;
4502 }
4503
4504 done:
4505 stats->mts_obytes = obytes;
4506 stats->mts_opackets = opackets;
4507 stats->mts_oerrors = oerrors;
4508 return (mp);
4509 }
4510
4511 /*
4512 * mac_tx_srs_ring_present
4513 *
4514 * Returns whether the specified ring is part of the specified SRS.
4515 */
4516 boolean_t
4517 mac_tx_srs_ring_present(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
4518 {
4519 int i;
4520 mac_soft_ring_t *soft_ring;
4521
4522 if (srs->srs_tx.st_arg2 == tx_ring)
4523 return (B_TRUE);
4524
4525 for (i = 0; i < srs->srs_tx_ring_count; i++) {
4526 soft_ring = srs->srs_tx_soft_rings[i];
4527 if (soft_ring->s_ring_tx_arg2 == tx_ring)
4528 return (B_TRUE);
4529 }
4530
4531 return (B_FALSE);
4532 }
4533
4534 /*
4535 * mac_tx_srs_get_soft_ring
4536 *
4537 * Returns the TX soft ring associated with the given ring, if present.
4538 */
4539 mac_soft_ring_t *
4540 mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
4541 {
4542 int i;
4543 mac_soft_ring_t *soft_ring;
4544
4545 if (srs->srs_tx.st_arg2 == tx_ring)
4546 return (NULL);
4547
4548 for (i = 0; i < srs->srs_tx_ring_count; i++) {
4549 soft_ring = srs->srs_tx_soft_rings[i];
4550 if (soft_ring->s_ring_tx_arg2 == tx_ring)
4551 return (soft_ring);
4552 }
4553
4554 return (NULL);
4555 }
4556
4557 /*
4558 * mac_tx_srs_wakeup
4559 *
4560 * Called when Tx desc become available. Wakeup the appropriate worker
4561 * thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the
4562 * state field.
4563 */
4564 void
4565 mac_tx_srs_wakeup(mac_soft_ring_set_t *mac_srs, mac_ring_handle_t ring)
4566 {
4567 int i;
4568 mac_soft_ring_t *sringp;
4569 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
4570
4571 mutex_enter(&mac_srs->srs_lock);
4572 /*
4573 * srs_tx_ring_count == 0 is the single ring mode case. In
4574 * this mode, there will not be Tx soft rings associated
4575 * with the SRS.
4576 */
4577 if (!MAC_TX_SOFT_RINGS(mac_srs)) {
4578 if (srs_tx->st_arg2 == ring &&
4579 mac_srs->srs_state & SRS_TX_BLOCKED) {
4580 mac_srs->srs_state &= ~SRS_TX_BLOCKED;
4581 srs_tx->st_stat.mts_unblockcnt++;
4582 cv_signal(&mac_srs->srs_async);
4583 }
4584 /*
4585 * A wakeup can come before tx_srs_drain() could
4586 * grab srs lock and set SRS_TX_BLOCKED. So
4587 * always set woken_up flag when we come here.
4588 */
4589 srs_tx->st_woken_up = B_TRUE;
4590 mutex_exit(&mac_srs->srs_lock);
4591 return;
4592 }
4593
4594 /*
4595 * If you are here, it is for FANOUT, BW_FANOUT,
4596 * AGGR_MODE or AGGR_BW_MODE case
4597 */
4598 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
4599 sringp = mac_srs->srs_tx_soft_rings[i];
4600 mutex_enter(&sringp->s_ring_lock);
4601 if (sringp->s_ring_tx_arg2 == ring) {
4602 if (sringp->s_ring_state & S_RING_BLOCK) {
4603 sringp->s_ring_state &= ~S_RING_BLOCK;
4604 sringp->s_st_stat.mts_unblockcnt++;
4605 cv_signal(&sringp->s_ring_async);
4606 }
4607 sringp->s_ring_tx_woken_up = B_TRUE;
4608 }
4609 mutex_exit(&sringp->s_ring_lock);
4610 }
4611 mutex_exit(&mac_srs->srs_lock);
4612 }
4613
4614 /*
4615 * Once the driver is done draining, send a MAC_NOTE_TX notification to unleash
4616 * the blocked clients again.
4617 */
4618 void
4619 mac_tx_notify(mac_impl_t *mip)
4620 {
4621 i_mac_notify(mip, MAC_NOTE_TX);
4622 }
4623
4624 /*
4625 * RX SOFTRING RELATED FUNCTIONS
4626 *
4627 * These functions really belong in mac_soft_ring.c and here for
4628 * a short period.
4629 */
4630
4631 #define SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
4632 /* \
4633 * Enqueue our mblk chain. \
4634 */ \
4635 ASSERT(MUTEX_HELD(&(ringp)->s_ring_lock)); \
4636 \
4637 if ((ringp)->s_ring_last != NULL) \
4638 (ringp)->s_ring_last->b_next = (mp); \
4639 else \
4640 (ringp)->s_ring_first = (mp); \
4641 (ringp)->s_ring_last = (tail); \
4642 (ringp)->s_ring_count += (cnt); \
4643 ASSERT((ringp)->s_ring_count > 0); \
4644 if ((ringp)->s_ring_type & ST_RING_BW_CTL) { \
4645 (ringp)->s_ring_size += sz; \
4646 } \
4647 }
4648
4649 /*
4650 * Default entry point to deliver a packet chain to a MAC client.
4651 * If the MAC client has flows, do the classification with these
4652 * flows as well.
4653 */
4654 /* ARGSUSED */
4655 void
4656 mac_rx_deliver(void *arg1, mac_resource_handle_t mrh, mblk_t *mp_chain,
4657 mac_header_info_t *arg3)
4658 {
4659 mac_client_impl_t *mcip = arg1;
4660
4661 if (mcip->mci_nvids == 1 &&
4662 !(mcip->mci_state_flags & MCIS_STRIP_DISABLE)) {
4663 /*
4664 * If the client has exactly one VID associated with it
4665 * and striping of VLAN header is not disabled,
4666 * remove the VLAN tag from the packet before
4667 * passing it on to the client's receive callback.
4668 * Note that this needs to be done after we dispatch
4669 * the packet to the promiscuous listeners of the
4670 * client, since they expect to see the whole
4671 * frame including the VLAN headers.
4672 */
4673 mp_chain = mac_strip_vlan_tag_chain(mp_chain);
4674 }
4675
4676 mcip->mci_rx_fn(mcip->mci_rx_arg, mrh, mp_chain, B_FALSE);
4677 }
4678
4679 /*
4680 * mac_rx_soft_ring_process
4681 *
4682 * process a chain for a given soft ring. The number of packets queued
4683 * in the SRS and its associated soft rings (including this one) is
4684 * very small (tracked by srs_poll_pkt_cnt), then allow the entering
4685 * thread (interrupt or poll thread) to do inline processing. This
4686 * helps keep the latency down under low load.
4687 *
4688 * The proc and arg for each mblk is already stored in the mblk in
4689 * appropriate places.
4690 */
4691 /* ARGSUSED */
4692 void
4693 mac_rx_soft_ring_process(mac_client_impl_t *mcip, mac_soft_ring_t *ringp,
4694 mblk_t *mp_chain, mblk_t *tail, int cnt, size_t sz)
4695 {
4696 mac_direct_rx_t proc;
4697 void *arg1;
4698 mac_resource_handle_t arg2;
4699 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4700
4701 ASSERT(ringp != NULL);
4702 ASSERT(mp_chain != NULL);
4703 ASSERT(tail != NULL);
4704 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4705
4706 mutex_enter(&ringp->s_ring_lock);
4707 ringp->s_ring_total_inpkt += cnt;
4708 ringp->s_ring_total_rbytes += sz;
4709 if ((mac_srs->srs_rx.sr_poll_pkt_cnt <= 1) &&
4710 !(ringp->s_ring_type & ST_RING_WORKER_ONLY)) {
4711 /* If on processor or blanking on, then enqueue and return */
4712 if (ringp->s_ring_state & S_RING_BLANK ||
4713 ringp->s_ring_state & S_RING_PROC) {
4714 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4715 mutex_exit(&ringp->s_ring_lock);
4716 return;
4717 }
4718 proc = ringp->s_ring_rx_func;
4719 arg1 = ringp->s_ring_rx_arg1;
4720 arg2 = ringp->s_ring_rx_arg2;
4721 /*
4722 * See if anything is already queued. If we are the
4723 * first packet, do inline processing else queue the
4724 * packet and do the drain.
4725 */
4726 if (ringp->s_ring_first == NULL) {
4727 /*
4728 * Fast-path, ok to process and nothing queued.
4729 */
4730 ringp->s_ring_run = curthread;
4731 ringp->s_ring_state |= (S_RING_PROC);
4732
4733 mutex_exit(&ringp->s_ring_lock);
4734
4735 /*
4736 * We are the chain of 1 packet so
4737 * go through this fast path.
4738 */
4739 ASSERT(mp_chain->b_next == NULL);
4740
4741 (*proc)(arg1, arg2, mp_chain, NULL);
4742
4743 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4744 /*
4745 * If we have a soft ring set which is doing
4746 * bandwidth control, we need to decrement
4747 * srs_size and count so it the SRS can have a
4748 * accurate idea of what is the real data
4749 * queued between SRS and its soft rings. We
4750 * decrement the counters only when the packet
4751 * gets processed by both SRS and the soft ring.
4752 */
4753 mutex_enter(&mac_srs->srs_lock);
4754 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
4755 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
4756 mutex_exit(&mac_srs->srs_lock);
4757
4758 mutex_enter(&ringp->s_ring_lock);
4759 ringp->s_ring_run = NULL;
4760 ringp->s_ring_state &= ~S_RING_PROC;
4761 if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
4762 cv_signal(&ringp->s_ring_client_cv);
4763
4764 if ((ringp->s_ring_first == NULL) ||
4765 (ringp->s_ring_state & S_RING_BLANK)) {
4766 /*
4767 * We processed inline our packet and
4768 * nothing new has arrived or our
4769 * receiver doesn't want to receive
4770 * any packets. We are done.
4771 */
4772 mutex_exit(&ringp->s_ring_lock);
4773 return;
4774 }
4775 } else {
4776 SOFT_RING_ENQUEUE_CHAIN(ringp,
4777 mp_chain, tail, cnt, sz);
4778 }
4779
4780 /*
4781 * We are here because either we couldn't do inline
4782 * processing (because something was already
4783 * queued), or we had a chain of more than one
4784 * packet, or something else arrived after we were
4785 * done with inline processing.
4786 */
4787 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
4788 ASSERT(ringp->s_ring_first != NULL);
4789
4790 ringp->s_ring_drain_func(ringp);
4791 mutex_exit(&ringp->s_ring_lock);
4792 return;
4793 } else {
4794 /* ST_RING_WORKER_ONLY case */
4795 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4796 mac_soft_ring_worker_wakeup(ringp);
4797 mutex_exit(&ringp->s_ring_lock);
4798 }
4799 }
4800
4801 /*
4802 * TX SOFTRING RELATED FUNCTIONS
4803 *
4804 * These functions really belong in mac_soft_ring.c and here for
4805 * a short period.
4806 */
4807
4808 #define TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
4809 ASSERT(MUTEX_HELD(&ringp->s_ring_lock)); \
4810 ringp->s_ring_state |= S_RING_ENQUEUED; \
4811 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); \
4812 }
4813
4814 /*
4815 * mac_tx_sring_queued
4816 *
4817 * When we are out of transmit descriptors and we already have a
4818 * queue that exceeds hiwat (or the client called us with
4819 * MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the
4820 * soft ring pointer as the opaque cookie for the client enable
4821 * flow control.
4822 */
4823 static mac_tx_cookie_t
4824 mac_tx_sring_enqueue(mac_soft_ring_t *ringp, mblk_t *mp_chain, uint16_t flag,
4825 mblk_t **ret_mp)
4826 {
4827 int cnt;
4828 size_t sz;
4829 mblk_t *tail;
4830 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4831 mac_tx_cookie_t cookie = NULL;
4832 boolean_t wakeup_worker = B_TRUE;
4833
4834 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
4835 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
4836 if (flag & MAC_DROP_ON_NO_DESC) {
4837 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
4838 /* increment freed stats */
4839 ringp->s_ring_drops += cnt;
4840 cookie = (mac_tx_cookie_t)ringp;
4841 } else {
4842 if (ringp->s_ring_first != NULL)
4843 wakeup_worker = B_FALSE;
4844
4845 if (flag & MAC_TX_NO_ENQUEUE) {
4846 /*
4847 * If QUEUED is not set, queue the packet
4848 * and let mac_tx_soft_ring_drain() set
4849 * the TX_BLOCKED bit for the reasons
4850 * explained above. Otherwise, return the
4851 * mblks.
4852 */
4853 if (wakeup_worker) {
4854 TX_SOFT_RING_ENQUEUE_CHAIN(ringp,
4855 mp_chain, tail, cnt, sz);
4856 } else {
4857 ringp->s_ring_state |= S_RING_WAKEUP_CLIENT;
4858 cookie = (mac_tx_cookie_t)ringp;
4859 *ret_mp = mp_chain;
4860 }
4861 } else {
4862 boolean_t enqueue = B_TRUE;
4863
4864 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
4865 /*
4866 * flow-controlled. Store ringp in cookie
4867 * so that it can be returned as
4868 * mac_tx_cookie_t to client
4869 */
4870 ringp->s_ring_state |= S_RING_TX_HIWAT;
4871 cookie = (mac_tx_cookie_t)ringp;
4872 ringp->s_ring_hiwat_cnt++;
4873 if (ringp->s_ring_count >
4874 ringp->s_ring_tx_max_q_cnt) {
4875 /* increment freed stats */
4876 ringp->s_ring_drops += cnt;
4877 /*
4878 * b_prev may be set to the fanout hint
4879 * hence can't use freemsg directly
4880 */
4881 mac_pkt_drop(NULL, NULL,
4882 mp_chain, B_FALSE);
4883 DTRACE_PROBE1(tx_queued_hiwat,
4884 mac_soft_ring_t *, ringp);
4885 enqueue = B_FALSE;
4886 }
4887 }
4888 if (enqueue) {
4889 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain,
4890 tail, cnt, sz);
4891 }
4892 }
4893 if (wakeup_worker)
4894 cv_signal(&ringp->s_ring_async);
4895 }
4896 return (cookie);
4897 }
4898
4899
4900 /*
4901 * mac_tx_soft_ring_process
4902 *
4903 * This routine is called when fanning out outgoing traffic among
4904 * multipe Tx rings.
4905 * Note that a soft ring is associated with a h/w Tx ring.
4906 */
4907 mac_tx_cookie_t
4908 mac_tx_soft_ring_process(mac_soft_ring_t *ringp, mblk_t *mp_chain,
4909 uint16_t flag, mblk_t **ret_mp)
4910 {
4911 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4912 int cnt;
4913 size_t sz;
4914 mblk_t *tail;
4915 mac_tx_cookie_t cookie = NULL;
4916
4917 ASSERT(ringp != NULL);
4918 ASSERT(mp_chain != NULL);
4919 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4920 /*
4921 * The following modes can come here: SRS_TX_BW_FANOUT,
4922 * SRS_TX_FANOUT, SRS_TX_AGGR, SRS_TX_BW_AGGR.
4923 */
4924 ASSERT(MAC_TX_SOFT_RINGS(mac_srs));
4925 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
4926 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT ||
4927 mac_srs->srs_tx.st_mode == SRS_TX_AGGR ||
4928 mac_srs->srs_tx.st_mode == SRS_TX_BW_AGGR);
4929
4930 if (ringp->s_ring_type & ST_RING_WORKER_ONLY) {
4931 /* Serialization mode */
4932
4933 mutex_enter(&ringp->s_ring_lock);
4934 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
4935 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
4936 flag, ret_mp);
4937 mutex_exit(&ringp->s_ring_lock);
4938 return (cookie);
4939 }
4940 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
4941 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4942 if (ringp->s_ring_state & (S_RING_BLOCK | S_RING_PROC)) {
4943 /*
4944 * If ring is blocked due to lack of Tx
4945 * descs, just return. Worker thread
4946 * will get scheduled when Tx desc's
4947 * become available.
4948 */
4949 mutex_exit(&ringp->s_ring_lock);
4950 return (cookie);
4951 }
4952 mac_soft_ring_worker_wakeup(ringp);
4953 mutex_exit(&ringp->s_ring_lock);
4954 return (cookie);
4955 } else {
4956 /* Default fanout mode */
4957 /*
4958 * S_RING_BLOCKED is set when underlying NIC runs
4959 * out of Tx descs and messages start getting
4960 * queued. It won't get reset until
4961 * tx_srs_drain() completely drains out the
4962 * messages.
4963 */
4964 mac_tx_stats_t stats;
4965
4966 if (ringp->s_ring_state & S_RING_ENQUEUED) {
4967 /* Tx descs/resources not available */
4968 mutex_enter(&ringp->s_ring_lock);
4969 if (ringp->s_ring_state & S_RING_ENQUEUED) {
4970 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
4971 flag, ret_mp);
4972 mutex_exit(&ringp->s_ring_lock);
4973 return (cookie);
4974 }
4975 /*
4976 * While we were computing mblk count, the
4977 * flow control condition got relieved.
4978 * Continue with the transmission.
4979 */
4980 mutex_exit(&ringp->s_ring_lock);
4981 }
4982
4983 mp_chain = mac_tx_send(ringp->s_ring_tx_arg1,
4984 ringp->s_ring_tx_arg2, mp_chain, &stats);
4985
4986 /*
4987 * Multiple threads could be here sending packets.
4988 * Under such conditions, it is not possible to
4989 * automically set S_RING_BLOCKED bit to indicate
4990 * out of tx desc condition. To atomically set
4991 * this, we queue the returned packet and do
4992 * the setting of S_RING_BLOCKED in
4993 * mac_tx_soft_ring_drain().
4994 */
4995 if (mp_chain != NULL) {
4996 mutex_enter(&ringp->s_ring_lock);
4997 cookie =
4998 mac_tx_sring_enqueue(ringp, mp_chain, flag, ret_mp);
4999 mutex_exit(&ringp->s_ring_lock);
5000 return (cookie);
5001 }
5002 SRS_TX_STATS_UPDATE(mac_srs, &stats);
5003 SOFTRING_TX_STATS_UPDATE(ringp, &stats);
5004
5005 return (NULL);
5006 }
5007 }