1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved.
14 * Copyright 2019 Joyent, Inc.
15 * Copyright 2017 Tegile Systems, Inc. All rights reserved.
16 */
17
18 /*
19 * Please see i40e_main.c for an introduction to the device driver, its layout,
20 * and more.
21 */
22
23 #ifndef _I40E_SW_H
24 #define _I40E_SW_H
25
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29
30 #include <sys/types.h>
31 #include <sys/conf.h>
32 #include <sys/debug.h>
33 #include <sys/stropts.h>
34 #include <sys/stream.h>
35 #include <sys/strsun.h>
36 #include <sys/strlog.h>
37 #include <sys/kmem.h>
38 #include <sys/stat.h>
39 #include <sys/kstat.h>
40 #include <sys/modctl.h>
41 #include <sys/errno.h>
42 #include <sys/dlpi.h>
43 #include <sys/mac_provider.h>
44 #include <sys/mac_ether.h>
45 #include <sys/vlan.h>
46 #include <sys/ddi.h>
47 #include <sys/sunddi.h>
48 #include <sys/pci.h>
49 #include <sys/pcie.h>
50 #include <sys/sdt.h>
51 #include <sys/ethernet.h>
52 #include <sys/pattr.h>
53 #include <sys/strsubr.h>
54 #include <sys/netlb.h>
55 #include <sys/random.h>
56 #include <inet/common.h>
57 #include <inet/tcp.h>
58 #include <inet/ip.h>
59 #include <inet/mi.h>
60 #include <inet/nd.h>
61 #include <netinet/udp.h>
62 #include <netinet/sctp.h>
63 #include <sys/bitmap.h>
64 #include <sys/cpuvar.h>
65 #include <sys/ddifm.h>
66 #include <sys/fm/protocol.h>
67 #include <sys/fm/util.h>
68 #include <sys/disp.h>
69 #include <sys/fm/io/ddi.h>
70 #include <sys/list.h>
71 #include <sys/debug.h>
72 #include <sys/sdt.h>
73 #include "i40e_type.h"
74 #include "i40e_osdep.h"
75 #include "i40e_prototype.h"
76 #include "i40e_xregs.h"
77
78 #define I40E_MODULE_NAME "i40e"
79
80 #define I40E_ADAPTER_REGSET 1
81
82 /*
83 * Configuration constants. Note that the hardware defines a minimum bound of 32
84 * descriptors and requires that the programming of the descriptor lengths be
85 * aligned in units of 32 descriptors.
86 */
87 #define I40E_MIN_TX_RING_SIZE 64
88 #define I40E_MAX_TX_RING_SIZE 4096
89 #define I40E_DEF_TX_RING_SIZE 1024
90
91 #define I40E_MIN_RX_RING_SIZE 64
92 #define I40E_MAX_RX_RING_SIZE 4096
93 #define I40E_DEF_RX_RING_SIZE 1024
94
95 #define I40E_DESC_ALIGN 32
96
97 /*
98 * Sizes used for asynchronous processing of the adminq. We allocate a fixed
99 * size buffer for each instance of the device during attach time, rather than
100 * allocating and freeing one during interrupt processing.
101 *
102 * We also define the descriptor size of the admin queue here.
103 */
104 #define I40E_ADMINQ_BUFSZ 4096
105 #define I40E_MAX_ADMINQ_SIZE 1024
106 #define I40E_DEF_ADMINQ_SIZE 256
107
108 /*
109 * Note, while the min and maximum values are based upon the sizing of the ring
110 * itself, the default is taken from ixgbe without much thought. It's basically
111 * been cargo culted. See i40e_transceiver.c for a bit more information.
112 */
113 #define I40E_MIN_RX_LIMIT_PER_INTR 16
114 #define I40E_MAX_RX_LIMIT_PER_INTR 4096
115 #define I40E_DEF_RX_LIMIT_PER_INTR 256
116
117 /*
118 * Valid MTU ranges. Note that the XL710's maximum payload is actually 9728.
119 * However, we need to adjust for the ETHERFCSL (4 bytes) and the Ethernet VLAN
120 * header size (18 bytes) to get the actual maximum frame we can use. If
121 * different adapters end up with different sizes, we should make this value a
122 * bit more dynamic.
123 */
124 #define I40E_MAX_MTU 9706
125 #define I40E_MIN_MTU ETHERMIN
126 #define I40E_DEF_MTU ETHERMTU
127
128 /*
129 * Interrupt throttling related values. Interrupt throttling values are defined
130 * in two microsecond increments. Note that a value of zero basically says do no
131 * ITR activity. A helpful way to think about these is that setting the ITR to a
132 * value will allow a certain number of interrupts per second.
133 *
134 * Our default values for RX allow 20k interrupts per second while our default
135 * values for TX allow for 5k interrupts per second. For other class interrupts,
136 * we limit ourselves to a rate of 2k/s.
137 */
138 #define I40E_MIN_ITR 0x0000
139 #define I40E_MAX_ITR 0x0FF0
140 #define I40E_DEF_RX_ITR 0x0019
141 #define I40E_DEF_TX_ITR 0x0064
142 #define I40E_DEF_OTHER_ITR 0x00FA
143
144 /*
145 * Indexes into the three ITR registers that we have.
146 */
147 typedef enum i40e_itr_index {
148 I40E_ITR_INDEX_RX = 0x0,
149 I40E_ITR_INDEX_TX = 0x1,
150 I40E_ITR_INDEX_OTHER = 0x2,
151 I40E_ITR_INDEX_NONE = 0x3
152 } i40e_itr_index_t;
153
154 /*
155 * The hardware claims to support LSO up to 256 KB, but due to the limitations
156 * imposed by the IP header for non-jumbo frames, we cap it at 64 KB.
157 */
158 #define I40E_LSO_MAXLEN (64 * 1024)
159
160 #define I40E_CYCLIC_PERIOD NANOSEC /* 1 second */
161 #define I40E_DRAIN_RX_WAIT (500 * MILLISEC) /* In us */
162
163 /*
164 * All the other queue types for are defined by the common code. However, this
165 * is the constant to indicate that it's terminated.
166 */
167 #define I40E_QUEUE_TYPE_EOL 0x7FF
168
169 /*
170 * See the comments in i40e_transceiver.c as to the purpose of this value and
171 * how it's used to ensure that the IP header is eventually aligned when it's
172 * received by the OS.
173 */
174 #define I40E_BUF_IPHDR_ALIGNMENT 2
175
176 /*
177 * The XL710 controller has a total of eight buffers available for the
178 * transmission of any single frame. This is defined in 8.4.1 - Transmit
179 * Packet in System Memory.
180 */
181 #define I40E_TX_MAX_COOKIE 8
182
183 /*
184 * An LSO frame can be as large as 64KB, so we allow a DMA bind to span more
185 * cookies than a non-LSO frame. The key here to is to select a value such
186 * that once the HW has chunked up the LSO frame into MSS-sized segments that no
187 * single segment spans more than 8 cookies (see comments for
188 * I40E_TX_MAX_COOKIE)
189 */
190 #define I40E_TX_LSO_MAX_COOKIE 32
191
192 /*
193 * Sizing to determine the amount of available descriptors at which we'll
194 * consider ourselves blocked. Also, when we have these available, we'll then
195 * consider ourselves available to transmit to MAC again. Strictly speaking, the
196 * MAX is based on the ring size. The default sizing is based on ixgbe.
197 */
198 #define I40E_MIN_TX_BLOCK_THRESH I40E_TX_MAX_COOKIE
199 #define I40E_DEF_TX_BLOCK_THRESH I40E_MIN_TX_BLOCK_THRESH
200
201 /*
202 * Sizing for DMA thresholds. These are used to indicate whether or not we
203 * should perform a bcopy or a DMA binding of a given message block. The range
204 * allows for setting things such that we'll always do a bcopy (a high value) or
205 * always perform a DMA binding (a low value).
206 */
207 #define I40E_MIN_RX_DMA_THRESH 0
208 #define I40E_DEF_RX_DMA_THRESH 256
209 #define I40E_MAX_RX_DMA_THRESH INT32_MAX
210
211 #define I40E_MIN_TX_DMA_THRESH 0
212 #define I40E_DEF_TX_DMA_THRESH 256
213 #define I40E_MAX_TX_DMA_THRESH INT32_MAX
214
215 /*
216 * The max size of each individual tx buffer is 16KB - 1.
217 * See table 8-17
218 */
219 #define I40E_MAX_TX_BUFSZ 0x0000000000003FFFull
220
221 /*
222 * Resource sizing counts. There are various aspects of hardware where we may
223 * have some variable number of elements that we need to handle. Such as the
224 * hardware capabilities and switch capacities. We cannot know a priori how many
225 * elements to do, so instead we take a starting guess and then will grow it up
226 * to an upper bound on a number of elements, to limit memory consumption in
227 * case of a hardware bug.
228 */
229 #define I40E_HW_CAP_DEFAULT 40
230 #define I40E_SWITCH_CAP_DEFAULT 25
231
232 /*
233 * Host Memory Context related constants.
234 */
235 #define I40E_HMC_RX_CTX_UNIT 128
236 #define I40E_HMC_RX_DBUFF_MIN 1024
237 #define I40E_HMC_RX_DBUFF_MAX (16 * 1024 - 128)
238 #define I40E_HMC_RX_DTYPE_NOSPLIT 0
239 #define I40E_HMC_RX_DSIZE_32BYTE 1
240 #define I40E_HMC_RX_CRCSTRIP_ENABLE 1
241 #define I40E_HMC_RX_FC_DISABLE 0
242 #define I40E_HMC_RX_L2TAGORDER 1
243 #define I40E_HMC_RX_HDRSPLIT_DISABLE 0
244 #define I40E_HMC_RX_INVLAN_DONTSTRIP 0
245 #define I40E_HMC_RX_TPH_DISABLE 0
246 #define I40E_HMC_RX_LOWRXQ_NOINTR 0
247 #define I40E_HMC_RX_PREFENA 1
248
249 #define I40E_HMC_TX_CTX_UNIT 128
250 #define I40E_HMC_TX_NEW_CONTEXT 1
251 #define I40E_HMC_TX_FC_DISABLE 0
252 #define I40E_HMC_TX_TS_DISABLE 0
253 #define I40E_HMC_TX_FD_DISABLE 0
254 #define I40E_HMC_TX_ALT_VLAN_DISABLE 0
255 #define I40E_HMC_TX_WB_ENABLE 1
256 #define I40E_HMC_TX_TPH_DISABLE 0
257
258 /*
259 * This defines the error mask that we care about from rx descriptors. Currently
260 * we're only concerned with the general errors and oversize errors.
261 */
262 #define I40E_RX_ERR_BITS ((1 << I40E_RX_DESC_ERROR_RXE_SHIFT) | \
263 (1 << I40E_RX_DESC_ERROR_OVERSIZE_SHIFT))
264
265 /*
266 * Property sizing macros for firmware versions, etc. They need to be large
267 * enough to hold 32-bit quantities transformed to strings as %d.%d or %x.
268 */
269 #define I40E_DDI_PROP_LEN 64
270
271 /*
272 * Place an artificial limit on the max number of groups. The X710
273 * series supports up to 384 VSIs to be partitioned across PFs as the
274 * driver sees fit. But until we support more interrupts this seems
275 * like a good place to start.
276 */
277 #define I40E_GROUP_MAX 32
278
279 #define I40E_GROUP_NOMSIX 1
280 #define I40E_TRQPAIR_NOMSIX 1
281
282 /*
283 * It seems reasonable to cast this to void because the only reason that we
284 * should be getting a DDI_FAILURE is due to the fact that we specify addresses
285 * out of range. Because we specify no offset or address, it shouldn't happen.
286 */
287 #ifdef DEBUG
288 #define I40E_DMA_SYNC(handle, flag) ASSERT0(ddi_dma_sync( \
289 (handle)->dmab_dma_handle, 0, 0, \
290 (flag)))
291 #else /* !DEBUG */
292 #define I40E_DMA_SYNC(handle, flag) ((void) ddi_dma_sync( \
293 (handle)->dmab_dma_handle, 0, 0, \
294 (flag)))
295 #endif /* DEBUG */
296
297 /*
298 * Constants related to ring startup and teardown. These refer to the amount of
299 * time that we're willing to wait for a ring to spin up and spin down.
300 */
301 #define I40E_RING_WAIT_NTRIES 10
302 #define I40E_RING_WAIT_PAUSE 10 /* ms */
303
304 /*
305 * Printed Board Assembly (PBA) length. These are derived from Table 6-2.
306 */
307 #define I40E_PBANUM_LENGTH 12
308 #define I40E_PBANUM_STRLEN 13
309
310 /*
311 * Define the maximum number of queues for a traffic class. These values come
312 * from the 'Number and offset of queue pairs per TCs' section of the 'Add VSI
313 * Command Buffer' table. For the 710 controller family this is table 7-62
314 * (r2.5) and for the 722 this is table 38-216 (r2.0).
315 */
316 #define I40E_710_MAX_TC_QUEUES 64
317 #define I40E_722_MAX_TC_QUEUES 128
318
319 /*
320 * Define the size of the HLUT table size. The HLUT table can either be 128 or
321 * 512 bytes. We always set the table size to be 512 bytes in i40e_chip_start().
322 * Note, this should not be confused with the common code's macro
323 * I40E_HASH_LUT_SIZE_512 which is the bit pattern needed to tell the card to
324 * use a 512 byte HLUT.
325 */
326 #define I40E_HLUT_TABLE_SIZE 512
327
328 /*
329 * Bit flags for attach_progress
330 */
331 typedef enum i40e_attach_state {
332 I40E_ATTACH_PCI_CONFIG = 0x0001, /* PCI config setup */
333 I40E_ATTACH_REGS_MAP = 0x0002, /* Registers mapped */
334 I40E_ATTACH_PROPS = 0x0004, /* Properties initialized */
335 I40E_ATTACH_ALLOC_INTR = 0x0008, /* Interrupts allocated */
336 I40E_ATTACH_ALLOC_RINGSLOCKS = 0x0010, /* Rings & locks allocated */
337 I40E_ATTACH_ADD_INTR = 0x0020, /* Intr handlers added */
338 I40E_ATTACH_COMMON_CODE = 0x0040, /* Intel code initialized */
339 I40E_ATTACH_INIT = 0x0080, /* Device initialized */
340 I40E_ATTACH_STATS = 0x0200, /* Kstats created */
341 I40E_ATTACH_MAC = 0x0800, /* MAC registered */
342 I40E_ATTACH_ENABLE_INTR = 0x1000, /* DDI interrupts enabled */
343 I40E_ATTACH_FM_INIT = 0x2000, /* FMA initialized */
344 I40E_ATTACH_LINK_TIMER = 0x4000, /* link check timer */
345 } i40e_attach_state_t;
346
347
348 /*
349 * State flags that what's going on in in the device. Some of these state flags
350 * indicate some aspirational work that needs to happen in the driver.
351 *
352 * I40E_UNKNOWN: The device has yet to be started.
353 * I40E_INITIALIZED: The device has been fully attached.
354 * I40E_STARTED: The device has come out of the GLDV3 start routine.
355 * I40E_SUSPENDED: The device is suspended and I/O among other things
356 * should not occur. This happens because of an actual
357 * DDI_SUSPEND or interrupt adjustments.
358 * I40E_STALL: The tx stall detection logic has found a stall.
359 * I40E_OVERTEMP: The device has encountered a temperature alarm.
360 * I40E_INTR_ADJUST: Our interrupts are being manipulated and therefore we
361 * shouldn't be manipulating their state.
362 * I40E_ERROR: We've detected an FM error and degraded the device.
363 */
364 typedef enum i40e_state {
365 I40E_UNKNOWN = 0x00,
366 I40E_INITIALIZED = 0x01,
367 I40E_STARTED = 0x02,
368 I40E_SUSPENDED = 0x04,
369 I40E_STALL = 0x08,
370 I40E_OVERTEMP = 0x20,
371 I40E_INTR_ADJUST = 0x40,
372 I40E_ERROR = 0x80
373 } i40e_state_t;
374
375
376 /*
377 * Definitions for common Intel things that we use and some slightly more usable
378 * names.
379 */
380 typedef struct i40e_hw i40e_hw_t;
381 typedef struct i40e_aqc_switch_resource_alloc_element_resp i40e_switch_rsrc_t;
382
383 /*
384 * Handles and addresses of DMA buffers.
385 */
386 typedef struct i40e_dma_buffer {
387 caddr_t dmab_address; /* Virtual address */
388 uint64_t dmab_dma_address; /* DMA (Hardware) address */
389 ddi_acc_handle_t dmab_acc_handle; /* Data access handle */
390 ddi_dma_handle_t dmab_dma_handle; /* DMA handle */
391 size_t dmab_size; /* Buffer size */
392 size_t dmab_len; /* Data length in the buffer */
393 } i40e_dma_buffer_t;
394
395 /*
396 * RX Control Block
397 */
398 typedef struct i40e_rx_control_block {
399 mblk_t *rcb_mp;
400 uint32_t rcb_ref;
401 i40e_dma_buffer_t rcb_dma;
402 frtn_t rcb_free_rtn;
403 struct i40e_rx_data *rcb_rxd;
404 } i40e_rx_control_block_t;
405
406 typedef enum {
407 I40E_TX_NONE,
408 I40E_TX_COPY,
409 I40E_TX_DMA,
410 I40E_TX_DESC,
411 } i40e_tx_type_t;
412
413 typedef struct i40e_tx_desc i40e_tx_desc_t;
414 typedef struct i40e_tx_context_desc i40e_tx_context_desc_t;
415 typedef union i40e_32byte_rx_desc i40e_rx_desc_t;
416
417 struct i40e_dma_bind_info {
418 caddr_t dbi_paddr;
419 size_t dbi_len;
420 };
421
422 typedef struct i40e_tx_control_block {
423 struct i40e_tx_control_block *tcb_next;
424 mblk_t *tcb_mp;
425 i40e_tx_type_t tcb_type;
426 ddi_dma_handle_t tcb_dma_handle;
427 ddi_dma_handle_t tcb_lso_dma_handle;
428 i40e_dma_buffer_t tcb_dma;
429 struct i40e_dma_bind_info *tcb_bind_info;
430 uint_t tcb_bind_ncookies;
431 boolean_t tcb_used_lso;
432 } i40e_tx_control_block_t;
433
434 /*
435 * Receive ring data (used below).
436 */
437 typedef struct i40e_rx_data {
438 struct i40e *rxd_i40e;
439
440 /*
441 * RX descriptor ring definitions
442 */
443 i40e_dma_buffer_t rxd_desc_area; /* DMA buffer of rx desc ring */
444 i40e_rx_desc_t *rxd_desc_ring; /* Rx desc ring */
445 uint32_t rxd_desc_next; /* Index of next rx desc */
446
447 /*
448 * RX control block list definitions
449 */
450 kmutex_t rxd_free_lock; /* Lock to protect free data */
451 i40e_rx_control_block_t *rxd_rcb_area; /* Array of control blocks */
452 i40e_rx_control_block_t **rxd_work_list; /* Work list of rcbs */
453 i40e_rx_control_block_t **rxd_free_list; /* Free list of rcbs */
454 uint32_t rxd_rcb_free; /* Number of free rcbs */
455
456 /*
457 * RX software ring settings
458 */
459 uint32_t rxd_ring_size; /* Rx descriptor ring size */
460 uint32_t rxd_free_list_size; /* Rx free list size */
461
462 /*
463 * RX outstanding data. This is used to keep track of outstanding loaned
464 * descriptors after we've shut down receiving information. Note these
465 * are protected by the i40e_t`i40e_rx_pending_lock.
466 */
467 uint32_t rxd_rcb_pending;
468 boolean_t rxd_shutdown;
469 } i40e_rx_data_t;
470
471 /*
472 * Structures for unicast and multicast addresses. Note that we keep the VSI id
473 * around for unicast addresses, since they may belong to different VSIs.
474 * However, since all multicast addresses belong to the default VSI, we don't
475 * duplicate that information.
476 */
477 typedef struct i40e_uaddr {
478 uint8_t iua_mac[ETHERADDRL];
479 int iua_vsi;
480 } i40e_uaddr_t;
481
482 typedef struct i40e_maddr {
483 uint8_t ima_mac[ETHERADDRL];
484 } i40e_maddr_t;
485
486 /*
487 * Collection of RX statistics on a given queue.
488 */
489 typedef struct i40e_rxq_stat {
490 /*
491 * The i40e hardware does not maintain statistics on a per-ring basis,
492 * only on a per-PF and per-VSI level. As such, to satisfy the GLDv3, we
493 * need to maintain our own stats for packets and bytes.
494 */
495 kstat_named_t irxs_bytes; /* Bytes in on queue */
496 kstat_named_t irxs_packets; /* Packets in on queue */
497
498 /*
499 * The following set of stats cover non-checksum data path issues.
500 */
501 kstat_named_t irxs_rx_desc_error; /* Error bit set on desc */
502 kstat_named_t irxs_rx_copy_nomem; /* allocb failure for copy */
503 kstat_named_t irxs_rx_intr_limit; /* Hit i40e_rx_limit_per_intr */
504 kstat_named_t irxs_rx_bind_norcb; /* No replacement rcb free */
505 kstat_named_t irxs_rx_bind_nomp; /* No mblk_t in bind rcb */
506
507 /*
508 * The following set of statistics covers rx checksum related activity.
509 * These are all primarily set in i40e_rx_hcksum. If rx checksum
510 * activity is disabled, then these should all be zero.
511 */
512 kstat_named_t irxs_hck_v4hdrok; /* Valid IPv4 Header */
513 kstat_named_t irxs_hck_l4hdrok; /* Valid L4 Header */
514 kstat_named_t irxs_hck_unknown; /* !pinfo.known */
515 kstat_named_t irxs_hck_nol3l4p; /* Missing L3L4P bit in desc */
516 kstat_named_t irxs_hck_iperr; /* IPE error bit set */
517 kstat_named_t irxs_hck_eiperr; /* EIPE error bit set */
518 kstat_named_t irxs_hck_l4err; /* L4E error bit set */
519 kstat_named_t irxs_hck_v6skip; /* IPv6 case hw fails on */
520 kstat_named_t irxs_hck_set; /* Total times we set cksum */
521 kstat_named_t irxs_hck_miss; /* Times with zero cksum bits */
522 } i40e_rxq_stat_t;
523
524 /*
525 * Collection of TX Statistics on a given queue
526 */
527 typedef struct i40e_txq_stat {
528 kstat_named_t itxs_bytes; /* Bytes out on queue */
529 kstat_named_t itxs_packets; /* Packets out on queue */
530 kstat_named_t itxs_descriptors; /* Descriptors issued */
531 kstat_named_t itxs_recycled; /* Descriptors reclaimed */
532 kstat_named_t itxs_force_copy; /* non-TSO force copy */
533 kstat_named_t itxs_tso_force_copy; /* TSO force copy */
534 /*
535 * Various failure conditions.
536 */
537 kstat_named_t itxs_hck_meoifail; /* ether offload failures */
538 kstat_named_t itxs_hck_nol2info; /* Missing l2 info */
539 kstat_named_t itxs_hck_nol3info; /* Missing l3 info */
540 kstat_named_t itxs_hck_nol4info; /* Missing l4 info */
541 kstat_named_t itxs_hck_badl3; /* Not IPv4/IPv6 */
542 kstat_named_t itxs_hck_badl4; /* Bad L4 Paylaod */
543 kstat_named_t itxs_lso_nohck; /* Missing offloads for LSO */
544 kstat_named_t itxs_bind_fails; /* DMA bind failures */
545
546 kstat_named_t itxs_err_notcb; /* No tcb's available */
547 kstat_named_t itxs_err_nodescs; /* No tcb's available */
548 kstat_named_t itxs_err_context; /* Total context failures */
549
550 kstat_named_t itxs_num_unblocked; /* Number of MAC unblocks */
551 } i40e_txq_stat_t;
552
553 /*
554 * An instance of an XL710 transmit/receive queue pair. This currently
555 * represents a combination of both a transmit and receive ring, though they
556 * should really be split apart into separate logical structures. Unfortunately,
557 * during initial work we mistakenly joined them together.
558 */
559 typedef struct i40e_trqpair {
560 struct i40e *itrq_i40e;
561
562 /* Receive-side structures. */
563 kmutex_t itrq_rx_lock;
564 mac_ring_handle_t itrq_macrxring; /* Receive ring handle. */
565 i40e_rx_data_t *itrq_rxdata; /* Receive ring rx data. */
566 uint64_t itrq_rxgen; /* Generation number for mac/GLDv3. */
567 uint32_t itrq_index; /* Queue index in the PF */
568 uint32_t itrq_rx_intrvec; /* Receive interrupt vector. */
569 boolean_t itrq_intr_poll; /* True when polling */
570
571 /* Receive-side stats. */
572 i40e_rxq_stat_t itrq_rxstat;
573 kstat_t *itrq_rxkstat;
574
575 /* Transmit-side structures. */
576 kmutex_t itrq_tx_lock;
577 mac_ring_handle_t itrq_mactxring; /* Transmit ring handle. */
578 uint32_t itrq_tx_intrvec; /* Transmit interrupt vector. */
579 boolean_t itrq_tx_blocked; /* Does MAC think we're blocked? */
580
581 /*
582 * TX data sizing
583 */
584 uint32_t itrq_tx_ring_size;
585 uint32_t itrq_tx_free_list_size;
586
587 /*
588 * TX descriptor ring data
589 */
590 i40e_dma_buffer_t itrq_desc_area; /* DMA buffer of tx desc ring */
591 i40e_tx_desc_t *itrq_desc_ring; /* TX Desc ring */
592 volatile uint32_t *itrq_desc_wbhead; /* TX write-back index */
593 uint32_t itrq_desc_head; /* Last index hw freed */
594 uint32_t itrq_desc_tail; /* Index of next free desc */
595 uint32_t itrq_desc_free; /* Number of free descriptors */
596
597 /*
598 * TX control block (tcb) data
599 */
600 kmutex_t itrq_tcb_lock;
601 i40e_tx_control_block_t *itrq_tcb_area; /* Array of control blocks */
602 i40e_tx_control_block_t **itrq_tcb_work_list; /* In use tcb */
603 i40e_tx_control_block_t **itrq_tcb_free_list; /* Available tcb */
604 uint32_t itrq_tcb_free; /* Count of free tcb */
605
606 /* Transmit-side stats. */
607 i40e_txq_stat_t itrq_txstat;
608 kstat_t *itrq_txkstat;
609
610 } i40e_trqpair_t;
611
612 /*
613 * VSI statistics.
614 *
615 * This mirrors the i40e_eth_stats structure but transforms it into a kstat.
616 * Note that the stock statistic structure also includes entries for tx
617 * discards. However, this is not actually implemented for the VSI (see Table
618 * 7-221), hence why we don't include the member which would always have a value
619 * of zero. This choice was made to minimize confusion to someone looking at
620 * these, as a value of zero does not necessarily equate to the fact that it's
621 * not implemented.
622 */
623 typedef struct i40e_vsi_stats {
624 uint64_t ivs_rx_bytes; /* gorc */
625 uint64_t ivs_rx_unicast; /* uprc */
626 uint64_t ivs_rx_multicast; /* mprc */
627 uint64_t ivs_rx_broadcast; /* bprc */
628 uint64_t ivs_rx_discards; /* rdpc */
629 uint64_t ivs_rx_unknown_protocol; /* rupp */
630 uint64_t ivs_tx_bytes; /* gotc */
631 uint64_t ivs_tx_unicast; /* uptc */
632 uint64_t ivs_tx_multicast; /* mptc */
633 uint64_t ivs_tx_broadcast; /* bptc */
634 uint64_t ivs_tx_errors; /* tepc */
635 } i40e_vsi_stats_t;
636
637 typedef struct i40e_vsi_kstats {
638 kstat_named_t ivk_rx_bytes;
639 kstat_named_t ivk_rx_unicast;
640 kstat_named_t ivk_rx_multicast;
641 kstat_named_t ivk_rx_broadcast;
642 kstat_named_t ivk_rx_discards;
643 kstat_named_t ivk_rx_unknown_protocol;
644 kstat_named_t ivk_tx_bytes;
645 kstat_named_t ivk_tx_unicast;
646 kstat_named_t ivk_tx_multicast;
647 kstat_named_t ivk_tx_broadcast;
648 kstat_named_t ivk_tx_errors;
649 } i40e_vsi_kstats_t;
650
651 /*
652 * For pf statistics, we opt not to use the standard statistics as defined by
653 * the Intel common code. This also currently combines statistics that are
654 * global across the entire device.
655 */
656 typedef struct i40e_pf_stats {
657 uint64_t ips_rx_bytes; /* gorc */
658 uint64_t ips_rx_unicast; /* uprc */
659 uint64_t ips_rx_multicast; /* mprc */
660 uint64_t ips_rx_broadcast; /* bprc */
661 uint64_t ips_tx_bytes; /* gotc */
662 uint64_t ips_tx_unicast; /* uptc */
663 uint64_t ips_tx_multicast; /* mptc */
664 uint64_t ips_tx_broadcast; /* bptc */
665
666 uint64_t ips_rx_size_64; /* prc64 */
667 uint64_t ips_rx_size_127; /* prc127 */
668 uint64_t ips_rx_size_255; /* prc255 */
669 uint64_t ips_rx_size_511; /* prc511 */
670 uint64_t ips_rx_size_1023; /* prc1023 */
671 uint64_t ips_rx_size_1522; /* prc1522 */
672 uint64_t ips_rx_size_9522; /* prc9522 */
673
674 uint64_t ips_tx_size_64; /* ptc64 */
675 uint64_t ips_tx_size_127; /* ptc127 */
676 uint64_t ips_tx_size_255; /* ptc255 */
677 uint64_t ips_tx_size_511; /* ptc511 */
678 uint64_t ips_tx_size_1023; /* ptc1023 */
679 uint64_t ips_tx_size_1522; /* ptc1522 */
680 uint64_t ips_tx_size_9522; /* ptc9522 */
681
682 uint64_t ips_link_xon_rx; /* lxonrxc */
683 uint64_t ips_link_xoff_rx; /* lxoffrxc */
684 uint64_t ips_link_xon_tx; /* lxontxc */
685 uint64_t ips_link_xoff_tx; /* lxofftxc */
686 uint64_t ips_priority_xon_rx[8]; /* pxonrxc[8] */
687 uint64_t ips_priority_xoff_rx[8]; /* pxoffrxc[8] */
688 uint64_t ips_priority_xon_tx[8]; /* pxontxc[8] */
689 uint64_t ips_priority_xoff_tx[8]; /* pxofftxc[8] */
690 uint64_t ips_priority_xon_2_xoff[8]; /* rxon2offcnt[8] */
691
692 uint64_t ips_crc_errors; /* crcerrs */
693 uint64_t ips_illegal_bytes; /* illerrc */
694 uint64_t ips_mac_local_faults; /* mlfc */
695 uint64_t ips_mac_remote_faults; /* mrfc */
696 uint64_t ips_rx_length_errors; /* rlec */
697 uint64_t ips_rx_undersize; /* ruc */
698 uint64_t ips_rx_fragments; /* rfc */
699 uint64_t ips_rx_oversize; /* roc */
700 uint64_t ips_rx_jabber; /* rjc */
701 uint64_t ips_rx_discards; /* rdpc */
702 uint64_t ips_rx_vm_discards; /* ldpc */
703 uint64_t ips_rx_short_discards; /* mspdc */
704 uint64_t ips_tx_dropped_link_down; /* tdold */
705 uint64_t ips_rx_unknown_protocol; /* rupp */
706 uint64_t ips_rx_err1; /* rxerr1 */
707 uint64_t ips_rx_err2; /* rxerr2 */
708 } i40e_pf_stats_t;
709
710 typedef struct i40e_pf_kstats {
711 kstat_named_t ipk_rx_bytes; /* gorc */
712 kstat_named_t ipk_rx_unicast; /* uprc */
713 kstat_named_t ipk_rx_multicast; /* mprc */
714 kstat_named_t ipk_rx_broadcast; /* bprc */
715 kstat_named_t ipk_tx_bytes; /* gotc */
716 kstat_named_t ipk_tx_unicast; /* uptc */
717 kstat_named_t ipk_tx_multicast; /* mptc */
718 kstat_named_t ipk_tx_broadcast; /* bptc */
719
720 kstat_named_t ipk_rx_size_64; /* prc64 */
721 kstat_named_t ipk_rx_size_127; /* prc127 */
722 kstat_named_t ipk_rx_size_255; /* prc255 */
723 kstat_named_t ipk_rx_size_511; /* prc511 */
724 kstat_named_t ipk_rx_size_1023; /* prc1023 */
725 kstat_named_t ipk_rx_size_1522; /* prc1522 */
726 kstat_named_t ipk_rx_size_9522; /* prc9522 */
727
728 kstat_named_t ipk_tx_size_64; /* ptc64 */
729 kstat_named_t ipk_tx_size_127; /* ptc127 */
730 kstat_named_t ipk_tx_size_255; /* ptc255 */
731 kstat_named_t ipk_tx_size_511; /* ptc511 */
732 kstat_named_t ipk_tx_size_1023; /* ptc1023 */
733 kstat_named_t ipk_tx_size_1522; /* ptc1522 */
734 kstat_named_t ipk_tx_size_9522; /* ptc9522 */
735
736 kstat_named_t ipk_link_xon_rx; /* lxonrxc */
737 kstat_named_t ipk_link_xoff_rx; /* lxoffrxc */
738 kstat_named_t ipk_link_xon_tx; /* lxontxc */
739 kstat_named_t ipk_link_xoff_tx; /* lxofftxc */
740 kstat_named_t ipk_priority_xon_rx[8]; /* pxonrxc[8] */
741 kstat_named_t ipk_priority_xoff_rx[8]; /* pxoffrxc[8] */
742 kstat_named_t ipk_priority_xon_tx[8]; /* pxontxc[8] */
743 kstat_named_t ipk_priority_xoff_tx[8]; /* pxofftxc[8] */
744 kstat_named_t ipk_priority_xon_2_xoff[8]; /* rxon2offcnt[8] */
745
746 kstat_named_t ipk_crc_errors; /* crcerrs */
747 kstat_named_t ipk_illegal_bytes; /* illerrc */
748 kstat_named_t ipk_mac_local_faults; /* mlfc */
749 kstat_named_t ipk_mac_remote_faults; /* mrfc */
750 kstat_named_t ipk_rx_length_errors; /* rlec */
751 kstat_named_t ipk_rx_undersize; /* ruc */
752 kstat_named_t ipk_rx_fragments; /* rfc */
753 kstat_named_t ipk_rx_oversize; /* roc */
754 kstat_named_t ipk_rx_jabber; /* rjc */
755 kstat_named_t ipk_rx_discards; /* rdpc */
756 kstat_named_t ipk_rx_vm_discards; /* ldpc */
757 kstat_named_t ipk_rx_short_discards; /* mspdc */
758 kstat_named_t ipk_tx_dropped_link_down; /* tdold */
759 kstat_named_t ipk_rx_unknown_protocol; /* rupp */
760 kstat_named_t ipk_rx_err1; /* rxerr1 */
761 kstat_named_t ipk_rx_err2; /* rxerr2 */
762 } i40e_pf_kstats_t;
763
764 /*
765 * Resources that are pooled and specific to a given i40e_t.
766 */
767 typedef struct i40e_func_rsrc {
768 uint_t ifr_nrx_queue;
769 uint_t ifr_nrx_queue_used;
770 uint_t ifr_ntx_queue;
771 uint_t ifr_trx_queue_used;
772 uint_t ifr_nvsis;
773 uint_t ifr_nvsis_used;
774 uint_t ifr_nmacfilt;
775 uint_t ifr_nmacfilt_used;
776 uint_t ifr_nmcastfilt;
777 uint_t ifr_nmcastfilt_used;
778 } i40e_func_rsrc_t;
779
780 typedef struct i40e_vsi {
781 uint16_t iv_seid;
782 uint16_t iv_number;
783 kstat_t *iv_kstats;
784 i40e_vsi_stats_t iv_stats;
785 uint16_t iv_stats_id;
786 } i40e_vsi_t;
787
788 /*
789 * While irg_index and irg_grp_hdl aren't used anywhere, they are
790 * still useful for debugging.
791 */
792 typedef struct i40e_rx_group {
793 uint32_t irg_index; /* index in i40e_rx_groups[] */
794 uint16_t irg_vsi_seid; /* SEID of VSI for this group */
795 mac_group_handle_t irg_grp_hdl; /* handle to mac_group_t */
796 struct i40e *irg_i40e; /* ref to i40e_t */
797 } i40e_rx_group_t;
798
799 /*
800 * Main i40e per-instance state.
801 */
802 typedef struct i40e {
803 list_node_t i40e_glink; /* Global list link */
804 list_node_t i40e_dlink; /* Device list link */
805 kmutex_t i40e_general_lock; /* General device lock */
806
807 /*
808 * General Data and management
809 */
810 dev_info_t *i40e_dip;
811 int i40e_instance;
812 int i40e_fm_capabilities;
813 uint_t i40e_state;
814 i40e_attach_state_t i40e_attach_progress;
815 mac_handle_t i40e_mac_hdl;
816 ddi_periodic_t i40e_periodic_id;
817
818 /*
819 * Pointers to common code data structures and memory for the common
820 * code.
821 */
822 struct i40e_hw i40e_hw_space;
823 struct i40e_osdep i40e_osdep_space;
824 struct i40e_aq_get_phy_abilities_resp i40e_phy;
825 void *i40e_aqbuf;
826
827 #define I40E_DEF_VSI_IDX 0
828 #define I40E_DEF_VSI(i40e) ((i40e)->i40e_vsis[I40E_DEF_VSI_IDX])
829 #define I40E_DEF_VSI_SEID(i40e) (I40E_DEF_VSI(i40e).iv_seid)
830
831 /*
832 * Device state, switch information, and resources.
833 */
834 i40e_vsi_t i40e_vsis[I40E_GROUP_MAX];
835 uint16_t i40e_mac_seid; /* SEID of physical MAC */
836 uint16_t i40e_veb_seid; /* switch atop MAC (SEID) */
837 uint16_t i40e_vsi_avail; /* VSIs avail to this PF */
838 uint16_t i40e_vsi_used; /* VSIs used by this PF */
839 struct i40e_device *i40e_device;
840 i40e_func_rsrc_t i40e_resources;
841 uint16_t i40e_switch_rsrc_alloc;
842 uint16_t i40e_switch_rsrc_actual;
843 i40e_switch_rsrc_t *i40e_switch_rsrcs;
844 i40e_uaddr_t *i40e_uaddrs;
845 i40e_maddr_t *i40e_maddrs;
846 int i40e_mcast_promisc_count;
847 boolean_t i40e_promisc_on;
848 link_state_t i40e_link_state;
849 uint32_t i40e_link_speed; /* In Mbps */
850 link_duplex_t i40e_link_duplex;
851 uint_t i40e_sdu;
852 uint_t i40e_frame_max;
853
854 /*
855 * Transmit and receive information, tunables, and MAC info.
856 */
857 i40e_trqpair_t *i40e_trqpairs;
858 boolean_t i40e_mr_enable;
859 uint_t i40e_num_trqpairs; /* total TRQPs (per PF) */
860 uint_t i40e_num_trqpairs_per_vsi; /* TRQPs per VSI */
861 uint_t i40e_other_itr;
862
863 i40e_rx_group_t *i40e_rx_groups;
864 uint_t i40e_num_rx_groups;
865 int i40e_num_rx_descs;
866 uint32_t i40e_rx_ring_size;
867 uint32_t i40e_rx_buf_size;
868 boolean_t i40e_rx_hcksum_enable;
869 uint32_t i40e_rx_dma_min;
870 uint32_t i40e_rx_limit_per_intr;
871 uint_t i40e_rx_itr;
872
873 int i40e_num_tx_descs;
874 uint32_t i40e_tx_ring_size;
875 uint32_t i40e_tx_buf_size;
876 uint32_t i40e_tx_block_thresh;
877 boolean_t i40e_tx_hcksum_enable;
878 boolean_t i40e_tx_lso_enable;
879 uint32_t i40e_tx_dma_min;
880 uint_t i40e_tx_itr;
881
882 /*
883 * Interrupt state
884 */
885 uint_t i40e_intr_pri;
886 uint_t i40e_intr_force;
887 uint_t i40e_intr_type;
888 int i40e_intr_cap;
889 uint32_t i40e_intr_count;
890 uint32_t i40e_intr_count_max;
891 uint32_t i40e_intr_count_min;
892 size_t i40e_intr_size;
893 ddi_intr_handle_t *i40e_intr_handles;
894 ddi_cb_handle_t i40e_callback_handle;
895
896 /*
897 * DMA attributes. See i40e_transceiver.c for why we have copies of them
898 * in the i40e_t.
899 */
900 ddi_dma_attr_t i40e_static_dma_attr;
901 ddi_dma_attr_t i40e_txbind_dma_attr;
902 ddi_dma_attr_t i40e_txbind_lso_dma_attr;
903 ddi_device_acc_attr_t i40e_desc_acc_attr;
904 ddi_device_acc_attr_t i40e_buf_acc_attr;
905
906 /*
907 * The following two fields are used to protect and keep track of
908 * outstanding, loaned buffers to MAC. If we have these, we can't
909 * detach as we have active DMA memory outstanding.
910 */
911 kmutex_t i40e_rx_pending_lock;
912 kcondvar_t i40e_rx_pending_cv;
913 uint32_t i40e_rx_pending;
914
915 /*
916 * PF statistics and VSI statistics.
917 */
918 kmutex_t i40e_stat_lock;
919 kstat_t *i40e_pf_kstat;
920 i40e_pf_stats_t i40e_pf_stat;
921
922 /*
923 * Misc. stats and counters that should maybe one day be kstats.
924 */
925 uint64_t i40e_s_link_status_errs;
926 uint32_t i40e_s_link_status_lasterr;
927
928 /*
929 * LED information. Note this state is only modified in
930 * i40e_gld_set_led() which is protected by MAC's serializer lock.
931 */
932 uint32_t i40e_led_status;
933 boolean_t i40e_led_saved;
934 } i40e_t;
935
936 /*
937 * The i40e_device represents a PCI device which encapsulates multiple physical
938 * functions which are represented as an i40e_t. This is used to track the use
939 * of pooled resources throughout all of the various devices.
940 */
941 typedef struct i40e_device {
942 list_node_t id_link;
943 dev_info_t *id_parent;
944 uint_t id_pci_bus;
945 uint_t id_pci_device;
946 uint_t id_nfuncs; /* Total number of functions */
947 uint_t id_nreg; /* Total number present */
948 list_t id_i40e_list; /* List of i40e_t's registered */
949 i40e_switch_rsrc_t *id_rsrcs; /* Switch resources for this PF */
950 uint_t id_rsrcs_alloc; /* Total allocated resources */
951 uint_t id_rsrcs_act; /* Actual number of resources */
952 } i40e_device_t;
953
954 /* Values for the interrupt forcing on the NIC. */
955 #define I40E_INTR_NONE 0
956 #define I40E_INTR_MSIX 1
957 #define I40E_INTR_MSI 2
958 #define I40E_INTR_LEGACY 3
959
960 /* Hint that we don't want to do any polling... */
961 #define I40E_POLL_NULL -1
962
963 /*
964 * Logging functions.
965 */
966 /*PRINTFLIKE2*/
967 extern void i40e_error(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
968 /*PRINTFLIKE2*/
969 extern void i40e_notice(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
970 /*PRINTFLIKE2*/
971 extern void i40e_log(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
972
973 /*
974 * General link handling functions.
975 */
976 extern void i40e_link_check(i40e_t *);
977 extern void i40e_update_mtu(i40e_t *);
978
979 /*
980 * FMA functions.
981 */
982 extern int i40e_check_acc_handle(ddi_acc_handle_t);
983 extern int i40e_check_dma_handle(ddi_dma_handle_t);
984 extern void i40e_fm_ereport(i40e_t *, char *);
985
986 /*
987 * Interrupt handlers and interrupt handler setup.
988 */
989 extern void i40e_intr_chip_init(i40e_t *);
990 extern void i40e_intr_chip_fini(i40e_t *);
991 extern uint_t i40e_intr_msix(void *, void *);
992 extern uint_t i40e_intr_msi(void *, void *);
993 extern uint_t i40e_intr_legacy(void *, void *);
994 extern void i40e_intr_io_enable_all(i40e_t *);
995 extern void i40e_intr_io_disable_all(i40e_t *);
996 extern void i40e_intr_io_clear_cause(i40e_t *);
997 extern void i40e_intr_rx_queue_disable(i40e_trqpair_t *);
998 extern void i40e_intr_rx_queue_enable(i40e_trqpair_t *);
999 extern void i40e_intr_set_itr(i40e_t *, i40e_itr_index_t, uint_t);
1000
1001 /*
1002 * Receive-side functions
1003 */
1004 extern mblk_t *i40e_ring_rx(i40e_trqpair_t *, int);
1005 extern mblk_t *i40e_ring_rx_poll(void *, int);
1006 extern void i40e_rx_recycle(caddr_t);
1007
1008 /*
1009 * Transmit-side functions
1010 */
1011 mblk_t *i40e_ring_tx(void *, mblk_t *);
1012 extern void i40e_tx_recycle_ring(i40e_trqpair_t *);
1013 extern void i40e_tx_cleanup_ring(i40e_trqpair_t *);
1014
1015 /*
1016 * Statistics functions.
1017 */
1018 extern boolean_t i40e_stats_init(i40e_t *);
1019 extern void i40e_stats_fini(i40e_t *);
1020 extern boolean_t i40e_stat_vsi_init(i40e_t *, uint_t);
1021 extern void i40e_stat_vsi_fini(i40e_t *, uint_t);
1022 extern boolean_t i40e_stats_trqpair_init(i40e_trqpair_t *);
1023 extern void i40e_stats_trqpair_fini(i40e_trqpair_t *);
1024 extern int i40e_m_stat(void *, uint_t, uint64_t *);
1025 extern int i40e_rx_ring_stat(mac_ring_driver_t, uint_t, uint64_t *);
1026 extern int i40e_tx_ring_stat(mac_ring_driver_t, uint_t, uint64_t *);
1027
1028 /*
1029 * MAC/GLDv3 functions, and functions called by MAC/GLDv3 support code.
1030 */
1031 extern boolean_t i40e_register_mac(i40e_t *);
1032 extern boolean_t i40e_start(i40e_t *, boolean_t);
1033 extern void i40e_stop(i40e_t *, boolean_t);
1034
1035 /*
1036 * DMA & buffer functions and attributes
1037 */
1038 extern void i40e_init_dma_attrs(i40e_t *, boolean_t);
1039 extern boolean_t i40e_alloc_ring_mem(i40e_t *);
1040 extern void i40e_free_ring_mem(i40e_t *, boolean_t);
1041
1042 #ifdef __cplusplus
1043 }
1044 #endif
1045
1046 #endif /* _I40E_SW_H */