1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2017, Joyent, Inc.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 */
34
35 #include "ixgbe_sw.h"
36
37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static void ixgbe_setup_rings(ixgbe_t *);
61 static void ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_setup_rss_table(ixgbe_t *);
69 static void ixgbe_init_unicst(ixgbe_t *);
70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
71 static void ixgbe_setup_multicst(ixgbe_t *);
72 static void ixgbe_get_hw_state(ixgbe_t *);
73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
74 static void ixgbe_get_conf(ixgbe_t *);
75 static void ixgbe_init_params(ixgbe_t *);
76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
77 static void ixgbe_driver_link_check(ixgbe_t *);
78 static void ixgbe_sfp_check(void *);
79 static void ixgbe_overtemp_check(void *);
80 static void ixgbe_phy_check(void *);
81 static void ixgbe_link_timer(void *);
82 static void ixgbe_local_timer(void *);
83 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
84 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
87 static boolean_t is_valid_mac_addr(uint8_t *);
88 static boolean_t ixgbe_stall_check(ixgbe_t *);
89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
91 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
92 static int ixgbe_alloc_intrs(ixgbe_t *);
93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
94 static int ixgbe_add_intr_handlers(ixgbe_t *);
95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
102 static void ixgbe_setup_adapter_vector(ixgbe_t *);
103 static void ixgbe_rem_intr_handlers(ixgbe_t *);
104 static void ixgbe_rem_intrs(ixgbe_t *);
105 static int ixgbe_enable_intrs(ixgbe_t *);
106 static int ixgbe_disable_intrs(ixgbe_t *);
107 static uint_t ixgbe_intr_legacy(void *, void *);
108 static uint_t ixgbe_intr_msi(void *, void *);
109 static uint_t ixgbe_intr_msix(void *, void *);
110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
113 static void ixgbe_get_driver_control(struct ixgbe_hw *);
114 static int ixgbe_addmac(void *, const uint8_t *);
115 static int ixgbe_remmac(void *, const uint8_t *);
116 static void ixgbe_release_driver_control(struct ixgbe_hw *);
117
118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
120 static int ixgbe_resume(dev_info_t *);
121 static int ixgbe_suspend(dev_info_t *);
122 static int ixgbe_quiesce(dev_info_t *);
123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
126 static int ixgbe_intr_cb_register(ixgbe_t *);
127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
128
129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
130 const void *impl_data);
131 static void ixgbe_fm_init(ixgbe_t *);
132 static void ixgbe_fm_fini(ixgbe_t *);
133
134 char *ixgbe_priv_props[] = {
135 "_tx_copy_thresh",
136 "_tx_recycle_thresh",
137 "_tx_overload_thresh",
138 "_tx_resched_thresh",
139 "_rx_copy_thresh",
140 "_rx_limit_per_intr",
141 "_intr_throttling",
142 "_adv_pause_cap",
143 "_adv_asym_pause_cap",
144 NULL
145 };
146
147 #define IXGBE_MAX_PRIV_PROPS \
148 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
149
150 static struct cb_ops ixgbe_cb_ops = {
151 nulldev, /* cb_open */
152 nulldev, /* cb_close */
153 nodev, /* cb_strategy */
154 nodev, /* cb_print */
155 nodev, /* cb_dump */
156 nodev, /* cb_read */
157 nodev, /* cb_write */
158 nodev, /* cb_ioctl */
159 nodev, /* cb_devmap */
160 nodev, /* cb_mmap */
161 nodev, /* cb_segmap */
162 nochpoll, /* cb_chpoll */
163 ddi_prop_op, /* cb_prop_op */
164 NULL, /* cb_stream */
165 D_MP | D_HOTPLUG, /* cb_flag */
166 CB_REV, /* cb_rev */
167 nodev, /* cb_aread */
168 nodev /* cb_awrite */
169 };
170
171 static struct dev_ops ixgbe_dev_ops = {
172 DEVO_REV, /* devo_rev */
173 0, /* devo_refcnt */
174 NULL, /* devo_getinfo */
175 nulldev, /* devo_identify */
176 nulldev, /* devo_probe */
177 ixgbe_attach, /* devo_attach */
178 ixgbe_detach, /* devo_detach */
179 nodev, /* devo_reset */
180 &ixgbe_cb_ops, /* devo_cb_ops */
181 NULL, /* devo_bus_ops */
182 ddi_power, /* devo_power */
183 ixgbe_quiesce, /* devo_quiesce */
184 };
185
186 static struct modldrv ixgbe_modldrv = {
187 &mod_driverops, /* Type of module. This one is a driver */
188 ixgbe_ident, /* Discription string */
189 &ixgbe_dev_ops /* driver ops */
190 };
191
192 static struct modlinkage ixgbe_modlinkage = {
193 MODREV_1, &ixgbe_modldrv, NULL
194 };
195
196 /*
197 * Access attributes for register mapping
198 */
199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
200 DDI_DEVICE_ATTR_V1,
201 DDI_STRUCTURE_LE_ACC,
202 DDI_STRICTORDER_ACC,
203 DDI_FLAGERR_ACC
204 };
205
206 /*
207 * Loopback property
208 */
209 static lb_property_t lb_normal = {
210 normal, "normal", IXGBE_LB_NONE
211 };
212
213 static lb_property_t lb_mac = {
214 internal, "MAC", IXGBE_LB_INTERNAL_MAC
215 };
216
217 static lb_property_t lb_external = {
218 external, "External", IXGBE_LB_EXTERNAL
219 };
220
221 #define IXGBE_M_CALLBACK_FLAGS \
222 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
223
224 static mac_callbacks_t ixgbe_m_callbacks = {
225 IXGBE_M_CALLBACK_FLAGS,
226 ixgbe_m_stat,
227 ixgbe_m_start,
228 ixgbe_m_stop,
229 ixgbe_m_promisc,
230 ixgbe_m_multicst,
231 NULL,
232 NULL,
233 NULL,
234 ixgbe_m_ioctl,
235 ixgbe_m_getcapab,
236 NULL,
237 NULL,
238 ixgbe_m_setprop,
239 ixgbe_m_getprop,
240 ixgbe_m_propinfo
241 };
242
243 /*
244 * Initialize capabilities of each supported adapter type
245 */
246 static adapter_info_t ixgbe_82598eb_cap = {
247 64, /* maximum number of rx queues */
248 1, /* minimum number of rx queues */
249 64, /* default number of rx queues */
250 16, /* maximum number of rx groups */
251 1, /* minimum number of rx groups */
252 1, /* default number of rx groups */
253 32, /* maximum number of tx queues */
254 1, /* minimum number of tx queues */
255 8, /* default number of tx queues */
256 16366, /* maximum MTU size */
257 0xFFFF, /* maximum interrupt throttle rate */
258 0, /* minimum interrupt throttle rate */
259 200, /* default interrupt throttle rate */
260 18, /* maximum total msix vectors */
261 16, /* maximum number of ring vectors */
262 2, /* maximum number of other vectors */
263 IXGBE_EICR_LSC, /* "other" interrupt types handled */
264 0, /* "other" interrupt types enable mask */
265 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
266 | IXGBE_FLAG_RSS_CAPABLE
267 | IXGBE_FLAG_VMDQ_CAPABLE)
268 };
269
270 static adapter_info_t ixgbe_82599eb_cap = {
271 128, /* maximum number of rx queues */
272 1, /* minimum number of rx queues */
273 128, /* default number of rx queues */
274 64, /* maximum number of rx groups */
275 1, /* minimum number of rx groups */
276 1, /* default number of rx groups */
277 128, /* maximum number of tx queues */
278 1, /* minimum number of tx queues */
279 8, /* default number of tx queues */
280 15500, /* maximum MTU size */
281 0xFF8, /* maximum interrupt throttle rate */
282 0, /* minimum interrupt throttle rate */
283 200, /* default interrupt throttle rate */
284 64, /* maximum total msix vectors */
285 16, /* maximum number of ring vectors */
286 2, /* maximum number of other vectors */
287 (IXGBE_EICR_LSC
288 | IXGBE_EICR_GPI_SDP1
289 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
290
291 (IXGBE_SDP1_GPIEN
292 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
293
294 (IXGBE_FLAG_DCA_CAPABLE
295 | IXGBE_FLAG_RSS_CAPABLE
296 | IXGBE_FLAG_VMDQ_CAPABLE
297 | IXGBE_FLAG_RSC_CAPABLE
298 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
299 };
300
301 static adapter_info_t ixgbe_X540_cap = {
302 128, /* maximum number of rx queues */
303 1, /* minimum number of rx queues */
304 128, /* default number of rx queues */
305 64, /* maximum number of rx groups */
306 1, /* minimum number of rx groups */
307 1, /* default number of rx groups */
308 128, /* maximum number of tx queues */
309 1, /* minimum number of tx queues */
310 8, /* default number of tx queues */
311 15500, /* maximum MTU size */
312 0xFF8, /* maximum interrupt throttle rate */
313 0, /* minimum interrupt throttle rate */
314 200, /* default interrupt throttle rate */
315 64, /* maximum total msix vectors */
316 16, /* maximum number of ring vectors */
317 2, /* maximum number of other vectors */
318 (IXGBE_EICR_LSC
319 | IXGBE_EICR_GPI_SDP1_X540
320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
321
322 (IXGBE_SDP1_GPIEN_X540
323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
324
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
329 };
330
331 static adapter_info_t ixgbe_X550_cap = {
332 128, /* maximum number of rx queues */
333 1, /* minimum number of rx queues */
334 128, /* default number of rx queues */
335 64, /* maximum number of rx groups */
336 1, /* minimum number of rx groups */
337 1, /* default number of rx groups */
338 128, /* maximum number of tx queues */
339 1, /* minimum number of tx queues */
340 8, /* default number of tx queues */
341 15500, /* maximum MTU size */
342 0xFF8, /* maximum interrupt throttle rate */
343 0, /* minimum interrupt throttle rate */
344 0x200, /* default interrupt throttle rate */
345 64, /* maximum total msix vectors */
346 16, /* maximum number of ring vectors */
347 2, /* maximum number of other vectors */
348 IXGBE_EICR_LSC, /* "other" interrupt types handled */
349 0, /* "other" interrupt types enable mask */
350 (IXGBE_FLAG_RSS_CAPABLE
351 | IXGBE_FLAG_VMDQ_CAPABLE
352 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
353 };
354
355 /*
356 * Module Initialization Functions.
357 */
358
359 int
360 _init(void)
361 {
362 int status;
363
364 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
365
366 status = mod_install(&ixgbe_modlinkage);
367
368 if (status != DDI_SUCCESS) {
369 mac_fini_ops(&ixgbe_dev_ops);
370 }
371
372 return (status);
373 }
374
375 int
376 _fini(void)
377 {
378 int status;
379
380 status = mod_remove(&ixgbe_modlinkage);
381
382 if (status == DDI_SUCCESS) {
383 mac_fini_ops(&ixgbe_dev_ops);
384 }
385
386 return (status);
387 }
388
389 int
390 _info(struct modinfo *modinfop)
391 {
392 int status;
393
394 status = mod_info(&ixgbe_modlinkage, modinfop);
395
396 return (status);
397 }
398
399 /*
400 * ixgbe_attach - Driver attach.
401 *
402 * This function is the device specific initialization entry
403 * point. This entry point is required and must be written.
404 * The DDI_ATTACH command must be provided in the attach entry
405 * point. When attach() is called with cmd set to DDI_ATTACH,
406 * all normal kernel services (such as kmem_alloc(9F)) are
407 * available for use by the driver.
408 *
409 * The attach() function will be called once for each instance
410 * of the device on the system with cmd set to DDI_ATTACH.
411 * Until attach() succeeds, the only driver entry points which
412 * may be called are open(9E) and getinfo(9E).
413 */
414 static int
415 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
416 {
417 ixgbe_t *ixgbe;
418 struct ixgbe_osdep *osdep;
419 struct ixgbe_hw *hw;
420 int instance;
421 char taskqname[32];
422
423 /*
424 * Check the command and perform corresponding operations
425 */
426 switch (cmd) {
427 default:
428 return (DDI_FAILURE);
429
430 case DDI_RESUME:
431 return (ixgbe_resume(devinfo));
432
433 case DDI_ATTACH:
434 break;
435 }
436
437 /* Get the device instance */
438 instance = ddi_get_instance(devinfo);
439
440 /* Allocate memory for the instance data structure */
441 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
442
443 ixgbe->dip = devinfo;
444 ixgbe->instance = instance;
445
446 hw = &ixgbe->hw;
447 osdep = &ixgbe->osdep;
448 hw->back = osdep;
449 osdep->ixgbe = ixgbe;
450
451 /* Attach the instance pointer to the dev_info data structure */
452 ddi_set_driver_private(devinfo, ixgbe);
453
454 /*
455 * Initialize for FMA support
456 */
457 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
458 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
459 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
460 ixgbe_fm_init(ixgbe);
461 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
462
463 /*
464 * Map PCI config space registers
465 */
466 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
467 ixgbe_error(ixgbe, "Failed to map PCI configurations");
468 goto attach_fail;
469 }
470 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
471
472 /*
473 * Identify the chipset family
474 */
475 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
476 ixgbe_error(ixgbe, "Failed to identify hardware");
477 goto attach_fail;
478 }
479
480 /*
481 * Map device registers
482 */
483 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
484 ixgbe_error(ixgbe, "Failed to map device registers");
485 goto attach_fail;
486 }
487 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
488
489 /*
490 * Initialize driver parameters
491 */
492 ixgbe_init_properties(ixgbe);
493 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
494
495 /*
496 * Register interrupt callback
497 */
498 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
499 ixgbe_error(ixgbe, "Failed to register interrupt callback");
500 goto attach_fail;
501 }
502
503 /*
504 * Allocate interrupts
505 */
506 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
507 ixgbe_error(ixgbe, "Failed to allocate interrupts");
508 goto attach_fail;
509 }
510 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
511
512 /*
513 * Allocate rx/tx rings based on the ring numbers.
514 * The actual numbers of rx/tx rings are decided by the number of
515 * allocated interrupt vectors, so we should allocate the rings after
516 * interrupts are allocated.
517 */
518 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
519 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
520 goto attach_fail;
521 }
522 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
523
524 /*
525 * Map rings to interrupt vectors
526 */
527 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
528 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
529 goto attach_fail;
530 }
531
532 /*
533 * Add interrupt handlers
534 */
535 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
536 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
537 goto attach_fail;
538 }
539 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
540
541 /*
542 * Create a taskq for sfp-change
543 */
544 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
545 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
546 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
547 ixgbe_error(ixgbe, "sfp_taskq create failed");
548 goto attach_fail;
549 }
550 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
551
552 /*
553 * Create a taskq for over-temp
554 */
555 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
556 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
557 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
558 ixgbe_error(ixgbe, "overtemp_taskq create failed");
559 goto attach_fail;
560 }
561 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
562
563 /*
564 * Create a taskq for processing external PHY interrupts
565 */
566 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
567 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
568 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
569 ixgbe_error(ixgbe, "phy_taskq create failed");
570 goto attach_fail;
571 }
572 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
573
574 /*
575 * Initialize driver parameters
576 */
577 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
578 ixgbe_error(ixgbe, "Failed to initialize driver settings");
579 goto attach_fail;
580 }
581
582 /*
583 * Initialize mutexes for this device.
584 * Do this before enabling the interrupt handler and
585 * register the softint to avoid the condition where
586 * interrupt handler can try using uninitialized mutex.
587 */
588 ixgbe_init_locks(ixgbe);
589 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
590
591 /*
592 * Initialize chipset hardware
593 */
594 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
595 ixgbe_error(ixgbe, "Failed to initialize adapter");
596 goto attach_fail;
597 }
598 ixgbe->link_check_complete = B_FALSE;
599 ixgbe->link_check_hrtime = gethrtime() +
600 (IXGBE_LINK_UP_TIME * 100000000ULL);
601 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
602
603 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
604 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
605 goto attach_fail;
606 }
607
608 /*
609 * Initialize adapter capabilities
610 */
611 ixgbe_init_params(ixgbe);
612
613 /*
614 * Initialize statistics
615 */
616 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
617 ixgbe_error(ixgbe, "Failed to initialize statistics");
618 goto attach_fail;
619 }
620 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
621
622 /*
623 * Register the driver to the MAC
624 */
625 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
626 ixgbe_error(ixgbe, "Failed to register MAC");
627 goto attach_fail;
628 }
629 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
630 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
631
632 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
633 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
634 if (ixgbe->periodic_id == 0) {
635 ixgbe_error(ixgbe, "Failed to add the link check timer");
636 goto attach_fail;
637 }
638 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
639
640 /*
641 * Now that mutex locks are initialized, and the chip is also
642 * initialized, enable interrupts.
643 */
644 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
645 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
646 goto attach_fail;
647 }
648 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
649
650 ixgbe_log(ixgbe, "%s", ixgbe_ident);
651 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
652
653 return (DDI_SUCCESS);
654
655 attach_fail:
656 ixgbe_unconfigure(devinfo, ixgbe);
657 return (DDI_FAILURE);
658 }
659
660 /*
661 * ixgbe_detach - Driver detach.
662 *
663 * The detach() function is the complement of the attach routine.
664 * If cmd is set to DDI_DETACH, detach() is used to remove the
665 * state associated with a given instance of a device node
666 * prior to the removal of that instance from the system.
667 *
668 * The detach() function will be called once for each instance
669 * of the device for which there has been a successful attach()
670 * once there are no longer any opens on the device.
671 *
672 * Interrupts routine are disabled, All memory allocated by this
673 * driver are freed.
674 */
675 static int
676 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
677 {
678 ixgbe_t *ixgbe;
679
680 /*
681 * Check detach command
682 */
683 switch (cmd) {
684 default:
685 return (DDI_FAILURE);
686
687 case DDI_SUSPEND:
688 return (ixgbe_suspend(devinfo));
689
690 case DDI_DETACH:
691 break;
692 }
693
694 /*
695 * Get the pointer to the driver private data structure
696 */
697 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
698 if (ixgbe == NULL)
699 return (DDI_FAILURE);
700
701 /*
702 * If the device is still running, it needs to be stopped first.
703 * This check is necessary because under some specific circumstances,
704 * the detach routine can be called without stopping the interface
705 * first.
706 */
707 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
708 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
709 mutex_enter(&ixgbe->gen_lock);
710 ixgbe_stop(ixgbe, B_TRUE);
711 mutex_exit(&ixgbe->gen_lock);
712 /* Disable and stop the watchdog timer */
713 ixgbe_disable_watchdog_timer(ixgbe);
714 }
715
716 /*
717 * Check if there are still rx buffers held by the upper layer.
718 * If so, fail the detach.
719 */
720 if (!ixgbe_rx_drain(ixgbe))
721 return (DDI_FAILURE);
722
723 /*
724 * Do the remaining unconfigure routines
725 */
726 ixgbe_unconfigure(devinfo, ixgbe);
727
728 return (DDI_SUCCESS);
729 }
730
731 /*
732 * quiesce(9E) entry point.
733 *
734 * This function is called when the system is single-threaded at high
735 * PIL with preemption disabled. Therefore, this function must not be
736 * blocked.
737 *
738 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
739 * DDI_FAILURE indicates an error condition and should almost never happen.
740 */
741 static int
742 ixgbe_quiesce(dev_info_t *devinfo)
743 {
744 ixgbe_t *ixgbe;
745 struct ixgbe_hw *hw;
746
747 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
748
749 if (ixgbe == NULL)
750 return (DDI_FAILURE);
751
752 hw = &ixgbe->hw;
753
754 /*
755 * Disable the adapter interrupts
756 */
757 ixgbe_disable_adapter_interrupts(ixgbe);
758
759 /*
760 * Tell firmware driver is no longer in control
761 */
762 ixgbe_release_driver_control(hw);
763
764 /*
765 * Reset the chipset
766 */
767 (void) ixgbe_reset_hw(hw);
768
769 /*
770 * Reset PHY
771 */
772 (void) ixgbe_reset_phy(hw);
773
774 return (DDI_SUCCESS);
775 }
776
777 static void
778 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
779 {
780 /*
781 * Disable interrupt
782 */
783 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
784 (void) ixgbe_disable_intrs(ixgbe);
785 }
786
787 /*
788 * remove the link check timer
789 */
790 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
791 if (ixgbe->periodic_id != NULL) {
792 ddi_periodic_delete(ixgbe->periodic_id);
793 ixgbe->periodic_id = NULL;
794 }
795 }
796
797 /*
798 * Unregister MAC
799 */
800 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
801 (void) mac_unregister(ixgbe->mac_hdl);
802 }
803
804 /*
805 * Free statistics
806 */
807 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
808 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
809 }
810
811 /*
812 * Remove interrupt handlers
813 */
814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
815 ixgbe_rem_intr_handlers(ixgbe);
816 }
817
818 /*
819 * Remove taskq for sfp-status-change
820 */
821 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
822 ddi_taskq_destroy(ixgbe->sfp_taskq);
823 }
824
825 /*
826 * Remove taskq for over-temp
827 */
828 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
829 ddi_taskq_destroy(ixgbe->overtemp_taskq);
830 }
831
832 /*
833 * Remove taskq for external PHYs
834 */
835 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
836 ddi_taskq_destroy(ixgbe->phy_taskq);
837 }
838
839 /*
840 * Remove interrupts
841 */
842 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
843 ixgbe_rem_intrs(ixgbe);
844 }
845
846 /*
847 * Unregister interrupt callback handler
848 */
849 if (ixgbe->cb_hdl != NULL) {
850 (void) ddi_cb_unregister(ixgbe->cb_hdl);
851 }
852
853 /*
854 * Remove driver properties
855 */
856 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
857 (void) ddi_prop_remove_all(devinfo);
858 }
859
860 /*
861 * Stop the chipset
862 */
863 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
864 mutex_enter(&ixgbe->gen_lock);
865 ixgbe_chip_stop(ixgbe);
866 mutex_exit(&ixgbe->gen_lock);
867 }
868
869 /*
870 * Free register handle
871 */
872 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
873 if (ixgbe->osdep.reg_handle != NULL)
874 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
875 }
876
877 /*
878 * Free PCI config handle
879 */
880 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
881 if (ixgbe->osdep.cfg_handle != NULL)
882 pci_config_teardown(&ixgbe->osdep.cfg_handle);
883 }
884
885 /*
886 * Free locks
887 */
888 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
889 ixgbe_destroy_locks(ixgbe);
890 }
891
892 /*
893 * Free the rx/tx rings
894 */
895 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
896 ixgbe_free_rings(ixgbe);
897 }
898
899 /*
900 * Unregister FMA capabilities
901 */
902 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
903 ixgbe_fm_fini(ixgbe);
904 }
905
906 /*
907 * Free the driver data structure
908 */
909 kmem_free(ixgbe, sizeof (ixgbe_t));
910
911 ddi_set_driver_private(devinfo, NULL);
912 }
913
914 /*
915 * ixgbe_register_mac - Register the driver and its function pointers with
916 * the GLD interface.
917 */
918 static int
919 ixgbe_register_mac(ixgbe_t *ixgbe)
920 {
921 struct ixgbe_hw *hw = &ixgbe->hw;
922 mac_register_t *mac;
923 int status;
924
925 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
926 return (IXGBE_FAILURE);
927
928 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
929 mac->m_driver = ixgbe;
930 mac->m_dip = ixgbe->dip;
931 mac->m_src_addr = hw->mac.addr;
932 mac->m_callbacks = &ixgbe_m_callbacks;
933 mac->m_min_sdu = 0;
934 mac->m_max_sdu = ixgbe->default_mtu;
935 mac->m_margin = VLAN_TAGSZ;
936 mac->m_priv_props = ixgbe_priv_props;
937 mac->m_v12n = MAC_VIRT_LEVEL1;
938
939 status = mac_register(mac, &ixgbe->mac_hdl);
940
941 mac_free(mac);
942
943 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
944 }
945
946 /*
947 * ixgbe_identify_hardware - Identify the type of the chipset.
948 */
949 static int
950 ixgbe_identify_hardware(ixgbe_t *ixgbe)
951 {
952 struct ixgbe_hw *hw = &ixgbe->hw;
953 struct ixgbe_osdep *osdep = &ixgbe->osdep;
954
955 /*
956 * Get the device id
957 */
958 hw->vendor_id =
959 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
960 hw->device_id =
961 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
962 hw->revision_id =
963 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
964 hw->subsystem_device_id =
965 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
966 hw->subsystem_vendor_id =
967 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
968
969 /*
970 * Set the mac type of the adapter based on the device id
971 */
972 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
973 return (IXGBE_FAILURE);
974 }
975
976 /*
977 * Install adapter capabilities
978 */
979 switch (hw->mac.type) {
980 case ixgbe_mac_82598EB:
981 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
982 ixgbe->capab = &ixgbe_82598eb_cap;
983
984 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
985 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
986 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
987 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
988 }
989 break;
990
991 case ixgbe_mac_82599EB:
992 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
993 ixgbe->capab = &ixgbe_82599eb_cap;
994
995 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
996 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
997 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
998 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
999 }
1000 break;
1001
1002 case ixgbe_mac_X540:
1003 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1004 ixgbe->capab = &ixgbe_X540_cap;
1005 /*
1006 * For now, X540 is all set in its capab structure.
1007 * As other X540 variants show up, things can change here.
1008 */
1009 break;
1010
1011 case ixgbe_mac_X550:
1012 case ixgbe_mac_X550EM_x:
1013 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1014 ixgbe->capab = &ixgbe_X550_cap;
1015
1016 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP)
1017 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1018
1019 /*
1020 * Link detection on X552 SFP+ and X552/X557-AT
1021 */
1022 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1023 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1024 ixgbe->capab->other_intr |=
1025 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1026 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1027 }
1028 break;
1029
1030 default:
1031 IXGBE_DEBUGLOG_1(ixgbe,
1032 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1033 hw->mac.type);
1034 return (IXGBE_FAILURE);
1035 }
1036
1037 return (IXGBE_SUCCESS);
1038 }
1039
1040 /*
1041 * ixgbe_regs_map - Map the device registers.
1042 *
1043 */
1044 static int
1045 ixgbe_regs_map(ixgbe_t *ixgbe)
1046 {
1047 dev_info_t *devinfo = ixgbe->dip;
1048 struct ixgbe_hw *hw = &ixgbe->hw;
1049 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1050 off_t mem_size;
1051
1052 /*
1053 * First get the size of device registers to be mapped.
1054 */
1055 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1056 != DDI_SUCCESS) {
1057 return (IXGBE_FAILURE);
1058 }
1059
1060 /*
1061 * Call ddi_regs_map_setup() to map registers
1062 */
1063 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1064 (caddr_t *)&hw->hw_addr, 0,
1065 mem_size, &ixgbe_regs_acc_attr,
1066 &osdep->reg_handle)) != DDI_SUCCESS) {
1067 return (IXGBE_FAILURE);
1068 }
1069
1070 return (IXGBE_SUCCESS);
1071 }
1072
1073 /*
1074 * ixgbe_init_properties - Initialize driver properties.
1075 */
1076 static void
1077 ixgbe_init_properties(ixgbe_t *ixgbe)
1078 {
1079 /*
1080 * Get conf file properties, including link settings
1081 * jumbo frames, ring number, descriptor number, etc.
1082 */
1083 ixgbe_get_conf(ixgbe);
1084 }
1085
1086 /*
1087 * ixgbe_init_driver_settings - Initialize driver settings.
1088 *
1089 * The settings include hardware function pointers, bus information,
1090 * rx/tx rings settings, link state, and any other parameters that
1091 * need to be setup during driver initialization.
1092 */
1093 static int
1094 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1095 {
1096 struct ixgbe_hw *hw = &ixgbe->hw;
1097 dev_info_t *devinfo = ixgbe->dip;
1098 ixgbe_rx_ring_t *rx_ring;
1099 ixgbe_rx_group_t *rx_group;
1100 ixgbe_tx_ring_t *tx_ring;
1101 uint32_t rx_size;
1102 uint32_t tx_size;
1103 uint32_t ring_per_group;
1104 int i;
1105
1106 /*
1107 * Initialize chipset specific hardware function pointers
1108 */
1109 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1110 return (IXGBE_FAILURE);
1111 }
1112
1113 /*
1114 * Get the system page size
1115 */
1116 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1117
1118 /*
1119 * Set rx buffer size
1120 *
1121 * The IP header alignment room is counted in the calculation.
1122 * The rx buffer size is in unit of 1K that is required by the
1123 * chipset hardware.
1124 */
1125 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1126 ixgbe->rx_buf_size = ((rx_size >> 10) +
1127 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1128
1129 /*
1130 * Set tx buffer size
1131 */
1132 tx_size = ixgbe->max_frame_size;
1133 ixgbe->tx_buf_size = ((tx_size >> 10) +
1134 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1135
1136 /*
1137 * Initialize rx/tx rings/groups parameters
1138 */
1139 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1140 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1141 rx_ring = &ixgbe->rx_rings[i];
1142 rx_ring->index = i;
1143 rx_ring->ixgbe = ixgbe;
1144 rx_ring->group_index = i / ring_per_group;
1145 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1146 }
1147
1148 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1149 rx_group = &ixgbe->rx_groups[i];
1150 rx_group->index = i;
1151 rx_group->ixgbe = ixgbe;
1152 }
1153
1154 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1155 tx_ring = &ixgbe->tx_rings[i];
1156 tx_ring->index = i;
1157 tx_ring->ixgbe = ixgbe;
1158 if (ixgbe->tx_head_wb_enable)
1159 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1160 else
1161 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1162
1163 tx_ring->ring_size = ixgbe->tx_ring_size;
1164 tx_ring->free_list_size = ixgbe->tx_ring_size +
1165 (ixgbe->tx_ring_size >> 1);
1166 }
1167
1168 /*
1169 * Initialize values of interrupt throttling rate
1170 */
1171 for (i = 1; i < MAX_INTR_VECTOR; i++)
1172 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1173
1174 /*
1175 * The initial link state should be "unknown"
1176 */
1177 ixgbe->link_state = LINK_STATE_UNKNOWN;
1178
1179 return (IXGBE_SUCCESS);
1180 }
1181
1182 /*
1183 * ixgbe_init_locks - Initialize locks.
1184 */
1185 static void
1186 ixgbe_init_locks(ixgbe_t *ixgbe)
1187 {
1188 ixgbe_rx_ring_t *rx_ring;
1189 ixgbe_tx_ring_t *tx_ring;
1190 int i;
1191
1192 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1193 rx_ring = &ixgbe->rx_rings[i];
1194 mutex_init(&rx_ring->rx_lock, NULL,
1195 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1196 }
1197
1198 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1199 tx_ring = &ixgbe->tx_rings[i];
1200 mutex_init(&tx_ring->tx_lock, NULL,
1201 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1202 mutex_init(&tx_ring->recycle_lock, NULL,
1203 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1204 mutex_init(&tx_ring->tcb_head_lock, NULL,
1205 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1206 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1207 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1208 }
1209
1210 mutex_init(&ixgbe->gen_lock, NULL,
1211 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1212
1213 mutex_init(&ixgbe->watchdog_lock, NULL,
1214 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1215 }
1216
1217 /*
1218 * ixgbe_destroy_locks - Destroy locks.
1219 */
1220 static void
1221 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1222 {
1223 ixgbe_rx_ring_t *rx_ring;
1224 ixgbe_tx_ring_t *tx_ring;
1225 int i;
1226
1227 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1228 rx_ring = &ixgbe->rx_rings[i];
1229 mutex_destroy(&rx_ring->rx_lock);
1230 }
1231
1232 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1233 tx_ring = &ixgbe->tx_rings[i];
1234 mutex_destroy(&tx_ring->tx_lock);
1235 mutex_destroy(&tx_ring->recycle_lock);
1236 mutex_destroy(&tx_ring->tcb_head_lock);
1237 mutex_destroy(&tx_ring->tcb_tail_lock);
1238 }
1239
1240 mutex_destroy(&ixgbe->gen_lock);
1241 mutex_destroy(&ixgbe->watchdog_lock);
1242 }
1243
1244 static int
1245 ixgbe_resume(dev_info_t *devinfo)
1246 {
1247 ixgbe_t *ixgbe;
1248 int i;
1249
1250 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1251 if (ixgbe == NULL)
1252 return (DDI_FAILURE);
1253
1254 mutex_enter(&ixgbe->gen_lock);
1255
1256 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1257 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1258 mutex_exit(&ixgbe->gen_lock);
1259 return (DDI_FAILURE);
1260 }
1261
1262 /*
1263 * Enable and start the watchdog timer
1264 */
1265 ixgbe_enable_watchdog_timer(ixgbe);
1266 }
1267
1268 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1269
1270 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1271 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1272 mac_tx_ring_update(ixgbe->mac_hdl,
1273 ixgbe->tx_rings[i].ring_handle);
1274 }
1275 }
1276
1277 mutex_exit(&ixgbe->gen_lock);
1278
1279 return (DDI_SUCCESS);
1280 }
1281
1282 static int
1283 ixgbe_suspend(dev_info_t *devinfo)
1284 {
1285 ixgbe_t *ixgbe;
1286
1287 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1288 if (ixgbe == NULL)
1289 return (DDI_FAILURE);
1290
1291 mutex_enter(&ixgbe->gen_lock);
1292
1293 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1294 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1295 mutex_exit(&ixgbe->gen_lock);
1296 return (DDI_SUCCESS);
1297 }
1298 ixgbe_stop(ixgbe, B_FALSE);
1299
1300 mutex_exit(&ixgbe->gen_lock);
1301
1302 /*
1303 * Disable and stop the watchdog timer
1304 */
1305 ixgbe_disable_watchdog_timer(ixgbe);
1306
1307 return (DDI_SUCCESS);
1308 }
1309
1310 /*
1311 * ixgbe_init - Initialize the device.
1312 */
1313 static int
1314 ixgbe_init(ixgbe_t *ixgbe)
1315 {
1316 struct ixgbe_hw *hw = &ixgbe->hw;
1317 u8 pbanum[IXGBE_PBANUM_LENGTH];
1318 int rv;
1319
1320 mutex_enter(&ixgbe->gen_lock);
1321
1322 /*
1323 * Configure/Initialize hardware
1324 */
1325 rv = ixgbe_init_hw(hw);
1326 if (rv != IXGBE_SUCCESS) {
1327 switch (rv) {
1328
1329 /*
1330 * The first three errors are not prohibitive to us progressing
1331 * further, and are maily advisory in nature. In the case of a
1332 * SFP module not being present or not deemed supported by the
1333 * common code, we adivse the operator of this fact but carry on
1334 * instead of failing hard, as SFPs can be inserted or replaced
1335 * while the driver is running. In the case of a unknown error,
1336 * we fail-hard, logging the reason and emitting a FMA event.
1337 */
1338 case IXGBE_ERR_EEPROM_VERSION:
1339 ixgbe_error(ixgbe,
1340 "This Intel 10Gb Ethernet device is pre-release and"
1341 " contains outdated firmware. Please contact your"
1342 " hardware vendor for a replacement.");
1343 break;
1344 case IXGBE_ERR_SFP_NOT_PRESENT:
1345 ixgbe_error(ixgbe,
1346 "No SFP+ module detected on this interface. Please "
1347 "install a supported SFP+ module for this "
1348 "interface to become operational.");
1349 break;
1350 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1351 ixgbe_error(ixgbe,
1352 "Unsupported SFP+ module detected. Please replace "
1353 "it with a supported SFP+ module per Intel "
1354 "documentation, or bypass this check with "
1355 "allow_unsupported_sfp=1 in ixgbe.conf.");
1356 break;
1357 default:
1358 ixgbe_error(ixgbe,
1359 "Failed to initialize hardware. ixgbe_init_hw "
1360 "returned %d", rv);
1361 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1362 goto init_fail;
1363 }
1364 }
1365
1366 /*
1367 * Need to init eeprom before validating the checksum.
1368 */
1369 if (ixgbe_init_eeprom_params(hw) < 0) {
1370 ixgbe_error(ixgbe,
1371 "Unable to intitialize the eeprom interface.");
1372 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1373 goto init_fail;
1374 }
1375
1376 /*
1377 * NVM validation
1378 */
1379 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1380 /*
1381 * Some PCI-E parts fail the first check due to
1382 * the link being in sleep state. Call it again,
1383 * if it fails a second time it's a real issue.
1384 */
1385 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1386 ixgbe_error(ixgbe,
1387 "Invalid NVM checksum. Please contact "
1388 "the vendor to update the NVM.");
1389 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1390 goto init_fail;
1391 }
1392 }
1393
1394 /*
1395 * Setup default flow control thresholds - enable/disable
1396 * & flow control type is controlled by ixgbe.conf
1397 */
1398 hw->fc.high_water[0] = DEFAULT_FCRTH;
1399 hw->fc.low_water[0] = DEFAULT_FCRTL;
1400 hw->fc.pause_time = DEFAULT_FCPAUSE;
1401 hw->fc.send_xon = B_TRUE;
1402
1403 /*
1404 * Initialize flow control
1405 */
1406 (void) ixgbe_start_hw(hw);
1407
1408 /*
1409 * Initialize link settings
1410 */
1411 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1412
1413 /*
1414 * Initialize the chipset hardware
1415 */
1416 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1417 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1418 goto init_fail;
1419 }
1420
1421 /*
1422 * Read identifying information and place in devinfo.
1423 */
1424 pbanum[0] = '\0';
1425 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1426 if (*pbanum != '\0') {
1427 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1428 "printed-board-assembly", (char *)pbanum);
1429 }
1430
1431 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1432 goto init_fail;
1433 }
1434
1435 mutex_exit(&ixgbe->gen_lock);
1436 return (IXGBE_SUCCESS);
1437
1438 init_fail:
1439 /*
1440 * Reset PHY
1441 */
1442 (void) ixgbe_reset_phy(hw);
1443
1444 mutex_exit(&ixgbe->gen_lock);
1445 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1446 return (IXGBE_FAILURE);
1447 }
1448
1449 /*
1450 * ixgbe_chip_start - Initialize and start the chipset hardware.
1451 */
1452 static int
1453 ixgbe_chip_start(ixgbe_t *ixgbe)
1454 {
1455 struct ixgbe_hw *hw = &ixgbe->hw;
1456 int i;
1457
1458 ASSERT(mutex_owned(&ixgbe->gen_lock));
1459
1460 /*
1461 * Get the mac address
1462 * This function should handle SPARC case correctly.
1463 */
1464 if (!ixgbe_find_mac_address(ixgbe)) {
1465 ixgbe_error(ixgbe, "Failed to get the mac address");
1466 return (IXGBE_FAILURE);
1467 }
1468
1469 /*
1470 * Validate the mac address
1471 */
1472 (void) ixgbe_init_rx_addrs(hw);
1473 if (!is_valid_mac_addr(hw->mac.addr)) {
1474 ixgbe_error(ixgbe, "Invalid mac address");
1475 return (IXGBE_FAILURE);
1476 }
1477
1478 /*
1479 * Re-enable relaxed ordering for performance. It is disabled
1480 * by default in the hardware init.
1481 */
1482 if (ixgbe->relax_order_enable == B_TRUE)
1483 ixgbe_enable_relaxed_ordering(hw);
1484
1485 /*
1486 * Setup adapter interrupt vectors
1487 */
1488 ixgbe_setup_adapter_vector(ixgbe);
1489
1490 /*
1491 * Initialize unicast addresses.
1492 */
1493 ixgbe_init_unicst(ixgbe);
1494
1495 /*
1496 * Setup and initialize the mctable structures.
1497 */
1498 ixgbe_setup_multicst(ixgbe);
1499
1500 /*
1501 * Set interrupt throttling rate
1502 */
1503 for (i = 0; i < ixgbe->intr_cnt; i++) {
1504 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1505 }
1506
1507 /*
1508 * Disable Wake-on-LAN
1509 */
1510 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1511
1512 /*
1513 * Some adapters offer Energy Efficient Ethernet (EEE) support.
1514 * Due to issues with EEE in e1000g/igb, we disable this by default
1515 * as a precautionary measure.
1516 *
1517 * Currently, the only known adapter which supports EEE in the ixgbe
1518 * line is 8086,15AB (IXGBE_DEV_ID_X550EM_X_KR), and only after the
1519 * first revision of it, as well as any X550 with MAC type 6 (non-EM)
1520 */
1521 (void) ixgbe_setup_eee(hw, B_FALSE);
1522
1523 /*
1524 * Turn on any present SFP Tx laser
1525 */
1526 ixgbe_enable_tx_laser(hw);
1527
1528 /*
1529 * Power on the PHY
1530 */
1531 (void) ixgbe_set_phy_power(hw, B_TRUE);
1532
1533 /*
1534 * Save the state of the PHY
1535 */
1536 ixgbe_get_hw_state(ixgbe);
1537
1538 /*
1539 * Make sure driver has control
1540 */
1541 ixgbe_get_driver_control(hw);
1542
1543 return (IXGBE_SUCCESS);
1544 }
1545
1546 /*
1547 * ixgbe_chip_stop - Stop the chipset hardware
1548 */
1549 static void
1550 ixgbe_chip_stop(ixgbe_t *ixgbe)
1551 {
1552 struct ixgbe_hw *hw = &ixgbe->hw;
1553 int rv;
1554
1555 ASSERT(mutex_owned(&ixgbe->gen_lock));
1556
1557 /*
1558 * Stop interupt generation and disable Tx unit
1559 */
1560 hw->adapter_stopped = B_FALSE;
1561 (void) ixgbe_stop_adapter(hw);
1562
1563 /*
1564 * Reset the chipset
1565 */
1566 (void) ixgbe_reset_hw(hw);
1567
1568 /*
1569 * Reset PHY
1570 */
1571 (void) ixgbe_reset_phy(hw);
1572
1573 /*
1574 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1575 * the PHY while doing so. Else, just power down the PHY.
1576 */
1577 if (hw->phy.ops.enter_lplu != NULL) {
1578 hw->phy.reset_disable = B_TRUE;
1579 rv = hw->phy.ops.enter_lplu(hw);
1580 if (rv != IXGBE_SUCCESS)
1581 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1582 hw->phy.reset_disable = B_FALSE;
1583 } else {
1584 (void) ixgbe_set_phy_power(hw, B_FALSE);
1585 }
1586
1587 /*
1588 * Turn off any present SFP Tx laser
1589 * Expected for health and safety reasons
1590 */
1591 ixgbe_disable_tx_laser(hw);
1592
1593 /*
1594 * Tell firmware driver is no longer in control
1595 */
1596 ixgbe_release_driver_control(hw);
1597
1598 }
1599
1600 /*
1601 * ixgbe_reset - Reset the chipset and re-start the driver.
1602 *
1603 * It involves stopping and re-starting the chipset,
1604 * and re-configuring the rx/tx rings.
1605 */
1606 static int
1607 ixgbe_reset(ixgbe_t *ixgbe)
1608 {
1609 int i;
1610
1611 /*
1612 * Disable and stop the watchdog timer
1613 */
1614 ixgbe_disable_watchdog_timer(ixgbe);
1615
1616 mutex_enter(&ixgbe->gen_lock);
1617
1618 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1619 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1620
1621 ixgbe_stop(ixgbe, B_FALSE);
1622
1623 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1624 mutex_exit(&ixgbe->gen_lock);
1625 return (IXGBE_FAILURE);
1626 }
1627
1628 /*
1629 * After resetting, need to recheck the link status.
1630 */
1631 ixgbe->link_check_complete = B_FALSE;
1632 ixgbe->link_check_hrtime = gethrtime() +
1633 (IXGBE_LINK_UP_TIME * 100000000ULL);
1634
1635 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1636
1637 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1638 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1639 mac_tx_ring_update(ixgbe->mac_hdl,
1640 ixgbe->tx_rings[i].ring_handle);
1641 }
1642 }
1643
1644 mutex_exit(&ixgbe->gen_lock);
1645
1646 /*
1647 * Enable and start the watchdog timer
1648 */
1649 ixgbe_enable_watchdog_timer(ixgbe);
1650
1651 return (IXGBE_SUCCESS);
1652 }
1653
1654 /*
1655 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1656 */
1657 static void
1658 ixgbe_tx_clean(ixgbe_t *ixgbe)
1659 {
1660 ixgbe_tx_ring_t *tx_ring;
1661 tx_control_block_t *tcb;
1662 link_list_t pending_list;
1663 uint32_t desc_num;
1664 int i, j;
1665
1666 LINK_LIST_INIT(&pending_list);
1667
1668 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1669 tx_ring = &ixgbe->tx_rings[i];
1670
1671 mutex_enter(&tx_ring->recycle_lock);
1672
1673 /*
1674 * Clean the pending tx data - the pending packets in the
1675 * work_list that have no chances to be transmitted again.
1676 *
1677 * We must ensure the chipset is stopped or the link is down
1678 * before cleaning the transmit packets.
1679 */
1680 desc_num = 0;
1681 for (j = 0; j < tx_ring->ring_size; j++) {
1682 tcb = tx_ring->work_list[j];
1683 if (tcb != NULL) {
1684 desc_num += tcb->desc_num;
1685
1686 tx_ring->work_list[j] = NULL;
1687
1688 ixgbe_free_tcb(tcb);
1689
1690 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1691 }
1692 }
1693
1694 if (desc_num > 0) {
1695 atomic_add_32(&tx_ring->tbd_free, desc_num);
1696 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1697
1698 /*
1699 * Reset the head and tail pointers of the tbd ring;
1700 * Reset the writeback head if it's enable.
1701 */
1702 tx_ring->tbd_head = 0;
1703 tx_ring->tbd_tail = 0;
1704 if (ixgbe->tx_head_wb_enable)
1705 *tx_ring->tbd_head_wb = 0;
1706
1707 IXGBE_WRITE_REG(&ixgbe->hw,
1708 IXGBE_TDH(tx_ring->index), 0);
1709 IXGBE_WRITE_REG(&ixgbe->hw,
1710 IXGBE_TDT(tx_ring->index), 0);
1711 }
1712
1713 mutex_exit(&tx_ring->recycle_lock);
1714
1715 /*
1716 * Add the tx control blocks in the pending list to
1717 * the free list.
1718 */
1719 ixgbe_put_free_list(tx_ring, &pending_list);
1720 }
1721 }
1722
1723 /*
1724 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1725 * transmitted.
1726 */
1727 static boolean_t
1728 ixgbe_tx_drain(ixgbe_t *ixgbe)
1729 {
1730 ixgbe_tx_ring_t *tx_ring;
1731 boolean_t done;
1732 int i, j;
1733
1734 /*
1735 * Wait for a specific time to allow pending tx packets
1736 * to be transmitted.
1737 *
1738 * Check the counter tbd_free to see if transmission is done.
1739 * No lock protection is needed here.
1740 *
1741 * Return B_TRUE if all pending packets have been transmitted;
1742 * Otherwise return B_FALSE;
1743 */
1744 for (i = 0; i < TX_DRAIN_TIME; i++) {
1745
1746 done = B_TRUE;
1747 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1748 tx_ring = &ixgbe->tx_rings[j];
1749 done = done &&
1750 (tx_ring->tbd_free == tx_ring->ring_size);
1751 }
1752
1753 if (done)
1754 break;
1755
1756 msec_delay(1);
1757 }
1758
1759 return (done);
1760 }
1761
1762 /*
1763 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1764 */
1765 static boolean_t
1766 ixgbe_rx_drain(ixgbe_t *ixgbe)
1767 {
1768 boolean_t done = B_TRUE;
1769 int i;
1770
1771 /*
1772 * Polling the rx free list to check if those rx buffers held by
1773 * the upper layer are released.
1774 *
1775 * Check the counter rcb_free to see if all pending buffers are
1776 * released. No lock protection is needed here.
1777 *
1778 * Return B_TRUE if all pending buffers have been released;
1779 * Otherwise return B_FALSE;
1780 */
1781 for (i = 0; i < RX_DRAIN_TIME; i++) {
1782 done = (ixgbe->rcb_pending == 0);
1783
1784 if (done)
1785 break;
1786
1787 msec_delay(1);
1788 }
1789
1790 return (done);
1791 }
1792
1793 /*
1794 * ixgbe_start - Start the driver/chipset.
1795 */
1796 int
1797 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1798 {
1799 struct ixgbe_hw *hw = &ixgbe->hw;
1800 int i;
1801
1802 ASSERT(mutex_owned(&ixgbe->gen_lock));
1803
1804 if (alloc_buffer) {
1805 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1806 ixgbe_error(ixgbe,
1807 "Failed to allocate software receive rings");
1808 return (IXGBE_FAILURE);
1809 }
1810
1811 /* Allocate buffers for all the rx/tx rings */
1812 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1813 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1814 return (IXGBE_FAILURE);
1815 }
1816
1817 ixgbe->tx_ring_init = B_TRUE;
1818 } else {
1819 ixgbe->tx_ring_init = B_FALSE;
1820 }
1821
1822 for (i = 0; i < ixgbe->num_rx_rings; i++)
1823 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1824 for (i = 0; i < ixgbe->num_tx_rings; i++)
1825 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1826
1827 /*
1828 * Start the chipset hardware
1829 */
1830 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1831 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1832 goto start_failure;
1833 }
1834
1835 /*
1836 * Configure link now for X550
1837 *
1838 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1839 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1840 * the resting state of the link would be the maximum speed that
1841 * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1842 * so we never bothered with explicitly setting the link to 10Gb as it
1843 * would already be at that state on driver attach. With X550, we must
1844 * trigger a re-negotiation of the link in order to switch from a LPLU
1845 * 1Gb link to 10Gb (cable and link partner permitting.)
1846 */
1847 if (hw->mac.type == ixgbe_mac_X550 ||
1848 hw->mac.type == ixgbe_mac_X550EM_x) {
1849 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1850 ixgbe_get_hw_state(ixgbe);
1851 }
1852
1853 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1854 goto start_failure;
1855 }
1856
1857 /*
1858 * Setup the rx/tx rings
1859 */
1860 ixgbe_setup_rings(ixgbe);
1861
1862 /*
1863 * ixgbe_start() will be called when resetting, however if reset
1864 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1865 * before enabling the interrupts.
1866 */
1867 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1868 | IXGBE_STALL| IXGBE_OVERTEMP));
1869
1870 /*
1871 * Enable adapter interrupts
1872 * The interrupts must be enabled after the driver state is START
1873 */
1874 ixgbe_enable_adapter_interrupts(ixgbe);
1875
1876 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1877 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1878 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1879 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1880
1881 return (IXGBE_SUCCESS);
1882
1883 start_failure:
1884 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1885 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1886 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1887 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1888
1889 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1890
1891 return (IXGBE_FAILURE);
1892 }
1893
1894 /*
1895 * ixgbe_stop - Stop the driver/chipset.
1896 */
1897 void
1898 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1899 {
1900 int i;
1901
1902 ASSERT(mutex_owned(&ixgbe->gen_lock));
1903
1904 /*
1905 * Disable the adapter interrupts
1906 */
1907 ixgbe_disable_adapter_interrupts(ixgbe);
1908
1909 /*
1910 * Drain the pending tx packets
1911 */
1912 (void) ixgbe_tx_drain(ixgbe);
1913
1914 for (i = 0; i < ixgbe->num_rx_rings; i++)
1915 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1916 for (i = 0; i < ixgbe->num_tx_rings; i++)
1917 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1918
1919 /*
1920 * Stop the chipset hardware
1921 */
1922 ixgbe_chip_stop(ixgbe);
1923
1924 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1925 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1926 }
1927
1928 /*
1929 * Clean the pending tx data/resources
1930 */
1931 ixgbe_tx_clean(ixgbe);
1932
1933 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1934 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1935 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1936 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1937
1938 if (ixgbe->link_state == LINK_STATE_UP) {
1939 ixgbe->link_state = LINK_STATE_UNKNOWN;
1940 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1941 }
1942
1943 if (free_buffer) {
1944 /*
1945 * Release the DMA/memory resources of rx/tx rings
1946 */
1947 ixgbe_free_dma(ixgbe);
1948 ixgbe_free_rx_data(ixgbe);
1949 }
1950 }
1951
1952 /*
1953 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1954 */
1955 /* ARGSUSED */
1956 static int
1957 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1958 void *arg1, void *arg2)
1959 {
1960 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1961
1962 switch (cbaction) {
1963 /* IRM callback */
1964 int count;
1965 case DDI_CB_INTR_ADD:
1966 case DDI_CB_INTR_REMOVE:
1967 count = (int)(uintptr_t)cbarg;
1968 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1969 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1970 int, ixgbe->intr_cnt);
1971 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1972 DDI_SUCCESS) {
1973 ixgbe_error(ixgbe,
1974 "IRM CB: Failed to adjust interrupts");
1975 goto cb_fail;
1976 }
1977 break;
1978 default:
1979 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1980 cbaction);
1981 return (DDI_ENOTSUP);
1982 }
1983 return (DDI_SUCCESS);
1984 cb_fail:
1985 return (DDI_FAILURE);
1986 }
1987
1988 /*
1989 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1990 */
1991 static int
1992 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1993 {
1994 int i, rc, actual;
1995
1996 if (count == 0)
1997 return (DDI_SUCCESS);
1998
1999 if ((cbaction == DDI_CB_INTR_ADD &&
2000 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
2001 (cbaction == DDI_CB_INTR_REMOVE &&
2002 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
2003 return (DDI_FAILURE);
2004
2005 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
2006 return (DDI_FAILURE);
2007 }
2008
2009 for (i = 0; i < ixgbe->num_rx_rings; i++)
2010 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
2011 for (i = 0; i < ixgbe->num_tx_rings; i++)
2012 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
2013
2014 mutex_enter(&ixgbe->gen_lock);
2015 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
2016 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
2017 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
2018 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
2019
2020 ixgbe_stop(ixgbe, B_FALSE);
2021 /*
2022 * Disable interrupts
2023 */
2024 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
2025 rc = ixgbe_disable_intrs(ixgbe);
2026 ASSERT(rc == IXGBE_SUCCESS);
2027 }
2028 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
2029
2030 /*
2031 * Remove interrupt handlers
2032 */
2033 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
2034 ixgbe_rem_intr_handlers(ixgbe);
2035 }
2036 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
2037
2038 /*
2039 * Clear vect_map
2040 */
2041 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
2042 switch (cbaction) {
2043 case DDI_CB_INTR_ADD:
2044 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
2045 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
2046 DDI_INTR_ALLOC_NORMAL);
2047 if (rc != DDI_SUCCESS || actual != count) {
2048 ixgbe_log(ixgbe, "Adjust interrupts failed."
2049 "return: %d, irm cb size: %d, actual: %d",
2050 rc, count, actual);
2051 goto intr_adjust_fail;
2052 }
2053 ixgbe->intr_cnt += count;
2054 break;
2055
2056 case DDI_CB_INTR_REMOVE:
2057 for (i = ixgbe->intr_cnt - count;
2058 i < ixgbe->intr_cnt; i ++) {
2059 rc = ddi_intr_free(ixgbe->htable[i]);
2060 ixgbe->htable[i] = NULL;
2061 if (rc != DDI_SUCCESS) {
2062 ixgbe_log(ixgbe, "Adjust interrupts failed."
2063 "return: %d, irm cb size: %d, actual: %d",
2064 rc, count, actual);
2065 goto intr_adjust_fail;
2066 }
2067 }
2068 ixgbe->intr_cnt -= count;
2069 break;
2070 }
2071
2072 /*
2073 * Get priority for first vector, assume remaining are all the same
2074 */
2075 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
2076 if (rc != DDI_SUCCESS) {
2077 ixgbe_log(ixgbe,
2078 "Get interrupt priority failed: %d", rc);
2079 goto intr_adjust_fail;
2080 }
2081 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
2082 if (rc != DDI_SUCCESS) {
2083 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
2084 goto intr_adjust_fail;
2085 }
2086 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
2087
2088 /*
2089 * Map rings to interrupt vectors
2090 */
2091 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
2092 ixgbe_error(ixgbe,
2093 "IRM CB: Failed to map interrupts to vectors");
2094 goto intr_adjust_fail;
2095 }
2096
2097 /*
2098 * Add interrupt handlers
2099 */
2100 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
2101 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
2102 goto intr_adjust_fail;
2103 }
2104 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
2105
2106 /*
2107 * Now that mutex locks are initialized, and the chip is also
2108 * initialized, enable interrupts.
2109 */
2110 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
2111 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
2112 goto intr_adjust_fail;
2113 }
2114 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
2115 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
2116 ixgbe_error(ixgbe, "IRM CB: Failed to start");
2117 goto intr_adjust_fail;
2118 }
2119 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
2120 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
2121 ixgbe->ixgbe_state |= IXGBE_STARTED;
2122 mutex_exit(&ixgbe->gen_lock);
2123
2124 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2125 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2126 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2127 }
2128 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2129 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2130 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2131 }
2132
2133 /* Wakeup all Tx rings */
2134 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2135 mac_tx_ring_update(ixgbe->mac_hdl,
2136 ixgbe->tx_rings[i].ring_handle);
2137 }
2138
2139 IXGBE_DEBUGLOG_3(ixgbe,
2140 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2141 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2142 return (DDI_SUCCESS);
2143
2144 intr_adjust_fail:
2145 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2146 mutex_exit(&ixgbe->gen_lock);
2147 return (DDI_FAILURE);
2148 }
2149
2150 /*
2151 * ixgbe_intr_cb_register - Register interrupt callback function.
2152 */
2153 static int
2154 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2155 {
2156 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2157 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2158 return (IXGBE_FAILURE);
2159 }
2160 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2161 return (IXGBE_SUCCESS);
2162 }
2163
2164 /*
2165 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2166 */
2167 static int
2168 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2169 {
2170 /*
2171 * Allocate memory space for rx rings
2172 */
2173 ixgbe->rx_rings = kmem_zalloc(
2174 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2175 KM_NOSLEEP);
2176
2177 if (ixgbe->rx_rings == NULL) {
2178 return (IXGBE_FAILURE);
2179 }
2180
2181 /*
2182 * Allocate memory space for tx rings
2183 */
2184 ixgbe->tx_rings = kmem_zalloc(
2185 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2186 KM_NOSLEEP);
2187
2188 if (ixgbe->tx_rings == NULL) {
2189 kmem_free(ixgbe->rx_rings,
2190 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2191 ixgbe->rx_rings = NULL;
2192 return (IXGBE_FAILURE);
2193 }
2194
2195 /*
2196 * Allocate memory space for rx ring groups
2197 */
2198 ixgbe->rx_groups = kmem_zalloc(
2199 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2200 KM_NOSLEEP);
2201
2202 if (ixgbe->rx_groups == NULL) {
2203 kmem_free(ixgbe->rx_rings,
2204 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2205 kmem_free(ixgbe->tx_rings,
2206 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2207 ixgbe->rx_rings = NULL;
2208 ixgbe->tx_rings = NULL;
2209 return (IXGBE_FAILURE);
2210 }
2211
2212 return (IXGBE_SUCCESS);
2213 }
2214
2215 /*
2216 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2217 */
2218 static void
2219 ixgbe_free_rings(ixgbe_t *ixgbe)
2220 {
2221 if (ixgbe->rx_rings != NULL) {
2222 kmem_free(ixgbe->rx_rings,
2223 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2224 ixgbe->rx_rings = NULL;
2225 }
2226
2227 if (ixgbe->tx_rings != NULL) {
2228 kmem_free(ixgbe->tx_rings,
2229 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2230 ixgbe->tx_rings = NULL;
2231 }
2232
2233 if (ixgbe->rx_groups != NULL) {
2234 kmem_free(ixgbe->rx_groups,
2235 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2236 ixgbe->rx_groups = NULL;
2237 }
2238 }
2239
2240 static int
2241 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2242 {
2243 ixgbe_rx_ring_t *rx_ring;
2244 int i;
2245
2246 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2247 rx_ring = &ixgbe->rx_rings[i];
2248 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2249 goto alloc_rx_rings_failure;
2250 }
2251 return (IXGBE_SUCCESS);
2252
2253 alloc_rx_rings_failure:
2254 ixgbe_free_rx_data(ixgbe);
2255 return (IXGBE_FAILURE);
2256 }
2257
2258 static void
2259 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2260 {
2261 ixgbe_rx_ring_t *rx_ring;
2262 ixgbe_rx_data_t *rx_data;
2263 int i;
2264
2265 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2266 rx_ring = &ixgbe->rx_rings[i];
2267
2268 mutex_enter(&ixgbe->rx_pending_lock);
2269 rx_data = rx_ring->rx_data;
2270
2271 if (rx_data != NULL) {
2272 rx_data->flag |= IXGBE_RX_STOPPED;
2273
2274 if (rx_data->rcb_pending == 0) {
2275 ixgbe_free_rx_ring_data(rx_data);
2276 rx_ring->rx_data = NULL;
2277 }
2278 }
2279
2280 mutex_exit(&ixgbe->rx_pending_lock);
2281 }
2282 }
2283
2284 /*
2285 * ixgbe_setup_rings - Setup rx/tx rings.
2286 */
2287 static void
2288 ixgbe_setup_rings(ixgbe_t *ixgbe)
2289 {
2290 /*
2291 * Setup the rx/tx rings, including the following:
2292 *
2293 * 1. Setup the descriptor ring and the control block buffers;
2294 * 2. Initialize necessary registers for receive/transmit;
2295 * 3. Initialize software pointers/parameters for receive/transmit;
2296 */
2297 ixgbe_setup_rx(ixgbe);
2298
2299 ixgbe_setup_tx(ixgbe);
2300 }
2301
2302 static void
2303 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2304 {
2305 ixgbe_t *ixgbe = rx_ring->ixgbe;
2306 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2307 struct ixgbe_hw *hw = &ixgbe->hw;
2308 rx_control_block_t *rcb;
2309 union ixgbe_adv_rx_desc *rbd;
2310 uint32_t size;
2311 uint32_t buf_low;
2312 uint32_t buf_high;
2313 uint32_t reg_val;
2314 int i;
2315
2316 ASSERT(mutex_owned(&rx_ring->rx_lock));
2317 ASSERT(mutex_owned(&ixgbe->gen_lock));
2318
2319 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2320 rcb = rx_data->work_list[i];
2321 rbd = &rx_data->rbd_ring[i];
2322
2323 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2324 rbd->read.hdr_addr = NULL;
2325 }
2326
2327 /*
2328 * Initialize the length register
2329 */
2330 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2331 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2332
2333 /*
2334 * Initialize the base address registers
2335 */
2336 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2337 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2338 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2339 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2340
2341 /*
2342 * Setup head & tail pointers
2343 */
2344 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2345 rx_data->ring_size - 1);
2346 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2347
2348 rx_data->rbd_next = 0;
2349 rx_data->lro_first = 0;
2350
2351 /*
2352 * Setup the Receive Descriptor Control Register (RXDCTL)
2353 * PTHRESH=32 descriptors (half the internal cache)
2354 * HTHRESH=0 descriptors (to minimize latency on fetch)
2355 * WTHRESH defaults to 1 (writeback each descriptor)
2356 */
2357 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2358 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2359
2360 /* Not a valid value for 82599, X540 or X550 */
2361 if (hw->mac.type == ixgbe_mac_82598EB) {
2362 reg_val |= 0x0020; /* pthresh */
2363 }
2364 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2365
2366 if (hw->mac.type == ixgbe_mac_82599EB ||
2367 hw->mac.type == ixgbe_mac_X540 ||
2368 hw->mac.type == ixgbe_mac_X550 ||
2369 hw->mac.type == ixgbe_mac_X550EM_x) {
2370 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2371 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2372 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2373 }
2374
2375 /*
2376 * Setup the Split and Replication Receive Control Register.
2377 * Set the rx buffer size and the advanced descriptor type.
2378 */
2379 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2380 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2381 reg_val |= IXGBE_SRRCTL_DROP_EN;
2382 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2383 }
2384
2385 static void
2386 ixgbe_setup_rx(ixgbe_t *ixgbe)
2387 {
2388 ixgbe_rx_ring_t *rx_ring;
2389 struct ixgbe_hw *hw = &ixgbe->hw;
2390 uint32_t reg_val;
2391 uint32_t ring_mapping;
2392 uint32_t i, index;
2393 uint32_t psrtype_rss_bit;
2394
2395 /*
2396 * Ensure that Rx is disabled while setting up
2397 * the Rx unit and Rx descriptor ring(s)
2398 */
2399 ixgbe_disable_rx(hw);
2400
2401 /* PSRTYPE must be configured for 82599 */
2402 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2403 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2404 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2405 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2406 reg_val |= IXGBE_PSRTYPE_L2HDR;
2407 reg_val |= 0x80000000;
2408 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2409 } else {
2410 if (ixgbe->num_rx_groups > 32) {
2411 psrtype_rss_bit = 0x20000000;
2412 } else {
2413 psrtype_rss_bit = 0x40000000;
2414 }
2415 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2416 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2417 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2418 reg_val |= IXGBE_PSRTYPE_L2HDR;
2419 reg_val |= psrtype_rss_bit;
2420 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2421 }
2422 }
2423
2424 /*
2425 * Set filter control in FCTRL to determine types of packets are passed
2426 * up to the driver.
2427 * - Pass broadcast packets.
2428 * - Do not pass flow control pause frames (82598-specific)
2429 */
2430 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2431 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2432 if (hw->mac.type == ixgbe_mac_82598EB) {
2433 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2434 }
2435 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2436
2437 /*
2438 * Hardware checksum settings
2439 */
2440 if (ixgbe->rx_hcksum_enable) {
2441 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2442 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2443 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2444 }
2445
2446 /*
2447 * Setup VMDq and RSS for multiple receive queues
2448 */
2449 switch (ixgbe->classify_mode) {
2450 case IXGBE_CLASSIFY_RSS:
2451 /*
2452 * One group, only RSS is needed when more than
2453 * one ring enabled.
2454 */
2455 ixgbe_setup_rss(ixgbe);
2456 break;
2457
2458 case IXGBE_CLASSIFY_VMDQ:
2459 /*
2460 * Multiple groups, each group has one ring,
2461 * only VMDq is needed.
2462 */
2463 ixgbe_setup_vmdq(ixgbe);
2464 break;
2465
2466 case IXGBE_CLASSIFY_VMDQ_RSS:
2467 /*
2468 * Multiple groups and multiple rings, both
2469 * VMDq and RSS are needed.
2470 */
2471 ixgbe_setup_vmdq_rss(ixgbe);
2472 break;
2473
2474 default:
2475 break;
2476 }
2477
2478 /*
2479 * Enable the receive unit. This must be done after filter
2480 * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2481 * 82598 is the only adapter which defines this RXCTRL option.
2482 */
2483 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2484 if (hw->mac.type == ixgbe_mac_82598EB)
2485 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2486 reg_val |= IXGBE_RXCTRL_RXEN;
2487 (void) ixgbe_enable_rx_dma(hw, reg_val);
2488
2489 /*
2490 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2491 */
2492 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2493 rx_ring = &ixgbe->rx_rings[i];
2494 ixgbe_setup_rx_ring(rx_ring);
2495 }
2496
2497 /*
2498 * Setup the per-ring statistics mapping.
2499 */
2500 ring_mapping = 0;
2501 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2502 index = ixgbe->rx_rings[i].hw_index;
2503 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2504 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2505 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2506 }
2507
2508 /*
2509 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2510 * by four bytes if the packet has a VLAN field, so includes MTU,
2511 * ethernet header and frame check sequence.
2512 * Register is MAXFRS in 82599.
2513 */
2514 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2515 reg_val &= ~IXGBE_MHADD_MFS_MASK;
2516 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2517 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2518 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2519
2520 /*
2521 * Setup Jumbo Frame enable bit
2522 */
2523 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2524 if (ixgbe->default_mtu > ETHERMTU)
2525 reg_val |= IXGBE_HLREG0_JUMBOEN;
2526 else
2527 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2528 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2529
2530 /*
2531 * Setup RSC for multiple receive queues.
2532 */
2533 if (ixgbe->lro_enable) {
2534 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2535 /*
2536 * Make sure rx_buf_size * MAXDESC not greater
2537 * than 65535.
2538 * Intel recommends 4 for MAXDESC field value.
2539 */
2540 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2541 reg_val |= IXGBE_RSCCTL_RSCEN;
2542 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2543 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2544 else
2545 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2546 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2547 }
2548
2549 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2550 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2551 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2552
2553 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2554 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2555 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2556 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2557
2558 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2559 }
2560 }
2561
2562 static void
2563 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2564 {
2565 ixgbe_t *ixgbe = tx_ring->ixgbe;
2566 struct ixgbe_hw *hw = &ixgbe->hw;
2567 uint32_t size;
2568 uint32_t buf_low;
2569 uint32_t buf_high;
2570 uint32_t reg_val;
2571
2572 ASSERT(mutex_owned(&tx_ring->tx_lock));
2573 ASSERT(mutex_owned(&ixgbe->gen_lock));
2574
2575 /*
2576 * Initialize the length register
2577 */
2578 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2579 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2580
2581 /*
2582 * Initialize the base address registers
2583 */
2584 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2585 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2586 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2587 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2588
2589 /*
2590 * Setup head & tail pointers
2591 */
2592 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2593 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2594
2595 /*
2596 * Setup head write-back
2597 */
2598 if (ixgbe->tx_head_wb_enable) {
2599 /*
2600 * The memory of the head write-back is allocated using
2601 * the extra tbd beyond the tail of the tbd ring.
2602 */
2603 tx_ring->tbd_head_wb = (uint32_t *)
2604 ((uintptr_t)tx_ring->tbd_area.address + size);
2605 *tx_ring->tbd_head_wb = 0;
2606
2607 buf_low = (uint32_t)
2608 (tx_ring->tbd_area.dma_address + size);
2609 buf_high = (uint32_t)
2610 ((tx_ring->tbd_area.dma_address + size) >> 32);
2611
2612 /* Set the head write-back enable bit */
2613 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2614
2615 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2616 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2617
2618 /*
2619 * Turn off relaxed ordering for head write back or it will
2620 * cause problems with the tx recycling
2621 */
2622
2623 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2624 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2625 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2626 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2627 if (hw->mac.type == ixgbe_mac_82598EB) {
2628 IXGBE_WRITE_REG(hw,
2629 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2630 } else {
2631 IXGBE_WRITE_REG(hw,
2632 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2633 }
2634 } else {
2635 tx_ring->tbd_head_wb = NULL;
2636 }
2637
2638 tx_ring->tbd_head = 0;
2639 tx_ring->tbd_tail = 0;
2640 tx_ring->tbd_free = tx_ring->ring_size;
2641
2642 if (ixgbe->tx_ring_init == B_TRUE) {
2643 tx_ring->tcb_head = 0;
2644 tx_ring->tcb_tail = 0;
2645 tx_ring->tcb_free = tx_ring->free_list_size;
2646 }
2647
2648 /*
2649 * Initialize the s/w context structure
2650 */
2651 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2652 }
2653
2654 static void
2655 ixgbe_setup_tx(ixgbe_t *ixgbe)
2656 {
2657 struct ixgbe_hw *hw = &ixgbe->hw;
2658 ixgbe_tx_ring_t *tx_ring;
2659 uint32_t reg_val;
2660 uint32_t ring_mapping;
2661 int i;
2662
2663 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2664 tx_ring = &ixgbe->tx_rings[i];
2665 ixgbe_setup_tx_ring(tx_ring);
2666 }
2667
2668 /*
2669 * Setup the per-ring statistics mapping.
2670 */
2671 ring_mapping = 0;
2672 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2673 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2674 if ((i & 0x3) == 0x3) {
2675 switch (hw->mac.type) {
2676 case ixgbe_mac_82598EB:
2677 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2678 ring_mapping);
2679 break;
2680
2681 case ixgbe_mac_82599EB:
2682 case ixgbe_mac_X540:
2683 case ixgbe_mac_X550:
2684 case ixgbe_mac_X550EM_x:
2685 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2686 ring_mapping);
2687 break;
2688
2689 default:
2690 break;
2691 }
2692
2693 ring_mapping = 0;
2694 }
2695 }
2696 if (i & 0x3) {
2697 switch (hw->mac.type) {
2698 case ixgbe_mac_82598EB:
2699 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2700 break;
2701
2702 case ixgbe_mac_82599EB:
2703 case ixgbe_mac_X540:
2704 case ixgbe_mac_X550:
2705 case ixgbe_mac_X550EM_x:
2706 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2707 break;
2708
2709 default:
2710 break;
2711 }
2712 }
2713
2714 /*
2715 * Enable CRC appending and TX padding (for short tx frames)
2716 */
2717 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2718 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2719 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2720
2721 /*
2722 * enable DMA for 82599, X540 and X550 parts
2723 */
2724 if (hw->mac.type == ixgbe_mac_82599EB ||
2725 hw->mac.type == ixgbe_mac_X540 ||
2726 hw->mac.type == ixgbe_mac_X550 ||
2727 hw->mac.type == ixgbe_mac_X550EM_x) {
2728 /* DMATXCTL.TE must be set after all Tx config is complete */
2729 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2730 reg_val |= IXGBE_DMATXCTL_TE;
2731 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2732
2733 /* Disable arbiter to set MTQC */
2734 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2735 reg_val |= IXGBE_RTTDCS_ARBDIS;
2736 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2737 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2738 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2739 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2740 }
2741
2742 /*
2743 * Enabling tx queues ..
2744 * For 82599 must be done after DMATXCTL.TE is set
2745 */
2746 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2747 tx_ring = &ixgbe->tx_rings[i];
2748 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2749 reg_val |= IXGBE_TXDCTL_ENABLE;
2750 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2751 }
2752 }
2753
2754 /*
2755 * ixgbe_setup_rss - Setup receive-side scaling feature.
2756 */
2757 static void
2758 ixgbe_setup_rss(ixgbe_t *ixgbe)
2759 {
2760 struct ixgbe_hw *hw = &ixgbe->hw;
2761 uint32_t mrqc;
2762
2763 /*
2764 * Initialize RETA/ERETA table
2765 */
2766 ixgbe_setup_rss_table(ixgbe);
2767
2768 /*
2769 * Enable RSS & perform hash on these packet types
2770 */
2771 mrqc = IXGBE_MRQC_RSSEN |
2772 IXGBE_MRQC_RSS_FIELD_IPV4 |
2773 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2774 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2775 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2776 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2777 IXGBE_MRQC_RSS_FIELD_IPV6 |
2778 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2779 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2780 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2781 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2782 }
2783
2784 /*
2785 * ixgbe_setup_vmdq - Setup MAC classification feature
2786 */
2787 static void
2788 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2789 {
2790 struct ixgbe_hw *hw = &ixgbe->hw;
2791 uint32_t vmdctl, i, vtctl;
2792
2793 /*
2794 * Setup the VMDq Control register, enable VMDq based on
2795 * packet destination MAC address:
2796 */
2797 switch (hw->mac.type) {
2798 case ixgbe_mac_82598EB:
2799 /*
2800 * VMDq Enable = 1;
2801 * VMDq Filter = 0; MAC filtering
2802 * Default VMDq output index = 0;
2803 */
2804 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2805 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2806 break;
2807
2808 case ixgbe_mac_82599EB:
2809 case ixgbe_mac_X540:
2810 case ixgbe_mac_X550:
2811 case ixgbe_mac_X550EM_x:
2812 /*
2813 * Enable VMDq-only.
2814 */
2815 vmdctl = IXGBE_MRQC_VMDQEN;
2816 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2817
2818 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2819 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2820 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2821 }
2822
2823 /*
2824 * Enable Virtualization and Replication.
2825 */
2826 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2827 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2828
2829 /*
2830 * Enable receiving packets to all VFs
2831 */
2832 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2833 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2834 break;
2835
2836 default:
2837 break;
2838 }
2839 }
2840
2841 /*
2842 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2843 */
2844 static void
2845 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2846 {
2847 struct ixgbe_hw *hw = &ixgbe->hw;
2848 uint32_t i, mrqc;
2849 uint32_t vtctl, vmdctl;
2850
2851 /*
2852 * Initialize RETA/ERETA table
2853 */
2854 ixgbe_setup_rss_table(ixgbe);
2855
2856 /*
2857 * Enable and setup RSS and VMDq
2858 */
2859 switch (hw->mac.type) {
2860 case ixgbe_mac_82598EB:
2861 /*
2862 * Enable RSS & Setup RSS Hash functions
2863 */
2864 mrqc = IXGBE_MRQC_RSSEN |
2865 IXGBE_MRQC_RSS_FIELD_IPV4 |
2866 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2867 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2868 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2869 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2870 IXGBE_MRQC_RSS_FIELD_IPV6 |
2871 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2872 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2873 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2874 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2875
2876 /*
2877 * Enable and Setup VMDq
2878 * VMDq Filter = 0; MAC filtering
2879 * Default VMDq output index = 0;
2880 */
2881 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2882 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2883 break;
2884
2885 case ixgbe_mac_82599EB:
2886 case ixgbe_mac_X540:
2887 case ixgbe_mac_X550:
2888 case ixgbe_mac_X550EM_x:
2889 /*
2890 * Enable RSS & Setup RSS Hash functions
2891 */
2892 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2893 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2894 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2895 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2896 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2897 IXGBE_MRQC_RSS_FIELD_IPV6 |
2898 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2899 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2900 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2901
2902 /*
2903 * Enable VMDq+RSS.
2904 */
2905 if (ixgbe->num_rx_groups > 32) {
2906 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2907 } else {
2908 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2909 }
2910
2911 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2912
2913 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2914 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2915 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2916 }
2917 break;
2918
2919 default:
2920 break;
2921
2922 }
2923
2924 if (hw->mac.type == ixgbe_mac_82599EB ||
2925 hw->mac.type == ixgbe_mac_X540 ||
2926 hw->mac.type == ixgbe_mac_X550 ||
2927 hw->mac.type == ixgbe_mac_X550EM_x) {
2928 /*
2929 * Enable Virtualization and Replication.
2930 */
2931 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2932 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2933
2934 /*
2935 * Enable receiving packets to all VFs
2936 */
2937 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2938 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2939 }
2940 }
2941
2942 /*
2943 * ixgbe_setup_rss_table - Setup RSS table
2944 */
2945 static void
2946 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
2947 {
2948 struct ixgbe_hw *hw = &ixgbe->hw;
2949 uint32_t i, j;
2950 uint32_t random;
2951 uint32_t reta;
2952 uint32_t ring_per_group;
2953 uint32_t ring;
2954 uint32_t table_size;
2955 uint32_t index_mult;
2956 uint32_t rxcsum;
2957
2958 /*
2959 * Set multiplier for RETA setup and table size based on MAC type.
2960 * RETA table sizes vary by model:
2961 *
2962 * 82598, 82599, X540: 128 table entries.
2963 * X550: 512 table entries.
2964 */
2965 index_mult = 0x1;
2966 table_size = 128;
2967 switch (ixgbe->hw.mac.type) {
2968 case ixgbe_mac_82598EB:
2969 index_mult = 0x11;
2970 break;
2971 case ixgbe_mac_X550:
2972 case ixgbe_mac_X550EM_x:
2973 table_size = 512;
2974 break;
2975 default:
2976 break;
2977 }
2978
2979 /*
2980 * Fill out RSS redirection table. The configuation of the indices is
2981 * hardware-dependent.
2982 *
2983 * 82598: 8 bits wide containing two 4 bit RSS indices
2984 * 82599, X540: 8 bits wide containing one 4 bit RSS index
2985 * X550: 8 bits wide containing one 6 bit RSS index
2986 */
2987 reta = 0;
2988 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2989
2990 for (i = 0, j = 0; i < table_size; i++, j++) {
2991 if (j == ring_per_group) j = 0;
2992
2993 /*
2994 * The low 8 bits are for hash value (n+0);
2995 * The next 8 bits are for hash value (n+1), etc.
2996 */
2997 ring = (j * index_mult);
2998 reta = reta >> 8;
2999 reta = reta | (((uint32_t)ring) << 24);
3000
3001 if ((i & 3) == 3) {
3002 /*
3003 * The first 128 table entries are programmed into the
3004 * RETA register, with any beyond that (eg; on X550)
3005 * into ERETA.
3006 */
3007 if (i < 128)
3008 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3009 else
3010 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3011 reta);
3012 reta = 0;
3013 }
3014 }
3015
3016 /*
3017 * Fill out hash function seeds with a random constant
3018 */
3019 for (i = 0; i < 10; i++) {
3020 (void) random_get_pseudo_bytes((uint8_t *)&random,
3021 sizeof (uint32_t));
3022 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3023 }
3024
3025 /*
3026 * Disable Packet Checksum to enable RSS for multiple receive queues.
3027 * It is an adapter hardware limitation that Packet Checksum is
3028 * mutually exclusive with RSS.
3029 */
3030 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3031 rxcsum |= IXGBE_RXCSUM_PCSD;
3032 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3033 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3034 }
3035
3036 /*
3037 * ixgbe_init_unicst - Initialize the unicast addresses.
3038 */
3039 static void
3040 ixgbe_init_unicst(ixgbe_t *ixgbe)
3041 {
3042 struct ixgbe_hw *hw = &ixgbe->hw;
3043 uint8_t *mac_addr;
3044 int slot;
3045 /*
3046 * Here we should consider two situations:
3047 *
3048 * 1. Chipset is initialized at the first time,
3049 * Clear all the multiple unicast addresses.
3050 *
3051 * 2. Chipset is reset
3052 * Recover the multiple unicast addresses from the
3053 * software data structure to the RAR registers.
3054 */
3055 if (!ixgbe->unicst_init) {
3056 /*
3057 * Initialize the multiple unicast addresses
3058 */
3059 ixgbe->unicst_total = hw->mac.num_rar_entries;
3060 ixgbe->unicst_avail = ixgbe->unicst_total;
3061 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3062 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3063 bzero(mac_addr, ETHERADDRL);
3064 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
3065 ixgbe->unicst_addr[slot].mac.set = 0;
3066 }
3067 ixgbe->unicst_init = B_TRUE;
3068 } else {
3069 /* Re-configure the RAR registers */
3070 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3071 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3072 if (ixgbe->unicst_addr[slot].mac.set == 1) {
3073 (void) ixgbe_set_rar(hw, slot, mac_addr,
3074 ixgbe->unicst_addr[slot].mac.group_index,
3075 IXGBE_RAH_AV);
3076 } else {
3077 bzero(mac_addr, ETHERADDRL);
3078 (void) ixgbe_set_rar(hw, slot, mac_addr,
3079 NULL, NULL);
3080 }
3081 }
3082 }
3083 }
3084
3085 /*
3086 * ixgbe_unicst_find - Find the slot for the specified unicast address
3087 */
3088 int
3089 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
3090 {
3091 int slot;
3092
3093 ASSERT(mutex_owned(&ixgbe->gen_lock));
3094
3095 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3096 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
3097 mac_addr, ETHERADDRL) == 0)
3098 return (slot);
3099 }
3100
3101 return (-1);
3102 }
3103
3104 /*
3105 * ixgbe_multicst_add - Add a multicst address.
3106 */
3107 int
3108 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3109 {
3110 ASSERT(mutex_owned(&ixgbe->gen_lock));
3111
3112 if ((multiaddr[0] & 01) == 0) {
3113 return (EINVAL);
3114 }
3115
3116 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
3117 return (ENOENT);
3118 }
3119
3120 bcopy(multiaddr,
3121 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
3122 ixgbe->mcast_count++;
3123
3124 /*
3125 * Update the multicast table in the hardware
3126 */
3127 ixgbe_setup_multicst(ixgbe);
3128
3129 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3130 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3131 return (EIO);
3132 }
3133
3134 return (0);
3135 }
3136
3137 /*
3138 * ixgbe_multicst_remove - Remove a multicst address.
3139 */
3140 int
3141 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3142 {
3143 int i;
3144
3145 ASSERT(mutex_owned(&ixgbe->gen_lock));
3146
3147 for (i = 0; i < ixgbe->mcast_count; i++) {
3148 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
3149 ETHERADDRL) == 0) {
3150 for (i++; i < ixgbe->mcast_count; i++) {
3151 ixgbe->mcast_table[i - 1] =
3152 ixgbe->mcast_table[i];
3153 }
3154 ixgbe->mcast_count--;
3155 break;
3156 }
3157 }
3158
3159 /*
3160 * Update the multicast table in the hardware
3161 */
3162 ixgbe_setup_multicst(ixgbe);
3163
3164 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3165 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3166 return (EIO);
3167 }
3168
3169 return (0);
3170 }
3171
3172 /*
3173 * ixgbe_setup_multicast - Setup multicast data structures.
3174 *
3175 * This routine initializes all of the multicast related structures
3176 * and save them in the hardware registers.
3177 */
3178 static void
3179 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3180 {
3181 uint8_t *mc_addr_list;
3182 uint32_t mc_addr_count;
3183 struct ixgbe_hw *hw = &ixgbe->hw;
3184
3185 ASSERT(mutex_owned(&ixgbe->gen_lock));
3186
3187 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3188
3189 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3190 mc_addr_count = ixgbe->mcast_count;
3191
3192 /*
3193 * Update the multicast addresses to the MTA registers
3194 */
3195 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3196 ixgbe_mc_table_itr, TRUE);
3197 }
3198
3199 /*
3200 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3201 *
3202 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3203 * Different chipsets may have different allowed configuration of vmdq and rss.
3204 */
3205 static void
3206 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3207 {
3208 struct ixgbe_hw *hw = &ixgbe->hw;
3209 uint32_t ring_per_group;
3210
3211 switch (hw->mac.type) {
3212 case ixgbe_mac_82598EB:
3213 /*
3214 * 82598 supports the following combination:
3215 * vmdq no. x rss no.
3216 * [5..16] x 1
3217 * [1..4] x [1..16]
3218 * However 8 rss queue per pool (vmdq) is sufficient for
3219 * most cases.
3220 */
3221 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3222 if (ixgbe->num_rx_groups > 4) {
3223 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3224 } else {
3225 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3226 min(8, ring_per_group);
3227 }
3228
3229 break;
3230
3231 case ixgbe_mac_82599EB:
3232 case ixgbe_mac_X540:
3233 case ixgbe_mac_X550:
3234 case ixgbe_mac_X550EM_x:
3235 /*
3236 * 82599 supports the following combination:
3237 * vmdq no. x rss no.
3238 * [33..64] x [1..2]
3239 * [2..32] x [1..4]
3240 * 1 x [1..16]
3241 * However 8 rss queue per pool (vmdq) is sufficient for
3242 * most cases.
3243 *
3244 * For now, treat X540 and X550 like the 82599.
3245 */
3246 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3247 if (ixgbe->num_rx_groups == 1) {
3248 ixgbe->num_rx_rings = min(8, ring_per_group);
3249 } else if (ixgbe->num_rx_groups <= 32) {
3250 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3251 min(4, ring_per_group);
3252 } else if (ixgbe->num_rx_groups <= 64) {
3253 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3254 min(2, ring_per_group);
3255 }
3256 break;
3257
3258 default:
3259 break;
3260 }
3261
3262 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3263
3264 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3265 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3266 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3267 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3268 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3269 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3270 } else {
3271 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3272 }
3273
3274 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3275 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3276 }
3277
3278 /*
3279 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3280 *
3281 * This routine gets user-configured values out of the configuration
3282 * file ixgbe.conf.
3283 *
3284 * For each configurable value, there is a minimum, a maximum, and a
3285 * default.
3286 * If user does not configure a value, use the default.
3287 * If user configures below the minimum, use the minumum.
3288 * If user configures above the maximum, use the maxumum.
3289 */
3290 static void
3291 ixgbe_get_conf(ixgbe_t *ixgbe)
3292 {
3293 struct ixgbe_hw *hw = &ixgbe->hw;
3294 uint32_t flow_control;
3295
3296 /*
3297 * ixgbe driver supports the following user configurations:
3298 *
3299 * Jumbo frame configuration:
3300 * default_mtu
3301 *
3302 * Ethernet flow control configuration:
3303 * flow_control
3304 *
3305 * Multiple rings configurations:
3306 * tx_queue_number
3307 * tx_ring_size
3308 * rx_queue_number
3309 * rx_ring_size
3310 *
3311 * Call ixgbe_get_prop() to get the value for a specific
3312 * configuration parameter.
3313 */
3314
3315 /*
3316 * Jumbo frame configuration - max_frame_size controls host buffer
3317 * allocation, so includes MTU, ethernet header, vlan tag and
3318 * frame check sequence.
3319 */
3320 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3321 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3322
3323 ixgbe->max_frame_size = ixgbe->default_mtu +
3324 sizeof (struct ether_vlan_header) + ETHERFCSL;
3325
3326 /*
3327 * Ethernet flow control configuration
3328 */
3329 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3330 ixgbe_fc_none, 3, ixgbe_fc_none);
3331 if (flow_control == 3)
3332 flow_control = ixgbe_fc_default;
3333
3334 /*
3335 * fc.requested mode is what the user requests. After autoneg,
3336 * fc.current_mode will be the flow_control mode that was negotiated.
3337 */
3338 hw->fc.requested_mode = flow_control;
3339
3340 /*
3341 * Multiple rings configurations
3342 */
3343 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3344 ixgbe->capab->min_tx_que_num,
3345 ixgbe->capab->max_tx_que_num,
3346 ixgbe->capab->def_tx_que_num);
3347 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3348 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3349
3350 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3351 ixgbe->capab->min_rx_que_num,
3352 ixgbe->capab->max_rx_que_num,
3353 ixgbe->capab->def_rx_que_num);
3354 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3355 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3356
3357 /*
3358 * Multiple groups configuration
3359 */
3360 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3361 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3362 ixgbe->capab->def_rx_grp_num);
3363
3364 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3365 0, 1, DEFAULT_MR_ENABLE);
3366
3367 if (ixgbe->mr_enable == B_FALSE) {
3368 ixgbe->num_tx_rings = 1;
3369 ixgbe->num_rx_rings = 1;
3370 ixgbe->num_rx_groups = 1;
3371 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3372 } else {
3373 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3374 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3375 /*
3376 * The combination of num_rx_rings and num_rx_groups
3377 * may be not supported by h/w. We need to adjust
3378 * them to appropriate values.
3379 */
3380 ixgbe_setup_vmdq_rss_conf(ixgbe);
3381 }
3382
3383 /*
3384 * Tunable used to force an interrupt type. The only use is
3385 * for testing of the lesser interrupt types.
3386 * 0 = don't force interrupt type
3387 * 1 = force interrupt type MSI-X
3388 * 2 = force interrupt type MSI
3389 * 3 = force interrupt type Legacy
3390 */
3391 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3392 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3393
3394 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3395 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3396 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3397 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3398 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3399 0, 1, DEFAULT_LSO_ENABLE);
3400 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3401 0, 1, DEFAULT_LRO_ENABLE);
3402 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3403 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3404 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3405 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3406
3407 /* Head Write Back not recommended for 82599, X540 and X550 */
3408 if (hw->mac.type == ixgbe_mac_82599EB ||
3409 hw->mac.type == ixgbe_mac_X540 ||
3410 hw->mac.type == ixgbe_mac_X550 ||
3411 hw->mac.type == ixgbe_mac_X550EM_x) {
3412 ixgbe->tx_head_wb_enable = B_FALSE;
3413 }
3414
3415 /*
3416 * ixgbe LSO needs the tx h/w checksum support.
3417 * LSO will be disabled if tx h/w checksum is not
3418 * enabled.
3419 */
3420 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3421 ixgbe->lso_enable = B_FALSE;
3422 }
3423
3424 /*
3425 * ixgbe LRO needs the rx h/w checksum support.
3426 * LRO will be disabled if rx h/w checksum is not
3427 * enabled.
3428 */
3429 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3430 ixgbe->lro_enable = B_FALSE;
3431 }
3432
3433 /*
3434 * ixgbe LRO only supported by 82599, X540 and X550
3435 */
3436 if (hw->mac.type == ixgbe_mac_82598EB) {
3437 ixgbe->lro_enable = B_FALSE;
3438 }
3439 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3440 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3441 DEFAULT_TX_COPY_THRESHOLD);
3442 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3443 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3444 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3445 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3446 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3447 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3448 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3449 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3450 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3451
3452 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3453 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3454 DEFAULT_RX_COPY_THRESHOLD);
3455 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3456 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3457 DEFAULT_RX_LIMIT_PER_INTR);
3458
3459 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3460 ixgbe->capab->min_intr_throttle,
3461 ixgbe->capab->max_intr_throttle,
3462 ixgbe->capab->def_intr_throttle);
3463 /*
3464 * 82599, X540 and X550 require the interrupt throttling rate is
3465 * a multiple of 8. This is enforced by the register definiton.
3466 */
3467 if (hw->mac.type == ixgbe_mac_82599EB ||
3468 hw->mac.type == ixgbe_mac_X540 ||
3469 hw->mac.type == ixgbe_mac_X550 ||
3470 hw->mac.type == ixgbe_mac_X550EM_x)
3471 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3472
3473 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3474 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3475 }
3476
3477 static void
3478 ixgbe_init_params(ixgbe_t *ixgbe)
3479 {
3480 struct ixgbe_hw *hw = &ixgbe->hw;
3481 ixgbe_link_speed speeds_supported = 0;
3482 boolean_t negotiate;
3483
3484 /*
3485 * Get a list of speeds the adapter supports. If the hw struct hasn't
3486 * been populated with this information yet, retrieve it from the
3487 * adapter and save it to our own variable.
3488 *
3489 * On certain adapters, such as ones which use SFPs, the contents of
3490 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3491 * updated, so we must rely on calling ixgbe_get_link_capabilities()
3492 * in order to ascertain the speeds which we are capable of supporting,
3493 * and in the case of SFP-equipped adapters, which speed we are
3494 * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3495 * we'll go with a default list of speeds as a last resort.
3496 */
3497 speeds_supported = hw->phy.speeds_supported;
3498
3499 if (speeds_supported == 0) {
3500 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3501 &negotiate) != IXGBE_SUCCESS) {
3502 if (hw->mac.type == ixgbe_mac_82598EB) {
3503 speeds_supported =
3504 IXGBE_LINK_SPEED_82598_AUTONEG;
3505 } else {
3506 speeds_supported =
3507 IXGBE_LINK_SPEED_82599_AUTONEG;
3508 }
3509 }
3510 }
3511 ixgbe->speeds_supported = speeds_supported;
3512
3513 /*
3514 * By default, all supported speeds are enabled and advertised.
3515 */
3516 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3517 ixgbe->param_en_10000fdx_cap = 1;
3518 ixgbe->param_adv_10000fdx_cap = 1;
3519 } else {
3520 ixgbe->param_en_10000fdx_cap = 0;
3521 ixgbe->param_adv_10000fdx_cap = 0;
3522 }
3523
3524 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3525 ixgbe->param_en_5000fdx_cap = 1;
3526 ixgbe->param_adv_5000fdx_cap = 1;
3527 } else {
3528 ixgbe->param_en_5000fdx_cap = 0;
3529 ixgbe->param_adv_5000fdx_cap = 0;
3530 }
3531
3532 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3533 ixgbe->param_en_2500fdx_cap = 1;
3534 ixgbe->param_adv_2500fdx_cap = 1;
3535 } else {
3536 ixgbe->param_en_2500fdx_cap = 0;
3537 ixgbe->param_adv_2500fdx_cap = 0;
3538 }
3539
3540 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3541 ixgbe->param_en_1000fdx_cap = 1;
3542 ixgbe->param_adv_1000fdx_cap = 1;
3543 } else {
3544 ixgbe->param_en_1000fdx_cap = 0;
3545 ixgbe->param_adv_1000fdx_cap = 0;
3546 }
3547
3548 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3549 ixgbe->param_en_100fdx_cap = 1;
3550 ixgbe->param_adv_100fdx_cap = 1;
3551 } else {
3552 ixgbe->param_en_100fdx_cap = 0;
3553 ixgbe->param_adv_100fdx_cap = 0;
3554 }
3555
3556 ixgbe->param_pause_cap = 1;
3557 ixgbe->param_asym_pause_cap = 1;
3558 ixgbe->param_rem_fault = 0;
3559
3560 ixgbe->param_adv_autoneg_cap = 1;
3561 ixgbe->param_adv_pause_cap = 1;
3562 ixgbe->param_adv_asym_pause_cap = 1;
3563 ixgbe->param_adv_rem_fault = 0;
3564
3565 ixgbe->param_lp_10000fdx_cap = 0;
3566 ixgbe->param_lp_5000fdx_cap = 0;
3567 ixgbe->param_lp_2500fdx_cap = 0;
3568 ixgbe->param_lp_1000fdx_cap = 0;
3569 ixgbe->param_lp_100fdx_cap = 0;
3570 ixgbe->param_lp_autoneg_cap = 0;
3571 ixgbe->param_lp_pause_cap = 0;
3572 ixgbe->param_lp_asym_pause_cap = 0;
3573 ixgbe->param_lp_rem_fault = 0;
3574 }
3575
3576 /*
3577 * ixgbe_get_prop - Get a property value out of the configuration file
3578 * ixgbe.conf.
3579 *
3580 * Caller provides the name of the property, a default value, a minimum
3581 * value, and a maximum value.
3582 *
3583 * Return configured value of the property, with default, minimum and
3584 * maximum properly applied.
3585 */
3586 static int
3587 ixgbe_get_prop(ixgbe_t *ixgbe,
3588 char *propname, /* name of the property */
3589 int minval, /* minimum acceptable value */
3590 int maxval, /* maximim acceptable value */
3591 int defval) /* default value */
3592 {
3593 int value;
3594
3595 /*
3596 * Call ddi_prop_get_int() to read the conf settings
3597 */
3598 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3599 DDI_PROP_DONTPASS, propname, defval);
3600 if (value > maxval)
3601 value = maxval;
3602
3603 if (value < minval)
3604 value = minval;
3605
3606 return (value);
3607 }
3608
3609 /*
3610 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3611 */
3612 int
3613 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3614 {
3615 struct ixgbe_hw *hw = &ixgbe->hw;
3616 ixgbe_link_speed advertised = 0;
3617
3618 /*
3619 * Assemble a list of enabled speeds to auto-negotiate with.
3620 */
3621 if (ixgbe->param_en_10000fdx_cap == 1)
3622 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3623
3624 if (ixgbe->param_en_5000fdx_cap == 1)
3625 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3626
3627 if (ixgbe->param_en_2500fdx_cap == 1)
3628 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3629
3630 if (ixgbe->param_en_1000fdx_cap == 1)
3631 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3632
3633 if (ixgbe->param_en_100fdx_cap == 1)
3634 advertised |= IXGBE_LINK_SPEED_100_FULL;
3635
3636 /*
3637 * As a last resort, autoneg with a default list of speeds.
3638 */
3639 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3640 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3641 "to autonegotiate with full capabilities.");
3642
3643 if (hw->mac.type == ixgbe_mac_82598EB)
3644 advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3645 else
3646 advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3647 }
3648
3649 if (setup_hw) {
3650 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3651 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3652 ixgbe_notice(ixgbe, "Setup link failed on this "
3653 "device.");
3654 return (IXGBE_FAILURE);
3655 }
3656 }
3657
3658 return (IXGBE_SUCCESS);
3659 }
3660
3661 /*
3662 * ixgbe_driver_link_check - Link status processing.
3663 *
3664 * This function can be called in both kernel context and interrupt context
3665 */
3666 static void
3667 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3668 {
3669 struct ixgbe_hw *hw = &ixgbe->hw;
3670 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3671 boolean_t link_up = B_FALSE;
3672 boolean_t link_changed = B_FALSE;
3673
3674 ASSERT(mutex_owned(&ixgbe->gen_lock));
3675
3676 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3677 if (link_up) {
3678 ixgbe->link_check_complete = B_TRUE;
3679
3680 /* Link is up, enable flow control settings */
3681 (void) ixgbe_fc_enable(hw);
3682
3683 /*
3684 * The Link is up, check whether it was marked as down earlier
3685 */
3686 if (ixgbe->link_state != LINK_STATE_UP) {
3687 switch (speed) {
3688 case IXGBE_LINK_SPEED_10GB_FULL:
3689 ixgbe->link_speed = SPEED_10GB;
3690 break;
3691 case IXGBE_LINK_SPEED_5GB_FULL:
3692 ixgbe->link_speed = SPEED_5GB;
3693 break;
3694 case IXGBE_LINK_SPEED_2_5GB_FULL:
3695 ixgbe->link_speed = SPEED_2_5GB;
3696 break;
3697 case IXGBE_LINK_SPEED_1GB_FULL:
3698 ixgbe->link_speed = SPEED_1GB;
3699 break;
3700 case IXGBE_LINK_SPEED_100_FULL:
3701 ixgbe->link_speed = SPEED_100;
3702 }
3703 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3704 ixgbe->link_state = LINK_STATE_UP;
3705 link_changed = B_TRUE;
3706 }
3707 } else {
3708 if (ixgbe->link_check_complete == B_TRUE ||
3709 (ixgbe->link_check_complete == B_FALSE &&
3710 gethrtime() >= ixgbe->link_check_hrtime)) {
3711 /*
3712 * The link is really down
3713 */
3714 ixgbe->link_check_complete = B_TRUE;
3715
3716 if (ixgbe->link_state != LINK_STATE_DOWN) {
3717 ixgbe->link_speed = 0;
3718 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3719 ixgbe->link_state = LINK_STATE_DOWN;
3720 link_changed = B_TRUE;
3721 }
3722 }
3723 }
3724
3725 /*
3726 * If we are in an interrupt context, need to re-enable the
3727 * interrupt, which was automasked
3728 */
3729 if (servicing_interrupt() != 0) {
3730 ixgbe->eims |= IXGBE_EICR_LSC;
3731 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3732 }
3733
3734 if (link_changed) {
3735 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3736 }
3737 }
3738
3739 /*
3740 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3741 */
3742 static void
3743 ixgbe_sfp_check(void *arg)
3744 {
3745 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3746 uint32_t eicr = ixgbe->eicr;
3747 struct ixgbe_hw *hw = &ixgbe->hw;
3748
3749 mutex_enter(&ixgbe->gen_lock);
3750 (void) hw->phy.ops.identify_sfp(hw);
3751 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3752 /* clear the interrupt */
3753 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3754
3755 /* if link up, do multispeed fiber setup */
3756 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3757 B_TRUE);
3758 ixgbe_driver_link_check(ixgbe);
3759 ixgbe_get_hw_state(ixgbe);
3760 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3761 /* clear the interrupt */
3762 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3763
3764 /* if link up, do sfp module setup */
3765 (void) hw->mac.ops.setup_sfp(hw);
3766
3767 /* do multispeed fiber setup */
3768 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3769 B_TRUE);
3770 ixgbe_driver_link_check(ixgbe);
3771 ixgbe_get_hw_state(ixgbe);
3772 }
3773 mutex_exit(&ixgbe->gen_lock);
3774
3775 /*
3776 * We need to fully re-check the link later.
3777 */
3778 ixgbe->link_check_complete = B_FALSE;
3779 ixgbe->link_check_hrtime = gethrtime() +
3780 (IXGBE_LINK_UP_TIME * 100000000ULL);
3781 }
3782
3783 /*
3784 * ixgbe_overtemp_check - overtemp module processing done in taskq
3785 *
3786 * This routine will only be called on adapters with temperature sensor.
3787 * The indication of over-temperature can be either SDP0 interrupt or the link
3788 * status change interrupt.
3789 */
3790 static void
3791 ixgbe_overtemp_check(void *arg)
3792 {
3793 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3794 struct ixgbe_hw *hw = &ixgbe->hw;
3795 uint32_t eicr = ixgbe->eicr;
3796 ixgbe_link_speed speed;
3797 boolean_t link_up;
3798
3799 mutex_enter(&ixgbe->gen_lock);
3800
3801 /* make sure we know current state of link */
3802 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3803
3804 /* check over-temp condition */
3805 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3806 (eicr & IXGBE_EICR_LSC)) {
3807 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3808 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3809
3810 /*
3811 * Disable the adapter interrupts
3812 */
3813 ixgbe_disable_adapter_interrupts(ixgbe);
3814
3815 /*
3816 * Disable Rx/Tx units
3817 */
3818 (void) ixgbe_stop_adapter(hw);
3819
3820 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3821 ixgbe_error(ixgbe,
3822 "Problem: Network adapter has been stopped "
3823 "because it has overheated");
3824 ixgbe_error(ixgbe,
3825 "Action: Restart the computer. "
3826 "If the problem persists, power off the system "
3827 "and replace the adapter");
3828 }
3829 }
3830
3831 /* write to clear the interrupt */
3832 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3833
3834 mutex_exit(&ixgbe->gen_lock);
3835 }
3836
3837 /*
3838 * ixgbe_phy_check - taskq to process interrupts from an external PHY
3839 *
3840 * This routine will only be called on adapters with external PHYs
3841 * (such as X550) that may be trying to raise our attention to some event.
3842 * Currently, this is limited to claiming PHY overtemperature and link status
3843 * change (LSC) events, however this may expand to include other things in
3844 * future adapters.
3845 */
3846 static void
3847 ixgbe_phy_check(void *arg)
3848 {
3849 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3850 struct ixgbe_hw *hw = &ixgbe->hw;
3851 int rv;
3852
3853 mutex_enter(&ixgbe->gen_lock);
3854
3855 /*
3856 * X550 baseT PHY overtemp and LSC events are handled here.
3857 *
3858 * If an overtemp event occurs, it will be reflected in the
3859 * return value of phy.ops.handle_lasi() and the common code will
3860 * automatically power off the baseT PHY. This is our cue to trigger
3861 * an FMA event.
3862 *
3863 * If a link status change event occurs, phy.ops.handle_lasi() will
3864 * automatically initiate a link setup between the integrated KR PHY
3865 * and the external X557 PHY to ensure that the link speed between
3866 * them matches the link speed of the baseT link.
3867 */
3868 rv = ixgbe_handle_lasi(hw);
3869
3870 if (rv == IXGBE_ERR_OVERTEMP) {
3871 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3872
3873 /*
3874 * Disable the adapter interrupts
3875 */
3876 ixgbe_disable_adapter_interrupts(ixgbe);
3877
3878 /*
3879 * Disable Rx/Tx units
3880 */
3881 (void) ixgbe_stop_adapter(hw);
3882
3883 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3884 ixgbe_error(ixgbe,
3885 "Problem: Network adapter has been stopped due to a "
3886 "overtemperature event being detected.");
3887 ixgbe_error(ixgbe,
3888 "Action: Shut down or restart the computer. If the issue "
3889 "persists, please take action in accordance with the "
3890 "recommendations from your system vendor.");
3891 }
3892
3893 mutex_exit(&ixgbe->gen_lock);
3894 }
3895
3896 /*
3897 * ixgbe_link_timer - timer for link status detection
3898 */
3899 static void
3900 ixgbe_link_timer(void *arg)
3901 {
3902 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3903
3904 mutex_enter(&ixgbe->gen_lock);
3905 ixgbe_driver_link_check(ixgbe);
3906 mutex_exit(&ixgbe->gen_lock);
3907 }
3908
3909 /*
3910 * ixgbe_local_timer - Driver watchdog function.
3911 *
3912 * This function will handle the transmit stall check and other routines.
3913 */
3914 static void
3915 ixgbe_local_timer(void *arg)
3916 {
3917 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3918
3919 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3920 goto out;
3921
3922 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3923 ixgbe->reset_count++;
3924 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3925 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3926 goto out;
3927 }
3928
3929 if (ixgbe_stall_check(ixgbe)) {
3930 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3931 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3932
3933 ixgbe->reset_count++;
3934 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3935 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3936 }
3937
3938 out:
3939 ixgbe_restart_watchdog_timer(ixgbe);
3940 }
3941
3942 /*
3943 * ixgbe_stall_check - Check for transmit stall.
3944 *
3945 * This function checks if the adapter is stalled (in transmit).
3946 *
3947 * It is called each time the watchdog timeout is invoked.
3948 * If the transmit descriptor reclaim continuously fails,
3949 * the watchdog value will increment by 1. If the watchdog
3950 * value exceeds the threshold, the ixgbe is assumed to
3951 * have stalled and need to be reset.
3952 */
3953 static boolean_t
3954 ixgbe_stall_check(ixgbe_t *ixgbe)
3955 {
3956 ixgbe_tx_ring_t *tx_ring;
3957 boolean_t result;
3958 int i;
3959
3960 if (ixgbe->link_state != LINK_STATE_UP)
3961 return (B_FALSE);
3962
3963 /*
3964 * If any tx ring is stalled, we'll reset the chipset
3965 */
3966 result = B_FALSE;
3967 for (i = 0; i < ixgbe->num_tx_rings; i++) {
3968 tx_ring = &ixgbe->tx_rings[i];
3969 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3970 tx_ring->tx_recycle(tx_ring);
3971 }
3972
3973 if (tx_ring->recycle_fail > 0)
3974 tx_ring->stall_watchdog++;
3975 else
3976 tx_ring->stall_watchdog = 0;
3977
3978 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3979 result = B_TRUE;
3980 break;
3981 }
3982 }
3983
3984 if (result) {
3985 tx_ring->stall_watchdog = 0;
3986 tx_ring->recycle_fail = 0;
3987 }
3988
3989 return (result);
3990 }
3991
3992
3993 /*
3994 * is_valid_mac_addr - Check if the mac address is valid.
3995 */
3996 static boolean_t
3997 is_valid_mac_addr(uint8_t *mac_addr)
3998 {
3999 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
4000 const uint8_t addr_test2[6] =
4001 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
4002
4003 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
4004 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
4005 return (B_FALSE);
4006
4007 return (B_TRUE);
4008 }
4009
4010 static boolean_t
4011 ixgbe_find_mac_address(ixgbe_t *ixgbe)
4012 {
4013 #ifdef __sparc
4014 struct ixgbe_hw *hw = &ixgbe->hw;
4015 uchar_t *bytes;
4016 struct ether_addr sysaddr;
4017 uint_t nelts;
4018 int err;
4019 boolean_t found = B_FALSE;
4020
4021 /*
4022 * The "vendor's factory-set address" may already have
4023 * been extracted from the chip, but if the property
4024 * "local-mac-address" is set we use that instead.
4025 *
4026 * We check whether it looks like an array of 6
4027 * bytes (which it should, if OBP set it). If we can't
4028 * make sense of it this way, we'll ignore it.
4029 */
4030 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4031 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
4032 if (err == DDI_PROP_SUCCESS) {
4033 if (nelts == ETHERADDRL) {
4034 while (nelts--)
4035 hw->mac.addr[nelts] = bytes[nelts];
4036 found = B_TRUE;
4037 }
4038 ddi_prop_free(bytes);
4039 }
4040
4041 /*
4042 * Look up the OBP property "local-mac-address?". If the user has set
4043 * 'local-mac-address? = false', use "the system address" instead.
4044 */
4045 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
4046 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
4047 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
4048 if (localetheraddr(NULL, &sysaddr) != 0) {
4049 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
4050 found = B_TRUE;
4051 }
4052 }
4053 ddi_prop_free(bytes);
4054 }
4055
4056 /*
4057 * Finally(!), if there's a valid "mac-address" property (created
4058 * if we netbooted from this interface), we must use this instead
4059 * of any of the above to ensure that the NFS/install server doesn't
4060 * get confused by the address changing as illumos takes over!
4061 */
4062 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4063 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
4064 if (err == DDI_PROP_SUCCESS) {
4065 if (nelts == ETHERADDRL) {
4066 while (nelts--)
4067 hw->mac.addr[nelts] = bytes[nelts];
4068 found = B_TRUE;
4069 }
4070 ddi_prop_free(bytes);
4071 }
4072
4073 if (found) {
4074 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
4075 return (B_TRUE);
4076 }
4077 #else
4078 _NOTE(ARGUNUSED(ixgbe));
4079 #endif
4080
4081 return (B_TRUE);
4082 }
4083
4084 #pragma inline(ixgbe_arm_watchdog_timer)
4085 static void
4086 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
4087 {
4088 /*
4089 * Fire a watchdog timer
4090 */
4091 ixgbe->watchdog_tid =
4092 timeout(ixgbe_local_timer,
4093 (void *)ixgbe, 1 * drv_usectohz(1000000));
4094
4095 }
4096
4097 /*
4098 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
4099 */
4100 void
4101 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
4102 {
4103 mutex_enter(&ixgbe->watchdog_lock);
4104
4105 if (!ixgbe->watchdog_enable) {
4106 ixgbe->watchdog_enable = B_TRUE;
4107 ixgbe->watchdog_start = B_TRUE;
4108 ixgbe_arm_watchdog_timer(ixgbe);
4109 }
4110
4111 mutex_exit(&ixgbe->watchdog_lock);
4112 }
4113
4114 /*
4115 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
4116 */
4117 void
4118 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
4119 {
4120 timeout_id_t tid;
4121
4122 mutex_enter(&ixgbe->watchdog_lock);
4123
4124 ixgbe->watchdog_enable = B_FALSE;
4125 ixgbe->watchdog_start = B_FALSE;
4126 tid = ixgbe->watchdog_tid;
4127 ixgbe->watchdog_tid = 0;
4128
4129 mutex_exit(&ixgbe->watchdog_lock);
4130
4131 if (tid != 0)
4132 (void) untimeout(tid);
4133 }
4134
4135 /*
4136 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
4137 */
4138 void
4139 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
4140 {
4141 mutex_enter(&ixgbe->watchdog_lock);
4142
4143 if (ixgbe->watchdog_enable) {
4144 if (!ixgbe->watchdog_start) {
4145 ixgbe->watchdog_start = B_TRUE;
4146 ixgbe_arm_watchdog_timer(ixgbe);
4147 }
4148 }
4149
4150 mutex_exit(&ixgbe->watchdog_lock);
4151 }
4152
4153 /*
4154 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
4155 */
4156 static void
4157 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
4158 {
4159 mutex_enter(&ixgbe->watchdog_lock);
4160
4161 if (ixgbe->watchdog_start)
4162 ixgbe_arm_watchdog_timer(ixgbe);
4163
4164 mutex_exit(&ixgbe->watchdog_lock);
4165 }
4166
4167 /*
4168 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
4169 */
4170 void
4171 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
4172 {
4173 timeout_id_t tid;
4174
4175 mutex_enter(&ixgbe->watchdog_lock);
4176
4177 ixgbe->watchdog_start = B_FALSE;
4178 tid = ixgbe->watchdog_tid;
4179 ixgbe->watchdog_tid = 0;
4180
4181 mutex_exit(&ixgbe->watchdog_lock);
4182
4183 if (tid != 0)
4184 (void) untimeout(tid);
4185 }
4186
4187 /*
4188 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
4189 */
4190 static void
4191 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
4192 {
4193 struct ixgbe_hw *hw = &ixgbe->hw;
4194
4195 /*
4196 * mask all interrupts off
4197 */
4198 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
4199
4200 /*
4201 * for MSI-X, also disable autoclear
4202 */
4203 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4204 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
4205 }
4206
4207 IXGBE_WRITE_FLUSH(hw);
4208 }
4209
4210 /*
4211 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
4212 */
4213 static void
4214 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
4215 {
4216 struct ixgbe_hw *hw = &ixgbe->hw;
4217 uint32_t eiac, eiam;
4218 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4219
4220 /* interrupt types to enable */
4221 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
4222 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
4223 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
4224
4225 /* enable automask on "other" causes that this adapter can generate */
4226 eiam = ixgbe->capab->other_intr;
4227
4228 /*
4229 * msi-x mode
4230 */
4231 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4232 /* enable autoclear but not on bits 29:20 */
4233 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4234
4235 /* general purpose interrupt enable */
4236 gpie |= (IXGBE_GPIE_MSIX_MODE
4237 | IXGBE_GPIE_PBA_SUPPORT
4238 | IXGBE_GPIE_OCD
4239 | IXGBE_GPIE_EIAME);
4240 /*
4241 * non-msi-x mode
4242 */
4243 } else {
4244
4245 /* disable autoclear, leave gpie at default */
4246 eiac = 0;
4247
4248 /*
4249 * General purpose interrupt enable.
4250 * For 82599, X540 and X550, extended interrupt
4251 * automask enable only in MSI or MSI-X mode
4252 */
4253 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4254 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4255 gpie |= IXGBE_GPIE_EIAME;
4256 }
4257 }
4258
4259 /* Enable specific "other" interrupt types */
4260 switch (hw->mac.type) {
4261 case ixgbe_mac_82598EB:
4262 gpie |= ixgbe->capab->other_gpie;
4263 break;
4264
4265 case ixgbe_mac_82599EB:
4266 case ixgbe_mac_X540:
4267 case ixgbe_mac_X550:
4268 case ixgbe_mac_X550EM_x:
4269 gpie |= ixgbe->capab->other_gpie;
4270
4271 /* Enable RSC Delay 8us when LRO enabled */
4272 if (ixgbe->lro_enable) {
4273 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4274 }
4275 break;
4276
4277 default:
4278 break;
4279 }
4280
4281 /* write to interrupt control registers */
4282 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4283 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4284 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4285 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4286 IXGBE_WRITE_FLUSH(hw);
4287 }
4288
4289 /*
4290 * ixgbe_loopback_ioctl - Loopback support.
4291 */
4292 enum ioc_reply
4293 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
4294 {
4295 lb_info_sz_t *lbsp;
4296 lb_property_t *lbpp;
4297 uint32_t *lbmp;
4298 uint32_t size;
4299 uint32_t value;
4300
4301 if (mp->b_cont == NULL)
4302 return (IOC_INVAL);
4303
4304 switch (iocp->ioc_cmd) {
4305 default:
4306 return (IOC_INVAL);
4307
4308 case LB_GET_INFO_SIZE:
4309 size = sizeof (lb_info_sz_t);
4310 if (iocp->ioc_count != size)
4311 return (IOC_INVAL);
4312
4313 value = sizeof (lb_normal);
4314 value += sizeof (lb_mac);
4315 value += sizeof (lb_external);
4316
4317 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
4318 *lbsp = value;
4319 break;
4320
4321 case LB_GET_INFO:
4322 value = sizeof (lb_normal);
4323 value += sizeof (lb_mac);
4324 value += sizeof (lb_external);
4325
4326 size = value;
4327 if (iocp->ioc_count != size)
4328 return (IOC_INVAL);
4329
4330 value = 0;
4331 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4332
4333 lbpp[value++] = lb_normal;
4334 lbpp[value++] = lb_mac;
4335 lbpp[value++] = lb_external;
4336 break;
4337
4338 case LB_GET_MODE:
4339 size = sizeof (uint32_t);
4340 if (iocp->ioc_count != size)
4341 return (IOC_INVAL);
4342
4343 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4344 *lbmp = ixgbe->loopback_mode;
4345 break;
4346
4347 case LB_SET_MODE:
4348 size = 0;
4349 if (iocp->ioc_count != sizeof (uint32_t))
4350 return (IOC_INVAL);
4351
4352 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4353 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4354 return (IOC_INVAL);
4355 break;
4356 }
4357
4358 iocp->ioc_count = size;
4359 iocp->ioc_error = 0;
4360
4361 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4362 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4363 return (IOC_INVAL);
4364 }
4365
4366 return (IOC_REPLY);
4367 }
4368
4369 /*
4370 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4371 */
4372 static boolean_t
4373 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4374 {
4375 if (mode == ixgbe->loopback_mode)
4376 return (B_TRUE);
4377
4378 ixgbe->loopback_mode = mode;
4379
4380 if (mode == IXGBE_LB_NONE) {
4381 /*
4382 * Reset the chip
4383 */
4384 (void) ixgbe_reset(ixgbe);
4385 return (B_TRUE);
4386 }
4387
4388 mutex_enter(&ixgbe->gen_lock);
4389
4390 switch (mode) {
4391 default:
4392 mutex_exit(&ixgbe->gen_lock);
4393 return (B_FALSE);
4394
4395 case IXGBE_LB_EXTERNAL:
4396 break;
4397
4398 case IXGBE_LB_INTERNAL_MAC:
4399 ixgbe_set_internal_mac_loopback(ixgbe);
4400 break;
4401 }
4402
4403 mutex_exit(&ixgbe->gen_lock);
4404
4405 return (B_TRUE);
4406 }
4407
4408 /*
4409 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4410 */
4411 static void
4412 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4413 {
4414 struct ixgbe_hw *hw;
4415 uint32_t reg;
4416 uint8_t atlas;
4417
4418 hw = &ixgbe->hw;
4419
4420 /*
4421 * Setup MAC loopback
4422 */
4423 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4424 reg |= IXGBE_HLREG0_LPBK;
4425 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4426
4427 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4428 reg &= ~IXGBE_AUTOC_LMS_MASK;
4429 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4430
4431 /*
4432 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4433 */
4434 switch (hw->mac.type) {
4435 case ixgbe_mac_82598EB:
4436 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4437 &atlas);
4438 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4439 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4440 atlas);
4441
4442 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4443 &atlas);
4444 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4445 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4446 atlas);
4447
4448 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4449 &atlas);
4450 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4451 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4452 atlas);
4453
4454 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4455 &atlas);
4456 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4457 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4458 atlas);
4459 break;
4460
4461 case ixgbe_mac_82599EB:
4462 case ixgbe_mac_X540:
4463 case ixgbe_mac_X550:
4464 case ixgbe_mac_X550EM_x:
4465 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4466 reg |= (IXGBE_AUTOC_FLU |
4467 IXGBE_AUTOC_10G_KX4);
4468 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4469
4470 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4471 B_FALSE);
4472 break;
4473
4474 default:
4475 break;
4476 }
4477 }
4478
4479 #pragma inline(ixgbe_intr_rx_work)
4480 /*
4481 * ixgbe_intr_rx_work - RX processing of ISR.
4482 */
4483 static void
4484 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4485 {
4486 mblk_t *mp;
4487
4488 mutex_enter(&rx_ring->rx_lock);
4489
4490 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4491 mutex_exit(&rx_ring->rx_lock);
4492
4493 if (mp != NULL)
4494 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4495 rx_ring->ring_gen_num);
4496 }
4497
4498 #pragma inline(ixgbe_intr_tx_work)
4499 /*
4500 * ixgbe_intr_tx_work - TX processing of ISR.
4501 */
4502 static void
4503 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4504 {
4505 ixgbe_t *ixgbe = tx_ring->ixgbe;
4506
4507 /*
4508 * Recycle the tx descriptors
4509 */
4510 tx_ring->tx_recycle(tx_ring);
4511
4512 /*
4513 * Schedule the re-transmit
4514 */
4515 if (tx_ring->reschedule &&
4516 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4517 tx_ring->reschedule = B_FALSE;
4518 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4519 tx_ring->ring_handle);
4520 tx_ring->stat_reschedule++;
4521 }
4522 }
4523
4524 #pragma inline(ixgbe_intr_other_work)
4525 /*
4526 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4527 */
4528 static void
4529 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4530 {
4531 struct ixgbe_hw *hw = &ixgbe->hw;
4532
4533 ASSERT(mutex_owned(&ixgbe->gen_lock));
4534
4535 /*
4536 * handle link status change
4537 */
4538 if (eicr & IXGBE_EICR_LSC) {
4539 ixgbe_driver_link_check(ixgbe);
4540 ixgbe_get_hw_state(ixgbe);
4541 }
4542
4543 /*
4544 * check for fan failure on adapters with fans
4545 */
4546 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4547 (eicr & IXGBE_EICR_GPI_SDP1)) {
4548 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4549
4550 /*
4551 * Disable the adapter interrupts
4552 */
4553 ixgbe_disable_adapter_interrupts(ixgbe);
4554
4555 /*
4556 * Disable Rx/Tx units
4557 */
4558 (void) ixgbe_stop_adapter(&ixgbe->hw);
4559
4560 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4561 ixgbe_error(ixgbe,
4562 "Problem: Network adapter has been stopped "
4563 "because the fan has stopped.\n");
4564 ixgbe_error(ixgbe,
4565 "Action: Replace the adapter.\n");
4566
4567 /* re-enable the interrupt, which was automasked */
4568 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4569 }
4570
4571 /*
4572 * Do SFP check for adapters with hot-plug capability
4573 */
4574 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4575 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4576 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4577 ixgbe->eicr = eicr;
4578 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4579 ixgbe_sfp_check, (void *)ixgbe,
4580 DDI_NOSLEEP)) != DDI_SUCCESS) {
4581 ixgbe_log(ixgbe, "No memory available to dispatch "
4582 "taskq for SFP check");
4583 }
4584 }
4585
4586 /*
4587 * Do over-temperature check for adapters with temp sensor
4588 */
4589 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4590 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4591 (eicr & IXGBE_EICR_LSC))) {
4592 ixgbe->eicr = eicr;
4593 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4594 ixgbe_overtemp_check, (void *)ixgbe,
4595 DDI_NOSLEEP)) != DDI_SUCCESS) {
4596 ixgbe_log(ixgbe, "No memory available to dispatch "
4597 "taskq for overtemp check");
4598 }
4599 }
4600
4601 /*
4602 * Process an external PHY interrupt
4603 */
4604 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4605 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4606 ixgbe->eicr = eicr;
4607 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4608 ixgbe_phy_check, (void *)ixgbe,
4609 DDI_NOSLEEP)) != DDI_SUCCESS) {
4610 ixgbe_log(ixgbe, "No memory available to dispatch "
4611 "taskq for PHY check");
4612 }
4613 }
4614 }
4615
4616 /*
4617 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4618 */
4619 static uint_t
4620 ixgbe_intr_legacy(void *arg1, void *arg2)
4621 {
4622 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4623 struct ixgbe_hw *hw = &ixgbe->hw;
4624 ixgbe_tx_ring_t *tx_ring;
4625 ixgbe_rx_ring_t *rx_ring;
4626 uint32_t eicr;
4627 mblk_t *mp;
4628 boolean_t tx_reschedule;
4629 uint_t result;
4630
4631 _NOTE(ARGUNUSED(arg2));
4632
4633 mutex_enter(&ixgbe->gen_lock);
4634 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4635 mutex_exit(&ixgbe->gen_lock);
4636 return (DDI_INTR_UNCLAIMED);
4637 }
4638
4639 mp = NULL;
4640 tx_reschedule = B_FALSE;
4641
4642 /*
4643 * Any bit set in eicr: claim this interrupt
4644 */
4645 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4646
4647 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4648 mutex_exit(&ixgbe->gen_lock);
4649 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4650 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4651 return (DDI_INTR_CLAIMED);
4652 }
4653
4654 if (eicr) {
4655 /*
4656 * For legacy interrupt, we have only one interrupt,
4657 * so we have only one rx ring and one tx ring enabled.
4658 */
4659 ASSERT(ixgbe->num_rx_rings == 1);
4660 ASSERT(ixgbe->num_tx_rings == 1);
4661
4662 /*
4663 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4664 */
4665 if (eicr & 0x1) {
4666 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4667 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4668 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4669 /*
4670 * Clean the rx descriptors
4671 */
4672 rx_ring = &ixgbe->rx_rings[0];
4673 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4674 }
4675
4676 /*
4677 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4678 */
4679 if (eicr & 0x2) {
4680 /*
4681 * Recycle the tx descriptors
4682 */
4683 tx_ring = &ixgbe->tx_rings[0];
4684 tx_ring->tx_recycle(tx_ring);
4685
4686 /*
4687 * Schedule the re-transmit
4688 */
4689 tx_reschedule = (tx_ring->reschedule &&
4690 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4691 }
4692
4693 /* any interrupt type other than tx/rx */
4694 if (eicr & ixgbe->capab->other_intr) {
4695 switch (hw->mac.type) {
4696 case ixgbe_mac_82598EB:
4697 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4698 break;
4699
4700 case ixgbe_mac_82599EB:
4701 case ixgbe_mac_X540:
4702 case ixgbe_mac_X550:
4703 case ixgbe_mac_X550EM_x:
4704 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4705 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4706 break;
4707
4708 default:
4709 break;
4710 }
4711 ixgbe_intr_other_work(ixgbe, eicr);
4712 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4713 }
4714
4715 mutex_exit(&ixgbe->gen_lock);
4716
4717 result = DDI_INTR_CLAIMED;
4718 } else {
4719 mutex_exit(&ixgbe->gen_lock);
4720
4721 /*
4722 * No interrupt cause bits set: don't claim this interrupt.
4723 */
4724 result = DDI_INTR_UNCLAIMED;
4725 }
4726
4727 /* re-enable the interrupts which were automasked */
4728 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4729
4730 /*
4731 * Do the following work outside of the gen_lock
4732 */
4733 if (mp != NULL) {
4734 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4735 rx_ring->ring_gen_num);
4736 }
4737
4738 if (tx_reschedule) {
4739 tx_ring->reschedule = B_FALSE;
4740 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4741 tx_ring->stat_reschedule++;
4742 }
4743
4744 return (result);
4745 }
4746
4747 /*
4748 * ixgbe_intr_msi - Interrupt handler for MSI.
4749 */
4750 static uint_t
4751 ixgbe_intr_msi(void *arg1, void *arg2)
4752 {
4753 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4754 struct ixgbe_hw *hw = &ixgbe->hw;
4755 uint32_t eicr;
4756
4757 _NOTE(ARGUNUSED(arg2));
4758
4759 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4760
4761 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4762 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4763 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4764 return (DDI_INTR_CLAIMED);
4765 }
4766
4767 /*
4768 * For MSI interrupt, we have only one vector,
4769 * so we have only one rx ring and one tx ring enabled.
4770 */
4771 ASSERT(ixgbe->num_rx_rings == 1);
4772 ASSERT(ixgbe->num_tx_rings == 1);
4773
4774 /*
4775 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4776 */
4777 if (eicr & 0x1) {
4778 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4779 }
4780
4781 /*
4782 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4783 */
4784 if (eicr & 0x2) {
4785 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4786 }
4787
4788 /* any interrupt type other than tx/rx */
4789 if (eicr & ixgbe->capab->other_intr) {
4790 mutex_enter(&ixgbe->gen_lock);
4791 switch (hw->mac.type) {
4792 case ixgbe_mac_82598EB:
4793 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4794 break;
4795
4796 case ixgbe_mac_82599EB:
4797 case ixgbe_mac_X540:
4798 case ixgbe_mac_X550:
4799 case ixgbe_mac_X550EM_x:
4800 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4801 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4802 break;
4803
4804 default:
4805 break;
4806 }
4807 ixgbe_intr_other_work(ixgbe, eicr);
4808 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4809 mutex_exit(&ixgbe->gen_lock);
4810 }
4811
4812 /* re-enable the interrupts which were automasked */
4813 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4814
4815 return (DDI_INTR_CLAIMED);
4816 }
4817
4818 /*
4819 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4820 */
4821 static uint_t
4822 ixgbe_intr_msix(void *arg1, void *arg2)
4823 {
4824 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4825 ixgbe_t *ixgbe = vect->ixgbe;
4826 struct ixgbe_hw *hw = &ixgbe->hw;
4827 uint32_t eicr;
4828 int r_idx = 0;
4829
4830 _NOTE(ARGUNUSED(arg2));
4831
4832 /*
4833 * Clean each rx ring that has its bit set in the map
4834 */
4835 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4836 while (r_idx >= 0) {
4837 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4838 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4839 (ixgbe->num_rx_rings - 1));
4840 }
4841
4842 /*
4843 * Clean each tx ring that has its bit set in the map
4844 */
4845 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4846 while (r_idx >= 0) {
4847 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4848 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4849 (ixgbe->num_tx_rings - 1));
4850 }
4851
4852
4853 /*
4854 * Clean other interrupt (link change) that has its bit set in the map
4855 */
4856 if (BT_TEST(vect->other_map, 0) == 1) {
4857 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4858
4859 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4860 DDI_FM_OK) {
4861 ddi_fm_service_impact(ixgbe->dip,
4862 DDI_SERVICE_DEGRADED);
4863 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4864 return (DDI_INTR_CLAIMED);
4865 }
4866
4867 /*
4868 * Check "other" cause bits: any interrupt type other than tx/rx
4869 */
4870 if (eicr & ixgbe->capab->other_intr) {
4871 mutex_enter(&ixgbe->gen_lock);
4872 switch (hw->mac.type) {
4873 case ixgbe_mac_82598EB:
4874 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4875 ixgbe_intr_other_work(ixgbe, eicr);
4876 break;
4877
4878 case ixgbe_mac_82599EB:
4879 case ixgbe_mac_X540:
4880 case ixgbe_mac_X550:
4881 case ixgbe_mac_X550EM_x:
4882 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4883 ixgbe_intr_other_work(ixgbe, eicr);
4884 break;
4885
4886 default:
4887 break;
4888 }
4889 mutex_exit(&ixgbe->gen_lock);
4890 }
4891
4892 /* re-enable the interrupts which were automasked */
4893 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4894 }
4895
4896 return (DDI_INTR_CLAIMED);
4897 }
4898
4899 /*
4900 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4901 *
4902 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4903 * if not successful, try Legacy.
4904 * ixgbe->intr_force can be used to force sequence to start with
4905 * any of the 3 types.
4906 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4907 */
4908 static int
4909 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4910 {
4911 dev_info_t *devinfo;
4912 int intr_types;
4913 int rc;
4914
4915 devinfo = ixgbe->dip;
4916
4917 /*
4918 * Get supported interrupt types
4919 */
4920 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4921
4922 if (rc != DDI_SUCCESS) {
4923 ixgbe_log(ixgbe,
4924 "Get supported interrupt types failed: %d", rc);
4925 return (IXGBE_FAILURE);
4926 }
4927 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4928
4929 ixgbe->intr_type = 0;
4930
4931 /*
4932 * Install MSI-X interrupts
4933 */
4934 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4935 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4936 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4937 if (rc == IXGBE_SUCCESS)
4938 return (IXGBE_SUCCESS);
4939
4940 ixgbe_log(ixgbe,
4941 "Allocate MSI-X failed, trying MSI interrupts...");
4942 }
4943
4944 /*
4945 * MSI-X not used, force rings and groups to 1
4946 */
4947 ixgbe->num_rx_rings = 1;
4948 ixgbe->num_rx_groups = 1;
4949 ixgbe->num_tx_rings = 1;
4950 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4951 ixgbe_log(ixgbe,
4952 "MSI-X not used, force rings and groups number to 1");
4953
4954 /*
4955 * Install MSI interrupts
4956 */
4957 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4958 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4959 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4960 if (rc == IXGBE_SUCCESS)
4961 return (IXGBE_SUCCESS);
4962
4963 ixgbe_log(ixgbe,
4964 "Allocate MSI failed, trying Legacy interrupts...");
4965 }
4966
4967 /*
4968 * Install legacy interrupts
4969 */
4970 if (intr_types & DDI_INTR_TYPE_FIXED) {
4971 /*
4972 * Disallow legacy interrupts for X550. X550 has a silicon
4973 * bug which prevents Shared Legacy interrupts from working.
4974 * For details, please reference:
4975 *
4976 * Intel Ethernet Controller X550 Specification Update rev. 2.1
4977 * May 2016, erratum 22: PCIe Interrupt Status Bit
4978 */
4979 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
4980 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
4981 ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
4982 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf) {
4983 ixgbe_log(ixgbe,
4984 "Legacy interrupts are not supported on this "
4985 "adapter. Please use MSI or MSI-X instead.");
4986 return (IXGBE_FAILURE);
4987 }
4988 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4989 if (rc == IXGBE_SUCCESS)
4990 return (IXGBE_SUCCESS);
4991
4992 ixgbe_log(ixgbe,
4993 "Allocate Legacy interrupts failed");
4994 }
4995
4996 /*
4997 * If none of the 3 types succeeded, return failure
4998 */
4999 return (IXGBE_FAILURE);
5000 }
5001
5002 /*
5003 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
5004 *
5005 * For legacy and MSI, only 1 handle is needed. For MSI-X,
5006 * if fewer than 2 handles are available, return failure.
5007 * Upon success, this maps the vectors to rx and tx rings for
5008 * interrupts.
5009 */
5010 static int
5011 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
5012 {
5013 dev_info_t *devinfo;
5014 int request, count, actual;
5015 int minimum;
5016 int rc;
5017 uint32_t ring_per_group;
5018
5019 devinfo = ixgbe->dip;
5020
5021 switch (intr_type) {
5022 case DDI_INTR_TYPE_FIXED:
5023 request = 1; /* Request 1 legacy interrupt handle */
5024 minimum = 1;
5025 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
5026 break;
5027
5028 case DDI_INTR_TYPE_MSI:
5029 request = 1; /* Request 1 MSI interrupt handle */
5030 minimum = 1;
5031 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
5032 break;
5033
5034 case DDI_INTR_TYPE_MSIX:
5035 /*
5036 * Best number of vectors for the adapter is
5037 * (# rx rings + # tx rings), however we will
5038 * limit the request number.
5039 */
5040 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
5041 if (request > ixgbe->capab->max_ring_vect)
5042 request = ixgbe->capab->max_ring_vect;
5043 minimum = 1;
5044 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
5045 break;
5046
5047 default:
5048 ixgbe_log(ixgbe,
5049 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
5050 intr_type);
5051 return (IXGBE_FAILURE);
5052 }
5053 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
5054 request, minimum);
5055
5056 /*
5057 * Get number of supported interrupts
5058 */
5059 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
5060 if ((rc != DDI_SUCCESS) || (count < minimum)) {
5061 ixgbe_log(ixgbe,
5062 "Get interrupt number failed. Return: %d, count: %d",
5063 rc, count);
5064 return (IXGBE_FAILURE);
5065 }
5066 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
5067
5068 actual = 0;
5069 ixgbe->intr_cnt = 0;
5070 ixgbe->intr_cnt_max = 0;
5071 ixgbe->intr_cnt_min = 0;
5072
5073 /*
5074 * Allocate an array of interrupt handles
5075 */
5076 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
5077 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
5078
5079 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
5080 request, &actual, DDI_INTR_ALLOC_NORMAL);
5081 if (rc != DDI_SUCCESS) {
5082 ixgbe_log(ixgbe, "Allocate interrupts failed. "
5083 "return: %d, request: %d, actual: %d",
5084 rc, request, actual);
5085 goto alloc_handle_fail;
5086 }
5087 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
5088
5089 /*
5090 * upper/lower limit of interrupts
5091 */
5092 ixgbe->intr_cnt = actual;
5093 ixgbe->intr_cnt_max = request;
5094 ixgbe->intr_cnt_min = minimum;
5095
5096 /*
5097 * rss number per group should not exceed the rx interrupt number,
5098 * else need to adjust rx ring number.
5099 */
5100 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5101 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
5102 if (actual < ring_per_group) {
5103 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
5104 ixgbe_setup_vmdq_rss_conf(ixgbe);
5105 }
5106
5107 /*
5108 * Now we know the actual number of vectors. Here we map the vector
5109 * to other, rx rings and tx ring.
5110 */
5111 if (actual < minimum) {
5112 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
5113 actual);
5114 goto alloc_handle_fail;
5115 }
5116
5117 /*
5118 * Get priority for first vector, assume remaining are all the same
5119 */
5120 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
5121 if (rc != DDI_SUCCESS) {
5122 ixgbe_log(ixgbe,
5123 "Get interrupt priority failed: %d", rc);
5124 goto alloc_handle_fail;
5125 }
5126
5127 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
5128 if (rc != DDI_SUCCESS) {
5129 ixgbe_log(ixgbe,
5130 "Get interrupt cap failed: %d", rc);
5131 goto alloc_handle_fail;
5132 }
5133
5134 ixgbe->intr_type = intr_type;
5135
5136 return (IXGBE_SUCCESS);
5137
5138 alloc_handle_fail:
5139 ixgbe_rem_intrs(ixgbe);
5140
5141 return (IXGBE_FAILURE);
5142 }
5143
5144 /*
5145 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
5146 *
5147 * Before adding the interrupt handlers, the interrupt vectors have
5148 * been allocated, and the rx/tx rings have also been allocated.
5149 */
5150 static int
5151 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
5152 {
5153 int vector = 0;
5154 int rc;
5155
5156 switch (ixgbe->intr_type) {
5157 case DDI_INTR_TYPE_MSIX:
5158 /*
5159 * Add interrupt handler for all vectors
5160 */
5161 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
5162 /*
5163 * install pointer to vect_map[vector]
5164 */
5165 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5166 (ddi_intr_handler_t *)ixgbe_intr_msix,
5167 (void *)&ixgbe->vect_map[vector], NULL);
5168
5169 if (rc != DDI_SUCCESS) {
5170 ixgbe_log(ixgbe,
5171 "Add interrupt handler failed. "
5172 "return: %d, vector: %d", rc, vector);
5173 for (vector--; vector >= 0; vector--) {
5174 (void) ddi_intr_remove_handler(
5175 ixgbe->htable[vector]);
5176 }
5177 return (IXGBE_FAILURE);
5178 }
5179 }
5180
5181 break;
5182
5183 case DDI_INTR_TYPE_MSI:
5184 /*
5185 * Add interrupt handlers for the only vector
5186 */
5187 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5188 (ddi_intr_handler_t *)ixgbe_intr_msi,
5189 (void *)ixgbe, NULL);
5190
5191 if (rc != DDI_SUCCESS) {
5192 ixgbe_log(ixgbe,
5193 "Add MSI interrupt handler failed: %d", rc);
5194 return (IXGBE_FAILURE);
5195 }
5196
5197 break;
5198
5199 case DDI_INTR_TYPE_FIXED:
5200 /*
5201 * Add interrupt handlers for the only vector
5202 */
5203 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5204 (ddi_intr_handler_t *)ixgbe_intr_legacy,
5205 (void *)ixgbe, NULL);
5206
5207 if (rc != DDI_SUCCESS) {
5208 ixgbe_log(ixgbe,
5209 "Add legacy interrupt handler failed: %d", rc);
5210 return (IXGBE_FAILURE);
5211 }
5212
5213 break;
5214
5215 default:
5216 return (IXGBE_FAILURE);
5217 }
5218
5219 return (IXGBE_SUCCESS);
5220 }
5221
5222 #pragma inline(ixgbe_map_rxring_to_vector)
5223 /*
5224 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
5225 */
5226 static void
5227 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
5228 {
5229 /*
5230 * Set bit in map
5231 */
5232 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5233
5234 /*
5235 * Count bits set
5236 */
5237 ixgbe->vect_map[v_idx].rxr_cnt++;
5238
5239 /*
5240 * Remember bit position
5241 */
5242 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
5243 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
5244 }
5245
5246 #pragma inline(ixgbe_map_txring_to_vector)
5247 /*
5248 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
5249 */
5250 static void
5251 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
5252 {
5253 /*
5254 * Set bit in map
5255 */
5256 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
5257
5258 /*
5259 * Count bits set
5260 */
5261 ixgbe->vect_map[v_idx].txr_cnt++;
5262
5263 /*
5264 * Remember bit position
5265 */
5266 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
5267 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
5268 }
5269
5270 /*
5271 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
5272 * allocation register (IVAR).
5273 * cause:
5274 * -1 : other cause
5275 * 0 : rx
5276 * 1 : tx
5277 */
5278 static void
5279 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
5280 int8_t cause)
5281 {
5282 struct ixgbe_hw *hw = &ixgbe->hw;
5283 u32 ivar, index;
5284
5285 switch (hw->mac.type) {
5286 case ixgbe_mac_82598EB:
5287 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5288 if (cause == -1) {
5289 cause = 0;
5290 }
5291 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5292 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5293 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5294 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5295 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5296 break;
5297
5298 case ixgbe_mac_82599EB:
5299 case ixgbe_mac_X540:
5300 case ixgbe_mac_X550:
5301 case ixgbe_mac_X550EM_x:
5302 if (cause == -1) {
5303 /* other causes */
5304 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5305 index = (intr_alloc_entry & 1) * 8;
5306 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5307 ivar &= ~(0xFF << index);
5308 ivar |= (msix_vector << index);
5309 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5310 } else {
5311 /* tx or rx causes */
5312 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5313 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5314 ivar = IXGBE_READ_REG(hw,
5315 IXGBE_IVAR(intr_alloc_entry >> 1));
5316 ivar &= ~(0xFF << index);
5317 ivar |= (msix_vector << index);
5318 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5319 ivar);
5320 }
5321 break;
5322
5323 default:
5324 break;
5325 }
5326 }
5327
5328 /*
5329 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
5330 * given interrupt vector allocation register (IVAR).
5331 * cause:
5332 * -1 : other cause
5333 * 0 : rx
5334 * 1 : tx
5335 */
5336 static void
5337 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5338 {
5339 struct ixgbe_hw *hw = &ixgbe->hw;
5340 u32 ivar, index;
5341
5342 switch (hw->mac.type) {
5343 case ixgbe_mac_82598EB:
5344 if (cause == -1) {
5345 cause = 0;
5346 }
5347 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5348 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5349 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5350 (intr_alloc_entry & 0x3)));
5351 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5352 break;
5353
5354 case ixgbe_mac_82599EB:
5355 case ixgbe_mac_X540:
5356 case ixgbe_mac_X550:
5357 case ixgbe_mac_X550EM_x:
5358 if (cause == -1) {
5359 /* other causes */
5360 index = (intr_alloc_entry & 1) * 8;
5361 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5362 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5363 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5364 } else {
5365 /* tx or rx causes */
5366 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5367 ivar = IXGBE_READ_REG(hw,
5368 IXGBE_IVAR(intr_alloc_entry >> 1));
5369 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5370 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5371 ivar);
5372 }
5373 break;
5374
5375 default:
5376 break;
5377 }
5378 }
5379
5380 /*
5381 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5382 * given interrupt vector allocation register (IVAR).
5383 * cause:
5384 * -1 : other cause
5385 * 0 : rx
5386 * 1 : tx
5387 */
5388 static void
5389 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5390 {
5391 struct ixgbe_hw *hw = &ixgbe->hw;
5392 u32 ivar, index;
5393
5394 switch (hw->mac.type) {
5395 case ixgbe_mac_82598EB:
5396 if (cause == -1) {
5397 cause = 0;
5398 }
5399 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5400 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5401 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5402 (intr_alloc_entry & 0x3)));
5403 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5404 break;
5405
5406 case ixgbe_mac_82599EB:
5407 case ixgbe_mac_X540:
5408 case ixgbe_mac_X550:
5409 case ixgbe_mac_X550EM_x:
5410 if (cause == -1) {
5411 /* other causes */
5412 index = (intr_alloc_entry & 1) * 8;
5413 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5414 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5415 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5416 } else {
5417 /* tx or rx causes */
5418 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5419 ivar = IXGBE_READ_REG(hw,
5420 IXGBE_IVAR(intr_alloc_entry >> 1));
5421 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5422 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5423 ivar);
5424 }
5425 break;
5426
5427 default:
5428 break;
5429 }
5430 }
5431
5432 /*
5433 * Convert the rx ring index driver maintained to the rx ring index
5434 * in h/w.
5435 */
5436 static uint32_t
5437 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5438 {
5439
5440 struct ixgbe_hw *hw = &ixgbe->hw;
5441 uint32_t rx_ring_per_group, hw_rx_index;
5442
5443 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5444 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5445 return (sw_rx_index);
5446 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5447 switch (hw->mac.type) {
5448 case ixgbe_mac_82598EB:
5449 return (sw_rx_index);
5450
5451 case ixgbe_mac_82599EB:
5452 case ixgbe_mac_X540:
5453 case ixgbe_mac_X550:
5454 case ixgbe_mac_X550EM_x:
5455 return (sw_rx_index * 2);
5456
5457 default:
5458 break;
5459 }
5460 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5461 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5462
5463 switch (hw->mac.type) {
5464 case ixgbe_mac_82598EB:
5465 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5466 16 + (sw_rx_index % rx_ring_per_group);
5467 return (hw_rx_index);
5468
5469 case ixgbe_mac_82599EB:
5470 case ixgbe_mac_X540:
5471 case ixgbe_mac_X550:
5472 case ixgbe_mac_X550EM_x:
5473 if (ixgbe->num_rx_groups > 32) {
5474 hw_rx_index = (sw_rx_index /
5475 rx_ring_per_group) * 2 +
5476 (sw_rx_index % rx_ring_per_group);
5477 } else {
5478 hw_rx_index = (sw_rx_index /
5479 rx_ring_per_group) * 4 +
5480 (sw_rx_index % rx_ring_per_group);
5481 }
5482 return (hw_rx_index);
5483
5484 default:
5485 break;
5486 }
5487 }
5488
5489 /*
5490 * Should never reach. Just to make compiler happy.
5491 */
5492 return (sw_rx_index);
5493 }
5494
5495 /*
5496 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5497 *
5498 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5499 * to vector[0 - (intr_cnt -1)].
5500 */
5501 static int
5502 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5503 {
5504 int i, vector = 0;
5505
5506 /* initialize vector map */
5507 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5508 for (i = 0; i < ixgbe->intr_cnt; i++) {
5509 ixgbe->vect_map[i].ixgbe = ixgbe;
5510 }
5511
5512 /*
5513 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5514 * tx rings[0] on RTxQ[1].
5515 */
5516 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5517 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5518 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5519 return (IXGBE_SUCCESS);
5520 }
5521
5522 /*
5523 * Interrupts/vectors mapping for MSI-X
5524 */
5525
5526 /*
5527 * Map other interrupt to vector 0,
5528 * Set bit in map and count the bits set.
5529 */
5530 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5531 ixgbe->vect_map[vector].other_cnt++;
5532
5533 /*
5534 * Map rx ring interrupts to vectors
5535 */
5536 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5537 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5538 vector = (vector +1) % ixgbe->intr_cnt;
5539 }
5540
5541 /*
5542 * Map tx ring interrupts to vectors
5543 */
5544 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5545 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5546 vector = (vector +1) % ixgbe->intr_cnt;
5547 }
5548
5549 return (IXGBE_SUCCESS);
5550 }
5551
5552 /*
5553 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5554 *
5555 * This relies on ring/vector mapping already set up in the
5556 * vect_map[] structures
5557 */
5558 static void
5559 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5560 {
5561 struct ixgbe_hw *hw = &ixgbe->hw;
5562 ixgbe_intr_vector_t *vect; /* vector bitmap */
5563 int r_idx; /* ring index */
5564 int v_idx; /* vector index */
5565 uint32_t hw_index;
5566
5567 /*
5568 * Clear any previous entries
5569 */
5570 switch (hw->mac.type) {
5571 case ixgbe_mac_82598EB:
5572 for (v_idx = 0; v_idx < 25; v_idx++)
5573 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5574 break;
5575
5576 case ixgbe_mac_82599EB:
5577 case ixgbe_mac_X540:
5578 case ixgbe_mac_X550:
5579 case ixgbe_mac_X550EM_x:
5580 for (v_idx = 0; v_idx < 64; v_idx++)
5581 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5582 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5583 break;
5584
5585 default:
5586 break;
5587 }
5588
5589 /*
5590 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5591 * tx rings[0] will use RTxQ[1].
5592 */
5593 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5594 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5595 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5596 return;
5597 }
5598
5599 /*
5600 * For MSI-X interrupt, "Other" is always on vector[0].
5601 */
5602 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5603
5604 /*
5605 * For each interrupt vector, populate the IVAR table
5606 */
5607 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5608 vect = &ixgbe->vect_map[v_idx];
5609
5610 /*
5611 * For each rx ring bit set
5612 */
5613 r_idx = bt_getlowbit(vect->rx_map, 0,
5614 (ixgbe->num_rx_rings - 1));
5615
5616 while (r_idx >= 0) {
5617 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5618 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5619 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5620 (ixgbe->num_rx_rings - 1));
5621 }
5622
5623 /*
5624 * For each tx ring bit set
5625 */
5626 r_idx = bt_getlowbit(vect->tx_map, 0,
5627 (ixgbe->num_tx_rings - 1));
5628
5629 while (r_idx >= 0) {
5630 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5631 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5632 (ixgbe->num_tx_rings - 1));
5633 }
5634 }
5635 }
5636
5637 /*
5638 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5639 */
5640 static void
5641 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5642 {
5643 int i;
5644 int rc;
5645
5646 for (i = 0; i < ixgbe->intr_cnt; i++) {
5647 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5648 if (rc != DDI_SUCCESS) {
5649 IXGBE_DEBUGLOG_1(ixgbe,
5650 "Remove intr handler failed: %d", rc);
5651 }
5652 }
5653 }
5654
5655 /*
5656 * ixgbe_rem_intrs - Remove the allocated interrupts.
5657 */
5658 static void
5659 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5660 {
5661 int i;
5662 int rc;
5663
5664 for (i = 0; i < ixgbe->intr_cnt; i++) {
5665 rc = ddi_intr_free(ixgbe->htable[i]);
5666 if (rc != DDI_SUCCESS) {
5667 IXGBE_DEBUGLOG_1(ixgbe,
5668 "Free intr failed: %d", rc);
5669 }
5670 }
5671
5672 kmem_free(ixgbe->htable, ixgbe->intr_size);
5673 ixgbe->htable = NULL;
5674 }
5675
5676 /*
5677 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5678 */
5679 static int
5680 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5681 {
5682 int i;
5683 int rc;
5684
5685 /*
5686 * Enable interrupts
5687 */
5688 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5689 /*
5690 * Call ddi_intr_block_enable() for MSI
5691 */
5692 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5693 if (rc != DDI_SUCCESS) {
5694 ixgbe_log(ixgbe,
5695 "Enable block intr failed: %d", rc);
5696 return (IXGBE_FAILURE);
5697 }
5698 } else {
5699 /*
5700 * Call ddi_intr_enable() for Legacy/MSI non block enable
5701 */
5702 for (i = 0; i < ixgbe->intr_cnt; i++) {
5703 rc = ddi_intr_enable(ixgbe->htable[i]);
5704 if (rc != DDI_SUCCESS) {
5705 ixgbe_log(ixgbe,
5706 "Enable intr failed: %d", rc);
5707 return (IXGBE_FAILURE);
5708 }
5709 }
5710 }
5711
5712 return (IXGBE_SUCCESS);
5713 }
5714
5715 /*
5716 * ixgbe_disable_intrs - Disable all the interrupts.
5717 */
5718 static int
5719 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5720 {
5721 int i;
5722 int rc;
5723
5724 /*
5725 * Disable all interrupts
5726 */
5727 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5728 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5729 if (rc != DDI_SUCCESS) {
5730 ixgbe_log(ixgbe,
5731 "Disable block intr failed: %d", rc);
5732 return (IXGBE_FAILURE);
5733 }
5734 } else {
5735 for (i = 0; i < ixgbe->intr_cnt; i++) {
5736 rc = ddi_intr_disable(ixgbe->htable[i]);
5737 if (rc != DDI_SUCCESS) {
5738 ixgbe_log(ixgbe,
5739 "Disable intr failed: %d", rc);
5740 return (IXGBE_FAILURE);
5741 }
5742 }
5743 }
5744
5745 return (IXGBE_SUCCESS);
5746 }
5747
5748 /*
5749 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5750 */
5751 static void
5752 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5753 {
5754 struct ixgbe_hw *hw = &ixgbe->hw;
5755 ixgbe_link_speed speed = 0;
5756 boolean_t link_up = B_FALSE;
5757 uint32_t pcs1g_anlp = 0;
5758
5759 ASSERT(mutex_owned(&ixgbe->gen_lock));
5760 ixgbe->param_lp_1000fdx_cap = 0;
5761 ixgbe->param_lp_100fdx_cap = 0;
5762
5763 /* check for link, don't wait */
5764 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5765
5766 /*
5767 * Update the observed Link Partner's capabilities. Not all adapters
5768 * can provide full information on the LP's capable speeds, so we
5769 * provide what we can.
5770 */
5771 if (link_up) {
5772 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5773
5774 ixgbe->param_lp_1000fdx_cap =
5775 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5776 ixgbe->param_lp_100fdx_cap =
5777 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5778 }
5779
5780 /*
5781 * Update GLD's notion of the adapter's currently advertised speeds.
5782 * Since the common code doesn't always record the current autonegotiate
5783 * settings in the phy struct for all parts (specifically, adapters with
5784 * SFPs) we first test to see if it is 0, and if so, we fall back to
5785 * using the adapter's speed capabilities which we saved during instance
5786 * init in ixgbe_init_params().
5787 *
5788 * Adapters with SFPs will always be shown as advertising all of their
5789 * supported speeds, and adapters with baseT PHYs (where the phy struct
5790 * is maintained by the common code) will always have a factual view of
5791 * their currently-advertised speeds. In the case of SFPs, this is
5792 * acceptable as we default to advertising all speeds that the adapter
5793 * claims to support, and those properties are immutable; unlike on
5794 * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5795 */
5796 speed = hw->phy.autoneg_advertised;
5797 if (speed == 0)
5798 speed = ixgbe->speeds_supported;
5799
5800 ixgbe->param_adv_10000fdx_cap =
5801 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5802 ixgbe->param_adv_5000fdx_cap =
5803 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5804 ixgbe->param_adv_2500fdx_cap =
5805 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5806 ixgbe->param_adv_1000fdx_cap =
5807 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5808 ixgbe->param_adv_100fdx_cap =
5809 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5810 }
5811
5812 /*
5813 * ixgbe_get_driver_control - Notify that driver is in control of device.
5814 */
5815 static void
5816 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5817 {
5818 uint32_t ctrl_ext;
5819
5820 /*
5821 * Notify firmware that driver is in control of device
5822 */
5823 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5824 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5825 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5826 }
5827
5828 /*
5829 * ixgbe_release_driver_control - Notify that driver is no longer in control
5830 * of device.
5831 */
5832 static void
5833 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5834 {
5835 uint32_t ctrl_ext;
5836
5837 /*
5838 * Notify firmware that driver is no longer in control of device
5839 */
5840 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5841 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5842 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5843 }
5844
5845 /*
5846 * ixgbe_atomic_reserve - Atomic decrease operation.
5847 */
5848 int
5849 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5850 {
5851 uint32_t oldval;
5852 uint32_t newval;
5853
5854 /*
5855 * ATOMICALLY
5856 */
5857 do {
5858 oldval = *count_p;
5859 if (oldval < n)
5860 return (-1);
5861 newval = oldval - n;
5862 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5863
5864 return (newval);
5865 }
5866
5867 /*
5868 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5869 */
5870 static uint8_t *
5871 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5872 {
5873 uint8_t *addr = *upd_ptr;
5874 uint8_t *new_ptr;
5875
5876 _NOTE(ARGUNUSED(hw));
5877 _NOTE(ARGUNUSED(vmdq));
5878
5879 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5880 *upd_ptr = new_ptr;
5881 return (addr);
5882 }
5883
5884 /*
5885 * FMA support
5886 */
5887 int
5888 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5889 {
5890 ddi_fm_error_t de;
5891
5892 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5893 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5894 return (de.fme_status);
5895 }
5896
5897 int
5898 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5899 {
5900 ddi_fm_error_t de;
5901
5902 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5903 return (de.fme_status);
5904 }
5905
5906 /*
5907 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5908 */
5909 static int
5910 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5911 {
5912 _NOTE(ARGUNUSED(impl_data));
5913 /*
5914 * as the driver can always deal with an error in any dma or
5915 * access handle, we can just return the fme_status value.
5916 */
5917 pci_ereport_post(dip, err, NULL);
5918 return (err->fme_status);
5919 }
5920
5921 static void
5922 ixgbe_fm_init(ixgbe_t *ixgbe)
5923 {
5924 ddi_iblock_cookie_t iblk;
5925 int fma_dma_flag;
5926
5927 /*
5928 * Only register with IO Fault Services if we have some capability
5929 */
5930 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5931 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5932 } else {
5933 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5934 }
5935
5936 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5937 fma_dma_flag = 1;
5938 } else {
5939 fma_dma_flag = 0;
5940 }
5941
5942 ixgbe_set_fma_flags(fma_dma_flag);
5943
5944 if (ixgbe->fm_capabilities) {
5945
5946 /*
5947 * Register capabilities with IO Fault Services
5948 */
5949 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5950
5951 /*
5952 * Initialize pci ereport capabilities if ereport capable
5953 */
5954 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5955 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5956 pci_ereport_setup(ixgbe->dip);
5957
5958 /*
5959 * Register error callback if error callback capable
5960 */
5961 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5962 ddi_fm_handler_register(ixgbe->dip,
5963 ixgbe_fm_error_cb, (void*) ixgbe);
5964 }
5965 }
5966
5967 static void
5968 ixgbe_fm_fini(ixgbe_t *ixgbe)
5969 {
5970 /*
5971 * Only unregister FMA capabilities if they are registered
5972 */
5973 if (ixgbe->fm_capabilities) {
5974
5975 /*
5976 * Release any resources allocated by pci_ereport_setup()
5977 */
5978 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5979 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5980 pci_ereport_teardown(ixgbe->dip);
5981
5982 /*
5983 * Un-register error callback if error callback capable
5984 */
5985 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5986 ddi_fm_handler_unregister(ixgbe->dip);
5987
5988 /*
5989 * Unregister from IO Fault Service
5990 */
5991 ddi_fm_fini(ixgbe->dip);
5992 }
5993 }
5994
5995 void
5996 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5997 {
5998 uint64_t ena;
5999 char buf[FM_MAX_CLASS];
6000
6001 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
6002 ena = fm_ena_generate(0, FM_ENA_FMT1);
6003 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
6004 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
6005 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
6006 }
6007 }
6008
6009 static int
6010 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
6011 {
6012 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
6013
6014 mutex_enter(&rx_ring->rx_lock);
6015 rx_ring->ring_gen_num = mr_gen_num;
6016 mutex_exit(&rx_ring->rx_lock);
6017 return (0);
6018 }
6019
6020 /*
6021 * Get the global ring index by a ring index within a group.
6022 */
6023 static int
6024 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
6025 {
6026 ixgbe_rx_ring_t *rx_ring;
6027 int i;
6028
6029 for (i = 0; i < ixgbe->num_rx_rings; i++) {
6030 rx_ring = &ixgbe->rx_rings[i];
6031 if (rx_ring->group_index == gindex)
6032 rindex--;
6033 if (rindex < 0)
6034 return (i);
6035 }
6036
6037 return (-1);
6038 }
6039
6040 /*
6041 * Callback funtion for MAC layer to register all rings.
6042 */
6043 /* ARGSUSED */
6044 void
6045 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
6046 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
6047 {
6048 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6049 mac_intr_t *mintr = &infop->mri_intr;
6050
6051 switch (rtype) {
6052 case MAC_RING_TYPE_RX: {
6053 /*
6054 * 'index' is the ring index within the group.
6055 * Need to get the global ring index by searching in groups.
6056 */
6057 int global_ring_index = ixgbe_get_rx_ring_index(
6058 ixgbe, group_index, ring_index);
6059
6060 ASSERT(global_ring_index >= 0);
6061
6062 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
6063 rx_ring->ring_handle = rh;
6064
6065 infop->mri_driver = (mac_ring_driver_t)rx_ring;
6066 infop->mri_start = ixgbe_ring_start;
6067 infop->mri_stop = NULL;
6068 infop->mri_poll = ixgbe_ring_rx_poll;
6069 infop->mri_stat = ixgbe_rx_ring_stat;
6070
6071 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
6072 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
6073 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
6074 if (ixgbe->intr_type &
6075 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6076 mintr->mi_ddi_handle =
6077 ixgbe->htable[rx_ring->intr_vector];
6078 }
6079
6080 break;
6081 }
6082 case MAC_RING_TYPE_TX: {
6083 ASSERT(group_index == -1);
6084 ASSERT(ring_index < ixgbe->num_tx_rings);
6085
6086 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
6087 tx_ring->ring_handle = rh;
6088
6089 infop->mri_driver = (mac_ring_driver_t)tx_ring;
6090 infop->mri_start = NULL;
6091 infop->mri_stop = NULL;
6092 infop->mri_tx = ixgbe_ring_tx;
6093 infop->mri_stat = ixgbe_tx_ring_stat;
6094 if (ixgbe->intr_type &
6095 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6096 mintr->mi_ddi_handle =
6097 ixgbe->htable[tx_ring->intr_vector];
6098 }
6099 break;
6100 }
6101 default:
6102 break;
6103 }
6104 }
6105
6106 /*
6107 * Callback funtion for MAC layer to register all groups.
6108 */
6109 void
6110 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
6111 mac_group_info_t *infop, mac_group_handle_t gh)
6112 {
6113 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6114
6115 switch (rtype) {
6116 case MAC_RING_TYPE_RX: {
6117 ixgbe_rx_group_t *rx_group;
6118
6119 rx_group = &ixgbe->rx_groups[index];
6120 rx_group->group_handle = gh;
6121
6122 infop->mgi_driver = (mac_group_driver_t)rx_group;
6123 infop->mgi_start = NULL;
6124 infop->mgi_stop = NULL;
6125 infop->mgi_addmac = ixgbe_addmac;
6126 infop->mgi_remmac = ixgbe_remmac;
6127 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
6128
6129 break;
6130 }
6131 case MAC_RING_TYPE_TX:
6132 break;
6133 default:
6134 break;
6135 }
6136 }
6137
6138 /*
6139 * Enable interrupt on the specificed rx ring.
6140 */
6141 int
6142 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
6143 {
6144 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6145 ixgbe_t *ixgbe = rx_ring->ixgbe;
6146 int r_idx = rx_ring->index;
6147 int hw_r_idx = rx_ring->hw_index;
6148 int v_idx = rx_ring->intr_vector;
6149
6150 mutex_enter(&ixgbe->gen_lock);
6151 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6152 mutex_exit(&ixgbe->gen_lock);
6153 /*
6154 * Simply return 0.
6155 * Interrupts are being adjusted. ixgbe_intr_adjust()
6156 * will eventually re-enable the interrupt when it's
6157 * done with the adjustment.
6158 */
6159 return (0);
6160 }
6161
6162 /*
6163 * To enable interrupt by setting the VAL bit of given interrupt
6164 * vector allocation register (IVAR).
6165 */
6166 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
6167
6168 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
6169
6170 /*
6171 * Trigger a Rx interrupt on this ring
6172 */
6173 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
6174 IXGBE_WRITE_FLUSH(&ixgbe->hw);
6175
6176 mutex_exit(&ixgbe->gen_lock);
6177
6178 return (0);
6179 }
6180
6181 /*
6182 * Disable interrupt on the specificed rx ring.
6183 */
6184 int
6185 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
6186 {
6187 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6188 ixgbe_t *ixgbe = rx_ring->ixgbe;
6189 int r_idx = rx_ring->index;
6190 int hw_r_idx = rx_ring->hw_index;
6191 int v_idx = rx_ring->intr_vector;
6192
6193 mutex_enter(&ixgbe->gen_lock);
6194 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6195 mutex_exit(&ixgbe->gen_lock);
6196 /*
6197 * Simply return 0.
6198 * In the rare case where an interrupt is being
6199 * disabled while interrupts are being adjusted,
6200 * we don't fail the operation. No interrupts will
6201 * be generated while they are adjusted, and
6202 * ixgbe_intr_adjust() will cause the interrupts
6203 * to be re-enabled once it completes. Note that
6204 * in this case, packets may be delivered to the
6205 * stack via interrupts before xgbe_rx_ring_intr_enable()
6206 * is called again. This is acceptable since interrupt
6207 * adjustment is infrequent, and the stack will be
6208 * able to handle these packets.
6209 */
6210 return (0);
6211 }
6212
6213 /*
6214 * To disable interrupt by clearing the VAL bit of given interrupt
6215 * vector allocation register (IVAR).
6216 */
6217 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
6218
6219 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
6220
6221 mutex_exit(&ixgbe->gen_lock);
6222
6223 return (0);
6224 }
6225
6226 /*
6227 * Add a mac address.
6228 */
6229 static int
6230 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
6231 {
6232 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6233 ixgbe_t *ixgbe = rx_group->ixgbe;
6234 struct ixgbe_hw *hw = &ixgbe->hw;
6235 int slot, i;
6236
6237 mutex_enter(&ixgbe->gen_lock);
6238
6239 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6240 mutex_exit(&ixgbe->gen_lock);
6241 return (ECANCELED);
6242 }
6243
6244 if (ixgbe->unicst_avail == 0) {
6245 /* no slots available */
6246 mutex_exit(&ixgbe->gen_lock);
6247 return (ENOSPC);
6248 }
6249
6250 /*
6251 * The first ixgbe->num_rx_groups slots are reserved for each respective
6252 * group. The rest slots are shared by all groups. While adding a
6253 * MAC address, reserved slots are firstly checked then the shared
6254 * slots are searched.
6255 */
6256 slot = -1;
6257 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
6258 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
6259 if (ixgbe->unicst_addr[i].mac.set == 0) {
6260 slot = i;
6261 break;
6262 }
6263 }
6264 } else {
6265 slot = rx_group->index;
6266 }
6267
6268 if (slot == -1) {
6269 /* no slots available */
6270 mutex_exit(&ixgbe->gen_lock);
6271 return (ENOSPC);
6272 }
6273
6274 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6275 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
6276 rx_group->index, IXGBE_RAH_AV);
6277 ixgbe->unicst_addr[slot].mac.set = 1;
6278 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
6279 ixgbe->unicst_avail--;
6280
6281 mutex_exit(&ixgbe->gen_lock);
6282
6283 return (0);
6284 }
6285
6286 /*
6287 * Remove a mac address.
6288 */
6289 static int
6290 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
6291 {
6292 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6293 ixgbe_t *ixgbe = rx_group->ixgbe;
6294 struct ixgbe_hw *hw = &ixgbe->hw;
6295 int slot;
6296
6297 mutex_enter(&ixgbe->gen_lock);
6298
6299 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6300 mutex_exit(&ixgbe->gen_lock);
6301 return (ECANCELED);
6302 }
6303
6304 slot = ixgbe_unicst_find(ixgbe, mac_addr);
6305 if (slot == -1) {
6306 mutex_exit(&ixgbe->gen_lock);
6307 return (EINVAL);
6308 }
6309
6310 if (ixgbe->unicst_addr[slot].mac.set == 0) {
6311 mutex_exit(&ixgbe->gen_lock);
6312 return (EINVAL);
6313 }
6314
6315 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6316 (void) ixgbe_clear_rar(hw, slot);
6317 ixgbe->unicst_addr[slot].mac.set = 0;
6318 ixgbe->unicst_avail++;
6319
6320 mutex_exit(&ixgbe->gen_lock);
6321
6322 return (0);
6323 }