1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 */
31
32 #include "ixgbe_sw.h"
33
34 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
35 static char ixgbe_version[] = "ixgbe 1.1.7";
36
37 /*
38 * Local function protoypes
39 */
40 static int ixgbe_register_mac(ixgbe_t *);
41 static int ixgbe_identify_hardware(ixgbe_t *);
42 static int ixgbe_regs_map(ixgbe_t *);
43 static void ixgbe_init_properties(ixgbe_t *);
44 static int ixgbe_init_driver_settings(ixgbe_t *);
45 static void ixgbe_init_locks(ixgbe_t *);
46 static void ixgbe_destroy_locks(ixgbe_t *);
47 static int ixgbe_init(ixgbe_t *);
48 static int ixgbe_chip_start(ixgbe_t *);
49 static void ixgbe_chip_stop(ixgbe_t *);
50 static int ixgbe_reset(ixgbe_t *);
51 static void ixgbe_tx_clean(ixgbe_t *);
52 static boolean_t ixgbe_tx_drain(ixgbe_t *);
53 static boolean_t ixgbe_rx_drain(ixgbe_t *);
54 static int ixgbe_alloc_rings(ixgbe_t *);
55 static void ixgbe_free_rings(ixgbe_t *);
56 static int ixgbe_alloc_rx_data(ixgbe_t *);
57 static void ixgbe_free_rx_data(ixgbe_t *);
58 static void ixgbe_setup_rings(ixgbe_t *);
59 static void ixgbe_setup_rx(ixgbe_t *);
60 static void ixgbe_setup_tx(ixgbe_t *);
61 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
62 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
63 static void ixgbe_setup_rss(ixgbe_t *);
64 static void ixgbe_setup_vmdq(ixgbe_t *);
65 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
66 static void ixgbe_init_unicst(ixgbe_t *);
67 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
68 static void ixgbe_setup_multicst(ixgbe_t *);
69 static void ixgbe_get_hw_state(ixgbe_t *);
70 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
71 static void ixgbe_get_conf(ixgbe_t *);
72 static void ixgbe_init_params(ixgbe_t *);
73 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
74 static void ixgbe_driver_link_check(ixgbe_t *);
75 static void ixgbe_sfp_check(void *);
76 static void ixgbe_overtemp_check(void *);
77 static void ixgbe_link_timer(void *);
78 static void ixgbe_local_timer(void *);
79 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
80 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
81 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
82 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
83 static boolean_t is_valid_mac_addr(uint8_t *);
84 static boolean_t ixgbe_stall_check(ixgbe_t *);
85 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
86 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
87 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
88 static int ixgbe_alloc_intrs(ixgbe_t *);
89 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
90 static int ixgbe_add_intr_handlers(ixgbe_t *);
91 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
92 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
93 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
94 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
95 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
96 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
97 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
98 static void ixgbe_setup_adapter_vector(ixgbe_t *);
99 static void ixgbe_rem_intr_handlers(ixgbe_t *);
100 static void ixgbe_rem_intrs(ixgbe_t *);
101 static int ixgbe_enable_intrs(ixgbe_t *);
102 static int ixgbe_disable_intrs(ixgbe_t *);
103 static uint_t ixgbe_intr_legacy(void *, void *);
104 static uint_t ixgbe_intr_msi(void *, void *);
105 static uint_t ixgbe_intr_msix(void *, void *);
106 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
107 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
108 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
109 static void ixgbe_get_driver_control(struct ixgbe_hw *);
110 static int ixgbe_addmac(void *, const uint8_t *);
111 static int ixgbe_remmac(void *, const uint8_t *);
112 static void ixgbe_release_driver_control(struct ixgbe_hw *);
113
114 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
115 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
116 static int ixgbe_resume(dev_info_t *);
117 static int ixgbe_suspend(dev_info_t *);
118 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
119 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
120 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
121 static int ixgbe_intr_cb_register(ixgbe_t *);
122 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
123
124 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
125 const void *impl_data);
126 static void ixgbe_fm_init(ixgbe_t *);
127 static void ixgbe_fm_fini(ixgbe_t *);
128
129 char *ixgbe_priv_props[] = {
130 "_tx_copy_thresh",
131 "_tx_recycle_thresh",
132 "_tx_overload_thresh",
133 "_tx_resched_thresh",
134 "_rx_copy_thresh",
135 "_rx_limit_per_intr",
136 "_intr_throttling",
137 "_adv_pause_cap",
138 "_adv_asym_pause_cap",
139 NULL
140 };
141
142 #define IXGBE_MAX_PRIV_PROPS \
143 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
144
145 static struct cb_ops ixgbe_cb_ops = {
146 nulldev, /* cb_open */
147 nulldev, /* cb_close */
148 nodev, /* cb_strategy */
149 nodev, /* cb_print */
150 nodev, /* cb_dump */
151 nodev, /* cb_read */
152 nodev, /* cb_write */
153 nodev, /* cb_ioctl */
154 nodev, /* cb_devmap */
155 nodev, /* cb_mmap */
156 nodev, /* cb_segmap */
157 nochpoll, /* cb_chpoll */
158 ddi_prop_op, /* cb_prop_op */
159 NULL, /* cb_stream */
160 D_MP | D_HOTPLUG, /* cb_flag */
161 CB_REV, /* cb_rev */
162 nodev, /* cb_aread */
163 nodev /* cb_awrite */
164 };
165
166 static struct dev_ops ixgbe_dev_ops = {
167 DEVO_REV, /* devo_rev */
168 0, /* devo_refcnt */
169 NULL, /* devo_getinfo */
170 nulldev, /* devo_identify */
171 nulldev, /* devo_probe */
172 ixgbe_attach, /* devo_attach */
173 ixgbe_detach, /* devo_detach */
174 nodev, /* devo_reset */
175 &ixgbe_cb_ops, /* devo_cb_ops */
176 NULL, /* devo_bus_ops */
177 ddi_power, /* devo_power */
178 ddi_quiesce_not_supported, /* devo_quiesce */
179 };
180
181 static struct modldrv ixgbe_modldrv = {
182 &mod_driverops, /* Type of module. This one is a driver */
183 ixgbe_ident, /* Discription string */
184 &ixgbe_dev_ops /* driver ops */
185 };
186
187 static struct modlinkage ixgbe_modlinkage = {
188 MODREV_1, &ixgbe_modldrv, NULL
189 };
190
191 /*
192 * Access attributes for register mapping
193 */
194 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
195 DDI_DEVICE_ATTR_V1,
196 DDI_STRUCTURE_LE_ACC,
197 DDI_STRICTORDER_ACC,
198 DDI_FLAGERR_ACC
199 };
200
201 /*
202 * Loopback property
203 */
204 static lb_property_t lb_normal = {
205 normal, "normal", IXGBE_LB_NONE
206 };
207
208 static lb_property_t lb_mac = {
209 internal, "MAC", IXGBE_LB_INTERNAL_MAC
210 };
211
212 static lb_property_t lb_external = {
213 external, "External", IXGBE_LB_EXTERNAL
214 };
215
216 #define IXGBE_M_CALLBACK_FLAGS \
217 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
218
219 static mac_callbacks_t ixgbe_m_callbacks = {
220 IXGBE_M_CALLBACK_FLAGS,
221 ixgbe_m_stat,
222 ixgbe_m_start,
223 ixgbe_m_stop,
224 ixgbe_m_promisc,
225 ixgbe_m_multicst,
226 NULL,
227 NULL,
228 NULL,
229 ixgbe_m_ioctl,
230 ixgbe_m_getcapab,
231 NULL,
232 NULL,
233 ixgbe_m_setprop,
234 ixgbe_m_getprop,
235 ixgbe_m_propinfo
236 };
237
238 /*
239 * Initialize capabilities of each supported adapter type
240 */
241 static adapter_info_t ixgbe_82598eb_cap = {
242 64, /* maximum number of rx queues */
243 1, /* minimum number of rx queues */
244 64, /* default number of rx queues */
245 16, /* maximum number of rx groups */
246 1, /* minimum number of rx groups */
247 1, /* default number of rx groups */
248 32, /* maximum number of tx queues */
249 1, /* minimum number of tx queues */
250 8, /* default number of tx queues */
251 16366, /* maximum MTU size */
252 0xFFFF, /* maximum interrupt throttle rate */
253 0, /* minimum interrupt throttle rate */
254 200, /* default interrupt throttle rate */
255 18, /* maximum total msix vectors */
256 16, /* maximum number of ring vectors */
257 2, /* maximum number of other vectors */
258 IXGBE_EICR_LSC, /* "other" interrupt types handled */
259 0, /* "other" interrupt types enable mask */
260 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
261 | IXGBE_FLAG_RSS_CAPABLE
262 | IXGBE_FLAG_VMDQ_CAPABLE)
263 };
264
265 static adapter_info_t ixgbe_82599eb_cap = {
266 128, /* maximum number of rx queues */
267 1, /* minimum number of rx queues */
268 128, /* default number of rx queues */
269 64, /* maximum number of rx groups */
270 1, /* minimum number of rx groups */
271 1, /* default number of rx groups */
272 128, /* maximum number of tx queues */
273 1, /* minimum number of tx queues */
274 8, /* default number of tx queues */
275 15500, /* maximum MTU size */
276 0xFF8, /* maximum interrupt throttle rate */
277 0, /* minimum interrupt throttle rate */
278 200, /* default interrupt throttle rate */
279 64, /* maximum total msix vectors */
280 16, /* maximum number of ring vectors */
281 2, /* maximum number of other vectors */
282 (IXGBE_EICR_LSC
283 | IXGBE_EICR_GPI_SDP1
284 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
285
286 (IXGBE_SDP1_GPIEN
287 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
288
289 (IXGBE_FLAG_DCA_CAPABLE
290 | IXGBE_FLAG_RSS_CAPABLE
291 | IXGBE_FLAG_VMDQ_CAPABLE
292 | IXGBE_FLAG_RSC_CAPABLE
293 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
294 };
295
296 static adapter_info_t ixgbe_X540_cap = {
297 128, /* maximum number of rx queues */
298 1, /* minimum number of rx queues */
299 128, /* default number of rx queues */
300 64, /* maximum number of rx groups */
301 1, /* minimum number of rx groups */
302 1, /* default number of rx groups */
303 128, /* maximum number of tx queues */
304 1, /* minimum number of tx queues */
305 8, /* default number of tx queues */
306 15500, /* maximum MTU size */
307 0xFF8, /* maximum interrupt throttle rate */
308 0, /* minimum interrupt throttle rate */
309 200, /* default interrupt throttle rate */
310 64, /* maximum total msix vectors */
311 16, /* maximum number of ring vectors */
312 2, /* maximum number of other vectors */
313 (IXGBE_EICR_LSC
314 | IXGBE_EICR_GPI_SDP1
315 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
316
317 (IXGBE_SDP1_GPIEN
318 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
319
320 (IXGBE_FLAG_DCA_CAPABLE
321 | IXGBE_FLAG_RSS_CAPABLE
322 | IXGBE_FLAG_VMDQ_CAPABLE
323 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
324 };
325
326 /*
327 * Module Initialization Functions.
328 */
329
330 int
331 _init(void)
332 {
333 int status;
334
335 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
336
337 status = mod_install(&ixgbe_modlinkage);
338
339 if (status != DDI_SUCCESS) {
340 mac_fini_ops(&ixgbe_dev_ops);
341 }
342
343 return (status);
344 }
345
346 int
347 _fini(void)
348 {
349 int status;
350
351 status = mod_remove(&ixgbe_modlinkage);
352
353 if (status == DDI_SUCCESS) {
354 mac_fini_ops(&ixgbe_dev_ops);
355 }
356
357 return (status);
358 }
359
360 int
361 _info(struct modinfo *modinfop)
362 {
363 int status;
364
365 status = mod_info(&ixgbe_modlinkage, modinfop);
366
367 return (status);
368 }
369
370 /*
371 * ixgbe_attach - Driver attach.
372 *
373 * This function is the device specific initialization entry
374 * point. This entry point is required and must be written.
375 * The DDI_ATTACH command must be provided in the attach entry
376 * point. When attach() is called with cmd set to DDI_ATTACH,
377 * all normal kernel services (such as kmem_alloc(9F)) are
378 * available for use by the driver.
379 *
380 * The attach() function will be called once for each instance
381 * of the device on the system with cmd set to DDI_ATTACH.
382 * Until attach() succeeds, the only driver entry points which
383 * may be called are open(9E) and getinfo(9E).
384 */
385 static int
386 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
387 {
388 ixgbe_t *ixgbe;
389 struct ixgbe_osdep *osdep;
390 struct ixgbe_hw *hw;
391 int instance;
392 char taskqname[32];
393
394 /*
395 * Check the command and perform corresponding operations
396 */
397 switch (cmd) {
398 default:
399 return (DDI_FAILURE);
400
401 case DDI_RESUME:
402 return (ixgbe_resume(devinfo));
403
404 case DDI_ATTACH:
405 break;
406 }
407
408 /* Get the device instance */
409 instance = ddi_get_instance(devinfo);
410
411 /* Allocate memory for the instance data structure */
412 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
413
414 ixgbe->dip = devinfo;
415 ixgbe->instance = instance;
416
417 hw = &ixgbe->hw;
418 osdep = &ixgbe->osdep;
419 hw->back = osdep;
420 osdep->ixgbe = ixgbe;
421
422 /* Attach the instance pointer to the dev_info data structure */
423 ddi_set_driver_private(devinfo, ixgbe);
424
425 /*
426 * Initialize for fma support
427 */
428 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
429 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
430 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
431 ixgbe_fm_init(ixgbe);
432 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
433
434 /*
435 * Map PCI config space registers
436 */
437 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
438 ixgbe_error(ixgbe, "Failed to map PCI configurations");
439 goto attach_fail;
440 }
441 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
442
443 /*
444 * Identify the chipset family
445 */
446 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
447 ixgbe_error(ixgbe, "Failed to identify hardware");
448 goto attach_fail;
449 }
450
451 /*
452 * Map device registers
453 */
454 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
455 ixgbe_error(ixgbe, "Failed to map device registers");
456 goto attach_fail;
457 }
458 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
459
460 /*
461 * Initialize driver parameters
462 */
463 ixgbe_init_properties(ixgbe);
464 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
465
466 /*
467 * Register interrupt callback
468 */
469 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
470 ixgbe_error(ixgbe, "Failed to register interrupt callback");
471 goto attach_fail;
472 }
473
474 /*
475 * Allocate interrupts
476 */
477 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
478 ixgbe_error(ixgbe, "Failed to allocate interrupts");
479 goto attach_fail;
480 }
481 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
482
483 /*
484 * Allocate rx/tx rings based on the ring numbers.
485 * The actual numbers of rx/tx rings are decided by the number of
486 * allocated interrupt vectors, so we should allocate the rings after
487 * interrupts are allocated.
488 */
489 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
490 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
491 goto attach_fail;
492 }
493 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
494
495 /*
496 * Map rings to interrupt vectors
497 */
498 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
499 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
500 goto attach_fail;
501 }
502
503 /*
504 * Add interrupt handlers
505 */
506 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
507 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
508 goto attach_fail;
509 }
510 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
511
512 /*
513 * Create a taskq for sfp-change
514 */
515 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
516 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
517 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
518 ixgbe_error(ixgbe, "sfp_taskq create failed");
519 goto attach_fail;
520 }
521 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
522
523 /*
524 * Create a taskq for over-temp
525 */
526 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
527 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
528 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
529 ixgbe_error(ixgbe, "overtemp_taskq create failed");
530 goto attach_fail;
531 }
532 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
533
534 /*
535 * Initialize driver parameters
536 */
537 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
538 ixgbe_error(ixgbe, "Failed to initialize driver settings");
539 goto attach_fail;
540 }
541
542 /*
543 * Initialize mutexes for this device.
544 * Do this before enabling the interrupt handler and
545 * register the softint to avoid the condition where
546 * interrupt handler can try using uninitialized mutex.
547 */
548 ixgbe_init_locks(ixgbe);
549 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
550
551 /*
552 * Initialize chipset hardware
553 */
554 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
555 ixgbe_error(ixgbe, "Failed to initialize adapter");
556 goto attach_fail;
557 }
558 ixgbe->link_check_complete = B_FALSE;
559 ixgbe->link_check_hrtime = gethrtime() +
560 (IXGBE_LINK_UP_TIME * 100000000ULL);
561 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
562
563 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
564 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
565 goto attach_fail;
566 }
567
568 /*
569 * Initialize statistics
570 */
571 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
572 ixgbe_error(ixgbe, "Failed to initialize statistics");
573 goto attach_fail;
574 }
575 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
576
577 /*
578 * Register the driver to the MAC
579 */
580 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
581 ixgbe_error(ixgbe, "Failed to register MAC");
582 goto attach_fail;
583 }
584 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
585 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
586
587 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
588 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
589 if (ixgbe->periodic_id == 0) {
590 ixgbe_error(ixgbe, "Failed to add the link check timer");
591 goto attach_fail;
592 }
593 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
594
595 /*
596 * Now that mutex locks are initialized, and the chip is also
597 * initialized, enable interrupts.
598 */
599 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
600 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
601 goto attach_fail;
602 }
603 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
604
605 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version);
606 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
607
608 return (DDI_SUCCESS);
609
610 attach_fail:
611 ixgbe_unconfigure(devinfo, ixgbe);
612 return (DDI_FAILURE);
613 }
614
615 /*
616 * ixgbe_detach - Driver detach.
617 *
618 * The detach() function is the complement of the attach routine.
619 * If cmd is set to DDI_DETACH, detach() is used to remove the
620 * state associated with a given instance of a device node
621 * prior to the removal of that instance from the system.
622 *
623 * The detach() function will be called once for each instance
624 * of the device for which there has been a successful attach()
625 * once there are no longer any opens on the device.
626 *
627 * Interrupts routine are disabled, All memory allocated by this
628 * driver are freed.
629 */
630 static int
631 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
632 {
633 ixgbe_t *ixgbe;
634
635 /*
636 * Check detach command
637 */
638 switch (cmd) {
639 default:
640 return (DDI_FAILURE);
641
642 case DDI_SUSPEND:
643 return (ixgbe_suspend(devinfo));
644
645 case DDI_DETACH:
646 break;
647 }
648
649 /*
650 * Get the pointer to the driver private data structure
651 */
652 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
653 if (ixgbe == NULL)
654 return (DDI_FAILURE);
655
656 /*
657 * If the device is still running, it needs to be stopped first.
658 * This check is necessary because under some specific circumstances,
659 * the detach routine can be called without stopping the interface
660 * first.
661 */
662 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
663 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
664 mutex_enter(&ixgbe->gen_lock);
665 ixgbe_stop(ixgbe, B_TRUE);
666 mutex_exit(&ixgbe->gen_lock);
667 /* Disable and stop the watchdog timer */
668 ixgbe_disable_watchdog_timer(ixgbe);
669 }
670
671 /*
672 * Check if there are still rx buffers held by the upper layer.
673 * If so, fail the detach.
674 */
675 if (!ixgbe_rx_drain(ixgbe))
676 return (DDI_FAILURE);
677
678 /*
679 * Do the remaining unconfigure routines
680 */
681 ixgbe_unconfigure(devinfo, ixgbe);
682
683 return (DDI_SUCCESS);
684 }
685
686 static void
687 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
688 {
689 /*
690 * Disable interrupt
691 */
692 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
693 (void) ixgbe_disable_intrs(ixgbe);
694 }
695
696 /*
697 * remove the link check timer
698 */
699 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
700 if (ixgbe->periodic_id != NULL) {
701 ddi_periodic_delete(ixgbe->periodic_id);
702 ixgbe->periodic_id = NULL;
703 }
704 }
705
706 /*
707 * Unregister MAC
708 */
709 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
710 (void) mac_unregister(ixgbe->mac_hdl);
711 }
712
713 /*
714 * Free statistics
715 */
716 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
717 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
718 }
719
720 /*
721 * Remove interrupt handlers
722 */
723 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
724 ixgbe_rem_intr_handlers(ixgbe);
725 }
726
727 /*
728 * Remove taskq for sfp-status-change
729 */
730 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
731 ddi_taskq_destroy(ixgbe->sfp_taskq);
732 }
733
734 /*
735 * Remove taskq for over-temp
736 */
737 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
738 ddi_taskq_destroy(ixgbe->overtemp_taskq);
739 }
740
741 /*
742 * Remove interrupts
743 */
744 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
745 ixgbe_rem_intrs(ixgbe);
746 }
747
748 /*
749 * Unregister interrupt callback handler
750 */
751 (void) ddi_cb_unregister(ixgbe->cb_hdl);
752
753 /*
754 * Remove driver properties
755 */
756 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
757 (void) ddi_prop_remove_all(devinfo);
758 }
759
760 /*
761 * Stop the chipset
762 */
763 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
764 mutex_enter(&ixgbe->gen_lock);
765 ixgbe_chip_stop(ixgbe);
766 mutex_exit(&ixgbe->gen_lock);
767 }
768
769 /*
770 * Free register handle
771 */
772 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
773 if (ixgbe->osdep.reg_handle != NULL)
774 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
775 }
776
777 /*
778 * Free PCI config handle
779 */
780 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
781 if (ixgbe->osdep.cfg_handle != NULL)
782 pci_config_teardown(&ixgbe->osdep.cfg_handle);
783 }
784
785 /*
786 * Free locks
787 */
788 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
789 ixgbe_destroy_locks(ixgbe);
790 }
791
792 /*
793 * Free the rx/tx rings
794 */
795 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
796 ixgbe_free_rings(ixgbe);
797 }
798
799 /*
800 * Unregister FMA capabilities
801 */
802 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
803 ixgbe_fm_fini(ixgbe);
804 }
805
806 /*
807 * Free the driver data structure
808 */
809 kmem_free(ixgbe, sizeof (ixgbe_t));
810
811 ddi_set_driver_private(devinfo, NULL);
812 }
813
814 /*
815 * ixgbe_register_mac - Register the driver and its function pointers with
816 * the GLD interface.
817 */
818 static int
819 ixgbe_register_mac(ixgbe_t *ixgbe)
820 {
821 struct ixgbe_hw *hw = &ixgbe->hw;
822 mac_register_t *mac;
823 int status;
824
825 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
826 return (IXGBE_FAILURE);
827
828 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
829 mac->m_driver = ixgbe;
830 mac->m_dip = ixgbe->dip;
831 mac->m_src_addr = hw->mac.addr;
832 mac->m_callbacks = &ixgbe_m_callbacks;
833 mac->m_min_sdu = 0;
834 mac->m_max_sdu = ixgbe->default_mtu;
835 mac->m_margin = VLAN_TAGSZ;
836 mac->m_priv_props = ixgbe_priv_props;
837 mac->m_v12n = MAC_VIRT_LEVEL1;
838
839 status = mac_register(mac, &ixgbe->mac_hdl);
840
841 mac_free(mac);
842
843 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
844 }
845
846 /*
847 * ixgbe_identify_hardware - Identify the type of the chipset.
848 */
849 static int
850 ixgbe_identify_hardware(ixgbe_t *ixgbe)
851 {
852 struct ixgbe_hw *hw = &ixgbe->hw;
853 struct ixgbe_osdep *osdep = &ixgbe->osdep;
854
855 /*
856 * Get the device id
857 */
858 hw->vendor_id =
859 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
860 hw->device_id =
861 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
862 hw->revision_id =
863 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
864 hw->subsystem_device_id =
865 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
866 hw->subsystem_vendor_id =
867 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
868
869 /*
870 * Set the mac type of the adapter based on the device id
871 */
872 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
873 return (IXGBE_FAILURE);
874 }
875
876 /*
877 * Install adapter capabilities
878 */
879 switch (hw->mac.type) {
880 case ixgbe_mac_82598EB:
881 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
882 ixgbe->capab = &ixgbe_82598eb_cap;
883
884 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
885 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
886 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
887 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
888 }
889 break;
890
891 case ixgbe_mac_82599EB:
892 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
893 ixgbe->capab = &ixgbe_82599eb_cap;
894
895 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
896 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
897 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
898 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
899 }
900 break;
901
902 case ixgbe_mac_X540:
903 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
904 ixgbe->capab = &ixgbe_X540_cap;
905 /*
906 * For now, X540 is all set in its capab structure.
907 * As other X540 variants show up, things can change here.
908 */
909 break;
910
911 default:
912 IXGBE_DEBUGLOG_1(ixgbe,
913 "adapter not supported in ixgbe_identify_hardware(): %d\n",
914 hw->mac.type);
915 return (IXGBE_FAILURE);
916 }
917
918 return (IXGBE_SUCCESS);
919 }
920
921 /*
922 * ixgbe_regs_map - Map the device registers.
923 *
924 */
925 static int
926 ixgbe_regs_map(ixgbe_t *ixgbe)
927 {
928 dev_info_t *devinfo = ixgbe->dip;
929 struct ixgbe_hw *hw = &ixgbe->hw;
930 struct ixgbe_osdep *osdep = &ixgbe->osdep;
931 off_t mem_size;
932
933 /*
934 * First get the size of device registers to be mapped.
935 */
936 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
937 != DDI_SUCCESS) {
938 return (IXGBE_FAILURE);
939 }
940
941 /*
942 * Call ddi_regs_map_setup() to map registers
943 */
944 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
945 (caddr_t *)&hw->hw_addr, 0,
946 mem_size, &ixgbe_regs_acc_attr,
947 &osdep->reg_handle)) != DDI_SUCCESS) {
948 return (IXGBE_FAILURE);
949 }
950
951 return (IXGBE_SUCCESS);
952 }
953
954 /*
955 * ixgbe_init_properties - Initialize driver properties.
956 */
957 static void
958 ixgbe_init_properties(ixgbe_t *ixgbe)
959 {
960 /*
961 * Get conf file properties, including link settings
962 * jumbo frames, ring number, descriptor number, etc.
963 */
964 ixgbe_get_conf(ixgbe);
965
966 ixgbe_init_params(ixgbe);
967 }
968
969 /*
970 * ixgbe_init_driver_settings - Initialize driver settings.
971 *
972 * The settings include hardware function pointers, bus information,
973 * rx/tx rings settings, link state, and any other parameters that
974 * need to be setup during driver initialization.
975 */
976 static int
977 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
978 {
979 struct ixgbe_hw *hw = &ixgbe->hw;
980 dev_info_t *devinfo = ixgbe->dip;
981 ixgbe_rx_ring_t *rx_ring;
982 ixgbe_rx_group_t *rx_group;
983 ixgbe_tx_ring_t *tx_ring;
984 uint32_t rx_size;
985 uint32_t tx_size;
986 uint32_t ring_per_group;
987 int i;
988
989 /*
990 * Initialize chipset specific hardware function pointers
991 */
992 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
993 return (IXGBE_FAILURE);
994 }
995
996 /*
997 * Get the system page size
998 */
999 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1000
1001 /*
1002 * Set rx buffer size
1003 *
1004 * The IP header alignment room is counted in the calculation.
1005 * The rx buffer size is in unit of 1K that is required by the
1006 * chipset hardware.
1007 */
1008 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1009 ixgbe->rx_buf_size = ((rx_size >> 10) +
1010 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1011
1012 /*
1013 * Set tx buffer size
1014 */
1015 tx_size = ixgbe->max_frame_size;
1016 ixgbe->tx_buf_size = ((tx_size >> 10) +
1017 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1018
1019 /*
1020 * Initialize rx/tx rings/groups parameters
1021 */
1022 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1023 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1024 rx_ring = &ixgbe->rx_rings[i];
1025 rx_ring->index = i;
1026 rx_ring->ixgbe = ixgbe;
1027 rx_ring->group_index = i / ring_per_group;
1028 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1029 }
1030
1031 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1032 rx_group = &ixgbe->rx_groups[i];
1033 rx_group->index = i;
1034 rx_group->ixgbe = ixgbe;
1035 }
1036
1037 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1038 tx_ring = &ixgbe->tx_rings[i];
1039 tx_ring->index = i;
1040 tx_ring->ixgbe = ixgbe;
1041 if (ixgbe->tx_head_wb_enable)
1042 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1043 else
1044 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1045
1046 tx_ring->ring_size = ixgbe->tx_ring_size;
1047 tx_ring->free_list_size = ixgbe->tx_ring_size +
1048 (ixgbe->tx_ring_size >> 1);
1049 }
1050
1051 /*
1052 * Initialize values of interrupt throttling rate
1053 */
1054 for (i = 1; i < MAX_INTR_VECTOR; i++)
1055 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1056
1057 /*
1058 * The initial link state should be "unknown"
1059 */
1060 ixgbe->link_state = LINK_STATE_UNKNOWN;
1061
1062 return (IXGBE_SUCCESS);
1063 }
1064
1065 /*
1066 * ixgbe_init_locks - Initialize locks.
1067 */
1068 static void
1069 ixgbe_init_locks(ixgbe_t *ixgbe)
1070 {
1071 ixgbe_rx_ring_t *rx_ring;
1072 ixgbe_tx_ring_t *tx_ring;
1073 int i;
1074
1075 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1076 rx_ring = &ixgbe->rx_rings[i];
1077 mutex_init(&rx_ring->rx_lock, NULL,
1078 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1079 }
1080
1081 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1082 tx_ring = &ixgbe->tx_rings[i];
1083 mutex_init(&tx_ring->tx_lock, NULL,
1084 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1085 mutex_init(&tx_ring->recycle_lock, NULL,
1086 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1087 mutex_init(&tx_ring->tcb_head_lock, NULL,
1088 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1089 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1090 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1091 }
1092
1093 mutex_init(&ixgbe->gen_lock, NULL,
1094 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1095
1096 mutex_init(&ixgbe->watchdog_lock, NULL,
1097 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1098 }
1099
1100 /*
1101 * ixgbe_destroy_locks - Destroy locks.
1102 */
1103 static void
1104 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1105 {
1106 ixgbe_rx_ring_t *rx_ring;
1107 ixgbe_tx_ring_t *tx_ring;
1108 int i;
1109
1110 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1111 rx_ring = &ixgbe->rx_rings[i];
1112 mutex_destroy(&rx_ring->rx_lock);
1113 }
1114
1115 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1116 tx_ring = &ixgbe->tx_rings[i];
1117 mutex_destroy(&tx_ring->tx_lock);
1118 mutex_destroy(&tx_ring->recycle_lock);
1119 mutex_destroy(&tx_ring->tcb_head_lock);
1120 mutex_destroy(&tx_ring->tcb_tail_lock);
1121 }
1122
1123 mutex_destroy(&ixgbe->gen_lock);
1124 mutex_destroy(&ixgbe->watchdog_lock);
1125 }
1126
1127 static int
1128 ixgbe_resume(dev_info_t *devinfo)
1129 {
1130 ixgbe_t *ixgbe;
1131 int i;
1132
1133 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1134 if (ixgbe == NULL)
1135 return (DDI_FAILURE);
1136
1137 mutex_enter(&ixgbe->gen_lock);
1138
1139 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1140 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1141 mutex_exit(&ixgbe->gen_lock);
1142 return (DDI_FAILURE);
1143 }
1144
1145 /*
1146 * Enable and start the watchdog timer
1147 */
1148 ixgbe_enable_watchdog_timer(ixgbe);
1149 }
1150
1151 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1152
1153 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1154 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1155 mac_tx_ring_update(ixgbe->mac_hdl,
1156 ixgbe->tx_rings[i].ring_handle);
1157 }
1158 }
1159
1160 mutex_exit(&ixgbe->gen_lock);
1161
1162 return (DDI_SUCCESS);
1163 }
1164
1165 static int
1166 ixgbe_suspend(dev_info_t *devinfo)
1167 {
1168 ixgbe_t *ixgbe;
1169
1170 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1171 if (ixgbe == NULL)
1172 return (DDI_FAILURE);
1173
1174 mutex_enter(&ixgbe->gen_lock);
1175
1176 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1177 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1178 mutex_exit(&ixgbe->gen_lock);
1179 return (DDI_SUCCESS);
1180 }
1181 ixgbe_stop(ixgbe, B_FALSE);
1182
1183 mutex_exit(&ixgbe->gen_lock);
1184
1185 /*
1186 * Disable and stop the watchdog timer
1187 */
1188 ixgbe_disable_watchdog_timer(ixgbe);
1189
1190 return (DDI_SUCCESS);
1191 }
1192
1193 /*
1194 * ixgbe_init - Initialize the device.
1195 */
1196 static int
1197 ixgbe_init(ixgbe_t *ixgbe)
1198 {
1199 struct ixgbe_hw *hw = &ixgbe->hw;
1200
1201 mutex_enter(&ixgbe->gen_lock);
1202
1203 /*
1204 * Reset chipset to put the hardware in a known state
1205 * before we try to do anything with the eeprom.
1206 */
1207 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1208 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1209 goto init_fail;
1210 }
1211
1212 /*
1213 * Need to init eeprom before validating the checksum.
1214 */
1215 if (ixgbe_init_eeprom_params(hw) < 0) {
1216 ixgbe_error(ixgbe,
1217 "Unable to intitialize the eeprom interface.");
1218 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1219 goto init_fail;
1220 }
1221
1222 /*
1223 * NVM validation
1224 */
1225 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1226 /*
1227 * Some PCI-E parts fail the first check due to
1228 * the link being in sleep state. Call it again,
1229 * if it fails a second time it's a real issue.
1230 */
1231 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1232 ixgbe_error(ixgbe,
1233 "Invalid NVM checksum. Please contact "
1234 "the vendor to update the NVM.");
1235 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1236 goto init_fail;
1237 }
1238 }
1239
1240 /*
1241 * Setup default flow control thresholds - enable/disable
1242 * & flow control type is controlled by ixgbe.conf
1243 */
1244 hw->fc.high_water[0] = DEFAULT_FCRTH;
1245 hw->fc.low_water[0] = DEFAULT_FCRTL;
1246 hw->fc.pause_time = DEFAULT_FCPAUSE;
1247 hw->fc.send_xon = B_TRUE;
1248
1249 /*
1250 * Initialize link settings
1251 */
1252 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1253
1254 /*
1255 * Initialize the chipset hardware
1256 */
1257 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1258 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1259 goto init_fail;
1260 }
1261
1262 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1263 goto init_fail;
1264 }
1265
1266 mutex_exit(&ixgbe->gen_lock);
1267 return (IXGBE_SUCCESS);
1268
1269 init_fail:
1270 /*
1271 * Reset PHY
1272 */
1273 (void) ixgbe_reset_phy(hw);
1274
1275 mutex_exit(&ixgbe->gen_lock);
1276 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1277 return (IXGBE_FAILURE);
1278 }
1279
1280 /*
1281 * ixgbe_chip_start - Initialize and start the chipset hardware.
1282 */
1283 static int
1284 ixgbe_chip_start(ixgbe_t *ixgbe)
1285 {
1286 struct ixgbe_hw *hw = &ixgbe->hw;
1287 int ret_val, i;
1288
1289 ASSERT(mutex_owned(&ixgbe->gen_lock));
1290
1291 /*
1292 * Get the mac address
1293 * This function should handle SPARC case correctly.
1294 */
1295 if (!ixgbe_find_mac_address(ixgbe)) {
1296 ixgbe_error(ixgbe, "Failed to get the mac address");
1297 return (IXGBE_FAILURE);
1298 }
1299
1300 /*
1301 * Validate the mac address
1302 */
1303 (void) ixgbe_init_rx_addrs(hw);
1304 if (!is_valid_mac_addr(hw->mac.addr)) {
1305 ixgbe_error(ixgbe, "Invalid mac address");
1306 return (IXGBE_FAILURE);
1307 }
1308
1309 /*
1310 * Configure/Initialize hardware
1311 */
1312 ret_val = ixgbe_init_hw(hw);
1313 if (ret_val != IXGBE_SUCCESS) {
1314 if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1315 ixgbe_error(ixgbe,
1316 "This 82599 device is pre-release and contains"
1317 " outdated firmware, please contact your hardware"
1318 " vendor for a replacement.");
1319 } else {
1320 ixgbe_error(ixgbe, "Failed to initialize hardware");
1321 return (IXGBE_FAILURE);
1322 }
1323 }
1324
1325 /*
1326 * Re-enable relaxed ordering for performance. It is disabled
1327 * by default in the hardware init.
1328 */
1329 if (ixgbe->relax_order_enable == B_TRUE)
1330 ixgbe_enable_relaxed_ordering(hw);
1331
1332 /*
1333 * Setup adapter interrupt vectors
1334 */
1335 ixgbe_setup_adapter_vector(ixgbe);
1336
1337 /*
1338 * Initialize unicast addresses.
1339 */
1340 ixgbe_init_unicst(ixgbe);
1341
1342 /*
1343 * Setup and initialize the mctable structures.
1344 */
1345 ixgbe_setup_multicst(ixgbe);
1346
1347 /*
1348 * Set interrupt throttling rate
1349 */
1350 for (i = 0; i < ixgbe->intr_cnt; i++) {
1351 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1352 }
1353
1354 /*
1355 * Save the state of the phy
1356 */
1357 ixgbe_get_hw_state(ixgbe);
1358
1359 /*
1360 * Make sure driver has control
1361 */
1362 ixgbe_get_driver_control(hw);
1363
1364 return (IXGBE_SUCCESS);
1365 }
1366
1367 /*
1368 * ixgbe_chip_stop - Stop the chipset hardware
1369 */
1370 static void
1371 ixgbe_chip_stop(ixgbe_t *ixgbe)
1372 {
1373 struct ixgbe_hw *hw = &ixgbe->hw;
1374
1375 ASSERT(mutex_owned(&ixgbe->gen_lock));
1376
1377 /*
1378 * Tell firmware driver is no longer in control
1379 */
1380 ixgbe_release_driver_control(hw);
1381
1382 /*
1383 * Reset the chipset
1384 */
1385 (void) ixgbe_reset_hw(hw);
1386
1387 /*
1388 * Reset PHY
1389 */
1390 (void) ixgbe_reset_phy(hw);
1391 }
1392
1393 /*
1394 * ixgbe_reset - Reset the chipset and re-start the driver.
1395 *
1396 * It involves stopping and re-starting the chipset,
1397 * and re-configuring the rx/tx rings.
1398 */
1399 static int
1400 ixgbe_reset(ixgbe_t *ixgbe)
1401 {
1402 int i;
1403
1404 /*
1405 * Disable and stop the watchdog timer
1406 */
1407 ixgbe_disable_watchdog_timer(ixgbe);
1408
1409 mutex_enter(&ixgbe->gen_lock);
1410
1411 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1412 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1413
1414 ixgbe_stop(ixgbe, B_FALSE);
1415
1416 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1417 mutex_exit(&ixgbe->gen_lock);
1418 return (IXGBE_FAILURE);
1419 }
1420
1421 /*
1422 * After resetting, need to recheck the link status.
1423 */
1424 ixgbe->link_check_complete = B_FALSE;
1425 ixgbe->link_check_hrtime = gethrtime() +
1426 (IXGBE_LINK_UP_TIME * 100000000ULL);
1427
1428 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1429
1430 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1431 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1432 mac_tx_ring_update(ixgbe->mac_hdl,
1433 ixgbe->tx_rings[i].ring_handle);
1434 }
1435 }
1436
1437 mutex_exit(&ixgbe->gen_lock);
1438
1439 /*
1440 * Enable and start the watchdog timer
1441 */
1442 ixgbe_enable_watchdog_timer(ixgbe);
1443
1444 return (IXGBE_SUCCESS);
1445 }
1446
1447 /*
1448 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1449 */
1450 static void
1451 ixgbe_tx_clean(ixgbe_t *ixgbe)
1452 {
1453 ixgbe_tx_ring_t *tx_ring;
1454 tx_control_block_t *tcb;
1455 link_list_t pending_list;
1456 uint32_t desc_num;
1457 int i, j;
1458
1459 LINK_LIST_INIT(&pending_list);
1460
1461 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1462 tx_ring = &ixgbe->tx_rings[i];
1463
1464 mutex_enter(&tx_ring->recycle_lock);
1465
1466 /*
1467 * Clean the pending tx data - the pending packets in the
1468 * work_list that have no chances to be transmitted again.
1469 *
1470 * We must ensure the chipset is stopped or the link is down
1471 * before cleaning the transmit packets.
1472 */
1473 desc_num = 0;
1474 for (j = 0; j < tx_ring->ring_size; j++) {
1475 tcb = tx_ring->work_list[j];
1476 if (tcb != NULL) {
1477 desc_num += tcb->desc_num;
1478
1479 tx_ring->work_list[j] = NULL;
1480
1481 ixgbe_free_tcb(tcb);
1482
1483 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1484 }
1485 }
1486
1487 if (desc_num > 0) {
1488 atomic_add_32(&tx_ring->tbd_free, desc_num);
1489 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1490
1491 /*
1492 * Reset the head and tail pointers of the tbd ring;
1493 * Reset the writeback head if it's enable.
1494 */
1495 tx_ring->tbd_head = 0;
1496 tx_ring->tbd_tail = 0;
1497 if (ixgbe->tx_head_wb_enable)
1498 *tx_ring->tbd_head_wb = 0;
1499
1500 IXGBE_WRITE_REG(&ixgbe->hw,
1501 IXGBE_TDH(tx_ring->index), 0);
1502 IXGBE_WRITE_REG(&ixgbe->hw,
1503 IXGBE_TDT(tx_ring->index), 0);
1504 }
1505
1506 mutex_exit(&tx_ring->recycle_lock);
1507
1508 /*
1509 * Add the tx control blocks in the pending list to
1510 * the free list.
1511 */
1512 ixgbe_put_free_list(tx_ring, &pending_list);
1513 }
1514 }
1515
1516 /*
1517 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1518 * transmitted.
1519 */
1520 static boolean_t
1521 ixgbe_tx_drain(ixgbe_t *ixgbe)
1522 {
1523 ixgbe_tx_ring_t *tx_ring;
1524 boolean_t done;
1525 int i, j;
1526
1527 /*
1528 * Wait for a specific time to allow pending tx packets
1529 * to be transmitted.
1530 *
1531 * Check the counter tbd_free to see if transmission is done.
1532 * No lock protection is needed here.
1533 *
1534 * Return B_TRUE if all pending packets have been transmitted;
1535 * Otherwise return B_FALSE;
1536 */
1537 for (i = 0; i < TX_DRAIN_TIME; i++) {
1538
1539 done = B_TRUE;
1540 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1541 tx_ring = &ixgbe->tx_rings[j];
1542 done = done &&
1543 (tx_ring->tbd_free == tx_ring->ring_size);
1544 }
1545
1546 if (done)
1547 break;
1548
1549 msec_delay(1);
1550 }
1551
1552 return (done);
1553 }
1554
1555 /*
1556 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1557 */
1558 static boolean_t
1559 ixgbe_rx_drain(ixgbe_t *ixgbe)
1560 {
1561 boolean_t done = B_TRUE;
1562 int i;
1563
1564 /*
1565 * Polling the rx free list to check if those rx buffers held by
1566 * the upper layer are released.
1567 *
1568 * Check the counter rcb_free to see if all pending buffers are
1569 * released. No lock protection is needed here.
1570 *
1571 * Return B_TRUE if all pending buffers have been released;
1572 * Otherwise return B_FALSE;
1573 */
1574 for (i = 0; i < RX_DRAIN_TIME; i++) {
1575 done = (ixgbe->rcb_pending == 0);
1576
1577 if (done)
1578 break;
1579
1580 msec_delay(1);
1581 }
1582
1583 return (done);
1584 }
1585
1586 /*
1587 * ixgbe_start - Start the driver/chipset.
1588 */
1589 int
1590 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1591 {
1592 int i;
1593
1594 ASSERT(mutex_owned(&ixgbe->gen_lock));
1595
1596 if (alloc_buffer) {
1597 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1598 ixgbe_error(ixgbe,
1599 "Failed to allocate software receive rings");
1600 return (IXGBE_FAILURE);
1601 }
1602
1603 /* Allocate buffers for all the rx/tx rings */
1604 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1605 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1606 return (IXGBE_FAILURE);
1607 }
1608
1609 ixgbe->tx_ring_init = B_TRUE;
1610 } else {
1611 ixgbe->tx_ring_init = B_FALSE;
1612 }
1613
1614 for (i = 0; i < ixgbe->num_rx_rings; i++)
1615 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1616 for (i = 0; i < ixgbe->num_tx_rings; i++)
1617 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1618
1619 /*
1620 * Start the chipset hardware
1621 */
1622 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1623 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1624 goto start_failure;
1625 }
1626
1627 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1628 goto start_failure;
1629 }
1630
1631 /*
1632 * Setup the rx/tx rings
1633 */
1634 ixgbe_setup_rings(ixgbe);
1635
1636 /*
1637 * ixgbe_start() will be called when resetting, however if reset
1638 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1639 * before enabling the interrupts.
1640 */
1641 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1642 | IXGBE_STALL| IXGBE_OVERTEMP));
1643
1644 /*
1645 * Enable adapter interrupts
1646 * The interrupts must be enabled after the driver state is START
1647 */
1648 ixgbe_enable_adapter_interrupts(ixgbe);
1649
1650 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1651 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1652 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1653 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1654
1655 return (IXGBE_SUCCESS);
1656
1657 start_failure:
1658 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1659 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1660 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1661 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1662
1663 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1664
1665 return (IXGBE_FAILURE);
1666 }
1667
1668 /*
1669 * ixgbe_stop - Stop the driver/chipset.
1670 */
1671 void
1672 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1673 {
1674 int i;
1675
1676 ASSERT(mutex_owned(&ixgbe->gen_lock));
1677
1678 /*
1679 * Disable the adapter interrupts
1680 */
1681 ixgbe_disable_adapter_interrupts(ixgbe);
1682
1683 /*
1684 * Drain the pending tx packets
1685 */
1686 (void) ixgbe_tx_drain(ixgbe);
1687
1688 for (i = 0; i < ixgbe->num_rx_rings; i++)
1689 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1690 for (i = 0; i < ixgbe->num_tx_rings; i++)
1691 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1692
1693 /*
1694 * Stop the chipset hardware
1695 */
1696 ixgbe_chip_stop(ixgbe);
1697
1698 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1699 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1700 }
1701
1702 /*
1703 * Clean the pending tx data/resources
1704 */
1705 ixgbe_tx_clean(ixgbe);
1706
1707 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1708 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1709 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1710 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1711
1712 if (ixgbe->link_state == LINK_STATE_UP) {
1713 ixgbe->link_state = LINK_STATE_UNKNOWN;
1714 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1715 }
1716
1717 if (free_buffer) {
1718 /*
1719 * Release the DMA/memory resources of rx/tx rings
1720 */
1721 ixgbe_free_dma(ixgbe);
1722 ixgbe_free_rx_data(ixgbe);
1723 }
1724 }
1725
1726 /*
1727 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1728 */
1729 /* ARGSUSED */
1730 static int
1731 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1732 void *arg1, void *arg2)
1733 {
1734 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1735
1736 switch (cbaction) {
1737 /* IRM callback */
1738 int count;
1739 case DDI_CB_INTR_ADD:
1740 case DDI_CB_INTR_REMOVE:
1741 count = (int)(uintptr_t)cbarg;
1742 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1743 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1744 int, ixgbe->intr_cnt);
1745 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1746 DDI_SUCCESS) {
1747 ixgbe_error(ixgbe,
1748 "IRM CB: Failed to adjust interrupts");
1749 goto cb_fail;
1750 }
1751 break;
1752 default:
1753 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1754 cbaction);
1755 return (DDI_ENOTSUP);
1756 }
1757 return (DDI_SUCCESS);
1758 cb_fail:
1759 return (DDI_FAILURE);
1760 }
1761
1762 /*
1763 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1764 */
1765 static int
1766 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1767 {
1768 int i, rc, actual;
1769
1770 if (count == 0)
1771 return (DDI_SUCCESS);
1772
1773 if ((cbaction == DDI_CB_INTR_ADD &&
1774 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1775 (cbaction == DDI_CB_INTR_REMOVE &&
1776 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
1777 return (DDI_FAILURE);
1778
1779 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1780 return (DDI_FAILURE);
1781 }
1782
1783 for (i = 0; i < ixgbe->num_rx_rings; i++)
1784 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
1785 for (i = 0; i < ixgbe->num_tx_rings; i++)
1786 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
1787
1788 mutex_enter(&ixgbe->gen_lock);
1789 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1790 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
1791 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
1792 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
1793
1794 ixgbe_stop(ixgbe, B_FALSE);
1795 /*
1796 * Disable interrupts
1797 */
1798 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1799 rc = ixgbe_disable_intrs(ixgbe);
1800 ASSERT(rc == IXGBE_SUCCESS);
1801 }
1802 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
1803
1804 /*
1805 * Remove interrupt handlers
1806 */
1807 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
1808 ixgbe_rem_intr_handlers(ixgbe);
1809 }
1810 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
1811
1812 /*
1813 * Clear vect_map
1814 */
1815 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
1816 switch (cbaction) {
1817 case DDI_CB_INTR_ADD:
1818 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
1819 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
1820 DDI_INTR_ALLOC_NORMAL);
1821 if (rc != DDI_SUCCESS || actual != count) {
1822 ixgbe_log(ixgbe, "Adjust interrupts failed."
1823 "return: %d, irm cb size: %d, actual: %d",
1824 rc, count, actual);
1825 goto intr_adjust_fail;
1826 }
1827 ixgbe->intr_cnt += count;
1828 break;
1829
1830 case DDI_CB_INTR_REMOVE:
1831 for (i = ixgbe->intr_cnt - count;
1832 i < ixgbe->intr_cnt; i ++) {
1833 rc = ddi_intr_free(ixgbe->htable[i]);
1834 ixgbe->htable[i] = NULL;
1835 if (rc != DDI_SUCCESS) {
1836 ixgbe_log(ixgbe, "Adjust interrupts failed."
1837 "return: %d, irm cb size: %d, actual: %d",
1838 rc, count, actual);
1839 goto intr_adjust_fail;
1840 }
1841 }
1842 ixgbe->intr_cnt -= count;
1843 break;
1844 }
1845
1846 /*
1847 * Get priority for first vector, assume remaining are all the same
1848 */
1849 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
1850 if (rc != DDI_SUCCESS) {
1851 ixgbe_log(ixgbe,
1852 "Get interrupt priority failed: %d", rc);
1853 goto intr_adjust_fail;
1854 }
1855 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
1856 if (rc != DDI_SUCCESS) {
1857 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
1858 goto intr_adjust_fail;
1859 }
1860 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
1861
1862 /*
1863 * Map rings to interrupt vectors
1864 */
1865 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
1866 ixgbe_error(ixgbe,
1867 "IRM CB: Failed to map interrupts to vectors");
1868 goto intr_adjust_fail;
1869 }
1870
1871 /*
1872 * Add interrupt handlers
1873 */
1874 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
1875 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
1876 goto intr_adjust_fail;
1877 }
1878 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
1879
1880 /*
1881 * Now that mutex locks are initialized, and the chip is also
1882 * initialized, enable interrupts.
1883 */
1884 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
1885 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
1886 goto intr_adjust_fail;
1887 }
1888 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
1889 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1890 ixgbe_error(ixgbe, "IRM CB: Failed to start");
1891 goto intr_adjust_fail;
1892 }
1893 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
1894 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
1895 ixgbe->ixgbe_state |= IXGBE_STARTED;
1896 mutex_exit(&ixgbe->gen_lock);
1897
1898 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1899 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
1900 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
1901 }
1902 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1903 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
1904 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
1905 }
1906
1907 /* Wakeup all Tx rings */
1908 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1909 mac_tx_ring_update(ixgbe->mac_hdl,
1910 ixgbe->tx_rings[i].ring_handle);
1911 }
1912
1913 IXGBE_DEBUGLOG_3(ixgbe,
1914 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
1915 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
1916 return (DDI_SUCCESS);
1917
1918 intr_adjust_fail:
1919 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1920 mutex_exit(&ixgbe->gen_lock);
1921 return (DDI_FAILURE);
1922 }
1923
1924 /*
1925 * ixgbe_intr_cb_register - Register interrupt callback function.
1926 */
1927 static int
1928 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
1929 {
1930 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
1931 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
1932 return (IXGBE_FAILURE);
1933 }
1934 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
1935 return (IXGBE_SUCCESS);
1936 }
1937
1938 /*
1939 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
1940 */
1941 static int
1942 ixgbe_alloc_rings(ixgbe_t *ixgbe)
1943 {
1944 /*
1945 * Allocate memory space for rx rings
1946 */
1947 ixgbe->rx_rings = kmem_zalloc(
1948 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
1949 KM_NOSLEEP);
1950
1951 if (ixgbe->rx_rings == NULL) {
1952 return (IXGBE_FAILURE);
1953 }
1954
1955 /*
1956 * Allocate memory space for tx rings
1957 */
1958 ixgbe->tx_rings = kmem_zalloc(
1959 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
1960 KM_NOSLEEP);
1961
1962 if (ixgbe->tx_rings == NULL) {
1963 kmem_free(ixgbe->rx_rings,
1964 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1965 ixgbe->rx_rings = NULL;
1966 return (IXGBE_FAILURE);
1967 }
1968
1969 /*
1970 * Allocate memory space for rx ring groups
1971 */
1972 ixgbe->rx_groups = kmem_zalloc(
1973 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
1974 KM_NOSLEEP);
1975
1976 if (ixgbe->rx_groups == NULL) {
1977 kmem_free(ixgbe->rx_rings,
1978 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1979 kmem_free(ixgbe->tx_rings,
1980 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
1981 ixgbe->rx_rings = NULL;
1982 ixgbe->tx_rings = NULL;
1983 return (IXGBE_FAILURE);
1984 }
1985
1986 return (IXGBE_SUCCESS);
1987 }
1988
1989 /*
1990 * ixgbe_free_rings - Free the memory space of rx/tx rings.
1991 */
1992 static void
1993 ixgbe_free_rings(ixgbe_t *ixgbe)
1994 {
1995 if (ixgbe->rx_rings != NULL) {
1996 kmem_free(ixgbe->rx_rings,
1997 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1998 ixgbe->rx_rings = NULL;
1999 }
2000
2001 if (ixgbe->tx_rings != NULL) {
2002 kmem_free(ixgbe->tx_rings,
2003 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2004 ixgbe->tx_rings = NULL;
2005 }
2006
2007 if (ixgbe->rx_groups != NULL) {
2008 kmem_free(ixgbe->rx_groups,
2009 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2010 ixgbe->rx_groups = NULL;
2011 }
2012 }
2013
2014 static int
2015 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2016 {
2017 ixgbe_rx_ring_t *rx_ring;
2018 int i;
2019
2020 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2021 rx_ring = &ixgbe->rx_rings[i];
2022 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2023 goto alloc_rx_rings_failure;
2024 }
2025 return (IXGBE_SUCCESS);
2026
2027 alloc_rx_rings_failure:
2028 ixgbe_free_rx_data(ixgbe);
2029 return (IXGBE_FAILURE);
2030 }
2031
2032 static void
2033 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2034 {
2035 ixgbe_rx_ring_t *rx_ring;
2036 ixgbe_rx_data_t *rx_data;
2037 int i;
2038
2039 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2040 rx_ring = &ixgbe->rx_rings[i];
2041
2042 mutex_enter(&ixgbe->rx_pending_lock);
2043 rx_data = rx_ring->rx_data;
2044
2045 if (rx_data != NULL) {
2046 rx_data->flag |= IXGBE_RX_STOPPED;
2047
2048 if (rx_data->rcb_pending == 0) {
2049 ixgbe_free_rx_ring_data(rx_data);
2050 rx_ring->rx_data = NULL;
2051 }
2052 }
2053
2054 mutex_exit(&ixgbe->rx_pending_lock);
2055 }
2056 }
2057
2058 /*
2059 * ixgbe_setup_rings - Setup rx/tx rings.
2060 */
2061 static void
2062 ixgbe_setup_rings(ixgbe_t *ixgbe)
2063 {
2064 /*
2065 * Setup the rx/tx rings, including the following:
2066 *
2067 * 1. Setup the descriptor ring and the control block buffers;
2068 * 2. Initialize necessary registers for receive/transmit;
2069 * 3. Initialize software pointers/parameters for receive/transmit;
2070 */
2071 ixgbe_setup_rx(ixgbe);
2072
2073 ixgbe_setup_tx(ixgbe);
2074 }
2075
2076 static void
2077 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2078 {
2079 ixgbe_t *ixgbe = rx_ring->ixgbe;
2080 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2081 struct ixgbe_hw *hw = &ixgbe->hw;
2082 rx_control_block_t *rcb;
2083 union ixgbe_adv_rx_desc *rbd;
2084 uint32_t size;
2085 uint32_t buf_low;
2086 uint32_t buf_high;
2087 uint32_t reg_val;
2088 int i;
2089
2090 ASSERT(mutex_owned(&rx_ring->rx_lock));
2091 ASSERT(mutex_owned(&ixgbe->gen_lock));
2092
2093 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2094 rcb = rx_data->work_list[i];
2095 rbd = &rx_data->rbd_ring[i];
2096
2097 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2098 rbd->read.hdr_addr = NULL;
2099 }
2100
2101 /*
2102 * Initialize the length register
2103 */
2104 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2105 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2106
2107 /*
2108 * Initialize the base address registers
2109 */
2110 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2111 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2112 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2113 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2114
2115 /*
2116 * Setup head & tail pointers
2117 */
2118 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2119 rx_data->ring_size - 1);
2120 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2121
2122 rx_data->rbd_next = 0;
2123 rx_data->lro_first = 0;
2124
2125 /*
2126 * Setup the Receive Descriptor Control Register (RXDCTL)
2127 * PTHRESH=32 descriptors (half the internal cache)
2128 * HTHRESH=0 descriptors (to minimize latency on fetch)
2129 * WTHRESH defaults to 1 (writeback each descriptor)
2130 */
2131 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2132 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2133
2134 /* Not a valid value for 82599 or X540 */
2135 if (hw->mac.type == ixgbe_mac_82598EB) {
2136 reg_val |= 0x0020; /* pthresh */
2137 }
2138 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2139
2140 if (hw->mac.type == ixgbe_mac_82599EB ||
2141 hw->mac.type == ixgbe_mac_X540) {
2142 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2143 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2144 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2145 }
2146
2147 /*
2148 * Setup the Split and Replication Receive Control Register.
2149 * Set the rx buffer size and the advanced descriptor type.
2150 */
2151 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2152 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2153 reg_val |= IXGBE_SRRCTL_DROP_EN;
2154 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2155 }
2156
2157 static void
2158 ixgbe_setup_rx(ixgbe_t *ixgbe)
2159 {
2160 ixgbe_rx_ring_t *rx_ring;
2161 struct ixgbe_hw *hw = &ixgbe->hw;
2162 uint32_t reg_val;
2163 uint32_t ring_mapping;
2164 uint32_t i, index;
2165 uint32_t psrtype_rss_bit;
2166
2167 /* PSRTYPE must be configured for 82599 */
2168 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2169 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2170 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2171 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2172 reg_val |= IXGBE_PSRTYPE_L2HDR;
2173 reg_val |= 0x80000000;
2174 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2175 } else {
2176 if (ixgbe->num_rx_groups > 32) {
2177 psrtype_rss_bit = 0x20000000;
2178 } else {
2179 psrtype_rss_bit = 0x40000000;
2180 }
2181 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2182 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2183 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2184 reg_val |= IXGBE_PSRTYPE_L2HDR;
2185 reg_val |= psrtype_rss_bit;
2186 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2187 }
2188 }
2189
2190 /*
2191 * Set filter control in FCTRL to accept broadcast packets and do
2192 * not pass pause frames to host. Flow control settings are already
2193 * in this register, so preserve them.
2194 */
2195 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2196 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */
2197 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */
2198 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2199
2200 /*
2201 * Hardware checksum settings
2202 */
2203 if (ixgbe->rx_hcksum_enable) {
2204 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2205 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2206 }
2207
2208 /*
2209 * Setup VMDq and RSS for multiple receive queues
2210 */
2211 switch (ixgbe->classify_mode) {
2212 case IXGBE_CLASSIFY_RSS:
2213 /*
2214 * One group, only RSS is needed when more than
2215 * one ring enabled.
2216 */
2217 ixgbe_setup_rss(ixgbe);
2218 break;
2219
2220 case IXGBE_CLASSIFY_VMDQ:
2221 /*
2222 * Multiple groups, each group has one ring,
2223 * only VMDq is needed.
2224 */
2225 ixgbe_setup_vmdq(ixgbe);
2226 break;
2227
2228 case IXGBE_CLASSIFY_VMDQ_RSS:
2229 /*
2230 * Multiple groups and multiple rings, both
2231 * VMDq and RSS are needed.
2232 */
2233 ixgbe_setup_vmdq_rss(ixgbe);
2234 break;
2235
2236 default:
2237 break;
2238 }
2239
2240 /*
2241 * Enable the receive unit. This must be done after filter
2242 * control is set in FCTRL.
2243 */
2244 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */
2245 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */
2246 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2247
2248 /*
2249 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2250 */
2251 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2252 rx_ring = &ixgbe->rx_rings[i];
2253 ixgbe_setup_rx_ring(rx_ring);
2254 }
2255
2256 /*
2257 * Setup the per-ring statistics mapping.
2258 */
2259 ring_mapping = 0;
2260 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2261 index = ixgbe->rx_rings[i].hw_index;
2262 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2263 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2264 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2265 }
2266
2267 /*
2268 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2269 * by four bytes if the packet has a VLAN field, so includes MTU,
2270 * ethernet header and frame check sequence.
2271 * Register is MAXFRS in 82599.
2272 */
2273 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2274 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2275 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2276
2277 /*
2278 * Setup Jumbo Frame enable bit
2279 */
2280 if (ixgbe->default_mtu > ETHERMTU) {
2281 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2282 reg_val |= IXGBE_HLREG0_JUMBOEN;
2283 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2284 }
2285
2286 /*
2287 * Setup RSC for multiple receive queues.
2288 */
2289 if (ixgbe->lro_enable) {
2290 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2291 /*
2292 * Make sure rx_buf_size * MAXDESC not greater
2293 * than 65535.
2294 * Intel recommends 4 for MAXDESC field value.
2295 */
2296 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2297 reg_val |= IXGBE_RSCCTL_RSCEN;
2298 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2299 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2300 else
2301 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2302 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2303 }
2304
2305 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2306 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2307 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2308
2309 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2310 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2311 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2312 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2313
2314 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2315 }
2316 }
2317
2318 static void
2319 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2320 {
2321 ixgbe_t *ixgbe = tx_ring->ixgbe;
2322 struct ixgbe_hw *hw = &ixgbe->hw;
2323 uint32_t size;
2324 uint32_t buf_low;
2325 uint32_t buf_high;
2326 uint32_t reg_val;
2327
2328 ASSERT(mutex_owned(&tx_ring->tx_lock));
2329 ASSERT(mutex_owned(&ixgbe->gen_lock));
2330
2331 /*
2332 * Initialize the length register
2333 */
2334 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2335 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2336
2337 /*
2338 * Initialize the base address registers
2339 */
2340 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2341 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2342 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2343 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2344
2345 /*
2346 * Setup head & tail pointers
2347 */
2348 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2349 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2350
2351 /*
2352 * Setup head write-back
2353 */
2354 if (ixgbe->tx_head_wb_enable) {
2355 /*
2356 * The memory of the head write-back is allocated using
2357 * the extra tbd beyond the tail of the tbd ring.
2358 */
2359 tx_ring->tbd_head_wb = (uint32_t *)
2360 ((uintptr_t)tx_ring->tbd_area.address + size);
2361 *tx_ring->tbd_head_wb = 0;
2362
2363 buf_low = (uint32_t)
2364 (tx_ring->tbd_area.dma_address + size);
2365 buf_high = (uint32_t)
2366 ((tx_ring->tbd_area.dma_address + size) >> 32);
2367
2368 /* Set the head write-back enable bit */
2369 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2370
2371 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2372 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2373
2374 /*
2375 * Turn off relaxed ordering for head write back or it will
2376 * cause problems with the tx recycling
2377 */
2378
2379 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2380 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2381 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2382 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2383 if (hw->mac.type == ixgbe_mac_82598EB) {
2384 IXGBE_WRITE_REG(hw,
2385 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2386 } else {
2387 IXGBE_WRITE_REG(hw,
2388 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2389 }
2390 } else {
2391 tx_ring->tbd_head_wb = NULL;
2392 }
2393
2394 tx_ring->tbd_head = 0;
2395 tx_ring->tbd_tail = 0;
2396 tx_ring->tbd_free = tx_ring->ring_size;
2397
2398 if (ixgbe->tx_ring_init == B_TRUE) {
2399 tx_ring->tcb_head = 0;
2400 tx_ring->tcb_tail = 0;
2401 tx_ring->tcb_free = tx_ring->free_list_size;
2402 }
2403
2404 /*
2405 * Initialize the s/w context structure
2406 */
2407 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2408 }
2409
2410 static void
2411 ixgbe_setup_tx(ixgbe_t *ixgbe)
2412 {
2413 struct ixgbe_hw *hw = &ixgbe->hw;
2414 ixgbe_tx_ring_t *tx_ring;
2415 uint32_t reg_val;
2416 uint32_t ring_mapping;
2417 int i;
2418
2419 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2420 tx_ring = &ixgbe->tx_rings[i];
2421 ixgbe_setup_tx_ring(tx_ring);
2422 }
2423
2424 /*
2425 * Setup the per-ring statistics mapping.
2426 */
2427 ring_mapping = 0;
2428 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2429 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2430 if ((i & 0x3) == 0x3) {
2431 switch (hw->mac.type) {
2432 case ixgbe_mac_82598EB:
2433 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2434 ring_mapping);
2435 break;
2436
2437 case ixgbe_mac_82599EB:
2438 case ixgbe_mac_X540:
2439 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2440 ring_mapping);
2441 break;
2442
2443 default:
2444 break;
2445 }
2446
2447 ring_mapping = 0;
2448 }
2449 }
2450 if (i & 0x3) {
2451 switch (hw->mac.type) {
2452 case ixgbe_mac_82598EB:
2453 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2454 break;
2455
2456 case ixgbe_mac_82599EB:
2457 case ixgbe_mac_X540:
2458 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2459 break;
2460
2461 default:
2462 break;
2463 }
2464 }
2465
2466 /*
2467 * Enable CRC appending and TX padding (for short tx frames)
2468 */
2469 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2470 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2471 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2472
2473 /*
2474 * enable DMA for 82599 and X540 parts
2475 */
2476 if (hw->mac.type == ixgbe_mac_82599EB ||
2477 hw->mac.type == ixgbe_mac_X540) {
2478 /* DMATXCTL.TE must be set after all Tx config is complete */
2479 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2480 reg_val |= IXGBE_DMATXCTL_TE;
2481 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2482
2483 /* Disable arbiter to set MTQC */
2484 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2485 reg_val |= IXGBE_RTTDCS_ARBDIS;
2486 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2487 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2488 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2489 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2490 }
2491
2492 /*
2493 * Enabling tx queues ..
2494 * For 82599 must be done after DMATXCTL.TE is set
2495 */
2496 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2497 tx_ring = &ixgbe->tx_rings[i];
2498 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2499 reg_val |= IXGBE_TXDCTL_ENABLE;
2500 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2501 }
2502 }
2503
2504 /*
2505 * ixgbe_setup_rss - Setup receive-side scaling feature.
2506 */
2507 static void
2508 ixgbe_setup_rss(ixgbe_t *ixgbe)
2509 {
2510 struct ixgbe_hw *hw = &ixgbe->hw;
2511 uint32_t i, mrqc, rxcsum;
2512 uint32_t random;
2513 uint32_t reta;
2514 uint32_t ring_per_group;
2515
2516 /*
2517 * Fill out redirection table
2518 */
2519 reta = 0;
2520 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2521
2522 for (i = 0; i < 128; i++) {
2523 reta = (reta << 8) | (i % ring_per_group) |
2524 ((i % ring_per_group) << 4);
2525 if ((i & 3) == 3)
2526 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2527 }
2528
2529 /*
2530 * Fill out hash function seeds with a random constant
2531 */
2532 for (i = 0; i < 10; i++) {
2533 (void) random_get_pseudo_bytes((uint8_t *)&random,
2534 sizeof (uint32_t));
2535 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2536 }
2537
2538 /*
2539 * Enable RSS & perform hash on these packet types
2540 */
2541 mrqc = IXGBE_MRQC_RSSEN |
2542 IXGBE_MRQC_RSS_FIELD_IPV4 |
2543 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2544 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2545 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2546 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2547 IXGBE_MRQC_RSS_FIELD_IPV6 |
2548 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2549 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2550 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2551 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2552
2553 /*
2554 * Disable Packet Checksum to enable RSS for multiple receive queues.
2555 * It is an adapter hardware limitation that Packet Checksum is
2556 * mutually exclusive with RSS.
2557 */
2558 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2559 rxcsum |= IXGBE_RXCSUM_PCSD;
2560 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2561 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2562 }
2563
2564 /*
2565 * ixgbe_setup_vmdq - Setup MAC classification feature
2566 */
2567 static void
2568 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2569 {
2570 struct ixgbe_hw *hw = &ixgbe->hw;
2571 uint32_t vmdctl, i, vtctl;
2572
2573 /*
2574 * Setup the VMDq Control register, enable VMDq based on
2575 * packet destination MAC address:
2576 */
2577 switch (hw->mac.type) {
2578 case ixgbe_mac_82598EB:
2579 /*
2580 * VMDq Enable = 1;
2581 * VMDq Filter = 0; MAC filtering
2582 * Default VMDq output index = 0;
2583 */
2584 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2585 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2586 break;
2587
2588 case ixgbe_mac_82599EB:
2589 case ixgbe_mac_X540:
2590 /*
2591 * Enable VMDq-only.
2592 */
2593 vmdctl = IXGBE_MRQC_VMDQEN;
2594 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2595
2596 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2597 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2598 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2599 }
2600
2601 /*
2602 * Enable Virtualization and Replication.
2603 */
2604 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2605 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2606
2607 /*
2608 * Enable receiving packets to all VFs
2609 */
2610 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2611 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2612 break;
2613
2614 default:
2615 break;
2616 }
2617 }
2618
2619 /*
2620 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2621 */
2622 static void
2623 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2624 {
2625 struct ixgbe_hw *hw = &ixgbe->hw;
2626 uint32_t i, mrqc, rxcsum;
2627 uint32_t random;
2628 uint32_t reta;
2629 uint32_t ring_per_group;
2630 uint32_t vmdctl, vtctl;
2631
2632 /*
2633 * Fill out redirection table
2634 */
2635 reta = 0;
2636 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2637 for (i = 0; i < 128; i++) {
2638 reta = (reta << 8) | (i % ring_per_group) |
2639 ((i % ring_per_group) << 4);
2640 if ((i & 3) == 3)
2641 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2642 }
2643
2644 /*
2645 * Fill out hash function seeds with a random constant
2646 */
2647 for (i = 0; i < 10; i++) {
2648 (void) random_get_pseudo_bytes((uint8_t *)&random,
2649 sizeof (uint32_t));
2650 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2651 }
2652
2653 /*
2654 * Enable and setup RSS and VMDq
2655 */
2656 switch (hw->mac.type) {
2657 case ixgbe_mac_82598EB:
2658 /*
2659 * Enable RSS & Setup RSS Hash functions
2660 */
2661 mrqc = IXGBE_MRQC_RSSEN |
2662 IXGBE_MRQC_RSS_FIELD_IPV4 |
2663 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2664 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2665 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2666 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2667 IXGBE_MRQC_RSS_FIELD_IPV6 |
2668 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2669 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2670 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2671 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2672
2673 /*
2674 * Enable and Setup VMDq
2675 * VMDq Filter = 0; MAC filtering
2676 * Default VMDq output index = 0;
2677 */
2678 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2679 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2680 break;
2681
2682 case ixgbe_mac_82599EB:
2683 case ixgbe_mac_X540:
2684 /*
2685 * Enable RSS & Setup RSS Hash functions
2686 */
2687 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2688 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2689 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2690 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2691 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2692 IXGBE_MRQC_RSS_FIELD_IPV6 |
2693 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2694 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2695 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2696
2697 /*
2698 * Enable VMDq+RSS.
2699 */
2700 if (ixgbe->num_rx_groups > 32) {
2701 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2702 } else {
2703 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2704 }
2705
2706 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2707
2708 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2709 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2710 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2711 }
2712 break;
2713
2714 default:
2715 break;
2716
2717 }
2718
2719 /*
2720 * Disable Packet Checksum to enable RSS for multiple receive queues.
2721 * It is an adapter hardware limitation that Packet Checksum is
2722 * mutually exclusive with RSS.
2723 */
2724 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2725 rxcsum |= IXGBE_RXCSUM_PCSD;
2726 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2727 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2728
2729 if (hw->mac.type == ixgbe_mac_82599EB ||
2730 hw->mac.type == ixgbe_mac_X540) {
2731 /*
2732 * Enable Virtualization and Replication.
2733 */
2734 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2735 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2736
2737 /*
2738 * Enable receiving packets to all VFs
2739 */
2740 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2741 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2742 }
2743 }
2744
2745 /*
2746 * ixgbe_init_unicst - Initialize the unicast addresses.
2747 */
2748 static void
2749 ixgbe_init_unicst(ixgbe_t *ixgbe)
2750 {
2751 struct ixgbe_hw *hw = &ixgbe->hw;
2752 uint8_t *mac_addr;
2753 int slot;
2754 /*
2755 * Here we should consider two situations:
2756 *
2757 * 1. Chipset is initialized at the first time,
2758 * Clear all the multiple unicast addresses.
2759 *
2760 * 2. Chipset is reset
2761 * Recover the multiple unicast addresses from the
2762 * software data structure to the RAR registers.
2763 */
2764 if (!ixgbe->unicst_init) {
2765 /*
2766 * Initialize the multiple unicast addresses
2767 */
2768 ixgbe->unicst_total = hw->mac.num_rar_entries;
2769 ixgbe->unicst_avail = ixgbe->unicst_total;
2770 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2771 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2772 bzero(mac_addr, ETHERADDRL);
2773 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2774 ixgbe->unicst_addr[slot].mac.set = 0;
2775 }
2776 ixgbe->unicst_init = B_TRUE;
2777 } else {
2778 /* Re-configure the RAR registers */
2779 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2780 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2781 if (ixgbe->unicst_addr[slot].mac.set == 1) {
2782 (void) ixgbe_set_rar(hw, slot, mac_addr,
2783 ixgbe->unicst_addr[slot].mac.group_index,
2784 IXGBE_RAH_AV);
2785 } else {
2786 bzero(mac_addr, ETHERADDRL);
2787 (void) ixgbe_set_rar(hw, slot, mac_addr,
2788 NULL, NULL);
2789 }
2790 }
2791 }
2792 }
2793
2794 /*
2795 * ixgbe_unicst_find - Find the slot for the specified unicast address
2796 */
2797 int
2798 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2799 {
2800 int slot;
2801
2802 ASSERT(mutex_owned(&ixgbe->gen_lock));
2803
2804 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2805 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2806 mac_addr, ETHERADDRL) == 0)
2807 return (slot);
2808 }
2809
2810 return (-1);
2811 }
2812
2813 /*
2814 * ixgbe_multicst_add - Add a multicst address.
2815 */
2816 int
2817 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2818 {
2819 ASSERT(mutex_owned(&ixgbe->gen_lock));
2820
2821 if ((multiaddr[0] & 01) == 0) {
2822 return (EINVAL);
2823 }
2824
2825 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2826 return (ENOENT);
2827 }
2828
2829 bcopy(multiaddr,
2830 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2831 ixgbe->mcast_count++;
2832
2833 /*
2834 * Update the multicast table in the hardware
2835 */
2836 ixgbe_setup_multicst(ixgbe);
2837
2838 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2839 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2840 return (EIO);
2841 }
2842
2843 return (0);
2844 }
2845
2846 /*
2847 * ixgbe_multicst_remove - Remove a multicst address.
2848 */
2849 int
2850 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2851 {
2852 int i;
2853
2854 ASSERT(mutex_owned(&ixgbe->gen_lock));
2855
2856 for (i = 0; i < ixgbe->mcast_count; i++) {
2857 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2858 ETHERADDRL) == 0) {
2859 for (i++; i < ixgbe->mcast_count; i++) {
2860 ixgbe->mcast_table[i - 1] =
2861 ixgbe->mcast_table[i];
2862 }
2863 ixgbe->mcast_count--;
2864 break;
2865 }
2866 }
2867
2868 /*
2869 * Update the multicast table in the hardware
2870 */
2871 ixgbe_setup_multicst(ixgbe);
2872
2873 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2874 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2875 return (EIO);
2876 }
2877
2878 return (0);
2879 }
2880
2881 /*
2882 * ixgbe_setup_multicast - Setup multicast data structures.
2883 *
2884 * This routine initializes all of the multicast related structures
2885 * and save them in the hardware registers.
2886 */
2887 static void
2888 ixgbe_setup_multicst(ixgbe_t *ixgbe)
2889 {
2890 uint8_t *mc_addr_list;
2891 uint32_t mc_addr_count;
2892 struct ixgbe_hw *hw = &ixgbe->hw;
2893
2894 ASSERT(mutex_owned(&ixgbe->gen_lock));
2895
2896 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
2897
2898 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
2899 mc_addr_count = ixgbe->mcast_count;
2900
2901 /*
2902 * Update the multicast addresses to the MTA registers
2903 */
2904 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
2905 ixgbe_mc_table_itr, TRUE);
2906 }
2907
2908 /*
2909 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
2910 *
2911 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
2912 * Different chipsets may have different allowed configuration of vmdq and rss.
2913 */
2914 static void
2915 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
2916 {
2917 struct ixgbe_hw *hw = &ixgbe->hw;
2918 uint32_t ring_per_group;
2919
2920 switch (hw->mac.type) {
2921 case ixgbe_mac_82598EB:
2922 /*
2923 * 82598 supports the following combination:
2924 * vmdq no. x rss no.
2925 * [5..16] x 1
2926 * [1..4] x [1..16]
2927 * However 8 rss queue per pool (vmdq) is sufficient for
2928 * most cases.
2929 */
2930 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2931 if (ixgbe->num_rx_groups > 4) {
2932 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
2933 } else {
2934 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2935 min(8, ring_per_group);
2936 }
2937
2938 break;
2939
2940 case ixgbe_mac_82599EB:
2941 case ixgbe_mac_X540:
2942 /*
2943 * 82599 supports the following combination:
2944 * vmdq no. x rss no.
2945 * [33..64] x [1..2]
2946 * [2..32] x [1..4]
2947 * 1 x [1..16]
2948 * However 8 rss queue per pool (vmdq) is sufficient for
2949 * most cases.
2950 *
2951 * For now, treat X540 like the 82599.
2952 */
2953 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2954 if (ixgbe->num_rx_groups == 1) {
2955 ixgbe->num_rx_rings = min(8, ring_per_group);
2956 } else if (ixgbe->num_rx_groups <= 32) {
2957 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2958 min(4, ring_per_group);
2959 } else if (ixgbe->num_rx_groups <= 64) {
2960 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2961 min(2, ring_per_group);
2962 }
2963 break;
2964
2965 default:
2966 break;
2967 }
2968
2969 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2970
2971 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
2972 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
2973 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
2974 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
2975 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
2976 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
2977 } else {
2978 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
2979 }
2980
2981 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
2982 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
2983 }
2984
2985 /*
2986 * ixgbe_get_conf - Get driver configurations set in driver.conf.
2987 *
2988 * This routine gets user-configured values out of the configuration
2989 * file ixgbe.conf.
2990 *
2991 * For each configurable value, there is a minimum, a maximum, and a
2992 * default.
2993 * If user does not configure a value, use the default.
2994 * If user configures below the minimum, use the minumum.
2995 * If user configures above the maximum, use the maxumum.
2996 */
2997 static void
2998 ixgbe_get_conf(ixgbe_t *ixgbe)
2999 {
3000 struct ixgbe_hw *hw = &ixgbe->hw;
3001 uint32_t flow_control;
3002
3003 /*
3004 * ixgbe driver supports the following user configurations:
3005 *
3006 * Jumbo frame configuration:
3007 * default_mtu
3008 *
3009 * Ethernet flow control configuration:
3010 * flow_control
3011 *
3012 * Multiple rings configurations:
3013 * tx_queue_number
3014 * tx_ring_size
3015 * rx_queue_number
3016 * rx_ring_size
3017 *
3018 * Call ixgbe_get_prop() to get the value for a specific
3019 * configuration parameter.
3020 */
3021
3022 /*
3023 * Jumbo frame configuration - max_frame_size controls host buffer
3024 * allocation, so includes MTU, ethernet header, vlan tag and
3025 * frame check sequence.
3026 */
3027 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3028 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3029
3030 ixgbe->max_frame_size = ixgbe->default_mtu +
3031 sizeof (struct ether_vlan_header) + ETHERFCSL;
3032
3033 /*
3034 * Ethernet flow control configuration
3035 */
3036 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3037 ixgbe_fc_none, 3, ixgbe_fc_none);
3038 if (flow_control == 3)
3039 flow_control = ixgbe_fc_default;
3040
3041 /*
3042 * fc.requested mode is what the user requests. After autoneg,
3043 * fc.current_mode will be the flow_control mode that was negotiated.
3044 */
3045 hw->fc.requested_mode = flow_control;
3046
3047 /*
3048 * Multiple rings configurations
3049 */
3050 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3051 ixgbe->capab->min_tx_que_num,
3052 ixgbe->capab->max_tx_que_num,
3053 ixgbe->capab->def_tx_que_num);
3054 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3055 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3056
3057 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3058 ixgbe->capab->min_rx_que_num,
3059 ixgbe->capab->max_rx_que_num,
3060 ixgbe->capab->def_rx_que_num);
3061 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3062 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3063
3064 /*
3065 * Multiple groups configuration
3066 */
3067 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3068 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3069 ixgbe->capab->def_rx_grp_num);
3070
3071 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3072 0, 1, DEFAULT_MR_ENABLE);
3073
3074 if (ixgbe->mr_enable == B_FALSE) {
3075 ixgbe->num_tx_rings = 1;
3076 ixgbe->num_rx_rings = 1;
3077 ixgbe->num_rx_groups = 1;
3078 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3079 } else {
3080 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3081 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3082 /*
3083 * The combination of num_rx_rings and num_rx_groups
3084 * may be not supported by h/w. We need to adjust
3085 * them to appropriate values.
3086 */
3087 ixgbe_setup_vmdq_rss_conf(ixgbe);
3088 }
3089
3090 /*
3091 * Tunable used to force an interrupt type. The only use is
3092 * for testing of the lesser interrupt types.
3093 * 0 = don't force interrupt type
3094 * 1 = force interrupt type MSI-X
3095 * 2 = force interrupt type MSI
3096 * 3 = force interrupt type Legacy
3097 */
3098 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3099 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3100
3101 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3102 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3103 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3104 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3105 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3106 0, 1, DEFAULT_LSO_ENABLE);
3107 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3108 0, 1, DEFAULT_LRO_ENABLE);
3109 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3110 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3111 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3112 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3113
3114 /* Head Write Back not recommended for 82599 and X540 */
3115 if (hw->mac.type == ixgbe_mac_82599EB ||
3116 hw->mac.type == ixgbe_mac_X540) {
3117 ixgbe->tx_head_wb_enable = B_FALSE;
3118 }
3119
3120 /*
3121 * ixgbe LSO needs the tx h/w checksum support.
3122 * LSO will be disabled if tx h/w checksum is not
3123 * enabled.
3124 */
3125 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3126 ixgbe->lso_enable = B_FALSE;
3127 }
3128
3129 /*
3130 * ixgbe LRO needs the rx h/w checksum support.
3131 * LRO will be disabled if rx h/w checksum is not
3132 * enabled.
3133 */
3134 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3135 ixgbe->lro_enable = B_FALSE;
3136 }
3137
3138 /*
3139 * ixgbe LRO only been supported by 82599 and X540 now
3140 */
3141 if (hw->mac.type == ixgbe_mac_82598EB) {
3142 ixgbe->lro_enable = B_FALSE;
3143 }
3144 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3145 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3146 DEFAULT_TX_COPY_THRESHOLD);
3147 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3148 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3149 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3150 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3151 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3152 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3153 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3154 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3155 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3156
3157 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3158 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3159 DEFAULT_RX_COPY_THRESHOLD);
3160 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3161 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3162 DEFAULT_RX_LIMIT_PER_INTR);
3163
3164 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3165 ixgbe->capab->min_intr_throttle,
3166 ixgbe->capab->max_intr_throttle,
3167 ixgbe->capab->def_intr_throttle);
3168 /*
3169 * 82599 and X540 require the interupt throttling rate is
3170 * a multiple of 8. This is enforced by the register
3171 * definiton.
3172 */
3173 if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540)
3174 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3175 }
3176
3177 static void
3178 ixgbe_init_params(ixgbe_t *ixgbe)
3179 {
3180 ixgbe->param_en_10000fdx_cap = 1;
3181 ixgbe->param_en_1000fdx_cap = 1;
3182 ixgbe->param_en_100fdx_cap = 1;
3183 ixgbe->param_adv_10000fdx_cap = 1;
3184 ixgbe->param_adv_1000fdx_cap = 1;
3185 ixgbe->param_adv_100fdx_cap = 1;
3186
3187 ixgbe->param_pause_cap = 1;
3188 ixgbe->param_asym_pause_cap = 1;
3189 ixgbe->param_rem_fault = 0;
3190
3191 ixgbe->param_adv_autoneg_cap = 1;
3192 ixgbe->param_adv_pause_cap = 1;
3193 ixgbe->param_adv_asym_pause_cap = 1;
3194 ixgbe->param_adv_rem_fault = 0;
3195
3196 ixgbe->param_lp_10000fdx_cap = 0;
3197 ixgbe->param_lp_1000fdx_cap = 0;
3198 ixgbe->param_lp_100fdx_cap = 0;
3199 ixgbe->param_lp_autoneg_cap = 0;
3200 ixgbe->param_lp_pause_cap = 0;
3201 ixgbe->param_lp_asym_pause_cap = 0;
3202 ixgbe->param_lp_rem_fault = 0;
3203 }
3204
3205 /*
3206 * ixgbe_get_prop - Get a property value out of the configuration file
3207 * ixgbe.conf.
3208 *
3209 * Caller provides the name of the property, a default value, a minimum
3210 * value, and a maximum value.
3211 *
3212 * Return configured value of the property, with default, minimum and
3213 * maximum properly applied.
3214 */
3215 static int
3216 ixgbe_get_prop(ixgbe_t *ixgbe,
3217 char *propname, /* name of the property */
3218 int minval, /* minimum acceptable value */
3219 int maxval, /* maximim acceptable value */
3220 int defval) /* default value */
3221 {
3222 int value;
3223
3224 /*
3225 * Call ddi_prop_get_int() to read the conf settings
3226 */
3227 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3228 DDI_PROP_DONTPASS, propname, defval);
3229 if (value > maxval)
3230 value = maxval;
3231
3232 if (value < minval)
3233 value = minval;
3234
3235 return (value);
3236 }
3237
3238 /*
3239 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3240 */
3241 int
3242 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3243 {
3244 u32 autoneg_advertised = 0;
3245
3246 /*
3247 * No half duplex support with 10Gb parts
3248 */
3249 if (ixgbe->param_adv_10000fdx_cap == 1)
3250 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3251
3252 if (ixgbe->param_adv_1000fdx_cap == 1)
3253 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3254
3255 if (ixgbe->param_adv_100fdx_cap == 1)
3256 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3257
3258 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3259 ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3260 "to autonegotiation with full link capabilities.");
3261
3262 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3263 IXGBE_LINK_SPEED_1GB_FULL |
3264 IXGBE_LINK_SPEED_100_FULL;
3265 }
3266
3267 if (setup_hw) {
3268 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3269 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) {
3270 ixgbe_notice(ixgbe, "Setup link failed on this "
3271 "device.");
3272 return (IXGBE_FAILURE);
3273 }
3274 }
3275
3276 return (IXGBE_SUCCESS);
3277 }
3278
3279 /*
3280 * ixgbe_driver_link_check - Link status processing.
3281 *
3282 * This function can be called in both kernel context and interrupt context
3283 */
3284 static void
3285 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3286 {
3287 struct ixgbe_hw *hw = &ixgbe->hw;
3288 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3289 boolean_t link_up = B_FALSE;
3290 boolean_t link_changed = B_FALSE;
3291
3292 ASSERT(mutex_owned(&ixgbe->gen_lock));
3293
3294 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3295 if (link_up) {
3296 ixgbe->link_check_complete = B_TRUE;
3297
3298 /* Link is up, enable flow control settings */
3299 (void) ixgbe_fc_enable(hw);
3300
3301 /*
3302 * The Link is up, check whether it was marked as down earlier
3303 */
3304 if (ixgbe->link_state != LINK_STATE_UP) {
3305 switch (speed) {
3306 case IXGBE_LINK_SPEED_10GB_FULL:
3307 ixgbe->link_speed = SPEED_10GB;
3308 break;
3309 case IXGBE_LINK_SPEED_1GB_FULL:
3310 ixgbe->link_speed = SPEED_1GB;
3311 break;
3312 case IXGBE_LINK_SPEED_100_FULL:
3313 ixgbe->link_speed = SPEED_100;
3314 }
3315 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3316 ixgbe->link_state = LINK_STATE_UP;
3317 link_changed = B_TRUE;
3318 }
3319 } else {
3320 if (ixgbe->link_check_complete == B_TRUE ||
3321 (ixgbe->link_check_complete == B_FALSE &&
3322 gethrtime() >= ixgbe->link_check_hrtime)) {
3323 /*
3324 * The link is really down
3325 */
3326 ixgbe->link_check_complete = B_TRUE;
3327
3328 if (ixgbe->link_state != LINK_STATE_DOWN) {
3329 ixgbe->link_speed = 0;
3330 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3331 ixgbe->link_state = LINK_STATE_DOWN;
3332 link_changed = B_TRUE;
3333 }
3334 }
3335 }
3336
3337 /*
3338 * If we are in an interrupt context, need to re-enable the
3339 * interrupt, which was automasked
3340 */
3341 if (servicing_interrupt() != 0) {
3342 ixgbe->eims |= IXGBE_EICR_LSC;
3343 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3344 }
3345
3346 if (link_changed) {
3347 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3348 }
3349 }
3350
3351 /*
3352 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3353 */
3354 static void
3355 ixgbe_sfp_check(void *arg)
3356 {
3357 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3358 uint32_t eicr = ixgbe->eicr;
3359 struct ixgbe_hw *hw = &ixgbe->hw;
3360
3361 mutex_enter(&ixgbe->gen_lock);
3362 if (eicr & IXGBE_EICR_GPI_SDP1) {
3363 /* clear the interrupt */
3364 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
3365
3366 /* if link up, do multispeed fiber setup */
3367 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3368 B_TRUE, B_TRUE);
3369 ixgbe_driver_link_check(ixgbe);
3370 ixgbe_get_hw_state(ixgbe);
3371 } else if (eicr & IXGBE_EICR_GPI_SDP2) {
3372 /* clear the interrupt */
3373 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
3374
3375 /* if link up, do sfp module setup */
3376 (void) hw->mac.ops.setup_sfp(hw);
3377
3378 /* do multispeed fiber setup */
3379 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3380 B_TRUE, B_TRUE);
3381 ixgbe_driver_link_check(ixgbe);
3382 ixgbe_get_hw_state(ixgbe);
3383 }
3384 mutex_exit(&ixgbe->gen_lock);
3385
3386 /*
3387 * We need to fully re-check the link later.
3388 */
3389 ixgbe->link_check_complete = B_FALSE;
3390 ixgbe->link_check_hrtime = gethrtime() +
3391 (IXGBE_LINK_UP_TIME * 100000000ULL);
3392 }
3393
3394 /*
3395 * ixgbe_overtemp_check - overtemp module processing done in taskq
3396 *
3397 * This routine will only be called on adapters with temperature sensor.
3398 * The indication of over-temperature can be either SDP0 interrupt or the link
3399 * status change interrupt.
3400 */
3401 static void
3402 ixgbe_overtemp_check(void *arg)
3403 {
3404 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3405 struct ixgbe_hw *hw = &ixgbe->hw;
3406 uint32_t eicr = ixgbe->eicr;
3407 ixgbe_link_speed speed;
3408 boolean_t link_up;
3409
3410 mutex_enter(&ixgbe->gen_lock);
3411
3412 /* make sure we know current state of link */
3413 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3414
3415 /* check over-temp condition */
3416 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
3417 (eicr & IXGBE_EICR_LSC)) {
3418 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3419 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3420
3421 /*
3422 * Disable the adapter interrupts
3423 */
3424 ixgbe_disable_adapter_interrupts(ixgbe);
3425
3426 /*
3427 * Disable Rx/Tx units
3428 */
3429 (void) ixgbe_stop_adapter(hw);
3430
3431 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3432 ixgbe_error(ixgbe,
3433 "Problem: Network adapter has been stopped "
3434 "because it has overheated");
3435 ixgbe_error(ixgbe,
3436 "Action: Restart the computer. "
3437 "If the problem persists, power off the system "
3438 "and replace the adapter");
3439 }
3440 }
3441
3442 /* write to clear the interrupt */
3443 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3444
3445 mutex_exit(&ixgbe->gen_lock);
3446 }
3447
3448 /*
3449 * ixgbe_link_timer - timer for link status detection
3450 */
3451 static void
3452 ixgbe_link_timer(void *arg)
3453 {
3454 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3455
3456 mutex_enter(&ixgbe->gen_lock);
3457 ixgbe_driver_link_check(ixgbe);
3458 mutex_exit(&ixgbe->gen_lock);
3459 }
3460
3461 /*
3462 * ixgbe_local_timer - Driver watchdog function.
3463 *
3464 * This function will handle the transmit stall check and other routines.
3465 */
3466 static void
3467 ixgbe_local_timer(void *arg)
3468 {
3469 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3470
3471 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3472 goto out;
3473
3474 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3475 ixgbe->reset_count++;
3476 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3477 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3478 goto out;
3479 }
3480
3481 if (ixgbe_stall_check(ixgbe)) {
3482 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3483 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3484
3485 ixgbe->reset_count++;
3486 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3487 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3488 }
3489
3490 out:
3491 ixgbe_restart_watchdog_timer(ixgbe);
3492 }
3493
3494 /*
3495 * ixgbe_stall_check - Check for transmit stall.
3496 *
3497 * This function checks if the adapter is stalled (in transmit).
3498 *
3499 * It is called each time the watchdog timeout is invoked.
3500 * If the transmit descriptor reclaim continuously fails,
3501 * the watchdog value will increment by 1. If the watchdog
3502 * value exceeds the threshold, the ixgbe is assumed to
3503 * have stalled and need to be reset.
3504 */
3505 static boolean_t
3506 ixgbe_stall_check(ixgbe_t *ixgbe)
3507 {
3508 ixgbe_tx_ring_t *tx_ring;
3509 boolean_t result;
3510 int i;
3511
3512 if (ixgbe->link_state != LINK_STATE_UP)
3513 return (B_FALSE);
3514
3515 /*
3516 * If any tx ring is stalled, we'll reset the chipset
3517 */
3518 result = B_FALSE;
3519 for (i = 0; i < ixgbe->num_tx_rings; i++) {
3520 tx_ring = &ixgbe->tx_rings[i];
3521 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3522 tx_ring->tx_recycle(tx_ring);
3523 }
3524
3525 if (tx_ring->recycle_fail > 0)
3526 tx_ring->stall_watchdog++;
3527 else
3528 tx_ring->stall_watchdog = 0;
3529
3530 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3531 result = B_TRUE;
3532 break;
3533 }
3534 }
3535
3536 if (result) {
3537 tx_ring->stall_watchdog = 0;
3538 tx_ring->recycle_fail = 0;
3539 }
3540
3541 return (result);
3542 }
3543
3544
3545 /*
3546 * is_valid_mac_addr - Check if the mac address is valid.
3547 */
3548 static boolean_t
3549 is_valid_mac_addr(uint8_t *mac_addr)
3550 {
3551 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3552 const uint8_t addr_test2[6] =
3553 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3554
3555 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3556 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3557 return (B_FALSE);
3558
3559 return (B_TRUE);
3560 }
3561
3562 static boolean_t
3563 ixgbe_find_mac_address(ixgbe_t *ixgbe)
3564 {
3565 #ifdef __sparc
3566 struct ixgbe_hw *hw = &ixgbe->hw;
3567 uchar_t *bytes;
3568 struct ether_addr sysaddr;
3569 uint_t nelts;
3570 int err;
3571 boolean_t found = B_FALSE;
3572
3573 /*
3574 * The "vendor's factory-set address" may already have
3575 * been extracted from the chip, but if the property
3576 * "local-mac-address" is set we use that instead.
3577 *
3578 * We check whether it looks like an array of 6
3579 * bytes (which it should, if OBP set it). If we can't
3580 * make sense of it this way, we'll ignore it.
3581 */
3582 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3583 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3584 if (err == DDI_PROP_SUCCESS) {
3585 if (nelts == ETHERADDRL) {
3586 while (nelts--)
3587 hw->mac.addr[nelts] = bytes[nelts];
3588 found = B_TRUE;
3589 }
3590 ddi_prop_free(bytes);
3591 }
3592
3593 /*
3594 * Look up the OBP property "local-mac-address?". If the user has set
3595 * 'local-mac-address? = false', use "the system address" instead.
3596 */
3597 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3598 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3599 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3600 if (localetheraddr(NULL, &sysaddr) != 0) {
3601 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3602 found = B_TRUE;
3603 }
3604 }
3605 ddi_prop_free(bytes);
3606 }
3607
3608 /*
3609 * Finally(!), if there's a valid "mac-address" property (created
3610 * if we netbooted from this interface), we must use this instead
3611 * of any of the above to ensure that the NFS/install server doesn't
3612 * get confused by the address changing as Solaris takes over!
3613 */
3614 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3615 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3616 if (err == DDI_PROP_SUCCESS) {
3617 if (nelts == ETHERADDRL) {
3618 while (nelts--)
3619 hw->mac.addr[nelts] = bytes[nelts];
3620 found = B_TRUE;
3621 }
3622 ddi_prop_free(bytes);
3623 }
3624
3625 if (found) {
3626 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3627 return (B_TRUE);
3628 }
3629 #else
3630 _NOTE(ARGUNUSED(ixgbe));
3631 #endif
3632
3633 return (B_TRUE);
3634 }
3635
3636 #pragma inline(ixgbe_arm_watchdog_timer)
3637 static void
3638 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
3639 {
3640 /*
3641 * Fire a watchdog timer
3642 */
3643 ixgbe->watchdog_tid =
3644 timeout(ixgbe_local_timer,
3645 (void *)ixgbe, 1 * drv_usectohz(1000000));
3646
3647 }
3648
3649 /*
3650 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
3651 */
3652 void
3653 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
3654 {
3655 mutex_enter(&ixgbe->watchdog_lock);
3656
3657 if (!ixgbe->watchdog_enable) {
3658 ixgbe->watchdog_enable = B_TRUE;
3659 ixgbe->watchdog_start = B_TRUE;
3660 ixgbe_arm_watchdog_timer(ixgbe);
3661 }
3662
3663 mutex_exit(&ixgbe->watchdog_lock);
3664 }
3665
3666 /*
3667 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
3668 */
3669 void
3670 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
3671 {
3672 timeout_id_t tid;
3673
3674 mutex_enter(&ixgbe->watchdog_lock);
3675
3676 ixgbe->watchdog_enable = B_FALSE;
3677 ixgbe->watchdog_start = B_FALSE;
3678 tid = ixgbe->watchdog_tid;
3679 ixgbe->watchdog_tid = 0;
3680
3681 mutex_exit(&ixgbe->watchdog_lock);
3682
3683 if (tid != 0)
3684 (void) untimeout(tid);
3685 }
3686
3687 /*
3688 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
3689 */
3690 void
3691 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
3692 {
3693 mutex_enter(&ixgbe->watchdog_lock);
3694
3695 if (ixgbe->watchdog_enable) {
3696 if (!ixgbe->watchdog_start) {
3697 ixgbe->watchdog_start = B_TRUE;
3698 ixgbe_arm_watchdog_timer(ixgbe);
3699 }
3700 }
3701
3702 mutex_exit(&ixgbe->watchdog_lock);
3703 }
3704
3705 /*
3706 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
3707 */
3708 static void
3709 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
3710 {
3711 mutex_enter(&ixgbe->watchdog_lock);
3712
3713 if (ixgbe->watchdog_start)
3714 ixgbe_arm_watchdog_timer(ixgbe);
3715
3716 mutex_exit(&ixgbe->watchdog_lock);
3717 }
3718
3719 /*
3720 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
3721 */
3722 void
3723 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
3724 {
3725 timeout_id_t tid;
3726
3727 mutex_enter(&ixgbe->watchdog_lock);
3728
3729 ixgbe->watchdog_start = B_FALSE;
3730 tid = ixgbe->watchdog_tid;
3731 ixgbe->watchdog_tid = 0;
3732
3733 mutex_exit(&ixgbe->watchdog_lock);
3734
3735 if (tid != 0)
3736 (void) untimeout(tid);
3737 }
3738
3739 /*
3740 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
3741 */
3742 static void
3743 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
3744 {
3745 struct ixgbe_hw *hw = &ixgbe->hw;
3746
3747 /*
3748 * mask all interrupts off
3749 */
3750 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
3751
3752 /*
3753 * for MSI-X, also disable autoclear
3754 */
3755 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3756 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
3757 }
3758
3759 IXGBE_WRITE_FLUSH(hw);
3760 }
3761
3762 /*
3763 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
3764 */
3765 static void
3766 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
3767 {
3768 struct ixgbe_hw *hw = &ixgbe->hw;
3769 uint32_t eiac, eiam;
3770 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3771
3772 /* interrupt types to enable */
3773 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
3774 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
3775 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
3776
3777 /* enable automask on "other" causes that this adapter can generate */
3778 eiam = ixgbe->capab->other_intr;
3779
3780 /*
3781 * msi-x mode
3782 */
3783 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3784 /* enable autoclear but not on bits 29:20 */
3785 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3786
3787 /* general purpose interrupt enable */
3788 gpie |= (IXGBE_GPIE_MSIX_MODE
3789 | IXGBE_GPIE_PBA_SUPPORT
3790 | IXGBE_GPIE_OCD
3791 | IXGBE_GPIE_EIAME);
3792 /*
3793 * non-msi-x mode
3794 */
3795 } else {
3796
3797 /* disable autoclear, leave gpie at default */
3798 eiac = 0;
3799
3800 /*
3801 * General purpose interrupt enable.
3802 * For 82599 or X540, extended interrupt automask enable
3803 * only in MSI or MSI-X mode
3804 */
3805 if ((hw->mac.type == ixgbe_mac_82598EB) ||
3806 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3807 gpie |= IXGBE_GPIE_EIAME;
3808 }
3809 }
3810
3811 /* Enable specific "other" interrupt types */
3812 switch (hw->mac.type) {
3813 case ixgbe_mac_82598EB:
3814 gpie |= ixgbe->capab->other_gpie;
3815 break;
3816
3817 case ixgbe_mac_82599EB:
3818 case ixgbe_mac_X540:
3819 gpie |= ixgbe->capab->other_gpie;
3820
3821 /* Enable RSC Delay 8us when LRO enabled */
3822 if (ixgbe->lro_enable) {
3823 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3824 }
3825 break;
3826
3827 default:
3828 break;
3829 }
3830
3831 /* write to interrupt control registers */
3832 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3833 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3834 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3835 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3836 IXGBE_WRITE_FLUSH(hw);
3837 }
3838
3839 /*
3840 * ixgbe_loopback_ioctl - Loopback support.
3841 */
3842 enum ioc_reply
3843 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
3844 {
3845 lb_info_sz_t *lbsp;
3846 lb_property_t *lbpp;
3847 uint32_t *lbmp;
3848 uint32_t size;
3849 uint32_t value;
3850
3851 if (mp->b_cont == NULL)
3852 return (IOC_INVAL);
3853
3854 switch (iocp->ioc_cmd) {
3855 default:
3856 return (IOC_INVAL);
3857
3858 case LB_GET_INFO_SIZE:
3859 size = sizeof (lb_info_sz_t);
3860 if (iocp->ioc_count != size)
3861 return (IOC_INVAL);
3862
3863 value = sizeof (lb_normal);
3864 value += sizeof (lb_mac);
3865 value += sizeof (lb_external);
3866
3867 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3868 *lbsp = value;
3869 break;
3870
3871 case LB_GET_INFO:
3872 value = sizeof (lb_normal);
3873 value += sizeof (lb_mac);
3874 value += sizeof (lb_external);
3875
3876 size = value;
3877 if (iocp->ioc_count != size)
3878 return (IOC_INVAL);
3879
3880 value = 0;
3881 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3882
3883 lbpp[value++] = lb_normal;
3884 lbpp[value++] = lb_mac;
3885 lbpp[value++] = lb_external;
3886 break;
3887
3888 case LB_GET_MODE:
3889 size = sizeof (uint32_t);
3890 if (iocp->ioc_count != size)
3891 return (IOC_INVAL);
3892
3893 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3894 *lbmp = ixgbe->loopback_mode;
3895 break;
3896
3897 case LB_SET_MODE:
3898 size = 0;
3899 if (iocp->ioc_count != sizeof (uint32_t))
3900 return (IOC_INVAL);
3901
3902 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3903 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
3904 return (IOC_INVAL);
3905 break;
3906 }
3907
3908 iocp->ioc_count = size;
3909 iocp->ioc_error = 0;
3910
3911 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3912 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3913 return (IOC_INVAL);
3914 }
3915
3916 return (IOC_REPLY);
3917 }
3918
3919 /*
3920 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
3921 */
3922 static boolean_t
3923 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
3924 {
3925 if (mode == ixgbe->loopback_mode)
3926 return (B_TRUE);
3927
3928 ixgbe->loopback_mode = mode;
3929
3930 if (mode == IXGBE_LB_NONE) {
3931 /*
3932 * Reset the chip
3933 */
3934 (void) ixgbe_reset(ixgbe);
3935 return (B_TRUE);
3936 }
3937
3938 mutex_enter(&ixgbe->gen_lock);
3939
3940 switch (mode) {
3941 default:
3942 mutex_exit(&ixgbe->gen_lock);
3943 return (B_FALSE);
3944
3945 case IXGBE_LB_EXTERNAL:
3946 break;
3947
3948 case IXGBE_LB_INTERNAL_MAC:
3949 ixgbe_set_internal_mac_loopback(ixgbe);
3950 break;
3951 }
3952
3953 mutex_exit(&ixgbe->gen_lock);
3954
3955 return (B_TRUE);
3956 }
3957
3958 /*
3959 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
3960 */
3961 static void
3962 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
3963 {
3964 struct ixgbe_hw *hw;
3965 uint32_t reg;
3966 uint8_t atlas;
3967
3968 hw = &ixgbe->hw;
3969
3970 /*
3971 * Setup MAC loopback
3972 */
3973 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
3974 reg |= IXGBE_HLREG0_LPBK;
3975 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
3976
3977 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
3978 reg &= ~IXGBE_AUTOC_LMS_MASK;
3979 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
3980
3981 /*
3982 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
3983 */
3984 switch (hw->mac.type) {
3985 case ixgbe_mac_82598EB:
3986 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
3987 &atlas);
3988 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
3989 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
3990 atlas);
3991
3992 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
3993 &atlas);
3994 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
3995 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
3996 atlas);
3997
3998 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
3999 &atlas);
4000 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4001 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4002 atlas);
4003
4004 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4005 &atlas);
4006 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4007 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4008 atlas);
4009 break;
4010
4011 case ixgbe_mac_82599EB:
4012 case ixgbe_mac_X540:
4013 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4014 reg |= (IXGBE_AUTOC_FLU |
4015 IXGBE_AUTOC_10G_KX4);
4016 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4017
4018 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4019 B_FALSE, B_TRUE);
4020 break;
4021
4022 default:
4023 break;
4024 }
4025 }
4026
4027 #pragma inline(ixgbe_intr_rx_work)
4028 /*
4029 * ixgbe_intr_rx_work - RX processing of ISR.
4030 */
4031 static void
4032 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4033 {
4034 mblk_t *mp;
4035
4036 mutex_enter(&rx_ring->rx_lock);
4037
4038 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4039 mutex_exit(&rx_ring->rx_lock);
4040
4041 if (mp != NULL)
4042 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4043 rx_ring->ring_gen_num);
4044 }
4045
4046 #pragma inline(ixgbe_intr_tx_work)
4047 /*
4048 * ixgbe_intr_tx_work - TX processing of ISR.
4049 */
4050 static void
4051 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4052 {
4053 ixgbe_t *ixgbe = tx_ring->ixgbe;
4054
4055 /*
4056 * Recycle the tx descriptors
4057 */
4058 tx_ring->tx_recycle(tx_ring);
4059
4060 /*
4061 * Schedule the re-transmit
4062 */
4063 if (tx_ring->reschedule &&
4064 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4065 tx_ring->reschedule = B_FALSE;
4066 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4067 tx_ring->ring_handle);
4068 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4069 }
4070 }
4071
4072 #pragma inline(ixgbe_intr_other_work)
4073 /*
4074 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4075 */
4076 static void
4077 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4078 {
4079 ASSERT(mutex_owned(&ixgbe->gen_lock));
4080
4081 /*
4082 * handle link status change
4083 */
4084 if (eicr & IXGBE_EICR_LSC) {
4085 ixgbe_driver_link_check(ixgbe);
4086 ixgbe_get_hw_state(ixgbe);
4087 }
4088
4089 /*
4090 * check for fan failure on adapters with fans
4091 */
4092 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4093 (eicr & IXGBE_EICR_GPI_SDP1)) {
4094 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4095
4096 /*
4097 * Disable the adapter interrupts
4098 */
4099 ixgbe_disable_adapter_interrupts(ixgbe);
4100
4101 /*
4102 * Disable Rx/Tx units
4103 */
4104 (void) ixgbe_stop_adapter(&ixgbe->hw);
4105
4106 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4107 ixgbe_error(ixgbe,
4108 "Problem: Network adapter has been stopped "
4109 "because the fan has stopped.\n");
4110 ixgbe_error(ixgbe,
4111 "Action: Replace the adapter.\n");
4112
4113 /* re-enable the interrupt, which was automasked */
4114 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4115 }
4116
4117 /*
4118 * Do SFP check for adapters with hot-plug capability
4119 */
4120 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4121 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) {
4122 ixgbe->eicr = eicr;
4123 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4124 ixgbe_sfp_check, (void *)ixgbe,
4125 DDI_NOSLEEP)) != DDI_SUCCESS) {
4126 ixgbe_log(ixgbe, "No memory available to dispatch "
4127 "taskq for SFP check");
4128 }
4129 }
4130
4131 /*
4132 * Do over-temperature check for adapters with temp sensor
4133 */
4134 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4135 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
4136 ixgbe->eicr = eicr;
4137 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4138 ixgbe_overtemp_check, (void *)ixgbe,
4139 DDI_NOSLEEP)) != DDI_SUCCESS) {
4140 ixgbe_log(ixgbe, "No memory available to dispatch "
4141 "taskq for overtemp check");
4142 }
4143 }
4144 }
4145
4146 /*
4147 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4148 */
4149 static uint_t
4150 ixgbe_intr_legacy(void *arg1, void *arg2)
4151 {
4152 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4153 struct ixgbe_hw *hw = &ixgbe->hw;
4154 ixgbe_tx_ring_t *tx_ring;
4155 ixgbe_rx_ring_t *rx_ring;
4156 uint32_t eicr;
4157 mblk_t *mp;
4158 boolean_t tx_reschedule;
4159 uint_t result;
4160
4161 _NOTE(ARGUNUSED(arg2));
4162
4163 mutex_enter(&ixgbe->gen_lock);
4164 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4165 mutex_exit(&ixgbe->gen_lock);
4166 return (DDI_INTR_UNCLAIMED);
4167 }
4168
4169 mp = NULL;
4170 tx_reschedule = B_FALSE;
4171
4172 /*
4173 * Any bit set in eicr: claim this interrupt
4174 */
4175 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4176
4177 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4178 mutex_exit(&ixgbe->gen_lock);
4179 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4180 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4181 return (DDI_INTR_CLAIMED);
4182 }
4183
4184 if (eicr) {
4185 /*
4186 * For legacy interrupt, we have only one interrupt,
4187 * so we have only one rx ring and one tx ring enabled.
4188 */
4189 ASSERT(ixgbe->num_rx_rings == 1);
4190 ASSERT(ixgbe->num_tx_rings == 1);
4191
4192 /*
4193 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4194 */
4195 if (eicr & 0x1) {
4196 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4197 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4198 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4199 /*
4200 * Clean the rx descriptors
4201 */
4202 rx_ring = &ixgbe->rx_rings[0];
4203 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4204 }
4205
4206 /*
4207 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4208 */
4209 if (eicr & 0x2) {
4210 /*
4211 * Recycle the tx descriptors
4212 */
4213 tx_ring = &ixgbe->tx_rings[0];
4214 tx_ring->tx_recycle(tx_ring);
4215
4216 /*
4217 * Schedule the re-transmit
4218 */
4219 tx_reschedule = (tx_ring->reschedule &&
4220 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4221 }
4222
4223 /* any interrupt type other than tx/rx */
4224 if (eicr & ixgbe->capab->other_intr) {
4225 switch (hw->mac.type) {
4226 case ixgbe_mac_82598EB:
4227 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4228 break;
4229
4230 case ixgbe_mac_82599EB:
4231 case ixgbe_mac_X540:
4232 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4233 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4234 break;
4235
4236 default:
4237 break;
4238 }
4239 ixgbe_intr_other_work(ixgbe, eicr);
4240 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4241 }
4242
4243 mutex_exit(&ixgbe->gen_lock);
4244
4245 result = DDI_INTR_CLAIMED;
4246 } else {
4247 mutex_exit(&ixgbe->gen_lock);
4248
4249 /*
4250 * No interrupt cause bits set: don't claim this interrupt.
4251 */
4252 result = DDI_INTR_UNCLAIMED;
4253 }
4254
4255 /* re-enable the interrupts which were automasked */
4256 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4257
4258 /*
4259 * Do the following work outside of the gen_lock
4260 */
4261 if (mp != NULL) {
4262 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4263 rx_ring->ring_gen_num);
4264 }
4265
4266 if (tx_reschedule) {
4267 tx_ring->reschedule = B_FALSE;
4268 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4269 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4270 }
4271
4272 return (result);
4273 }
4274
4275 /*
4276 * ixgbe_intr_msi - Interrupt handler for MSI.
4277 */
4278 static uint_t
4279 ixgbe_intr_msi(void *arg1, void *arg2)
4280 {
4281 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4282 struct ixgbe_hw *hw = &ixgbe->hw;
4283 uint32_t eicr;
4284
4285 _NOTE(ARGUNUSED(arg2));
4286
4287 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4288
4289 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4290 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4291 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4292 return (DDI_INTR_CLAIMED);
4293 }
4294
4295 /*
4296 * For MSI interrupt, we have only one vector,
4297 * so we have only one rx ring and one tx ring enabled.
4298 */
4299 ASSERT(ixgbe->num_rx_rings == 1);
4300 ASSERT(ixgbe->num_tx_rings == 1);
4301
4302 /*
4303 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4304 */
4305 if (eicr & 0x1) {
4306 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4307 }
4308
4309 /*
4310 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4311 */
4312 if (eicr & 0x2) {
4313 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4314 }
4315
4316 /* any interrupt type other than tx/rx */
4317 if (eicr & ixgbe->capab->other_intr) {
4318 mutex_enter(&ixgbe->gen_lock);
4319 switch (hw->mac.type) {
4320 case ixgbe_mac_82598EB:
4321 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4322 break;
4323
4324 case ixgbe_mac_82599EB:
4325 case ixgbe_mac_X540:
4326 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4327 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4328 break;
4329
4330 default:
4331 break;
4332 }
4333 ixgbe_intr_other_work(ixgbe, eicr);
4334 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4335 mutex_exit(&ixgbe->gen_lock);
4336 }
4337
4338 /* re-enable the interrupts which were automasked */
4339 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4340
4341 return (DDI_INTR_CLAIMED);
4342 }
4343
4344 /*
4345 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4346 */
4347 static uint_t
4348 ixgbe_intr_msix(void *arg1, void *arg2)
4349 {
4350 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4351 ixgbe_t *ixgbe = vect->ixgbe;
4352 struct ixgbe_hw *hw = &ixgbe->hw;
4353 uint32_t eicr;
4354 int r_idx = 0;
4355
4356 _NOTE(ARGUNUSED(arg2));
4357
4358 /*
4359 * Clean each rx ring that has its bit set in the map
4360 */
4361 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4362 while (r_idx >= 0) {
4363 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4364 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4365 (ixgbe->num_rx_rings - 1));
4366 }
4367
4368 /*
4369 * Clean each tx ring that has its bit set in the map
4370 */
4371 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4372 while (r_idx >= 0) {
4373 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4374 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4375 (ixgbe->num_tx_rings - 1));
4376 }
4377
4378
4379 /*
4380 * Clean other interrupt (link change) that has its bit set in the map
4381 */
4382 if (BT_TEST(vect->other_map, 0) == 1) {
4383 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4384
4385 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4386 DDI_FM_OK) {
4387 ddi_fm_service_impact(ixgbe->dip,
4388 DDI_SERVICE_DEGRADED);
4389 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4390 return (DDI_INTR_CLAIMED);
4391 }
4392
4393 /*
4394 * Check "other" cause bits: any interrupt type other than tx/rx
4395 */
4396 if (eicr & ixgbe->capab->other_intr) {
4397 mutex_enter(&ixgbe->gen_lock);
4398 switch (hw->mac.type) {
4399 case ixgbe_mac_82598EB:
4400 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4401 ixgbe_intr_other_work(ixgbe, eicr);
4402 break;
4403
4404 case ixgbe_mac_82599EB:
4405 case ixgbe_mac_X540:
4406 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4407 ixgbe_intr_other_work(ixgbe, eicr);
4408 break;
4409
4410 default:
4411 break;
4412 }
4413 mutex_exit(&ixgbe->gen_lock);
4414 }
4415
4416 /* re-enable the interrupts which were automasked */
4417 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4418 }
4419
4420 return (DDI_INTR_CLAIMED);
4421 }
4422
4423 /*
4424 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4425 *
4426 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4427 * if not successful, try Legacy.
4428 * ixgbe->intr_force can be used to force sequence to start with
4429 * any of the 3 types.
4430 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4431 */
4432 static int
4433 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4434 {
4435 dev_info_t *devinfo;
4436 int intr_types;
4437 int rc;
4438
4439 devinfo = ixgbe->dip;
4440
4441 /*
4442 * Get supported interrupt types
4443 */
4444 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4445
4446 if (rc != DDI_SUCCESS) {
4447 ixgbe_log(ixgbe,
4448 "Get supported interrupt types failed: %d", rc);
4449 return (IXGBE_FAILURE);
4450 }
4451 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4452
4453 ixgbe->intr_type = 0;
4454
4455 /*
4456 * Install MSI-X interrupts
4457 */
4458 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4459 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4460 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4461 if (rc == IXGBE_SUCCESS)
4462 return (IXGBE_SUCCESS);
4463
4464 ixgbe_log(ixgbe,
4465 "Allocate MSI-X failed, trying MSI interrupts...");
4466 }
4467
4468 /*
4469 * MSI-X not used, force rings and groups to 1
4470 */
4471 ixgbe->num_rx_rings = 1;
4472 ixgbe->num_rx_groups = 1;
4473 ixgbe->num_tx_rings = 1;
4474 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4475 ixgbe_log(ixgbe,
4476 "MSI-X not used, force rings and groups number to 1");
4477
4478 /*
4479 * Install MSI interrupts
4480 */
4481 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4482 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4483 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4484 if (rc == IXGBE_SUCCESS)
4485 return (IXGBE_SUCCESS);
4486
4487 ixgbe_log(ixgbe,
4488 "Allocate MSI failed, trying Legacy interrupts...");
4489 }
4490
4491 /*
4492 * Install legacy interrupts
4493 */
4494 if (intr_types & DDI_INTR_TYPE_FIXED) {
4495 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4496 if (rc == IXGBE_SUCCESS)
4497 return (IXGBE_SUCCESS);
4498
4499 ixgbe_log(ixgbe,
4500 "Allocate Legacy interrupts failed");
4501 }
4502
4503 /*
4504 * If none of the 3 types succeeded, return failure
4505 */
4506 return (IXGBE_FAILURE);
4507 }
4508
4509 /*
4510 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4511 *
4512 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4513 * if fewer than 2 handles are available, return failure.
4514 * Upon success, this maps the vectors to rx and tx rings for
4515 * interrupts.
4516 */
4517 static int
4518 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4519 {
4520 dev_info_t *devinfo;
4521 int request, count, actual;
4522 int minimum;
4523 int rc;
4524 uint32_t ring_per_group;
4525
4526 devinfo = ixgbe->dip;
4527
4528 switch (intr_type) {
4529 case DDI_INTR_TYPE_FIXED:
4530 request = 1; /* Request 1 legacy interrupt handle */
4531 minimum = 1;
4532 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4533 break;
4534
4535 case DDI_INTR_TYPE_MSI:
4536 request = 1; /* Request 1 MSI interrupt handle */
4537 minimum = 1;
4538 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4539 break;
4540
4541 case DDI_INTR_TYPE_MSIX:
4542 /*
4543 * Best number of vectors for the adapter is
4544 * (# rx rings + # tx rings), however we will
4545 * limit the request number.
4546 */
4547 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4548 if (request > ixgbe->capab->max_ring_vect)
4549 request = ixgbe->capab->max_ring_vect;
4550 minimum = 1;
4551 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4552 break;
4553
4554 default:
4555 ixgbe_log(ixgbe,
4556 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4557 intr_type);
4558 return (IXGBE_FAILURE);
4559 }
4560 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
4561 request, minimum);
4562
4563 /*
4564 * Get number of supported interrupts
4565 */
4566 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4567 if ((rc != DDI_SUCCESS) || (count < minimum)) {
4568 ixgbe_log(ixgbe,
4569 "Get interrupt number failed. Return: %d, count: %d",
4570 rc, count);
4571 return (IXGBE_FAILURE);
4572 }
4573 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4574
4575 actual = 0;
4576 ixgbe->intr_cnt = 0;
4577 ixgbe->intr_cnt_max = 0;
4578 ixgbe->intr_cnt_min = 0;
4579
4580 /*
4581 * Allocate an array of interrupt handles
4582 */
4583 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
4584 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
4585
4586 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
4587 request, &actual, DDI_INTR_ALLOC_NORMAL);
4588 if (rc != DDI_SUCCESS) {
4589 ixgbe_log(ixgbe, "Allocate interrupts failed. "
4590 "return: %d, request: %d, actual: %d",
4591 rc, request, actual);
4592 goto alloc_handle_fail;
4593 }
4594 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
4595
4596 /*
4597 * upper/lower limit of interrupts
4598 */
4599 ixgbe->intr_cnt = actual;
4600 ixgbe->intr_cnt_max = request;
4601 ixgbe->intr_cnt_min = minimum;
4602
4603 /*
4604 * rss number per group should not exceed the rx interrupt number,
4605 * else need to adjust rx ring number.
4606 */
4607 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4608 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
4609 if (actual < ring_per_group) {
4610 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
4611 ixgbe_setup_vmdq_rss_conf(ixgbe);
4612 }
4613
4614 /*
4615 * Now we know the actual number of vectors. Here we map the vector
4616 * to other, rx rings and tx ring.
4617 */
4618 if (actual < minimum) {
4619 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
4620 actual);
4621 goto alloc_handle_fail;
4622 }
4623
4624 /*
4625 * Get priority for first vector, assume remaining are all the same
4626 */
4627 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
4628 if (rc != DDI_SUCCESS) {
4629 ixgbe_log(ixgbe,
4630 "Get interrupt priority failed: %d", rc);
4631 goto alloc_handle_fail;
4632 }
4633
4634 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
4635 if (rc != DDI_SUCCESS) {
4636 ixgbe_log(ixgbe,
4637 "Get interrupt cap failed: %d", rc);
4638 goto alloc_handle_fail;
4639 }
4640
4641 ixgbe->intr_type = intr_type;
4642
4643 return (IXGBE_SUCCESS);
4644
4645 alloc_handle_fail:
4646 ixgbe_rem_intrs(ixgbe);
4647
4648 return (IXGBE_FAILURE);
4649 }
4650
4651 /*
4652 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
4653 *
4654 * Before adding the interrupt handlers, the interrupt vectors have
4655 * been allocated, and the rx/tx rings have also been allocated.
4656 */
4657 static int
4658 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
4659 {
4660 int vector = 0;
4661 int rc;
4662
4663 switch (ixgbe->intr_type) {
4664 case DDI_INTR_TYPE_MSIX:
4665 /*
4666 * Add interrupt handler for all vectors
4667 */
4668 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
4669 /*
4670 * install pointer to vect_map[vector]
4671 */
4672 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4673 (ddi_intr_handler_t *)ixgbe_intr_msix,
4674 (void *)&ixgbe->vect_map[vector], NULL);
4675
4676 if (rc != DDI_SUCCESS) {
4677 ixgbe_log(ixgbe,
4678 "Add interrupt handler failed. "
4679 "return: %d, vector: %d", rc, vector);
4680 for (vector--; vector >= 0; vector--) {
4681 (void) ddi_intr_remove_handler(
4682 ixgbe->htable[vector]);
4683 }
4684 return (IXGBE_FAILURE);
4685 }
4686 }
4687
4688 break;
4689
4690 case DDI_INTR_TYPE_MSI:
4691 /*
4692 * Add interrupt handlers for the only vector
4693 */
4694 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4695 (ddi_intr_handler_t *)ixgbe_intr_msi,
4696 (void *)ixgbe, NULL);
4697
4698 if (rc != DDI_SUCCESS) {
4699 ixgbe_log(ixgbe,
4700 "Add MSI interrupt handler failed: %d", rc);
4701 return (IXGBE_FAILURE);
4702 }
4703
4704 break;
4705
4706 case DDI_INTR_TYPE_FIXED:
4707 /*
4708 * Add interrupt handlers for the only vector
4709 */
4710 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4711 (ddi_intr_handler_t *)ixgbe_intr_legacy,
4712 (void *)ixgbe, NULL);
4713
4714 if (rc != DDI_SUCCESS) {
4715 ixgbe_log(ixgbe,
4716 "Add legacy interrupt handler failed: %d", rc);
4717 return (IXGBE_FAILURE);
4718 }
4719
4720 break;
4721
4722 default:
4723 return (IXGBE_FAILURE);
4724 }
4725
4726 return (IXGBE_SUCCESS);
4727 }
4728
4729 #pragma inline(ixgbe_map_rxring_to_vector)
4730 /*
4731 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
4732 */
4733 static void
4734 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
4735 {
4736 /*
4737 * Set bit in map
4738 */
4739 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4740
4741 /*
4742 * Count bits set
4743 */
4744 ixgbe->vect_map[v_idx].rxr_cnt++;
4745
4746 /*
4747 * Remember bit position
4748 */
4749 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
4750 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
4751 }
4752
4753 #pragma inline(ixgbe_map_txring_to_vector)
4754 /*
4755 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
4756 */
4757 static void
4758 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
4759 {
4760 /*
4761 * Set bit in map
4762 */
4763 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
4764
4765 /*
4766 * Count bits set
4767 */
4768 ixgbe->vect_map[v_idx].txr_cnt++;
4769
4770 /*
4771 * Remember bit position
4772 */
4773 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
4774 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
4775 }
4776
4777 /*
4778 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
4779 * allocation register (IVAR).
4780 * cause:
4781 * -1 : other cause
4782 * 0 : rx
4783 * 1 : tx
4784 */
4785 static void
4786 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
4787 int8_t cause)
4788 {
4789 struct ixgbe_hw *hw = &ixgbe->hw;
4790 u32 ivar, index;
4791
4792 switch (hw->mac.type) {
4793 case ixgbe_mac_82598EB:
4794 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4795 if (cause == -1) {
4796 cause = 0;
4797 }
4798 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4799 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4800 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4801 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4802 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4803 break;
4804
4805 case ixgbe_mac_82599EB:
4806 case ixgbe_mac_X540:
4807 if (cause == -1) {
4808 /* other causes */
4809 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4810 index = (intr_alloc_entry & 1) * 8;
4811 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4812 ivar &= ~(0xFF << index);
4813 ivar |= (msix_vector << index);
4814 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4815 } else {
4816 /* tx or rx causes */
4817 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4818 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4819 ivar = IXGBE_READ_REG(hw,
4820 IXGBE_IVAR(intr_alloc_entry >> 1));
4821 ivar &= ~(0xFF << index);
4822 ivar |= (msix_vector << index);
4823 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4824 ivar);
4825 }
4826 break;
4827
4828 default:
4829 break;
4830 }
4831 }
4832
4833 /*
4834 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
4835 * given interrupt vector allocation register (IVAR).
4836 * cause:
4837 * -1 : other cause
4838 * 0 : rx
4839 * 1 : tx
4840 */
4841 static void
4842 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4843 {
4844 struct ixgbe_hw *hw = &ixgbe->hw;
4845 u32 ivar, index;
4846
4847 switch (hw->mac.type) {
4848 case ixgbe_mac_82598EB:
4849 if (cause == -1) {
4850 cause = 0;
4851 }
4852 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4853 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4854 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4855 (intr_alloc_entry & 0x3)));
4856 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4857 break;
4858
4859 case ixgbe_mac_82599EB:
4860 case ixgbe_mac_X540:
4861 if (cause == -1) {
4862 /* other causes */
4863 index = (intr_alloc_entry & 1) * 8;
4864 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4865 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4866 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4867 } else {
4868 /* tx or rx causes */
4869 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4870 ivar = IXGBE_READ_REG(hw,
4871 IXGBE_IVAR(intr_alloc_entry >> 1));
4872 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4873 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4874 ivar);
4875 }
4876 break;
4877
4878 default:
4879 break;
4880 }
4881 }
4882
4883 /*
4884 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
4885 * given interrupt vector allocation register (IVAR).
4886 * cause:
4887 * -1 : other cause
4888 * 0 : rx
4889 * 1 : tx
4890 */
4891 static void
4892 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4893 {
4894 struct ixgbe_hw *hw = &ixgbe->hw;
4895 u32 ivar, index;
4896
4897 switch (hw->mac.type) {
4898 case ixgbe_mac_82598EB:
4899 if (cause == -1) {
4900 cause = 0;
4901 }
4902 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4903 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4904 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
4905 (intr_alloc_entry & 0x3)));
4906 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4907 break;
4908
4909 case ixgbe_mac_82599EB:
4910 case ixgbe_mac_X540:
4911 if (cause == -1) {
4912 /* other causes */
4913 index = (intr_alloc_entry & 1) * 8;
4914 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4915 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4916 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4917 } else {
4918 /* tx or rx causes */
4919 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4920 ivar = IXGBE_READ_REG(hw,
4921 IXGBE_IVAR(intr_alloc_entry >> 1));
4922 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4923 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4924 ivar);
4925 }
4926 break;
4927
4928 default:
4929 break;
4930 }
4931 }
4932
4933 /*
4934 * Convert the rx ring index driver maintained to the rx ring index
4935 * in h/w.
4936 */
4937 static uint32_t
4938 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
4939 {
4940
4941 struct ixgbe_hw *hw = &ixgbe->hw;
4942 uint32_t rx_ring_per_group, hw_rx_index;
4943
4944 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
4945 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
4946 return (sw_rx_index);
4947 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
4948 switch (hw->mac.type) {
4949 case ixgbe_mac_82598EB:
4950 return (sw_rx_index);
4951
4952 case ixgbe_mac_82599EB:
4953 case ixgbe_mac_X540:
4954 return (sw_rx_index * 2);
4955
4956 default:
4957 break;
4958 }
4959 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
4960 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4961
4962 switch (hw->mac.type) {
4963 case ixgbe_mac_82598EB:
4964 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
4965 16 + (sw_rx_index % rx_ring_per_group);
4966 return (hw_rx_index);
4967
4968 case ixgbe_mac_82599EB:
4969 case ixgbe_mac_X540:
4970 if (ixgbe->num_rx_groups > 32) {
4971 hw_rx_index = (sw_rx_index /
4972 rx_ring_per_group) * 2 +
4973 (sw_rx_index % rx_ring_per_group);
4974 } else {
4975 hw_rx_index = (sw_rx_index /
4976 rx_ring_per_group) * 4 +
4977 (sw_rx_index % rx_ring_per_group);
4978 }
4979 return (hw_rx_index);
4980
4981 default:
4982 break;
4983 }
4984 }
4985
4986 /*
4987 * Should never reach. Just to make compiler happy.
4988 */
4989 return (sw_rx_index);
4990 }
4991
4992 /*
4993 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
4994 *
4995 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
4996 * to vector[0 - (intr_cnt -1)].
4997 */
4998 static int
4999 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5000 {
5001 int i, vector = 0;
5002
5003 /* initialize vector map */
5004 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5005 for (i = 0; i < ixgbe->intr_cnt; i++) {
5006 ixgbe->vect_map[i].ixgbe = ixgbe;
5007 }
5008
5009 /*
5010 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5011 * tx rings[0] on RTxQ[1].
5012 */
5013 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5014 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5015 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5016 return (IXGBE_SUCCESS);
5017 }
5018
5019 /*
5020 * Interrupts/vectors mapping for MSI-X
5021 */
5022
5023 /*
5024 * Map other interrupt to vector 0,
5025 * Set bit in map and count the bits set.
5026 */
5027 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5028 ixgbe->vect_map[vector].other_cnt++;
5029
5030 /*
5031 * Map rx ring interrupts to vectors
5032 */
5033 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5034 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5035 vector = (vector +1) % ixgbe->intr_cnt;
5036 }
5037
5038 /*
5039 * Map tx ring interrupts to vectors
5040 */
5041 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5042 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5043 vector = (vector +1) % ixgbe->intr_cnt;
5044 }
5045
5046 return (IXGBE_SUCCESS);
5047 }
5048
5049 /*
5050 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5051 *
5052 * This relies on ring/vector mapping already set up in the
5053 * vect_map[] structures
5054 */
5055 static void
5056 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5057 {
5058 struct ixgbe_hw *hw = &ixgbe->hw;
5059 ixgbe_intr_vector_t *vect; /* vector bitmap */
5060 int r_idx; /* ring index */
5061 int v_idx; /* vector index */
5062 uint32_t hw_index;
5063
5064 /*
5065 * Clear any previous entries
5066 */
5067 switch (hw->mac.type) {
5068 case ixgbe_mac_82598EB:
5069 for (v_idx = 0; v_idx < 25; v_idx++)
5070 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5071 break;
5072
5073 case ixgbe_mac_82599EB:
5074 case ixgbe_mac_X540:
5075 for (v_idx = 0; v_idx < 64; v_idx++)
5076 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5077 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5078 break;
5079
5080 default:
5081 break;
5082 }
5083
5084 /*
5085 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5086 * tx rings[0] will use RTxQ[1].
5087 */
5088 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5089 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5090 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5091 return;
5092 }
5093
5094 /*
5095 * For MSI-X interrupt, "Other" is always on vector[0].
5096 */
5097 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5098
5099 /*
5100 * For each interrupt vector, populate the IVAR table
5101 */
5102 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5103 vect = &ixgbe->vect_map[v_idx];
5104
5105 /*
5106 * For each rx ring bit set
5107 */
5108 r_idx = bt_getlowbit(vect->rx_map, 0,
5109 (ixgbe->num_rx_rings - 1));
5110
5111 while (r_idx >= 0) {
5112 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5113 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5114 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5115 (ixgbe->num_rx_rings - 1));
5116 }
5117
5118 /*
5119 * For each tx ring bit set
5120 */
5121 r_idx = bt_getlowbit(vect->tx_map, 0,
5122 (ixgbe->num_tx_rings - 1));
5123
5124 while (r_idx >= 0) {
5125 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5126 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5127 (ixgbe->num_tx_rings - 1));
5128 }
5129 }
5130 }
5131
5132 /*
5133 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5134 */
5135 static void
5136 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5137 {
5138 int i;
5139 int rc;
5140
5141 for (i = 0; i < ixgbe->intr_cnt; i++) {
5142 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5143 if (rc != DDI_SUCCESS) {
5144 IXGBE_DEBUGLOG_1(ixgbe,
5145 "Remove intr handler failed: %d", rc);
5146 }
5147 }
5148 }
5149
5150 /*
5151 * ixgbe_rem_intrs - Remove the allocated interrupts.
5152 */
5153 static void
5154 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5155 {
5156 int i;
5157 int rc;
5158
5159 for (i = 0; i < ixgbe->intr_cnt; i++) {
5160 rc = ddi_intr_free(ixgbe->htable[i]);
5161 if (rc != DDI_SUCCESS) {
5162 IXGBE_DEBUGLOG_1(ixgbe,
5163 "Free intr failed: %d", rc);
5164 }
5165 }
5166
5167 kmem_free(ixgbe->htable, ixgbe->intr_size);
5168 ixgbe->htable = NULL;
5169 }
5170
5171 /*
5172 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5173 */
5174 static int
5175 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5176 {
5177 int i;
5178 int rc;
5179
5180 /*
5181 * Enable interrupts
5182 */
5183 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5184 /*
5185 * Call ddi_intr_block_enable() for MSI
5186 */
5187 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5188 if (rc != DDI_SUCCESS) {
5189 ixgbe_log(ixgbe,
5190 "Enable block intr failed: %d", rc);
5191 return (IXGBE_FAILURE);
5192 }
5193 } else {
5194 /*
5195 * Call ddi_intr_enable() for Legacy/MSI non block enable
5196 */
5197 for (i = 0; i < ixgbe->intr_cnt; i++) {
5198 rc = ddi_intr_enable(ixgbe->htable[i]);
5199 if (rc != DDI_SUCCESS) {
5200 ixgbe_log(ixgbe,
5201 "Enable intr failed: %d", rc);
5202 return (IXGBE_FAILURE);
5203 }
5204 }
5205 }
5206
5207 return (IXGBE_SUCCESS);
5208 }
5209
5210 /*
5211 * ixgbe_disable_intrs - Disable all the interrupts.
5212 */
5213 static int
5214 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5215 {
5216 int i;
5217 int rc;
5218
5219 /*
5220 * Disable all interrupts
5221 */
5222 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5223 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5224 if (rc != DDI_SUCCESS) {
5225 ixgbe_log(ixgbe,
5226 "Disable block intr failed: %d", rc);
5227 return (IXGBE_FAILURE);
5228 }
5229 } else {
5230 for (i = 0; i < ixgbe->intr_cnt; i++) {
5231 rc = ddi_intr_disable(ixgbe->htable[i]);
5232 if (rc != DDI_SUCCESS) {
5233 ixgbe_log(ixgbe,
5234 "Disable intr failed: %d", rc);
5235 return (IXGBE_FAILURE);
5236 }
5237 }
5238 }
5239
5240 return (IXGBE_SUCCESS);
5241 }
5242
5243 /*
5244 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5245 */
5246 static void
5247 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5248 {
5249 struct ixgbe_hw *hw = &ixgbe->hw;
5250 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5251 boolean_t link_up = B_FALSE;
5252 uint32_t pcs1g_anlp = 0;
5253 uint32_t pcs1g_ana = 0;
5254 boolean_t autoneg = B_FALSE;
5255
5256 ASSERT(mutex_owned(&ixgbe->gen_lock));
5257 ixgbe->param_lp_1000fdx_cap = 0;
5258 ixgbe->param_lp_100fdx_cap = 0;
5259
5260 /* check for link, don't wait */
5261 (void) ixgbe_check_link(hw, &speed, &link_up, false);
5262 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5263
5264 if (link_up) {
5265 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5266
5267 ixgbe->param_lp_1000fdx_cap =
5268 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5269 ixgbe->param_lp_100fdx_cap =
5270 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5271 }
5272
5273 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5274
5275 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5276 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5277 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5278 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5279 }
5280
5281 /*
5282 * ixgbe_get_driver_control - Notify that driver is in control of device.
5283 */
5284 static void
5285 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5286 {
5287 uint32_t ctrl_ext;
5288
5289 /*
5290 * Notify firmware that driver is in control of device
5291 */
5292 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5293 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5294 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5295 }
5296
5297 /*
5298 * ixgbe_release_driver_control - Notify that driver is no longer in control
5299 * of device.
5300 */
5301 static void
5302 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5303 {
5304 uint32_t ctrl_ext;
5305
5306 /*
5307 * Notify firmware that driver is no longer in control of device
5308 */
5309 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5310 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5311 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5312 }
5313
5314 /*
5315 * ixgbe_atomic_reserve - Atomic decrease operation.
5316 */
5317 int
5318 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5319 {
5320 uint32_t oldval;
5321 uint32_t newval;
5322
5323 /*
5324 * ATOMICALLY
5325 */
5326 do {
5327 oldval = *count_p;
5328 if (oldval < n)
5329 return (-1);
5330 newval = oldval - n;
5331 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5332
5333 return (newval);
5334 }
5335
5336 /*
5337 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5338 */
5339 static uint8_t *
5340 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5341 {
5342 uint8_t *addr = *upd_ptr;
5343 uint8_t *new_ptr;
5344
5345 _NOTE(ARGUNUSED(hw));
5346 _NOTE(ARGUNUSED(vmdq));
5347
5348 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5349 *upd_ptr = new_ptr;
5350 return (addr);
5351 }
5352
5353 /*
5354 * FMA support
5355 */
5356 int
5357 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5358 {
5359 ddi_fm_error_t de;
5360
5361 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5362 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5363 return (de.fme_status);
5364 }
5365
5366 int
5367 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5368 {
5369 ddi_fm_error_t de;
5370
5371 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5372 return (de.fme_status);
5373 }
5374
5375 /*
5376 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5377 */
5378 static int
5379 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5380 {
5381 _NOTE(ARGUNUSED(impl_data));
5382 /*
5383 * as the driver can always deal with an error in any dma or
5384 * access handle, we can just return the fme_status value.
5385 */
5386 pci_ereport_post(dip, err, NULL);
5387 return (err->fme_status);
5388 }
5389
5390 static void
5391 ixgbe_fm_init(ixgbe_t *ixgbe)
5392 {
5393 ddi_iblock_cookie_t iblk;
5394 int fma_dma_flag;
5395
5396 /*
5397 * Only register with IO Fault Services if we have some capability
5398 */
5399 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5400 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5401 } else {
5402 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5403 }
5404
5405 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5406 fma_dma_flag = 1;
5407 } else {
5408 fma_dma_flag = 0;
5409 }
5410
5411 ixgbe_set_fma_flags(fma_dma_flag);
5412
5413 if (ixgbe->fm_capabilities) {
5414
5415 /*
5416 * Register capabilities with IO Fault Services
5417 */
5418 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5419
5420 /*
5421 * Initialize pci ereport capabilities if ereport capable
5422 */
5423 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5424 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5425 pci_ereport_setup(ixgbe->dip);
5426
5427 /*
5428 * Register error callback if error callback capable
5429 */
5430 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5431 ddi_fm_handler_register(ixgbe->dip,
5432 ixgbe_fm_error_cb, (void*) ixgbe);
5433 }
5434 }
5435
5436 static void
5437 ixgbe_fm_fini(ixgbe_t *ixgbe)
5438 {
5439 /*
5440 * Only unregister FMA capabilities if they are registered
5441 */
5442 if (ixgbe->fm_capabilities) {
5443
5444 /*
5445 * Release any resources allocated by pci_ereport_setup()
5446 */
5447 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5448 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5449 pci_ereport_teardown(ixgbe->dip);
5450
5451 /*
5452 * Un-register error callback if error callback capable
5453 */
5454 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5455 ddi_fm_handler_unregister(ixgbe->dip);
5456
5457 /*
5458 * Unregister from IO Fault Service
5459 */
5460 ddi_fm_fini(ixgbe->dip);
5461 }
5462 }
5463
5464 void
5465 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5466 {
5467 uint64_t ena;
5468 char buf[FM_MAX_CLASS];
5469
5470 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5471 ena = fm_ena_generate(0, FM_ENA_FMT1);
5472 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5473 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5474 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5475 }
5476 }
5477
5478 static int
5479 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5480 {
5481 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5482
5483 mutex_enter(&rx_ring->rx_lock);
5484 rx_ring->ring_gen_num = mr_gen_num;
5485 mutex_exit(&rx_ring->rx_lock);
5486 return (0);
5487 }
5488
5489 /*
5490 * Get the global ring index by a ring index within a group.
5491 */
5492 static int
5493 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5494 {
5495 ixgbe_rx_ring_t *rx_ring;
5496 int i;
5497
5498 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5499 rx_ring = &ixgbe->rx_rings[i];
5500 if (rx_ring->group_index == gindex)
5501 rindex--;
5502 if (rindex < 0)
5503 return (i);
5504 }
5505
5506 return (-1);
5507 }
5508
5509 /*
5510 * Callback funtion for MAC layer to register all rings.
5511 */
5512 /* ARGSUSED */
5513 void
5514 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5515 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5516 {
5517 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5518 mac_intr_t *mintr = &infop->mri_intr;
5519
5520 switch (rtype) {
5521 case MAC_RING_TYPE_RX: {
5522 /*
5523 * 'index' is the ring index within the group.
5524 * Need to get the global ring index by searching in groups.
5525 */
5526 int global_ring_index = ixgbe_get_rx_ring_index(
5527 ixgbe, group_index, ring_index);
5528
5529 ASSERT(global_ring_index >= 0);
5530
5531 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5532 rx_ring->ring_handle = rh;
5533
5534 infop->mri_driver = (mac_ring_driver_t)rx_ring;
5535 infop->mri_start = ixgbe_ring_start;
5536 infop->mri_stop = NULL;
5537 infop->mri_poll = ixgbe_ring_rx_poll;
5538 infop->mri_stat = ixgbe_rx_ring_stat;
5539
5540 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
5541 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
5542 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
5543 if (ixgbe->intr_type &
5544 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5545 mintr->mi_ddi_handle =
5546 ixgbe->htable[rx_ring->intr_vector];
5547 }
5548
5549 break;
5550 }
5551 case MAC_RING_TYPE_TX: {
5552 ASSERT(group_index == -1);
5553 ASSERT(ring_index < ixgbe->num_tx_rings);
5554
5555 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
5556 tx_ring->ring_handle = rh;
5557
5558 infop->mri_driver = (mac_ring_driver_t)tx_ring;
5559 infop->mri_start = NULL;
5560 infop->mri_stop = NULL;
5561 infop->mri_tx = ixgbe_ring_tx;
5562 infop->mri_stat = ixgbe_tx_ring_stat;
5563 if (ixgbe->intr_type &
5564 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5565 mintr->mi_ddi_handle =
5566 ixgbe->htable[tx_ring->intr_vector];
5567 }
5568 break;
5569 }
5570 default:
5571 break;
5572 }
5573 }
5574
5575 /*
5576 * Callback funtion for MAC layer to register all groups.
5577 */
5578 void
5579 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
5580 mac_group_info_t *infop, mac_group_handle_t gh)
5581 {
5582 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5583
5584 switch (rtype) {
5585 case MAC_RING_TYPE_RX: {
5586 ixgbe_rx_group_t *rx_group;
5587
5588 rx_group = &ixgbe->rx_groups[index];
5589 rx_group->group_handle = gh;
5590
5591 infop->mgi_driver = (mac_group_driver_t)rx_group;
5592 infop->mgi_start = NULL;
5593 infop->mgi_stop = NULL;
5594 infop->mgi_addmac = ixgbe_addmac;
5595 infop->mgi_remmac = ixgbe_remmac;
5596 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
5597
5598 break;
5599 }
5600 case MAC_RING_TYPE_TX:
5601 break;
5602 default:
5603 break;
5604 }
5605 }
5606
5607 /*
5608 * Enable interrupt on the specificed rx ring.
5609 */
5610 int
5611 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
5612 {
5613 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5614 ixgbe_t *ixgbe = rx_ring->ixgbe;
5615 int r_idx = rx_ring->index;
5616 int hw_r_idx = rx_ring->hw_index;
5617 int v_idx = rx_ring->intr_vector;
5618
5619 mutex_enter(&ixgbe->gen_lock);
5620 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5621 mutex_exit(&ixgbe->gen_lock);
5622 /*
5623 * Simply return 0.
5624 * Interrupts are being adjusted. ixgbe_intr_adjust()
5625 * will eventually re-enable the interrupt when it's
5626 * done with the adjustment.
5627 */
5628 return (0);
5629 }
5630
5631 /*
5632 * To enable interrupt by setting the VAL bit of given interrupt
5633 * vector allocation register (IVAR).
5634 */
5635 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
5636
5637 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5638
5639 /*
5640 * Trigger a Rx interrupt on this ring
5641 */
5642 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
5643 IXGBE_WRITE_FLUSH(&ixgbe->hw);
5644
5645 mutex_exit(&ixgbe->gen_lock);
5646
5647 return (0);
5648 }
5649
5650 /*
5651 * Disable interrupt on the specificed rx ring.
5652 */
5653 int
5654 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
5655 {
5656 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5657 ixgbe_t *ixgbe = rx_ring->ixgbe;
5658 int r_idx = rx_ring->index;
5659 int hw_r_idx = rx_ring->hw_index;
5660 int v_idx = rx_ring->intr_vector;
5661
5662 mutex_enter(&ixgbe->gen_lock);
5663 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5664 mutex_exit(&ixgbe->gen_lock);
5665 /*
5666 * Simply return 0.
5667 * In the rare case where an interrupt is being
5668 * disabled while interrupts are being adjusted,
5669 * we don't fail the operation. No interrupts will
5670 * be generated while they are adjusted, and
5671 * ixgbe_intr_adjust() will cause the interrupts
5672 * to be re-enabled once it completes. Note that
5673 * in this case, packets may be delivered to the
5674 * stack via interrupts before xgbe_rx_ring_intr_enable()
5675 * is called again. This is acceptable since interrupt
5676 * adjustment is infrequent, and the stack will be
5677 * able to handle these packets.
5678 */
5679 return (0);
5680 }
5681
5682 /*
5683 * To disable interrupt by clearing the VAL bit of given interrupt
5684 * vector allocation register (IVAR).
5685 */
5686 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
5687
5688 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
5689
5690 mutex_exit(&ixgbe->gen_lock);
5691
5692 return (0);
5693 }
5694
5695 /*
5696 * Add a mac address.
5697 */
5698 static int
5699 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
5700 {
5701 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5702 ixgbe_t *ixgbe = rx_group->ixgbe;
5703 struct ixgbe_hw *hw = &ixgbe->hw;
5704 int slot, i;
5705
5706 mutex_enter(&ixgbe->gen_lock);
5707
5708 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5709 mutex_exit(&ixgbe->gen_lock);
5710 return (ECANCELED);
5711 }
5712
5713 if (ixgbe->unicst_avail == 0) {
5714 /* no slots available */
5715 mutex_exit(&ixgbe->gen_lock);
5716 return (ENOSPC);
5717 }
5718
5719 /*
5720 * The first ixgbe->num_rx_groups slots are reserved for each respective
5721 * group. The rest slots are shared by all groups. While adding a
5722 * MAC address, reserved slots are firstly checked then the shared
5723 * slots are searched.
5724 */
5725 slot = -1;
5726 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
5727 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
5728 if (ixgbe->unicst_addr[i].mac.set == 0) {
5729 slot = i;
5730 break;
5731 }
5732 }
5733 } else {
5734 slot = rx_group->index;
5735 }
5736
5737 if (slot == -1) {
5738 /* no slots available */
5739 mutex_exit(&ixgbe->gen_lock);
5740 return (ENOSPC);
5741 }
5742
5743 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5744 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
5745 rx_group->index, IXGBE_RAH_AV);
5746 ixgbe->unicst_addr[slot].mac.set = 1;
5747 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
5748 ixgbe->unicst_avail--;
5749
5750 mutex_exit(&ixgbe->gen_lock);
5751
5752 return (0);
5753 }
5754
5755 /*
5756 * Remove a mac address.
5757 */
5758 static int
5759 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
5760 {
5761 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5762 ixgbe_t *ixgbe = rx_group->ixgbe;
5763 struct ixgbe_hw *hw = &ixgbe->hw;
5764 int slot;
5765
5766 mutex_enter(&ixgbe->gen_lock);
5767
5768 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5769 mutex_exit(&ixgbe->gen_lock);
5770 return (ECANCELED);
5771 }
5772
5773 slot = ixgbe_unicst_find(ixgbe, mac_addr);
5774 if (slot == -1) {
5775 mutex_exit(&ixgbe->gen_lock);
5776 return (EINVAL);
5777 }
5778
5779 if (ixgbe->unicst_addr[slot].mac.set == 0) {
5780 mutex_exit(&ixgbe->gen_lock);
5781 return (EINVAL);
5782 }
5783
5784 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5785 (void) ixgbe_clear_rar(hw, slot);
5786 ixgbe->unicst_addr[slot].mac.set = 0;
5787 ixgbe->unicst_avail++;
5788
5789 mutex_exit(&ixgbe->gen_lock);
5790
5791 return (0);
5792 }