1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2007-2012 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 */
29
30 #include "igb_sw.h"
31
32 static char ident[] = "Intel 1Gb Ethernet";
33 static char igb_version[] = "igb 1.1.18";
34
35 /*
36 * Local function protoypes
37 */
38 static int igb_register_mac(igb_t *);
39 static int igb_identify_hardware(igb_t *);
40 static int igb_regs_map(igb_t *);
41 static void igb_init_properties(igb_t *);
42 static int igb_init_driver_settings(igb_t *);
43 static void igb_init_locks(igb_t *);
44 static void igb_destroy_locks(igb_t *);
45 static int igb_init_mac_address(igb_t *);
46 static int igb_init(igb_t *);
47 static int igb_init_adapter(igb_t *);
48 static void igb_stop_adapter(igb_t *);
49 static int igb_reset(igb_t *);
50 static void igb_tx_clean(igb_t *);
51 static boolean_t igb_tx_drain(igb_t *);
52 static boolean_t igb_rx_drain(igb_t *);
53 static int igb_alloc_rings(igb_t *);
54 static int igb_alloc_rx_data(igb_t *);
55 static void igb_free_rx_data(igb_t *);
56 static void igb_free_rings(igb_t *);
57 static void igb_setup_rings(igb_t *);
58 static void igb_setup_rx(igb_t *);
59 static void igb_setup_tx(igb_t *);
60 static void igb_setup_rx_ring(igb_rx_ring_t *);
61 static void igb_setup_tx_ring(igb_tx_ring_t *);
62 static void igb_setup_rss(igb_t *);
63 static void igb_setup_mac_rss_classify(igb_t *);
64 static void igb_setup_mac_classify(igb_t *);
65 static void igb_init_unicst(igb_t *);
66 static void igb_setup_multicst(igb_t *);
67 static void igb_get_phy_state(igb_t *);
68 static void igb_param_sync(igb_t *);
69 static void igb_get_conf(igb_t *);
70 static int igb_get_prop(igb_t *, char *, int, int, int);
71 static boolean_t igb_is_link_up(igb_t *);
72 static boolean_t igb_link_check(igb_t *);
73 static void igb_local_timer(void *);
74 static void igb_link_timer(void *);
75 static void igb_arm_watchdog_timer(igb_t *);
76 static void igb_start_watchdog_timer(igb_t *);
77 static void igb_restart_watchdog_timer(igb_t *);
78 static void igb_stop_watchdog_timer(igb_t *);
79 static void igb_start_link_timer(igb_t *);
80 static void igb_stop_link_timer(igb_t *);
81 static void igb_disable_adapter_interrupts(igb_t *);
82 static void igb_enable_adapter_interrupts_82575(igb_t *);
83 static void igb_enable_adapter_interrupts_82576(igb_t *);
84 static void igb_enable_adapter_interrupts_82580(igb_t *);
85 static boolean_t is_valid_mac_addr(uint8_t *);
86 static boolean_t igb_stall_check(igb_t *);
87 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t);
88 static void igb_set_external_loopback(igb_t *);
89 static void igb_set_internal_phy_loopback(igb_t *);
90 static void igb_set_internal_serdes_loopback(igb_t *);
91 static boolean_t igb_find_mac_address(igb_t *);
92 static int igb_alloc_intrs(igb_t *);
93 static int igb_alloc_intr_handles(igb_t *, int);
94 static int igb_add_intr_handlers(igb_t *);
95 static void igb_rem_intr_handlers(igb_t *);
96 static void igb_rem_intrs(igb_t *);
97 static int igb_enable_intrs(igb_t *);
98 static int igb_disable_intrs(igb_t *);
99 static void igb_setup_msix_82575(igb_t *);
100 static void igb_setup_msix_82576(igb_t *);
101 static void igb_setup_msix_82580(igb_t *);
102 static uint_t igb_intr_legacy(void *, void *);
103 static uint_t igb_intr_msi(void *, void *);
104 static uint_t igb_intr_rx(void *, void *);
105 static uint_t igb_intr_tx(void *, void *);
106 static uint_t igb_intr_tx_other(void *, void *);
107 static void igb_intr_rx_work(igb_rx_ring_t *);
108 static void igb_intr_tx_work(igb_tx_ring_t *);
109 static void igb_intr_link_work(igb_t *);
110 static void igb_get_driver_control(struct e1000_hw *);
111 static void igb_release_driver_control(struct e1000_hw *);
112
113 static int igb_attach(dev_info_t *, ddi_attach_cmd_t);
114 static int igb_detach(dev_info_t *, ddi_detach_cmd_t);
115 static int igb_resume(dev_info_t *);
116 static int igb_suspend(dev_info_t *);
117 static int igb_quiesce(dev_info_t *);
118 static void igb_unconfigure(dev_info_t *, igb_t *);
119 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *,
120 const void *);
121 static void igb_fm_init(igb_t *);
122 static void igb_fm_fini(igb_t *);
123 static void igb_release_multicast(igb_t *);
124
125 char *igb_priv_props[] = {
126 "_tx_copy_thresh",
127 "_tx_recycle_thresh",
128 "_tx_overload_thresh",
129 "_tx_resched_thresh",
130 "_rx_copy_thresh",
131 "_rx_limit_per_intr",
132 "_intr_throttling",
133 "_adv_pause_cap",
134 "_adv_asym_pause_cap",
135 NULL
136 };
137
138 static struct cb_ops igb_cb_ops = {
139 nulldev, /* cb_open */
140 nulldev, /* cb_close */
141 nodev, /* cb_strategy */
142 nodev, /* cb_print */
143 nodev, /* cb_dump */
144 nodev, /* cb_read */
145 nodev, /* cb_write */
146 nodev, /* cb_ioctl */
147 nodev, /* cb_devmap */
148 nodev, /* cb_mmap */
149 nodev, /* cb_segmap */
150 nochpoll, /* cb_chpoll */
151 ddi_prop_op, /* cb_prop_op */
152 NULL, /* cb_stream */
153 D_MP | D_HOTPLUG, /* cb_flag */
154 CB_REV, /* cb_rev */
155 nodev, /* cb_aread */
156 nodev /* cb_awrite */
157 };
158
159 static struct dev_ops igb_dev_ops = {
160 DEVO_REV, /* devo_rev */
161 0, /* devo_refcnt */
162 NULL, /* devo_getinfo */
163 nulldev, /* devo_identify */
164 nulldev, /* devo_probe */
165 igb_attach, /* devo_attach */
166 igb_detach, /* devo_detach */
167 nodev, /* devo_reset */
168 &igb_cb_ops, /* devo_cb_ops */
169 NULL, /* devo_bus_ops */
170 ddi_power, /* devo_power */
171 igb_quiesce, /* devo_quiesce */
172 };
173
174 static struct modldrv igb_modldrv = {
175 &mod_driverops, /* Type of module. This one is a driver */
176 ident, /* Discription string */
177 &igb_dev_ops, /* driver ops */
178 };
179
180 static struct modlinkage igb_modlinkage = {
181 MODREV_1, &igb_modldrv, NULL
182 };
183
184 /* Access attributes for register mapping */
185 ddi_device_acc_attr_t igb_regs_acc_attr = {
186 DDI_DEVICE_ATTR_V1,
187 DDI_STRUCTURE_LE_ACC,
188 DDI_STRICTORDER_ACC,
189 DDI_FLAGERR_ACC
190 };
191
192 #define IGB_M_CALLBACK_FLAGS \
193 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
194
195 static mac_callbacks_t igb_m_callbacks = {
196 IGB_M_CALLBACK_FLAGS,
197 igb_m_stat,
198 igb_m_start,
199 igb_m_stop,
200 igb_m_promisc,
201 igb_m_multicst,
202 NULL,
203 NULL,
204 NULL,
205 igb_m_ioctl,
206 igb_m_getcapab,
207 NULL,
208 NULL,
209 igb_m_setprop,
210 igb_m_getprop,
211 igb_m_propinfo
212 };
213
214 /*
215 * Initialize capabilities of each supported adapter type
216 */
217 static adapter_info_t igb_82575_cap = {
218 /* limits */
219 4, /* maximum number of rx queues */
220 1, /* minimum number of rx queues */
221 4, /* default number of rx queues */
222 4, /* maximum number of tx queues */
223 1, /* minimum number of tx queues */
224 4, /* default number of tx queues */
225 65535, /* maximum interrupt throttle rate */
226 0, /* minimum interrupt throttle rate */
227 200, /* default interrupt throttle rate */
228
229 /* function pointers */
230 igb_enable_adapter_interrupts_82575,
231 igb_setup_msix_82575,
232
233 /* capabilities */
234 (IGB_FLAG_HAS_DCA | /* capability flags */
235 IGB_FLAG_VMDQ_POOL),
236
237 0xffc00000 /* mask for RXDCTL register */
238 };
239
240 static adapter_info_t igb_82576_cap = {
241 /* limits */
242 16, /* maximum number of rx queues */
243 1, /* minimum number of rx queues */
244 4, /* default number of rx queues */
245 16, /* maximum number of tx queues */
246 1, /* minimum number of tx queues */
247 4, /* default number of tx queues */
248 65535, /* maximum interrupt throttle rate */
249 0, /* minimum interrupt throttle rate */
250 200, /* default interrupt throttle rate */
251
252 /* function pointers */
253 igb_enable_adapter_interrupts_82576,
254 igb_setup_msix_82576,
255
256 /* capabilities */
257 (IGB_FLAG_HAS_DCA | /* capability flags */
258 IGB_FLAG_VMDQ_POOL |
259 IGB_FLAG_NEED_CTX_IDX),
260
261 0xffe00000 /* mask for RXDCTL register */
262 };
263
264 static adapter_info_t igb_82580_cap = {
265 /* limits */
266 8, /* maximum number of rx queues */
267 1, /* minimum number of rx queues */
268 4, /* default number of rx queues */
269 8, /* maximum number of tx queues */
270 1, /* minimum number of tx queues */
271 4, /* default number of tx queues */
272 65535, /* maximum interrupt throttle rate */
273 0, /* minimum interrupt throttle rate */
274 200, /* default interrupt throttle rate */
275
276 /* function pointers */
277 igb_enable_adapter_interrupts_82580,
278 igb_setup_msix_82580,
279
280 /* capabilities */
281 (IGB_FLAG_HAS_DCA | /* capability flags */
282 IGB_FLAG_VMDQ_POOL |
283 IGB_FLAG_NEED_CTX_IDX),
284
285 0xffe00000 /* mask for RXDCTL register */
286 };
287
288 static adapter_info_t igb_i350_cap = {
289 /* limits */
290 8, /* maximum number of rx queues */
291 1, /* minimum number of rx queues */
292 4, /* default number of rx queues */
293 8, /* maximum number of tx queues */
294 1, /* minimum number of tx queues */
295 4, /* default number of tx queues */
296 65535, /* maximum interrupt throttle rate */
297 0, /* minimum interrupt throttle rate */
298 200, /* default interrupt throttle rate */
299
300 /* function pointers */
301 igb_enable_adapter_interrupts_82580,
302 igb_setup_msix_82580,
303
304 /* capabilities */
305 (IGB_FLAG_HAS_DCA | /* capability flags */
306 IGB_FLAG_VMDQ_POOL |
307 IGB_FLAG_NEED_CTX_IDX),
308
309 0xffe00000 /* mask for RXDCTL register */
310 };
311
312 /*
313 * Module Initialization Functions
314 */
315
316 int
317 _init(void)
318 {
319 int status;
320
321 mac_init_ops(&igb_dev_ops, MODULE_NAME);
322
323 status = mod_install(&igb_modlinkage);
324
325 if (status != DDI_SUCCESS) {
326 mac_fini_ops(&igb_dev_ops);
327 }
328
329 return (status);
330 }
331
332 int
333 _fini(void)
334 {
335 int status;
336
337 status = mod_remove(&igb_modlinkage);
338
339 if (status == DDI_SUCCESS) {
340 mac_fini_ops(&igb_dev_ops);
341 }
342
343 return (status);
344
345 }
346
347 int
348 _info(struct modinfo *modinfop)
349 {
350 int status;
351
352 status = mod_info(&igb_modlinkage, modinfop);
353
354 return (status);
355 }
356
357 /*
358 * igb_attach - driver attach
359 *
360 * This function is the device specific initialization entry
361 * point. This entry point is required and must be written.
362 * The DDI_ATTACH command must be provided in the attach entry
363 * point. When attach() is called with cmd set to DDI_ATTACH,
364 * all normal kernel services (such as kmem_alloc(9F)) are
365 * available for use by the driver.
366 *
367 * The attach() function will be called once for each instance
368 * of the device on the system with cmd set to DDI_ATTACH.
369 * Until attach() succeeds, the only driver entry points which
370 * may be called are open(9E) and getinfo(9E).
371 */
372 static int
373 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
374 {
375 igb_t *igb;
376 struct igb_osdep *osdep;
377 struct e1000_hw *hw;
378 int instance;
379
380 /*
381 * Check the command and perform corresponding operations
382 */
383 switch (cmd) {
384 default:
385 return (DDI_FAILURE);
386
387 case DDI_RESUME:
388 return (igb_resume(devinfo));
389
390 case DDI_ATTACH:
391 break;
392 }
393
394 /* Get the device instance */
395 instance = ddi_get_instance(devinfo);
396
397 /* Allocate memory for the instance data structure */
398 igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP);
399
400 igb->dip = devinfo;
401 igb->instance = instance;
402
403 hw = &igb->hw;
404 osdep = &igb->osdep;
405 hw->back = osdep;
406 osdep->igb = igb;
407
408 /* Attach the instance pointer to the dev_info data structure */
409 ddi_set_driver_private(devinfo, igb);
410
411
412 /* Initialize for fma support */
413 igb->fm_capabilities = igb_get_prop(igb, "fm-capable",
414 0, 0x0f,
415 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
416 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
417 igb_fm_init(igb);
418 igb->attach_progress |= ATTACH_PROGRESS_FMINIT;
419
420 /*
421 * Map PCI config space registers
422 */
423 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
424 igb_error(igb, "Failed to map PCI configurations");
425 goto attach_fail;
426 }
427 igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
428
429 /*
430 * Identify the chipset family
431 */
432 if (igb_identify_hardware(igb) != IGB_SUCCESS) {
433 igb_error(igb, "Failed to identify hardware");
434 goto attach_fail;
435 }
436
437 /*
438 * Map device registers
439 */
440 if (igb_regs_map(igb) != IGB_SUCCESS) {
441 igb_error(igb, "Failed to map device registers");
442 goto attach_fail;
443 }
444 igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
445
446 /*
447 * Initialize driver parameters
448 */
449 igb_init_properties(igb);
450 igb->attach_progress |= ATTACH_PROGRESS_PROPS;
451
452 /*
453 * Allocate interrupts
454 */
455 if (igb_alloc_intrs(igb) != IGB_SUCCESS) {
456 igb_error(igb, "Failed to allocate interrupts");
457 goto attach_fail;
458 }
459 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
460
461 /*
462 * Allocate rx/tx rings based on the ring numbers.
463 * The actual numbers of rx/tx rings are decided by the number of
464 * allocated interrupt vectors, so we should allocate the rings after
465 * interrupts are allocated.
466 */
467 if (igb_alloc_rings(igb) != IGB_SUCCESS) {
468 igb_error(igb, "Failed to allocate rx/tx rings or groups");
469 goto attach_fail;
470 }
471 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
472
473 /*
474 * Add interrupt handlers
475 */
476 if (igb_add_intr_handlers(igb) != IGB_SUCCESS) {
477 igb_error(igb, "Failed to add interrupt handlers");
478 goto attach_fail;
479 }
480 igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
481
482 /*
483 * Initialize driver parameters
484 */
485 if (igb_init_driver_settings(igb) != IGB_SUCCESS) {
486 igb_error(igb, "Failed to initialize driver settings");
487 goto attach_fail;
488 }
489
490 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) {
491 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
492 goto attach_fail;
493 }
494
495 /*
496 * Initialize mutexes for this device.
497 * Do this before enabling the interrupt handler and
498 * register the softint to avoid the condition where
499 * interrupt handler can try using uninitialized mutex
500 */
501 igb_init_locks(igb);
502 igb->attach_progress |= ATTACH_PROGRESS_LOCKS;
503
504 /*
505 * Initialize the adapter
506 */
507 if (igb_init(igb) != IGB_SUCCESS) {
508 igb_error(igb, "Failed to initialize adapter");
509 goto attach_fail;
510 }
511 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
512
513 /*
514 * Initialize statistics
515 */
516 if (igb_init_stats(igb) != IGB_SUCCESS) {
517 igb_error(igb, "Failed to initialize statistics");
518 goto attach_fail;
519 }
520 igb->attach_progress |= ATTACH_PROGRESS_STATS;
521
522 /*
523 * Register the driver to the MAC
524 */
525 if (igb_register_mac(igb) != IGB_SUCCESS) {
526 igb_error(igb, "Failed to register MAC");
527 goto attach_fail;
528 }
529 igb->attach_progress |= ATTACH_PROGRESS_MAC;
530
531 /*
532 * Now that mutex locks are initialized, and the chip is also
533 * initialized, enable interrupts.
534 */
535 if (igb_enable_intrs(igb) != IGB_SUCCESS) {
536 igb_error(igb, "Failed to enable DDI interrupts");
537 goto attach_fail;
538 }
539 igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
540
541 igb_log(igb, "%s", igb_version);
542 atomic_or_32(&igb->igb_state, IGB_INITIALIZED);
543
544 /*
545 * Newer models have Energy Efficient Ethernet, let's disable this by
546 * default.
547 */
548 if (igb->hw.mac.type == e1000_i350)
549 (void) e1000_set_eee_i350(&igb->hw);
550
551 return (DDI_SUCCESS);
552
553 attach_fail:
554 igb_unconfigure(devinfo, igb);
555 return (DDI_FAILURE);
556 }
557
558 /*
559 * igb_detach - driver detach
560 *
561 * The detach() function is the complement of the attach routine.
562 * If cmd is set to DDI_DETACH, detach() is used to remove the
563 * state associated with a given instance of a device node
564 * prior to the removal of that instance from the system.
565 *
566 * The detach() function will be called once for each instance
567 * of the device for which there has been a successful attach()
568 * once there are no longer any opens on the device.
569 *
570 * Interrupts routine are disabled, All memory allocated by this
571 * driver are freed.
572 */
573 static int
574 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
575 {
576 igb_t *igb;
577
578 /*
579 * Check detach command
580 */
581 switch (cmd) {
582 default:
583 return (DDI_FAILURE);
584
585 case DDI_SUSPEND:
586 return (igb_suspend(devinfo));
587
588 case DDI_DETACH:
589 break;
590 }
591
592
593 /*
594 * Get the pointer to the driver private data structure
595 */
596 igb = (igb_t *)ddi_get_driver_private(devinfo);
597 if (igb == NULL)
598 return (DDI_FAILURE);
599
600 /*
601 * Unregister MAC. If failed, we have to fail the detach
602 */
603 if (mac_unregister(igb->mac_hdl) != 0) {
604 igb_error(igb, "Failed to unregister MAC");
605 return (DDI_FAILURE);
606 }
607 igb->attach_progress &= ~ATTACH_PROGRESS_MAC;
608
609 /*
610 * If the device is still running, it needs to be stopped first.
611 * This check is necessary because under some specific circumstances,
612 * the detach routine can be called without stopping the interface
613 * first.
614 */
615 mutex_enter(&igb->gen_lock);
616 if (igb->igb_state & IGB_STARTED) {
617 atomic_and_32(&igb->igb_state, ~IGB_STARTED);
618 igb_stop(igb, B_TRUE);
619 mutex_exit(&igb->gen_lock);
620 /* Disable and stop the watchdog timer */
621 igb_disable_watchdog_timer(igb);
622 } else
623 mutex_exit(&igb->gen_lock);
624
625 /*
626 * Check if there are still rx buffers held by the upper layer.
627 * If so, fail the detach.
628 */
629 if (!igb_rx_drain(igb))
630 return (DDI_FAILURE);
631
632 /*
633 * Do the remaining unconfigure routines
634 */
635 igb_unconfigure(devinfo, igb);
636
637 return (DDI_SUCCESS);
638 }
639
640 /*
641 * quiesce(9E) entry point.
642 *
643 * This function is called when the system is single-threaded at high
644 * PIL with preemption disabled. Therefore, this function must not be
645 * blocked.
646 *
647 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
648 * DDI_FAILURE indicates an error condition and should almost never happen.
649 */
650 static int
651 igb_quiesce(dev_info_t *devinfo)
652 {
653 igb_t *igb;
654 struct e1000_hw *hw;
655
656 igb = (igb_t *)ddi_get_driver_private(devinfo);
657
658 if (igb == NULL)
659 return (DDI_FAILURE);
660
661 hw = &igb->hw;
662
663 /*
664 * Disable the adapter interrupts
665 */
666 igb_disable_adapter_interrupts(igb);
667
668 /* Tell firmware driver is no longer in control */
669 igb_release_driver_control(hw);
670
671 /*
672 * Reset the chipset
673 */
674 (void) e1000_reset_hw(hw);
675
676 /*
677 * Reset PHY if possible
678 */
679 if (e1000_check_reset_block(hw) == E1000_SUCCESS)
680 (void) e1000_phy_hw_reset(hw);
681
682 return (DDI_SUCCESS);
683 }
684
685 /*
686 * igb_unconfigure - release all resources held by this instance
687 */
688 static void
689 igb_unconfigure(dev_info_t *devinfo, igb_t *igb)
690 {
691 /*
692 * Disable interrupt
693 */
694 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
695 (void) igb_disable_intrs(igb);
696 }
697
698 /*
699 * Unregister MAC
700 */
701 if (igb->attach_progress & ATTACH_PROGRESS_MAC) {
702 (void) mac_unregister(igb->mac_hdl);
703 }
704
705 /*
706 * Free statistics
707 */
708 if (igb->attach_progress & ATTACH_PROGRESS_STATS) {
709 kstat_delete((kstat_t *)igb->igb_ks);
710 }
711
712 /*
713 * Remove interrupt handlers
714 */
715 if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
716 igb_rem_intr_handlers(igb);
717 }
718
719 /*
720 * Remove interrupts
721 */
722 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
723 igb_rem_intrs(igb);
724 }
725
726 /*
727 * Remove driver properties
728 */
729 if (igb->attach_progress & ATTACH_PROGRESS_PROPS) {
730 (void) ddi_prop_remove_all(devinfo);
731 }
732
733 /*
734 * Stop the adapter
735 */
736 if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) {
737 mutex_enter(&igb->gen_lock);
738 igb_stop_adapter(igb);
739 mutex_exit(&igb->gen_lock);
740 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
741 ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED);
742 }
743
744 /*
745 * Free multicast table
746 */
747 igb_release_multicast(igb);
748
749 /*
750 * Free register handle
751 */
752 if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
753 if (igb->osdep.reg_handle != NULL)
754 ddi_regs_map_free(&igb->osdep.reg_handle);
755 }
756
757 /*
758 * Free PCI config handle
759 */
760 if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
761 if (igb->osdep.cfg_handle != NULL)
762 pci_config_teardown(&igb->osdep.cfg_handle);
763 }
764
765 /*
766 * Free locks
767 */
768 if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) {
769 igb_destroy_locks(igb);
770 }
771
772 /*
773 * Free the rx/tx rings
774 */
775 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
776 igb_free_rings(igb);
777 }
778
779 /*
780 * Remove FMA
781 */
782 if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) {
783 igb_fm_fini(igb);
784 }
785
786 /*
787 * Free the driver data structure
788 */
789 kmem_free(igb, sizeof (igb_t));
790
791 ddi_set_driver_private(devinfo, NULL);
792 }
793
794 /*
795 * igb_register_mac - Register the driver and its function pointers with
796 * the GLD interface
797 */
798 static int
799 igb_register_mac(igb_t *igb)
800 {
801 struct e1000_hw *hw = &igb->hw;
802 mac_register_t *mac;
803 int status;
804
805 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
806 return (IGB_FAILURE);
807
808 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
809 mac->m_driver = igb;
810 mac->m_dip = igb->dip;
811 mac->m_src_addr = hw->mac.addr;
812 mac->m_callbacks = &igb_m_callbacks;
813 mac->m_min_sdu = 0;
814 mac->m_max_sdu = igb->max_frame_size -
815 sizeof (struct ether_vlan_header) - ETHERFCSL;
816 mac->m_margin = VLAN_TAGSZ;
817 mac->m_priv_props = igb_priv_props;
818 mac->m_v12n = MAC_VIRT_LEVEL1;
819
820 status = mac_register(mac, &igb->mac_hdl);
821
822 mac_free(mac);
823
824 return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE);
825 }
826
827 /*
828 * igb_identify_hardware - Identify the type of the chipset
829 */
830 static int
831 igb_identify_hardware(igb_t *igb)
832 {
833 struct e1000_hw *hw = &igb->hw;
834 struct igb_osdep *osdep = &igb->osdep;
835
836 /*
837 * Get the device id
838 */
839 hw->vendor_id =
840 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
841 hw->device_id =
842 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
843 hw->revision_id =
844 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
845 hw->subsystem_device_id =
846 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
847 hw->subsystem_vendor_id =
848 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
849
850 /*
851 * Set the mac type of the adapter based on the device id
852 */
853 if (e1000_set_mac_type(hw) != E1000_SUCCESS) {
854 return (IGB_FAILURE);
855 }
856
857 /*
858 * Install adapter capabilities based on mac type
859 */
860 switch (hw->mac.type) {
861 case e1000_82575:
862 igb->capab = &igb_82575_cap;
863 break;
864 case e1000_82576:
865 igb->capab = &igb_82576_cap;
866 break;
867 case e1000_82580:
868 igb->capab = &igb_82580_cap;
869 break;
870 case e1000_i350:
871 igb->capab = &igb_i350_cap;
872 break;
873 default:
874 return (IGB_FAILURE);
875 }
876
877 return (IGB_SUCCESS);
878 }
879
880 /*
881 * igb_regs_map - Map the device registers
882 */
883 static int
884 igb_regs_map(igb_t *igb)
885 {
886 dev_info_t *devinfo = igb->dip;
887 struct e1000_hw *hw = &igb->hw;
888 struct igb_osdep *osdep = &igb->osdep;
889 off_t mem_size;
890
891 /*
892 * First get the size of device registers to be mapped.
893 */
894 if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) !=
895 DDI_SUCCESS) {
896 return (IGB_FAILURE);
897 }
898
899 /*
900 * Call ddi_regs_map_setup() to map registers
901 */
902 if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET,
903 (caddr_t *)&hw->hw_addr, 0,
904 mem_size, &igb_regs_acc_attr,
905 &osdep->reg_handle)) != DDI_SUCCESS) {
906 return (IGB_FAILURE);
907 }
908
909 return (IGB_SUCCESS);
910 }
911
912 /*
913 * igb_init_properties - Initialize driver properties
914 */
915 static void
916 igb_init_properties(igb_t *igb)
917 {
918 /*
919 * Get conf file properties, including link settings
920 * jumbo frames, ring number, descriptor number, etc.
921 */
922 igb_get_conf(igb);
923 }
924
925 /*
926 * igb_init_driver_settings - Initialize driver settings
927 *
928 * The settings include hardware function pointers, bus information,
929 * rx/tx rings settings, link state, and any other parameters that
930 * need to be setup during driver initialization.
931 */
932 static int
933 igb_init_driver_settings(igb_t *igb)
934 {
935 struct e1000_hw *hw = &igb->hw;
936 igb_rx_ring_t *rx_ring;
937 igb_tx_ring_t *tx_ring;
938 uint32_t rx_size;
939 uint32_t tx_size;
940 int i;
941
942 /*
943 * Initialize chipset specific hardware function pointers
944 */
945 if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) {
946 return (IGB_FAILURE);
947 }
948
949 /*
950 * Get bus information
951 */
952 if (e1000_get_bus_info(hw) != E1000_SUCCESS) {
953 return (IGB_FAILURE);
954 }
955
956 /*
957 * Get the system page size
958 */
959 igb->page_size = ddi_ptob(igb->dip, (ulong_t)1);
960
961 /*
962 * Set rx buffer size
963 * The IP header alignment room is counted in the calculation.
964 * The rx buffer size is in unit of 1K that is required by the
965 * chipset hardware.
966 */
967 rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM;
968 igb->rx_buf_size = ((rx_size >> 10) +
969 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
970
971 /*
972 * Set tx buffer size
973 */
974 tx_size = igb->max_frame_size;
975 igb->tx_buf_size = ((tx_size >> 10) +
976 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
977
978 /*
979 * Initialize rx/tx rings parameters
980 */
981 for (i = 0; i < igb->num_rx_rings; i++) {
982 rx_ring = &igb->rx_rings[i];
983 rx_ring->index = i;
984 rx_ring->igb = igb;
985 }
986
987 for (i = 0; i < igb->num_tx_rings; i++) {
988 tx_ring = &igb->tx_rings[i];
989 tx_ring->index = i;
990 tx_ring->igb = igb;
991 if (igb->tx_head_wb_enable)
992 tx_ring->tx_recycle = igb_tx_recycle_head_wb;
993 else
994 tx_ring->tx_recycle = igb_tx_recycle_legacy;
995
996 tx_ring->ring_size = igb->tx_ring_size;
997 tx_ring->free_list_size = igb->tx_ring_size +
998 (igb->tx_ring_size >> 1);
999 }
1000
1001 /*
1002 * Initialize values of interrupt throttling rates
1003 */
1004 for (i = 1; i < MAX_NUM_EITR; i++)
1005 igb->intr_throttling[i] = igb->intr_throttling[0];
1006
1007 /*
1008 * The initial link state should be "unknown"
1009 */
1010 igb->link_state = LINK_STATE_UNKNOWN;
1011
1012 return (IGB_SUCCESS);
1013 }
1014
1015 /*
1016 * igb_init_locks - Initialize locks
1017 */
1018 static void
1019 igb_init_locks(igb_t *igb)
1020 {
1021 igb_rx_ring_t *rx_ring;
1022 igb_tx_ring_t *tx_ring;
1023 int i;
1024
1025 for (i = 0; i < igb->num_rx_rings; i++) {
1026 rx_ring = &igb->rx_rings[i];
1027 mutex_init(&rx_ring->rx_lock, NULL,
1028 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1029 }
1030
1031 for (i = 0; i < igb->num_tx_rings; i++) {
1032 tx_ring = &igb->tx_rings[i];
1033 mutex_init(&tx_ring->tx_lock, NULL,
1034 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1035 mutex_init(&tx_ring->recycle_lock, NULL,
1036 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1037 mutex_init(&tx_ring->tcb_head_lock, NULL,
1038 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1039 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1040 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1041 }
1042
1043 mutex_init(&igb->gen_lock, NULL,
1044 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1045
1046 mutex_init(&igb->watchdog_lock, NULL,
1047 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1048
1049 mutex_init(&igb->link_lock, NULL,
1050 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1051 }
1052
1053 /*
1054 * igb_destroy_locks - Destroy locks
1055 */
1056 static void
1057 igb_destroy_locks(igb_t *igb)
1058 {
1059 igb_rx_ring_t *rx_ring;
1060 igb_tx_ring_t *tx_ring;
1061 int i;
1062
1063 for (i = 0; i < igb->num_rx_rings; i++) {
1064 rx_ring = &igb->rx_rings[i];
1065 mutex_destroy(&rx_ring->rx_lock);
1066 }
1067
1068 for (i = 0; i < igb->num_tx_rings; i++) {
1069 tx_ring = &igb->tx_rings[i];
1070 mutex_destroy(&tx_ring->tx_lock);
1071 mutex_destroy(&tx_ring->recycle_lock);
1072 mutex_destroy(&tx_ring->tcb_head_lock);
1073 mutex_destroy(&tx_ring->tcb_tail_lock);
1074 }
1075
1076 mutex_destroy(&igb->gen_lock);
1077 mutex_destroy(&igb->watchdog_lock);
1078 mutex_destroy(&igb->link_lock);
1079 }
1080
1081 static int
1082 igb_resume(dev_info_t *devinfo)
1083 {
1084 igb_t *igb;
1085
1086 igb = (igb_t *)ddi_get_driver_private(devinfo);
1087 if (igb == NULL)
1088 return (DDI_FAILURE);
1089
1090 mutex_enter(&igb->gen_lock);
1091
1092 /*
1093 * Enable interrupts
1094 */
1095 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1096 if (igb_enable_intrs(igb) != IGB_SUCCESS) {
1097 igb_error(igb, "Failed to enable DDI interrupts");
1098 mutex_exit(&igb->gen_lock);
1099 return (DDI_FAILURE);
1100 }
1101 }
1102
1103 if (igb->igb_state & IGB_STARTED) {
1104 if (igb_start(igb, B_FALSE) != IGB_SUCCESS) {
1105 mutex_exit(&igb->gen_lock);
1106 return (DDI_FAILURE);
1107 }
1108
1109 /*
1110 * Enable and start the watchdog timer
1111 */
1112 igb_enable_watchdog_timer(igb);
1113 }
1114
1115 atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED);
1116
1117 mutex_exit(&igb->gen_lock);
1118
1119 return (DDI_SUCCESS);
1120 }
1121
1122 static int
1123 igb_suspend(dev_info_t *devinfo)
1124 {
1125 igb_t *igb;
1126
1127 igb = (igb_t *)ddi_get_driver_private(devinfo);
1128 if (igb == NULL)
1129 return (DDI_FAILURE);
1130
1131 mutex_enter(&igb->gen_lock);
1132
1133 atomic_or_32(&igb->igb_state, IGB_SUSPENDED);
1134
1135 /*
1136 * Disable interrupts
1137 */
1138 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1139 (void) igb_disable_intrs(igb);
1140 }
1141
1142 if (!(igb->igb_state & IGB_STARTED)) {
1143 mutex_exit(&igb->gen_lock);
1144 return (DDI_SUCCESS);
1145 }
1146
1147 igb_stop(igb, B_FALSE);
1148
1149 mutex_exit(&igb->gen_lock);
1150
1151 /*
1152 * Disable and stop the watchdog timer
1153 */
1154 igb_disable_watchdog_timer(igb);
1155
1156 return (DDI_SUCCESS);
1157 }
1158
1159 static int
1160 igb_init(igb_t *igb)
1161 {
1162 mutex_enter(&igb->gen_lock);
1163
1164 /*
1165 * Initilize the adapter
1166 */
1167 if (igb_init_adapter(igb) != IGB_SUCCESS) {
1168 mutex_exit(&igb->gen_lock);
1169 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1170 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1171 return (IGB_FAILURE);
1172 }
1173
1174 mutex_exit(&igb->gen_lock);
1175
1176 return (IGB_SUCCESS);
1177 }
1178
1179 /*
1180 * igb_init_mac_address - Initialize the default MAC address
1181 *
1182 * On success, the MAC address is entered in the igb->hw.mac.addr
1183 * and hw->mac.perm_addr fields and the adapter's RAR(0) receive
1184 * address register.
1185 *
1186 * Important side effects:
1187 * 1. adapter is reset - this is required to put it in a known state.
1188 * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where
1189 * MAC address and all default settings are stored, so a valid checksum
1190 * is required.
1191 */
1192 static int
1193 igb_init_mac_address(igb_t *igb)
1194 {
1195 struct e1000_hw *hw = &igb->hw;
1196
1197 ASSERT(mutex_owned(&igb->gen_lock));
1198
1199 /*
1200 * Reset chipset to put the hardware in a known state
1201 * before we try to get MAC address from NVM.
1202 */
1203 if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1204 igb_error(igb, "Adapter reset failed.");
1205 goto init_mac_fail;
1206 }
1207
1208 /*
1209 * NVM validation
1210 */
1211 if (e1000_validate_nvm_checksum(hw) < 0) {
1212 /*
1213 * Some PCI-E parts fail the first check due to
1214 * the link being in sleep state. Call it again,
1215 * if it fails a second time its a real issue.
1216 */
1217 if (e1000_validate_nvm_checksum(hw) < 0) {
1218 igb_error(igb,
1219 "Invalid NVM checksum. Please contact "
1220 "the vendor to update the NVM.");
1221 goto init_mac_fail;
1222 }
1223 }
1224
1225 /*
1226 * Get the mac address
1227 * This function should handle SPARC case correctly.
1228 */
1229 if (!igb_find_mac_address(igb)) {
1230 igb_error(igb, "Failed to get the mac address");
1231 goto init_mac_fail;
1232 }
1233
1234 /* Validate mac address */
1235 if (!is_valid_mac_addr(hw->mac.addr)) {
1236 igb_error(igb, "Invalid mac address");
1237 goto init_mac_fail;
1238 }
1239
1240 return (IGB_SUCCESS);
1241
1242 init_mac_fail:
1243 return (IGB_FAILURE);
1244 }
1245
1246 /*
1247 * igb_init_adapter - Initialize the adapter
1248 */
1249 static int
1250 igb_init_adapter(igb_t *igb)
1251 {
1252 struct e1000_hw *hw = &igb->hw;
1253 uint32_t pba;
1254 uint32_t high_water;
1255 int i;
1256
1257 ASSERT(mutex_owned(&igb->gen_lock));
1258
1259 /*
1260 * In order to obtain the default MAC address, this will reset the
1261 * adapter and validate the NVM that the address and many other
1262 * default settings come from.
1263 */
1264 if (igb_init_mac_address(igb) != IGB_SUCCESS) {
1265 igb_error(igb, "Failed to initialize MAC address");
1266 goto init_adapter_fail;
1267 }
1268
1269 /*
1270 * Setup flow control
1271 *
1272 * These parameters set thresholds for the adapter's generation(Tx)
1273 * and response(Rx) to Ethernet PAUSE frames. These are just threshold
1274 * settings. Flow control is enabled or disabled in the configuration
1275 * file.
1276 * High-water mark is set down from the top of the rx fifo (not
1277 * sensitive to max_frame_size) and low-water is set just below
1278 * high-water mark.
1279 * The high water mark must be low enough to fit one full frame above
1280 * it in the rx FIFO. Should be the lower of:
1281 * 90% of the Rx FIFO size, or the full Rx FIFO size minus one full
1282 * frame.
1283 */
1284 /*
1285 * The default setting of PBA is correct for 82575 and other supported
1286 * adapters do not have the E1000_PBA register, so PBA value is only
1287 * used for calculation here and is never written to the adapter.
1288 */
1289 if (hw->mac.type == e1000_82575) {
1290 pba = E1000_PBA_34K;
1291 } else {
1292 pba = E1000_PBA_64K;
1293 }
1294
1295 high_water = min(((pba << 10) * 9 / 10),
1296 ((pba << 10) - igb->max_frame_size));
1297
1298 if (hw->mac.type == e1000_82575) {
1299 /* 8-byte granularity */
1300 hw->fc.high_water = high_water & 0xFFF8;
1301 hw->fc.low_water = hw->fc.high_water - 8;
1302 } else {
1303 /* 16-byte granularity */
1304 hw->fc.high_water = high_water & 0xFFF0;
1305 hw->fc.low_water = hw->fc.high_water - 16;
1306 }
1307
1308 hw->fc.pause_time = E1000_FC_PAUSE_TIME;
1309 hw->fc.send_xon = B_TRUE;
1310
1311 (void) e1000_validate_mdi_setting(hw);
1312
1313 /*
1314 * Reset the chipset hardware the second time to put PBA settings
1315 * into effect.
1316 */
1317 if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1318 igb_error(igb, "Second reset failed");
1319 goto init_adapter_fail;
1320 }
1321
1322 /*
1323 * Don't wait for auto-negotiation to complete
1324 */
1325 hw->phy.autoneg_wait_to_complete = B_FALSE;
1326
1327 /*
1328 * Copper options
1329 */
1330 if (hw->phy.media_type == e1000_media_type_copper) {
1331 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1332 hw->phy.disable_polarity_correction = B_FALSE;
1333 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */
1334 }
1335
1336 /*
1337 * Initialize link settings
1338 */
1339 (void) igb_setup_link(igb, B_FALSE);
1340
1341 /*
1342 * Configure/Initialize hardware
1343 */
1344 if (e1000_init_hw(hw) != E1000_SUCCESS) {
1345 igb_error(igb, "Failed to initialize hardware");
1346 goto init_adapter_fail;
1347 }
1348
1349 /*
1350 * Start the link setup timer
1351 */
1352 igb_start_link_timer(igb);
1353
1354 /*
1355 * Disable wakeup control by default
1356 */
1357 E1000_WRITE_REG(hw, E1000_WUC, 0);
1358
1359 /*
1360 * Record phy info in hw struct
1361 */
1362 (void) e1000_get_phy_info(hw);
1363
1364 /*
1365 * Make sure driver has control
1366 */
1367 igb_get_driver_control(hw);
1368
1369 /*
1370 * Restore LED settings to the default from EEPROM
1371 * to meet the standard for Sun platforms.
1372 */
1373 (void) e1000_cleanup_led(hw);
1374
1375 /*
1376 * Setup MSI-X interrupts
1377 */
1378 if (igb->intr_type == DDI_INTR_TYPE_MSIX)
1379 igb->capab->setup_msix(igb);
1380
1381 /*
1382 * Initialize unicast addresses.
1383 */
1384 igb_init_unicst(igb);
1385
1386 /*
1387 * Setup and initialize the mctable structures.
1388 */
1389 igb_setup_multicst(igb);
1390
1391 /*
1392 * Set interrupt throttling rate
1393 */
1394 for (i = 0; i < igb->intr_cnt; i++)
1395 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]);
1396
1397 /*
1398 * Save the state of the phy
1399 */
1400 igb_get_phy_state(igb);
1401
1402 igb_param_sync(igb);
1403
1404 return (IGB_SUCCESS);
1405
1406 init_adapter_fail:
1407 /*
1408 * Reset PHY if possible
1409 */
1410 if (e1000_check_reset_block(hw) == E1000_SUCCESS)
1411 (void) e1000_phy_hw_reset(hw);
1412
1413 return (IGB_FAILURE);
1414 }
1415
1416 /*
1417 * igb_stop_adapter - Stop the adapter
1418 */
1419 static void
1420 igb_stop_adapter(igb_t *igb)
1421 {
1422 struct e1000_hw *hw = &igb->hw;
1423
1424 ASSERT(mutex_owned(&igb->gen_lock));
1425
1426 /* Stop the link setup timer */
1427 igb_stop_link_timer(igb);
1428
1429 /* Tell firmware driver is no longer in control */
1430 igb_release_driver_control(hw);
1431
1432 /*
1433 * Reset the chipset
1434 */
1435 if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1436 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1437 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1438 }
1439
1440 /*
1441 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient
1442 */
1443 }
1444
1445 /*
1446 * igb_reset - Reset the chipset and restart the driver.
1447 *
1448 * It involves stopping and re-starting the chipset,
1449 * and re-configuring the rx/tx rings.
1450 */
1451 static int
1452 igb_reset(igb_t *igb)
1453 {
1454 int i;
1455
1456 mutex_enter(&igb->gen_lock);
1457
1458 ASSERT(igb->igb_state & IGB_STARTED);
1459 atomic_and_32(&igb->igb_state, ~IGB_STARTED);
1460
1461 /*
1462 * Disable the adapter interrupts to stop any rx/tx activities
1463 * before draining pending data and resetting hardware.
1464 */
1465 igb_disable_adapter_interrupts(igb);
1466
1467 /*
1468 * Drain the pending transmit packets
1469 */
1470 (void) igb_tx_drain(igb);
1471
1472 for (i = 0; i < igb->num_rx_rings; i++)
1473 mutex_enter(&igb->rx_rings[i].rx_lock);
1474 for (i = 0; i < igb->num_tx_rings; i++)
1475 mutex_enter(&igb->tx_rings[i].tx_lock);
1476
1477 /*
1478 * Stop the adapter
1479 */
1480 igb_stop_adapter(igb);
1481
1482 /*
1483 * Clean the pending tx data/resources
1484 */
1485 igb_tx_clean(igb);
1486
1487 /*
1488 * Start the adapter
1489 */
1490 if (igb_init_adapter(igb) != IGB_SUCCESS) {
1491 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1492 goto reset_failure;
1493 }
1494
1495 /*
1496 * Setup the rx/tx rings
1497 */
1498 igb->tx_ring_init = B_FALSE;
1499 igb_setup_rings(igb);
1500
1501 atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL));
1502
1503 /*
1504 * Enable adapter interrupts
1505 * The interrupts must be enabled after the driver state is START
1506 */
1507 igb->capab->enable_intr(igb);
1508
1509 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1510 goto reset_failure;
1511
1512 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1513 goto reset_failure;
1514
1515 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1516 mutex_exit(&igb->tx_rings[i].tx_lock);
1517 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1518 mutex_exit(&igb->rx_rings[i].rx_lock);
1519
1520 atomic_or_32(&igb->igb_state, IGB_STARTED);
1521
1522 mutex_exit(&igb->gen_lock);
1523
1524 return (IGB_SUCCESS);
1525
1526 reset_failure:
1527 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1528 mutex_exit(&igb->tx_rings[i].tx_lock);
1529 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1530 mutex_exit(&igb->rx_rings[i].rx_lock);
1531
1532 mutex_exit(&igb->gen_lock);
1533
1534 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1535
1536 return (IGB_FAILURE);
1537 }
1538
1539 /*
1540 * igb_tx_clean - Clean the pending transmit packets and DMA resources
1541 */
1542 static void
1543 igb_tx_clean(igb_t *igb)
1544 {
1545 igb_tx_ring_t *tx_ring;
1546 tx_control_block_t *tcb;
1547 link_list_t pending_list;
1548 uint32_t desc_num;
1549 int i, j;
1550
1551 LINK_LIST_INIT(&pending_list);
1552
1553 for (i = 0; i < igb->num_tx_rings; i++) {
1554 tx_ring = &igb->tx_rings[i];
1555
1556 mutex_enter(&tx_ring->recycle_lock);
1557
1558 /*
1559 * Clean the pending tx data - the pending packets in the
1560 * work_list that have no chances to be transmitted again.
1561 *
1562 * We must ensure the chipset is stopped or the link is down
1563 * before cleaning the transmit packets.
1564 */
1565 desc_num = 0;
1566 for (j = 0; j < tx_ring->ring_size; j++) {
1567 tcb = tx_ring->work_list[j];
1568 if (tcb != NULL) {
1569 desc_num += tcb->desc_num;
1570
1571 tx_ring->work_list[j] = NULL;
1572
1573 igb_free_tcb(tcb);
1574
1575 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1576 }
1577 }
1578
1579 if (desc_num > 0) {
1580 atomic_add_32(&tx_ring->tbd_free, desc_num);
1581 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1582
1583 /*
1584 * Reset the head and tail pointers of the tbd ring;
1585 * Reset the head write-back if it is enabled.
1586 */
1587 tx_ring->tbd_head = 0;
1588 tx_ring->tbd_tail = 0;
1589 if (igb->tx_head_wb_enable)
1590 *tx_ring->tbd_head_wb = 0;
1591
1592 E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0);
1593 E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0);
1594 }
1595
1596 mutex_exit(&tx_ring->recycle_lock);
1597
1598 /*
1599 * Add the tx control blocks in the pending list to
1600 * the free list.
1601 */
1602 igb_put_free_list(tx_ring, &pending_list);
1603 }
1604 }
1605
1606 /*
1607 * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted
1608 */
1609 static boolean_t
1610 igb_tx_drain(igb_t *igb)
1611 {
1612 igb_tx_ring_t *tx_ring;
1613 boolean_t done;
1614 int i, j;
1615
1616 /*
1617 * Wait for a specific time to allow pending tx packets
1618 * to be transmitted.
1619 *
1620 * Check the counter tbd_free to see if transmission is done.
1621 * No lock protection is needed here.
1622 *
1623 * Return B_TRUE if all pending packets have been transmitted;
1624 * Otherwise return B_FALSE;
1625 */
1626 for (i = 0; i < TX_DRAIN_TIME; i++) {
1627
1628 done = B_TRUE;
1629 for (j = 0; j < igb->num_tx_rings; j++) {
1630 tx_ring = &igb->tx_rings[j];
1631 done = done &&
1632 (tx_ring->tbd_free == tx_ring->ring_size);
1633 }
1634
1635 if (done)
1636 break;
1637
1638 msec_delay(1);
1639 }
1640
1641 return (done);
1642 }
1643
1644 /*
1645 * igb_rx_drain - Wait for all rx buffers to be released by upper layer
1646 */
1647 static boolean_t
1648 igb_rx_drain(igb_t *igb)
1649 {
1650 boolean_t done;
1651 int i;
1652
1653 /*
1654 * Polling the rx free list to check if those rx buffers held by
1655 * the upper layer are released.
1656 *
1657 * Check the counter rcb_free to see if all pending buffers are
1658 * released. No lock protection is needed here.
1659 *
1660 * Return B_TRUE if all pending buffers have been released;
1661 * Otherwise return B_FALSE;
1662 */
1663 for (i = 0; i < RX_DRAIN_TIME; i++) {
1664 done = (igb->rcb_pending == 0);
1665
1666 if (done)
1667 break;
1668
1669 msec_delay(1);
1670 }
1671
1672 return (done);
1673 }
1674
1675 /*
1676 * igb_start - Start the driver/chipset
1677 */
1678 int
1679 igb_start(igb_t *igb, boolean_t alloc_buffer)
1680 {
1681 int i;
1682
1683 ASSERT(mutex_owned(&igb->gen_lock));
1684
1685 if (alloc_buffer) {
1686 if (igb_alloc_rx_data(igb) != IGB_SUCCESS) {
1687 igb_error(igb,
1688 "Failed to allocate software receive rings");
1689 return (IGB_FAILURE);
1690 }
1691
1692 /* Allocate buffers for all the rx/tx rings */
1693 if (igb_alloc_dma(igb) != IGB_SUCCESS) {
1694 igb_error(igb, "Failed to allocate DMA resource");
1695 return (IGB_FAILURE);
1696 }
1697
1698 igb->tx_ring_init = B_TRUE;
1699 } else {
1700 igb->tx_ring_init = B_FALSE;
1701 }
1702
1703 for (i = 0; i < igb->num_rx_rings; i++)
1704 mutex_enter(&igb->rx_rings[i].rx_lock);
1705 for (i = 0; i < igb->num_tx_rings; i++)
1706 mutex_enter(&igb->tx_rings[i].tx_lock);
1707
1708 /*
1709 * Start the adapter
1710 */
1711 if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) {
1712 if (igb_init_adapter(igb) != IGB_SUCCESS) {
1713 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1714 goto start_failure;
1715 }
1716 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
1717 }
1718
1719 /*
1720 * Setup the rx/tx rings
1721 */
1722 igb_setup_rings(igb);
1723
1724 /*
1725 * Enable adapter interrupts
1726 * The interrupts must be enabled after the driver state is START
1727 */
1728 igb->capab->enable_intr(igb);
1729
1730 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1731 goto start_failure;
1732
1733 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1734 goto start_failure;
1735
1736 if (igb->hw.mac.type == e1000_i350)
1737 (void) e1000_set_eee_i350(&igb->hw);
1738
1739 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1740 mutex_exit(&igb->tx_rings[i].tx_lock);
1741 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1742 mutex_exit(&igb->rx_rings[i].rx_lock);
1743
1744 return (IGB_SUCCESS);
1745
1746 start_failure:
1747 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1748 mutex_exit(&igb->tx_rings[i].tx_lock);
1749 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1750 mutex_exit(&igb->rx_rings[i].rx_lock);
1751
1752 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1753
1754 return (IGB_FAILURE);
1755 }
1756
1757 /*
1758 * igb_stop - Stop the driver/chipset
1759 */
1760 void
1761 igb_stop(igb_t *igb, boolean_t free_buffer)
1762 {
1763 int i;
1764
1765 ASSERT(mutex_owned(&igb->gen_lock));
1766
1767 igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER;
1768
1769 /*
1770 * Disable the adapter interrupts
1771 */
1772 igb_disable_adapter_interrupts(igb);
1773
1774 /*
1775 * Drain the pending tx packets
1776 */
1777 (void) igb_tx_drain(igb);
1778
1779 for (i = 0; i < igb->num_rx_rings; i++)
1780 mutex_enter(&igb->rx_rings[i].rx_lock);
1781 for (i = 0; i < igb->num_tx_rings; i++)
1782 mutex_enter(&igb->tx_rings[i].tx_lock);
1783
1784 /*
1785 * Stop the adapter
1786 */
1787 igb_stop_adapter(igb);
1788
1789 /*
1790 * Clean the pending tx data/resources
1791 */
1792 igb_tx_clean(igb);
1793
1794 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1795 mutex_exit(&igb->tx_rings[i].tx_lock);
1796 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1797 mutex_exit(&igb->rx_rings[i].rx_lock);
1798
1799 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1800 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1801
1802 if (igb->link_state == LINK_STATE_UP) {
1803 igb->link_state = LINK_STATE_UNKNOWN;
1804 mac_link_update(igb->mac_hdl, igb->link_state);
1805 }
1806
1807 if (free_buffer) {
1808 /*
1809 * Release the DMA/memory resources of rx/tx rings
1810 */
1811 igb_free_dma(igb);
1812 igb_free_rx_data(igb);
1813 }
1814 }
1815
1816 /*
1817 * igb_alloc_rings - Allocate memory space for rx/tx rings
1818 */
1819 static int
1820 igb_alloc_rings(igb_t *igb)
1821 {
1822 /*
1823 * Allocate memory space for rx rings
1824 */
1825 igb->rx_rings = kmem_zalloc(
1826 sizeof (igb_rx_ring_t) * igb->num_rx_rings,
1827 KM_NOSLEEP);
1828
1829 if (igb->rx_rings == NULL) {
1830 return (IGB_FAILURE);
1831 }
1832
1833 /*
1834 * Allocate memory space for tx rings
1835 */
1836 igb->tx_rings = kmem_zalloc(
1837 sizeof (igb_tx_ring_t) * igb->num_tx_rings,
1838 KM_NOSLEEP);
1839
1840 if (igb->tx_rings == NULL) {
1841 kmem_free(igb->rx_rings,
1842 sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1843 igb->rx_rings = NULL;
1844 return (IGB_FAILURE);
1845 }
1846
1847 /*
1848 * Allocate memory space for rx ring groups
1849 */
1850 igb->rx_groups = kmem_zalloc(
1851 sizeof (igb_rx_group_t) * igb->num_rx_groups,
1852 KM_NOSLEEP);
1853
1854 if (igb->rx_groups == NULL) {
1855 kmem_free(igb->rx_rings,
1856 sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1857 kmem_free(igb->tx_rings,
1858 sizeof (igb_tx_ring_t) * igb->num_tx_rings);
1859 igb->rx_rings = NULL;
1860 igb->tx_rings = NULL;
1861 return (IGB_FAILURE);
1862 }
1863
1864 return (IGB_SUCCESS);
1865 }
1866
1867 /*
1868 * igb_free_rings - Free the memory space of rx/tx rings.
1869 */
1870 static void
1871 igb_free_rings(igb_t *igb)
1872 {
1873 if (igb->rx_rings != NULL) {
1874 kmem_free(igb->rx_rings,
1875 sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1876 igb->rx_rings = NULL;
1877 }
1878
1879 if (igb->tx_rings != NULL) {
1880 kmem_free(igb->tx_rings,
1881 sizeof (igb_tx_ring_t) * igb->num_tx_rings);
1882 igb->tx_rings = NULL;
1883 }
1884
1885 if (igb->rx_groups != NULL) {
1886 kmem_free(igb->rx_groups,
1887 sizeof (igb_rx_group_t) * igb->num_rx_groups);
1888 igb->rx_groups = NULL;
1889 }
1890 }
1891
1892 static int
1893 igb_alloc_rx_data(igb_t *igb)
1894 {
1895 igb_rx_ring_t *rx_ring;
1896 int i;
1897
1898 for (i = 0; i < igb->num_rx_rings; i++) {
1899 rx_ring = &igb->rx_rings[i];
1900 if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS)
1901 goto alloc_rx_rings_failure;
1902 }
1903 return (IGB_SUCCESS);
1904
1905 alloc_rx_rings_failure:
1906 igb_free_rx_data(igb);
1907 return (IGB_FAILURE);
1908 }
1909
1910 static void
1911 igb_free_rx_data(igb_t *igb)
1912 {
1913 igb_rx_ring_t *rx_ring;
1914 igb_rx_data_t *rx_data;
1915 int i;
1916
1917 for (i = 0; i < igb->num_rx_rings; i++) {
1918 rx_ring = &igb->rx_rings[i];
1919
1920 mutex_enter(&igb->rx_pending_lock);
1921 rx_data = rx_ring->rx_data;
1922
1923 if (rx_data != NULL) {
1924 rx_data->flag |= IGB_RX_STOPPED;
1925
1926 if (rx_data->rcb_pending == 0) {
1927 igb_free_rx_ring_data(rx_data);
1928 rx_ring->rx_data = NULL;
1929 }
1930 }
1931
1932 mutex_exit(&igb->rx_pending_lock);
1933 }
1934 }
1935
1936 /*
1937 * igb_setup_rings - Setup rx/tx rings
1938 */
1939 static void
1940 igb_setup_rings(igb_t *igb)
1941 {
1942 /*
1943 * Setup the rx/tx rings, including the following:
1944 *
1945 * 1. Setup the descriptor ring and the control block buffers;
1946 * 2. Initialize necessary registers for receive/transmit;
1947 * 3. Initialize software pointers/parameters for receive/transmit;
1948 */
1949 igb_setup_rx(igb);
1950
1951 igb_setup_tx(igb);
1952 }
1953
1954 static void
1955 igb_setup_rx_ring(igb_rx_ring_t *rx_ring)
1956 {
1957 igb_t *igb = rx_ring->igb;
1958 igb_rx_data_t *rx_data = rx_ring->rx_data;
1959 struct e1000_hw *hw = &igb->hw;
1960 rx_control_block_t *rcb;
1961 union e1000_adv_rx_desc *rbd;
1962 uint32_t size;
1963 uint32_t buf_low;
1964 uint32_t buf_high;
1965 uint32_t rxdctl;
1966 int i;
1967
1968 ASSERT(mutex_owned(&rx_ring->rx_lock));
1969 ASSERT(mutex_owned(&igb->gen_lock));
1970
1971 /*
1972 * Initialize descriptor ring with buffer addresses
1973 */
1974 for (i = 0; i < igb->rx_ring_size; i++) {
1975 rcb = rx_data->work_list[i];
1976 rbd = &rx_data->rbd_ring[i];
1977
1978 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
1979 rbd->read.hdr_addr = NULL;
1980 }
1981
1982 /*
1983 * Initialize the base address registers
1984 */
1985 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
1986 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
1987 E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high);
1988 E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low);
1989
1990 /*
1991 * Initialize the length register
1992 */
1993 size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc);
1994 E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size);
1995
1996 /*
1997 * Initialize buffer size & descriptor type
1998 */
1999 E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index),
2000 ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) |
2001 E1000_SRRCTL_DESCTYPE_ADV_ONEBUF));
2002
2003 /*
2004 * Setup the Receive Descriptor Control Register (RXDCTL)
2005 */
2006 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index));
2007 rxdctl &= igb->capab->rxdctl_mask;
2008 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2009 rxdctl |= 16; /* pthresh */
2010 rxdctl |= 8 << 8; /* hthresh */
2011 rxdctl |= 1 << 16; /* wthresh */
2012 E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl);
2013
2014 rx_data->rbd_next = 0;
2015 }
2016
2017 static void
2018 igb_setup_rx(igb_t *igb)
2019 {
2020 igb_rx_ring_t *rx_ring;
2021 igb_rx_data_t *rx_data;
2022 igb_rx_group_t *rx_group;
2023 struct e1000_hw *hw = &igb->hw;
2024 uint32_t rctl, rxcsum;
2025 uint32_t ring_per_group;
2026 int i;
2027
2028 /*
2029 * Setup the Receive Control Register (RCTL), and enable the
2030 * receiver. The initial configuration is to: enable the receiver,
2031 * accept broadcasts, discard bad packets, accept long packets,
2032 * disable VLAN filter checking, and set receive buffer size to
2033 * 2k. For 82575, also set the receive descriptor minimum
2034 * threshold size to 1/2 the ring.
2035 */
2036 rctl = E1000_READ_REG(hw, E1000_RCTL);
2037
2038 /*
2039 * Clear the field used for wakeup control. This driver doesn't do
2040 * wakeup but leave this here for completeness.
2041 */
2042 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2043 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2044
2045 rctl |= (E1000_RCTL_EN | /* Enable Receive Unit */
2046 E1000_RCTL_BAM | /* Accept Broadcast Packets */
2047 E1000_RCTL_LPE | /* Large Packet Enable */
2048 /* Multicast filter offset */
2049 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) |
2050 E1000_RCTL_RDMTS_HALF | /* rx descriptor threshold */
2051 E1000_RCTL_SECRC); /* Strip Ethernet CRC */
2052
2053 for (i = 0; i < igb->num_rx_groups; i++) {
2054 rx_group = &igb->rx_groups[i];
2055 rx_group->index = i;
2056 rx_group->igb = igb;
2057 }
2058
2059 /*
2060 * Set up all rx descriptor rings - must be called before receive unit
2061 * enabled.
2062 */
2063 ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2064 for (i = 0; i < igb->num_rx_rings; i++) {
2065 rx_ring = &igb->rx_rings[i];
2066 igb_setup_rx_ring(rx_ring);
2067
2068 /*
2069 * Map a ring to a group by assigning a group index
2070 */
2071 rx_ring->group_index = i / ring_per_group;
2072 }
2073
2074 /*
2075 * Setup the Rx Long Packet Max Length register
2076 */
2077 E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size);
2078
2079 /*
2080 * Hardware checksum settings
2081 */
2082 if (igb->rx_hcksum_enable) {
2083 rxcsum =
2084 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum */
2085 E1000_RXCSUM_IPOFL; /* IP checksum */
2086
2087 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2088 }
2089
2090 /*
2091 * Setup classify and RSS for multiple receive queues
2092 */
2093 switch (igb->vmdq_mode) {
2094 case E1000_VMDQ_OFF:
2095 /*
2096 * One ring group, only RSS is needed when more than
2097 * one ring enabled.
2098 */
2099 if (igb->num_rx_rings > 1)
2100 igb_setup_rss(igb);
2101 break;
2102 case E1000_VMDQ_MAC:
2103 /*
2104 * Multiple groups, each group has one ring,
2105 * only the MAC classification is needed.
2106 */
2107 igb_setup_mac_classify(igb);
2108 break;
2109 case E1000_VMDQ_MAC_RSS:
2110 /*
2111 * Multiple groups and multiple rings, both
2112 * MAC classification and RSS are needed.
2113 */
2114 igb_setup_mac_rss_classify(igb);
2115 break;
2116 }
2117
2118 /*
2119 * Enable the receive unit - must be done after all
2120 * the rx setup above.
2121 */
2122 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2123
2124 /*
2125 * Initialize all adapter ring head & tail pointers - must
2126 * be done after receive unit is enabled
2127 */
2128 for (i = 0; i < igb->num_rx_rings; i++) {
2129 rx_ring = &igb->rx_rings[i];
2130 rx_data = rx_ring->rx_data;
2131 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
2132 E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1);
2133 }
2134
2135 /*
2136 * 82575 with manageability enabled needs a special flush to make
2137 * sure the fifos start clean.
2138 */
2139 if ((hw->mac.type == e1000_82575) &&
2140 (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) {
2141 e1000_rx_fifo_flush_82575(hw);
2142 }
2143 }
2144
2145 static void
2146 igb_setup_tx_ring(igb_tx_ring_t *tx_ring)
2147 {
2148 igb_t *igb = tx_ring->igb;
2149 struct e1000_hw *hw = &igb->hw;
2150 uint32_t size;
2151 uint32_t buf_low;
2152 uint32_t buf_high;
2153 uint32_t reg_val;
2154
2155 ASSERT(mutex_owned(&tx_ring->tx_lock));
2156 ASSERT(mutex_owned(&igb->gen_lock));
2157
2158
2159 /*
2160 * Initialize the length register
2161 */
2162 size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc);
2163 E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size);
2164
2165 /*
2166 * Initialize the base address registers
2167 */
2168 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2169 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2170 E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low);
2171 E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high);
2172
2173 /*
2174 * Setup head & tail pointers
2175 */
2176 E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0);
2177 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0);
2178
2179 /*
2180 * Setup head write-back
2181 */
2182 if (igb->tx_head_wb_enable) {
2183 /*
2184 * The memory of the head write-back is allocated using
2185 * the extra tbd beyond the tail of the tbd ring.
2186 */
2187 tx_ring->tbd_head_wb = (uint32_t *)
2188 ((uintptr_t)tx_ring->tbd_area.address + size);
2189 *tx_ring->tbd_head_wb = 0;
2190
2191 buf_low = (uint32_t)
2192 (tx_ring->tbd_area.dma_address + size);
2193 buf_high = (uint32_t)
2194 ((tx_ring->tbd_area.dma_address + size) >> 32);
2195
2196 /* Set the head write-back enable bit */
2197 buf_low |= E1000_TX_HEAD_WB_ENABLE;
2198
2199 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low);
2200 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high);
2201
2202 /*
2203 * Turn off relaxed ordering for head write back or it will
2204 * cause problems with the tx recycling
2205 */
2206 reg_val = E1000_READ_REG(hw,
2207 E1000_DCA_TXCTRL(tx_ring->index));
2208 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
2209 E1000_WRITE_REG(hw,
2210 E1000_DCA_TXCTRL(tx_ring->index), reg_val);
2211 } else {
2212 tx_ring->tbd_head_wb = NULL;
2213 }
2214
2215 tx_ring->tbd_head = 0;
2216 tx_ring->tbd_tail = 0;
2217 tx_ring->tbd_free = tx_ring->ring_size;
2218
2219 if (igb->tx_ring_init == B_TRUE) {
2220 tx_ring->tcb_head = 0;
2221 tx_ring->tcb_tail = 0;
2222 tx_ring->tcb_free = tx_ring->free_list_size;
2223 }
2224
2225 /*
2226 * Enable TXDCTL per queue
2227 */
2228 reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index));
2229 reg_val |= E1000_TXDCTL_QUEUE_ENABLE;
2230 E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val);
2231
2232 /*
2233 * Initialize hardware checksum offload settings
2234 */
2235 bzero(&tx_ring->tx_context, sizeof (tx_context_t));
2236 }
2237
2238 static void
2239 igb_setup_tx(igb_t *igb)
2240 {
2241 igb_tx_ring_t *tx_ring;
2242 struct e1000_hw *hw = &igb->hw;
2243 uint32_t reg_val;
2244 int i;
2245
2246 for (i = 0; i < igb->num_tx_rings; i++) {
2247 tx_ring = &igb->tx_rings[i];
2248 igb_setup_tx_ring(tx_ring);
2249 }
2250
2251 /*
2252 * Setup the Transmit Control Register (TCTL)
2253 */
2254 reg_val = E1000_READ_REG(hw, E1000_TCTL);
2255 reg_val &= ~E1000_TCTL_CT;
2256 reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2257 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2258
2259 /* Enable transmits */
2260 reg_val |= E1000_TCTL_EN;
2261
2262 E1000_WRITE_REG(hw, E1000_TCTL, reg_val);
2263 }
2264
2265 /*
2266 * igb_setup_rss - Setup receive-side scaling feature
2267 */
2268 static void
2269 igb_setup_rss(igb_t *igb)
2270 {
2271 struct e1000_hw *hw = &igb->hw;
2272 uint32_t i, mrqc, rxcsum;
2273 int shift = 0;
2274 uint32_t random;
2275 union e1000_reta {
2276 uint32_t dword;
2277 uint8_t bytes[4];
2278 } reta;
2279
2280 /* Setup the Redirection Table */
2281 if (hw->mac.type == e1000_82576) {
2282 shift = 3;
2283 } else if (hw->mac.type == e1000_82575) {
2284 shift = 6;
2285 }
2286 for (i = 0; i < (32 * 4); i++) {
2287 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift;
2288 if ((i & 3) == 3) {
2289 E1000_WRITE_REG(hw,
2290 (E1000_RETA(0) + (i & ~3)), reta.dword);
2291 }
2292 }
2293
2294 /* Fill out hash function seeds */
2295 for (i = 0; i < 10; i++) {
2296 (void) random_get_pseudo_bytes((uint8_t *)&random,
2297 sizeof (uint32_t));
2298 E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2299 }
2300
2301 /* Setup the Multiple Receive Queue Control register */
2302 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2303 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2304 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2305 E1000_MRQC_RSS_FIELD_IPV6 |
2306 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2307 E1000_MRQC_RSS_FIELD_IPV4_UDP |
2308 E1000_MRQC_RSS_FIELD_IPV6_UDP |
2309 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2310 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2311
2312 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2313
2314 /*
2315 * Disable Packet Checksum to enable RSS for multiple receive queues.
2316 *
2317 * The Packet Checksum is not ethernet CRC. It is another kind of
2318 * checksum offloading provided by the 82575 chipset besides the IP
2319 * header checksum offloading and the TCP/UDP checksum offloading.
2320 * The Packet Checksum is by default computed over the entire packet
2321 * from the first byte of the DA through the last byte of the CRC,
2322 * including the Ethernet and IP headers.
2323 *
2324 * It is a hardware limitation that Packet Checksum is mutually
2325 * exclusive with RSS.
2326 */
2327 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2328 rxcsum |= E1000_RXCSUM_PCSD;
2329 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2330 }
2331
2332 /*
2333 * igb_setup_mac_rss_classify - Setup MAC classification and rss
2334 */
2335 static void
2336 igb_setup_mac_rss_classify(igb_t *igb)
2337 {
2338 struct e1000_hw *hw = &igb->hw;
2339 uint32_t i, mrqc, vmdctl, rxcsum;
2340 uint32_t ring_per_group;
2341 int shift_group0, shift_group1;
2342 uint32_t random;
2343 union e1000_reta {
2344 uint32_t dword;
2345 uint8_t bytes[4];
2346 } reta;
2347
2348 ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2349
2350 /* Setup the Redirection Table, it is shared between two groups */
2351 shift_group0 = 2;
2352 shift_group1 = 6;
2353 for (i = 0; i < (32 * 4); i++) {
2354 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) |
2355 ((ring_per_group + (i % ring_per_group)) << shift_group1);
2356 if ((i & 3) == 3) {
2357 E1000_WRITE_REG(hw,
2358 (E1000_RETA(0) + (i & ~3)), reta.dword);
2359 }
2360 }
2361
2362 /* Fill out hash function seeds */
2363 for (i = 0; i < 10; i++) {
2364 (void) random_get_pseudo_bytes((uint8_t *)&random,
2365 sizeof (uint32_t));
2366 E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2367 }
2368
2369 /*
2370 * Setup the Multiple Receive Queue Control register,
2371 * enable VMDq based on packet destination MAC address and RSS.
2372 */
2373 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP;
2374 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2375 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2376 E1000_MRQC_RSS_FIELD_IPV6 |
2377 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2378 E1000_MRQC_RSS_FIELD_IPV4_UDP |
2379 E1000_MRQC_RSS_FIELD_IPV6_UDP |
2380 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2381 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2382
2383 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2384
2385
2386 /* Define the default group and default queues */
2387 vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE;
2388 E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl);
2389
2390 /*
2391 * Disable Packet Checksum to enable RSS for multiple receive queues.
2392 *
2393 * The Packet Checksum is not ethernet CRC. It is another kind of
2394 * checksum offloading provided by the 82575 chipset besides the IP
2395 * header checksum offloading and the TCP/UDP checksum offloading.
2396 * The Packet Checksum is by default computed over the entire packet
2397 * from the first byte of the DA through the last byte of the CRC,
2398 * including the Ethernet and IP headers.
2399 *
2400 * It is a hardware limitation that Packet Checksum is mutually
2401 * exclusive with RSS.
2402 */
2403 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2404 rxcsum |= E1000_RXCSUM_PCSD;
2405 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2406 }
2407
2408 /*
2409 * igb_setup_mac_classify - Setup MAC classification feature
2410 */
2411 static void
2412 igb_setup_mac_classify(igb_t *igb)
2413 {
2414 struct e1000_hw *hw = &igb->hw;
2415 uint32_t mrqc, rxcsum;
2416
2417 /*
2418 * Setup the Multiple Receive Queue Control register,
2419 * enable VMDq based on packet destination MAC address.
2420 */
2421 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP;
2422 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2423
2424 /*
2425 * Disable Packet Checksum to enable RSS for multiple receive queues.
2426 *
2427 * The Packet Checksum is not ethernet CRC. It is another kind of
2428 * checksum offloading provided by the 82575 chipset besides the IP
2429 * header checksum offloading and the TCP/UDP checksum offloading.
2430 * The Packet Checksum is by default computed over the entire packet
2431 * from the first byte of the DA through the last byte of the CRC,
2432 * including the Ethernet and IP headers.
2433 *
2434 * It is a hardware limitation that Packet Checksum is mutually
2435 * exclusive with RSS.
2436 */
2437 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2438 rxcsum |= E1000_RXCSUM_PCSD;
2439 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2440
2441 }
2442
2443 /*
2444 * igb_init_unicst - Initialize the unicast addresses
2445 */
2446 static void
2447 igb_init_unicst(igb_t *igb)
2448 {
2449 struct e1000_hw *hw = &igb->hw;
2450 int slot;
2451
2452 /*
2453 * Here we should consider two situations:
2454 *
2455 * 1. Chipset is initialized the first time
2456 * Initialize the multiple unicast addresses, and
2457 * save the default MAC address.
2458 *
2459 * 2. Chipset is reset
2460 * Recover the multiple unicast addresses from the
2461 * software data structure to the RAR registers.
2462 */
2463
2464 /*
2465 * Clear the default MAC address in the RAR0 rgister,
2466 * which is loaded from EEPROM when system boot or chipreset,
2467 * this will cause the conficts with add_mac/rem_mac entry
2468 * points when VMDq is enabled. For this reason, the RAR0
2469 * must be cleared for both cases mentioned above.
2470 */
2471 e1000_rar_clear(hw, 0);
2472
2473 if (!igb->unicst_init) {
2474
2475 /* Initialize the multiple unicast addresses */
2476 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES;
2477 igb->unicst_avail = igb->unicst_total;
2478
2479 for (slot = 0; slot < igb->unicst_total; slot++)
2480 igb->unicst_addr[slot].mac.set = 0;
2481
2482 igb->unicst_init = B_TRUE;
2483 } else {
2484 /* Re-configure the RAR registers */
2485 for (slot = 0; slot < igb->unicst_total; slot++) {
2486 e1000_rar_set_vmdq(hw, igb->unicst_addr[slot].mac.addr,
2487 slot, igb->vmdq_mode,
2488 igb->unicst_addr[slot].mac.group_index);
2489 }
2490 }
2491 }
2492
2493 /*
2494 * igb_unicst_find - Find the slot for the specified unicast address
2495 */
2496 int
2497 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr)
2498 {
2499 int slot;
2500
2501 ASSERT(mutex_owned(&igb->gen_lock));
2502
2503 for (slot = 0; slot < igb->unicst_total; slot++) {
2504 if (bcmp(igb->unicst_addr[slot].mac.addr,
2505 mac_addr, ETHERADDRL) == 0)
2506 return (slot);
2507 }
2508
2509 return (-1);
2510 }
2511
2512 /*
2513 * igb_unicst_set - Set the unicast address to the specified slot
2514 */
2515 int
2516 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr,
2517 int slot)
2518 {
2519 struct e1000_hw *hw = &igb->hw;
2520
2521 ASSERT(mutex_owned(&igb->gen_lock));
2522
2523 /*
2524 * Save the unicast address in the software data structure
2525 */
2526 bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL);
2527
2528 /*
2529 * Set the unicast address to the RAR register
2530 */
2531 e1000_rar_set(hw, (uint8_t *)mac_addr, slot);
2532
2533 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2534 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2535 return (EIO);
2536 }
2537
2538 return (0);
2539 }
2540
2541 /*
2542 * igb_multicst_add - Add a multicst address
2543 */
2544 int
2545 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr)
2546 {
2547 struct ether_addr *new_table;
2548 size_t new_len;
2549 size_t old_len;
2550
2551 ASSERT(mutex_owned(&igb->gen_lock));
2552
2553 if ((multiaddr[0] & 01) == 0) {
2554 igb_error(igb, "Illegal multicast address");
2555 return (EINVAL);
2556 }
2557
2558 if (igb->mcast_count >= igb->mcast_max_num) {
2559 igb_error(igb, "Adapter requested more than %d mcast addresses",
2560 igb->mcast_max_num);
2561 return (ENOENT);
2562 }
2563
2564 if (igb->mcast_count == igb->mcast_alloc_count) {
2565 old_len = igb->mcast_alloc_count *
2566 sizeof (struct ether_addr);
2567 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) *
2568 sizeof (struct ether_addr);
2569
2570 new_table = kmem_alloc(new_len, KM_NOSLEEP);
2571 if (new_table == NULL) {
2572 igb_error(igb,
2573 "Not enough memory to alloc mcast table");
2574 return (ENOMEM);
2575 }
2576
2577 if (igb->mcast_table != NULL) {
2578 bcopy(igb->mcast_table, new_table, old_len);
2579 kmem_free(igb->mcast_table, old_len);
2580 }
2581 igb->mcast_alloc_count += MCAST_ALLOC_COUNT;
2582 igb->mcast_table = new_table;
2583 }
2584
2585 bcopy(multiaddr,
2586 &igb->mcast_table[igb->mcast_count], ETHERADDRL);
2587 igb->mcast_count++;
2588
2589 /*
2590 * Update the multicast table in the hardware
2591 */
2592 igb_setup_multicst(igb);
2593
2594 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2595 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2596 return (EIO);
2597 }
2598
2599 return (0);
2600 }
2601
2602 /*
2603 * igb_multicst_remove - Remove a multicst address
2604 */
2605 int
2606 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr)
2607 {
2608 struct ether_addr *new_table;
2609 size_t new_len;
2610 size_t old_len;
2611 int i;
2612
2613 ASSERT(mutex_owned(&igb->gen_lock));
2614
2615 for (i = 0; i < igb->mcast_count; i++) {
2616 if (bcmp(multiaddr, &igb->mcast_table[i],
2617 ETHERADDRL) == 0) {
2618 for (i++; i < igb->mcast_count; i++) {
2619 igb->mcast_table[i - 1] =
2620 igb->mcast_table[i];
2621 }
2622 igb->mcast_count--;
2623 break;
2624 }
2625 }
2626
2627 if ((igb->mcast_alloc_count - igb->mcast_count) >
2628 MCAST_ALLOC_COUNT) {
2629 old_len = igb->mcast_alloc_count *
2630 sizeof (struct ether_addr);
2631 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) *
2632 sizeof (struct ether_addr);
2633
2634 new_table = kmem_alloc(new_len, KM_NOSLEEP);
2635 if (new_table != NULL) {
2636 bcopy(igb->mcast_table, new_table, new_len);
2637 kmem_free(igb->mcast_table, old_len);
2638 igb->mcast_alloc_count -= MCAST_ALLOC_COUNT;
2639 igb->mcast_table = new_table;
2640 }
2641 }
2642
2643 /*
2644 * Update the multicast table in the hardware
2645 */
2646 igb_setup_multicst(igb);
2647
2648 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2649 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2650 return (EIO);
2651 }
2652
2653 return (0);
2654 }
2655
2656 static void
2657 igb_release_multicast(igb_t *igb)
2658 {
2659 if (igb->mcast_table != NULL) {
2660 kmem_free(igb->mcast_table,
2661 igb->mcast_alloc_count * sizeof (struct ether_addr));
2662 igb->mcast_table = NULL;
2663 }
2664 }
2665
2666 /*
2667 * igb_setup_multicast - setup multicast data structures
2668 *
2669 * This routine initializes all of the multicast related structures
2670 * and save them in the hardware registers.
2671 */
2672 static void
2673 igb_setup_multicst(igb_t *igb)
2674 {
2675 uint8_t *mc_addr_list;
2676 uint32_t mc_addr_count;
2677 struct e1000_hw *hw = &igb->hw;
2678
2679 ASSERT(mutex_owned(&igb->gen_lock));
2680 ASSERT(igb->mcast_count <= igb->mcast_max_num);
2681
2682 mc_addr_list = (uint8_t *)igb->mcast_table;
2683 mc_addr_count = igb->mcast_count;
2684
2685 /*
2686 * Update the multicase addresses to the MTA registers
2687 */
2688 e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count);
2689 }
2690
2691 /*
2692 * igb_get_conf - Get driver configurations set in driver.conf
2693 *
2694 * This routine gets user-configured values out of the configuration
2695 * file igb.conf.
2696 *
2697 * For each configurable value, there is a minimum, a maximum, and a
2698 * default.
2699 * If user does not configure a value, use the default.
2700 * If user configures below the minimum, use the minumum.
2701 * If user configures above the maximum, use the maxumum.
2702 */
2703 static void
2704 igb_get_conf(igb_t *igb)
2705 {
2706 struct e1000_hw *hw = &igb->hw;
2707 uint32_t default_mtu;
2708 uint32_t flow_control;
2709 uint32_t ring_per_group;
2710 int i;
2711
2712 /*
2713 * igb driver supports the following user configurations:
2714 *
2715 * Link configurations:
2716 * adv_autoneg_cap
2717 * adv_1000fdx_cap
2718 * adv_100fdx_cap
2719 * adv_100hdx_cap
2720 * adv_10fdx_cap
2721 * adv_10hdx_cap
2722 * Note: 1000hdx is not supported.
2723 *
2724 * Jumbo frame configuration:
2725 * default_mtu
2726 *
2727 * Ethernet flow control configuration:
2728 * flow_control
2729 *
2730 * Multiple rings configurations:
2731 * tx_queue_number
2732 * tx_ring_size
2733 * rx_queue_number
2734 * rx_ring_size
2735 *
2736 * Call igb_get_prop() to get the value for a specific
2737 * configuration parameter.
2738 */
2739
2740 /*
2741 * Link configurations
2742 */
2743 igb->param_adv_autoneg_cap = igb_get_prop(igb,
2744 PROP_ADV_AUTONEG_CAP, 0, 1, 1);
2745 igb->param_adv_1000fdx_cap = igb_get_prop(igb,
2746 PROP_ADV_1000FDX_CAP, 0, 1, 1);
2747 igb->param_adv_100fdx_cap = igb_get_prop(igb,
2748 PROP_ADV_100FDX_CAP, 0, 1, 1);
2749 igb->param_adv_100hdx_cap = igb_get_prop(igb,
2750 PROP_ADV_100HDX_CAP, 0, 1, 1);
2751 igb->param_adv_10fdx_cap = igb_get_prop(igb,
2752 PROP_ADV_10FDX_CAP, 0, 1, 1);
2753 igb->param_adv_10hdx_cap = igb_get_prop(igb,
2754 PROP_ADV_10HDX_CAP, 0, 1, 1);
2755
2756 /*
2757 * Jumbo frame configurations
2758 */
2759 default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
2760 MIN_MTU, MAX_MTU, DEFAULT_MTU);
2761
2762 igb->max_frame_size = default_mtu +
2763 sizeof (struct ether_vlan_header) + ETHERFCSL;
2764
2765 /*
2766 * Ethernet flow control configuration
2767 */
2768 flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL,
2769 e1000_fc_none, 4, e1000_fc_full);
2770 if (flow_control == 4)
2771 flow_control = e1000_fc_default;
2772
2773 hw->fc.requested_mode = flow_control;
2774
2775 /*
2776 * Multiple rings configurations
2777 */
2778 igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE,
2779 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
2780 igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE,
2781 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
2782
2783 igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0);
2784 igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM,
2785 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM);
2786 /*
2787 * Currently we do not support VMDq for 82576 and 82580.
2788 * If it is e1000_82576, set num_rx_groups to 1.
2789 */
2790 if (hw->mac.type >= e1000_82576)
2791 igb->num_rx_groups = 1;
2792
2793 if (igb->mr_enable) {
2794 igb->num_tx_rings = igb->capab->def_tx_que_num;
2795 igb->num_rx_rings = igb->capab->def_rx_que_num;
2796 } else {
2797 igb->num_tx_rings = 1;
2798 igb->num_rx_rings = 1;
2799
2800 if (igb->num_rx_groups > 1) {
2801 igb_error(igb,
2802 "Invalid rx groups number. Please enable multiple "
2803 "rings first");
2804 igb->num_rx_groups = 1;
2805 }
2806 }
2807
2808 /*
2809 * Check the divisibility between rx rings and rx groups.
2810 */
2811 for (i = igb->num_rx_groups; i > 0; i--) {
2812 if ((igb->num_rx_rings % i) == 0)
2813 break;
2814 }
2815 if (i != igb->num_rx_groups) {
2816 igb_error(igb,
2817 "Invalid rx groups number. Downgrade the rx group "
2818 "number to %d.", i);
2819 igb->num_rx_groups = i;
2820 }
2821
2822 /*
2823 * Get the ring number per group.
2824 */
2825 ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2826
2827 if (igb->num_rx_groups == 1) {
2828 /*
2829 * One rx ring group, the rx ring number is num_rx_rings.
2830 */
2831 igb->vmdq_mode = E1000_VMDQ_OFF;
2832 } else if (ring_per_group == 1) {
2833 /*
2834 * Multiple rx groups, each group has one rx ring.
2835 */
2836 igb->vmdq_mode = E1000_VMDQ_MAC;
2837 } else {
2838 /*
2839 * Multiple groups and multiple rings.
2840 */
2841 igb->vmdq_mode = E1000_VMDQ_MAC_RSS;
2842 }
2843
2844 /*
2845 * Tunable used to force an interrupt type. The only use is
2846 * for testing of the lesser interrupt types.
2847 * 0 = don't force interrupt type
2848 * 1 = force interrupt type MSIX
2849 * 2 = force interrupt type MSI
2850 * 3 = force interrupt type Legacy
2851 */
2852 igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE,
2853 IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE);
2854
2855 igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE,
2856 0, 1, 1);
2857 igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE,
2858 0, 1, 1);
2859 igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE,
2860 0, 1, 1);
2861 igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE,
2862 0, 1, 1);
2863
2864 /*
2865 * igb LSO needs the tx h/w checksum support.
2866 * Here LSO will be disabled if tx h/w checksum has been disabled.
2867 */
2868 if (igb->tx_hcksum_enable == B_FALSE)
2869 igb->lso_enable = B_FALSE;
2870
2871 igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD,
2872 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
2873 DEFAULT_TX_COPY_THRESHOLD);
2874 igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD,
2875 MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD,
2876 DEFAULT_TX_RECYCLE_THRESHOLD);
2877 igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD,
2878 MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD,
2879 DEFAULT_TX_OVERLOAD_THRESHOLD);
2880 igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD,
2881 MIN_TX_RESCHED_THRESHOLD,
2882 MIN(igb->tx_ring_size, MAX_TX_RESCHED_THRESHOLD),
2883 igb->tx_ring_size > DEFAULT_TX_RESCHED_THRESHOLD ?
2884 DEFAULT_TX_RESCHED_THRESHOLD : DEFAULT_TX_RESCHED_THRESHOLD_LOW);
2885
2886 igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD,
2887 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
2888 DEFAULT_RX_COPY_THRESHOLD);
2889 igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR,
2890 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
2891 DEFAULT_RX_LIMIT_PER_INTR);
2892
2893 igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING,
2894 igb->capab->min_intr_throttle,
2895 igb->capab->max_intr_throttle,
2896 igb->capab->def_intr_throttle);
2897
2898 /*
2899 * Max number of multicast addresses
2900 */
2901 igb->mcast_max_num =
2902 igb_get_prop(igb, PROP_MCAST_MAX_NUM,
2903 MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM);
2904 }
2905
2906 /*
2907 * igb_get_prop - Get a property value out of the configuration file igb.conf
2908 *
2909 * Caller provides the name of the property, a default value, a minimum
2910 * value, and a maximum value.
2911 *
2912 * Return configured value of the property, with default, minimum and
2913 * maximum properly applied.
2914 */
2915 static int
2916 igb_get_prop(igb_t *igb,
2917 char *propname, /* name of the property */
2918 int minval, /* minimum acceptable value */
2919 int maxval, /* maximim acceptable value */
2920 int defval) /* default value */
2921 {
2922 int value;
2923
2924 /*
2925 * Call ddi_prop_get_int() to read the conf settings
2926 */
2927 value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip,
2928 DDI_PROP_DONTPASS, propname, defval);
2929
2930 if (value > maxval)
2931 value = maxval;
2932
2933 if (value < minval)
2934 value = minval;
2935
2936 return (value);
2937 }
2938
2939 /*
2940 * igb_setup_link - Using the link properties to setup the link
2941 */
2942 int
2943 igb_setup_link(igb_t *igb, boolean_t setup_hw)
2944 {
2945 struct e1000_mac_info *mac;
2946 struct e1000_phy_info *phy;
2947 boolean_t invalid;
2948
2949 mac = &igb->hw.mac;
2950 phy = &igb->hw.phy;
2951 invalid = B_FALSE;
2952
2953 if (igb->param_adv_autoneg_cap == 1) {
2954 mac->autoneg = B_TRUE;
2955 phy->autoneg_advertised = 0;
2956
2957 /*
2958 * 1000hdx is not supported for autonegotiation
2959 */
2960 if (igb->param_adv_1000fdx_cap == 1)
2961 phy->autoneg_advertised |= ADVERTISE_1000_FULL;
2962
2963 if (igb->param_adv_100fdx_cap == 1)
2964 phy->autoneg_advertised |= ADVERTISE_100_FULL;
2965
2966 if (igb->param_adv_100hdx_cap == 1)
2967 phy->autoneg_advertised |= ADVERTISE_100_HALF;
2968
2969 if (igb->param_adv_10fdx_cap == 1)
2970 phy->autoneg_advertised |= ADVERTISE_10_FULL;
2971
2972 if (igb->param_adv_10hdx_cap == 1)
2973 phy->autoneg_advertised |= ADVERTISE_10_HALF;
2974
2975 if (phy->autoneg_advertised == 0)
2976 invalid = B_TRUE;
2977 } else {
2978 mac->autoneg = B_FALSE;
2979
2980 /*
2981 * 1000fdx and 1000hdx are not supported for forced link
2982 */
2983 if (igb->param_adv_100fdx_cap == 1)
2984 mac->forced_speed_duplex = ADVERTISE_100_FULL;
2985 else if (igb->param_adv_100hdx_cap == 1)
2986 mac->forced_speed_duplex = ADVERTISE_100_HALF;
2987 else if (igb->param_adv_10fdx_cap == 1)
2988 mac->forced_speed_duplex = ADVERTISE_10_FULL;
2989 else if (igb->param_adv_10hdx_cap == 1)
2990 mac->forced_speed_duplex = ADVERTISE_10_HALF;
2991 else
2992 invalid = B_TRUE;
2993 }
2994
2995 if (invalid) {
2996 igb_notice(igb, "Invalid link settings. Setup link to "
2997 "autonegotiation with full link capabilities.");
2998 mac->autoneg = B_TRUE;
2999 phy->autoneg_advertised = ADVERTISE_1000_FULL |
3000 ADVERTISE_100_FULL | ADVERTISE_100_HALF |
3001 ADVERTISE_10_FULL | ADVERTISE_10_HALF;
3002 }
3003
3004 if (setup_hw) {
3005 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS)
3006 return (IGB_FAILURE);
3007 }
3008
3009 return (IGB_SUCCESS);
3010 }
3011
3012
3013 /*
3014 * igb_is_link_up - Check if the link is up
3015 */
3016 static boolean_t
3017 igb_is_link_up(igb_t *igb)
3018 {
3019 struct e1000_hw *hw = &igb->hw;
3020 boolean_t link_up = B_FALSE;
3021
3022 ASSERT(mutex_owned(&igb->gen_lock));
3023
3024 /*
3025 * get_link_status is set in the interrupt handler on link-status-change
3026 * or rx sequence error interrupt. get_link_status will stay
3027 * false until the e1000_check_for_link establishes link only
3028 * for copper adapters.
3029 */
3030 switch (hw->phy.media_type) {
3031 case e1000_media_type_copper:
3032 if (hw->mac.get_link_status) {
3033 (void) e1000_check_for_link(hw);
3034 link_up = !hw->mac.get_link_status;
3035 } else {
3036 link_up = B_TRUE;
3037 }
3038 break;
3039 case e1000_media_type_fiber:
3040 (void) e1000_check_for_link(hw);
3041 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU);
3042 break;
3043 case e1000_media_type_internal_serdes:
3044 (void) e1000_check_for_link(hw);
3045 link_up = hw->mac.serdes_has_link;
3046 break;
3047 }
3048
3049 return (link_up);
3050 }
3051
3052 /*
3053 * igb_link_check - Link status processing
3054 */
3055 static boolean_t
3056 igb_link_check(igb_t *igb)
3057 {
3058 struct e1000_hw *hw = &igb->hw;
3059 uint16_t speed = 0, duplex = 0;
3060 boolean_t link_changed = B_FALSE;
3061
3062 ASSERT(mutex_owned(&igb->gen_lock));
3063
3064 if (igb_is_link_up(igb)) {
3065 /*
3066 * The Link is up, check whether it was marked as down earlier
3067 */
3068 if (igb->link_state != LINK_STATE_UP) {
3069 (void) e1000_get_speed_and_duplex(hw, &speed, &duplex);
3070 igb->link_speed = speed;
3071 igb->link_duplex = duplex;
3072 igb->link_state = LINK_STATE_UP;
3073 link_changed = B_TRUE;
3074 if (!igb->link_complete)
3075 igb_stop_link_timer(igb);
3076 }
3077 } else if (igb->link_complete) {
3078 if (igb->link_state != LINK_STATE_DOWN) {
3079 igb->link_speed = 0;
3080 igb->link_duplex = 0;
3081 igb->link_state = LINK_STATE_DOWN;
3082 link_changed = B_TRUE;
3083 }
3084 }
3085
3086 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3087 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3088 return (B_FALSE);
3089 }
3090
3091 return (link_changed);
3092 }
3093
3094 /*
3095 * igb_local_timer - driver watchdog function
3096 *
3097 * This function will handle the hardware stall check, link status
3098 * check and other routines.
3099 */
3100 static void
3101 igb_local_timer(void *arg)
3102 {
3103 igb_t *igb = (igb_t *)arg;
3104 boolean_t link_changed = B_FALSE;
3105
3106 if (igb->igb_state & IGB_ERROR) {
3107 igb->reset_count++;
3108 if (igb_reset(igb) == IGB_SUCCESS)
3109 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3110
3111 igb_restart_watchdog_timer(igb);
3112 return;
3113 }
3114
3115 if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) {
3116 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL);
3117 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
3118 igb->reset_count++;
3119 if (igb_reset(igb) == IGB_SUCCESS)
3120 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3121
3122 igb_restart_watchdog_timer(igb);
3123 return;
3124 }
3125
3126 mutex_enter(&igb->gen_lock);
3127 if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED))
3128 link_changed = igb_link_check(igb);
3129 mutex_exit(&igb->gen_lock);
3130
3131 if (link_changed)
3132 mac_link_update(igb->mac_hdl, igb->link_state);
3133
3134 igb_restart_watchdog_timer(igb);
3135 }
3136
3137 /*
3138 * igb_link_timer - link setup timer function
3139 *
3140 * It is called when the timer for link setup is expired, which indicates
3141 * the completion of the link setup. The link state will not be updated
3142 * until the link setup is completed. And the link state will not be sent
3143 * to the upper layer through mac_link_update() in this function. It will
3144 * be updated in the local timer routine or the interrupts service routine
3145 * after the interface is started (plumbed).
3146 */
3147 static void
3148 igb_link_timer(void *arg)
3149 {
3150 igb_t *igb = (igb_t *)arg;
3151
3152 mutex_enter(&igb->link_lock);
3153 igb->link_complete = B_TRUE;
3154 igb->link_tid = 0;
3155 mutex_exit(&igb->link_lock);
3156 }
3157 /*
3158 * igb_stall_check - check for transmit stall
3159 *
3160 * This function checks if the adapter is stalled (in transmit).
3161 *
3162 * It is called each time the watchdog timeout is invoked.
3163 * If the transmit descriptor reclaim continuously fails,
3164 * the watchdog value will increment by 1. If the watchdog
3165 * value exceeds the threshold, the igb is assumed to
3166 * have stalled and need to be reset.
3167 */
3168 static boolean_t
3169 igb_stall_check(igb_t *igb)
3170 {
3171 igb_tx_ring_t *tx_ring;
3172 struct e1000_hw *hw = &igb->hw;
3173 boolean_t result;
3174 int i;
3175
3176 if (igb->link_state != LINK_STATE_UP)
3177 return (B_FALSE);
3178
3179 /*
3180 * If any tx ring is stalled, we'll reset the chipset
3181 */
3182 result = B_FALSE;
3183 for (i = 0; i < igb->num_tx_rings; i++) {
3184 tx_ring = &igb->tx_rings[i];
3185
3186 if (tx_ring->recycle_fail > 0)
3187 tx_ring->stall_watchdog++;
3188 else
3189 tx_ring->stall_watchdog = 0;
3190
3191 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3192 result = B_TRUE;
3193 if (hw->mac.type == e1000_82580) {
3194 hw->dev_spec._82575.global_device_reset
3195 = B_TRUE;
3196 }
3197 break;
3198 }
3199 }
3200
3201 if (result) {
3202 tx_ring->stall_watchdog = 0;
3203 tx_ring->recycle_fail = 0;
3204 }
3205
3206 return (result);
3207 }
3208
3209
3210 /*
3211 * is_valid_mac_addr - Check if the mac address is valid
3212 */
3213 static boolean_t
3214 is_valid_mac_addr(uint8_t *mac_addr)
3215 {
3216 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3217 const uint8_t addr_test2[6] =
3218 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3219
3220 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3221 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3222 return (B_FALSE);
3223
3224 return (B_TRUE);
3225 }
3226
3227 static boolean_t
3228 igb_find_mac_address(igb_t *igb)
3229 {
3230 struct e1000_hw *hw = &igb->hw;
3231 #ifdef __sparc
3232 uchar_t *bytes;
3233 struct ether_addr sysaddr;
3234 uint_t nelts;
3235 int err;
3236 boolean_t found = B_FALSE;
3237
3238 /*
3239 * The "vendor's factory-set address" may already have
3240 * been extracted from the chip, but if the property
3241 * "local-mac-address" is set we use that instead.
3242 *
3243 * We check whether it looks like an array of 6
3244 * bytes (which it should, if OBP set it). If we can't
3245 * make sense of it this way, we'll ignore it.
3246 */
3247 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3248 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3249 if (err == DDI_PROP_SUCCESS) {
3250 if (nelts == ETHERADDRL) {
3251 while (nelts--)
3252 hw->mac.addr[nelts] = bytes[nelts];
3253 found = B_TRUE;
3254 }
3255 ddi_prop_free(bytes);
3256 }
3257
3258 /*
3259 * Look up the OBP property "local-mac-address?". If the user has set
3260 * 'local-mac-address? = false', use "the system address" instead.
3261 */
3262 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0,
3263 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3264 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3265 if (localetheraddr(NULL, &sysaddr) != 0) {
3266 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3267 found = B_TRUE;
3268 }
3269 }
3270 ddi_prop_free(bytes);
3271 }
3272
3273 /*
3274 * Finally(!), if there's a valid "mac-address" property (created
3275 * if we netbooted from this interface), we must use this instead
3276 * of any of the above to ensure that the NFS/install server doesn't
3277 * get confused by the address changing as Solaris takes over!
3278 */
3279 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3280 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3281 if (err == DDI_PROP_SUCCESS) {
3282 if (nelts == ETHERADDRL) {
3283 while (nelts--)
3284 hw->mac.addr[nelts] = bytes[nelts];
3285 found = B_TRUE;
3286 }
3287 ddi_prop_free(bytes);
3288 }
3289
3290 if (found) {
3291 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3292 return (B_TRUE);
3293 }
3294 #endif
3295
3296 /*
3297 * Read the device MAC address from the EEPROM
3298 */
3299 if (e1000_read_mac_addr(hw) != E1000_SUCCESS)
3300 return (B_FALSE);
3301
3302 return (B_TRUE);
3303 }
3304
3305 #pragma inline(igb_arm_watchdog_timer)
3306
3307 static void
3308 igb_arm_watchdog_timer(igb_t *igb)
3309 {
3310 /*
3311 * Fire a watchdog timer
3312 */
3313 igb->watchdog_tid =
3314 timeout(igb_local_timer,
3315 (void *)igb, 1 * drv_usectohz(1000000));
3316
3317 }
3318
3319 /*
3320 * igb_enable_watchdog_timer - Enable and start the driver watchdog timer
3321 */
3322 void
3323 igb_enable_watchdog_timer(igb_t *igb)
3324 {
3325 mutex_enter(&igb->watchdog_lock);
3326
3327 if (!igb->watchdog_enable) {
3328 igb->watchdog_enable = B_TRUE;
3329 igb->watchdog_start = B_TRUE;
3330 igb_arm_watchdog_timer(igb);
3331 }
3332
3333 mutex_exit(&igb->watchdog_lock);
3334
3335 }
3336
3337 /*
3338 * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer
3339 */
3340 void
3341 igb_disable_watchdog_timer(igb_t *igb)
3342 {
3343 timeout_id_t tid;
3344
3345 mutex_enter(&igb->watchdog_lock);
3346
3347 igb->watchdog_enable = B_FALSE;
3348 igb->watchdog_start = B_FALSE;
3349 tid = igb->watchdog_tid;
3350 igb->watchdog_tid = 0;
3351
3352 mutex_exit(&igb->watchdog_lock);
3353
3354 if (tid != 0)
3355 (void) untimeout(tid);
3356
3357 }
3358
3359 /*
3360 * igb_start_watchdog_timer - Start the driver watchdog timer
3361 */
3362 static void
3363 igb_start_watchdog_timer(igb_t *igb)
3364 {
3365 mutex_enter(&igb->watchdog_lock);
3366
3367 if (igb->watchdog_enable) {
3368 if (!igb->watchdog_start) {
3369 igb->watchdog_start = B_TRUE;
3370 igb_arm_watchdog_timer(igb);
3371 }
3372 }
3373
3374 mutex_exit(&igb->watchdog_lock);
3375 }
3376
3377 /*
3378 * igb_restart_watchdog_timer - Restart the driver watchdog timer
3379 */
3380 static void
3381 igb_restart_watchdog_timer(igb_t *igb)
3382 {
3383 mutex_enter(&igb->watchdog_lock);
3384
3385 if (igb->watchdog_start)
3386 igb_arm_watchdog_timer(igb);
3387
3388 mutex_exit(&igb->watchdog_lock);
3389 }
3390
3391 /*
3392 * igb_stop_watchdog_timer - Stop the driver watchdog timer
3393 */
3394 static void
3395 igb_stop_watchdog_timer(igb_t *igb)
3396 {
3397 timeout_id_t tid;
3398
3399 mutex_enter(&igb->watchdog_lock);
3400
3401 igb->watchdog_start = B_FALSE;
3402 tid = igb->watchdog_tid;
3403 igb->watchdog_tid = 0;
3404
3405 mutex_exit(&igb->watchdog_lock);
3406
3407 if (tid != 0)
3408 (void) untimeout(tid);
3409 }
3410
3411 /*
3412 * igb_start_link_timer - Start the link setup timer
3413 */
3414 static void
3415 igb_start_link_timer(struct igb *igb)
3416 {
3417 struct e1000_hw *hw = &igb->hw;
3418 clock_t link_timeout;
3419
3420 if (hw->mac.autoneg)
3421 link_timeout = PHY_AUTO_NEG_LIMIT *
3422 drv_usectohz(100000);
3423 else
3424 link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000);
3425
3426 mutex_enter(&igb->link_lock);
3427 if (hw->phy.autoneg_wait_to_complete) {
3428 igb->link_complete = B_TRUE;
3429 } else {
3430 igb->link_complete = B_FALSE;
3431 igb->link_tid = timeout(igb_link_timer, (void *)igb,
3432 link_timeout);
3433 }
3434 mutex_exit(&igb->link_lock);
3435 }
3436
3437 /*
3438 * igb_stop_link_timer - Stop the link setup timer
3439 */
3440 static void
3441 igb_stop_link_timer(struct igb *igb)
3442 {
3443 timeout_id_t tid;
3444
3445 mutex_enter(&igb->link_lock);
3446 igb->link_complete = B_TRUE;
3447 tid = igb->link_tid;
3448 igb->link_tid = 0;
3449 mutex_exit(&igb->link_lock);
3450
3451 if (tid != 0)
3452 (void) untimeout(tid);
3453 }
3454
3455 /*
3456 * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts
3457 */
3458 static void
3459 igb_disable_adapter_interrupts(igb_t *igb)
3460 {
3461 struct e1000_hw *hw = &igb->hw;
3462
3463 /*
3464 * Set the IMC register to mask all the interrupts,
3465 * including the tx interrupts.
3466 */
3467 E1000_WRITE_REG(hw, E1000_IMC, ~0);
3468 E1000_WRITE_REG(hw, E1000_IAM, 0);
3469
3470 /*
3471 * Additional disabling for MSI-X
3472 */
3473 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3474 E1000_WRITE_REG(hw, E1000_EIMC, ~0);
3475 E1000_WRITE_REG(hw, E1000_EIAC, 0);
3476 E1000_WRITE_REG(hw, E1000_EIAM, 0);
3477 }
3478
3479 E1000_WRITE_FLUSH(hw);
3480 }
3481
3482 /*
3483 * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580
3484 */
3485 static void
3486 igb_enable_adapter_interrupts_82580(igb_t *igb)
3487 {
3488 struct e1000_hw *hw = &igb->hw;
3489
3490 /* Clear any pending interrupts */
3491 (void) E1000_READ_REG(hw, E1000_ICR);
3492 igb->ims_mask |= E1000_IMS_DRSTA;
3493
3494 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3495
3496 /* Interrupt enabling for MSI-X */
3497 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3498 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3499 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA);
3500 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3501 } else { /* Interrupt enabling for MSI and legacy */
3502 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3503 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3504 igb->ims_mask |= E1000_IMS_DRSTA;
3505 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3506 }
3507
3508 /* Disable auto-mask for ICR interrupt bits */
3509 E1000_WRITE_REG(hw, E1000_IAM, 0);
3510
3511 E1000_WRITE_FLUSH(hw);
3512 }
3513
3514 /*
3515 * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576
3516 */
3517 static void
3518 igb_enable_adapter_interrupts_82576(igb_t *igb)
3519 {
3520 struct e1000_hw *hw = &igb->hw;
3521
3522 /* Clear any pending interrupts */
3523 (void) E1000_READ_REG(hw, E1000_ICR);
3524
3525 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3526
3527 /* Interrupt enabling for MSI-X */
3528 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3529 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3530 igb->ims_mask = E1000_IMS_LSC;
3531 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3532 } else {
3533 /* Interrupt enabling for MSI and legacy */
3534 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3535 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3536 E1000_WRITE_REG(hw, E1000_IMS,
3537 (IMS_ENABLE_MASK | E1000_IMS_TXQE));
3538 }
3539
3540 /* Disable auto-mask for ICR interrupt bits */
3541 E1000_WRITE_REG(hw, E1000_IAM, 0);
3542
3543 E1000_WRITE_FLUSH(hw);
3544 }
3545
3546 /*
3547 * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575
3548 */
3549 static void
3550 igb_enable_adapter_interrupts_82575(igb_t *igb)
3551 {
3552 struct e1000_hw *hw = &igb->hw;
3553 uint32_t reg;
3554
3555 /* Clear any pending interrupts */
3556 (void) E1000_READ_REG(hw, E1000_ICR);
3557
3558 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3559 /* Interrupt enabling for MSI-X */
3560 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3561 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3562 igb->ims_mask = E1000_IMS_LSC;
3563 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3564
3565 /* Enable MSI-X PBA support */
3566 reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
3567 reg |= E1000_CTRL_EXT_PBA_CLR;
3568
3569 /* Non-selective interrupt clear-on-read */
3570 reg |= E1000_CTRL_EXT_IRCA; /* Called NSICR in the EAS */
3571
3572 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
3573 } else {
3574 /* Interrupt enabling for MSI and legacy */
3575 igb->ims_mask = IMS_ENABLE_MASK;
3576 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK);
3577 }
3578
3579 E1000_WRITE_FLUSH(hw);
3580 }
3581
3582 /*
3583 * Loopback Support
3584 */
3585 static lb_property_t lb_normal =
3586 { normal, "normal", IGB_LB_NONE };
3587 static lb_property_t lb_external =
3588 { external, "External", IGB_LB_EXTERNAL };
3589 static lb_property_t lb_phy =
3590 { internal, "PHY", IGB_LB_INTERNAL_PHY };
3591 static lb_property_t lb_serdes =
3592 { internal, "SerDes", IGB_LB_INTERNAL_SERDES };
3593
3594 enum ioc_reply
3595 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp)
3596 {
3597 lb_info_sz_t *lbsp;
3598 lb_property_t *lbpp;
3599 struct e1000_hw *hw;
3600 uint32_t *lbmp;
3601 uint32_t size;
3602 uint32_t value;
3603
3604 hw = &igb->hw;
3605
3606 if (mp->b_cont == NULL)
3607 return (IOC_INVAL);
3608
3609 switch (iocp->ioc_cmd) {
3610 default:
3611 return (IOC_INVAL);
3612
3613 case LB_GET_INFO_SIZE:
3614 size = sizeof (lb_info_sz_t);
3615 if (iocp->ioc_count != size)
3616 return (IOC_INVAL);
3617
3618 value = sizeof (lb_normal);
3619 if (hw->phy.media_type == e1000_media_type_copper)
3620 value += sizeof (lb_phy);
3621 else
3622 value += sizeof (lb_serdes);
3623 value += sizeof (lb_external);
3624
3625 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3626 *lbsp = value;
3627 break;
3628
3629 case LB_GET_INFO:
3630 value = sizeof (lb_normal);
3631 if (hw->phy.media_type == e1000_media_type_copper)
3632 value += sizeof (lb_phy);
3633 else
3634 value += sizeof (lb_serdes);
3635 value += sizeof (lb_external);
3636
3637 size = value;
3638 if (iocp->ioc_count != size)
3639 return (IOC_INVAL);
3640
3641 value = 0;
3642 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3643
3644 lbpp[value++] = lb_normal;
3645 if (hw->phy.media_type == e1000_media_type_copper)
3646 lbpp[value++] = lb_phy;
3647 else
3648 lbpp[value++] = lb_serdes;
3649 lbpp[value++] = lb_external;
3650 break;
3651
3652 case LB_GET_MODE:
3653 size = sizeof (uint32_t);
3654 if (iocp->ioc_count != size)
3655 return (IOC_INVAL);
3656
3657 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3658 *lbmp = igb->loopback_mode;
3659 break;
3660
3661 case LB_SET_MODE:
3662 size = 0;
3663 if (iocp->ioc_count != sizeof (uint32_t))
3664 return (IOC_INVAL);
3665
3666 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3667 if (!igb_set_loopback_mode(igb, *lbmp))
3668 return (IOC_INVAL);
3669 break;
3670 }
3671
3672 iocp->ioc_count = size;
3673 iocp->ioc_error = 0;
3674
3675 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3676 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3677 return (IOC_INVAL);
3678 }
3679
3680 return (IOC_REPLY);
3681 }
3682
3683 /*
3684 * igb_set_loopback_mode - Setup loopback based on the loopback mode
3685 */
3686 static boolean_t
3687 igb_set_loopback_mode(igb_t *igb, uint32_t mode)
3688 {
3689 struct e1000_hw *hw;
3690 int i;
3691
3692 if (mode == igb->loopback_mode)
3693 return (B_TRUE);
3694
3695 hw = &igb->hw;
3696
3697 igb->loopback_mode = mode;
3698
3699 if (mode == IGB_LB_NONE) {
3700 /* Reset the chip */
3701 hw->phy.autoneg_wait_to_complete = B_TRUE;
3702 (void) igb_reset(igb);
3703 hw->phy.autoneg_wait_to_complete = B_FALSE;
3704 return (B_TRUE);
3705 }
3706
3707 mutex_enter(&igb->gen_lock);
3708
3709 switch (mode) {
3710 default:
3711 mutex_exit(&igb->gen_lock);
3712 return (B_FALSE);
3713
3714 case IGB_LB_EXTERNAL:
3715 igb_set_external_loopback(igb);
3716 break;
3717
3718 case IGB_LB_INTERNAL_PHY:
3719 igb_set_internal_phy_loopback(igb);
3720 break;
3721
3722 case IGB_LB_INTERNAL_SERDES:
3723 igb_set_internal_serdes_loopback(igb);
3724 break;
3725 }
3726
3727 mutex_exit(&igb->gen_lock);
3728
3729 /*
3730 * When external loopback is set, wait up to 1000ms to get the link up.
3731 * According to test, 1000ms can work and it's an experimental value.
3732 */
3733 if (mode == IGB_LB_EXTERNAL) {
3734 for (i = 0; i <= 10; i++) {
3735 mutex_enter(&igb->gen_lock);
3736 (void) igb_link_check(igb);
3737 mutex_exit(&igb->gen_lock);
3738
3739 if (igb->link_state == LINK_STATE_UP)
3740 break;
3741
3742 msec_delay(100);
3743 }
3744
3745 if (igb->link_state != LINK_STATE_UP) {
3746 /*
3747 * Does not support external loopback.
3748 * Reset driver to loopback none.
3749 */
3750 igb->loopback_mode = IGB_LB_NONE;
3751
3752 /* Reset the chip */
3753 hw->phy.autoneg_wait_to_complete = B_TRUE;
3754 (void) igb_reset(igb);
3755 hw->phy.autoneg_wait_to_complete = B_FALSE;
3756
3757 IGB_DEBUGLOG_0(igb, "Set external loopback failed, "
3758 "reset to loopback none.");
3759
3760 return (B_FALSE);
3761 }
3762 }
3763
3764 return (B_TRUE);
3765 }
3766
3767 /*
3768 * igb_set_external_loopback - Set the external loopback mode
3769 */
3770 static void
3771 igb_set_external_loopback(igb_t *igb)
3772 {
3773 struct e1000_hw *hw;
3774 uint32_t ctrl_ext;
3775
3776 hw = &igb->hw;
3777
3778 /* Set link mode to PHY (00b) in the Extended Control register */
3779 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3780 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3781 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3782
3783 (void) e1000_write_phy_reg(hw, 0x0, 0x0140);
3784 (void) e1000_write_phy_reg(hw, 0x9, 0x1a00);
3785 (void) e1000_write_phy_reg(hw, 0x12, 0x1610);
3786 (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c);
3787 }
3788
3789 /*
3790 * igb_set_internal_phy_loopback - Set the internal PHY loopback mode
3791 */
3792 static void
3793 igb_set_internal_phy_loopback(igb_t *igb)
3794 {
3795 struct e1000_hw *hw;
3796 uint32_t ctrl_ext;
3797 uint16_t phy_ctrl;
3798 uint16_t phy_pconf;
3799
3800 hw = &igb->hw;
3801
3802 /* Set link mode to PHY (00b) in the Extended Control register */
3803 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3804 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3805 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3806
3807 /*
3808 * Set PHY control register (0x4140):
3809 * Set full duplex mode
3810 * Set loopback bit
3811 * Clear auto-neg enable bit
3812 * Set PHY speed
3813 */
3814 phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK;
3815 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl);
3816
3817 /* Set the link disable bit in the Port Configuration register */
3818 (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf);
3819 phy_pconf |= (uint16_t)1 << 14;
3820 (void) e1000_write_phy_reg(hw, 0x10, phy_pconf);
3821 }
3822
3823 /*
3824 * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode
3825 */
3826 static void
3827 igb_set_internal_serdes_loopback(igb_t *igb)
3828 {
3829 struct e1000_hw *hw;
3830 uint32_t ctrl_ext;
3831 uint32_t ctrl;
3832 uint32_t pcs_lctl;
3833 uint32_t connsw;
3834
3835 hw = &igb->hw;
3836
3837 /* Set link mode to SerDes (11b) in the Extended Control register */
3838 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3839 ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
3840 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3841
3842 /* Configure the SerDes to loopback */
3843 E1000_WRITE_REG(hw, E1000_SCTL, 0x410);
3844
3845 /* Set Device Control register */
3846 ctrl = E1000_READ_REG(hw, E1000_CTRL);
3847 ctrl |= (E1000_CTRL_FD | /* Force full duplex */
3848 E1000_CTRL_SLU); /* Force link up */
3849 ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */
3850 E1000_CTRL_TFCE | /* Disable transmit flow control */
3851 E1000_CTRL_LRST); /* Clear link reset */
3852 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
3853
3854 /* Set PCS Link Control register */
3855 pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL);
3856 pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK |
3857 E1000_PCS_LCTL_FSD |
3858 E1000_PCS_LCTL_FDV_FULL |
3859 E1000_PCS_LCTL_FLV_LINK_UP);
3860 pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE;
3861 E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl);
3862
3863 /* Set the Copper/Fiber Switch Control - CONNSW register */
3864 connsw = E1000_READ_REG(hw, E1000_CONNSW);
3865 connsw &= ~E1000_CONNSW_ENRGSRC;
3866 E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
3867 }
3868
3869 #pragma inline(igb_intr_rx_work)
3870 /*
3871 * igb_intr_rx_work - rx processing of ISR
3872 */
3873 static void
3874 igb_intr_rx_work(igb_rx_ring_t *rx_ring)
3875 {
3876 mblk_t *mp;
3877
3878 mutex_enter(&rx_ring->rx_lock);
3879 mp = igb_rx(rx_ring, IGB_NO_POLL);
3880 mutex_exit(&rx_ring->rx_lock);
3881
3882 if (mp != NULL)
3883 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp,
3884 rx_ring->ring_gen_num);
3885 }
3886
3887 #pragma inline(igb_intr_tx_work)
3888 /*
3889 * igb_intr_tx_work - tx processing of ISR
3890 */
3891 static void
3892 igb_intr_tx_work(igb_tx_ring_t *tx_ring)
3893 {
3894 igb_t *igb = tx_ring->igb;
3895
3896 /* Recycle the tx descriptors */
3897 tx_ring->tx_recycle(tx_ring);
3898
3899 /* Schedule the re-transmit */
3900 if (tx_ring->reschedule &&
3901 (tx_ring->tbd_free >= igb->tx_resched_thresh)) {
3902 tx_ring->reschedule = B_FALSE;
3903 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle);
3904 IGB_DEBUG_STAT(tx_ring->stat_reschedule);
3905 }
3906 }
3907
3908 #pragma inline(igb_intr_link_work)
3909 /*
3910 * igb_intr_link_work - link-status-change processing of ISR
3911 */
3912 static void
3913 igb_intr_link_work(igb_t *igb)
3914 {
3915 boolean_t link_changed;
3916
3917 igb_stop_watchdog_timer(igb);
3918
3919 mutex_enter(&igb->gen_lock);
3920
3921 /*
3922 * Because we got a link-status-change interrupt, force
3923 * e1000_check_for_link() to look at phy
3924 */
3925 igb->hw.mac.get_link_status = B_TRUE;
3926
3927 /* igb_link_check takes care of link status change */
3928 link_changed = igb_link_check(igb);
3929
3930 /* Get new phy state */
3931 igb_get_phy_state(igb);
3932
3933 mutex_exit(&igb->gen_lock);
3934
3935 if (link_changed)
3936 mac_link_update(igb->mac_hdl, igb->link_state);
3937
3938 igb_start_watchdog_timer(igb);
3939 }
3940
3941 /*
3942 * igb_intr_legacy - Interrupt handler for legacy interrupts
3943 */
3944 static uint_t
3945 igb_intr_legacy(void *arg1, void *arg2)
3946 {
3947 igb_t *igb = (igb_t *)arg1;
3948 igb_tx_ring_t *tx_ring;
3949 uint32_t icr;
3950 mblk_t *mp;
3951 boolean_t tx_reschedule;
3952 boolean_t link_changed;
3953 uint_t result;
3954
3955 _NOTE(ARGUNUSED(arg2));
3956
3957 mutex_enter(&igb->gen_lock);
3958
3959 if (igb->igb_state & IGB_SUSPENDED) {
3960 mutex_exit(&igb->gen_lock);
3961 return (DDI_INTR_UNCLAIMED);
3962 }
3963
3964 mp = NULL;
3965 tx_reschedule = B_FALSE;
3966 link_changed = B_FALSE;
3967 icr = E1000_READ_REG(&igb->hw, E1000_ICR);
3968
3969 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3970 mutex_exit(&igb->gen_lock);
3971 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3972 atomic_or_32(&igb->igb_state, IGB_ERROR);
3973 return (DDI_INTR_UNCLAIMED);
3974 }
3975
3976 if (icr & E1000_ICR_INT_ASSERTED) {
3977 /*
3978 * E1000_ICR_INT_ASSERTED bit was set:
3979 * Read(Clear) the ICR, claim this interrupt,
3980 * look for work to do.
3981 */
3982 ASSERT(igb->num_rx_rings == 1);
3983 ASSERT(igb->num_tx_rings == 1);
3984
3985 /* Make sure all interrupt causes cleared */
3986 (void) E1000_READ_REG(&igb->hw, E1000_EICR);
3987
3988 if (icr & E1000_ICR_RXT0) {
3989 mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL);
3990 }
3991
3992 if (icr & E1000_ICR_TXDW) {
3993 tx_ring = &igb->tx_rings[0];
3994
3995 /* Recycle the tx descriptors */
3996 tx_ring->tx_recycle(tx_ring);
3997
3998 /* Schedule the re-transmit */
3999 tx_reschedule = (tx_ring->reschedule &&
4000 (tx_ring->tbd_free >= igb->tx_resched_thresh));
4001 }
4002
4003 if (icr & E1000_ICR_LSC) {
4004 /*
4005 * Because we got a link-status-change interrupt, force
4006 * e1000_check_for_link() to look at phy
4007 */
4008 igb->hw.mac.get_link_status = B_TRUE;
4009
4010 /* igb_link_check takes care of link status change */
4011 link_changed = igb_link_check(igb);
4012
4013 /* Get new phy state */
4014 igb_get_phy_state(igb);
4015 }
4016
4017 if (icr & E1000_ICR_DRSTA) {
4018 /* 82580 Full Device Reset needed */
4019 atomic_or_32(&igb->igb_state, IGB_STALL);
4020 }
4021
4022 result = DDI_INTR_CLAIMED;
4023 } else {
4024 /*
4025 * E1000_ICR_INT_ASSERTED bit was not set:
4026 * Don't claim this interrupt.
4027 */
4028 result = DDI_INTR_UNCLAIMED;
4029 }
4030
4031 mutex_exit(&igb->gen_lock);
4032
4033 /*
4034 * Do the following work outside of the gen_lock
4035 */
4036 if (mp != NULL)
4037 mac_rx(igb->mac_hdl, NULL, mp);
4038
4039 if (tx_reschedule) {
4040 tx_ring->reschedule = B_FALSE;
4041 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle);
4042 IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4043 }
4044
4045 if (link_changed)
4046 mac_link_update(igb->mac_hdl, igb->link_state);
4047
4048 return (result);
4049 }
4050
4051 /*
4052 * igb_intr_msi - Interrupt handler for MSI
4053 */
4054 static uint_t
4055 igb_intr_msi(void *arg1, void *arg2)
4056 {
4057 igb_t *igb = (igb_t *)arg1;
4058 uint32_t icr;
4059
4060 _NOTE(ARGUNUSED(arg2));
4061
4062 icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4063
4064 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4065 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4066 atomic_or_32(&igb->igb_state, IGB_ERROR);
4067 return (DDI_INTR_CLAIMED);
4068 }
4069
4070 /* Make sure all interrupt causes cleared */
4071 (void) E1000_READ_REG(&igb->hw, E1000_EICR);
4072
4073 /*
4074 * For MSI interrupt, we have only one vector,
4075 * so we have only one rx ring and one tx ring enabled.
4076 */
4077 ASSERT(igb->num_rx_rings == 1);
4078 ASSERT(igb->num_tx_rings == 1);
4079
4080 if (icr & E1000_ICR_RXT0) {
4081 igb_intr_rx_work(&igb->rx_rings[0]);
4082 }
4083
4084 if (icr & E1000_ICR_TXDW) {
4085 igb_intr_tx_work(&igb->tx_rings[0]);
4086 }
4087
4088 if (icr & E1000_ICR_LSC) {
4089 igb_intr_link_work(igb);
4090 }
4091
4092 if (icr & E1000_ICR_DRSTA) {
4093 /* 82580 Full Device Reset needed */
4094 atomic_or_32(&igb->igb_state, IGB_STALL);
4095 }
4096
4097 return (DDI_INTR_CLAIMED);
4098 }
4099
4100 /*
4101 * igb_intr_rx - Interrupt handler for rx
4102 */
4103 static uint_t
4104 igb_intr_rx(void *arg1, void *arg2)
4105 {
4106 igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1;
4107
4108 _NOTE(ARGUNUSED(arg2));
4109
4110 /*
4111 * Only used via MSI-X vector so don't check cause bits
4112 * and only clean the given ring.
4113 */
4114 igb_intr_rx_work(rx_ring);
4115
4116 return (DDI_INTR_CLAIMED);
4117 }
4118
4119 /*
4120 * igb_intr_tx - Interrupt handler for tx
4121 */
4122 static uint_t
4123 igb_intr_tx(void *arg1, void *arg2)
4124 {
4125 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1;
4126
4127 _NOTE(ARGUNUSED(arg2));
4128
4129 /*
4130 * Only used via MSI-X vector so don't check cause bits
4131 * and only clean the given ring.
4132 */
4133 igb_intr_tx_work(tx_ring);
4134
4135 return (DDI_INTR_CLAIMED);
4136 }
4137
4138 /*
4139 * igb_intr_tx_other - Interrupt handler for both tx and other
4140 *
4141 */
4142 static uint_t
4143 igb_intr_tx_other(void *arg1, void *arg2)
4144 {
4145 igb_t *igb = (igb_t *)arg1;
4146 uint32_t icr;
4147
4148 _NOTE(ARGUNUSED(arg2));
4149
4150 icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4151
4152 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4153 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4154 atomic_or_32(&igb->igb_state, IGB_ERROR);
4155 return (DDI_INTR_CLAIMED);
4156 }
4157
4158 /*
4159 * Look for tx reclaiming work first. Remember, in the
4160 * case of only interrupt sharing, only one tx ring is
4161 * used
4162 */
4163 igb_intr_tx_work(&igb->tx_rings[0]);
4164
4165 /*
4166 * Check for "other" causes.
4167 */
4168 if (icr & E1000_ICR_LSC) {
4169 igb_intr_link_work(igb);
4170 }
4171
4172 /*
4173 * The DOUTSYNC bit indicates a tx packet dropped because
4174 * DMA engine gets "out of sync". There isn't a real fix
4175 * for this. The Intel recommendation is to count the number
4176 * of occurrences so user can detect when it is happening.
4177 * The issue is non-fatal and there's no recovery action
4178 * available.
4179 */
4180 if (icr & E1000_ICR_DOUTSYNC) {
4181 IGB_STAT(igb->dout_sync);
4182 }
4183
4184 if (icr & E1000_ICR_DRSTA) {
4185 /* 82580 Full Device Reset needed */
4186 atomic_or_32(&igb->igb_state, IGB_STALL);
4187 }
4188
4189 return (DDI_INTR_CLAIMED);
4190 }
4191
4192 /*
4193 * igb_alloc_intrs - Allocate interrupts for the driver
4194 *
4195 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4196 * if not successful, try Legacy.
4197 * igb->intr_force can be used to force sequence to start with
4198 * any of the 3 types.
4199 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4200 */
4201 static int
4202 igb_alloc_intrs(igb_t *igb)
4203 {
4204 dev_info_t *devinfo;
4205 int intr_types;
4206 int rc;
4207
4208 devinfo = igb->dip;
4209
4210 /* Get supported interrupt types */
4211 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4212
4213 if (rc != DDI_SUCCESS) {
4214 igb_log(igb,
4215 "Get supported interrupt types failed: %d", rc);
4216 return (IGB_FAILURE);
4217 }
4218 IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types);
4219
4220 igb->intr_type = 0;
4221
4222 /* Install MSI-X interrupts */
4223 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4224 (igb->intr_force <= IGB_INTR_MSIX)) {
4225 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX);
4226
4227 if (rc == IGB_SUCCESS)
4228 return (IGB_SUCCESS);
4229
4230 igb_log(igb,
4231 "Allocate MSI-X failed, trying MSI interrupts...");
4232 }
4233
4234 /* MSI-X not used, force rings to 1 */
4235 igb->num_rx_rings = 1;
4236 igb->num_tx_rings = 1;
4237 igb_log(igb,
4238 "MSI-X not used, force rx and tx queue number to 1");
4239
4240 /* Install MSI interrupts */
4241 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4242 (igb->intr_force <= IGB_INTR_MSI)) {
4243 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI);
4244
4245 if (rc == IGB_SUCCESS)
4246 return (IGB_SUCCESS);
4247
4248 igb_log(igb,
4249 "Allocate MSI failed, trying Legacy interrupts...");
4250 }
4251
4252 /* Install legacy interrupts */
4253 if (intr_types & DDI_INTR_TYPE_FIXED) {
4254 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED);
4255
4256 if (rc == IGB_SUCCESS)
4257 return (IGB_SUCCESS);
4258
4259 igb_log(igb,
4260 "Allocate Legacy interrupts failed");
4261 }
4262
4263 /* If none of the 3 types succeeded, return failure */
4264 return (IGB_FAILURE);
4265 }
4266
4267 /*
4268 * igb_alloc_intr_handles - Allocate interrupt handles.
4269 *
4270 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4271 * if fewer than 2 handles are available, return failure.
4272 * Upon success, this sets the number of Rx rings to a number that
4273 * matches the handles available for Rx interrupts.
4274 */
4275 static int
4276 igb_alloc_intr_handles(igb_t *igb, int intr_type)
4277 {
4278 dev_info_t *devinfo;
4279 int orig, request, count, avail, actual;
4280 int diff, minimum;
4281 int rc;
4282
4283 devinfo = igb->dip;
4284
4285 switch (intr_type) {
4286 case DDI_INTR_TYPE_FIXED:
4287 request = 1; /* Request 1 legacy interrupt handle */
4288 minimum = 1;
4289 IGB_DEBUGLOG_0(igb, "interrupt type: legacy");
4290 break;
4291
4292 case DDI_INTR_TYPE_MSI:
4293 request = 1; /* Request 1 MSI interrupt handle */
4294 minimum = 1;
4295 IGB_DEBUGLOG_0(igb, "interrupt type: MSI");
4296 break;
4297
4298 case DDI_INTR_TYPE_MSIX:
4299 /*
4300 * Number of vectors for the adapter is
4301 * # rx rings + # tx rings
4302 * One of tx vectors is for tx & other
4303 */
4304 request = igb->num_rx_rings + igb->num_tx_rings;
4305 orig = request;
4306 minimum = 2;
4307 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X");
4308 break;
4309
4310 default:
4311 igb_log(igb,
4312 "invalid call to igb_alloc_intr_handles(): %d\n",
4313 intr_type);
4314 return (IGB_FAILURE);
4315 }
4316 IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d minimum: %d",
4317 request, minimum);
4318
4319 /*
4320 * Get number of supported interrupts
4321 */
4322 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4323 if ((rc != DDI_SUCCESS) || (count < minimum)) {
4324 igb_log(igb,
4325 "Get supported interrupt number failed. "
4326 "Return: %d, count: %d", rc, count);
4327 return (IGB_FAILURE);
4328 }
4329 IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count);
4330
4331 /*
4332 * Get number of available interrupts
4333 */
4334 rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
4335 if ((rc != DDI_SUCCESS) || (avail < minimum)) {
4336 igb_log(igb,
4337 "Get available interrupt number failed. "
4338 "Return: %d, available: %d", rc, avail);
4339 return (IGB_FAILURE);
4340 }
4341 IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail);
4342
4343 if (avail < request) {
4344 igb_log(igb, "Request %d handles, %d available",
4345 request, avail);
4346 request = avail;
4347 }
4348
4349 actual = 0;
4350 igb->intr_cnt = 0;
4351
4352 /*
4353 * Allocate an array of interrupt handles
4354 */
4355 igb->intr_size = request * sizeof (ddi_intr_handle_t);
4356 igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP);
4357
4358 rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0,
4359 request, &actual, DDI_INTR_ALLOC_NORMAL);
4360 if (rc != DDI_SUCCESS) {
4361 igb_log(igb, "Allocate interrupts failed. "
4362 "return: %d, request: %d, actual: %d",
4363 rc, request, actual);
4364 goto alloc_handle_fail;
4365 }
4366 IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual);
4367
4368 igb->intr_cnt = actual;
4369
4370 if (actual < minimum) {
4371 igb_log(igb, "Insufficient interrupt handles allocated: %d",
4372 actual);
4373 goto alloc_handle_fail;
4374 }
4375
4376 /*
4377 * For MSI-X, actual might force us to reduce number of tx & rx rings
4378 */
4379 if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) {
4380 diff = orig - actual;
4381 if (diff < igb->num_tx_rings) {
4382 igb_log(igb,
4383 "MSI-X vectors force Tx queue number to %d",
4384 igb->num_tx_rings - diff);
4385 igb->num_tx_rings -= diff;
4386 } else {
4387 igb_log(igb,
4388 "MSI-X vectors force Tx queue number to 1");
4389 igb->num_tx_rings = 1;
4390
4391 igb_log(igb,
4392 "MSI-X vectors force Rx queue number to %d",
4393 actual - 1);
4394 igb->num_rx_rings = actual - 1;
4395 }
4396 }
4397
4398 /*
4399 * Get priority for first vector, assume remaining are all the same
4400 */
4401 rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri);
4402 if (rc != DDI_SUCCESS) {
4403 igb_log(igb,
4404 "Get interrupt priority failed: %d", rc);
4405 goto alloc_handle_fail;
4406 }
4407
4408 rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap);
4409 if (rc != DDI_SUCCESS) {
4410 igb_log(igb,
4411 "Get interrupt cap failed: %d", rc);
4412 goto alloc_handle_fail;
4413 }
4414
4415 igb->intr_type = intr_type;
4416
4417 return (IGB_SUCCESS);
4418
4419 alloc_handle_fail:
4420 igb_rem_intrs(igb);
4421
4422 return (IGB_FAILURE);
4423 }
4424
4425 /*
4426 * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type
4427 *
4428 * Before adding the interrupt handlers, the interrupt vectors have
4429 * been allocated, and the rx/tx rings have also been allocated.
4430 */
4431 static int
4432 igb_add_intr_handlers(igb_t *igb)
4433 {
4434 igb_rx_ring_t *rx_ring;
4435 igb_tx_ring_t *tx_ring;
4436 int vector;
4437 int rc;
4438 int i;
4439
4440 vector = 0;
4441
4442 switch (igb->intr_type) {
4443 case DDI_INTR_TYPE_MSIX:
4444 /* Add interrupt handler for tx + other */
4445 tx_ring = &igb->tx_rings[0];
4446 rc = ddi_intr_add_handler(igb->htable[vector],
4447 (ddi_intr_handler_t *)igb_intr_tx_other,
4448 (void *)igb, NULL);
4449
4450 if (rc != DDI_SUCCESS) {
4451 igb_log(igb,
4452 "Add tx/other interrupt handler failed: %d", rc);
4453 return (IGB_FAILURE);
4454 }
4455 tx_ring->intr_vector = vector;
4456 vector++;
4457
4458 /* Add interrupt handler for each rx ring */
4459 for (i = 0; i < igb->num_rx_rings; i++) {
4460 rx_ring = &igb->rx_rings[i];
4461
4462 rc = ddi_intr_add_handler(igb->htable[vector],
4463 (ddi_intr_handler_t *)igb_intr_rx,
4464 (void *)rx_ring, NULL);
4465
4466 if (rc != DDI_SUCCESS) {
4467 igb_log(igb,
4468 "Add rx interrupt handler failed. "
4469 "return: %d, rx ring: %d", rc, i);
4470 for (vector--; vector >= 0; vector--) {
4471 (void) ddi_intr_remove_handler(
4472 igb->htable[vector]);
4473 }
4474 return (IGB_FAILURE);
4475 }
4476
4477 rx_ring->intr_vector = vector;
4478
4479 vector++;
4480 }
4481
4482 /* Add interrupt handler for each tx ring from 2nd ring */
4483 for (i = 1; i < igb->num_tx_rings; i++) {
4484 tx_ring = &igb->tx_rings[i];
4485
4486 rc = ddi_intr_add_handler(igb->htable[vector],
4487 (ddi_intr_handler_t *)igb_intr_tx,
4488 (void *)tx_ring, NULL);
4489
4490 if (rc != DDI_SUCCESS) {
4491 igb_log(igb,
4492 "Add tx interrupt handler failed. "
4493 "return: %d, tx ring: %d", rc, i);
4494 for (vector--; vector >= 0; vector--) {
4495 (void) ddi_intr_remove_handler(
4496 igb->htable[vector]);
4497 }
4498 return (IGB_FAILURE);
4499 }
4500
4501 tx_ring->intr_vector = vector;
4502
4503 vector++;
4504 }
4505
4506 break;
4507
4508 case DDI_INTR_TYPE_MSI:
4509 /* Add interrupt handlers for the only vector */
4510 rc = ddi_intr_add_handler(igb->htable[vector],
4511 (ddi_intr_handler_t *)igb_intr_msi,
4512 (void *)igb, NULL);
4513
4514 if (rc != DDI_SUCCESS) {
4515 igb_log(igb,
4516 "Add MSI interrupt handler failed: %d", rc);
4517 return (IGB_FAILURE);
4518 }
4519
4520 rx_ring = &igb->rx_rings[0];
4521 rx_ring->intr_vector = vector;
4522
4523 vector++;
4524 break;
4525
4526 case DDI_INTR_TYPE_FIXED:
4527 /* Add interrupt handlers for the only vector */
4528 rc = ddi_intr_add_handler(igb->htable[vector],
4529 (ddi_intr_handler_t *)igb_intr_legacy,
4530 (void *)igb, NULL);
4531
4532 if (rc != DDI_SUCCESS) {
4533 igb_log(igb,
4534 "Add legacy interrupt handler failed: %d", rc);
4535 return (IGB_FAILURE);
4536 }
4537
4538 rx_ring = &igb->rx_rings[0];
4539 rx_ring->intr_vector = vector;
4540
4541 vector++;
4542 break;
4543
4544 default:
4545 return (IGB_FAILURE);
4546 }
4547
4548 ASSERT(vector == igb->intr_cnt);
4549
4550 return (IGB_SUCCESS);
4551 }
4552
4553 /*
4554 * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts
4555 *
4556 * For each vector enabled on the adapter, Set the MSIXBM register accordingly
4557 */
4558 static void
4559 igb_setup_msix_82575(igb_t *igb)
4560 {
4561 uint32_t eims = 0;
4562 int i, vector;
4563 struct e1000_hw *hw = &igb->hw;
4564
4565 /*
4566 * Set vector for tx ring 0 and other causes.
4567 * NOTE assumption that it is vector 0.
4568 */
4569 vector = 0;
4570
4571 igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER;
4572 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask);
4573 vector++;
4574
4575 for (i = 0; i < igb->num_rx_rings; i++) {
4576 /*
4577 * Set vector for each rx ring
4578 */
4579 eims = (E1000_EICR_RX_QUEUE0 << i);
4580 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4581
4582 /*
4583 * Accumulate bits to enable in
4584 * igb_enable_adapter_interrupts_82575()
4585 */
4586 igb->eims_mask |= eims;
4587
4588 vector++;
4589 }
4590
4591 for (i = 1; i < igb->num_tx_rings; i++) {
4592 /*
4593 * Set vector for each tx ring from 2nd tx ring
4594 */
4595 eims = (E1000_EICR_TX_QUEUE0 << i);
4596 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4597
4598 /*
4599 * Accumulate bits to enable in
4600 * igb_enable_adapter_interrupts_82575()
4601 */
4602 igb->eims_mask |= eims;
4603
4604 vector++;
4605 }
4606
4607 ASSERT(vector == igb->intr_cnt);
4608
4609 /*
4610 * Disable IAM for ICR interrupt bits
4611 */
4612 E1000_WRITE_REG(hw, E1000_IAM, 0);
4613 E1000_WRITE_FLUSH(hw);
4614 }
4615
4616 /*
4617 * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts
4618 *
4619 * 82576 uses a table based method for assigning vectors. Each queue has a
4620 * single entry in the table to which we write a vector number along with a
4621 * "valid" bit. The entry is a single byte in a 4-byte register. Vectors
4622 * take a different position in the 4-byte register depending on whether
4623 * they are numbered above or below 8.
4624 */
4625 static void
4626 igb_setup_msix_82576(igb_t *igb)
4627 {
4628 struct e1000_hw *hw = &igb->hw;
4629 uint32_t ivar, index, vector;
4630 int i;
4631
4632 /* must enable msi-x capability before IVAR settings */
4633 E1000_WRITE_REG(hw, E1000_GPIE,
4634 (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR));
4635
4636 /*
4637 * Set vector for tx ring 0 and other causes.
4638 * NOTE assumption that it is vector 0.
4639 * This is also interdependent with installation of interrupt service
4640 * routines in igb_add_intr_handlers().
4641 */
4642
4643 /* assign "other" causes to vector 0 */
4644 vector = 0;
4645 ivar = ((vector | E1000_IVAR_VALID) << 8);
4646 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4647
4648 /* assign tx ring 0 to vector 0 */
4649 ivar = ((vector | E1000_IVAR_VALID) << 8);
4650 E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4651
4652 /* prepare to enable tx & other interrupt causes */
4653 igb->eims_mask = (1 << vector);
4654
4655 vector ++;
4656 for (i = 0; i < igb->num_rx_rings; i++) {
4657 /*
4658 * Set vector for each rx ring
4659 */
4660 index = (i & 0x7);
4661 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4662
4663 if (i < 8) {
4664 /* vector goes into low byte of register */
4665 ivar = ivar & 0xFFFFFF00;
4666 ivar |= (vector | E1000_IVAR_VALID);
4667 } else {
4668 /* vector goes into third byte of register */
4669 ivar = ivar & 0xFF00FFFF;
4670 ivar |= ((vector | E1000_IVAR_VALID) << 16);
4671 }
4672 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4673
4674 /* Accumulate interrupt-cause bits to enable */
4675 igb->eims_mask |= (1 << vector);
4676
4677 vector ++;
4678 }
4679
4680 for (i = 1; i < igb->num_tx_rings; i++) {
4681 /*
4682 * Set vector for each tx ring from 2nd tx ring.
4683 * Note assumption that tx vectors numericall follow rx vectors.
4684 */
4685 index = (i & 0x7);
4686 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4687
4688 if (i < 8) {
4689 /* vector goes into second byte of register */
4690 ivar = ivar & 0xFFFF00FF;
4691 ivar |= ((vector | E1000_IVAR_VALID) << 8);
4692 } else {
4693 /* vector goes into fourth byte of register */
4694 ivar = ivar & 0x00FFFFFF;
4695 ivar |= (vector | E1000_IVAR_VALID) << 24;
4696 }
4697 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4698
4699 /* Accumulate interrupt-cause bits to enable */
4700 igb->eims_mask |= (1 << vector);
4701
4702 vector ++;
4703 }
4704
4705 ASSERT(vector == igb->intr_cnt);
4706 }
4707
4708 /*
4709 * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts
4710 *
4711 * 82580 uses same table approach at 82576 but has fewer entries. Each
4712 * queue has a single entry in the table to which we write a vector number
4713 * along with a "valid" bit. Vectors take a different position in the
4714 * register depending on * whether * they are numbered above or below 4.
4715 */
4716 static void
4717 igb_setup_msix_82580(igb_t *igb)
4718 {
4719 struct e1000_hw *hw = &igb->hw;
4720 uint32_t ivar, index, vector;
4721 int i;
4722
4723 /* must enable msi-x capability before IVAR settings */
4724 E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE |
4725 E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME));
4726 /*
4727 * Set vector for tx ring 0 and other causes.
4728 * NOTE assumption that it is vector 0.
4729 * This is also interdependent with installation of interrupt service
4730 * routines in igb_add_intr_handlers().
4731 */
4732
4733 /* assign "other" causes to vector 0 */
4734 vector = 0;
4735 ivar = ((vector | E1000_IVAR_VALID) << 8);
4736 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4737
4738 /* assign tx ring 0 to vector 0 */
4739 ivar = ((vector | E1000_IVAR_VALID) << 8);
4740 E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4741
4742 /* prepare to enable tx & other interrupt causes */
4743 igb->eims_mask = (1 << vector);
4744
4745 vector ++;
4746
4747 for (i = 0; i < igb->num_rx_rings; i++) {
4748 /*
4749 * Set vector for each rx ring
4750 */
4751 index = (i >> 1);
4752 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4753
4754 if (i & 1) {
4755 /* vector goes into third byte of register */
4756 ivar = ivar & 0xFF00FFFF;
4757 ivar |= ((vector | E1000_IVAR_VALID) << 16);
4758 } else {
4759 /* vector goes into low byte of register */
4760 ivar = ivar & 0xFFFFFF00;
4761 ivar |= (vector | E1000_IVAR_VALID);
4762 }
4763 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4764
4765 /* Accumulate interrupt-cause bits to enable */
4766 igb->eims_mask |= (1 << vector);
4767
4768 vector ++;
4769 }
4770
4771 for (i = 1; i < igb->num_tx_rings; i++) {
4772 /*
4773 * Set vector for each tx ring from 2nd tx ring.
4774 * Note assumption that tx vectors numericall follow rx vectors.
4775 */
4776 index = (i >> 1);
4777 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4778
4779 if (i & 1) {
4780 /* vector goes into high byte of register */
4781 ivar = ivar & 0x00FFFFFF;
4782 ivar |= ((vector | E1000_IVAR_VALID) << 24);
4783 } else {
4784 /* vector goes into second byte of register */
4785 ivar = ivar & 0xFFFF00FF;
4786 ivar |= (vector | E1000_IVAR_VALID) << 8;
4787 }
4788 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4789
4790 /* Accumulate interrupt-cause bits to enable */
4791 igb->eims_mask |= (1 << vector);
4792
4793 vector ++;
4794 }
4795 ASSERT(vector == igb->intr_cnt);
4796 }
4797
4798 /*
4799 * igb_rem_intr_handlers - remove the interrupt handlers
4800 */
4801 static void
4802 igb_rem_intr_handlers(igb_t *igb)
4803 {
4804 int i;
4805 int rc;
4806
4807 for (i = 0; i < igb->intr_cnt; i++) {
4808 rc = ddi_intr_remove_handler(igb->htable[i]);
4809 if (rc != DDI_SUCCESS) {
4810 IGB_DEBUGLOG_1(igb,
4811 "Remove intr handler failed: %d", rc);
4812 }
4813 }
4814 }
4815
4816 /*
4817 * igb_rem_intrs - remove the allocated interrupts
4818 */
4819 static void
4820 igb_rem_intrs(igb_t *igb)
4821 {
4822 int i;
4823 int rc;
4824
4825 for (i = 0; i < igb->intr_cnt; i++) {
4826 rc = ddi_intr_free(igb->htable[i]);
4827 if (rc != DDI_SUCCESS) {
4828 IGB_DEBUGLOG_1(igb,
4829 "Free intr failed: %d", rc);
4830 }
4831 }
4832
4833 kmem_free(igb->htable, igb->intr_size);
4834 igb->htable = NULL;
4835 }
4836
4837 /*
4838 * igb_enable_intrs - enable all the ddi interrupts
4839 */
4840 static int
4841 igb_enable_intrs(igb_t *igb)
4842 {
4843 int i;
4844 int rc;
4845
4846 /* Enable interrupts */
4847 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
4848 /* Call ddi_intr_block_enable() for MSI */
4849 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt);
4850 if (rc != DDI_SUCCESS) {
4851 igb_log(igb,
4852 "Enable block intr failed: %d", rc);
4853 return (IGB_FAILURE);
4854 }
4855 } else {
4856 /* Call ddi_intr_enable() for Legacy/MSI non block enable */
4857 for (i = 0; i < igb->intr_cnt; i++) {
4858 rc = ddi_intr_enable(igb->htable[i]);
4859 if (rc != DDI_SUCCESS) {
4860 igb_log(igb,
4861 "Enable intr failed: %d", rc);
4862 return (IGB_FAILURE);
4863 }
4864 }
4865 }
4866
4867 return (IGB_SUCCESS);
4868 }
4869
4870 /*
4871 * igb_disable_intrs - disable all the ddi interrupts
4872 */
4873 static int
4874 igb_disable_intrs(igb_t *igb)
4875 {
4876 int i;
4877 int rc;
4878
4879 /* Disable all interrupts */
4880 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
4881 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt);
4882 if (rc != DDI_SUCCESS) {
4883 igb_log(igb,
4884 "Disable block intr failed: %d", rc);
4885 return (IGB_FAILURE);
4886 }
4887 } else {
4888 for (i = 0; i < igb->intr_cnt; i++) {
4889 rc = ddi_intr_disable(igb->htable[i]);
4890 if (rc != DDI_SUCCESS) {
4891 igb_log(igb,
4892 "Disable intr failed: %d", rc);
4893 return (IGB_FAILURE);
4894 }
4895 }
4896 }
4897
4898 return (IGB_SUCCESS);
4899 }
4900
4901 /*
4902 * igb_get_phy_state - Get and save the parameters read from PHY registers
4903 */
4904 static void
4905 igb_get_phy_state(igb_t *igb)
4906 {
4907 struct e1000_hw *hw = &igb->hw;
4908 uint16_t phy_ctrl;
4909 uint16_t phy_status;
4910 uint16_t phy_an_adv;
4911 uint16_t phy_an_exp;
4912 uint16_t phy_ext_status;
4913 uint16_t phy_1000t_ctrl;
4914 uint16_t phy_1000t_status;
4915 uint16_t phy_lp_able;
4916
4917 ASSERT(mutex_owned(&igb->gen_lock));
4918
4919 if (hw->phy.media_type == e1000_media_type_copper) {
4920 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl);
4921 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status);
4922 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv);
4923 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp);
4924 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status);
4925 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl);
4926 (void) e1000_read_phy_reg(hw,
4927 PHY_1000T_STATUS, &phy_1000t_status);
4928 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able);
4929
4930 igb->param_autoneg_cap =
4931 (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0;
4932 igb->param_pause_cap =
4933 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
4934 igb->param_asym_pause_cap =
4935 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
4936 igb->param_1000fdx_cap =
4937 ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) ||
4938 (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0;
4939 igb->param_1000hdx_cap =
4940 ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) ||
4941 (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0;
4942 igb->param_100t4_cap =
4943 (phy_status & MII_SR_100T4_CAPS) ? 1 : 0;
4944 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) ||
4945 (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0;
4946 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) ||
4947 (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0;
4948 igb->param_10fdx_cap =
4949 (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0;
4950 igb->param_10hdx_cap =
4951 (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0;
4952 igb->param_rem_fault =
4953 (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0;
4954
4955 igb->param_adv_autoneg_cap = hw->mac.autoneg;
4956 igb->param_adv_pause_cap =
4957 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
4958 igb->param_adv_asym_pause_cap =
4959 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
4960 igb->param_adv_1000hdx_cap =
4961 (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0;
4962 igb->param_adv_100t4_cap =
4963 (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0;
4964 igb->param_adv_rem_fault =
4965 (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0;
4966 if (igb->param_adv_autoneg_cap == 1) {
4967 igb->param_adv_1000fdx_cap =
4968 (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0;
4969 igb->param_adv_100fdx_cap =
4970 (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0;
4971 igb->param_adv_100hdx_cap =
4972 (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0;
4973 igb->param_adv_10fdx_cap =
4974 (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0;
4975 igb->param_adv_10hdx_cap =
4976 (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0;
4977 }
4978
4979 igb->param_lp_autoneg_cap =
4980 (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0;
4981 igb->param_lp_pause_cap =
4982 (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0;
4983 igb->param_lp_asym_pause_cap =
4984 (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0;
4985 igb->param_lp_1000fdx_cap =
4986 (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0;
4987 igb->param_lp_1000hdx_cap =
4988 (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0;
4989 igb->param_lp_100t4_cap =
4990 (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0;
4991 igb->param_lp_100fdx_cap =
4992 (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0;
4993 igb->param_lp_100hdx_cap =
4994 (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0;
4995 igb->param_lp_10fdx_cap =
4996 (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0;
4997 igb->param_lp_10hdx_cap =
4998 (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0;
4999 igb->param_lp_rem_fault =
5000 (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0;
5001 } else {
5002 /*
5003 * 1Gig Fiber adapter only offers 1Gig Full Duplex.
5004 */
5005 igb->param_autoneg_cap = 0;
5006 igb->param_pause_cap = 1;
5007 igb->param_asym_pause_cap = 1;
5008 igb->param_1000fdx_cap = 1;
5009 igb->param_1000hdx_cap = 0;
5010 igb->param_100t4_cap = 0;
5011 igb->param_100fdx_cap = 0;
5012 igb->param_100hdx_cap = 0;
5013 igb->param_10fdx_cap = 0;
5014 igb->param_10hdx_cap = 0;
5015
5016 igb->param_adv_autoneg_cap = 0;
5017 igb->param_adv_pause_cap = 1;
5018 igb->param_adv_asym_pause_cap = 1;
5019 igb->param_adv_1000fdx_cap = 1;
5020 igb->param_adv_1000hdx_cap = 0;
5021 igb->param_adv_100t4_cap = 0;
5022 igb->param_adv_100fdx_cap = 0;
5023 igb->param_adv_100hdx_cap = 0;
5024 igb->param_adv_10fdx_cap = 0;
5025 igb->param_adv_10hdx_cap = 0;
5026
5027 igb->param_lp_autoneg_cap = 0;
5028 igb->param_lp_pause_cap = 0;
5029 igb->param_lp_asym_pause_cap = 0;
5030 igb->param_lp_1000fdx_cap = 0;
5031 igb->param_lp_1000hdx_cap = 0;
5032 igb->param_lp_100t4_cap = 0;
5033 igb->param_lp_100fdx_cap = 0;
5034 igb->param_lp_100hdx_cap = 0;
5035 igb->param_lp_10fdx_cap = 0;
5036 igb->param_lp_10hdx_cap = 0;
5037 igb->param_lp_rem_fault = 0;
5038 }
5039 }
5040
5041 /*
5042 * synchronize the adv* and en* parameters.
5043 *
5044 * See comments in <sys/dld.h> for details of the *_en_*
5045 * parameters. The usage of ndd for setting adv parameters will
5046 * synchronize all the en parameters with the e1000g parameters,
5047 * implicitly disabling any settings made via dladm.
5048 */
5049 static void
5050 igb_param_sync(igb_t *igb)
5051 {
5052 igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap;
5053 igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap;
5054 igb->param_en_100t4_cap = igb->param_adv_100t4_cap;
5055 igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap;
5056 igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap;
5057 igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap;
5058 igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap;
5059 }
5060
5061 /*
5062 * igb_get_driver_control
5063 */
5064 static void
5065 igb_get_driver_control(struct e1000_hw *hw)
5066 {
5067 uint32_t ctrl_ext;
5068
5069 /* Notify firmware that driver is in control of device */
5070 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5071 ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD;
5072 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5073 }
5074
5075 /*
5076 * igb_release_driver_control
5077 */
5078 static void
5079 igb_release_driver_control(struct e1000_hw *hw)
5080 {
5081 uint32_t ctrl_ext;
5082
5083 /* Notify firmware that driver is no longer in control of device */
5084 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5085 ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD;
5086 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5087 }
5088
5089 /*
5090 * igb_atomic_reserve - Atomic decrease operation
5091 */
5092 int
5093 igb_atomic_reserve(uint32_t *count_p, uint32_t n)
5094 {
5095 uint32_t oldval;
5096 uint32_t newval;
5097
5098 /* ATOMICALLY */
5099 do {
5100 oldval = *count_p;
5101 if (oldval < n)
5102 return (-1);
5103 newval = oldval - n;
5104 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5105
5106 return (newval);
5107 }
5108
5109 /*
5110 * FMA support
5111 */
5112
5113 int
5114 igb_check_acc_handle(ddi_acc_handle_t handle)
5115 {
5116 ddi_fm_error_t de;
5117
5118 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5119 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5120 return (de.fme_status);
5121 }
5122
5123 int
5124 igb_check_dma_handle(ddi_dma_handle_t handle)
5125 {
5126 ddi_fm_error_t de;
5127
5128 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5129 return (de.fme_status);
5130 }
5131
5132 /*
5133 * The IO fault service error handling callback function
5134 */
5135 /*ARGSUSED*/
5136 static int
5137 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5138 {
5139 /*
5140 * as the driver can always deal with an error in any dma or
5141 * access handle, we can just return the fme_status value.
5142 */
5143 pci_ereport_post(dip, err, NULL);
5144 return (err->fme_status);
5145 }
5146
5147 static void
5148 igb_fm_init(igb_t *igb)
5149 {
5150 ddi_iblock_cookie_t iblk;
5151 int fma_dma_flag;
5152
5153 /* Only register with IO Fault Services if we have some capability */
5154 if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5155 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5156 } else {
5157 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5158 }
5159
5160 if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5161 fma_dma_flag = 1;
5162 } else {
5163 fma_dma_flag = 0;
5164 }
5165
5166 (void) igb_set_fma_flags(fma_dma_flag);
5167
5168 if (igb->fm_capabilities) {
5169
5170 /* Register capabilities with IO Fault Services */
5171 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk);
5172
5173 /*
5174 * Initialize pci ereport capabilities if ereport capable
5175 */
5176 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5177 DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5178 pci_ereport_setup(igb->dip);
5179
5180 /*
5181 * Register error callback if error callback capable
5182 */
5183 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5184 ddi_fm_handler_register(igb->dip,
5185 igb_fm_error_cb, (void*) igb);
5186 }
5187 }
5188
5189 static void
5190 igb_fm_fini(igb_t *igb)
5191 {
5192 /* Only unregister FMA capabilities if we registered some */
5193 if (igb->fm_capabilities) {
5194
5195 /*
5196 * Release any resources allocated by pci_ereport_setup()
5197 */
5198 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5199 DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5200 pci_ereport_teardown(igb->dip);
5201
5202 /*
5203 * Un-register error callback if error callback capable
5204 */
5205 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5206 ddi_fm_handler_unregister(igb->dip);
5207
5208 /* Unregister from IO Fault Services */
5209 ddi_fm_fini(igb->dip);
5210 }
5211 }
5212
5213 void
5214 igb_fm_ereport(igb_t *igb, char *detail)
5215 {
5216 uint64_t ena;
5217 char buf[FM_MAX_CLASS];
5218
5219 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5220 ena = fm_ena_generate(0, FM_ENA_FMT1);
5221 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) {
5222 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP,
5223 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5224 }
5225 }