1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 */
29
30 #include "igb_sw.h"
31
32 static char ident[] = "Intel 1Gb Ethernet";
33 static char igb_version[] = "igb 1.1.17";
34
35 /*
36 * Local function protoypes
37 */
38 static int igb_register_mac(igb_t *);
39 static int igb_identify_hardware(igb_t *);
40 static int igb_regs_map(igb_t *);
41 static void igb_init_properties(igb_t *);
42 static int igb_init_driver_settings(igb_t *);
43 static void igb_init_locks(igb_t *);
44 static void igb_destroy_locks(igb_t *);
45 static int igb_init_mac_address(igb_t *);
46 static int igb_init(igb_t *);
47 static int igb_init_adapter(igb_t *);
48 static void igb_stop_adapter(igb_t *);
49 static int igb_reset(igb_t *);
50 static void igb_tx_clean(igb_t *);
51 static boolean_t igb_tx_drain(igb_t *);
52 static boolean_t igb_rx_drain(igb_t *);
53 static int igb_alloc_rings(igb_t *);
54 static int igb_alloc_rx_data(igb_t *);
55 static void igb_free_rx_data(igb_t *);
56 static void igb_free_rings(igb_t *);
57 static void igb_setup_rings(igb_t *);
58 static void igb_setup_rx(igb_t *);
59 static void igb_setup_tx(igb_t *);
60 static void igb_setup_rx_ring(igb_rx_ring_t *);
61 static void igb_setup_tx_ring(igb_tx_ring_t *);
62 static void igb_setup_rss(igb_t *);
63 static void igb_setup_mac_rss_classify(igb_t *);
64 static void igb_setup_mac_classify(igb_t *);
65 static void igb_init_unicst(igb_t *);
66 static void igb_setup_multicst(igb_t *);
67 static void igb_get_phy_state(igb_t *);
68 static void igb_param_sync(igb_t *);
69 static void igb_get_conf(igb_t *);
70 static int igb_get_prop(igb_t *, char *, int, int, int);
71 static boolean_t igb_is_link_up(igb_t *);
72 static boolean_t igb_link_check(igb_t *);
73 static void igb_local_timer(void *);
74 static void igb_link_timer(void *);
75 static void igb_arm_watchdog_timer(igb_t *);
76 static void igb_start_watchdog_timer(igb_t *);
77 static void igb_restart_watchdog_timer(igb_t *);
78 static void igb_stop_watchdog_timer(igb_t *);
79 static void igb_start_link_timer(igb_t *);
80 static void igb_stop_link_timer(igb_t *);
81 static void igb_disable_adapter_interrupts(igb_t *);
82 static void igb_enable_adapter_interrupts_82575(igb_t *);
83 static void igb_enable_adapter_interrupts_82576(igb_t *);
84 static void igb_enable_adapter_interrupts_82580(igb_t *);
85 static boolean_t is_valid_mac_addr(uint8_t *);
86 static boolean_t igb_stall_check(igb_t *);
87 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t);
88 static void igb_set_external_loopback(igb_t *);
89 static void igb_set_internal_phy_loopback(igb_t *);
90 static void igb_set_internal_serdes_loopback(igb_t *);
91 static boolean_t igb_find_mac_address(igb_t *);
92 static int igb_alloc_intrs(igb_t *);
93 static int igb_alloc_intr_handles(igb_t *, int);
94 static int igb_add_intr_handlers(igb_t *);
95 static void igb_rem_intr_handlers(igb_t *);
96 static void igb_rem_intrs(igb_t *);
97 static int igb_enable_intrs(igb_t *);
98 static int igb_disable_intrs(igb_t *);
99 static void igb_setup_msix_82575(igb_t *);
100 static void igb_setup_msix_82576(igb_t *);
101 static void igb_setup_msix_82580(igb_t *);
102 static uint_t igb_intr_legacy(void *, void *);
103 static uint_t igb_intr_msi(void *, void *);
104 static uint_t igb_intr_rx(void *, void *);
105 static uint_t igb_intr_tx(void *, void *);
106 static uint_t igb_intr_tx_other(void *, void *);
107 static void igb_intr_rx_work(igb_rx_ring_t *);
108 static void igb_intr_tx_work(igb_tx_ring_t *);
109 static void igb_intr_link_work(igb_t *);
110 static void igb_get_driver_control(struct e1000_hw *);
111 static void igb_release_driver_control(struct e1000_hw *);
112
113 static int igb_attach(dev_info_t *, ddi_attach_cmd_t);
114 static int igb_detach(dev_info_t *, ddi_detach_cmd_t);
115 static int igb_resume(dev_info_t *);
116 static int igb_suspend(dev_info_t *);
117 static int igb_quiesce(dev_info_t *);
118 static void igb_unconfigure(dev_info_t *, igb_t *);
119 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *,
120 const void *);
121 static void igb_fm_init(igb_t *);
122 static void igb_fm_fini(igb_t *);
123 static void igb_release_multicast(igb_t *);
124
125 char *igb_priv_props[] = {
126 "_tx_copy_thresh",
127 "_tx_recycle_thresh",
128 "_tx_overload_thresh",
129 "_tx_resched_thresh",
130 "_rx_copy_thresh",
131 "_rx_limit_per_intr",
132 "_intr_throttling",
133 "_adv_pause_cap",
134 "_adv_asym_pause_cap",
135 NULL
136 };
137
138 static struct cb_ops igb_cb_ops = {
139 nulldev, /* cb_open */
140 nulldev, /* cb_close */
141 nodev, /* cb_strategy */
142 nodev, /* cb_print */
143 nodev, /* cb_dump */
144 nodev, /* cb_read */
145 nodev, /* cb_write */
146 nodev, /* cb_ioctl */
147 nodev, /* cb_devmap */
148 nodev, /* cb_mmap */
149 nodev, /* cb_segmap */
150 nochpoll, /* cb_chpoll */
151 ddi_prop_op, /* cb_prop_op */
152 NULL, /* cb_stream */
153 D_MP | D_HOTPLUG, /* cb_flag */
154 CB_REV, /* cb_rev */
155 nodev, /* cb_aread */
156 nodev /* cb_awrite */
157 };
158
159 static struct dev_ops igb_dev_ops = {
160 DEVO_REV, /* devo_rev */
161 0, /* devo_refcnt */
162 NULL, /* devo_getinfo */
163 nulldev, /* devo_identify */
164 nulldev, /* devo_probe */
165 igb_attach, /* devo_attach */
166 igb_detach, /* devo_detach */
167 nodev, /* devo_reset */
168 &igb_cb_ops, /* devo_cb_ops */
169 NULL, /* devo_bus_ops */
170 ddi_power, /* devo_power */
171 igb_quiesce, /* devo_quiesce */
172 };
173
174 static struct modldrv igb_modldrv = {
175 &mod_driverops, /* Type of module. This one is a driver */
176 ident, /* Discription string */
177 &igb_dev_ops, /* driver ops */
178 };
179
180 static struct modlinkage igb_modlinkage = {
181 MODREV_1, &igb_modldrv, NULL
182 };
183
184 /* Access attributes for register mapping */
185 ddi_device_acc_attr_t igb_regs_acc_attr = {
186 DDI_DEVICE_ATTR_V1,
187 DDI_STRUCTURE_LE_ACC,
188 DDI_STRICTORDER_ACC,
189 DDI_FLAGERR_ACC
190 };
191
192 #define IGB_M_CALLBACK_FLAGS \
193 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
194
195 static mac_callbacks_t igb_m_callbacks = {
196 IGB_M_CALLBACK_FLAGS,
197 igb_m_stat,
198 igb_m_start,
199 igb_m_stop,
200 igb_m_promisc,
201 igb_m_multicst,
202 NULL,
203 NULL,
204 NULL,
205 igb_m_ioctl,
206 igb_m_getcapab,
207 NULL,
208 NULL,
209 igb_m_setprop,
210 igb_m_getprop,
211 igb_m_propinfo
212 };
213
214 /*
215 * Initialize capabilities of each supported adapter type
216 */
217 static adapter_info_t igb_82575_cap = {
218 /* limits */
219 4, /* maximum number of rx queues */
220 1, /* minimum number of rx queues */
221 4, /* default number of rx queues */
222 4, /* maximum number of tx queues */
223 1, /* minimum number of tx queues */
224 4, /* default number of tx queues */
225 65535, /* maximum interrupt throttle rate */
226 0, /* minimum interrupt throttle rate */
227 200, /* default interrupt throttle rate */
228
229 /* function pointers */
230 igb_enable_adapter_interrupts_82575,
231 igb_setup_msix_82575,
232
233 /* capabilities */
234 (IGB_FLAG_HAS_DCA | /* capability flags */
235 IGB_FLAG_VMDQ_POOL),
236
237 0xffc00000 /* mask for RXDCTL register */
238 };
239
240 static adapter_info_t igb_82576_cap = {
241 /* limits */
242 16, /* maximum number of rx queues */
243 1, /* minimum number of rx queues */
244 4, /* default number of rx queues */
245 16, /* maximum number of tx queues */
246 1, /* minimum number of tx queues */
247 4, /* default number of tx queues */
248 65535, /* maximum interrupt throttle rate */
249 0, /* minimum interrupt throttle rate */
250 200, /* default interrupt throttle rate */
251
252 /* function pointers */
253 igb_enable_adapter_interrupts_82576,
254 igb_setup_msix_82576,
255
256 /* capabilities */
257 (IGB_FLAG_HAS_DCA | /* capability flags */
258 IGB_FLAG_VMDQ_POOL |
259 IGB_FLAG_NEED_CTX_IDX),
260
261 0xffe00000 /* mask for RXDCTL register */
262 };
263
264 static adapter_info_t igb_82580_cap = {
265 /* limits */
266 8, /* maximum number of rx queues */
267 1, /* minimum number of rx queues */
268 4, /* default number of rx queues */
269 8, /* maximum number of tx queues */
270 1, /* minimum number of tx queues */
271 4, /* default number of tx queues */
272 65535, /* maximum interrupt throttle rate */
273 0, /* minimum interrupt throttle rate */
274 200, /* default interrupt throttle rate */
275
276 /* function pointers */
277 igb_enable_adapter_interrupts_82580,
278 igb_setup_msix_82580,
279
280 /* capabilities */
281 (IGB_FLAG_HAS_DCA | /* capability flags */
282 IGB_FLAG_VMDQ_POOL |
283 IGB_FLAG_NEED_CTX_IDX),
284
285 0xffe00000 /* mask for RXDCTL register */
286 };
287
288 /*
289 * Module Initialization Functions
290 */
291
292 int
293 _init(void)
294 {
295 int status;
296
297 mac_init_ops(&igb_dev_ops, MODULE_NAME);
298
299 status = mod_install(&igb_modlinkage);
300
301 if (status != DDI_SUCCESS) {
302 mac_fini_ops(&igb_dev_ops);
303 }
304
305 return (status);
306 }
307
308 int
309 _fini(void)
310 {
311 int status;
312
313 status = mod_remove(&igb_modlinkage);
314
315 if (status == DDI_SUCCESS) {
316 mac_fini_ops(&igb_dev_ops);
317 }
318
319 return (status);
320
321 }
322
323 int
324 _info(struct modinfo *modinfop)
325 {
326 int status;
327
328 status = mod_info(&igb_modlinkage, modinfop);
329
330 return (status);
331 }
332
333 /*
334 * igb_attach - driver attach
335 *
336 * This function is the device specific initialization entry
337 * point. This entry point is required and must be written.
338 * The DDI_ATTACH command must be provided in the attach entry
339 * point. When attach() is called with cmd set to DDI_ATTACH,
340 * all normal kernel services (such as kmem_alloc(9F)) are
341 * available for use by the driver.
342 *
343 * The attach() function will be called once for each instance
344 * of the device on the system with cmd set to DDI_ATTACH.
345 * Until attach() succeeds, the only driver entry points which
346 * may be called are open(9E) and getinfo(9E).
347 */
348 static int
349 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
350 {
351 igb_t *igb;
352 struct igb_osdep *osdep;
353 struct e1000_hw *hw;
354 int instance;
355
356 /*
357 * Check the command and perform corresponding operations
358 */
359 switch (cmd) {
360 default:
361 return (DDI_FAILURE);
362
363 case DDI_RESUME:
364 return (igb_resume(devinfo));
365
366 case DDI_ATTACH:
367 break;
368 }
369
370 /* Get the device instance */
371 instance = ddi_get_instance(devinfo);
372
373 /* Allocate memory for the instance data structure */
374 igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP);
375
376 igb->dip = devinfo;
377 igb->instance = instance;
378
379 hw = &igb->hw;
380 osdep = &igb->osdep;
381 hw->back = osdep;
382 osdep->igb = igb;
383
384 /* Attach the instance pointer to the dev_info data structure */
385 ddi_set_driver_private(devinfo, igb);
386
387
388 /* Initialize for fma support */
389 igb->fm_capabilities = igb_get_prop(igb, "fm-capable",
390 0, 0x0f,
391 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
392 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
393 igb_fm_init(igb);
394 igb->attach_progress |= ATTACH_PROGRESS_FMINIT;
395
396 /*
397 * Map PCI config space registers
398 */
399 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
400 igb_error(igb, "Failed to map PCI configurations");
401 goto attach_fail;
402 }
403 igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
404
405 /*
406 * Identify the chipset family
407 */
408 if (igb_identify_hardware(igb) != IGB_SUCCESS) {
409 igb_error(igb, "Failed to identify hardware");
410 goto attach_fail;
411 }
412
413 /*
414 * Map device registers
415 */
416 if (igb_regs_map(igb) != IGB_SUCCESS) {
417 igb_error(igb, "Failed to map device registers");
418 goto attach_fail;
419 }
420 igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
421
422 /*
423 * Initialize driver parameters
424 */
425 igb_init_properties(igb);
426 igb->attach_progress |= ATTACH_PROGRESS_PROPS;
427
428 /*
429 * Allocate interrupts
430 */
431 if (igb_alloc_intrs(igb) != IGB_SUCCESS) {
432 igb_error(igb, "Failed to allocate interrupts");
433 goto attach_fail;
434 }
435 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
436
437 /*
438 * Allocate rx/tx rings based on the ring numbers.
439 * The actual numbers of rx/tx rings are decided by the number of
440 * allocated interrupt vectors, so we should allocate the rings after
441 * interrupts are allocated.
442 */
443 if (igb_alloc_rings(igb) != IGB_SUCCESS) {
444 igb_error(igb, "Failed to allocate rx/tx rings or groups");
445 goto attach_fail;
446 }
447 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
448
449 /*
450 * Add interrupt handlers
451 */
452 if (igb_add_intr_handlers(igb) != IGB_SUCCESS) {
453 igb_error(igb, "Failed to add interrupt handlers");
454 goto attach_fail;
455 }
456 igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
457
458 /*
459 * Initialize driver parameters
460 */
461 if (igb_init_driver_settings(igb) != IGB_SUCCESS) {
462 igb_error(igb, "Failed to initialize driver settings");
463 goto attach_fail;
464 }
465
466 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) {
467 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
468 goto attach_fail;
469 }
470
471 /*
472 * Initialize mutexes for this device.
473 * Do this before enabling the interrupt handler and
474 * register the softint to avoid the condition where
475 * interrupt handler can try using uninitialized mutex
476 */
477 igb_init_locks(igb);
478 igb->attach_progress |= ATTACH_PROGRESS_LOCKS;
479
480 /*
481 * Initialize the adapter
482 */
483 if (igb_init(igb) != IGB_SUCCESS) {
484 igb_error(igb, "Failed to initialize adapter");
485 goto attach_fail;
486 }
487 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
488
489 /*
490 * Initialize statistics
491 */
492 if (igb_init_stats(igb) != IGB_SUCCESS) {
493 igb_error(igb, "Failed to initialize statistics");
494 goto attach_fail;
495 }
496 igb->attach_progress |= ATTACH_PROGRESS_STATS;
497
498 /*
499 * Register the driver to the MAC
500 */
501 if (igb_register_mac(igb) != IGB_SUCCESS) {
502 igb_error(igb, "Failed to register MAC");
503 goto attach_fail;
504 }
505 igb->attach_progress |= ATTACH_PROGRESS_MAC;
506
507 /*
508 * Now that mutex locks are initialized, and the chip is also
509 * initialized, enable interrupts.
510 */
511 if (igb_enable_intrs(igb) != IGB_SUCCESS) {
512 igb_error(igb, "Failed to enable DDI interrupts");
513 goto attach_fail;
514 }
515 igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
516
517 igb_log(igb, "%s", igb_version);
518 atomic_or_32(&igb->igb_state, IGB_INITIALIZED);
519
520 return (DDI_SUCCESS);
521
522 attach_fail:
523 igb_unconfigure(devinfo, igb);
524 return (DDI_FAILURE);
525 }
526
527 /*
528 * igb_detach - driver detach
529 *
530 * The detach() function is the complement of the attach routine.
531 * If cmd is set to DDI_DETACH, detach() is used to remove the
532 * state associated with a given instance of a device node
533 * prior to the removal of that instance from the system.
534 *
535 * The detach() function will be called once for each instance
536 * of the device for which there has been a successful attach()
537 * once there are no longer any opens on the device.
538 *
539 * Interrupts routine are disabled, All memory allocated by this
540 * driver are freed.
541 */
542 static int
543 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
544 {
545 igb_t *igb;
546
547 /*
548 * Check detach command
549 */
550 switch (cmd) {
551 default:
552 return (DDI_FAILURE);
553
554 case DDI_SUSPEND:
555 return (igb_suspend(devinfo));
556
557 case DDI_DETACH:
558 break;
559 }
560
561
562 /*
563 * Get the pointer to the driver private data structure
564 */
565 igb = (igb_t *)ddi_get_driver_private(devinfo);
566 if (igb == NULL)
567 return (DDI_FAILURE);
568
569 /*
570 * Unregister MAC. If failed, we have to fail the detach
571 */
572 if (mac_unregister(igb->mac_hdl) != 0) {
573 igb_error(igb, "Failed to unregister MAC");
574 return (DDI_FAILURE);
575 }
576 igb->attach_progress &= ~ATTACH_PROGRESS_MAC;
577
578 /*
579 * If the device is still running, it needs to be stopped first.
580 * This check is necessary because under some specific circumstances,
581 * the detach routine can be called without stopping the interface
582 * first.
583 */
584 mutex_enter(&igb->gen_lock);
585 if (igb->igb_state & IGB_STARTED) {
586 atomic_and_32(&igb->igb_state, ~IGB_STARTED);
587 igb_stop(igb, B_TRUE);
588 mutex_exit(&igb->gen_lock);
589 /* Disable and stop the watchdog timer */
590 igb_disable_watchdog_timer(igb);
591 } else
592 mutex_exit(&igb->gen_lock);
593
594 /*
595 * Check if there are still rx buffers held by the upper layer.
596 * If so, fail the detach.
597 */
598 if (!igb_rx_drain(igb))
599 return (DDI_FAILURE);
600
601 /*
602 * Do the remaining unconfigure routines
603 */
604 igb_unconfigure(devinfo, igb);
605
606 return (DDI_SUCCESS);
607 }
608
609 /*
610 * quiesce(9E) entry point.
611 *
612 * This function is called when the system is single-threaded at high
613 * PIL with preemption disabled. Therefore, this function must not be
614 * blocked.
615 *
616 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
617 * DDI_FAILURE indicates an error condition and should almost never happen.
618 */
619 static int
620 igb_quiesce(dev_info_t *devinfo)
621 {
622 igb_t *igb;
623 struct e1000_hw *hw;
624
625 igb = (igb_t *)ddi_get_driver_private(devinfo);
626
627 if (igb == NULL)
628 return (DDI_FAILURE);
629
630 hw = &igb->hw;
631
632 /*
633 * Disable the adapter interrupts
634 */
635 igb_disable_adapter_interrupts(igb);
636
637 /* Tell firmware driver is no longer in control */
638 igb_release_driver_control(hw);
639
640 /*
641 * Reset the chipset
642 */
643 (void) e1000_reset_hw(hw);
644
645 /*
646 * Reset PHY if possible
647 */
648 if (e1000_check_reset_block(hw) == E1000_SUCCESS)
649 (void) e1000_phy_hw_reset(hw);
650
651 return (DDI_SUCCESS);
652 }
653
654 /*
655 * igb_unconfigure - release all resources held by this instance
656 */
657 static void
658 igb_unconfigure(dev_info_t *devinfo, igb_t *igb)
659 {
660 /*
661 * Disable interrupt
662 */
663 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
664 (void) igb_disable_intrs(igb);
665 }
666
667 /*
668 * Unregister MAC
669 */
670 if (igb->attach_progress & ATTACH_PROGRESS_MAC) {
671 (void) mac_unregister(igb->mac_hdl);
672 }
673
674 /*
675 * Free statistics
676 */
677 if (igb->attach_progress & ATTACH_PROGRESS_STATS) {
678 kstat_delete((kstat_t *)igb->igb_ks);
679 }
680
681 /*
682 * Remove interrupt handlers
683 */
684 if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
685 igb_rem_intr_handlers(igb);
686 }
687
688 /*
689 * Remove interrupts
690 */
691 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
692 igb_rem_intrs(igb);
693 }
694
695 /*
696 * Remove driver properties
697 */
698 if (igb->attach_progress & ATTACH_PROGRESS_PROPS) {
699 (void) ddi_prop_remove_all(devinfo);
700 }
701
702 /*
703 * Stop the adapter
704 */
705 if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) {
706 mutex_enter(&igb->gen_lock);
707 igb_stop_adapter(igb);
708 mutex_exit(&igb->gen_lock);
709 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
710 ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED);
711 }
712
713 /*
714 * Free multicast table
715 */
716 igb_release_multicast(igb);
717
718 /*
719 * Free register handle
720 */
721 if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
722 if (igb->osdep.reg_handle != NULL)
723 ddi_regs_map_free(&igb->osdep.reg_handle);
724 }
725
726 /*
727 * Free PCI config handle
728 */
729 if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
730 if (igb->osdep.cfg_handle != NULL)
731 pci_config_teardown(&igb->osdep.cfg_handle);
732 }
733
734 /*
735 * Free locks
736 */
737 if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) {
738 igb_destroy_locks(igb);
739 }
740
741 /*
742 * Free the rx/tx rings
743 */
744 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
745 igb_free_rings(igb);
746 }
747
748 /*
749 * Remove FMA
750 */
751 if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) {
752 igb_fm_fini(igb);
753 }
754
755 /*
756 * Free the driver data structure
757 */
758 kmem_free(igb, sizeof (igb_t));
759
760 ddi_set_driver_private(devinfo, NULL);
761 }
762
763 /*
764 * igb_register_mac - Register the driver and its function pointers with
765 * the GLD interface
766 */
767 static int
768 igb_register_mac(igb_t *igb)
769 {
770 struct e1000_hw *hw = &igb->hw;
771 mac_register_t *mac;
772 int status;
773
774 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
775 return (IGB_FAILURE);
776
777 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
778 mac->m_driver = igb;
779 mac->m_dip = igb->dip;
780 mac->m_src_addr = hw->mac.addr;
781 mac->m_callbacks = &igb_m_callbacks;
782 mac->m_min_sdu = 0;
783 mac->m_max_sdu = igb->max_frame_size -
784 sizeof (struct ether_vlan_header) - ETHERFCSL;
785 mac->m_margin = VLAN_TAGSZ;
786 mac->m_priv_props = igb_priv_props;
787 mac->m_v12n = MAC_VIRT_LEVEL1;
788
789 status = mac_register(mac, &igb->mac_hdl);
790
791 mac_free(mac);
792
793 return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE);
794 }
795
796 /*
797 * igb_identify_hardware - Identify the type of the chipset
798 */
799 static int
800 igb_identify_hardware(igb_t *igb)
801 {
802 struct e1000_hw *hw = &igb->hw;
803 struct igb_osdep *osdep = &igb->osdep;
804
805 /*
806 * Get the device id
807 */
808 hw->vendor_id =
809 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
810 hw->device_id =
811 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
812 hw->revision_id =
813 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
814 hw->subsystem_device_id =
815 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
816 hw->subsystem_vendor_id =
817 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
818
819 /*
820 * Set the mac type of the adapter based on the device id
821 */
822 if (e1000_set_mac_type(hw) != E1000_SUCCESS) {
823 return (IGB_FAILURE);
824 }
825
826 /*
827 * Install adapter capabilities based on mac type
828 */
829 switch (hw->mac.type) {
830 case e1000_82575:
831 igb->capab = &igb_82575_cap;
832 break;
833 case e1000_82576:
834 igb->capab = &igb_82576_cap;
835 break;
836 case e1000_82580:
837 igb->capab = &igb_82580_cap;
838 break;
839 default:
840 return (IGB_FAILURE);
841 }
842
843 return (IGB_SUCCESS);
844 }
845
846 /*
847 * igb_regs_map - Map the device registers
848 */
849 static int
850 igb_regs_map(igb_t *igb)
851 {
852 dev_info_t *devinfo = igb->dip;
853 struct e1000_hw *hw = &igb->hw;
854 struct igb_osdep *osdep = &igb->osdep;
855 off_t mem_size;
856
857 /*
858 * First get the size of device registers to be mapped.
859 */
860 if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) !=
861 DDI_SUCCESS) {
862 return (IGB_FAILURE);
863 }
864
865 /*
866 * Call ddi_regs_map_setup() to map registers
867 */
868 if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET,
869 (caddr_t *)&hw->hw_addr, 0,
870 mem_size, &igb_regs_acc_attr,
871 &osdep->reg_handle)) != DDI_SUCCESS) {
872 return (IGB_FAILURE);
873 }
874
875 return (IGB_SUCCESS);
876 }
877
878 /*
879 * igb_init_properties - Initialize driver properties
880 */
881 static void
882 igb_init_properties(igb_t *igb)
883 {
884 /*
885 * Get conf file properties, including link settings
886 * jumbo frames, ring number, descriptor number, etc.
887 */
888 igb_get_conf(igb);
889 }
890
891 /*
892 * igb_init_driver_settings - Initialize driver settings
893 *
894 * The settings include hardware function pointers, bus information,
895 * rx/tx rings settings, link state, and any other parameters that
896 * need to be setup during driver initialization.
897 */
898 static int
899 igb_init_driver_settings(igb_t *igb)
900 {
901 struct e1000_hw *hw = &igb->hw;
902 igb_rx_ring_t *rx_ring;
903 igb_tx_ring_t *tx_ring;
904 uint32_t rx_size;
905 uint32_t tx_size;
906 int i;
907
908 /*
909 * Initialize chipset specific hardware function pointers
910 */
911 if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) {
912 return (IGB_FAILURE);
913 }
914
915 /*
916 * Get bus information
917 */
918 if (e1000_get_bus_info(hw) != E1000_SUCCESS) {
919 return (IGB_FAILURE);
920 }
921
922 /*
923 * Get the system page size
924 */
925 igb->page_size = ddi_ptob(igb->dip, (ulong_t)1);
926
927 /*
928 * Set rx buffer size
929 * The IP header alignment room is counted in the calculation.
930 * The rx buffer size is in unit of 1K that is required by the
931 * chipset hardware.
932 */
933 rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM;
934 igb->rx_buf_size = ((rx_size >> 10) +
935 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
936
937 /*
938 * Set tx buffer size
939 */
940 tx_size = igb->max_frame_size;
941 igb->tx_buf_size = ((tx_size >> 10) +
942 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
943
944 /*
945 * Initialize rx/tx rings parameters
946 */
947 for (i = 0; i < igb->num_rx_rings; i++) {
948 rx_ring = &igb->rx_rings[i];
949 rx_ring->index = i;
950 rx_ring->igb = igb;
951 }
952
953 for (i = 0; i < igb->num_tx_rings; i++) {
954 tx_ring = &igb->tx_rings[i];
955 tx_ring->index = i;
956 tx_ring->igb = igb;
957 if (igb->tx_head_wb_enable)
958 tx_ring->tx_recycle = igb_tx_recycle_head_wb;
959 else
960 tx_ring->tx_recycle = igb_tx_recycle_legacy;
961
962 tx_ring->ring_size = igb->tx_ring_size;
963 tx_ring->free_list_size = igb->tx_ring_size +
964 (igb->tx_ring_size >> 1);
965 }
966
967 /*
968 * Initialize values of interrupt throttling rates
969 */
970 for (i = 1; i < MAX_NUM_EITR; i++)
971 igb->intr_throttling[i] = igb->intr_throttling[0];
972
973 /*
974 * The initial link state should be "unknown"
975 */
976 igb->link_state = LINK_STATE_UNKNOWN;
977
978 return (IGB_SUCCESS);
979 }
980
981 /*
982 * igb_init_locks - Initialize locks
983 */
984 static void
985 igb_init_locks(igb_t *igb)
986 {
987 igb_rx_ring_t *rx_ring;
988 igb_tx_ring_t *tx_ring;
989 int i;
990
991 for (i = 0; i < igb->num_rx_rings; i++) {
992 rx_ring = &igb->rx_rings[i];
993 mutex_init(&rx_ring->rx_lock, NULL,
994 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
995 }
996
997 for (i = 0; i < igb->num_tx_rings; i++) {
998 tx_ring = &igb->tx_rings[i];
999 mutex_init(&tx_ring->tx_lock, NULL,
1000 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1001 mutex_init(&tx_ring->recycle_lock, NULL,
1002 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1003 mutex_init(&tx_ring->tcb_head_lock, NULL,
1004 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1005 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1006 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1007 }
1008
1009 mutex_init(&igb->gen_lock, NULL,
1010 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1011
1012 mutex_init(&igb->watchdog_lock, NULL,
1013 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1014
1015 mutex_init(&igb->link_lock, NULL,
1016 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1017 }
1018
1019 /*
1020 * igb_destroy_locks - Destroy locks
1021 */
1022 static void
1023 igb_destroy_locks(igb_t *igb)
1024 {
1025 igb_rx_ring_t *rx_ring;
1026 igb_tx_ring_t *tx_ring;
1027 int i;
1028
1029 for (i = 0; i < igb->num_rx_rings; i++) {
1030 rx_ring = &igb->rx_rings[i];
1031 mutex_destroy(&rx_ring->rx_lock);
1032 }
1033
1034 for (i = 0; i < igb->num_tx_rings; i++) {
1035 tx_ring = &igb->tx_rings[i];
1036 mutex_destroy(&tx_ring->tx_lock);
1037 mutex_destroy(&tx_ring->recycle_lock);
1038 mutex_destroy(&tx_ring->tcb_head_lock);
1039 mutex_destroy(&tx_ring->tcb_tail_lock);
1040 }
1041
1042 mutex_destroy(&igb->gen_lock);
1043 mutex_destroy(&igb->watchdog_lock);
1044 mutex_destroy(&igb->link_lock);
1045 }
1046
1047 static int
1048 igb_resume(dev_info_t *devinfo)
1049 {
1050 igb_t *igb;
1051
1052 igb = (igb_t *)ddi_get_driver_private(devinfo);
1053 if (igb == NULL)
1054 return (DDI_FAILURE);
1055
1056 mutex_enter(&igb->gen_lock);
1057
1058 /*
1059 * Enable interrupts
1060 */
1061 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1062 if (igb_enable_intrs(igb) != IGB_SUCCESS) {
1063 igb_error(igb, "Failed to enable DDI interrupts");
1064 mutex_exit(&igb->gen_lock);
1065 return (DDI_FAILURE);
1066 }
1067 }
1068
1069 if (igb->igb_state & IGB_STARTED) {
1070 if (igb_start(igb, B_FALSE) != IGB_SUCCESS) {
1071 mutex_exit(&igb->gen_lock);
1072 return (DDI_FAILURE);
1073 }
1074
1075 /*
1076 * Enable and start the watchdog timer
1077 */
1078 igb_enable_watchdog_timer(igb);
1079 }
1080
1081 atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED);
1082
1083 mutex_exit(&igb->gen_lock);
1084
1085 return (DDI_SUCCESS);
1086 }
1087
1088 static int
1089 igb_suspend(dev_info_t *devinfo)
1090 {
1091 igb_t *igb;
1092
1093 igb = (igb_t *)ddi_get_driver_private(devinfo);
1094 if (igb == NULL)
1095 return (DDI_FAILURE);
1096
1097 mutex_enter(&igb->gen_lock);
1098
1099 atomic_or_32(&igb->igb_state, IGB_SUSPENDED);
1100
1101 /*
1102 * Disable interrupts
1103 */
1104 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1105 (void) igb_disable_intrs(igb);
1106 }
1107
1108 if (!(igb->igb_state & IGB_STARTED)) {
1109 mutex_exit(&igb->gen_lock);
1110 return (DDI_SUCCESS);
1111 }
1112
1113 igb_stop(igb, B_FALSE);
1114
1115 mutex_exit(&igb->gen_lock);
1116
1117 /*
1118 * Disable and stop the watchdog timer
1119 */
1120 igb_disable_watchdog_timer(igb);
1121
1122 return (DDI_SUCCESS);
1123 }
1124
1125 static int
1126 igb_init(igb_t *igb)
1127 {
1128 mutex_enter(&igb->gen_lock);
1129
1130 /*
1131 * Initilize the adapter
1132 */
1133 if (igb_init_adapter(igb) != IGB_SUCCESS) {
1134 mutex_exit(&igb->gen_lock);
1135 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1136 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1137 return (IGB_FAILURE);
1138 }
1139
1140 mutex_exit(&igb->gen_lock);
1141
1142 return (IGB_SUCCESS);
1143 }
1144
1145 /*
1146 * igb_init_mac_address - Initialize the default MAC address
1147 *
1148 * On success, the MAC address is entered in the igb->hw.mac.addr
1149 * and hw->mac.perm_addr fields and the adapter's RAR(0) receive
1150 * address register.
1151 *
1152 * Important side effects:
1153 * 1. adapter is reset - this is required to put it in a known state.
1154 * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where
1155 * MAC address and all default settings are stored, so a valid checksum
1156 * is required.
1157 */
1158 static int
1159 igb_init_mac_address(igb_t *igb)
1160 {
1161 struct e1000_hw *hw = &igb->hw;
1162
1163 ASSERT(mutex_owned(&igb->gen_lock));
1164
1165 /*
1166 * Reset chipset to put the hardware in a known state
1167 * before we try to get MAC address from NVM.
1168 */
1169 if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1170 igb_error(igb, "Adapter reset failed.");
1171 goto init_mac_fail;
1172 }
1173
1174 /*
1175 * NVM validation
1176 */
1177 if (e1000_validate_nvm_checksum(hw) < 0) {
1178 /*
1179 * Some PCI-E parts fail the first check due to
1180 * the link being in sleep state. Call it again,
1181 * if it fails a second time its a real issue.
1182 */
1183 if (e1000_validate_nvm_checksum(hw) < 0) {
1184 igb_error(igb,
1185 "Invalid NVM checksum. Please contact "
1186 "the vendor to update the NVM.");
1187 goto init_mac_fail;
1188 }
1189 }
1190
1191 /*
1192 * Get the mac address
1193 * This function should handle SPARC case correctly.
1194 */
1195 if (!igb_find_mac_address(igb)) {
1196 igb_error(igb, "Failed to get the mac address");
1197 goto init_mac_fail;
1198 }
1199
1200 /* Validate mac address */
1201 if (!is_valid_mac_addr(hw->mac.addr)) {
1202 igb_error(igb, "Invalid mac address");
1203 goto init_mac_fail;
1204 }
1205
1206 return (IGB_SUCCESS);
1207
1208 init_mac_fail:
1209 return (IGB_FAILURE);
1210 }
1211
1212 /*
1213 * igb_init_adapter - Initialize the adapter
1214 */
1215 static int
1216 igb_init_adapter(igb_t *igb)
1217 {
1218 struct e1000_hw *hw = &igb->hw;
1219 uint32_t pba;
1220 uint32_t high_water;
1221 int i;
1222
1223 ASSERT(mutex_owned(&igb->gen_lock));
1224
1225 /*
1226 * In order to obtain the default MAC address, this will reset the
1227 * adapter and validate the NVM that the address and many other
1228 * default settings come from.
1229 */
1230 if (igb_init_mac_address(igb) != IGB_SUCCESS) {
1231 igb_error(igb, "Failed to initialize MAC address");
1232 goto init_adapter_fail;
1233 }
1234
1235 /*
1236 * Setup flow control
1237 *
1238 * These parameters set thresholds for the adapter's generation(Tx)
1239 * and response(Rx) to Ethernet PAUSE frames. These are just threshold
1240 * settings. Flow control is enabled or disabled in the configuration
1241 * file.
1242 * High-water mark is set down from the top of the rx fifo (not
1243 * sensitive to max_frame_size) and low-water is set just below
1244 * high-water mark.
1245 * The high water mark must be low enough to fit one full frame above
1246 * it in the rx FIFO. Should be the lower of:
1247 * 90% of the Rx FIFO size, or the full Rx FIFO size minus one full
1248 * frame.
1249 */
1250 /*
1251 * The default setting of PBA is correct for 82575 and other supported
1252 * adapters do not have the E1000_PBA register, so PBA value is only
1253 * used for calculation here and is never written to the adapter.
1254 */
1255 if (hw->mac.type == e1000_82575) {
1256 pba = E1000_PBA_34K;
1257 } else {
1258 pba = E1000_PBA_64K;
1259 }
1260
1261 high_water = min(((pba << 10) * 9 / 10),
1262 ((pba << 10) - igb->max_frame_size));
1263
1264 if (hw->mac.type == e1000_82575) {
1265 /* 8-byte granularity */
1266 hw->fc.high_water = high_water & 0xFFF8;
1267 hw->fc.low_water = hw->fc.high_water - 8;
1268 } else {
1269 /* 16-byte granularity */
1270 hw->fc.high_water = high_water & 0xFFF0;
1271 hw->fc.low_water = hw->fc.high_water - 16;
1272 }
1273
1274 hw->fc.pause_time = E1000_FC_PAUSE_TIME;
1275 hw->fc.send_xon = B_TRUE;
1276
1277 (void) e1000_validate_mdi_setting(hw);
1278
1279 /*
1280 * Reset the chipset hardware the second time to put PBA settings
1281 * into effect.
1282 */
1283 if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1284 igb_error(igb, "Second reset failed");
1285 goto init_adapter_fail;
1286 }
1287
1288 /*
1289 * Don't wait for auto-negotiation to complete
1290 */
1291 hw->phy.autoneg_wait_to_complete = B_FALSE;
1292
1293 /*
1294 * Copper options
1295 */
1296 if (hw->phy.media_type == e1000_media_type_copper) {
1297 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1298 hw->phy.disable_polarity_correction = B_FALSE;
1299 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */
1300 }
1301
1302 /*
1303 * Initialize link settings
1304 */
1305 (void) igb_setup_link(igb, B_FALSE);
1306
1307 /*
1308 * Configure/Initialize hardware
1309 */
1310 if (e1000_init_hw(hw) != E1000_SUCCESS) {
1311 igb_error(igb, "Failed to initialize hardware");
1312 goto init_adapter_fail;
1313 }
1314
1315 /*
1316 * Start the link setup timer
1317 */
1318 igb_start_link_timer(igb);
1319
1320 /*
1321 * Disable wakeup control by default
1322 */
1323 E1000_WRITE_REG(hw, E1000_WUC, 0);
1324
1325 /*
1326 * Record phy info in hw struct
1327 */
1328 (void) e1000_get_phy_info(hw);
1329
1330 /*
1331 * Make sure driver has control
1332 */
1333 igb_get_driver_control(hw);
1334
1335 /*
1336 * Restore LED settings to the default from EEPROM
1337 * to meet the standard for Sun platforms.
1338 */
1339 (void) e1000_cleanup_led(hw);
1340
1341 /*
1342 * Setup MSI-X interrupts
1343 */
1344 if (igb->intr_type == DDI_INTR_TYPE_MSIX)
1345 igb->capab->setup_msix(igb);
1346
1347 /*
1348 * Initialize unicast addresses.
1349 */
1350 igb_init_unicst(igb);
1351
1352 /*
1353 * Setup and initialize the mctable structures.
1354 */
1355 igb_setup_multicst(igb);
1356
1357 /*
1358 * Set interrupt throttling rate
1359 */
1360 for (i = 0; i < igb->intr_cnt; i++)
1361 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]);
1362
1363 /*
1364 * Save the state of the phy
1365 */
1366 igb_get_phy_state(igb);
1367
1368 igb_param_sync(igb);
1369
1370 return (IGB_SUCCESS);
1371
1372 init_adapter_fail:
1373 /*
1374 * Reset PHY if possible
1375 */
1376 if (e1000_check_reset_block(hw) == E1000_SUCCESS)
1377 (void) e1000_phy_hw_reset(hw);
1378
1379 return (IGB_FAILURE);
1380 }
1381
1382 /*
1383 * igb_stop_adapter - Stop the adapter
1384 */
1385 static void
1386 igb_stop_adapter(igb_t *igb)
1387 {
1388 struct e1000_hw *hw = &igb->hw;
1389
1390 ASSERT(mutex_owned(&igb->gen_lock));
1391
1392 /* Stop the link setup timer */
1393 igb_stop_link_timer(igb);
1394
1395 /* Tell firmware driver is no longer in control */
1396 igb_release_driver_control(hw);
1397
1398 /*
1399 * Reset the chipset
1400 */
1401 if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1402 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1403 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1404 }
1405
1406 /*
1407 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient
1408 */
1409 }
1410
1411 /*
1412 * igb_reset - Reset the chipset and restart the driver.
1413 *
1414 * It involves stopping and re-starting the chipset,
1415 * and re-configuring the rx/tx rings.
1416 */
1417 static int
1418 igb_reset(igb_t *igb)
1419 {
1420 int i;
1421
1422 mutex_enter(&igb->gen_lock);
1423
1424 ASSERT(igb->igb_state & IGB_STARTED);
1425 atomic_and_32(&igb->igb_state, ~IGB_STARTED);
1426
1427 /*
1428 * Disable the adapter interrupts to stop any rx/tx activities
1429 * before draining pending data and resetting hardware.
1430 */
1431 igb_disable_adapter_interrupts(igb);
1432
1433 /*
1434 * Drain the pending transmit packets
1435 */
1436 (void) igb_tx_drain(igb);
1437
1438 for (i = 0; i < igb->num_rx_rings; i++)
1439 mutex_enter(&igb->rx_rings[i].rx_lock);
1440 for (i = 0; i < igb->num_tx_rings; i++)
1441 mutex_enter(&igb->tx_rings[i].tx_lock);
1442
1443 /*
1444 * Stop the adapter
1445 */
1446 igb_stop_adapter(igb);
1447
1448 /*
1449 * Clean the pending tx data/resources
1450 */
1451 igb_tx_clean(igb);
1452
1453 /*
1454 * Start the adapter
1455 */
1456 if (igb_init_adapter(igb) != IGB_SUCCESS) {
1457 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1458 goto reset_failure;
1459 }
1460
1461 /*
1462 * Setup the rx/tx rings
1463 */
1464 igb->tx_ring_init = B_FALSE;
1465 igb_setup_rings(igb);
1466
1467 atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL));
1468
1469 /*
1470 * Enable adapter interrupts
1471 * The interrupts must be enabled after the driver state is START
1472 */
1473 igb->capab->enable_intr(igb);
1474
1475 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1476 goto reset_failure;
1477
1478 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1479 goto reset_failure;
1480
1481 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1482 mutex_exit(&igb->tx_rings[i].tx_lock);
1483 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1484 mutex_exit(&igb->rx_rings[i].rx_lock);
1485
1486 atomic_or_32(&igb->igb_state, IGB_STARTED);
1487
1488 mutex_exit(&igb->gen_lock);
1489
1490 return (IGB_SUCCESS);
1491
1492 reset_failure:
1493 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1494 mutex_exit(&igb->tx_rings[i].tx_lock);
1495 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1496 mutex_exit(&igb->rx_rings[i].rx_lock);
1497
1498 mutex_exit(&igb->gen_lock);
1499
1500 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1501
1502 return (IGB_FAILURE);
1503 }
1504
1505 /*
1506 * igb_tx_clean - Clean the pending transmit packets and DMA resources
1507 */
1508 static void
1509 igb_tx_clean(igb_t *igb)
1510 {
1511 igb_tx_ring_t *tx_ring;
1512 tx_control_block_t *tcb;
1513 link_list_t pending_list;
1514 uint32_t desc_num;
1515 int i, j;
1516
1517 LINK_LIST_INIT(&pending_list);
1518
1519 for (i = 0; i < igb->num_tx_rings; i++) {
1520 tx_ring = &igb->tx_rings[i];
1521
1522 mutex_enter(&tx_ring->recycle_lock);
1523
1524 /*
1525 * Clean the pending tx data - the pending packets in the
1526 * work_list that have no chances to be transmitted again.
1527 *
1528 * We must ensure the chipset is stopped or the link is down
1529 * before cleaning the transmit packets.
1530 */
1531 desc_num = 0;
1532 for (j = 0; j < tx_ring->ring_size; j++) {
1533 tcb = tx_ring->work_list[j];
1534 if (tcb != NULL) {
1535 desc_num += tcb->desc_num;
1536
1537 tx_ring->work_list[j] = NULL;
1538
1539 igb_free_tcb(tcb);
1540
1541 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1542 }
1543 }
1544
1545 if (desc_num > 0) {
1546 atomic_add_32(&tx_ring->tbd_free, desc_num);
1547 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1548
1549 /*
1550 * Reset the head and tail pointers of the tbd ring;
1551 * Reset the head write-back if it is enabled.
1552 */
1553 tx_ring->tbd_head = 0;
1554 tx_ring->tbd_tail = 0;
1555 if (igb->tx_head_wb_enable)
1556 *tx_ring->tbd_head_wb = 0;
1557
1558 E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0);
1559 E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0);
1560 }
1561
1562 mutex_exit(&tx_ring->recycle_lock);
1563
1564 /*
1565 * Add the tx control blocks in the pending list to
1566 * the free list.
1567 */
1568 igb_put_free_list(tx_ring, &pending_list);
1569 }
1570 }
1571
1572 /*
1573 * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted
1574 */
1575 static boolean_t
1576 igb_tx_drain(igb_t *igb)
1577 {
1578 igb_tx_ring_t *tx_ring;
1579 boolean_t done;
1580 int i, j;
1581
1582 /*
1583 * Wait for a specific time to allow pending tx packets
1584 * to be transmitted.
1585 *
1586 * Check the counter tbd_free to see if transmission is done.
1587 * No lock protection is needed here.
1588 *
1589 * Return B_TRUE if all pending packets have been transmitted;
1590 * Otherwise return B_FALSE;
1591 */
1592 for (i = 0; i < TX_DRAIN_TIME; i++) {
1593
1594 done = B_TRUE;
1595 for (j = 0; j < igb->num_tx_rings; j++) {
1596 tx_ring = &igb->tx_rings[j];
1597 done = done &&
1598 (tx_ring->tbd_free == tx_ring->ring_size);
1599 }
1600
1601 if (done)
1602 break;
1603
1604 msec_delay(1);
1605 }
1606
1607 return (done);
1608 }
1609
1610 /*
1611 * igb_rx_drain - Wait for all rx buffers to be released by upper layer
1612 */
1613 static boolean_t
1614 igb_rx_drain(igb_t *igb)
1615 {
1616 boolean_t done;
1617 int i;
1618
1619 /*
1620 * Polling the rx free list to check if those rx buffers held by
1621 * the upper layer are released.
1622 *
1623 * Check the counter rcb_free to see if all pending buffers are
1624 * released. No lock protection is needed here.
1625 *
1626 * Return B_TRUE if all pending buffers have been released;
1627 * Otherwise return B_FALSE;
1628 */
1629 for (i = 0; i < RX_DRAIN_TIME; i++) {
1630 done = (igb->rcb_pending == 0);
1631
1632 if (done)
1633 break;
1634
1635 msec_delay(1);
1636 }
1637
1638 return (done);
1639 }
1640
1641 /*
1642 * igb_start - Start the driver/chipset
1643 */
1644 int
1645 igb_start(igb_t *igb, boolean_t alloc_buffer)
1646 {
1647 int i;
1648
1649 ASSERT(mutex_owned(&igb->gen_lock));
1650
1651 if (alloc_buffer) {
1652 if (igb_alloc_rx_data(igb) != IGB_SUCCESS) {
1653 igb_error(igb,
1654 "Failed to allocate software receive rings");
1655 return (IGB_FAILURE);
1656 }
1657
1658 /* Allocate buffers for all the rx/tx rings */
1659 if (igb_alloc_dma(igb) != IGB_SUCCESS) {
1660 igb_error(igb, "Failed to allocate DMA resource");
1661 return (IGB_FAILURE);
1662 }
1663
1664 igb->tx_ring_init = B_TRUE;
1665 } else {
1666 igb->tx_ring_init = B_FALSE;
1667 }
1668
1669 for (i = 0; i < igb->num_rx_rings; i++)
1670 mutex_enter(&igb->rx_rings[i].rx_lock);
1671 for (i = 0; i < igb->num_tx_rings; i++)
1672 mutex_enter(&igb->tx_rings[i].tx_lock);
1673
1674 /*
1675 * Start the adapter
1676 */
1677 if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) {
1678 if (igb_init_adapter(igb) != IGB_SUCCESS) {
1679 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1680 goto start_failure;
1681 }
1682 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
1683 }
1684
1685 /*
1686 * Setup the rx/tx rings
1687 */
1688 igb_setup_rings(igb);
1689
1690 /*
1691 * Enable adapter interrupts
1692 * The interrupts must be enabled after the driver state is START
1693 */
1694 igb->capab->enable_intr(igb);
1695
1696 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1697 goto start_failure;
1698
1699 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1700 goto start_failure;
1701
1702 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1703 mutex_exit(&igb->tx_rings[i].tx_lock);
1704 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1705 mutex_exit(&igb->rx_rings[i].rx_lock);
1706
1707 return (IGB_SUCCESS);
1708
1709 start_failure:
1710 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1711 mutex_exit(&igb->tx_rings[i].tx_lock);
1712 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1713 mutex_exit(&igb->rx_rings[i].rx_lock);
1714
1715 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1716
1717 return (IGB_FAILURE);
1718 }
1719
1720 /*
1721 * igb_stop - Stop the driver/chipset
1722 */
1723 void
1724 igb_stop(igb_t *igb, boolean_t free_buffer)
1725 {
1726 int i;
1727
1728 ASSERT(mutex_owned(&igb->gen_lock));
1729
1730 igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER;
1731
1732 /*
1733 * Disable the adapter interrupts
1734 */
1735 igb_disable_adapter_interrupts(igb);
1736
1737 /*
1738 * Drain the pending tx packets
1739 */
1740 (void) igb_tx_drain(igb);
1741
1742 for (i = 0; i < igb->num_rx_rings; i++)
1743 mutex_enter(&igb->rx_rings[i].rx_lock);
1744 for (i = 0; i < igb->num_tx_rings; i++)
1745 mutex_enter(&igb->tx_rings[i].tx_lock);
1746
1747 /*
1748 * Stop the adapter
1749 */
1750 igb_stop_adapter(igb);
1751
1752 /*
1753 * Clean the pending tx data/resources
1754 */
1755 igb_tx_clean(igb);
1756
1757 for (i = igb->num_tx_rings - 1; i >= 0; i--)
1758 mutex_exit(&igb->tx_rings[i].tx_lock);
1759 for (i = igb->num_rx_rings - 1; i >= 0; i--)
1760 mutex_exit(&igb->rx_rings[i].rx_lock);
1761
1762 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1763 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1764
1765 if (igb->link_state == LINK_STATE_UP) {
1766 igb->link_state = LINK_STATE_UNKNOWN;
1767 mac_link_update(igb->mac_hdl, igb->link_state);
1768 }
1769
1770 if (free_buffer) {
1771 /*
1772 * Release the DMA/memory resources of rx/tx rings
1773 */
1774 igb_free_dma(igb);
1775 igb_free_rx_data(igb);
1776 }
1777 }
1778
1779 /*
1780 * igb_alloc_rings - Allocate memory space for rx/tx rings
1781 */
1782 static int
1783 igb_alloc_rings(igb_t *igb)
1784 {
1785 /*
1786 * Allocate memory space for rx rings
1787 */
1788 igb->rx_rings = kmem_zalloc(
1789 sizeof (igb_rx_ring_t) * igb->num_rx_rings,
1790 KM_NOSLEEP);
1791
1792 if (igb->rx_rings == NULL) {
1793 return (IGB_FAILURE);
1794 }
1795
1796 /*
1797 * Allocate memory space for tx rings
1798 */
1799 igb->tx_rings = kmem_zalloc(
1800 sizeof (igb_tx_ring_t) * igb->num_tx_rings,
1801 KM_NOSLEEP);
1802
1803 if (igb->tx_rings == NULL) {
1804 kmem_free(igb->rx_rings,
1805 sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1806 igb->rx_rings = NULL;
1807 return (IGB_FAILURE);
1808 }
1809
1810 /*
1811 * Allocate memory space for rx ring groups
1812 */
1813 igb->rx_groups = kmem_zalloc(
1814 sizeof (igb_rx_group_t) * igb->num_rx_groups,
1815 KM_NOSLEEP);
1816
1817 if (igb->rx_groups == NULL) {
1818 kmem_free(igb->rx_rings,
1819 sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1820 kmem_free(igb->tx_rings,
1821 sizeof (igb_tx_ring_t) * igb->num_tx_rings);
1822 igb->rx_rings = NULL;
1823 igb->tx_rings = NULL;
1824 return (IGB_FAILURE);
1825 }
1826
1827 return (IGB_SUCCESS);
1828 }
1829
1830 /*
1831 * igb_free_rings - Free the memory space of rx/tx rings.
1832 */
1833 static void
1834 igb_free_rings(igb_t *igb)
1835 {
1836 if (igb->rx_rings != NULL) {
1837 kmem_free(igb->rx_rings,
1838 sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1839 igb->rx_rings = NULL;
1840 }
1841
1842 if (igb->tx_rings != NULL) {
1843 kmem_free(igb->tx_rings,
1844 sizeof (igb_tx_ring_t) * igb->num_tx_rings);
1845 igb->tx_rings = NULL;
1846 }
1847
1848 if (igb->rx_groups != NULL) {
1849 kmem_free(igb->rx_groups,
1850 sizeof (igb_rx_group_t) * igb->num_rx_groups);
1851 igb->rx_groups = NULL;
1852 }
1853 }
1854
1855 static int
1856 igb_alloc_rx_data(igb_t *igb)
1857 {
1858 igb_rx_ring_t *rx_ring;
1859 int i;
1860
1861 for (i = 0; i < igb->num_rx_rings; i++) {
1862 rx_ring = &igb->rx_rings[i];
1863 if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS)
1864 goto alloc_rx_rings_failure;
1865 }
1866 return (IGB_SUCCESS);
1867
1868 alloc_rx_rings_failure:
1869 igb_free_rx_data(igb);
1870 return (IGB_FAILURE);
1871 }
1872
1873 static void
1874 igb_free_rx_data(igb_t *igb)
1875 {
1876 igb_rx_ring_t *rx_ring;
1877 igb_rx_data_t *rx_data;
1878 int i;
1879
1880 for (i = 0; i < igb->num_rx_rings; i++) {
1881 rx_ring = &igb->rx_rings[i];
1882
1883 mutex_enter(&igb->rx_pending_lock);
1884 rx_data = rx_ring->rx_data;
1885
1886 if (rx_data != NULL) {
1887 rx_data->flag |= IGB_RX_STOPPED;
1888
1889 if (rx_data->rcb_pending == 0) {
1890 igb_free_rx_ring_data(rx_data);
1891 rx_ring->rx_data = NULL;
1892 }
1893 }
1894
1895 mutex_exit(&igb->rx_pending_lock);
1896 }
1897 }
1898
1899 /*
1900 * igb_setup_rings - Setup rx/tx rings
1901 */
1902 static void
1903 igb_setup_rings(igb_t *igb)
1904 {
1905 /*
1906 * Setup the rx/tx rings, including the following:
1907 *
1908 * 1. Setup the descriptor ring and the control block buffers;
1909 * 2. Initialize necessary registers for receive/transmit;
1910 * 3. Initialize software pointers/parameters for receive/transmit;
1911 */
1912 igb_setup_rx(igb);
1913
1914 igb_setup_tx(igb);
1915 }
1916
1917 static void
1918 igb_setup_rx_ring(igb_rx_ring_t *rx_ring)
1919 {
1920 igb_t *igb = rx_ring->igb;
1921 igb_rx_data_t *rx_data = rx_ring->rx_data;
1922 struct e1000_hw *hw = &igb->hw;
1923 rx_control_block_t *rcb;
1924 union e1000_adv_rx_desc *rbd;
1925 uint32_t size;
1926 uint32_t buf_low;
1927 uint32_t buf_high;
1928 uint32_t rxdctl;
1929 int i;
1930
1931 ASSERT(mutex_owned(&rx_ring->rx_lock));
1932 ASSERT(mutex_owned(&igb->gen_lock));
1933
1934 /*
1935 * Initialize descriptor ring with buffer addresses
1936 */
1937 for (i = 0; i < igb->rx_ring_size; i++) {
1938 rcb = rx_data->work_list[i];
1939 rbd = &rx_data->rbd_ring[i];
1940
1941 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
1942 rbd->read.hdr_addr = NULL;
1943 }
1944
1945 /*
1946 * Initialize the base address registers
1947 */
1948 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
1949 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
1950 E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high);
1951 E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low);
1952
1953 /*
1954 * Initialize the length register
1955 */
1956 size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc);
1957 E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size);
1958
1959 /*
1960 * Initialize buffer size & descriptor type
1961 */
1962 E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index),
1963 ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) |
1964 E1000_SRRCTL_DESCTYPE_ADV_ONEBUF));
1965
1966 /*
1967 * Setup the Receive Descriptor Control Register (RXDCTL)
1968 */
1969 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index));
1970 rxdctl &= igb->capab->rxdctl_mask;
1971 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1972 rxdctl |= 16; /* pthresh */
1973 rxdctl |= 8 << 8; /* hthresh */
1974 rxdctl |= 1 << 16; /* wthresh */
1975 E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl);
1976
1977 rx_data->rbd_next = 0;
1978 }
1979
1980 static void
1981 igb_setup_rx(igb_t *igb)
1982 {
1983 igb_rx_ring_t *rx_ring;
1984 igb_rx_data_t *rx_data;
1985 igb_rx_group_t *rx_group;
1986 struct e1000_hw *hw = &igb->hw;
1987 uint32_t rctl, rxcsum;
1988 uint32_t ring_per_group;
1989 int i;
1990
1991 /*
1992 * Setup the Receive Control Register (RCTL), and enable the
1993 * receiver. The initial configuration is to: enable the receiver,
1994 * accept broadcasts, discard bad packets, accept long packets,
1995 * disable VLAN filter checking, and set receive buffer size to
1996 * 2k. For 82575, also set the receive descriptor minimum
1997 * threshold size to 1/2 the ring.
1998 */
1999 rctl = E1000_READ_REG(hw, E1000_RCTL);
2000
2001 /*
2002 * Clear the field used for wakeup control. This driver doesn't do
2003 * wakeup but leave this here for completeness.
2004 */
2005 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2006 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2007
2008 rctl |= (E1000_RCTL_EN | /* Enable Receive Unit */
2009 E1000_RCTL_BAM | /* Accept Broadcast Packets */
2010 E1000_RCTL_LPE | /* Large Packet Enable */
2011 /* Multicast filter offset */
2012 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) |
2013 E1000_RCTL_RDMTS_HALF | /* rx descriptor threshold */
2014 E1000_RCTL_SECRC); /* Strip Ethernet CRC */
2015
2016 for (i = 0; i < igb->num_rx_groups; i++) {
2017 rx_group = &igb->rx_groups[i];
2018 rx_group->index = i;
2019 rx_group->igb = igb;
2020 }
2021
2022 /*
2023 * Set up all rx descriptor rings - must be called before receive unit
2024 * enabled.
2025 */
2026 ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2027 for (i = 0; i < igb->num_rx_rings; i++) {
2028 rx_ring = &igb->rx_rings[i];
2029 igb_setup_rx_ring(rx_ring);
2030
2031 /*
2032 * Map a ring to a group by assigning a group index
2033 */
2034 rx_ring->group_index = i / ring_per_group;
2035 }
2036
2037 /*
2038 * Setup the Rx Long Packet Max Length register
2039 */
2040 E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size);
2041
2042 /*
2043 * Hardware checksum settings
2044 */
2045 if (igb->rx_hcksum_enable) {
2046 rxcsum =
2047 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum */
2048 E1000_RXCSUM_IPOFL; /* IP checksum */
2049
2050 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2051 }
2052
2053 /*
2054 * Setup classify and RSS for multiple receive queues
2055 */
2056 switch (igb->vmdq_mode) {
2057 case E1000_VMDQ_OFF:
2058 /*
2059 * One ring group, only RSS is needed when more than
2060 * one ring enabled.
2061 */
2062 if (igb->num_rx_rings > 1)
2063 igb_setup_rss(igb);
2064 break;
2065 case E1000_VMDQ_MAC:
2066 /*
2067 * Multiple groups, each group has one ring,
2068 * only the MAC classification is needed.
2069 */
2070 igb_setup_mac_classify(igb);
2071 break;
2072 case E1000_VMDQ_MAC_RSS:
2073 /*
2074 * Multiple groups and multiple rings, both
2075 * MAC classification and RSS are needed.
2076 */
2077 igb_setup_mac_rss_classify(igb);
2078 break;
2079 }
2080
2081 /*
2082 * Enable the receive unit - must be done after all
2083 * the rx setup above.
2084 */
2085 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2086
2087 /*
2088 * Initialize all adapter ring head & tail pointers - must
2089 * be done after receive unit is enabled
2090 */
2091 for (i = 0; i < igb->num_rx_rings; i++) {
2092 rx_ring = &igb->rx_rings[i];
2093 rx_data = rx_ring->rx_data;
2094 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
2095 E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1);
2096 }
2097
2098 /*
2099 * 82575 with manageability enabled needs a special flush to make
2100 * sure the fifos start clean.
2101 */
2102 if ((hw->mac.type == e1000_82575) &&
2103 (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) {
2104 e1000_rx_fifo_flush_82575(hw);
2105 }
2106 }
2107
2108 static void
2109 igb_setup_tx_ring(igb_tx_ring_t *tx_ring)
2110 {
2111 igb_t *igb = tx_ring->igb;
2112 struct e1000_hw *hw = &igb->hw;
2113 uint32_t size;
2114 uint32_t buf_low;
2115 uint32_t buf_high;
2116 uint32_t reg_val;
2117
2118 ASSERT(mutex_owned(&tx_ring->tx_lock));
2119 ASSERT(mutex_owned(&igb->gen_lock));
2120
2121
2122 /*
2123 * Initialize the length register
2124 */
2125 size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc);
2126 E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size);
2127
2128 /*
2129 * Initialize the base address registers
2130 */
2131 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2132 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2133 E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low);
2134 E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high);
2135
2136 /*
2137 * Setup head & tail pointers
2138 */
2139 E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0);
2140 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0);
2141
2142 /*
2143 * Setup head write-back
2144 */
2145 if (igb->tx_head_wb_enable) {
2146 /*
2147 * The memory of the head write-back is allocated using
2148 * the extra tbd beyond the tail of the tbd ring.
2149 */
2150 tx_ring->tbd_head_wb = (uint32_t *)
2151 ((uintptr_t)tx_ring->tbd_area.address + size);
2152 *tx_ring->tbd_head_wb = 0;
2153
2154 buf_low = (uint32_t)
2155 (tx_ring->tbd_area.dma_address + size);
2156 buf_high = (uint32_t)
2157 ((tx_ring->tbd_area.dma_address + size) >> 32);
2158
2159 /* Set the head write-back enable bit */
2160 buf_low |= E1000_TX_HEAD_WB_ENABLE;
2161
2162 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low);
2163 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high);
2164
2165 /*
2166 * Turn off relaxed ordering for head write back or it will
2167 * cause problems with the tx recycling
2168 */
2169 reg_val = E1000_READ_REG(hw,
2170 E1000_DCA_TXCTRL(tx_ring->index));
2171 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
2172 E1000_WRITE_REG(hw,
2173 E1000_DCA_TXCTRL(tx_ring->index), reg_val);
2174 } else {
2175 tx_ring->tbd_head_wb = NULL;
2176 }
2177
2178 tx_ring->tbd_head = 0;
2179 tx_ring->tbd_tail = 0;
2180 tx_ring->tbd_free = tx_ring->ring_size;
2181
2182 if (igb->tx_ring_init == B_TRUE) {
2183 tx_ring->tcb_head = 0;
2184 tx_ring->tcb_tail = 0;
2185 tx_ring->tcb_free = tx_ring->free_list_size;
2186 }
2187
2188 /*
2189 * Enable TXDCTL per queue
2190 */
2191 reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index));
2192 reg_val |= E1000_TXDCTL_QUEUE_ENABLE;
2193 E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val);
2194
2195 /*
2196 * Initialize hardware checksum offload settings
2197 */
2198 bzero(&tx_ring->tx_context, sizeof (tx_context_t));
2199 }
2200
2201 static void
2202 igb_setup_tx(igb_t *igb)
2203 {
2204 igb_tx_ring_t *tx_ring;
2205 struct e1000_hw *hw = &igb->hw;
2206 uint32_t reg_val;
2207 int i;
2208
2209 for (i = 0; i < igb->num_tx_rings; i++) {
2210 tx_ring = &igb->tx_rings[i];
2211 igb_setup_tx_ring(tx_ring);
2212 }
2213
2214 /*
2215 * Setup the Transmit Control Register (TCTL)
2216 */
2217 reg_val = E1000_READ_REG(hw, E1000_TCTL);
2218 reg_val &= ~E1000_TCTL_CT;
2219 reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2220 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2221
2222 /* Enable transmits */
2223 reg_val |= E1000_TCTL_EN;
2224
2225 E1000_WRITE_REG(hw, E1000_TCTL, reg_val);
2226 }
2227
2228 /*
2229 * igb_setup_rss - Setup receive-side scaling feature
2230 */
2231 static void
2232 igb_setup_rss(igb_t *igb)
2233 {
2234 struct e1000_hw *hw = &igb->hw;
2235 uint32_t i, mrqc, rxcsum;
2236 int shift = 0;
2237 uint32_t random;
2238 union e1000_reta {
2239 uint32_t dword;
2240 uint8_t bytes[4];
2241 } reta;
2242
2243 /* Setup the Redirection Table */
2244 if (hw->mac.type == e1000_82576) {
2245 shift = 3;
2246 } else if (hw->mac.type == e1000_82575) {
2247 shift = 6;
2248 }
2249 for (i = 0; i < (32 * 4); i++) {
2250 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift;
2251 if ((i & 3) == 3) {
2252 E1000_WRITE_REG(hw,
2253 (E1000_RETA(0) + (i & ~3)), reta.dword);
2254 }
2255 }
2256
2257 /* Fill out hash function seeds */
2258 for (i = 0; i < 10; i++) {
2259 (void) random_get_pseudo_bytes((uint8_t *)&random,
2260 sizeof (uint32_t));
2261 E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2262 }
2263
2264 /* Setup the Multiple Receive Queue Control register */
2265 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2266 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2267 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2268 E1000_MRQC_RSS_FIELD_IPV6 |
2269 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2270 E1000_MRQC_RSS_FIELD_IPV4_UDP |
2271 E1000_MRQC_RSS_FIELD_IPV6_UDP |
2272 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2273 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2274
2275 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2276
2277 /*
2278 * Disable Packet Checksum to enable RSS for multiple receive queues.
2279 *
2280 * The Packet Checksum is not ethernet CRC. It is another kind of
2281 * checksum offloading provided by the 82575 chipset besides the IP
2282 * header checksum offloading and the TCP/UDP checksum offloading.
2283 * The Packet Checksum is by default computed over the entire packet
2284 * from the first byte of the DA through the last byte of the CRC,
2285 * including the Ethernet and IP headers.
2286 *
2287 * It is a hardware limitation that Packet Checksum is mutually
2288 * exclusive with RSS.
2289 */
2290 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2291 rxcsum |= E1000_RXCSUM_PCSD;
2292 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2293 }
2294
2295 /*
2296 * igb_setup_mac_rss_classify - Setup MAC classification and rss
2297 */
2298 static void
2299 igb_setup_mac_rss_classify(igb_t *igb)
2300 {
2301 struct e1000_hw *hw = &igb->hw;
2302 uint32_t i, mrqc, vmdctl, rxcsum;
2303 uint32_t ring_per_group;
2304 int shift_group0, shift_group1;
2305 uint32_t random;
2306 union e1000_reta {
2307 uint32_t dword;
2308 uint8_t bytes[4];
2309 } reta;
2310
2311 ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2312
2313 /* Setup the Redirection Table, it is shared between two groups */
2314 shift_group0 = 2;
2315 shift_group1 = 6;
2316 for (i = 0; i < (32 * 4); i++) {
2317 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) |
2318 ((ring_per_group + (i % ring_per_group)) << shift_group1);
2319 if ((i & 3) == 3) {
2320 E1000_WRITE_REG(hw,
2321 (E1000_RETA(0) + (i & ~3)), reta.dword);
2322 }
2323 }
2324
2325 /* Fill out hash function seeds */
2326 for (i = 0; i < 10; i++) {
2327 (void) random_get_pseudo_bytes((uint8_t *)&random,
2328 sizeof (uint32_t));
2329 E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2330 }
2331
2332 /*
2333 * Setup the Multiple Receive Queue Control register,
2334 * enable VMDq based on packet destination MAC address and RSS.
2335 */
2336 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP;
2337 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2338 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2339 E1000_MRQC_RSS_FIELD_IPV6 |
2340 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2341 E1000_MRQC_RSS_FIELD_IPV4_UDP |
2342 E1000_MRQC_RSS_FIELD_IPV6_UDP |
2343 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2344 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2345
2346 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2347
2348
2349 /* Define the default group and default queues */
2350 vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE;
2351 E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl);
2352
2353 /*
2354 * Disable Packet Checksum to enable RSS for multiple receive queues.
2355 *
2356 * The Packet Checksum is not ethernet CRC. It is another kind of
2357 * checksum offloading provided by the 82575 chipset besides the IP
2358 * header checksum offloading and the TCP/UDP checksum offloading.
2359 * The Packet Checksum is by default computed over the entire packet
2360 * from the first byte of the DA through the last byte of the CRC,
2361 * including the Ethernet and IP headers.
2362 *
2363 * It is a hardware limitation that Packet Checksum is mutually
2364 * exclusive with RSS.
2365 */
2366 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2367 rxcsum |= E1000_RXCSUM_PCSD;
2368 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2369 }
2370
2371 /*
2372 * igb_setup_mac_classify - Setup MAC classification feature
2373 */
2374 static void
2375 igb_setup_mac_classify(igb_t *igb)
2376 {
2377 struct e1000_hw *hw = &igb->hw;
2378 uint32_t mrqc, rxcsum;
2379
2380 /*
2381 * Setup the Multiple Receive Queue Control register,
2382 * enable VMDq based on packet destination MAC address.
2383 */
2384 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP;
2385 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2386
2387 /*
2388 * Disable Packet Checksum to enable RSS for multiple receive queues.
2389 *
2390 * The Packet Checksum is not ethernet CRC. It is another kind of
2391 * checksum offloading provided by the 82575 chipset besides the IP
2392 * header checksum offloading and the TCP/UDP checksum offloading.
2393 * The Packet Checksum is by default computed over the entire packet
2394 * from the first byte of the DA through the last byte of the CRC,
2395 * including the Ethernet and IP headers.
2396 *
2397 * It is a hardware limitation that Packet Checksum is mutually
2398 * exclusive with RSS.
2399 */
2400 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2401 rxcsum |= E1000_RXCSUM_PCSD;
2402 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2403
2404 }
2405
2406 /*
2407 * igb_init_unicst - Initialize the unicast addresses
2408 */
2409 static void
2410 igb_init_unicst(igb_t *igb)
2411 {
2412 struct e1000_hw *hw = &igb->hw;
2413 int slot;
2414
2415 /*
2416 * Here we should consider two situations:
2417 *
2418 * 1. Chipset is initialized the first time
2419 * Initialize the multiple unicast addresses, and
2420 * save the default MAC address.
2421 *
2422 * 2. Chipset is reset
2423 * Recover the multiple unicast addresses from the
2424 * software data structure to the RAR registers.
2425 */
2426
2427 /*
2428 * Clear the default MAC address in the RAR0 rgister,
2429 * which is loaded from EEPROM when system boot or chipreset,
2430 * this will cause the conficts with add_mac/rem_mac entry
2431 * points when VMDq is enabled. For this reason, the RAR0
2432 * must be cleared for both cases mentioned above.
2433 */
2434 e1000_rar_clear(hw, 0);
2435
2436 if (!igb->unicst_init) {
2437
2438 /* Initialize the multiple unicast addresses */
2439 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES;
2440 igb->unicst_avail = igb->unicst_total;
2441
2442 for (slot = 0; slot < igb->unicst_total; slot++)
2443 igb->unicst_addr[slot].mac.set = 0;
2444
2445 igb->unicst_init = B_TRUE;
2446 } else {
2447 /* Re-configure the RAR registers */
2448 for (slot = 0; slot < igb->unicst_total; slot++) {
2449 e1000_rar_set_vmdq(hw, igb->unicst_addr[slot].mac.addr,
2450 slot, igb->vmdq_mode,
2451 igb->unicst_addr[slot].mac.group_index);
2452 }
2453 }
2454 }
2455
2456 /*
2457 * igb_unicst_find - Find the slot for the specified unicast address
2458 */
2459 int
2460 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr)
2461 {
2462 int slot;
2463
2464 ASSERT(mutex_owned(&igb->gen_lock));
2465
2466 for (slot = 0; slot < igb->unicst_total; slot++) {
2467 if (bcmp(igb->unicst_addr[slot].mac.addr,
2468 mac_addr, ETHERADDRL) == 0)
2469 return (slot);
2470 }
2471
2472 return (-1);
2473 }
2474
2475 /*
2476 * igb_unicst_set - Set the unicast address to the specified slot
2477 */
2478 int
2479 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr,
2480 int slot)
2481 {
2482 struct e1000_hw *hw = &igb->hw;
2483
2484 ASSERT(mutex_owned(&igb->gen_lock));
2485
2486 /*
2487 * Save the unicast address in the software data structure
2488 */
2489 bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL);
2490
2491 /*
2492 * Set the unicast address to the RAR register
2493 */
2494 e1000_rar_set(hw, (uint8_t *)mac_addr, slot);
2495
2496 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2497 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2498 return (EIO);
2499 }
2500
2501 return (0);
2502 }
2503
2504 /*
2505 * igb_multicst_add - Add a multicst address
2506 */
2507 int
2508 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr)
2509 {
2510 struct ether_addr *new_table;
2511 size_t new_len;
2512 size_t old_len;
2513
2514 ASSERT(mutex_owned(&igb->gen_lock));
2515
2516 if ((multiaddr[0] & 01) == 0) {
2517 igb_error(igb, "Illegal multicast address");
2518 return (EINVAL);
2519 }
2520
2521 if (igb->mcast_count >= igb->mcast_max_num) {
2522 igb_error(igb, "Adapter requested more than %d mcast addresses",
2523 igb->mcast_max_num);
2524 return (ENOENT);
2525 }
2526
2527 if (igb->mcast_count == igb->mcast_alloc_count) {
2528 old_len = igb->mcast_alloc_count *
2529 sizeof (struct ether_addr);
2530 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) *
2531 sizeof (struct ether_addr);
2532
2533 new_table = kmem_alloc(new_len, KM_NOSLEEP);
2534 if (new_table == NULL) {
2535 igb_error(igb,
2536 "Not enough memory to alloc mcast table");
2537 return (ENOMEM);
2538 }
2539
2540 if (igb->mcast_table != NULL) {
2541 bcopy(igb->mcast_table, new_table, old_len);
2542 kmem_free(igb->mcast_table, old_len);
2543 }
2544 igb->mcast_alloc_count += MCAST_ALLOC_COUNT;
2545 igb->mcast_table = new_table;
2546 }
2547
2548 bcopy(multiaddr,
2549 &igb->mcast_table[igb->mcast_count], ETHERADDRL);
2550 igb->mcast_count++;
2551
2552 /*
2553 * Update the multicast table in the hardware
2554 */
2555 igb_setup_multicst(igb);
2556
2557 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2558 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2559 return (EIO);
2560 }
2561
2562 return (0);
2563 }
2564
2565 /*
2566 * igb_multicst_remove - Remove a multicst address
2567 */
2568 int
2569 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr)
2570 {
2571 struct ether_addr *new_table;
2572 size_t new_len;
2573 size_t old_len;
2574 int i;
2575
2576 ASSERT(mutex_owned(&igb->gen_lock));
2577
2578 for (i = 0; i < igb->mcast_count; i++) {
2579 if (bcmp(multiaddr, &igb->mcast_table[i],
2580 ETHERADDRL) == 0) {
2581 for (i++; i < igb->mcast_count; i++) {
2582 igb->mcast_table[i - 1] =
2583 igb->mcast_table[i];
2584 }
2585 igb->mcast_count--;
2586 break;
2587 }
2588 }
2589
2590 if ((igb->mcast_alloc_count - igb->mcast_count) >
2591 MCAST_ALLOC_COUNT) {
2592 old_len = igb->mcast_alloc_count *
2593 sizeof (struct ether_addr);
2594 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) *
2595 sizeof (struct ether_addr);
2596
2597 new_table = kmem_alloc(new_len, KM_NOSLEEP);
2598 if (new_table != NULL) {
2599 bcopy(igb->mcast_table, new_table, new_len);
2600 kmem_free(igb->mcast_table, old_len);
2601 igb->mcast_alloc_count -= MCAST_ALLOC_COUNT;
2602 igb->mcast_table = new_table;
2603 }
2604 }
2605
2606 /*
2607 * Update the multicast table in the hardware
2608 */
2609 igb_setup_multicst(igb);
2610
2611 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2612 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2613 return (EIO);
2614 }
2615
2616 return (0);
2617 }
2618
2619 static void
2620 igb_release_multicast(igb_t *igb)
2621 {
2622 if (igb->mcast_table != NULL) {
2623 kmem_free(igb->mcast_table,
2624 igb->mcast_alloc_count * sizeof (struct ether_addr));
2625 igb->mcast_table = NULL;
2626 }
2627 }
2628
2629 /*
2630 * igb_setup_multicast - setup multicast data structures
2631 *
2632 * This routine initializes all of the multicast related structures
2633 * and save them in the hardware registers.
2634 */
2635 static void
2636 igb_setup_multicst(igb_t *igb)
2637 {
2638 uint8_t *mc_addr_list;
2639 uint32_t mc_addr_count;
2640 struct e1000_hw *hw = &igb->hw;
2641
2642 ASSERT(mutex_owned(&igb->gen_lock));
2643 ASSERT(igb->mcast_count <= igb->mcast_max_num);
2644
2645 mc_addr_list = (uint8_t *)igb->mcast_table;
2646 mc_addr_count = igb->mcast_count;
2647
2648 /*
2649 * Update the multicase addresses to the MTA registers
2650 */
2651 e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count);
2652 }
2653
2654 /*
2655 * igb_get_conf - Get driver configurations set in driver.conf
2656 *
2657 * This routine gets user-configured values out of the configuration
2658 * file igb.conf.
2659 *
2660 * For each configurable value, there is a minimum, a maximum, and a
2661 * default.
2662 * If user does not configure a value, use the default.
2663 * If user configures below the minimum, use the minumum.
2664 * If user configures above the maximum, use the maxumum.
2665 */
2666 static void
2667 igb_get_conf(igb_t *igb)
2668 {
2669 struct e1000_hw *hw = &igb->hw;
2670 uint32_t default_mtu;
2671 uint32_t flow_control;
2672 uint32_t ring_per_group;
2673 int i;
2674
2675 /*
2676 * igb driver supports the following user configurations:
2677 *
2678 * Link configurations:
2679 * adv_autoneg_cap
2680 * adv_1000fdx_cap
2681 * adv_100fdx_cap
2682 * adv_100hdx_cap
2683 * adv_10fdx_cap
2684 * adv_10hdx_cap
2685 * Note: 1000hdx is not supported.
2686 *
2687 * Jumbo frame configuration:
2688 * default_mtu
2689 *
2690 * Ethernet flow control configuration:
2691 * flow_control
2692 *
2693 * Multiple rings configurations:
2694 * tx_queue_number
2695 * tx_ring_size
2696 * rx_queue_number
2697 * rx_ring_size
2698 *
2699 * Call igb_get_prop() to get the value for a specific
2700 * configuration parameter.
2701 */
2702
2703 /*
2704 * Link configurations
2705 */
2706 igb->param_adv_autoneg_cap = igb_get_prop(igb,
2707 PROP_ADV_AUTONEG_CAP, 0, 1, 1);
2708 igb->param_adv_1000fdx_cap = igb_get_prop(igb,
2709 PROP_ADV_1000FDX_CAP, 0, 1, 1);
2710 igb->param_adv_100fdx_cap = igb_get_prop(igb,
2711 PROP_ADV_100FDX_CAP, 0, 1, 1);
2712 igb->param_adv_100hdx_cap = igb_get_prop(igb,
2713 PROP_ADV_100HDX_CAP, 0, 1, 1);
2714 igb->param_adv_10fdx_cap = igb_get_prop(igb,
2715 PROP_ADV_10FDX_CAP, 0, 1, 1);
2716 igb->param_adv_10hdx_cap = igb_get_prop(igb,
2717 PROP_ADV_10HDX_CAP, 0, 1, 1);
2718
2719 /*
2720 * Jumbo frame configurations
2721 */
2722 default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
2723 MIN_MTU, MAX_MTU, DEFAULT_MTU);
2724
2725 igb->max_frame_size = default_mtu +
2726 sizeof (struct ether_vlan_header) + ETHERFCSL;
2727
2728 /*
2729 * Ethernet flow control configuration
2730 */
2731 flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL,
2732 e1000_fc_none, 4, e1000_fc_full);
2733 if (flow_control == 4)
2734 flow_control = e1000_fc_default;
2735
2736 hw->fc.requested_mode = flow_control;
2737
2738 /*
2739 * Multiple rings configurations
2740 */
2741 igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE,
2742 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
2743 igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE,
2744 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
2745
2746 igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0);
2747 igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM,
2748 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM);
2749 /*
2750 * Currently we do not support VMDq for 82576 and 82580.
2751 * If it is e1000_82576, set num_rx_groups to 1.
2752 */
2753 if (hw->mac.type >= e1000_82576)
2754 igb->num_rx_groups = 1;
2755
2756 if (igb->mr_enable) {
2757 igb->num_tx_rings = igb->capab->def_tx_que_num;
2758 igb->num_rx_rings = igb->capab->def_rx_que_num;
2759 } else {
2760 igb->num_tx_rings = 1;
2761 igb->num_rx_rings = 1;
2762
2763 if (igb->num_rx_groups > 1) {
2764 igb_error(igb,
2765 "Invalid rx groups number. Please enable multiple "
2766 "rings first");
2767 igb->num_rx_groups = 1;
2768 }
2769 }
2770
2771 /*
2772 * Check the divisibility between rx rings and rx groups.
2773 */
2774 for (i = igb->num_rx_groups; i > 0; i--) {
2775 if ((igb->num_rx_rings % i) == 0)
2776 break;
2777 }
2778 if (i != igb->num_rx_groups) {
2779 igb_error(igb,
2780 "Invalid rx groups number. Downgrade the rx group "
2781 "number to %d.", i);
2782 igb->num_rx_groups = i;
2783 }
2784
2785 /*
2786 * Get the ring number per group.
2787 */
2788 ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2789
2790 if (igb->num_rx_groups == 1) {
2791 /*
2792 * One rx ring group, the rx ring number is num_rx_rings.
2793 */
2794 igb->vmdq_mode = E1000_VMDQ_OFF;
2795 } else if (ring_per_group == 1) {
2796 /*
2797 * Multiple rx groups, each group has one rx ring.
2798 */
2799 igb->vmdq_mode = E1000_VMDQ_MAC;
2800 } else {
2801 /*
2802 * Multiple groups and multiple rings.
2803 */
2804 igb->vmdq_mode = E1000_VMDQ_MAC_RSS;
2805 }
2806
2807 /*
2808 * Tunable used to force an interrupt type. The only use is
2809 * for testing of the lesser interrupt types.
2810 * 0 = don't force interrupt type
2811 * 1 = force interrupt type MSIX
2812 * 2 = force interrupt type MSI
2813 * 3 = force interrupt type Legacy
2814 */
2815 igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE,
2816 IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE);
2817
2818 igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE,
2819 0, 1, 1);
2820 igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE,
2821 0, 1, 1);
2822 igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE,
2823 0, 1, 1);
2824 igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE,
2825 0, 1, 1);
2826
2827 /*
2828 * igb LSO needs the tx h/w checksum support.
2829 * Here LSO will be disabled if tx h/w checksum has been disabled.
2830 */
2831 if (igb->tx_hcksum_enable == B_FALSE)
2832 igb->lso_enable = B_FALSE;
2833
2834 igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD,
2835 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
2836 DEFAULT_TX_COPY_THRESHOLD);
2837 igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD,
2838 MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD,
2839 DEFAULT_TX_RECYCLE_THRESHOLD);
2840 igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD,
2841 MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD,
2842 DEFAULT_TX_OVERLOAD_THRESHOLD);
2843 igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD,
2844 MIN_TX_RESCHED_THRESHOLD,
2845 MIN(igb->tx_ring_size, MAX_TX_RESCHED_THRESHOLD),
2846 igb->tx_ring_size > DEFAULT_TX_RESCHED_THRESHOLD ?
2847 DEFAULT_TX_RESCHED_THRESHOLD : DEFAULT_TX_RESCHED_THRESHOLD_LOW);
2848
2849 igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD,
2850 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
2851 DEFAULT_RX_COPY_THRESHOLD);
2852 igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR,
2853 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
2854 DEFAULT_RX_LIMIT_PER_INTR);
2855
2856 igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING,
2857 igb->capab->min_intr_throttle,
2858 igb->capab->max_intr_throttle,
2859 igb->capab->def_intr_throttle);
2860
2861 /*
2862 * Max number of multicast addresses
2863 */
2864 igb->mcast_max_num =
2865 igb_get_prop(igb, PROP_MCAST_MAX_NUM,
2866 MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM);
2867 }
2868
2869 /*
2870 * igb_get_prop - Get a property value out of the configuration file igb.conf
2871 *
2872 * Caller provides the name of the property, a default value, a minimum
2873 * value, and a maximum value.
2874 *
2875 * Return configured value of the property, with default, minimum and
2876 * maximum properly applied.
2877 */
2878 static int
2879 igb_get_prop(igb_t *igb,
2880 char *propname, /* name of the property */
2881 int minval, /* minimum acceptable value */
2882 int maxval, /* maximim acceptable value */
2883 int defval) /* default value */
2884 {
2885 int value;
2886
2887 /*
2888 * Call ddi_prop_get_int() to read the conf settings
2889 */
2890 value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip,
2891 DDI_PROP_DONTPASS, propname, defval);
2892
2893 if (value > maxval)
2894 value = maxval;
2895
2896 if (value < minval)
2897 value = minval;
2898
2899 return (value);
2900 }
2901
2902 /*
2903 * igb_setup_link - Using the link properties to setup the link
2904 */
2905 int
2906 igb_setup_link(igb_t *igb, boolean_t setup_hw)
2907 {
2908 struct e1000_mac_info *mac;
2909 struct e1000_phy_info *phy;
2910 boolean_t invalid;
2911
2912 mac = &igb->hw.mac;
2913 phy = &igb->hw.phy;
2914 invalid = B_FALSE;
2915
2916 if (igb->param_adv_autoneg_cap == 1) {
2917 mac->autoneg = B_TRUE;
2918 phy->autoneg_advertised = 0;
2919
2920 /*
2921 * 1000hdx is not supported for autonegotiation
2922 */
2923 if (igb->param_adv_1000fdx_cap == 1)
2924 phy->autoneg_advertised |= ADVERTISE_1000_FULL;
2925
2926 if (igb->param_adv_100fdx_cap == 1)
2927 phy->autoneg_advertised |= ADVERTISE_100_FULL;
2928
2929 if (igb->param_adv_100hdx_cap == 1)
2930 phy->autoneg_advertised |= ADVERTISE_100_HALF;
2931
2932 if (igb->param_adv_10fdx_cap == 1)
2933 phy->autoneg_advertised |= ADVERTISE_10_FULL;
2934
2935 if (igb->param_adv_10hdx_cap == 1)
2936 phy->autoneg_advertised |= ADVERTISE_10_HALF;
2937
2938 if (phy->autoneg_advertised == 0)
2939 invalid = B_TRUE;
2940 } else {
2941 mac->autoneg = B_FALSE;
2942
2943 /*
2944 * 1000fdx and 1000hdx are not supported for forced link
2945 */
2946 if (igb->param_adv_100fdx_cap == 1)
2947 mac->forced_speed_duplex = ADVERTISE_100_FULL;
2948 else if (igb->param_adv_100hdx_cap == 1)
2949 mac->forced_speed_duplex = ADVERTISE_100_HALF;
2950 else if (igb->param_adv_10fdx_cap == 1)
2951 mac->forced_speed_duplex = ADVERTISE_10_FULL;
2952 else if (igb->param_adv_10hdx_cap == 1)
2953 mac->forced_speed_duplex = ADVERTISE_10_HALF;
2954 else
2955 invalid = B_TRUE;
2956 }
2957
2958 if (invalid) {
2959 igb_notice(igb, "Invalid link settings. Setup link to "
2960 "autonegotiation with full link capabilities.");
2961 mac->autoneg = B_TRUE;
2962 phy->autoneg_advertised = ADVERTISE_1000_FULL |
2963 ADVERTISE_100_FULL | ADVERTISE_100_HALF |
2964 ADVERTISE_10_FULL | ADVERTISE_10_HALF;
2965 }
2966
2967 if (setup_hw) {
2968 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS)
2969 return (IGB_FAILURE);
2970 }
2971
2972 return (IGB_SUCCESS);
2973 }
2974
2975
2976 /*
2977 * igb_is_link_up - Check if the link is up
2978 */
2979 static boolean_t
2980 igb_is_link_up(igb_t *igb)
2981 {
2982 struct e1000_hw *hw = &igb->hw;
2983 boolean_t link_up = B_FALSE;
2984
2985 ASSERT(mutex_owned(&igb->gen_lock));
2986
2987 /*
2988 * get_link_status is set in the interrupt handler on link-status-change
2989 * or rx sequence error interrupt. get_link_status will stay
2990 * false until the e1000_check_for_link establishes link only
2991 * for copper adapters.
2992 */
2993 switch (hw->phy.media_type) {
2994 case e1000_media_type_copper:
2995 if (hw->mac.get_link_status) {
2996 (void) e1000_check_for_link(hw);
2997 link_up = !hw->mac.get_link_status;
2998 } else {
2999 link_up = B_TRUE;
3000 }
3001 break;
3002 case e1000_media_type_fiber:
3003 (void) e1000_check_for_link(hw);
3004 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU);
3005 break;
3006 case e1000_media_type_internal_serdes:
3007 (void) e1000_check_for_link(hw);
3008 link_up = hw->mac.serdes_has_link;
3009 break;
3010 }
3011
3012 return (link_up);
3013 }
3014
3015 /*
3016 * igb_link_check - Link status processing
3017 */
3018 static boolean_t
3019 igb_link_check(igb_t *igb)
3020 {
3021 struct e1000_hw *hw = &igb->hw;
3022 uint16_t speed = 0, duplex = 0;
3023 boolean_t link_changed = B_FALSE;
3024
3025 ASSERT(mutex_owned(&igb->gen_lock));
3026
3027 if (igb_is_link_up(igb)) {
3028 /*
3029 * The Link is up, check whether it was marked as down earlier
3030 */
3031 if (igb->link_state != LINK_STATE_UP) {
3032 (void) e1000_get_speed_and_duplex(hw, &speed, &duplex);
3033 igb->link_speed = speed;
3034 igb->link_duplex = duplex;
3035 igb->link_state = LINK_STATE_UP;
3036 link_changed = B_TRUE;
3037 if (!igb->link_complete)
3038 igb_stop_link_timer(igb);
3039 }
3040 } else if (igb->link_complete) {
3041 if (igb->link_state != LINK_STATE_DOWN) {
3042 igb->link_speed = 0;
3043 igb->link_duplex = 0;
3044 igb->link_state = LINK_STATE_DOWN;
3045 link_changed = B_TRUE;
3046 }
3047 }
3048
3049 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3050 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3051 return (B_FALSE);
3052 }
3053
3054 return (link_changed);
3055 }
3056
3057 /*
3058 * igb_local_timer - driver watchdog function
3059 *
3060 * This function will handle the hardware stall check, link status
3061 * check and other routines.
3062 */
3063 static void
3064 igb_local_timer(void *arg)
3065 {
3066 igb_t *igb = (igb_t *)arg;
3067 boolean_t link_changed = B_FALSE;
3068
3069 if (igb->igb_state & IGB_ERROR) {
3070 igb->reset_count++;
3071 if (igb_reset(igb) == IGB_SUCCESS)
3072 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3073
3074 igb_restart_watchdog_timer(igb);
3075 return;
3076 }
3077
3078 if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) {
3079 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL);
3080 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
3081 igb->reset_count++;
3082 if (igb_reset(igb) == IGB_SUCCESS)
3083 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3084
3085 igb_restart_watchdog_timer(igb);
3086 return;
3087 }
3088
3089 mutex_enter(&igb->gen_lock);
3090 if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED))
3091 link_changed = igb_link_check(igb);
3092 mutex_exit(&igb->gen_lock);
3093
3094 if (link_changed)
3095 mac_link_update(igb->mac_hdl, igb->link_state);
3096
3097 igb_restart_watchdog_timer(igb);
3098 }
3099
3100 /*
3101 * igb_link_timer - link setup timer function
3102 *
3103 * It is called when the timer for link setup is expired, which indicates
3104 * the completion of the link setup. The link state will not be updated
3105 * until the link setup is completed. And the link state will not be sent
3106 * to the upper layer through mac_link_update() in this function. It will
3107 * be updated in the local timer routine or the interrupts service routine
3108 * after the interface is started (plumbed).
3109 */
3110 static void
3111 igb_link_timer(void *arg)
3112 {
3113 igb_t *igb = (igb_t *)arg;
3114
3115 mutex_enter(&igb->link_lock);
3116 igb->link_complete = B_TRUE;
3117 igb->link_tid = 0;
3118 mutex_exit(&igb->link_lock);
3119 }
3120 /*
3121 * igb_stall_check - check for transmit stall
3122 *
3123 * This function checks if the adapter is stalled (in transmit).
3124 *
3125 * It is called each time the watchdog timeout is invoked.
3126 * If the transmit descriptor reclaim continuously fails,
3127 * the watchdog value will increment by 1. If the watchdog
3128 * value exceeds the threshold, the igb is assumed to
3129 * have stalled and need to be reset.
3130 */
3131 static boolean_t
3132 igb_stall_check(igb_t *igb)
3133 {
3134 igb_tx_ring_t *tx_ring;
3135 struct e1000_hw *hw = &igb->hw;
3136 boolean_t result;
3137 int i;
3138
3139 if (igb->link_state != LINK_STATE_UP)
3140 return (B_FALSE);
3141
3142 /*
3143 * If any tx ring is stalled, we'll reset the chipset
3144 */
3145 result = B_FALSE;
3146 for (i = 0; i < igb->num_tx_rings; i++) {
3147 tx_ring = &igb->tx_rings[i];
3148
3149 if (tx_ring->recycle_fail > 0)
3150 tx_ring->stall_watchdog++;
3151 else
3152 tx_ring->stall_watchdog = 0;
3153
3154 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3155 result = B_TRUE;
3156 if (hw->mac.type == e1000_82580) {
3157 hw->dev_spec._82575.global_device_reset
3158 = B_TRUE;
3159 }
3160 break;
3161 }
3162 }
3163
3164 if (result) {
3165 tx_ring->stall_watchdog = 0;
3166 tx_ring->recycle_fail = 0;
3167 }
3168
3169 return (result);
3170 }
3171
3172
3173 /*
3174 * is_valid_mac_addr - Check if the mac address is valid
3175 */
3176 static boolean_t
3177 is_valid_mac_addr(uint8_t *mac_addr)
3178 {
3179 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3180 const uint8_t addr_test2[6] =
3181 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3182
3183 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3184 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3185 return (B_FALSE);
3186
3187 return (B_TRUE);
3188 }
3189
3190 static boolean_t
3191 igb_find_mac_address(igb_t *igb)
3192 {
3193 struct e1000_hw *hw = &igb->hw;
3194 #ifdef __sparc
3195 uchar_t *bytes;
3196 struct ether_addr sysaddr;
3197 uint_t nelts;
3198 int err;
3199 boolean_t found = B_FALSE;
3200
3201 /*
3202 * The "vendor's factory-set address" may already have
3203 * been extracted from the chip, but if the property
3204 * "local-mac-address" is set we use that instead.
3205 *
3206 * We check whether it looks like an array of 6
3207 * bytes (which it should, if OBP set it). If we can't
3208 * make sense of it this way, we'll ignore it.
3209 */
3210 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3211 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3212 if (err == DDI_PROP_SUCCESS) {
3213 if (nelts == ETHERADDRL) {
3214 while (nelts--)
3215 hw->mac.addr[nelts] = bytes[nelts];
3216 found = B_TRUE;
3217 }
3218 ddi_prop_free(bytes);
3219 }
3220
3221 /*
3222 * Look up the OBP property "local-mac-address?". If the user has set
3223 * 'local-mac-address? = false', use "the system address" instead.
3224 */
3225 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0,
3226 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3227 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3228 if (localetheraddr(NULL, &sysaddr) != 0) {
3229 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3230 found = B_TRUE;
3231 }
3232 }
3233 ddi_prop_free(bytes);
3234 }
3235
3236 /*
3237 * Finally(!), if there's a valid "mac-address" property (created
3238 * if we netbooted from this interface), we must use this instead
3239 * of any of the above to ensure that the NFS/install server doesn't
3240 * get confused by the address changing as Solaris takes over!
3241 */
3242 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3243 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3244 if (err == DDI_PROP_SUCCESS) {
3245 if (nelts == ETHERADDRL) {
3246 while (nelts--)
3247 hw->mac.addr[nelts] = bytes[nelts];
3248 found = B_TRUE;
3249 }
3250 ddi_prop_free(bytes);
3251 }
3252
3253 if (found) {
3254 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3255 return (B_TRUE);
3256 }
3257 #endif
3258
3259 /*
3260 * Read the device MAC address from the EEPROM
3261 */
3262 if (e1000_read_mac_addr(hw) != E1000_SUCCESS)
3263 return (B_FALSE);
3264
3265 return (B_TRUE);
3266 }
3267
3268 #pragma inline(igb_arm_watchdog_timer)
3269
3270 static void
3271 igb_arm_watchdog_timer(igb_t *igb)
3272 {
3273 /*
3274 * Fire a watchdog timer
3275 */
3276 igb->watchdog_tid =
3277 timeout(igb_local_timer,
3278 (void *)igb, 1 * drv_usectohz(1000000));
3279
3280 }
3281
3282 /*
3283 * igb_enable_watchdog_timer - Enable and start the driver watchdog timer
3284 */
3285 void
3286 igb_enable_watchdog_timer(igb_t *igb)
3287 {
3288 mutex_enter(&igb->watchdog_lock);
3289
3290 if (!igb->watchdog_enable) {
3291 igb->watchdog_enable = B_TRUE;
3292 igb->watchdog_start = B_TRUE;
3293 igb_arm_watchdog_timer(igb);
3294 }
3295
3296 mutex_exit(&igb->watchdog_lock);
3297
3298 }
3299
3300 /*
3301 * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer
3302 */
3303 void
3304 igb_disable_watchdog_timer(igb_t *igb)
3305 {
3306 timeout_id_t tid;
3307
3308 mutex_enter(&igb->watchdog_lock);
3309
3310 igb->watchdog_enable = B_FALSE;
3311 igb->watchdog_start = B_FALSE;
3312 tid = igb->watchdog_tid;
3313 igb->watchdog_tid = 0;
3314
3315 mutex_exit(&igb->watchdog_lock);
3316
3317 if (tid != 0)
3318 (void) untimeout(tid);
3319
3320 }
3321
3322 /*
3323 * igb_start_watchdog_timer - Start the driver watchdog timer
3324 */
3325 static void
3326 igb_start_watchdog_timer(igb_t *igb)
3327 {
3328 mutex_enter(&igb->watchdog_lock);
3329
3330 if (igb->watchdog_enable) {
3331 if (!igb->watchdog_start) {
3332 igb->watchdog_start = B_TRUE;
3333 igb_arm_watchdog_timer(igb);
3334 }
3335 }
3336
3337 mutex_exit(&igb->watchdog_lock);
3338 }
3339
3340 /*
3341 * igb_restart_watchdog_timer - Restart the driver watchdog timer
3342 */
3343 static void
3344 igb_restart_watchdog_timer(igb_t *igb)
3345 {
3346 mutex_enter(&igb->watchdog_lock);
3347
3348 if (igb->watchdog_start)
3349 igb_arm_watchdog_timer(igb);
3350
3351 mutex_exit(&igb->watchdog_lock);
3352 }
3353
3354 /*
3355 * igb_stop_watchdog_timer - Stop the driver watchdog timer
3356 */
3357 static void
3358 igb_stop_watchdog_timer(igb_t *igb)
3359 {
3360 timeout_id_t tid;
3361
3362 mutex_enter(&igb->watchdog_lock);
3363
3364 igb->watchdog_start = B_FALSE;
3365 tid = igb->watchdog_tid;
3366 igb->watchdog_tid = 0;
3367
3368 mutex_exit(&igb->watchdog_lock);
3369
3370 if (tid != 0)
3371 (void) untimeout(tid);
3372 }
3373
3374 /*
3375 * igb_start_link_timer - Start the link setup timer
3376 */
3377 static void
3378 igb_start_link_timer(struct igb *igb)
3379 {
3380 struct e1000_hw *hw = &igb->hw;
3381 clock_t link_timeout;
3382
3383 if (hw->mac.autoneg)
3384 link_timeout = PHY_AUTO_NEG_LIMIT *
3385 drv_usectohz(100000);
3386 else
3387 link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000);
3388
3389 mutex_enter(&igb->link_lock);
3390 if (hw->phy.autoneg_wait_to_complete) {
3391 igb->link_complete = B_TRUE;
3392 } else {
3393 igb->link_complete = B_FALSE;
3394 igb->link_tid = timeout(igb_link_timer, (void *)igb,
3395 link_timeout);
3396 }
3397 mutex_exit(&igb->link_lock);
3398 }
3399
3400 /*
3401 * igb_stop_link_timer - Stop the link setup timer
3402 */
3403 static void
3404 igb_stop_link_timer(struct igb *igb)
3405 {
3406 timeout_id_t tid;
3407
3408 mutex_enter(&igb->link_lock);
3409 igb->link_complete = B_TRUE;
3410 tid = igb->link_tid;
3411 igb->link_tid = 0;
3412 mutex_exit(&igb->link_lock);
3413
3414 if (tid != 0)
3415 (void) untimeout(tid);
3416 }
3417
3418 /*
3419 * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts
3420 */
3421 static void
3422 igb_disable_adapter_interrupts(igb_t *igb)
3423 {
3424 struct e1000_hw *hw = &igb->hw;
3425
3426 /*
3427 * Set the IMC register to mask all the interrupts,
3428 * including the tx interrupts.
3429 */
3430 E1000_WRITE_REG(hw, E1000_IMC, ~0);
3431 E1000_WRITE_REG(hw, E1000_IAM, 0);
3432
3433 /*
3434 * Additional disabling for MSI-X
3435 */
3436 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3437 E1000_WRITE_REG(hw, E1000_EIMC, ~0);
3438 E1000_WRITE_REG(hw, E1000_EIAC, 0);
3439 E1000_WRITE_REG(hw, E1000_EIAM, 0);
3440 }
3441
3442 E1000_WRITE_FLUSH(hw);
3443 }
3444
3445 /*
3446 * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580
3447 */
3448 static void
3449 igb_enable_adapter_interrupts_82580(igb_t *igb)
3450 {
3451 struct e1000_hw *hw = &igb->hw;
3452
3453 /* Clear any pending interrupts */
3454 (void) E1000_READ_REG(hw, E1000_ICR);
3455 igb->ims_mask |= E1000_IMS_DRSTA;
3456
3457 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3458
3459 /* Interrupt enabling for MSI-X */
3460 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3461 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3462 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA);
3463 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3464 } else { /* Interrupt enabling for MSI and legacy */
3465 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3466 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3467 igb->ims_mask |= E1000_IMS_DRSTA;
3468 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3469 }
3470
3471 /* Disable auto-mask for ICR interrupt bits */
3472 E1000_WRITE_REG(hw, E1000_IAM, 0);
3473
3474 E1000_WRITE_FLUSH(hw);
3475 }
3476
3477 /*
3478 * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576
3479 */
3480 static void
3481 igb_enable_adapter_interrupts_82576(igb_t *igb)
3482 {
3483 struct e1000_hw *hw = &igb->hw;
3484
3485 /* Clear any pending interrupts */
3486 (void) E1000_READ_REG(hw, E1000_ICR);
3487
3488 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3489
3490 /* Interrupt enabling for MSI-X */
3491 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3492 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3493 igb->ims_mask = E1000_IMS_LSC;
3494 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3495 } else {
3496 /* Interrupt enabling for MSI and legacy */
3497 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3498 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3499 E1000_WRITE_REG(hw, E1000_IMS,
3500 (IMS_ENABLE_MASK | E1000_IMS_TXQE));
3501 }
3502
3503 /* Disable auto-mask for ICR interrupt bits */
3504 E1000_WRITE_REG(hw, E1000_IAM, 0);
3505
3506 E1000_WRITE_FLUSH(hw);
3507 }
3508
3509 /*
3510 * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575
3511 */
3512 static void
3513 igb_enable_adapter_interrupts_82575(igb_t *igb)
3514 {
3515 struct e1000_hw *hw = &igb->hw;
3516 uint32_t reg;
3517
3518 /* Clear any pending interrupts */
3519 (void) E1000_READ_REG(hw, E1000_ICR);
3520
3521 if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3522 /* Interrupt enabling for MSI-X */
3523 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3524 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3525 igb->ims_mask = E1000_IMS_LSC;
3526 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3527
3528 /* Enable MSI-X PBA support */
3529 reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
3530 reg |= E1000_CTRL_EXT_PBA_CLR;
3531
3532 /* Non-selective interrupt clear-on-read */
3533 reg |= E1000_CTRL_EXT_IRCA; /* Called NSICR in the EAS */
3534
3535 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
3536 } else {
3537 /* Interrupt enabling for MSI and legacy */
3538 igb->ims_mask = IMS_ENABLE_MASK;
3539 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK);
3540 }
3541
3542 E1000_WRITE_FLUSH(hw);
3543 }
3544
3545 /*
3546 * Loopback Support
3547 */
3548 static lb_property_t lb_normal =
3549 { normal, "normal", IGB_LB_NONE };
3550 static lb_property_t lb_external =
3551 { external, "External", IGB_LB_EXTERNAL };
3552 static lb_property_t lb_phy =
3553 { internal, "PHY", IGB_LB_INTERNAL_PHY };
3554 static lb_property_t lb_serdes =
3555 { internal, "SerDes", IGB_LB_INTERNAL_SERDES };
3556
3557 enum ioc_reply
3558 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp)
3559 {
3560 lb_info_sz_t *lbsp;
3561 lb_property_t *lbpp;
3562 struct e1000_hw *hw;
3563 uint32_t *lbmp;
3564 uint32_t size;
3565 uint32_t value;
3566
3567 hw = &igb->hw;
3568
3569 if (mp->b_cont == NULL)
3570 return (IOC_INVAL);
3571
3572 switch (iocp->ioc_cmd) {
3573 default:
3574 return (IOC_INVAL);
3575
3576 case LB_GET_INFO_SIZE:
3577 size = sizeof (lb_info_sz_t);
3578 if (iocp->ioc_count != size)
3579 return (IOC_INVAL);
3580
3581 value = sizeof (lb_normal);
3582 if (hw->phy.media_type == e1000_media_type_copper)
3583 value += sizeof (lb_phy);
3584 else
3585 value += sizeof (lb_serdes);
3586 value += sizeof (lb_external);
3587
3588 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3589 *lbsp = value;
3590 break;
3591
3592 case LB_GET_INFO:
3593 value = sizeof (lb_normal);
3594 if (hw->phy.media_type == e1000_media_type_copper)
3595 value += sizeof (lb_phy);
3596 else
3597 value += sizeof (lb_serdes);
3598 value += sizeof (lb_external);
3599
3600 size = value;
3601 if (iocp->ioc_count != size)
3602 return (IOC_INVAL);
3603
3604 value = 0;
3605 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3606
3607 lbpp[value++] = lb_normal;
3608 if (hw->phy.media_type == e1000_media_type_copper)
3609 lbpp[value++] = lb_phy;
3610 else
3611 lbpp[value++] = lb_serdes;
3612 lbpp[value++] = lb_external;
3613 break;
3614
3615 case LB_GET_MODE:
3616 size = sizeof (uint32_t);
3617 if (iocp->ioc_count != size)
3618 return (IOC_INVAL);
3619
3620 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3621 *lbmp = igb->loopback_mode;
3622 break;
3623
3624 case LB_SET_MODE:
3625 size = 0;
3626 if (iocp->ioc_count != sizeof (uint32_t))
3627 return (IOC_INVAL);
3628
3629 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3630 if (!igb_set_loopback_mode(igb, *lbmp))
3631 return (IOC_INVAL);
3632 break;
3633 }
3634
3635 iocp->ioc_count = size;
3636 iocp->ioc_error = 0;
3637
3638 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3639 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3640 return (IOC_INVAL);
3641 }
3642
3643 return (IOC_REPLY);
3644 }
3645
3646 /*
3647 * igb_set_loopback_mode - Setup loopback based on the loopback mode
3648 */
3649 static boolean_t
3650 igb_set_loopback_mode(igb_t *igb, uint32_t mode)
3651 {
3652 struct e1000_hw *hw;
3653 int i;
3654
3655 if (mode == igb->loopback_mode)
3656 return (B_TRUE);
3657
3658 hw = &igb->hw;
3659
3660 igb->loopback_mode = mode;
3661
3662 if (mode == IGB_LB_NONE) {
3663 /* Reset the chip */
3664 hw->phy.autoneg_wait_to_complete = B_TRUE;
3665 (void) igb_reset(igb);
3666 hw->phy.autoneg_wait_to_complete = B_FALSE;
3667 return (B_TRUE);
3668 }
3669
3670 mutex_enter(&igb->gen_lock);
3671
3672 switch (mode) {
3673 default:
3674 mutex_exit(&igb->gen_lock);
3675 return (B_FALSE);
3676
3677 case IGB_LB_EXTERNAL:
3678 igb_set_external_loopback(igb);
3679 break;
3680
3681 case IGB_LB_INTERNAL_PHY:
3682 igb_set_internal_phy_loopback(igb);
3683 break;
3684
3685 case IGB_LB_INTERNAL_SERDES:
3686 igb_set_internal_serdes_loopback(igb);
3687 break;
3688 }
3689
3690 mutex_exit(&igb->gen_lock);
3691
3692 /*
3693 * When external loopback is set, wait up to 1000ms to get the link up.
3694 * According to test, 1000ms can work and it's an experimental value.
3695 */
3696 if (mode == IGB_LB_EXTERNAL) {
3697 for (i = 0; i <= 10; i++) {
3698 mutex_enter(&igb->gen_lock);
3699 (void) igb_link_check(igb);
3700 mutex_exit(&igb->gen_lock);
3701
3702 if (igb->link_state == LINK_STATE_UP)
3703 break;
3704
3705 msec_delay(100);
3706 }
3707
3708 if (igb->link_state != LINK_STATE_UP) {
3709 /*
3710 * Does not support external loopback.
3711 * Reset driver to loopback none.
3712 */
3713 igb->loopback_mode = IGB_LB_NONE;
3714
3715 /* Reset the chip */
3716 hw->phy.autoneg_wait_to_complete = B_TRUE;
3717 (void) igb_reset(igb);
3718 hw->phy.autoneg_wait_to_complete = B_FALSE;
3719
3720 IGB_DEBUGLOG_0(igb, "Set external loopback failed, "
3721 "reset to loopback none.");
3722
3723 return (B_FALSE);
3724 }
3725 }
3726
3727 return (B_TRUE);
3728 }
3729
3730 /*
3731 * igb_set_external_loopback - Set the external loopback mode
3732 */
3733 static void
3734 igb_set_external_loopback(igb_t *igb)
3735 {
3736 struct e1000_hw *hw;
3737 uint32_t ctrl_ext;
3738
3739 hw = &igb->hw;
3740
3741 /* Set link mode to PHY (00b) in the Extended Control register */
3742 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3743 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3744 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3745
3746 (void) e1000_write_phy_reg(hw, 0x0, 0x0140);
3747 (void) e1000_write_phy_reg(hw, 0x9, 0x1a00);
3748 (void) e1000_write_phy_reg(hw, 0x12, 0x1610);
3749 (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c);
3750 }
3751
3752 /*
3753 * igb_set_internal_phy_loopback - Set the internal PHY loopback mode
3754 */
3755 static void
3756 igb_set_internal_phy_loopback(igb_t *igb)
3757 {
3758 struct e1000_hw *hw;
3759 uint32_t ctrl_ext;
3760 uint16_t phy_ctrl;
3761 uint16_t phy_pconf;
3762
3763 hw = &igb->hw;
3764
3765 /* Set link mode to PHY (00b) in the Extended Control register */
3766 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3767 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3768 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3769
3770 /*
3771 * Set PHY control register (0x4140):
3772 * Set full duplex mode
3773 * Set loopback bit
3774 * Clear auto-neg enable bit
3775 * Set PHY speed
3776 */
3777 phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK;
3778 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl);
3779
3780 /* Set the link disable bit in the Port Configuration register */
3781 (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf);
3782 phy_pconf |= (uint16_t)1 << 14;
3783 (void) e1000_write_phy_reg(hw, 0x10, phy_pconf);
3784 }
3785
3786 /*
3787 * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode
3788 */
3789 static void
3790 igb_set_internal_serdes_loopback(igb_t *igb)
3791 {
3792 struct e1000_hw *hw;
3793 uint32_t ctrl_ext;
3794 uint32_t ctrl;
3795 uint32_t pcs_lctl;
3796 uint32_t connsw;
3797
3798 hw = &igb->hw;
3799
3800 /* Set link mode to SerDes (11b) in the Extended Control register */
3801 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3802 ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
3803 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3804
3805 /* Configure the SerDes to loopback */
3806 E1000_WRITE_REG(hw, E1000_SCTL, 0x410);
3807
3808 /* Set Device Control register */
3809 ctrl = E1000_READ_REG(hw, E1000_CTRL);
3810 ctrl |= (E1000_CTRL_FD | /* Force full duplex */
3811 E1000_CTRL_SLU); /* Force link up */
3812 ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */
3813 E1000_CTRL_TFCE | /* Disable transmit flow control */
3814 E1000_CTRL_LRST); /* Clear link reset */
3815 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
3816
3817 /* Set PCS Link Control register */
3818 pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL);
3819 pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK |
3820 E1000_PCS_LCTL_FSD |
3821 E1000_PCS_LCTL_FDV_FULL |
3822 E1000_PCS_LCTL_FLV_LINK_UP);
3823 pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE;
3824 E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl);
3825
3826 /* Set the Copper/Fiber Switch Control - CONNSW register */
3827 connsw = E1000_READ_REG(hw, E1000_CONNSW);
3828 connsw &= ~E1000_CONNSW_ENRGSRC;
3829 E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
3830 }
3831
3832 #pragma inline(igb_intr_rx_work)
3833 /*
3834 * igb_intr_rx_work - rx processing of ISR
3835 */
3836 static void
3837 igb_intr_rx_work(igb_rx_ring_t *rx_ring)
3838 {
3839 mblk_t *mp;
3840
3841 mutex_enter(&rx_ring->rx_lock);
3842 mp = igb_rx(rx_ring, IGB_NO_POLL);
3843 mutex_exit(&rx_ring->rx_lock);
3844
3845 if (mp != NULL)
3846 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp,
3847 rx_ring->ring_gen_num);
3848 }
3849
3850 #pragma inline(igb_intr_tx_work)
3851 /*
3852 * igb_intr_tx_work - tx processing of ISR
3853 */
3854 static void
3855 igb_intr_tx_work(igb_tx_ring_t *tx_ring)
3856 {
3857 igb_t *igb = tx_ring->igb;
3858
3859 /* Recycle the tx descriptors */
3860 tx_ring->tx_recycle(tx_ring);
3861
3862 /* Schedule the re-transmit */
3863 if (tx_ring->reschedule &&
3864 (tx_ring->tbd_free >= igb->tx_resched_thresh)) {
3865 tx_ring->reschedule = B_FALSE;
3866 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle);
3867 IGB_DEBUG_STAT(tx_ring->stat_reschedule);
3868 }
3869 }
3870
3871 #pragma inline(igb_intr_link_work)
3872 /*
3873 * igb_intr_link_work - link-status-change processing of ISR
3874 */
3875 static void
3876 igb_intr_link_work(igb_t *igb)
3877 {
3878 boolean_t link_changed;
3879
3880 igb_stop_watchdog_timer(igb);
3881
3882 mutex_enter(&igb->gen_lock);
3883
3884 /*
3885 * Because we got a link-status-change interrupt, force
3886 * e1000_check_for_link() to look at phy
3887 */
3888 igb->hw.mac.get_link_status = B_TRUE;
3889
3890 /* igb_link_check takes care of link status change */
3891 link_changed = igb_link_check(igb);
3892
3893 /* Get new phy state */
3894 igb_get_phy_state(igb);
3895
3896 mutex_exit(&igb->gen_lock);
3897
3898 if (link_changed)
3899 mac_link_update(igb->mac_hdl, igb->link_state);
3900
3901 igb_start_watchdog_timer(igb);
3902 }
3903
3904 /*
3905 * igb_intr_legacy - Interrupt handler for legacy interrupts
3906 */
3907 static uint_t
3908 igb_intr_legacy(void *arg1, void *arg2)
3909 {
3910 igb_t *igb = (igb_t *)arg1;
3911 igb_tx_ring_t *tx_ring;
3912 uint32_t icr;
3913 mblk_t *mp;
3914 boolean_t tx_reschedule;
3915 boolean_t link_changed;
3916 uint_t result;
3917
3918 _NOTE(ARGUNUSED(arg2));
3919
3920 mutex_enter(&igb->gen_lock);
3921
3922 if (igb->igb_state & IGB_SUSPENDED) {
3923 mutex_exit(&igb->gen_lock);
3924 return (DDI_INTR_UNCLAIMED);
3925 }
3926
3927 mp = NULL;
3928 tx_reschedule = B_FALSE;
3929 link_changed = B_FALSE;
3930 icr = E1000_READ_REG(&igb->hw, E1000_ICR);
3931
3932 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3933 mutex_exit(&igb->gen_lock);
3934 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3935 atomic_or_32(&igb->igb_state, IGB_ERROR);
3936 return (DDI_INTR_UNCLAIMED);
3937 }
3938
3939 if (icr & E1000_ICR_INT_ASSERTED) {
3940 /*
3941 * E1000_ICR_INT_ASSERTED bit was set:
3942 * Read(Clear) the ICR, claim this interrupt,
3943 * look for work to do.
3944 */
3945 ASSERT(igb->num_rx_rings == 1);
3946 ASSERT(igb->num_tx_rings == 1);
3947
3948 /* Make sure all interrupt causes cleared */
3949 (void) E1000_READ_REG(&igb->hw, E1000_EICR);
3950
3951 if (icr & E1000_ICR_RXT0) {
3952 mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL);
3953 }
3954
3955 if (icr & E1000_ICR_TXDW) {
3956 tx_ring = &igb->tx_rings[0];
3957
3958 /* Recycle the tx descriptors */
3959 tx_ring->tx_recycle(tx_ring);
3960
3961 /* Schedule the re-transmit */
3962 tx_reschedule = (tx_ring->reschedule &&
3963 (tx_ring->tbd_free >= igb->tx_resched_thresh));
3964 }
3965
3966 if (icr & E1000_ICR_LSC) {
3967 /*
3968 * Because we got a link-status-change interrupt, force
3969 * e1000_check_for_link() to look at phy
3970 */
3971 igb->hw.mac.get_link_status = B_TRUE;
3972
3973 /* igb_link_check takes care of link status change */
3974 link_changed = igb_link_check(igb);
3975
3976 /* Get new phy state */
3977 igb_get_phy_state(igb);
3978 }
3979
3980 if (icr & E1000_ICR_DRSTA) {
3981 /* 82580 Full Device Reset needed */
3982 atomic_or_32(&igb->igb_state, IGB_STALL);
3983 }
3984
3985 result = DDI_INTR_CLAIMED;
3986 } else {
3987 /*
3988 * E1000_ICR_INT_ASSERTED bit was not set:
3989 * Don't claim this interrupt.
3990 */
3991 result = DDI_INTR_UNCLAIMED;
3992 }
3993
3994 mutex_exit(&igb->gen_lock);
3995
3996 /*
3997 * Do the following work outside of the gen_lock
3998 */
3999 if (mp != NULL)
4000 mac_rx(igb->mac_hdl, NULL, mp);
4001
4002 if (tx_reschedule) {
4003 tx_ring->reschedule = B_FALSE;
4004 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle);
4005 IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4006 }
4007
4008 if (link_changed)
4009 mac_link_update(igb->mac_hdl, igb->link_state);
4010
4011 return (result);
4012 }
4013
4014 /*
4015 * igb_intr_msi - Interrupt handler for MSI
4016 */
4017 static uint_t
4018 igb_intr_msi(void *arg1, void *arg2)
4019 {
4020 igb_t *igb = (igb_t *)arg1;
4021 uint32_t icr;
4022
4023 _NOTE(ARGUNUSED(arg2));
4024
4025 icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4026
4027 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4028 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4029 atomic_or_32(&igb->igb_state, IGB_ERROR);
4030 return (DDI_INTR_CLAIMED);
4031 }
4032
4033 /* Make sure all interrupt causes cleared */
4034 (void) E1000_READ_REG(&igb->hw, E1000_EICR);
4035
4036 /*
4037 * For MSI interrupt, we have only one vector,
4038 * so we have only one rx ring and one tx ring enabled.
4039 */
4040 ASSERT(igb->num_rx_rings == 1);
4041 ASSERT(igb->num_tx_rings == 1);
4042
4043 if (icr & E1000_ICR_RXT0) {
4044 igb_intr_rx_work(&igb->rx_rings[0]);
4045 }
4046
4047 if (icr & E1000_ICR_TXDW) {
4048 igb_intr_tx_work(&igb->tx_rings[0]);
4049 }
4050
4051 if (icr & E1000_ICR_LSC) {
4052 igb_intr_link_work(igb);
4053 }
4054
4055 if (icr & E1000_ICR_DRSTA) {
4056 /* 82580 Full Device Reset needed */
4057 atomic_or_32(&igb->igb_state, IGB_STALL);
4058 }
4059
4060 return (DDI_INTR_CLAIMED);
4061 }
4062
4063 /*
4064 * igb_intr_rx - Interrupt handler for rx
4065 */
4066 static uint_t
4067 igb_intr_rx(void *arg1, void *arg2)
4068 {
4069 igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1;
4070
4071 _NOTE(ARGUNUSED(arg2));
4072
4073 /*
4074 * Only used via MSI-X vector so don't check cause bits
4075 * and only clean the given ring.
4076 */
4077 igb_intr_rx_work(rx_ring);
4078
4079 return (DDI_INTR_CLAIMED);
4080 }
4081
4082 /*
4083 * igb_intr_tx - Interrupt handler for tx
4084 */
4085 static uint_t
4086 igb_intr_tx(void *arg1, void *arg2)
4087 {
4088 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1;
4089
4090 _NOTE(ARGUNUSED(arg2));
4091
4092 /*
4093 * Only used via MSI-X vector so don't check cause bits
4094 * and only clean the given ring.
4095 */
4096 igb_intr_tx_work(tx_ring);
4097
4098 return (DDI_INTR_CLAIMED);
4099 }
4100
4101 /*
4102 * igb_intr_tx_other - Interrupt handler for both tx and other
4103 *
4104 */
4105 static uint_t
4106 igb_intr_tx_other(void *arg1, void *arg2)
4107 {
4108 igb_t *igb = (igb_t *)arg1;
4109 uint32_t icr;
4110
4111 _NOTE(ARGUNUSED(arg2));
4112
4113 icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4114
4115 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4116 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4117 atomic_or_32(&igb->igb_state, IGB_ERROR);
4118 return (DDI_INTR_CLAIMED);
4119 }
4120
4121 /*
4122 * Look for tx reclaiming work first. Remember, in the
4123 * case of only interrupt sharing, only one tx ring is
4124 * used
4125 */
4126 igb_intr_tx_work(&igb->tx_rings[0]);
4127
4128 /*
4129 * Check for "other" causes.
4130 */
4131 if (icr & E1000_ICR_LSC) {
4132 igb_intr_link_work(igb);
4133 }
4134
4135 /*
4136 * The DOUTSYNC bit indicates a tx packet dropped because
4137 * DMA engine gets "out of sync". There isn't a real fix
4138 * for this. The Intel recommendation is to count the number
4139 * of occurrences so user can detect when it is happening.
4140 * The issue is non-fatal and there's no recovery action
4141 * available.
4142 */
4143 if (icr & E1000_ICR_DOUTSYNC) {
4144 IGB_STAT(igb->dout_sync);
4145 }
4146
4147 if (icr & E1000_ICR_DRSTA) {
4148 /* 82580 Full Device Reset needed */
4149 atomic_or_32(&igb->igb_state, IGB_STALL);
4150 }
4151
4152 return (DDI_INTR_CLAIMED);
4153 }
4154
4155 /*
4156 * igb_alloc_intrs - Allocate interrupts for the driver
4157 *
4158 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4159 * if not successful, try Legacy.
4160 * igb->intr_force can be used to force sequence to start with
4161 * any of the 3 types.
4162 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4163 */
4164 static int
4165 igb_alloc_intrs(igb_t *igb)
4166 {
4167 dev_info_t *devinfo;
4168 int intr_types;
4169 int rc;
4170
4171 devinfo = igb->dip;
4172
4173 /* Get supported interrupt types */
4174 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4175
4176 if (rc != DDI_SUCCESS) {
4177 igb_log(igb,
4178 "Get supported interrupt types failed: %d", rc);
4179 return (IGB_FAILURE);
4180 }
4181 IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types);
4182
4183 igb->intr_type = 0;
4184
4185 /* Install MSI-X interrupts */
4186 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4187 (igb->intr_force <= IGB_INTR_MSIX)) {
4188 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX);
4189
4190 if (rc == IGB_SUCCESS)
4191 return (IGB_SUCCESS);
4192
4193 igb_log(igb,
4194 "Allocate MSI-X failed, trying MSI interrupts...");
4195 }
4196
4197 /* MSI-X not used, force rings to 1 */
4198 igb->num_rx_rings = 1;
4199 igb->num_tx_rings = 1;
4200 igb_log(igb,
4201 "MSI-X not used, force rx and tx queue number to 1");
4202
4203 /* Install MSI interrupts */
4204 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4205 (igb->intr_force <= IGB_INTR_MSI)) {
4206 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI);
4207
4208 if (rc == IGB_SUCCESS)
4209 return (IGB_SUCCESS);
4210
4211 igb_log(igb,
4212 "Allocate MSI failed, trying Legacy interrupts...");
4213 }
4214
4215 /* Install legacy interrupts */
4216 if (intr_types & DDI_INTR_TYPE_FIXED) {
4217 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED);
4218
4219 if (rc == IGB_SUCCESS)
4220 return (IGB_SUCCESS);
4221
4222 igb_log(igb,
4223 "Allocate Legacy interrupts failed");
4224 }
4225
4226 /* If none of the 3 types succeeded, return failure */
4227 return (IGB_FAILURE);
4228 }
4229
4230 /*
4231 * igb_alloc_intr_handles - Allocate interrupt handles.
4232 *
4233 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4234 * if fewer than 2 handles are available, return failure.
4235 * Upon success, this sets the number of Rx rings to a number that
4236 * matches the handles available for Rx interrupts.
4237 */
4238 static int
4239 igb_alloc_intr_handles(igb_t *igb, int intr_type)
4240 {
4241 dev_info_t *devinfo;
4242 int orig, request, count, avail, actual;
4243 int diff, minimum;
4244 int rc;
4245
4246 devinfo = igb->dip;
4247
4248 switch (intr_type) {
4249 case DDI_INTR_TYPE_FIXED:
4250 request = 1; /* Request 1 legacy interrupt handle */
4251 minimum = 1;
4252 IGB_DEBUGLOG_0(igb, "interrupt type: legacy");
4253 break;
4254
4255 case DDI_INTR_TYPE_MSI:
4256 request = 1; /* Request 1 MSI interrupt handle */
4257 minimum = 1;
4258 IGB_DEBUGLOG_0(igb, "interrupt type: MSI");
4259 break;
4260
4261 case DDI_INTR_TYPE_MSIX:
4262 /*
4263 * Number of vectors for the adapter is
4264 * # rx rings + # tx rings
4265 * One of tx vectors is for tx & other
4266 */
4267 request = igb->num_rx_rings + igb->num_tx_rings;
4268 orig = request;
4269 minimum = 2;
4270 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X");
4271 break;
4272
4273 default:
4274 igb_log(igb,
4275 "invalid call to igb_alloc_intr_handles(): %d\n",
4276 intr_type);
4277 return (IGB_FAILURE);
4278 }
4279 IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d minimum: %d",
4280 request, minimum);
4281
4282 /*
4283 * Get number of supported interrupts
4284 */
4285 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4286 if ((rc != DDI_SUCCESS) || (count < minimum)) {
4287 igb_log(igb,
4288 "Get supported interrupt number failed. "
4289 "Return: %d, count: %d", rc, count);
4290 return (IGB_FAILURE);
4291 }
4292 IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count);
4293
4294 /*
4295 * Get number of available interrupts
4296 */
4297 rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
4298 if ((rc != DDI_SUCCESS) || (avail < minimum)) {
4299 igb_log(igb,
4300 "Get available interrupt number failed. "
4301 "Return: %d, available: %d", rc, avail);
4302 return (IGB_FAILURE);
4303 }
4304 IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail);
4305
4306 if (avail < request) {
4307 igb_log(igb, "Request %d handles, %d available",
4308 request, avail);
4309 request = avail;
4310 }
4311
4312 actual = 0;
4313 igb->intr_cnt = 0;
4314
4315 /*
4316 * Allocate an array of interrupt handles
4317 */
4318 igb->intr_size = request * sizeof (ddi_intr_handle_t);
4319 igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP);
4320
4321 rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0,
4322 request, &actual, DDI_INTR_ALLOC_NORMAL);
4323 if (rc != DDI_SUCCESS) {
4324 igb_log(igb, "Allocate interrupts failed. "
4325 "return: %d, request: %d, actual: %d",
4326 rc, request, actual);
4327 goto alloc_handle_fail;
4328 }
4329 IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual);
4330
4331 igb->intr_cnt = actual;
4332
4333 if (actual < minimum) {
4334 igb_log(igb, "Insufficient interrupt handles allocated: %d",
4335 actual);
4336 goto alloc_handle_fail;
4337 }
4338
4339 /*
4340 * For MSI-X, actual might force us to reduce number of tx & rx rings
4341 */
4342 if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) {
4343 diff = orig - actual;
4344 if (diff < igb->num_tx_rings) {
4345 igb_log(igb,
4346 "MSI-X vectors force Tx queue number to %d",
4347 igb->num_tx_rings - diff);
4348 igb->num_tx_rings -= diff;
4349 } else {
4350 igb_log(igb,
4351 "MSI-X vectors force Tx queue number to 1");
4352 igb->num_tx_rings = 1;
4353
4354 igb_log(igb,
4355 "MSI-X vectors force Rx queue number to %d",
4356 actual - 1);
4357 igb->num_rx_rings = actual - 1;
4358 }
4359 }
4360
4361 /*
4362 * Get priority for first vector, assume remaining are all the same
4363 */
4364 rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri);
4365 if (rc != DDI_SUCCESS) {
4366 igb_log(igb,
4367 "Get interrupt priority failed: %d", rc);
4368 goto alloc_handle_fail;
4369 }
4370
4371 rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap);
4372 if (rc != DDI_SUCCESS) {
4373 igb_log(igb,
4374 "Get interrupt cap failed: %d", rc);
4375 goto alloc_handle_fail;
4376 }
4377
4378 igb->intr_type = intr_type;
4379
4380 return (IGB_SUCCESS);
4381
4382 alloc_handle_fail:
4383 igb_rem_intrs(igb);
4384
4385 return (IGB_FAILURE);
4386 }
4387
4388 /*
4389 * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type
4390 *
4391 * Before adding the interrupt handlers, the interrupt vectors have
4392 * been allocated, and the rx/tx rings have also been allocated.
4393 */
4394 static int
4395 igb_add_intr_handlers(igb_t *igb)
4396 {
4397 igb_rx_ring_t *rx_ring;
4398 igb_tx_ring_t *tx_ring;
4399 int vector;
4400 int rc;
4401 int i;
4402
4403 vector = 0;
4404
4405 switch (igb->intr_type) {
4406 case DDI_INTR_TYPE_MSIX:
4407 /* Add interrupt handler for tx + other */
4408 tx_ring = &igb->tx_rings[0];
4409 rc = ddi_intr_add_handler(igb->htable[vector],
4410 (ddi_intr_handler_t *)igb_intr_tx_other,
4411 (void *)igb, NULL);
4412
4413 if (rc != DDI_SUCCESS) {
4414 igb_log(igb,
4415 "Add tx/other interrupt handler failed: %d", rc);
4416 return (IGB_FAILURE);
4417 }
4418 tx_ring->intr_vector = vector;
4419 vector++;
4420
4421 /* Add interrupt handler for each rx ring */
4422 for (i = 0; i < igb->num_rx_rings; i++) {
4423 rx_ring = &igb->rx_rings[i];
4424
4425 rc = ddi_intr_add_handler(igb->htable[vector],
4426 (ddi_intr_handler_t *)igb_intr_rx,
4427 (void *)rx_ring, NULL);
4428
4429 if (rc != DDI_SUCCESS) {
4430 igb_log(igb,
4431 "Add rx interrupt handler failed. "
4432 "return: %d, rx ring: %d", rc, i);
4433 for (vector--; vector >= 0; vector--) {
4434 (void) ddi_intr_remove_handler(
4435 igb->htable[vector]);
4436 }
4437 return (IGB_FAILURE);
4438 }
4439
4440 rx_ring->intr_vector = vector;
4441
4442 vector++;
4443 }
4444
4445 /* Add interrupt handler for each tx ring from 2nd ring */
4446 for (i = 1; i < igb->num_tx_rings; i++) {
4447 tx_ring = &igb->tx_rings[i];
4448
4449 rc = ddi_intr_add_handler(igb->htable[vector],
4450 (ddi_intr_handler_t *)igb_intr_tx,
4451 (void *)tx_ring, NULL);
4452
4453 if (rc != DDI_SUCCESS) {
4454 igb_log(igb,
4455 "Add tx interrupt handler failed. "
4456 "return: %d, tx ring: %d", rc, i);
4457 for (vector--; vector >= 0; vector--) {
4458 (void) ddi_intr_remove_handler(
4459 igb->htable[vector]);
4460 }
4461 return (IGB_FAILURE);
4462 }
4463
4464 tx_ring->intr_vector = vector;
4465
4466 vector++;
4467 }
4468
4469 break;
4470
4471 case DDI_INTR_TYPE_MSI:
4472 /* Add interrupt handlers for the only vector */
4473 rc = ddi_intr_add_handler(igb->htable[vector],
4474 (ddi_intr_handler_t *)igb_intr_msi,
4475 (void *)igb, NULL);
4476
4477 if (rc != DDI_SUCCESS) {
4478 igb_log(igb,
4479 "Add MSI interrupt handler failed: %d", rc);
4480 return (IGB_FAILURE);
4481 }
4482
4483 rx_ring = &igb->rx_rings[0];
4484 rx_ring->intr_vector = vector;
4485
4486 vector++;
4487 break;
4488
4489 case DDI_INTR_TYPE_FIXED:
4490 /* Add interrupt handlers for the only vector */
4491 rc = ddi_intr_add_handler(igb->htable[vector],
4492 (ddi_intr_handler_t *)igb_intr_legacy,
4493 (void *)igb, NULL);
4494
4495 if (rc != DDI_SUCCESS) {
4496 igb_log(igb,
4497 "Add legacy interrupt handler failed: %d", rc);
4498 return (IGB_FAILURE);
4499 }
4500
4501 rx_ring = &igb->rx_rings[0];
4502 rx_ring->intr_vector = vector;
4503
4504 vector++;
4505 break;
4506
4507 default:
4508 return (IGB_FAILURE);
4509 }
4510
4511 ASSERT(vector == igb->intr_cnt);
4512
4513 return (IGB_SUCCESS);
4514 }
4515
4516 /*
4517 * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts
4518 *
4519 * For each vector enabled on the adapter, Set the MSIXBM register accordingly
4520 */
4521 static void
4522 igb_setup_msix_82575(igb_t *igb)
4523 {
4524 uint32_t eims = 0;
4525 int i, vector;
4526 struct e1000_hw *hw = &igb->hw;
4527
4528 /*
4529 * Set vector for tx ring 0 and other causes.
4530 * NOTE assumption that it is vector 0.
4531 */
4532 vector = 0;
4533
4534 igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER;
4535 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask);
4536 vector++;
4537
4538 for (i = 0; i < igb->num_rx_rings; i++) {
4539 /*
4540 * Set vector for each rx ring
4541 */
4542 eims = (E1000_EICR_RX_QUEUE0 << i);
4543 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4544
4545 /*
4546 * Accumulate bits to enable in
4547 * igb_enable_adapter_interrupts_82575()
4548 */
4549 igb->eims_mask |= eims;
4550
4551 vector++;
4552 }
4553
4554 for (i = 1; i < igb->num_tx_rings; i++) {
4555 /*
4556 * Set vector for each tx ring from 2nd tx ring
4557 */
4558 eims = (E1000_EICR_TX_QUEUE0 << i);
4559 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4560
4561 /*
4562 * Accumulate bits to enable in
4563 * igb_enable_adapter_interrupts_82575()
4564 */
4565 igb->eims_mask |= eims;
4566
4567 vector++;
4568 }
4569
4570 ASSERT(vector == igb->intr_cnt);
4571
4572 /*
4573 * Disable IAM for ICR interrupt bits
4574 */
4575 E1000_WRITE_REG(hw, E1000_IAM, 0);
4576 E1000_WRITE_FLUSH(hw);
4577 }
4578
4579 /*
4580 * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts
4581 *
4582 * 82576 uses a table based method for assigning vectors. Each queue has a
4583 * single entry in the table to which we write a vector number along with a
4584 * "valid" bit. The entry is a single byte in a 4-byte register. Vectors
4585 * take a different position in the 4-byte register depending on whether
4586 * they are numbered above or below 8.
4587 */
4588 static void
4589 igb_setup_msix_82576(igb_t *igb)
4590 {
4591 struct e1000_hw *hw = &igb->hw;
4592 uint32_t ivar, index, vector;
4593 int i;
4594
4595 /* must enable msi-x capability before IVAR settings */
4596 E1000_WRITE_REG(hw, E1000_GPIE,
4597 (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR));
4598
4599 /*
4600 * Set vector for tx ring 0 and other causes.
4601 * NOTE assumption that it is vector 0.
4602 * This is also interdependent with installation of interrupt service
4603 * routines in igb_add_intr_handlers().
4604 */
4605
4606 /* assign "other" causes to vector 0 */
4607 vector = 0;
4608 ivar = ((vector | E1000_IVAR_VALID) << 8);
4609 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4610
4611 /* assign tx ring 0 to vector 0 */
4612 ivar = ((vector | E1000_IVAR_VALID) << 8);
4613 E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4614
4615 /* prepare to enable tx & other interrupt causes */
4616 igb->eims_mask = (1 << vector);
4617
4618 vector ++;
4619 for (i = 0; i < igb->num_rx_rings; i++) {
4620 /*
4621 * Set vector for each rx ring
4622 */
4623 index = (i & 0x7);
4624 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4625
4626 if (i < 8) {
4627 /* vector goes into low byte of register */
4628 ivar = ivar & 0xFFFFFF00;
4629 ivar |= (vector | E1000_IVAR_VALID);
4630 } else {
4631 /* vector goes into third byte of register */
4632 ivar = ivar & 0xFF00FFFF;
4633 ivar |= ((vector | E1000_IVAR_VALID) << 16);
4634 }
4635 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4636
4637 /* Accumulate interrupt-cause bits to enable */
4638 igb->eims_mask |= (1 << vector);
4639
4640 vector ++;
4641 }
4642
4643 for (i = 1; i < igb->num_tx_rings; i++) {
4644 /*
4645 * Set vector for each tx ring from 2nd tx ring.
4646 * Note assumption that tx vectors numericall follow rx vectors.
4647 */
4648 index = (i & 0x7);
4649 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4650
4651 if (i < 8) {
4652 /* vector goes into second byte of register */
4653 ivar = ivar & 0xFFFF00FF;
4654 ivar |= ((vector | E1000_IVAR_VALID) << 8);
4655 } else {
4656 /* vector goes into fourth byte of register */
4657 ivar = ivar & 0x00FFFFFF;
4658 ivar |= (vector | E1000_IVAR_VALID) << 24;
4659 }
4660 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4661
4662 /* Accumulate interrupt-cause bits to enable */
4663 igb->eims_mask |= (1 << vector);
4664
4665 vector ++;
4666 }
4667
4668 ASSERT(vector == igb->intr_cnt);
4669 }
4670
4671 /*
4672 * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts
4673 *
4674 * 82580 uses same table approach at 82576 but has fewer entries. Each
4675 * queue has a single entry in the table to which we write a vector number
4676 * along with a "valid" bit. Vectors take a different position in the
4677 * register depending on * whether * they are numbered above or below 4.
4678 */
4679 static void
4680 igb_setup_msix_82580(igb_t *igb)
4681 {
4682 struct e1000_hw *hw = &igb->hw;
4683 uint32_t ivar, index, vector;
4684 int i;
4685
4686 /* must enable msi-x capability before IVAR settings */
4687 E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE |
4688 E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME));
4689 /*
4690 * Set vector for tx ring 0 and other causes.
4691 * NOTE assumption that it is vector 0.
4692 * This is also interdependent with installation of interrupt service
4693 * routines in igb_add_intr_handlers().
4694 */
4695
4696 /* assign "other" causes to vector 0 */
4697 vector = 0;
4698 ivar = ((vector | E1000_IVAR_VALID) << 8);
4699 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4700
4701 /* assign tx ring 0 to vector 0 */
4702 ivar = ((vector | E1000_IVAR_VALID) << 8);
4703 E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4704
4705 /* prepare to enable tx & other interrupt causes */
4706 igb->eims_mask = (1 << vector);
4707
4708 vector ++;
4709
4710 for (i = 0; i < igb->num_rx_rings; i++) {
4711 /*
4712 * Set vector for each rx ring
4713 */
4714 index = (i >> 1);
4715 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4716
4717 if (i & 1) {
4718 /* vector goes into third byte of register */
4719 ivar = ivar & 0xFF00FFFF;
4720 ivar |= ((vector | E1000_IVAR_VALID) << 16);
4721 } else {
4722 /* vector goes into low byte of register */
4723 ivar = ivar & 0xFFFFFF00;
4724 ivar |= (vector | E1000_IVAR_VALID);
4725 }
4726 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4727
4728 /* Accumulate interrupt-cause bits to enable */
4729 igb->eims_mask |= (1 << vector);
4730
4731 vector ++;
4732 }
4733
4734 for (i = 1; i < igb->num_tx_rings; i++) {
4735 /*
4736 * Set vector for each tx ring from 2nd tx ring.
4737 * Note assumption that tx vectors numericall follow rx vectors.
4738 */
4739 index = (i >> 1);
4740 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4741
4742 if (i & 1) {
4743 /* vector goes into high byte of register */
4744 ivar = ivar & 0x00FFFFFF;
4745 ivar |= ((vector | E1000_IVAR_VALID) << 24);
4746 } else {
4747 /* vector goes into second byte of register */
4748 ivar = ivar & 0xFFFF00FF;
4749 ivar |= (vector | E1000_IVAR_VALID) << 8;
4750 }
4751 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4752
4753 /* Accumulate interrupt-cause bits to enable */
4754 igb->eims_mask |= (1 << vector);
4755
4756 vector ++;
4757 }
4758 ASSERT(vector == igb->intr_cnt);
4759 }
4760
4761 /*
4762 * igb_rem_intr_handlers - remove the interrupt handlers
4763 */
4764 static void
4765 igb_rem_intr_handlers(igb_t *igb)
4766 {
4767 int i;
4768 int rc;
4769
4770 for (i = 0; i < igb->intr_cnt; i++) {
4771 rc = ddi_intr_remove_handler(igb->htable[i]);
4772 if (rc != DDI_SUCCESS) {
4773 IGB_DEBUGLOG_1(igb,
4774 "Remove intr handler failed: %d", rc);
4775 }
4776 }
4777 }
4778
4779 /*
4780 * igb_rem_intrs - remove the allocated interrupts
4781 */
4782 static void
4783 igb_rem_intrs(igb_t *igb)
4784 {
4785 int i;
4786 int rc;
4787
4788 for (i = 0; i < igb->intr_cnt; i++) {
4789 rc = ddi_intr_free(igb->htable[i]);
4790 if (rc != DDI_SUCCESS) {
4791 IGB_DEBUGLOG_1(igb,
4792 "Free intr failed: %d", rc);
4793 }
4794 }
4795
4796 kmem_free(igb->htable, igb->intr_size);
4797 igb->htable = NULL;
4798 }
4799
4800 /*
4801 * igb_enable_intrs - enable all the ddi interrupts
4802 */
4803 static int
4804 igb_enable_intrs(igb_t *igb)
4805 {
4806 int i;
4807 int rc;
4808
4809 /* Enable interrupts */
4810 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
4811 /* Call ddi_intr_block_enable() for MSI */
4812 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt);
4813 if (rc != DDI_SUCCESS) {
4814 igb_log(igb,
4815 "Enable block intr failed: %d", rc);
4816 return (IGB_FAILURE);
4817 }
4818 } else {
4819 /* Call ddi_intr_enable() for Legacy/MSI non block enable */
4820 for (i = 0; i < igb->intr_cnt; i++) {
4821 rc = ddi_intr_enable(igb->htable[i]);
4822 if (rc != DDI_SUCCESS) {
4823 igb_log(igb,
4824 "Enable intr failed: %d", rc);
4825 return (IGB_FAILURE);
4826 }
4827 }
4828 }
4829
4830 return (IGB_SUCCESS);
4831 }
4832
4833 /*
4834 * igb_disable_intrs - disable all the ddi interrupts
4835 */
4836 static int
4837 igb_disable_intrs(igb_t *igb)
4838 {
4839 int i;
4840 int rc;
4841
4842 /* Disable all interrupts */
4843 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
4844 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt);
4845 if (rc != DDI_SUCCESS) {
4846 igb_log(igb,
4847 "Disable block intr failed: %d", rc);
4848 return (IGB_FAILURE);
4849 }
4850 } else {
4851 for (i = 0; i < igb->intr_cnt; i++) {
4852 rc = ddi_intr_disable(igb->htable[i]);
4853 if (rc != DDI_SUCCESS) {
4854 igb_log(igb,
4855 "Disable intr failed: %d", rc);
4856 return (IGB_FAILURE);
4857 }
4858 }
4859 }
4860
4861 return (IGB_SUCCESS);
4862 }
4863
4864 /*
4865 * igb_get_phy_state - Get and save the parameters read from PHY registers
4866 */
4867 static void
4868 igb_get_phy_state(igb_t *igb)
4869 {
4870 struct e1000_hw *hw = &igb->hw;
4871 uint16_t phy_ctrl;
4872 uint16_t phy_status;
4873 uint16_t phy_an_adv;
4874 uint16_t phy_an_exp;
4875 uint16_t phy_ext_status;
4876 uint16_t phy_1000t_ctrl;
4877 uint16_t phy_1000t_status;
4878 uint16_t phy_lp_able;
4879
4880 ASSERT(mutex_owned(&igb->gen_lock));
4881
4882 if (hw->phy.media_type == e1000_media_type_copper) {
4883 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl);
4884 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status);
4885 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv);
4886 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp);
4887 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status);
4888 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl);
4889 (void) e1000_read_phy_reg(hw,
4890 PHY_1000T_STATUS, &phy_1000t_status);
4891 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able);
4892
4893 igb->param_autoneg_cap =
4894 (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0;
4895 igb->param_pause_cap =
4896 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
4897 igb->param_asym_pause_cap =
4898 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
4899 igb->param_1000fdx_cap =
4900 ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) ||
4901 (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0;
4902 igb->param_1000hdx_cap =
4903 ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) ||
4904 (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0;
4905 igb->param_100t4_cap =
4906 (phy_status & MII_SR_100T4_CAPS) ? 1 : 0;
4907 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) ||
4908 (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0;
4909 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) ||
4910 (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0;
4911 igb->param_10fdx_cap =
4912 (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0;
4913 igb->param_10hdx_cap =
4914 (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0;
4915 igb->param_rem_fault =
4916 (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0;
4917
4918 igb->param_adv_autoneg_cap = hw->mac.autoneg;
4919 igb->param_adv_pause_cap =
4920 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
4921 igb->param_adv_asym_pause_cap =
4922 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
4923 igb->param_adv_1000hdx_cap =
4924 (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0;
4925 igb->param_adv_100t4_cap =
4926 (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0;
4927 igb->param_adv_rem_fault =
4928 (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0;
4929 if (igb->param_adv_autoneg_cap == 1) {
4930 igb->param_adv_1000fdx_cap =
4931 (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0;
4932 igb->param_adv_100fdx_cap =
4933 (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0;
4934 igb->param_adv_100hdx_cap =
4935 (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0;
4936 igb->param_adv_10fdx_cap =
4937 (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0;
4938 igb->param_adv_10hdx_cap =
4939 (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0;
4940 }
4941
4942 igb->param_lp_autoneg_cap =
4943 (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0;
4944 igb->param_lp_pause_cap =
4945 (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0;
4946 igb->param_lp_asym_pause_cap =
4947 (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0;
4948 igb->param_lp_1000fdx_cap =
4949 (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0;
4950 igb->param_lp_1000hdx_cap =
4951 (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0;
4952 igb->param_lp_100t4_cap =
4953 (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0;
4954 igb->param_lp_100fdx_cap =
4955 (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0;
4956 igb->param_lp_100hdx_cap =
4957 (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0;
4958 igb->param_lp_10fdx_cap =
4959 (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0;
4960 igb->param_lp_10hdx_cap =
4961 (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0;
4962 igb->param_lp_rem_fault =
4963 (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0;
4964 } else {
4965 /*
4966 * 1Gig Fiber adapter only offers 1Gig Full Duplex.
4967 */
4968 igb->param_autoneg_cap = 0;
4969 igb->param_pause_cap = 1;
4970 igb->param_asym_pause_cap = 1;
4971 igb->param_1000fdx_cap = 1;
4972 igb->param_1000hdx_cap = 0;
4973 igb->param_100t4_cap = 0;
4974 igb->param_100fdx_cap = 0;
4975 igb->param_100hdx_cap = 0;
4976 igb->param_10fdx_cap = 0;
4977 igb->param_10hdx_cap = 0;
4978
4979 igb->param_adv_autoneg_cap = 0;
4980 igb->param_adv_pause_cap = 1;
4981 igb->param_adv_asym_pause_cap = 1;
4982 igb->param_adv_1000fdx_cap = 1;
4983 igb->param_adv_1000hdx_cap = 0;
4984 igb->param_adv_100t4_cap = 0;
4985 igb->param_adv_100fdx_cap = 0;
4986 igb->param_adv_100hdx_cap = 0;
4987 igb->param_adv_10fdx_cap = 0;
4988 igb->param_adv_10hdx_cap = 0;
4989
4990 igb->param_lp_autoneg_cap = 0;
4991 igb->param_lp_pause_cap = 0;
4992 igb->param_lp_asym_pause_cap = 0;
4993 igb->param_lp_1000fdx_cap = 0;
4994 igb->param_lp_1000hdx_cap = 0;
4995 igb->param_lp_100t4_cap = 0;
4996 igb->param_lp_100fdx_cap = 0;
4997 igb->param_lp_100hdx_cap = 0;
4998 igb->param_lp_10fdx_cap = 0;
4999 igb->param_lp_10hdx_cap = 0;
5000 igb->param_lp_rem_fault = 0;
5001 }
5002 }
5003
5004 /*
5005 * synchronize the adv* and en* parameters.
5006 *
5007 * See comments in <sys/dld.h> for details of the *_en_*
5008 * parameters. The usage of ndd for setting adv parameters will
5009 * synchronize all the en parameters with the e1000g parameters,
5010 * implicitly disabling any settings made via dladm.
5011 */
5012 static void
5013 igb_param_sync(igb_t *igb)
5014 {
5015 igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap;
5016 igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap;
5017 igb->param_en_100t4_cap = igb->param_adv_100t4_cap;
5018 igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap;
5019 igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap;
5020 igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap;
5021 igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap;
5022 }
5023
5024 /*
5025 * igb_get_driver_control
5026 */
5027 static void
5028 igb_get_driver_control(struct e1000_hw *hw)
5029 {
5030 uint32_t ctrl_ext;
5031
5032 /* Notify firmware that driver is in control of device */
5033 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5034 ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD;
5035 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5036 }
5037
5038 /*
5039 * igb_release_driver_control
5040 */
5041 static void
5042 igb_release_driver_control(struct e1000_hw *hw)
5043 {
5044 uint32_t ctrl_ext;
5045
5046 /* Notify firmware that driver is no longer in control of device */
5047 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5048 ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD;
5049 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5050 }
5051
5052 /*
5053 * igb_atomic_reserve - Atomic decrease operation
5054 */
5055 int
5056 igb_atomic_reserve(uint32_t *count_p, uint32_t n)
5057 {
5058 uint32_t oldval;
5059 uint32_t newval;
5060
5061 /* ATOMICALLY */
5062 do {
5063 oldval = *count_p;
5064 if (oldval < n)
5065 return (-1);
5066 newval = oldval - n;
5067 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5068
5069 return (newval);
5070 }
5071
5072 /*
5073 * FMA support
5074 */
5075
5076 int
5077 igb_check_acc_handle(ddi_acc_handle_t handle)
5078 {
5079 ddi_fm_error_t de;
5080
5081 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5082 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5083 return (de.fme_status);
5084 }
5085
5086 int
5087 igb_check_dma_handle(ddi_dma_handle_t handle)
5088 {
5089 ddi_fm_error_t de;
5090
5091 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5092 return (de.fme_status);
5093 }
5094
5095 /*
5096 * The IO fault service error handling callback function
5097 */
5098 /*ARGSUSED*/
5099 static int
5100 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5101 {
5102 /*
5103 * as the driver can always deal with an error in any dma or
5104 * access handle, we can just return the fme_status value.
5105 */
5106 pci_ereport_post(dip, err, NULL);
5107 return (err->fme_status);
5108 }
5109
5110 static void
5111 igb_fm_init(igb_t *igb)
5112 {
5113 ddi_iblock_cookie_t iblk;
5114 int fma_dma_flag;
5115
5116 /* Only register with IO Fault Services if we have some capability */
5117 if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5118 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5119 } else {
5120 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5121 }
5122
5123 if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5124 fma_dma_flag = 1;
5125 } else {
5126 fma_dma_flag = 0;
5127 }
5128
5129 (void) igb_set_fma_flags(fma_dma_flag);
5130
5131 if (igb->fm_capabilities) {
5132
5133 /* Register capabilities with IO Fault Services */
5134 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk);
5135
5136 /*
5137 * Initialize pci ereport capabilities if ereport capable
5138 */
5139 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5140 DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5141 pci_ereport_setup(igb->dip);
5142
5143 /*
5144 * Register error callback if error callback capable
5145 */
5146 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5147 ddi_fm_handler_register(igb->dip,
5148 igb_fm_error_cb, (void*) igb);
5149 }
5150 }
5151
5152 static void
5153 igb_fm_fini(igb_t *igb)
5154 {
5155 /* Only unregister FMA capabilities if we registered some */
5156 if (igb->fm_capabilities) {
5157
5158 /*
5159 * Release any resources allocated by pci_ereport_setup()
5160 */
5161 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5162 DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5163 pci_ereport_teardown(igb->dip);
5164
5165 /*
5166 * Un-register error callback if error callback capable
5167 */
5168 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5169 ddi_fm_handler_unregister(igb->dip);
5170
5171 /* Unregister from IO Fault Services */
5172 ddi_fm_fini(igb->dip);
5173 }
5174 }
5175
5176 void
5177 igb_fm_ereport(igb_t *igb, char *detail)
5178 {
5179 uint64_t ena;
5180 char buf[FM_MAX_CLASS];
5181
5182 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5183 ena = fm_ena_generate(0, FM_ENA_FMT1);
5184 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) {
5185 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP,
5186 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5187 }
5188 }