1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 */
33
34 #include "ixgbe_sw.h"
35
36 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
37 static char ixgbe_version[] = "ixgbe 1.1.7";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static void ixgbe_setup_rings(ixgbe_t *);
61 static void ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_init_unicst(ixgbe_t *);
69 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
70 static void ixgbe_setup_multicst(ixgbe_t *);
71 static void ixgbe_get_hw_state(ixgbe_t *);
72 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
73 static void ixgbe_get_conf(ixgbe_t *);
74 static void ixgbe_init_params(ixgbe_t *);
75 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
76 static void ixgbe_driver_link_check(ixgbe_t *);
77 static void ixgbe_sfp_check(void *);
78 static void ixgbe_overtemp_check(void *);
79 static void ixgbe_link_timer(void *);
80 static void ixgbe_local_timer(void *);
81 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
82 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
83 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
84 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
85 static boolean_t is_valid_mac_addr(uint8_t *);
86 static boolean_t ixgbe_stall_check(ixgbe_t *);
87 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
88 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
89 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
90 static int ixgbe_alloc_intrs(ixgbe_t *);
91 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
92 static int ixgbe_add_intr_handlers(ixgbe_t *);
93 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
94 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
95 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
96 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
97 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
98 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
99 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
100 static void ixgbe_setup_adapter_vector(ixgbe_t *);
101 static void ixgbe_rem_intr_handlers(ixgbe_t *);
102 static void ixgbe_rem_intrs(ixgbe_t *);
103 static int ixgbe_enable_intrs(ixgbe_t *);
104 static int ixgbe_disable_intrs(ixgbe_t *);
105 static uint_t ixgbe_intr_legacy(void *, void *);
106 static uint_t ixgbe_intr_msi(void *, void *);
107 static uint_t ixgbe_intr_msix(void *, void *);
108 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
109 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
110 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
111 static void ixgbe_get_driver_control(struct ixgbe_hw *);
112 static int ixgbe_addmac(void *, const uint8_t *);
113 static int ixgbe_remmac(void *, const uint8_t *);
114 static void ixgbe_release_driver_control(struct ixgbe_hw *);
115
116 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
117 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
118 static int ixgbe_resume(dev_info_t *);
119 static int ixgbe_suspend(dev_info_t *);
120 static int ixgbe_quiesce(dev_info_t *);
121 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
122 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
123 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
124 static int ixgbe_intr_cb_register(ixgbe_t *);
125 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
126
127 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
128 const void *impl_data);
129 static void ixgbe_fm_init(ixgbe_t *);
130 static void ixgbe_fm_fini(ixgbe_t *);
131
132 char *ixgbe_priv_props[] = {
133 "_tx_copy_thresh",
134 "_tx_recycle_thresh",
135 "_tx_overload_thresh",
136 "_tx_resched_thresh",
137 "_rx_copy_thresh",
138 "_rx_limit_per_intr",
139 "_intr_throttling",
140 "_adv_pause_cap",
141 "_adv_asym_pause_cap",
142 NULL
143 };
144
145 #define IXGBE_MAX_PRIV_PROPS \
146 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
147
148 static struct cb_ops ixgbe_cb_ops = {
149 nulldev, /* cb_open */
150 nulldev, /* cb_close */
151 nodev, /* cb_strategy */
152 nodev, /* cb_print */
153 nodev, /* cb_dump */
154 nodev, /* cb_read */
155 nodev, /* cb_write */
156 nodev, /* cb_ioctl */
157 nodev, /* cb_devmap */
158 nodev, /* cb_mmap */
159 nodev, /* cb_segmap */
160 nochpoll, /* cb_chpoll */
161 ddi_prop_op, /* cb_prop_op */
162 NULL, /* cb_stream */
163 D_MP | D_HOTPLUG, /* cb_flag */
164 CB_REV, /* cb_rev */
165 nodev, /* cb_aread */
166 nodev /* cb_awrite */
167 };
168
169 static struct dev_ops ixgbe_dev_ops = {
170 DEVO_REV, /* devo_rev */
171 0, /* devo_refcnt */
172 NULL, /* devo_getinfo */
173 nulldev, /* devo_identify */
174 nulldev, /* devo_probe */
175 ixgbe_attach, /* devo_attach */
176 ixgbe_detach, /* devo_detach */
177 nodev, /* devo_reset */
178 &ixgbe_cb_ops, /* devo_cb_ops */
179 NULL, /* devo_bus_ops */
180 ddi_power, /* devo_power */
181 ixgbe_quiesce, /* devo_quiesce */
182 };
183
184 static struct modldrv ixgbe_modldrv = {
185 &mod_driverops, /* Type of module. This one is a driver */
186 ixgbe_ident, /* Discription string */
187 &ixgbe_dev_ops /* driver ops */
188 };
189
190 static struct modlinkage ixgbe_modlinkage = {
191 MODREV_1, &ixgbe_modldrv, NULL
192 };
193
194 /*
195 * Access attributes for register mapping
196 */
197 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
198 DDI_DEVICE_ATTR_V1,
199 DDI_STRUCTURE_LE_ACC,
200 DDI_STRICTORDER_ACC,
201 DDI_FLAGERR_ACC
202 };
203
204 /*
205 * Loopback property
206 */
207 static lb_property_t lb_normal = {
208 normal, "normal", IXGBE_LB_NONE
209 };
210
211 static lb_property_t lb_mac = {
212 internal, "MAC", IXGBE_LB_INTERNAL_MAC
213 };
214
215 static lb_property_t lb_external = {
216 external, "External", IXGBE_LB_EXTERNAL
217 };
218
219 #define IXGBE_M_CALLBACK_FLAGS \
220 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
221
222 static mac_callbacks_t ixgbe_m_callbacks = {
223 IXGBE_M_CALLBACK_FLAGS,
224 ixgbe_m_stat,
225 ixgbe_m_start,
226 ixgbe_m_stop,
227 ixgbe_m_promisc,
228 ixgbe_m_multicst,
229 NULL,
230 NULL,
231 NULL,
232 ixgbe_m_ioctl,
233 ixgbe_m_getcapab,
234 NULL,
235 NULL,
236 ixgbe_m_setprop,
237 ixgbe_m_getprop,
238 ixgbe_m_propinfo
239 };
240
241 /*
242 * Initialize capabilities of each supported adapter type
243 */
244 static adapter_info_t ixgbe_82598eb_cap = {
245 64, /* maximum number of rx queues */
246 1, /* minimum number of rx queues */
247 64, /* default number of rx queues */
248 16, /* maximum number of rx groups */
249 1, /* minimum number of rx groups */
250 1, /* default number of rx groups */
251 32, /* maximum number of tx queues */
252 1, /* minimum number of tx queues */
253 8, /* default number of tx queues */
254 16366, /* maximum MTU size */
255 0xFFFF, /* maximum interrupt throttle rate */
256 0, /* minimum interrupt throttle rate */
257 200, /* default interrupt throttle rate */
258 18, /* maximum total msix vectors */
259 16, /* maximum number of ring vectors */
260 2, /* maximum number of other vectors */
261 IXGBE_EICR_LSC, /* "other" interrupt types handled */
262 0, /* "other" interrupt types enable mask */
263 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
264 | IXGBE_FLAG_RSS_CAPABLE
265 | IXGBE_FLAG_VMDQ_CAPABLE)
266 };
267
268 static adapter_info_t ixgbe_82599eb_cap = {
269 128, /* maximum number of rx queues */
270 1, /* minimum number of rx queues */
271 128, /* default number of rx queues */
272 64, /* maximum number of rx groups */
273 1, /* minimum number of rx groups */
274 1, /* default number of rx groups */
275 128, /* maximum number of tx queues */
276 1, /* minimum number of tx queues */
277 8, /* default number of tx queues */
278 15500, /* maximum MTU size */
279 0xFF8, /* maximum interrupt throttle rate */
280 0, /* minimum interrupt throttle rate */
281 200, /* default interrupt throttle rate */
282 64, /* maximum total msix vectors */
283 16, /* maximum number of ring vectors */
284 2, /* maximum number of other vectors */
285 (IXGBE_EICR_LSC
286 | IXGBE_EICR_GPI_SDP1
287 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
288
289 (IXGBE_SDP1_GPIEN
290 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
291
292 (IXGBE_FLAG_DCA_CAPABLE
293 | IXGBE_FLAG_RSS_CAPABLE
294 | IXGBE_FLAG_VMDQ_CAPABLE
295 | IXGBE_FLAG_RSC_CAPABLE
296 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
297 };
298
299 static adapter_info_t ixgbe_X540_cap = {
300 128, /* maximum number of rx queues */
301 1, /* minimum number of rx queues */
302 128, /* default number of rx queues */
303 64, /* maximum number of rx groups */
304 1, /* minimum number of rx groups */
305 1, /* default number of rx groups */
306 128, /* maximum number of tx queues */
307 1, /* minimum number of tx queues */
308 8, /* default number of tx queues */
309 15500, /* maximum MTU size */
310 0xFF8, /* maximum interrupt throttle rate */
311 0, /* minimum interrupt throttle rate */
312 200, /* default interrupt throttle rate */
313 64, /* maximum total msix vectors */
314 16, /* maximum number of ring vectors */
315 2, /* maximum number of other vectors */
316 (IXGBE_EICR_LSC
317 | IXGBE_EICR_GPI_SDP1
318 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
319
320 (IXGBE_SDP1_GPIEN
321 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
322
323 (IXGBE_FLAG_DCA_CAPABLE
324 | IXGBE_FLAG_RSS_CAPABLE
325 | IXGBE_FLAG_VMDQ_CAPABLE
326 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
327 };
328
329 /*
330 * Module Initialization Functions.
331 */
332
333 int
334 _init(void)
335 {
336 int status;
337
338 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
339
340 status = mod_install(&ixgbe_modlinkage);
341
342 if (status != DDI_SUCCESS) {
343 mac_fini_ops(&ixgbe_dev_ops);
344 }
345
346 return (status);
347 }
348
349 int
350 _fini(void)
351 {
352 int status;
353
354 status = mod_remove(&ixgbe_modlinkage);
355
356 if (status == DDI_SUCCESS) {
357 mac_fini_ops(&ixgbe_dev_ops);
358 }
359
360 return (status);
361 }
362
363 int
364 _info(struct modinfo *modinfop)
365 {
366 int status;
367
368 status = mod_info(&ixgbe_modlinkage, modinfop);
369
370 return (status);
371 }
372
373 /*
374 * ixgbe_attach - Driver attach.
375 *
376 * This function is the device specific initialization entry
377 * point. This entry point is required and must be written.
378 * The DDI_ATTACH command must be provided in the attach entry
379 * point. When attach() is called with cmd set to DDI_ATTACH,
380 * all normal kernel services (such as kmem_alloc(9F)) are
381 * available for use by the driver.
382 *
383 * The attach() function will be called once for each instance
384 * of the device on the system with cmd set to DDI_ATTACH.
385 * Until attach() succeeds, the only driver entry points which
386 * may be called are open(9E) and getinfo(9E).
387 */
388 static int
389 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
390 {
391 ixgbe_t *ixgbe;
392 struct ixgbe_osdep *osdep;
393 struct ixgbe_hw *hw;
394 int instance;
395 char taskqname[32];
396
397 /*
398 * Check the command and perform corresponding operations
399 */
400 switch (cmd) {
401 default:
402 return (DDI_FAILURE);
403
404 case DDI_RESUME:
405 return (ixgbe_resume(devinfo));
406
407 case DDI_ATTACH:
408 break;
409 }
410
411 /* Get the device instance */
412 instance = ddi_get_instance(devinfo);
413
414 /* Allocate memory for the instance data structure */
415 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
416
417 ixgbe->dip = devinfo;
418 ixgbe->instance = instance;
419
420 hw = &ixgbe->hw;
421 osdep = &ixgbe->osdep;
422 hw->back = osdep;
423 osdep->ixgbe = ixgbe;
424
425 /* Attach the instance pointer to the dev_info data structure */
426 ddi_set_driver_private(devinfo, ixgbe);
427
428 /*
429 * Initialize for fma support
430 */
431 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
432 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
433 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
434 ixgbe_fm_init(ixgbe);
435 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
436
437 /*
438 * Map PCI config space registers
439 */
440 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
441 ixgbe_error(ixgbe, "Failed to map PCI configurations");
442 goto attach_fail;
443 }
444 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
445
446 /*
447 * Identify the chipset family
448 */
449 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
450 ixgbe_error(ixgbe, "Failed to identify hardware");
451 goto attach_fail;
452 }
453
454 /*
455 * Map device registers
456 */
457 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
458 ixgbe_error(ixgbe, "Failed to map device registers");
459 goto attach_fail;
460 }
461 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
462
463 /*
464 * Initialize driver parameters
465 */
466 ixgbe_init_properties(ixgbe);
467 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
468
469 /*
470 * Register interrupt callback
471 */
472 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
473 ixgbe_error(ixgbe, "Failed to register interrupt callback");
474 goto attach_fail;
475 }
476
477 /*
478 * Allocate interrupts
479 */
480 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
481 ixgbe_error(ixgbe, "Failed to allocate interrupts");
482 goto attach_fail;
483 }
484 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
485
486 /*
487 * Allocate rx/tx rings based on the ring numbers.
488 * The actual numbers of rx/tx rings are decided by the number of
489 * allocated interrupt vectors, so we should allocate the rings after
490 * interrupts are allocated.
491 */
492 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
493 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
494 goto attach_fail;
495 }
496 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
497
498 /*
499 * Map rings to interrupt vectors
500 */
501 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
502 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
503 goto attach_fail;
504 }
505
506 /*
507 * Add interrupt handlers
508 */
509 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
510 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
511 goto attach_fail;
512 }
513 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
514
515 /*
516 * Create a taskq for sfp-change
517 */
518 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
519 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
520 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
521 ixgbe_error(ixgbe, "sfp_taskq create failed");
522 goto attach_fail;
523 }
524 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
525
526 /*
527 * Create a taskq for over-temp
528 */
529 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
530 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
531 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
532 ixgbe_error(ixgbe, "overtemp_taskq create failed");
533 goto attach_fail;
534 }
535 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
536
537 /*
538 * Initialize driver parameters
539 */
540 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
541 ixgbe_error(ixgbe, "Failed to initialize driver settings");
542 goto attach_fail;
543 }
544
545 /*
546 * Initialize mutexes for this device.
547 * Do this before enabling the interrupt handler and
548 * register the softint to avoid the condition where
549 * interrupt handler can try using uninitialized mutex.
550 */
551 ixgbe_init_locks(ixgbe);
552 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
553
554 /*
555 * Initialize chipset hardware
556 */
557 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
558 ixgbe_error(ixgbe, "Failed to initialize adapter");
559 goto attach_fail;
560 }
561 ixgbe->link_check_complete = B_FALSE;
562 ixgbe->link_check_hrtime = gethrtime() +
563 (IXGBE_LINK_UP_TIME * 100000000ULL);
564 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
565
566 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
567 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
568 goto attach_fail;
569 }
570
571 /*
572 * Initialize statistics
573 */
574 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
575 ixgbe_error(ixgbe, "Failed to initialize statistics");
576 goto attach_fail;
577 }
578 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
579
580 /*
581 * Register the driver to the MAC
582 */
583 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
584 ixgbe_error(ixgbe, "Failed to register MAC");
585 goto attach_fail;
586 }
587 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
588 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
589
590 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
591 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
592 if (ixgbe->periodic_id == 0) {
593 ixgbe_error(ixgbe, "Failed to add the link check timer");
594 goto attach_fail;
595 }
596 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
597
598 /*
599 * Now that mutex locks are initialized, and the chip is also
600 * initialized, enable interrupts.
601 */
602 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
603 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
604 goto attach_fail;
605 }
606 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
607
608 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version);
609 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
610
611 return (DDI_SUCCESS);
612
613 attach_fail:
614 ixgbe_unconfigure(devinfo, ixgbe);
615 return (DDI_FAILURE);
616 }
617
618 /*
619 * ixgbe_detach - Driver detach.
620 *
621 * The detach() function is the complement of the attach routine.
622 * If cmd is set to DDI_DETACH, detach() is used to remove the
623 * state associated with a given instance of a device node
624 * prior to the removal of that instance from the system.
625 *
626 * The detach() function will be called once for each instance
627 * of the device for which there has been a successful attach()
628 * once there are no longer any opens on the device.
629 *
630 * Interrupts routine are disabled, All memory allocated by this
631 * driver are freed.
632 */
633 static int
634 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
635 {
636 ixgbe_t *ixgbe;
637
638 /*
639 * Check detach command
640 */
641 switch (cmd) {
642 default:
643 return (DDI_FAILURE);
644
645 case DDI_SUSPEND:
646 return (ixgbe_suspend(devinfo));
647
648 case DDI_DETACH:
649 break;
650 }
651
652 /*
653 * Get the pointer to the driver private data structure
654 */
655 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
656 if (ixgbe == NULL)
657 return (DDI_FAILURE);
658
659 /*
660 * If the device is still running, it needs to be stopped first.
661 * This check is necessary because under some specific circumstances,
662 * the detach routine can be called without stopping the interface
663 * first.
664 */
665 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
666 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
667 mutex_enter(&ixgbe->gen_lock);
668 ixgbe_stop(ixgbe, B_TRUE);
669 mutex_exit(&ixgbe->gen_lock);
670 /* Disable and stop the watchdog timer */
671 ixgbe_disable_watchdog_timer(ixgbe);
672 }
673
674 /*
675 * Check if there are still rx buffers held by the upper layer.
676 * If so, fail the detach.
677 */
678 if (!ixgbe_rx_drain(ixgbe))
679 return (DDI_FAILURE);
680
681 /*
682 * Do the remaining unconfigure routines
683 */
684 ixgbe_unconfigure(devinfo, ixgbe);
685
686 return (DDI_SUCCESS);
687 }
688
689 /*
690 * quiesce(9E) entry point.
691 *
692 * This function is called when the system is single-threaded at high
693 * PIL with preemption disabled. Therefore, this function must not be
694 * blocked.
695 *
696 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
697 * DDI_FAILURE indicates an error condition and should almost never happen.
698 */
699 static int
700 ixgbe_quiesce(dev_info_t *devinfo)
701 {
702 ixgbe_t *ixgbe;
703 struct ixgbe_hw *hw;
704
705 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
706
707 if (ixgbe == NULL)
708 return (DDI_FAILURE);
709
710 hw = &ixgbe->hw;
711
712 /*
713 * Disable the adapter interrupts
714 */
715 ixgbe_disable_adapter_interrupts(ixgbe);
716
717 /*
718 * Tell firmware driver is no longer in control
719 */
720 ixgbe_release_driver_control(hw);
721
722 /*
723 * Reset the chipset
724 */
725 (void) ixgbe_reset_hw(hw);
726
727 /*
728 * Reset PHY
729 */
730 (void) ixgbe_reset_phy(hw);
731
732 return (DDI_SUCCESS);
733 }
734
735 static void
736 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
737 {
738 /*
739 * Disable interrupt
740 */
741 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
742 (void) ixgbe_disable_intrs(ixgbe);
743 }
744
745 /*
746 * remove the link check timer
747 */
748 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
749 if (ixgbe->periodic_id != NULL) {
750 ddi_periodic_delete(ixgbe->periodic_id);
751 ixgbe->periodic_id = NULL;
752 }
753 }
754
755 /*
756 * Unregister MAC
757 */
758 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
759 (void) mac_unregister(ixgbe->mac_hdl);
760 }
761
762 /*
763 * Free statistics
764 */
765 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
766 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
767 }
768
769 /*
770 * Remove interrupt handlers
771 */
772 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
773 ixgbe_rem_intr_handlers(ixgbe);
774 }
775
776 /*
777 * Remove taskq for sfp-status-change
778 */
779 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
780 ddi_taskq_destroy(ixgbe->sfp_taskq);
781 }
782
783 /*
784 * Remove taskq for over-temp
785 */
786 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
787 ddi_taskq_destroy(ixgbe->overtemp_taskq);
788 }
789
790 /*
791 * Remove interrupts
792 */
793 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
794 ixgbe_rem_intrs(ixgbe);
795 }
796
797 /*
798 * Unregister interrupt callback handler
799 */
800 (void) ddi_cb_unregister(ixgbe->cb_hdl);
801
802 /*
803 * Remove driver properties
804 */
805 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
806 (void) ddi_prop_remove_all(devinfo);
807 }
808
809 /*
810 * Stop the chipset
811 */
812 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
813 mutex_enter(&ixgbe->gen_lock);
814 ixgbe_chip_stop(ixgbe);
815 mutex_exit(&ixgbe->gen_lock);
816 }
817
818 /*
819 * Free register handle
820 */
821 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
822 if (ixgbe->osdep.reg_handle != NULL)
823 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
824 }
825
826 /*
827 * Free PCI config handle
828 */
829 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
830 if (ixgbe->osdep.cfg_handle != NULL)
831 pci_config_teardown(&ixgbe->osdep.cfg_handle);
832 }
833
834 /*
835 * Free locks
836 */
837 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
838 ixgbe_destroy_locks(ixgbe);
839 }
840
841 /*
842 * Free the rx/tx rings
843 */
844 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
845 ixgbe_free_rings(ixgbe);
846 }
847
848 /*
849 * Unregister FMA capabilities
850 */
851 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
852 ixgbe_fm_fini(ixgbe);
853 }
854
855 /*
856 * Free the driver data structure
857 */
858 kmem_free(ixgbe, sizeof (ixgbe_t));
859
860 ddi_set_driver_private(devinfo, NULL);
861 }
862
863 /*
864 * ixgbe_register_mac - Register the driver and its function pointers with
865 * the GLD interface.
866 */
867 static int
868 ixgbe_register_mac(ixgbe_t *ixgbe)
869 {
870 struct ixgbe_hw *hw = &ixgbe->hw;
871 mac_register_t *mac;
872 int status;
873
874 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
875 return (IXGBE_FAILURE);
876
877 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
878 mac->m_driver = ixgbe;
879 mac->m_dip = ixgbe->dip;
880 mac->m_src_addr = hw->mac.addr;
881 mac->m_callbacks = &ixgbe_m_callbacks;
882 mac->m_min_sdu = 0;
883 mac->m_max_sdu = ixgbe->default_mtu;
884 mac->m_margin = VLAN_TAGSZ;
885 mac->m_priv_props = ixgbe_priv_props;
886 mac->m_v12n = MAC_VIRT_LEVEL1;
887
888 status = mac_register(mac, &ixgbe->mac_hdl);
889
890 mac_free(mac);
891
892 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
893 }
894
895 /*
896 * ixgbe_identify_hardware - Identify the type of the chipset.
897 */
898 static int
899 ixgbe_identify_hardware(ixgbe_t *ixgbe)
900 {
901 struct ixgbe_hw *hw = &ixgbe->hw;
902 struct ixgbe_osdep *osdep = &ixgbe->osdep;
903
904 /*
905 * Get the device id
906 */
907 hw->vendor_id =
908 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
909 hw->device_id =
910 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
911 hw->revision_id =
912 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
913 hw->subsystem_device_id =
914 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
915 hw->subsystem_vendor_id =
916 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
917
918 /*
919 * Set the mac type of the adapter based on the device id
920 */
921 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
922 return (IXGBE_FAILURE);
923 }
924
925 /*
926 * Install adapter capabilities
927 */
928 switch (hw->mac.type) {
929 case ixgbe_mac_82598EB:
930 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
931 ixgbe->capab = &ixgbe_82598eb_cap;
932
933 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
934 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
935 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
936 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
937 }
938 break;
939
940 case ixgbe_mac_82599EB:
941 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
942 ixgbe->capab = &ixgbe_82599eb_cap;
943
944 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
945 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
946 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
947 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
948 }
949 break;
950
951 case ixgbe_mac_X540:
952 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
953 ixgbe->capab = &ixgbe_X540_cap;
954 /*
955 * For now, X540 is all set in its capab structure.
956 * As other X540 variants show up, things can change here.
957 */
958 break;
959
960 default:
961 IXGBE_DEBUGLOG_1(ixgbe,
962 "adapter not supported in ixgbe_identify_hardware(): %d\n",
963 hw->mac.type);
964 return (IXGBE_FAILURE);
965 }
966
967 return (IXGBE_SUCCESS);
968 }
969
970 /*
971 * ixgbe_regs_map - Map the device registers.
972 *
973 */
974 static int
975 ixgbe_regs_map(ixgbe_t *ixgbe)
976 {
977 dev_info_t *devinfo = ixgbe->dip;
978 struct ixgbe_hw *hw = &ixgbe->hw;
979 struct ixgbe_osdep *osdep = &ixgbe->osdep;
980 off_t mem_size;
981
982 /*
983 * First get the size of device registers to be mapped.
984 */
985 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
986 != DDI_SUCCESS) {
987 return (IXGBE_FAILURE);
988 }
989
990 /*
991 * Call ddi_regs_map_setup() to map registers
992 */
993 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
994 (caddr_t *)&hw->hw_addr, 0,
995 mem_size, &ixgbe_regs_acc_attr,
996 &osdep->reg_handle)) != DDI_SUCCESS) {
997 return (IXGBE_FAILURE);
998 }
999
1000 return (IXGBE_SUCCESS);
1001 }
1002
1003 /*
1004 * ixgbe_init_properties - Initialize driver properties.
1005 */
1006 static void
1007 ixgbe_init_properties(ixgbe_t *ixgbe)
1008 {
1009 /*
1010 * Get conf file properties, including link settings
1011 * jumbo frames, ring number, descriptor number, etc.
1012 */
1013 ixgbe_get_conf(ixgbe);
1014
1015 ixgbe_init_params(ixgbe);
1016 }
1017
1018 /*
1019 * ixgbe_init_driver_settings - Initialize driver settings.
1020 *
1021 * The settings include hardware function pointers, bus information,
1022 * rx/tx rings settings, link state, and any other parameters that
1023 * need to be setup during driver initialization.
1024 */
1025 static int
1026 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1027 {
1028 struct ixgbe_hw *hw = &ixgbe->hw;
1029 dev_info_t *devinfo = ixgbe->dip;
1030 ixgbe_rx_ring_t *rx_ring;
1031 ixgbe_rx_group_t *rx_group;
1032 ixgbe_tx_ring_t *tx_ring;
1033 uint32_t rx_size;
1034 uint32_t tx_size;
1035 uint32_t ring_per_group;
1036 int i;
1037
1038 /*
1039 * Initialize chipset specific hardware function pointers
1040 */
1041 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1042 return (IXGBE_FAILURE);
1043 }
1044
1045 /*
1046 * Get the system page size
1047 */
1048 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1049
1050 /*
1051 * Set rx buffer size
1052 *
1053 * The IP header alignment room is counted in the calculation.
1054 * The rx buffer size is in unit of 1K that is required by the
1055 * chipset hardware.
1056 */
1057 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1058 ixgbe->rx_buf_size = ((rx_size >> 10) +
1059 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1060
1061 /*
1062 * Set tx buffer size
1063 */
1064 tx_size = ixgbe->max_frame_size;
1065 ixgbe->tx_buf_size = ((tx_size >> 10) +
1066 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1067
1068 /*
1069 * Initialize rx/tx rings/groups parameters
1070 */
1071 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1072 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1073 rx_ring = &ixgbe->rx_rings[i];
1074 rx_ring->index = i;
1075 rx_ring->ixgbe = ixgbe;
1076 rx_ring->group_index = i / ring_per_group;
1077 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1078 }
1079
1080 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1081 rx_group = &ixgbe->rx_groups[i];
1082 rx_group->index = i;
1083 rx_group->ixgbe = ixgbe;
1084 }
1085
1086 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1087 tx_ring = &ixgbe->tx_rings[i];
1088 tx_ring->index = i;
1089 tx_ring->ixgbe = ixgbe;
1090 if (ixgbe->tx_head_wb_enable)
1091 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1092 else
1093 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1094
1095 tx_ring->ring_size = ixgbe->tx_ring_size;
1096 tx_ring->free_list_size = ixgbe->tx_ring_size +
1097 (ixgbe->tx_ring_size >> 1);
1098 }
1099
1100 /*
1101 * Initialize values of interrupt throttling rate
1102 */
1103 for (i = 1; i < MAX_INTR_VECTOR; i++)
1104 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1105
1106 /*
1107 * The initial link state should be "unknown"
1108 */
1109 ixgbe->link_state = LINK_STATE_UNKNOWN;
1110
1111 return (IXGBE_SUCCESS);
1112 }
1113
1114 /*
1115 * ixgbe_init_locks - Initialize locks.
1116 */
1117 static void
1118 ixgbe_init_locks(ixgbe_t *ixgbe)
1119 {
1120 ixgbe_rx_ring_t *rx_ring;
1121 ixgbe_tx_ring_t *tx_ring;
1122 int i;
1123
1124 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1125 rx_ring = &ixgbe->rx_rings[i];
1126 mutex_init(&rx_ring->rx_lock, NULL,
1127 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1128 }
1129
1130 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1131 tx_ring = &ixgbe->tx_rings[i];
1132 mutex_init(&tx_ring->tx_lock, NULL,
1133 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1134 mutex_init(&tx_ring->recycle_lock, NULL,
1135 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1136 mutex_init(&tx_ring->tcb_head_lock, NULL,
1137 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1138 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1139 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1140 }
1141
1142 mutex_init(&ixgbe->gen_lock, NULL,
1143 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1144
1145 mutex_init(&ixgbe->watchdog_lock, NULL,
1146 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1147 }
1148
1149 /*
1150 * ixgbe_destroy_locks - Destroy locks.
1151 */
1152 static void
1153 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1154 {
1155 ixgbe_rx_ring_t *rx_ring;
1156 ixgbe_tx_ring_t *tx_ring;
1157 int i;
1158
1159 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1160 rx_ring = &ixgbe->rx_rings[i];
1161 mutex_destroy(&rx_ring->rx_lock);
1162 }
1163
1164 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1165 tx_ring = &ixgbe->tx_rings[i];
1166 mutex_destroy(&tx_ring->tx_lock);
1167 mutex_destroy(&tx_ring->recycle_lock);
1168 mutex_destroy(&tx_ring->tcb_head_lock);
1169 mutex_destroy(&tx_ring->tcb_tail_lock);
1170 }
1171
1172 mutex_destroy(&ixgbe->gen_lock);
1173 mutex_destroy(&ixgbe->watchdog_lock);
1174 }
1175
1176 static int
1177 ixgbe_resume(dev_info_t *devinfo)
1178 {
1179 ixgbe_t *ixgbe;
1180 int i;
1181
1182 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1183 if (ixgbe == NULL)
1184 return (DDI_FAILURE);
1185
1186 mutex_enter(&ixgbe->gen_lock);
1187
1188 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1189 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1190 mutex_exit(&ixgbe->gen_lock);
1191 return (DDI_FAILURE);
1192 }
1193
1194 /*
1195 * Enable and start the watchdog timer
1196 */
1197 ixgbe_enable_watchdog_timer(ixgbe);
1198 }
1199
1200 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1201
1202 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1203 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1204 mac_tx_ring_update(ixgbe->mac_hdl,
1205 ixgbe->tx_rings[i].ring_handle);
1206 }
1207 }
1208
1209 mutex_exit(&ixgbe->gen_lock);
1210
1211 return (DDI_SUCCESS);
1212 }
1213
1214 static int
1215 ixgbe_suspend(dev_info_t *devinfo)
1216 {
1217 ixgbe_t *ixgbe;
1218
1219 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1220 if (ixgbe == NULL)
1221 return (DDI_FAILURE);
1222
1223 mutex_enter(&ixgbe->gen_lock);
1224
1225 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1226 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1227 mutex_exit(&ixgbe->gen_lock);
1228 return (DDI_SUCCESS);
1229 }
1230 ixgbe_stop(ixgbe, B_FALSE);
1231
1232 mutex_exit(&ixgbe->gen_lock);
1233
1234 /*
1235 * Disable and stop the watchdog timer
1236 */
1237 ixgbe_disable_watchdog_timer(ixgbe);
1238
1239 return (DDI_SUCCESS);
1240 }
1241
1242 /*
1243 * ixgbe_init - Initialize the device.
1244 */
1245 static int
1246 ixgbe_init(ixgbe_t *ixgbe)
1247 {
1248 struct ixgbe_hw *hw = &ixgbe->hw;
1249
1250 mutex_enter(&ixgbe->gen_lock);
1251
1252 /*
1253 * Reset chipset to put the hardware in a known state
1254 * before we try to do anything with the eeprom.
1255 */
1256 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1257 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1258 goto init_fail;
1259 }
1260
1261 /*
1262 * Need to init eeprom before validating the checksum.
1263 */
1264 if (ixgbe_init_eeprom_params(hw) < 0) {
1265 ixgbe_error(ixgbe,
1266 "Unable to intitialize the eeprom interface.");
1267 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1268 goto init_fail;
1269 }
1270
1271 /*
1272 * NVM validation
1273 */
1274 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1275 /*
1276 * Some PCI-E parts fail the first check due to
1277 * the link being in sleep state. Call it again,
1278 * if it fails a second time it's a real issue.
1279 */
1280 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1281 ixgbe_error(ixgbe,
1282 "Invalid NVM checksum. Please contact "
1283 "the vendor to update the NVM.");
1284 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1285 goto init_fail;
1286 }
1287 }
1288
1289 /*
1290 * Setup default flow control thresholds - enable/disable
1291 * & flow control type is controlled by ixgbe.conf
1292 */
1293 hw->fc.high_water[0] = DEFAULT_FCRTH;
1294 hw->fc.low_water[0] = DEFAULT_FCRTL;
1295 hw->fc.pause_time = DEFAULT_FCPAUSE;
1296 hw->fc.send_xon = B_TRUE;
1297
1298 /*
1299 * Initialize link settings
1300 */
1301 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1302
1303 /*
1304 * Initialize the chipset hardware
1305 */
1306 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1307 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1308 goto init_fail;
1309 }
1310
1311 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1312 goto init_fail;
1313 }
1314
1315 mutex_exit(&ixgbe->gen_lock);
1316 return (IXGBE_SUCCESS);
1317
1318 init_fail:
1319 /*
1320 * Reset PHY
1321 */
1322 (void) ixgbe_reset_phy(hw);
1323
1324 mutex_exit(&ixgbe->gen_lock);
1325 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1326 return (IXGBE_FAILURE);
1327 }
1328
1329 /*
1330 * ixgbe_chip_start - Initialize and start the chipset hardware.
1331 */
1332 static int
1333 ixgbe_chip_start(ixgbe_t *ixgbe)
1334 {
1335 struct ixgbe_hw *hw = &ixgbe->hw;
1336 int ret_val, i;
1337
1338 ASSERT(mutex_owned(&ixgbe->gen_lock));
1339
1340 /*
1341 * Get the mac address
1342 * This function should handle SPARC case correctly.
1343 */
1344 if (!ixgbe_find_mac_address(ixgbe)) {
1345 ixgbe_error(ixgbe, "Failed to get the mac address");
1346 return (IXGBE_FAILURE);
1347 }
1348
1349 /*
1350 * Validate the mac address
1351 */
1352 (void) ixgbe_init_rx_addrs(hw);
1353 if (!is_valid_mac_addr(hw->mac.addr)) {
1354 ixgbe_error(ixgbe, "Invalid mac address");
1355 return (IXGBE_FAILURE);
1356 }
1357
1358 /*
1359 * Configure/Initialize hardware
1360 */
1361 ret_val = ixgbe_init_hw(hw);
1362 if (ret_val != IXGBE_SUCCESS) {
1363 if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1364 ixgbe_error(ixgbe,
1365 "This 82599 device is pre-release and contains"
1366 " outdated firmware, please contact your hardware"
1367 " vendor for a replacement.");
1368 } else {
1369 ixgbe_error(ixgbe, "Failed to initialize hardware");
1370 return (IXGBE_FAILURE);
1371 }
1372 }
1373
1374 /*
1375 * Re-enable relaxed ordering for performance. It is disabled
1376 * by default in the hardware init.
1377 */
1378 if (ixgbe->relax_order_enable == B_TRUE)
1379 ixgbe_enable_relaxed_ordering(hw);
1380
1381 /*
1382 * Setup adapter interrupt vectors
1383 */
1384 ixgbe_setup_adapter_vector(ixgbe);
1385
1386 /*
1387 * Initialize unicast addresses.
1388 */
1389 ixgbe_init_unicst(ixgbe);
1390
1391 /*
1392 * Setup and initialize the mctable structures.
1393 */
1394 ixgbe_setup_multicst(ixgbe);
1395
1396 /*
1397 * Set interrupt throttling rate
1398 */
1399 for (i = 0; i < ixgbe->intr_cnt; i++) {
1400 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1401 }
1402
1403 /*
1404 * Save the state of the phy
1405 */
1406 ixgbe_get_hw_state(ixgbe);
1407
1408 /*
1409 * Make sure driver has control
1410 */
1411 ixgbe_get_driver_control(hw);
1412
1413 return (IXGBE_SUCCESS);
1414 }
1415
1416 /*
1417 * ixgbe_chip_stop - Stop the chipset hardware
1418 */
1419 static void
1420 ixgbe_chip_stop(ixgbe_t *ixgbe)
1421 {
1422 struct ixgbe_hw *hw = &ixgbe->hw;
1423
1424 ASSERT(mutex_owned(&ixgbe->gen_lock));
1425
1426 /*
1427 * Tell firmware driver is no longer in control
1428 */
1429 ixgbe_release_driver_control(hw);
1430
1431 /*
1432 * Reset the chipset
1433 */
1434 (void) ixgbe_reset_hw(hw);
1435
1436 /*
1437 * Reset PHY
1438 */
1439 (void) ixgbe_reset_phy(hw);
1440 }
1441
1442 /*
1443 * ixgbe_reset - Reset the chipset and re-start the driver.
1444 *
1445 * It involves stopping and re-starting the chipset,
1446 * and re-configuring the rx/tx rings.
1447 */
1448 static int
1449 ixgbe_reset(ixgbe_t *ixgbe)
1450 {
1451 int i;
1452
1453 /*
1454 * Disable and stop the watchdog timer
1455 */
1456 ixgbe_disable_watchdog_timer(ixgbe);
1457
1458 mutex_enter(&ixgbe->gen_lock);
1459
1460 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1461 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1462
1463 ixgbe_stop(ixgbe, B_FALSE);
1464
1465 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1466 mutex_exit(&ixgbe->gen_lock);
1467 return (IXGBE_FAILURE);
1468 }
1469
1470 /*
1471 * After resetting, need to recheck the link status.
1472 */
1473 ixgbe->link_check_complete = B_FALSE;
1474 ixgbe->link_check_hrtime = gethrtime() +
1475 (IXGBE_LINK_UP_TIME * 100000000ULL);
1476
1477 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1478
1479 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1480 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1481 mac_tx_ring_update(ixgbe->mac_hdl,
1482 ixgbe->tx_rings[i].ring_handle);
1483 }
1484 }
1485
1486 mutex_exit(&ixgbe->gen_lock);
1487
1488 /*
1489 * Enable and start the watchdog timer
1490 */
1491 ixgbe_enable_watchdog_timer(ixgbe);
1492
1493 return (IXGBE_SUCCESS);
1494 }
1495
1496 /*
1497 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1498 */
1499 static void
1500 ixgbe_tx_clean(ixgbe_t *ixgbe)
1501 {
1502 ixgbe_tx_ring_t *tx_ring;
1503 tx_control_block_t *tcb;
1504 link_list_t pending_list;
1505 uint32_t desc_num;
1506 int i, j;
1507
1508 LINK_LIST_INIT(&pending_list);
1509
1510 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1511 tx_ring = &ixgbe->tx_rings[i];
1512
1513 mutex_enter(&tx_ring->recycle_lock);
1514
1515 /*
1516 * Clean the pending tx data - the pending packets in the
1517 * work_list that have no chances to be transmitted again.
1518 *
1519 * We must ensure the chipset is stopped or the link is down
1520 * before cleaning the transmit packets.
1521 */
1522 desc_num = 0;
1523 for (j = 0; j < tx_ring->ring_size; j++) {
1524 tcb = tx_ring->work_list[j];
1525 if (tcb != NULL) {
1526 desc_num += tcb->desc_num;
1527
1528 tx_ring->work_list[j] = NULL;
1529
1530 ixgbe_free_tcb(tcb);
1531
1532 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1533 }
1534 }
1535
1536 if (desc_num > 0) {
1537 atomic_add_32(&tx_ring->tbd_free, desc_num);
1538 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1539
1540 /*
1541 * Reset the head and tail pointers of the tbd ring;
1542 * Reset the writeback head if it's enable.
1543 */
1544 tx_ring->tbd_head = 0;
1545 tx_ring->tbd_tail = 0;
1546 if (ixgbe->tx_head_wb_enable)
1547 *tx_ring->tbd_head_wb = 0;
1548
1549 IXGBE_WRITE_REG(&ixgbe->hw,
1550 IXGBE_TDH(tx_ring->index), 0);
1551 IXGBE_WRITE_REG(&ixgbe->hw,
1552 IXGBE_TDT(tx_ring->index), 0);
1553 }
1554
1555 mutex_exit(&tx_ring->recycle_lock);
1556
1557 /*
1558 * Add the tx control blocks in the pending list to
1559 * the free list.
1560 */
1561 ixgbe_put_free_list(tx_ring, &pending_list);
1562 }
1563 }
1564
1565 /*
1566 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1567 * transmitted.
1568 */
1569 static boolean_t
1570 ixgbe_tx_drain(ixgbe_t *ixgbe)
1571 {
1572 ixgbe_tx_ring_t *tx_ring;
1573 boolean_t done;
1574 int i, j;
1575
1576 /*
1577 * Wait for a specific time to allow pending tx packets
1578 * to be transmitted.
1579 *
1580 * Check the counter tbd_free to see if transmission is done.
1581 * No lock protection is needed here.
1582 *
1583 * Return B_TRUE if all pending packets have been transmitted;
1584 * Otherwise return B_FALSE;
1585 */
1586 for (i = 0; i < TX_DRAIN_TIME; i++) {
1587
1588 done = B_TRUE;
1589 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1590 tx_ring = &ixgbe->tx_rings[j];
1591 done = done &&
1592 (tx_ring->tbd_free == tx_ring->ring_size);
1593 }
1594
1595 if (done)
1596 break;
1597
1598 msec_delay(1);
1599 }
1600
1601 return (done);
1602 }
1603
1604 /*
1605 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1606 */
1607 static boolean_t
1608 ixgbe_rx_drain(ixgbe_t *ixgbe)
1609 {
1610 boolean_t done = B_TRUE;
1611 int i;
1612
1613 /*
1614 * Polling the rx free list to check if those rx buffers held by
1615 * the upper layer are released.
1616 *
1617 * Check the counter rcb_free to see if all pending buffers are
1618 * released. No lock protection is needed here.
1619 *
1620 * Return B_TRUE if all pending buffers have been released;
1621 * Otherwise return B_FALSE;
1622 */
1623 for (i = 0; i < RX_DRAIN_TIME; i++) {
1624 done = (ixgbe->rcb_pending == 0);
1625
1626 if (done)
1627 break;
1628
1629 msec_delay(1);
1630 }
1631
1632 return (done);
1633 }
1634
1635 /*
1636 * ixgbe_start - Start the driver/chipset.
1637 */
1638 int
1639 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1640 {
1641 int i;
1642
1643 ASSERT(mutex_owned(&ixgbe->gen_lock));
1644
1645 if (alloc_buffer) {
1646 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1647 ixgbe_error(ixgbe,
1648 "Failed to allocate software receive rings");
1649 return (IXGBE_FAILURE);
1650 }
1651
1652 /* Allocate buffers for all the rx/tx rings */
1653 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1654 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1655 return (IXGBE_FAILURE);
1656 }
1657
1658 ixgbe->tx_ring_init = B_TRUE;
1659 } else {
1660 ixgbe->tx_ring_init = B_FALSE;
1661 }
1662
1663 for (i = 0; i < ixgbe->num_rx_rings; i++)
1664 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1665 for (i = 0; i < ixgbe->num_tx_rings; i++)
1666 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1667
1668 /*
1669 * Start the chipset hardware
1670 */
1671 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1672 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1673 goto start_failure;
1674 }
1675
1676 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1677 goto start_failure;
1678 }
1679
1680 /*
1681 * Setup the rx/tx rings
1682 */
1683 ixgbe_setup_rings(ixgbe);
1684
1685 /*
1686 * ixgbe_start() will be called when resetting, however if reset
1687 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1688 * before enabling the interrupts.
1689 */
1690 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1691 | IXGBE_STALL| IXGBE_OVERTEMP));
1692
1693 /*
1694 * Enable adapter interrupts
1695 * The interrupts must be enabled after the driver state is START
1696 */
1697 ixgbe_enable_adapter_interrupts(ixgbe);
1698
1699 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1700 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1701 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1702 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1703
1704 return (IXGBE_SUCCESS);
1705
1706 start_failure:
1707 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1708 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1709 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1710 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1711
1712 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1713
1714 return (IXGBE_FAILURE);
1715 }
1716
1717 /*
1718 * ixgbe_stop - Stop the driver/chipset.
1719 */
1720 void
1721 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1722 {
1723 int i;
1724
1725 ASSERT(mutex_owned(&ixgbe->gen_lock));
1726
1727 /*
1728 * Disable the adapter interrupts
1729 */
1730 ixgbe_disable_adapter_interrupts(ixgbe);
1731
1732 /*
1733 * Drain the pending tx packets
1734 */
1735 (void) ixgbe_tx_drain(ixgbe);
1736
1737 for (i = 0; i < ixgbe->num_rx_rings; i++)
1738 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1739 for (i = 0; i < ixgbe->num_tx_rings; i++)
1740 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1741
1742 /*
1743 * Stop the chipset hardware
1744 */
1745 ixgbe_chip_stop(ixgbe);
1746
1747 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1748 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1749 }
1750
1751 /*
1752 * Clean the pending tx data/resources
1753 */
1754 ixgbe_tx_clean(ixgbe);
1755
1756 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1757 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1758 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1759 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1760
1761 if (ixgbe->link_state == LINK_STATE_UP) {
1762 ixgbe->link_state = LINK_STATE_UNKNOWN;
1763 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1764 }
1765
1766 if (free_buffer) {
1767 /*
1768 * Release the DMA/memory resources of rx/tx rings
1769 */
1770 ixgbe_free_dma(ixgbe);
1771 ixgbe_free_rx_data(ixgbe);
1772 }
1773 }
1774
1775 /*
1776 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1777 */
1778 /* ARGSUSED */
1779 static int
1780 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1781 void *arg1, void *arg2)
1782 {
1783 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1784
1785 switch (cbaction) {
1786 /* IRM callback */
1787 int count;
1788 case DDI_CB_INTR_ADD:
1789 case DDI_CB_INTR_REMOVE:
1790 count = (int)(uintptr_t)cbarg;
1791 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1792 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1793 int, ixgbe->intr_cnt);
1794 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1795 DDI_SUCCESS) {
1796 ixgbe_error(ixgbe,
1797 "IRM CB: Failed to adjust interrupts");
1798 goto cb_fail;
1799 }
1800 break;
1801 default:
1802 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1803 cbaction);
1804 return (DDI_ENOTSUP);
1805 }
1806 return (DDI_SUCCESS);
1807 cb_fail:
1808 return (DDI_FAILURE);
1809 }
1810
1811 /*
1812 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1813 */
1814 static int
1815 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1816 {
1817 int i, rc, actual;
1818
1819 if (count == 0)
1820 return (DDI_SUCCESS);
1821
1822 if ((cbaction == DDI_CB_INTR_ADD &&
1823 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1824 (cbaction == DDI_CB_INTR_REMOVE &&
1825 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
1826 return (DDI_FAILURE);
1827
1828 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1829 return (DDI_FAILURE);
1830 }
1831
1832 for (i = 0; i < ixgbe->num_rx_rings; i++)
1833 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
1834 for (i = 0; i < ixgbe->num_tx_rings; i++)
1835 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
1836
1837 mutex_enter(&ixgbe->gen_lock);
1838 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1839 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
1840 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
1841 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
1842
1843 ixgbe_stop(ixgbe, B_FALSE);
1844 /*
1845 * Disable interrupts
1846 */
1847 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1848 rc = ixgbe_disable_intrs(ixgbe);
1849 ASSERT(rc == IXGBE_SUCCESS);
1850 }
1851 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
1852
1853 /*
1854 * Remove interrupt handlers
1855 */
1856 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
1857 ixgbe_rem_intr_handlers(ixgbe);
1858 }
1859 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
1860
1861 /*
1862 * Clear vect_map
1863 */
1864 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
1865 switch (cbaction) {
1866 case DDI_CB_INTR_ADD:
1867 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
1868 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
1869 DDI_INTR_ALLOC_NORMAL);
1870 if (rc != DDI_SUCCESS || actual != count) {
1871 ixgbe_log(ixgbe, "Adjust interrupts failed."
1872 "return: %d, irm cb size: %d, actual: %d",
1873 rc, count, actual);
1874 goto intr_adjust_fail;
1875 }
1876 ixgbe->intr_cnt += count;
1877 break;
1878
1879 case DDI_CB_INTR_REMOVE:
1880 for (i = ixgbe->intr_cnt - count;
1881 i < ixgbe->intr_cnt; i ++) {
1882 rc = ddi_intr_free(ixgbe->htable[i]);
1883 ixgbe->htable[i] = NULL;
1884 if (rc != DDI_SUCCESS) {
1885 ixgbe_log(ixgbe, "Adjust interrupts failed."
1886 "return: %d, irm cb size: %d, actual: %d",
1887 rc, count, actual);
1888 goto intr_adjust_fail;
1889 }
1890 }
1891 ixgbe->intr_cnt -= count;
1892 break;
1893 }
1894
1895 /*
1896 * Get priority for first vector, assume remaining are all the same
1897 */
1898 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
1899 if (rc != DDI_SUCCESS) {
1900 ixgbe_log(ixgbe,
1901 "Get interrupt priority failed: %d", rc);
1902 goto intr_adjust_fail;
1903 }
1904 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
1905 if (rc != DDI_SUCCESS) {
1906 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
1907 goto intr_adjust_fail;
1908 }
1909 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
1910
1911 /*
1912 * Map rings to interrupt vectors
1913 */
1914 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
1915 ixgbe_error(ixgbe,
1916 "IRM CB: Failed to map interrupts to vectors");
1917 goto intr_adjust_fail;
1918 }
1919
1920 /*
1921 * Add interrupt handlers
1922 */
1923 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
1924 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
1925 goto intr_adjust_fail;
1926 }
1927 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
1928
1929 /*
1930 * Now that mutex locks are initialized, and the chip is also
1931 * initialized, enable interrupts.
1932 */
1933 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
1934 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
1935 goto intr_adjust_fail;
1936 }
1937 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
1938 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1939 ixgbe_error(ixgbe, "IRM CB: Failed to start");
1940 goto intr_adjust_fail;
1941 }
1942 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
1943 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
1944 ixgbe->ixgbe_state |= IXGBE_STARTED;
1945 mutex_exit(&ixgbe->gen_lock);
1946
1947 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1948 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
1949 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
1950 }
1951 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1952 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
1953 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
1954 }
1955
1956 /* Wakeup all Tx rings */
1957 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1958 mac_tx_ring_update(ixgbe->mac_hdl,
1959 ixgbe->tx_rings[i].ring_handle);
1960 }
1961
1962 IXGBE_DEBUGLOG_3(ixgbe,
1963 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
1964 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
1965 return (DDI_SUCCESS);
1966
1967 intr_adjust_fail:
1968 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1969 mutex_exit(&ixgbe->gen_lock);
1970 return (DDI_FAILURE);
1971 }
1972
1973 /*
1974 * ixgbe_intr_cb_register - Register interrupt callback function.
1975 */
1976 static int
1977 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
1978 {
1979 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
1980 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
1981 return (IXGBE_FAILURE);
1982 }
1983 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
1984 return (IXGBE_SUCCESS);
1985 }
1986
1987 /*
1988 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
1989 */
1990 static int
1991 ixgbe_alloc_rings(ixgbe_t *ixgbe)
1992 {
1993 /*
1994 * Allocate memory space for rx rings
1995 */
1996 ixgbe->rx_rings = kmem_zalloc(
1997 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
1998 KM_NOSLEEP);
1999
2000 if (ixgbe->rx_rings == NULL) {
2001 return (IXGBE_FAILURE);
2002 }
2003
2004 /*
2005 * Allocate memory space for tx rings
2006 */
2007 ixgbe->tx_rings = kmem_zalloc(
2008 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2009 KM_NOSLEEP);
2010
2011 if (ixgbe->tx_rings == NULL) {
2012 kmem_free(ixgbe->rx_rings,
2013 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2014 ixgbe->rx_rings = NULL;
2015 return (IXGBE_FAILURE);
2016 }
2017
2018 /*
2019 * Allocate memory space for rx ring groups
2020 */
2021 ixgbe->rx_groups = kmem_zalloc(
2022 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2023 KM_NOSLEEP);
2024
2025 if (ixgbe->rx_groups == NULL) {
2026 kmem_free(ixgbe->rx_rings,
2027 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2028 kmem_free(ixgbe->tx_rings,
2029 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2030 ixgbe->rx_rings = NULL;
2031 ixgbe->tx_rings = NULL;
2032 return (IXGBE_FAILURE);
2033 }
2034
2035 return (IXGBE_SUCCESS);
2036 }
2037
2038 /*
2039 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2040 */
2041 static void
2042 ixgbe_free_rings(ixgbe_t *ixgbe)
2043 {
2044 if (ixgbe->rx_rings != NULL) {
2045 kmem_free(ixgbe->rx_rings,
2046 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2047 ixgbe->rx_rings = NULL;
2048 }
2049
2050 if (ixgbe->tx_rings != NULL) {
2051 kmem_free(ixgbe->tx_rings,
2052 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2053 ixgbe->tx_rings = NULL;
2054 }
2055
2056 if (ixgbe->rx_groups != NULL) {
2057 kmem_free(ixgbe->rx_groups,
2058 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2059 ixgbe->rx_groups = NULL;
2060 }
2061 }
2062
2063 static int
2064 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2065 {
2066 ixgbe_rx_ring_t *rx_ring;
2067 int i;
2068
2069 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2070 rx_ring = &ixgbe->rx_rings[i];
2071 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2072 goto alloc_rx_rings_failure;
2073 }
2074 return (IXGBE_SUCCESS);
2075
2076 alloc_rx_rings_failure:
2077 ixgbe_free_rx_data(ixgbe);
2078 return (IXGBE_FAILURE);
2079 }
2080
2081 static void
2082 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2083 {
2084 ixgbe_rx_ring_t *rx_ring;
2085 ixgbe_rx_data_t *rx_data;
2086 int i;
2087
2088 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2089 rx_ring = &ixgbe->rx_rings[i];
2090
2091 mutex_enter(&ixgbe->rx_pending_lock);
2092 rx_data = rx_ring->rx_data;
2093
2094 if (rx_data != NULL) {
2095 rx_data->flag |= IXGBE_RX_STOPPED;
2096
2097 if (rx_data->rcb_pending == 0) {
2098 ixgbe_free_rx_ring_data(rx_data);
2099 rx_ring->rx_data = NULL;
2100 }
2101 }
2102
2103 mutex_exit(&ixgbe->rx_pending_lock);
2104 }
2105 }
2106
2107 /*
2108 * ixgbe_setup_rings - Setup rx/tx rings.
2109 */
2110 static void
2111 ixgbe_setup_rings(ixgbe_t *ixgbe)
2112 {
2113 /*
2114 * Setup the rx/tx rings, including the following:
2115 *
2116 * 1. Setup the descriptor ring and the control block buffers;
2117 * 2. Initialize necessary registers for receive/transmit;
2118 * 3. Initialize software pointers/parameters for receive/transmit;
2119 */
2120 ixgbe_setup_rx(ixgbe);
2121
2122 ixgbe_setup_tx(ixgbe);
2123 }
2124
2125 static void
2126 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2127 {
2128 ixgbe_t *ixgbe = rx_ring->ixgbe;
2129 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2130 struct ixgbe_hw *hw = &ixgbe->hw;
2131 rx_control_block_t *rcb;
2132 union ixgbe_adv_rx_desc *rbd;
2133 uint32_t size;
2134 uint32_t buf_low;
2135 uint32_t buf_high;
2136 uint32_t reg_val;
2137 int i;
2138
2139 ASSERT(mutex_owned(&rx_ring->rx_lock));
2140 ASSERT(mutex_owned(&ixgbe->gen_lock));
2141
2142 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2143 rcb = rx_data->work_list[i];
2144 rbd = &rx_data->rbd_ring[i];
2145
2146 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2147 rbd->read.hdr_addr = NULL;
2148 }
2149
2150 /*
2151 * Initialize the length register
2152 */
2153 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2154 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2155
2156 /*
2157 * Initialize the base address registers
2158 */
2159 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2160 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2161 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2162 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2163
2164 /*
2165 * Setup head & tail pointers
2166 */
2167 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2168 rx_data->ring_size - 1);
2169 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2170
2171 rx_data->rbd_next = 0;
2172 rx_data->lro_first = 0;
2173
2174 /*
2175 * Setup the Receive Descriptor Control Register (RXDCTL)
2176 * PTHRESH=32 descriptors (half the internal cache)
2177 * HTHRESH=0 descriptors (to minimize latency on fetch)
2178 * WTHRESH defaults to 1 (writeback each descriptor)
2179 */
2180 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2181 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2182
2183 /* Not a valid value for 82599 or X540 */
2184 if (hw->mac.type == ixgbe_mac_82598EB) {
2185 reg_val |= 0x0020; /* pthresh */
2186 }
2187 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2188
2189 if (hw->mac.type == ixgbe_mac_82599EB ||
2190 hw->mac.type == ixgbe_mac_X540) {
2191 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2192 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2193 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2194 }
2195
2196 /*
2197 * Setup the Split and Replication Receive Control Register.
2198 * Set the rx buffer size and the advanced descriptor type.
2199 */
2200 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2201 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2202 reg_val |= IXGBE_SRRCTL_DROP_EN;
2203 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2204 }
2205
2206 static void
2207 ixgbe_setup_rx(ixgbe_t *ixgbe)
2208 {
2209 ixgbe_rx_ring_t *rx_ring;
2210 struct ixgbe_hw *hw = &ixgbe->hw;
2211 uint32_t reg_val;
2212 uint32_t ring_mapping;
2213 uint32_t i, index;
2214 uint32_t psrtype_rss_bit;
2215
2216 /* PSRTYPE must be configured for 82599 */
2217 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2218 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2219 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2220 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2221 reg_val |= IXGBE_PSRTYPE_L2HDR;
2222 reg_val |= 0x80000000;
2223 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2224 } else {
2225 if (ixgbe->num_rx_groups > 32) {
2226 psrtype_rss_bit = 0x20000000;
2227 } else {
2228 psrtype_rss_bit = 0x40000000;
2229 }
2230 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2231 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2232 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2233 reg_val |= IXGBE_PSRTYPE_L2HDR;
2234 reg_val |= psrtype_rss_bit;
2235 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2236 }
2237 }
2238
2239 /*
2240 * Set filter control in FCTRL to accept broadcast packets and do
2241 * not pass pause frames to host. Flow control settings are already
2242 * in this register, so preserve them.
2243 */
2244 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2245 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */
2246 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */
2247 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2248
2249 /*
2250 * Hardware checksum settings
2251 */
2252 if (ixgbe->rx_hcksum_enable) {
2253 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2254 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2255 }
2256
2257 /*
2258 * Setup VMDq and RSS for multiple receive queues
2259 */
2260 switch (ixgbe->classify_mode) {
2261 case IXGBE_CLASSIFY_RSS:
2262 /*
2263 * One group, only RSS is needed when more than
2264 * one ring enabled.
2265 */
2266 ixgbe_setup_rss(ixgbe);
2267 break;
2268
2269 case IXGBE_CLASSIFY_VMDQ:
2270 /*
2271 * Multiple groups, each group has one ring,
2272 * only VMDq is needed.
2273 */
2274 ixgbe_setup_vmdq(ixgbe);
2275 break;
2276
2277 case IXGBE_CLASSIFY_VMDQ_RSS:
2278 /*
2279 * Multiple groups and multiple rings, both
2280 * VMDq and RSS are needed.
2281 */
2282 ixgbe_setup_vmdq_rss(ixgbe);
2283 break;
2284
2285 default:
2286 break;
2287 }
2288
2289 /*
2290 * Enable the receive unit. This must be done after filter
2291 * control is set in FCTRL.
2292 */
2293 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */
2294 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */
2295 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2296
2297 /*
2298 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2299 */
2300 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2301 rx_ring = &ixgbe->rx_rings[i];
2302 ixgbe_setup_rx_ring(rx_ring);
2303 }
2304
2305 /*
2306 * Setup the per-ring statistics mapping.
2307 */
2308 ring_mapping = 0;
2309 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2310 index = ixgbe->rx_rings[i].hw_index;
2311 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2312 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2313 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2314 }
2315
2316 /*
2317 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2318 * by four bytes if the packet has a VLAN field, so includes MTU,
2319 * ethernet header and frame check sequence.
2320 * Register is MAXFRS in 82599.
2321 */
2322 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2323 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2324 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2325
2326 /*
2327 * Setup Jumbo Frame enable bit
2328 */
2329 if (ixgbe->default_mtu > ETHERMTU) {
2330 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2331 reg_val |= IXGBE_HLREG0_JUMBOEN;
2332 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2333 }
2334
2335 /*
2336 * Setup RSC for multiple receive queues.
2337 */
2338 if (ixgbe->lro_enable) {
2339 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2340 /*
2341 * Make sure rx_buf_size * MAXDESC not greater
2342 * than 65535.
2343 * Intel recommends 4 for MAXDESC field value.
2344 */
2345 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2346 reg_val |= IXGBE_RSCCTL_RSCEN;
2347 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2348 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2349 else
2350 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2351 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2352 }
2353
2354 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2355 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2356 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2357
2358 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2359 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2360 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2361 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2362
2363 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2364 }
2365 }
2366
2367 static void
2368 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2369 {
2370 ixgbe_t *ixgbe = tx_ring->ixgbe;
2371 struct ixgbe_hw *hw = &ixgbe->hw;
2372 uint32_t size;
2373 uint32_t buf_low;
2374 uint32_t buf_high;
2375 uint32_t reg_val;
2376
2377 ASSERT(mutex_owned(&tx_ring->tx_lock));
2378 ASSERT(mutex_owned(&ixgbe->gen_lock));
2379
2380 /*
2381 * Initialize the length register
2382 */
2383 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2384 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2385
2386 /*
2387 * Initialize the base address registers
2388 */
2389 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2390 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2391 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2392 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2393
2394 /*
2395 * Setup head & tail pointers
2396 */
2397 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2398 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2399
2400 /*
2401 * Setup head write-back
2402 */
2403 if (ixgbe->tx_head_wb_enable) {
2404 /*
2405 * The memory of the head write-back is allocated using
2406 * the extra tbd beyond the tail of the tbd ring.
2407 */
2408 tx_ring->tbd_head_wb = (uint32_t *)
2409 ((uintptr_t)tx_ring->tbd_area.address + size);
2410 *tx_ring->tbd_head_wb = 0;
2411
2412 buf_low = (uint32_t)
2413 (tx_ring->tbd_area.dma_address + size);
2414 buf_high = (uint32_t)
2415 ((tx_ring->tbd_area.dma_address + size) >> 32);
2416
2417 /* Set the head write-back enable bit */
2418 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2419
2420 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2421 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2422
2423 /*
2424 * Turn off relaxed ordering for head write back or it will
2425 * cause problems with the tx recycling
2426 */
2427
2428 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2429 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2430 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2431 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2432 if (hw->mac.type == ixgbe_mac_82598EB) {
2433 IXGBE_WRITE_REG(hw,
2434 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2435 } else {
2436 IXGBE_WRITE_REG(hw,
2437 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2438 }
2439 } else {
2440 tx_ring->tbd_head_wb = NULL;
2441 }
2442
2443 tx_ring->tbd_head = 0;
2444 tx_ring->tbd_tail = 0;
2445 tx_ring->tbd_free = tx_ring->ring_size;
2446
2447 if (ixgbe->tx_ring_init == B_TRUE) {
2448 tx_ring->tcb_head = 0;
2449 tx_ring->tcb_tail = 0;
2450 tx_ring->tcb_free = tx_ring->free_list_size;
2451 }
2452
2453 /*
2454 * Initialize the s/w context structure
2455 */
2456 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2457 }
2458
2459 static void
2460 ixgbe_setup_tx(ixgbe_t *ixgbe)
2461 {
2462 struct ixgbe_hw *hw = &ixgbe->hw;
2463 ixgbe_tx_ring_t *tx_ring;
2464 uint32_t reg_val;
2465 uint32_t ring_mapping;
2466 int i;
2467
2468 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2469 tx_ring = &ixgbe->tx_rings[i];
2470 ixgbe_setup_tx_ring(tx_ring);
2471 }
2472
2473 /*
2474 * Setup the per-ring statistics mapping.
2475 */
2476 ring_mapping = 0;
2477 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2478 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2479 if ((i & 0x3) == 0x3) {
2480 switch (hw->mac.type) {
2481 case ixgbe_mac_82598EB:
2482 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2483 ring_mapping);
2484 break;
2485
2486 case ixgbe_mac_82599EB:
2487 case ixgbe_mac_X540:
2488 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2489 ring_mapping);
2490 break;
2491
2492 default:
2493 break;
2494 }
2495
2496 ring_mapping = 0;
2497 }
2498 }
2499 if (i & 0x3) {
2500 switch (hw->mac.type) {
2501 case ixgbe_mac_82598EB:
2502 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2503 break;
2504
2505 case ixgbe_mac_82599EB:
2506 case ixgbe_mac_X540:
2507 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2508 break;
2509
2510 default:
2511 break;
2512 }
2513 }
2514
2515 /*
2516 * Enable CRC appending and TX padding (for short tx frames)
2517 */
2518 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2519 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2520 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2521
2522 /*
2523 * enable DMA for 82599 and X540 parts
2524 */
2525 if (hw->mac.type == ixgbe_mac_82599EB ||
2526 hw->mac.type == ixgbe_mac_X540) {
2527 /* DMATXCTL.TE must be set after all Tx config is complete */
2528 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2529 reg_val |= IXGBE_DMATXCTL_TE;
2530 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2531
2532 /* Disable arbiter to set MTQC */
2533 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2534 reg_val |= IXGBE_RTTDCS_ARBDIS;
2535 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2536 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2537 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2538 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2539 }
2540
2541 /*
2542 * Enabling tx queues ..
2543 * For 82599 must be done after DMATXCTL.TE is set
2544 */
2545 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2546 tx_ring = &ixgbe->tx_rings[i];
2547 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2548 reg_val |= IXGBE_TXDCTL_ENABLE;
2549 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2550 }
2551 }
2552
2553 /*
2554 * ixgbe_setup_rss - Setup receive-side scaling feature.
2555 */
2556 static void
2557 ixgbe_setup_rss(ixgbe_t *ixgbe)
2558 {
2559 struct ixgbe_hw *hw = &ixgbe->hw;
2560 uint32_t i, mrqc, rxcsum;
2561 uint32_t random;
2562 uint32_t reta;
2563 uint32_t ring_per_group;
2564
2565 /*
2566 * Fill out redirection table
2567 */
2568 reta = 0;
2569 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2570
2571 for (i = 0; i < 128; i++) {
2572 reta = (reta << 8) | (i % ring_per_group) |
2573 ((i % ring_per_group) << 4);
2574 if ((i & 3) == 3)
2575 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2576 }
2577
2578 /*
2579 * Fill out hash function seeds with a random constant
2580 */
2581 for (i = 0; i < 10; i++) {
2582 (void) random_get_pseudo_bytes((uint8_t *)&random,
2583 sizeof (uint32_t));
2584 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2585 }
2586
2587 /*
2588 * Enable RSS & perform hash on these packet types
2589 */
2590 mrqc = IXGBE_MRQC_RSSEN |
2591 IXGBE_MRQC_RSS_FIELD_IPV4 |
2592 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2593 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2594 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2595 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2596 IXGBE_MRQC_RSS_FIELD_IPV6 |
2597 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2598 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2599 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2600 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2601
2602 /*
2603 * Disable Packet Checksum to enable RSS for multiple receive queues.
2604 * It is an adapter hardware limitation that Packet Checksum is
2605 * mutually exclusive with RSS.
2606 */
2607 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2608 rxcsum |= IXGBE_RXCSUM_PCSD;
2609 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2610 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2611 }
2612
2613 /*
2614 * ixgbe_setup_vmdq - Setup MAC classification feature
2615 */
2616 static void
2617 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2618 {
2619 struct ixgbe_hw *hw = &ixgbe->hw;
2620 uint32_t vmdctl, i, vtctl;
2621
2622 /*
2623 * Setup the VMDq Control register, enable VMDq based on
2624 * packet destination MAC address:
2625 */
2626 switch (hw->mac.type) {
2627 case ixgbe_mac_82598EB:
2628 /*
2629 * VMDq Enable = 1;
2630 * VMDq Filter = 0; MAC filtering
2631 * Default VMDq output index = 0;
2632 */
2633 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2634 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2635 break;
2636
2637 case ixgbe_mac_82599EB:
2638 case ixgbe_mac_X540:
2639 /*
2640 * Enable VMDq-only.
2641 */
2642 vmdctl = IXGBE_MRQC_VMDQEN;
2643 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2644
2645 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2646 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2647 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2648 }
2649
2650 /*
2651 * Enable Virtualization and Replication.
2652 */
2653 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2654 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2655
2656 /*
2657 * Enable receiving packets to all VFs
2658 */
2659 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2660 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2661 break;
2662
2663 default:
2664 break;
2665 }
2666 }
2667
2668 /*
2669 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2670 */
2671 static void
2672 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2673 {
2674 struct ixgbe_hw *hw = &ixgbe->hw;
2675 uint32_t i, mrqc, rxcsum;
2676 uint32_t random;
2677 uint32_t reta;
2678 uint32_t ring_per_group;
2679 uint32_t vmdctl, vtctl;
2680
2681 /*
2682 * Fill out redirection table
2683 */
2684 reta = 0;
2685 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2686 for (i = 0; i < 128; i++) {
2687 reta = (reta << 8) | (i % ring_per_group) |
2688 ((i % ring_per_group) << 4);
2689 if ((i & 3) == 3)
2690 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2691 }
2692
2693 /*
2694 * Fill out hash function seeds with a random constant
2695 */
2696 for (i = 0; i < 10; i++) {
2697 (void) random_get_pseudo_bytes((uint8_t *)&random,
2698 sizeof (uint32_t));
2699 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2700 }
2701
2702 /*
2703 * Enable and setup RSS and VMDq
2704 */
2705 switch (hw->mac.type) {
2706 case ixgbe_mac_82598EB:
2707 /*
2708 * Enable RSS & Setup RSS Hash functions
2709 */
2710 mrqc = IXGBE_MRQC_RSSEN |
2711 IXGBE_MRQC_RSS_FIELD_IPV4 |
2712 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2713 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2714 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2715 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2716 IXGBE_MRQC_RSS_FIELD_IPV6 |
2717 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2718 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2719 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2720 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2721
2722 /*
2723 * Enable and Setup VMDq
2724 * VMDq Filter = 0; MAC filtering
2725 * Default VMDq output index = 0;
2726 */
2727 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2728 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2729 break;
2730
2731 case ixgbe_mac_82599EB:
2732 case ixgbe_mac_X540:
2733 /*
2734 * Enable RSS & Setup RSS Hash functions
2735 */
2736 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2737 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2738 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2739 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2740 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2741 IXGBE_MRQC_RSS_FIELD_IPV6 |
2742 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2743 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2744 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2745
2746 /*
2747 * Enable VMDq+RSS.
2748 */
2749 if (ixgbe->num_rx_groups > 32) {
2750 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2751 } else {
2752 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2753 }
2754
2755 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2756
2757 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2758 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2759 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2760 }
2761 break;
2762
2763 default:
2764 break;
2765
2766 }
2767
2768 /*
2769 * Disable Packet Checksum to enable RSS for multiple receive queues.
2770 * It is an adapter hardware limitation that Packet Checksum is
2771 * mutually exclusive with RSS.
2772 */
2773 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2774 rxcsum |= IXGBE_RXCSUM_PCSD;
2775 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2776 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2777
2778 if (hw->mac.type == ixgbe_mac_82599EB ||
2779 hw->mac.type == ixgbe_mac_X540) {
2780 /*
2781 * Enable Virtualization and Replication.
2782 */
2783 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2784 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2785
2786 /*
2787 * Enable receiving packets to all VFs
2788 */
2789 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2790 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2791 }
2792 }
2793
2794 /*
2795 * ixgbe_init_unicst - Initialize the unicast addresses.
2796 */
2797 static void
2798 ixgbe_init_unicst(ixgbe_t *ixgbe)
2799 {
2800 struct ixgbe_hw *hw = &ixgbe->hw;
2801 uint8_t *mac_addr;
2802 int slot;
2803 /*
2804 * Here we should consider two situations:
2805 *
2806 * 1. Chipset is initialized at the first time,
2807 * Clear all the multiple unicast addresses.
2808 *
2809 * 2. Chipset is reset
2810 * Recover the multiple unicast addresses from the
2811 * software data structure to the RAR registers.
2812 */
2813 if (!ixgbe->unicst_init) {
2814 /*
2815 * Initialize the multiple unicast addresses
2816 */
2817 ixgbe->unicst_total = hw->mac.num_rar_entries;
2818 ixgbe->unicst_avail = ixgbe->unicst_total;
2819 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2820 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2821 bzero(mac_addr, ETHERADDRL);
2822 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2823 ixgbe->unicst_addr[slot].mac.set = 0;
2824 }
2825 ixgbe->unicst_init = B_TRUE;
2826 } else {
2827 /* Re-configure the RAR registers */
2828 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2829 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2830 if (ixgbe->unicst_addr[slot].mac.set == 1) {
2831 (void) ixgbe_set_rar(hw, slot, mac_addr,
2832 ixgbe->unicst_addr[slot].mac.group_index,
2833 IXGBE_RAH_AV);
2834 } else {
2835 bzero(mac_addr, ETHERADDRL);
2836 (void) ixgbe_set_rar(hw, slot, mac_addr,
2837 NULL, NULL);
2838 }
2839 }
2840 }
2841 }
2842
2843 /*
2844 * ixgbe_unicst_find - Find the slot for the specified unicast address
2845 */
2846 int
2847 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2848 {
2849 int slot;
2850
2851 ASSERT(mutex_owned(&ixgbe->gen_lock));
2852
2853 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2854 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2855 mac_addr, ETHERADDRL) == 0)
2856 return (slot);
2857 }
2858
2859 return (-1);
2860 }
2861
2862 /*
2863 * ixgbe_multicst_add - Add a multicst address.
2864 */
2865 int
2866 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2867 {
2868 ASSERT(mutex_owned(&ixgbe->gen_lock));
2869
2870 if ((multiaddr[0] & 01) == 0) {
2871 return (EINVAL);
2872 }
2873
2874 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2875 return (ENOENT);
2876 }
2877
2878 bcopy(multiaddr,
2879 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2880 ixgbe->mcast_count++;
2881
2882 /*
2883 * Update the multicast table in the hardware
2884 */
2885 ixgbe_setup_multicst(ixgbe);
2886
2887 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2888 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2889 return (EIO);
2890 }
2891
2892 return (0);
2893 }
2894
2895 /*
2896 * ixgbe_multicst_remove - Remove a multicst address.
2897 */
2898 int
2899 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2900 {
2901 int i;
2902
2903 ASSERT(mutex_owned(&ixgbe->gen_lock));
2904
2905 for (i = 0; i < ixgbe->mcast_count; i++) {
2906 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2907 ETHERADDRL) == 0) {
2908 for (i++; i < ixgbe->mcast_count; i++) {
2909 ixgbe->mcast_table[i - 1] =
2910 ixgbe->mcast_table[i];
2911 }
2912 ixgbe->mcast_count--;
2913 break;
2914 }
2915 }
2916
2917 /*
2918 * Update the multicast table in the hardware
2919 */
2920 ixgbe_setup_multicst(ixgbe);
2921
2922 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2923 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2924 return (EIO);
2925 }
2926
2927 return (0);
2928 }
2929
2930 /*
2931 * ixgbe_setup_multicast - Setup multicast data structures.
2932 *
2933 * This routine initializes all of the multicast related structures
2934 * and save them in the hardware registers.
2935 */
2936 static void
2937 ixgbe_setup_multicst(ixgbe_t *ixgbe)
2938 {
2939 uint8_t *mc_addr_list;
2940 uint32_t mc_addr_count;
2941 struct ixgbe_hw *hw = &ixgbe->hw;
2942
2943 ASSERT(mutex_owned(&ixgbe->gen_lock));
2944
2945 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
2946
2947 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
2948 mc_addr_count = ixgbe->mcast_count;
2949
2950 /*
2951 * Update the multicast addresses to the MTA registers
2952 */
2953 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
2954 ixgbe_mc_table_itr, TRUE);
2955 }
2956
2957 /*
2958 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
2959 *
2960 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
2961 * Different chipsets may have different allowed configuration of vmdq and rss.
2962 */
2963 static void
2964 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
2965 {
2966 struct ixgbe_hw *hw = &ixgbe->hw;
2967 uint32_t ring_per_group;
2968
2969 switch (hw->mac.type) {
2970 case ixgbe_mac_82598EB:
2971 /*
2972 * 82598 supports the following combination:
2973 * vmdq no. x rss no.
2974 * [5..16] x 1
2975 * [1..4] x [1..16]
2976 * However 8 rss queue per pool (vmdq) is sufficient for
2977 * most cases.
2978 */
2979 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2980 if (ixgbe->num_rx_groups > 4) {
2981 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
2982 } else {
2983 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2984 min(8, ring_per_group);
2985 }
2986
2987 break;
2988
2989 case ixgbe_mac_82599EB:
2990 case ixgbe_mac_X540:
2991 /*
2992 * 82599 supports the following combination:
2993 * vmdq no. x rss no.
2994 * [33..64] x [1..2]
2995 * [2..32] x [1..4]
2996 * 1 x [1..16]
2997 * However 8 rss queue per pool (vmdq) is sufficient for
2998 * most cases.
2999 *
3000 * For now, treat X540 like the 82599.
3001 */
3002 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3003 if (ixgbe->num_rx_groups == 1) {
3004 ixgbe->num_rx_rings = min(8, ring_per_group);
3005 } else if (ixgbe->num_rx_groups <= 32) {
3006 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3007 min(4, ring_per_group);
3008 } else if (ixgbe->num_rx_groups <= 64) {
3009 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3010 min(2, ring_per_group);
3011 }
3012 break;
3013
3014 default:
3015 break;
3016 }
3017
3018 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3019
3020 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3021 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3022 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3023 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3024 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3025 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3026 } else {
3027 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3028 }
3029
3030 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3031 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3032 }
3033
3034 /*
3035 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3036 *
3037 * This routine gets user-configured values out of the configuration
3038 * file ixgbe.conf.
3039 *
3040 * For each configurable value, there is a minimum, a maximum, and a
3041 * default.
3042 * If user does not configure a value, use the default.
3043 * If user configures below the minimum, use the minumum.
3044 * If user configures above the maximum, use the maxumum.
3045 */
3046 static void
3047 ixgbe_get_conf(ixgbe_t *ixgbe)
3048 {
3049 struct ixgbe_hw *hw = &ixgbe->hw;
3050 uint32_t flow_control;
3051
3052 /*
3053 * ixgbe driver supports the following user configurations:
3054 *
3055 * Jumbo frame configuration:
3056 * default_mtu
3057 *
3058 * Ethernet flow control configuration:
3059 * flow_control
3060 *
3061 * Multiple rings configurations:
3062 * tx_queue_number
3063 * tx_ring_size
3064 * rx_queue_number
3065 * rx_ring_size
3066 *
3067 * Call ixgbe_get_prop() to get the value for a specific
3068 * configuration parameter.
3069 */
3070
3071 /*
3072 * Jumbo frame configuration - max_frame_size controls host buffer
3073 * allocation, so includes MTU, ethernet header, vlan tag and
3074 * frame check sequence.
3075 */
3076 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3077 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3078
3079 ixgbe->max_frame_size = ixgbe->default_mtu +
3080 sizeof (struct ether_vlan_header) + ETHERFCSL;
3081
3082 /*
3083 * Ethernet flow control configuration
3084 */
3085 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3086 ixgbe_fc_none, 3, ixgbe_fc_none);
3087 if (flow_control == 3)
3088 flow_control = ixgbe_fc_default;
3089
3090 /*
3091 * fc.requested mode is what the user requests. After autoneg,
3092 * fc.current_mode will be the flow_control mode that was negotiated.
3093 */
3094 hw->fc.requested_mode = flow_control;
3095
3096 /*
3097 * Multiple rings configurations
3098 */
3099 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3100 ixgbe->capab->min_tx_que_num,
3101 ixgbe->capab->max_tx_que_num,
3102 ixgbe->capab->def_tx_que_num);
3103 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3104 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3105
3106 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3107 ixgbe->capab->min_rx_que_num,
3108 ixgbe->capab->max_rx_que_num,
3109 ixgbe->capab->def_rx_que_num);
3110 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3111 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3112
3113 /*
3114 * Multiple groups configuration
3115 */
3116 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3117 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3118 ixgbe->capab->def_rx_grp_num);
3119
3120 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3121 0, 1, DEFAULT_MR_ENABLE);
3122
3123 if (ixgbe->mr_enable == B_FALSE) {
3124 ixgbe->num_tx_rings = 1;
3125 ixgbe->num_rx_rings = 1;
3126 ixgbe->num_rx_groups = 1;
3127 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3128 } else {
3129 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3130 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3131 /*
3132 * The combination of num_rx_rings and num_rx_groups
3133 * may be not supported by h/w. We need to adjust
3134 * them to appropriate values.
3135 */
3136 ixgbe_setup_vmdq_rss_conf(ixgbe);
3137 }
3138
3139 /*
3140 * Tunable used to force an interrupt type. The only use is
3141 * for testing of the lesser interrupt types.
3142 * 0 = don't force interrupt type
3143 * 1 = force interrupt type MSI-X
3144 * 2 = force interrupt type MSI
3145 * 3 = force interrupt type Legacy
3146 */
3147 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3148 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3149
3150 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3151 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3152 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3153 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3154 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3155 0, 1, DEFAULT_LSO_ENABLE);
3156 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3157 0, 1, DEFAULT_LRO_ENABLE);
3158 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3159 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3160 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3161 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3162
3163 /* Head Write Back not recommended for 82599 and X540 */
3164 if (hw->mac.type == ixgbe_mac_82599EB ||
3165 hw->mac.type == ixgbe_mac_X540) {
3166 ixgbe->tx_head_wb_enable = B_FALSE;
3167 }
3168
3169 /*
3170 * ixgbe LSO needs the tx h/w checksum support.
3171 * LSO will be disabled if tx h/w checksum is not
3172 * enabled.
3173 */
3174 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3175 ixgbe->lso_enable = B_FALSE;
3176 }
3177
3178 /*
3179 * ixgbe LRO needs the rx h/w checksum support.
3180 * LRO will be disabled if rx h/w checksum is not
3181 * enabled.
3182 */
3183 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3184 ixgbe->lro_enable = B_FALSE;
3185 }
3186
3187 /*
3188 * ixgbe LRO only been supported by 82599 and X540 now
3189 */
3190 if (hw->mac.type == ixgbe_mac_82598EB) {
3191 ixgbe->lro_enable = B_FALSE;
3192 }
3193 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3194 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3195 DEFAULT_TX_COPY_THRESHOLD);
3196 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3197 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3198 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3199 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3200 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3201 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3202 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3203 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3204 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3205
3206 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3207 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3208 DEFAULT_RX_COPY_THRESHOLD);
3209 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3210 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3211 DEFAULT_RX_LIMIT_PER_INTR);
3212
3213 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3214 ixgbe->capab->min_intr_throttle,
3215 ixgbe->capab->max_intr_throttle,
3216 ixgbe->capab->def_intr_throttle);
3217 /*
3218 * 82599 and X540 require the interrupt throttling rate is
3219 * a multiple of 8. This is enforced by the register
3220 * definiton.
3221 */
3222 if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540)
3223 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3224
3225 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3226 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3227 }
3228
3229 static void
3230 ixgbe_init_params(ixgbe_t *ixgbe)
3231 {
3232 ixgbe->param_en_10000fdx_cap = 1;
3233 ixgbe->param_en_1000fdx_cap = 1;
3234 ixgbe->param_en_100fdx_cap = 1;
3235 ixgbe->param_adv_10000fdx_cap = 1;
3236 ixgbe->param_adv_1000fdx_cap = 1;
3237 ixgbe->param_adv_100fdx_cap = 1;
3238
3239 ixgbe->param_pause_cap = 1;
3240 ixgbe->param_asym_pause_cap = 1;
3241 ixgbe->param_rem_fault = 0;
3242
3243 ixgbe->param_adv_autoneg_cap = 1;
3244 ixgbe->param_adv_pause_cap = 1;
3245 ixgbe->param_adv_asym_pause_cap = 1;
3246 ixgbe->param_adv_rem_fault = 0;
3247
3248 ixgbe->param_lp_10000fdx_cap = 0;
3249 ixgbe->param_lp_1000fdx_cap = 0;
3250 ixgbe->param_lp_100fdx_cap = 0;
3251 ixgbe->param_lp_autoneg_cap = 0;
3252 ixgbe->param_lp_pause_cap = 0;
3253 ixgbe->param_lp_asym_pause_cap = 0;
3254 ixgbe->param_lp_rem_fault = 0;
3255 }
3256
3257 /*
3258 * ixgbe_get_prop - Get a property value out of the configuration file
3259 * ixgbe.conf.
3260 *
3261 * Caller provides the name of the property, a default value, a minimum
3262 * value, and a maximum value.
3263 *
3264 * Return configured value of the property, with default, minimum and
3265 * maximum properly applied.
3266 */
3267 static int
3268 ixgbe_get_prop(ixgbe_t *ixgbe,
3269 char *propname, /* name of the property */
3270 int minval, /* minimum acceptable value */
3271 int maxval, /* maximim acceptable value */
3272 int defval) /* default value */
3273 {
3274 int value;
3275
3276 /*
3277 * Call ddi_prop_get_int() to read the conf settings
3278 */
3279 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3280 DDI_PROP_DONTPASS, propname, defval);
3281 if (value > maxval)
3282 value = maxval;
3283
3284 if (value < minval)
3285 value = minval;
3286
3287 return (value);
3288 }
3289
3290 /*
3291 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3292 */
3293 int
3294 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3295 {
3296 u32 autoneg_advertised = 0;
3297
3298 /*
3299 * No half duplex support with 10Gb parts
3300 */
3301 if (ixgbe->param_adv_10000fdx_cap == 1)
3302 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3303
3304 if (ixgbe->param_adv_1000fdx_cap == 1)
3305 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3306
3307 if (ixgbe->param_adv_100fdx_cap == 1)
3308 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3309
3310 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3311 ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3312 "to autonegotiation with full link capabilities.");
3313
3314 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3315 IXGBE_LINK_SPEED_1GB_FULL |
3316 IXGBE_LINK_SPEED_100_FULL;
3317 }
3318
3319 if (setup_hw) {
3320 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3321 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) {
3322 ixgbe_notice(ixgbe, "Setup link failed on this "
3323 "device.");
3324 return (IXGBE_FAILURE);
3325 }
3326 }
3327
3328 return (IXGBE_SUCCESS);
3329 }
3330
3331 /*
3332 * ixgbe_driver_link_check - Link status processing.
3333 *
3334 * This function can be called in both kernel context and interrupt context
3335 */
3336 static void
3337 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3338 {
3339 struct ixgbe_hw *hw = &ixgbe->hw;
3340 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3341 boolean_t link_up = B_FALSE;
3342 boolean_t link_changed = B_FALSE;
3343
3344 ASSERT(mutex_owned(&ixgbe->gen_lock));
3345
3346 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3347 if (link_up) {
3348 ixgbe->link_check_complete = B_TRUE;
3349
3350 /* Link is up, enable flow control settings */
3351 (void) ixgbe_fc_enable(hw);
3352
3353 /*
3354 * The Link is up, check whether it was marked as down earlier
3355 */
3356 if (ixgbe->link_state != LINK_STATE_UP) {
3357 switch (speed) {
3358 case IXGBE_LINK_SPEED_10GB_FULL:
3359 ixgbe->link_speed = SPEED_10GB;
3360 break;
3361 case IXGBE_LINK_SPEED_1GB_FULL:
3362 ixgbe->link_speed = SPEED_1GB;
3363 break;
3364 case IXGBE_LINK_SPEED_100_FULL:
3365 ixgbe->link_speed = SPEED_100;
3366 }
3367 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3368 ixgbe->link_state = LINK_STATE_UP;
3369 link_changed = B_TRUE;
3370 }
3371 } else {
3372 if (ixgbe->link_check_complete == B_TRUE ||
3373 (ixgbe->link_check_complete == B_FALSE &&
3374 gethrtime() >= ixgbe->link_check_hrtime)) {
3375 /*
3376 * The link is really down
3377 */
3378 ixgbe->link_check_complete = B_TRUE;
3379
3380 if (ixgbe->link_state != LINK_STATE_DOWN) {
3381 ixgbe->link_speed = 0;
3382 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3383 ixgbe->link_state = LINK_STATE_DOWN;
3384 link_changed = B_TRUE;
3385 }
3386 }
3387 }
3388
3389 /*
3390 * If we are in an interrupt context, need to re-enable the
3391 * interrupt, which was automasked
3392 */
3393 if (servicing_interrupt() != 0) {
3394 ixgbe->eims |= IXGBE_EICR_LSC;
3395 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3396 }
3397
3398 if (link_changed) {
3399 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3400 }
3401 }
3402
3403 /*
3404 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3405 */
3406 static void
3407 ixgbe_sfp_check(void *arg)
3408 {
3409 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3410 uint32_t eicr = ixgbe->eicr;
3411 struct ixgbe_hw *hw = &ixgbe->hw;
3412
3413 mutex_enter(&ixgbe->gen_lock);
3414 if (eicr & IXGBE_EICR_GPI_SDP1) {
3415 /* clear the interrupt */
3416 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
3417
3418 /* if link up, do multispeed fiber setup */
3419 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3420 B_TRUE, B_TRUE);
3421 ixgbe_driver_link_check(ixgbe);
3422 ixgbe_get_hw_state(ixgbe);
3423 } else if (eicr & IXGBE_EICR_GPI_SDP2) {
3424 /* clear the interrupt */
3425 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
3426
3427 /* if link up, do sfp module setup */
3428 (void) hw->mac.ops.setup_sfp(hw);
3429
3430 /* do multispeed fiber setup */
3431 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3432 B_TRUE, B_TRUE);
3433 ixgbe_driver_link_check(ixgbe);
3434 ixgbe_get_hw_state(ixgbe);
3435 }
3436 mutex_exit(&ixgbe->gen_lock);
3437
3438 /*
3439 * We need to fully re-check the link later.
3440 */
3441 ixgbe->link_check_complete = B_FALSE;
3442 ixgbe->link_check_hrtime = gethrtime() +
3443 (IXGBE_LINK_UP_TIME * 100000000ULL);
3444 }
3445
3446 /*
3447 * ixgbe_overtemp_check - overtemp module processing done in taskq
3448 *
3449 * This routine will only be called on adapters with temperature sensor.
3450 * The indication of over-temperature can be either SDP0 interrupt or the link
3451 * status change interrupt.
3452 */
3453 static void
3454 ixgbe_overtemp_check(void *arg)
3455 {
3456 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3457 struct ixgbe_hw *hw = &ixgbe->hw;
3458 uint32_t eicr = ixgbe->eicr;
3459 ixgbe_link_speed speed;
3460 boolean_t link_up;
3461
3462 mutex_enter(&ixgbe->gen_lock);
3463
3464 /* make sure we know current state of link */
3465 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3466
3467 /* check over-temp condition */
3468 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
3469 (eicr & IXGBE_EICR_LSC)) {
3470 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3471 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3472
3473 /*
3474 * Disable the adapter interrupts
3475 */
3476 ixgbe_disable_adapter_interrupts(ixgbe);
3477
3478 /*
3479 * Disable Rx/Tx units
3480 */
3481 (void) ixgbe_stop_adapter(hw);
3482
3483 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3484 ixgbe_error(ixgbe,
3485 "Problem: Network adapter has been stopped "
3486 "because it has overheated");
3487 ixgbe_error(ixgbe,
3488 "Action: Restart the computer. "
3489 "If the problem persists, power off the system "
3490 "and replace the adapter");
3491 }
3492 }
3493
3494 /* write to clear the interrupt */
3495 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3496
3497 mutex_exit(&ixgbe->gen_lock);
3498 }
3499
3500 /*
3501 * ixgbe_link_timer - timer for link status detection
3502 */
3503 static void
3504 ixgbe_link_timer(void *arg)
3505 {
3506 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3507
3508 mutex_enter(&ixgbe->gen_lock);
3509 ixgbe_driver_link_check(ixgbe);
3510 mutex_exit(&ixgbe->gen_lock);
3511 }
3512
3513 /*
3514 * ixgbe_local_timer - Driver watchdog function.
3515 *
3516 * This function will handle the transmit stall check and other routines.
3517 */
3518 static void
3519 ixgbe_local_timer(void *arg)
3520 {
3521 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3522
3523 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3524 goto out;
3525
3526 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3527 ixgbe->reset_count++;
3528 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3529 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3530 goto out;
3531 }
3532
3533 if (ixgbe_stall_check(ixgbe)) {
3534 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3535 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3536
3537 ixgbe->reset_count++;
3538 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3539 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3540 }
3541
3542 out:
3543 ixgbe_restart_watchdog_timer(ixgbe);
3544 }
3545
3546 /*
3547 * ixgbe_stall_check - Check for transmit stall.
3548 *
3549 * This function checks if the adapter is stalled (in transmit).
3550 *
3551 * It is called each time the watchdog timeout is invoked.
3552 * If the transmit descriptor reclaim continuously fails,
3553 * the watchdog value will increment by 1. If the watchdog
3554 * value exceeds the threshold, the ixgbe is assumed to
3555 * have stalled and need to be reset.
3556 */
3557 static boolean_t
3558 ixgbe_stall_check(ixgbe_t *ixgbe)
3559 {
3560 ixgbe_tx_ring_t *tx_ring;
3561 boolean_t result;
3562 int i;
3563
3564 if (ixgbe->link_state != LINK_STATE_UP)
3565 return (B_FALSE);
3566
3567 /*
3568 * If any tx ring is stalled, we'll reset the chipset
3569 */
3570 result = B_FALSE;
3571 for (i = 0; i < ixgbe->num_tx_rings; i++) {
3572 tx_ring = &ixgbe->tx_rings[i];
3573 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3574 tx_ring->tx_recycle(tx_ring);
3575 }
3576
3577 if (tx_ring->recycle_fail > 0)
3578 tx_ring->stall_watchdog++;
3579 else
3580 tx_ring->stall_watchdog = 0;
3581
3582 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3583 result = B_TRUE;
3584 break;
3585 }
3586 }
3587
3588 if (result) {
3589 tx_ring->stall_watchdog = 0;
3590 tx_ring->recycle_fail = 0;
3591 }
3592
3593 return (result);
3594 }
3595
3596
3597 /*
3598 * is_valid_mac_addr - Check if the mac address is valid.
3599 */
3600 static boolean_t
3601 is_valid_mac_addr(uint8_t *mac_addr)
3602 {
3603 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3604 const uint8_t addr_test2[6] =
3605 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3606
3607 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3608 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3609 return (B_FALSE);
3610
3611 return (B_TRUE);
3612 }
3613
3614 static boolean_t
3615 ixgbe_find_mac_address(ixgbe_t *ixgbe)
3616 {
3617 #ifdef __sparc
3618 struct ixgbe_hw *hw = &ixgbe->hw;
3619 uchar_t *bytes;
3620 struct ether_addr sysaddr;
3621 uint_t nelts;
3622 int err;
3623 boolean_t found = B_FALSE;
3624
3625 /*
3626 * The "vendor's factory-set address" may already have
3627 * been extracted from the chip, but if the property
3628 * "local-mac-address" is set we use that instead.
3629 *
3630 * We check whether it looks like an array of 6
3631 * bytes (which it should, if OBP set it). If we can't
3632 * make sense of it this way, we'll ignore it.
3633 */
3634 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3635 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3636 if (err == DDI_PROP_SUCCESS) {
3637 if (nelts == ETHERADDRL) {
3638 while (nelts--)
3639 hw->mac.addr[nelts] = bytes[nelts];
3640 found = B_TRUE;
3641 }
3642 ddi_prop_free(bytes);
3643 }
3644
3645 /*
3646 * Look up the OBP property "local-mac-address?". If the user has set
3647 * 'local-mac-address? = false', use "the system address" instead.
3648 */
3649 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3650 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3651 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3652 if (localetheraddr(NULL, &sysaddr) != 0) {
3653 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3654 found = B_TRUE;
3655 }
3656 }
3657 ddi_prop_free(bytes);
3658 }
3659
3660 /*
3661 * Finally(!), if there's a valid "mac-address" property (created
3662 * if we netbooted from this interface), we must use this instead
3663 * of any of the above to ensure that the NFS/install server doesn't
3664 * get confused by the address changing as Solaris takes over!
3665 */
3666 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3667 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3668 if (err == DDI_PROP_SUCCESS) {
3669 if (nelts == ETHERADDRL) {
3670 while (nelts--)
3671 hw->mac.addr[nelts] = bytes[nelts];
3672 found = B_TRUE;
3673 }
3674 ddi_prop_free(bytes);
3675 }
3676
3677 if (found) {
3678 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3679 return (B_TRUE);
3680 }
3681 #else
3682 _NOTE(ARGUNUSED(ixgbe));
3683 #endif
3684
3685 return (B_TRUE);
3686 }
3687
3688 #pragma inline(ixgbe_arm_watchdog_timer)
3689 static void
3690 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
3691 {
3692 /*
3693 * Fire a watchdog timer
3694 */
3695 ixgbe->watchdog_tid =
3696 timeout(ixgbe_local_timer,
3697 (void *)ixgbe, 1 * drv_usectohz(1000000));
3698
3699 }
3700
3701 /*
3702 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
3703 */
3704 void
3705 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
3706 {
3707 mutex_enter(&ixgbe->watchdog_lock);
3708
3709 if (!ixgbe->watchdog_enable) {
3710 ixgbe->watchdog_enable = B_TRUE;
3711 ixgbe->watchdog_start = B_TRUE;
3712 ixgbe_arm_watchdog_timer(ixgbe);
3713 }
3714
3715 mutex_exit(&ixgbe->watchdog_lock);
3716 }
3717
3718 /*
3719 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
3720 */
3721 void
3722 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
3723 {
3724 timeout_id_t tid;
3725
3726 mutex_enter(&ixgbe->watchdog_lock);
3727
3728 ixgbe->watchdog_enable = B_FALSE;
3729 ixgbe->watchdog_start = B_FALSE;
3730 tid = ixgbe->watchdog_tid;
3731 ixgbe->watchdog_tid = 0;
3732
3733 mutex_exit(&ixgbe->watchdog_lock);
3734
3735 if (tid != 0)
3736 (void) untimeout(tid);
3737 }
3738
3739 /*
3740 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
3741 */
3742 void
3743 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
3744 {
3745 mutex_enter(&ixgbe->watchdog_lock);
3746
3747 if (ixgbe->watchdog_enable) {
3748 if (!ixgbe->watchdog_start) {
3749 ixgbe->watchdog_start = B_TRUE;
3750 ixgbe_arm_watchdog_timer(ixgbe);
3751 }
3752 }
3753
3754 mutex_exit(&ixgbe->watchdog_lock);
3755 }
3756
3757 /*
3758 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
3759 */
3760 static void
3761 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
3762 {
3763 mutex_enter(&ixgbe->watchdog_lock);
3764
3765 if (ixgbe->watchdog_start)
3766 ixgbe_arm_watchdog_timer(ixgbe);
3767
3768 mutex_exit(&ixgbe->watchdog_lock);
3769 }
3770
3771 /*
3772 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
3773 */
3774 void
3775 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
3776 {
3777 timeout_id_t tid;
3778
3779 mutex_enter(&ixgbe->watchdog_lock);
3780
3781 ixgbe->watchdog_start = B_FALSE;
3782 tid = ixgbe->watchdog_tid;
3783 ixgbe->watchdog_tid = 0;
3784
3785 mutex_exit(&ixgbe->watchdog_lock);
3786
3787 if (tid != 0)
3788 (void) untimeout(tid);
3789 }
3790
3791 /*
3792 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
3793 */
3794 static void
3795 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
3796 {
3797 struct ixgbe_hw *hw = &ixgbe->hw;
3798
3799 /*
3800 * mask all interrupts off
3801 */
3802 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
3803
3804 /*
3805 * for MSI-X, also disable autoclear
3806 */
3807 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3808 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
3809 }
3810
3811 IXGBE_WRITE_FLUSH(hw);
3812 }
3813
3814 /*
3815 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
3816 */
3817 static void
3818 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
3819 {
3820 struct ixgbe_hw *hw = &ixgbe->hw;
3821 uint32_t eiac, eiam;
3822 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3823
3824 /* interrupt types to enable */
3825 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
3826 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
3827 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
3828
3829 /* enable automask on "other" causes that this adapter can generate */
3830 eiam = ixgbe->capab->other_intr;
3831
3832 /*
3833 * msi-x mode
3834 */
3835 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3836 /* enable autoclear but not on bits 29:20 */
3837 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3838
3839 /* general purpose interrupt enable */
3840 gpie |= (IXGBE_GPIE_MSIX_MODE
3841 | IXGBE_GPIE_PBA_SUPPORT
3842 | IXGBE_GPIE_OCD
3843 | IXGBE_GPIE_EIAME);
3844 /*
3845 * non-msi-x mode
3846 */
3847 } else {
3848
3849 /* disable autoclear, leave gpie at default */
3850 eiac = 0;
3851
3852 /*
3853 * General purpose interrupt enable.
3854 * For 82599 or X540, extended interrupt automask enable
3855 * only in MSI or MSI-X mode
3856 */
3857 if ((hw->mac.type == ixgbe_mac_82598EB) ||
3858 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3859 gpie |= IXGBE_GPIE_EIAME;
3860 }
3861 }
3862
3863 /* Enable specific "other" interrupt types */
3864 switch (hw->mac.type) {
3865 case ixgbe_mac_82598EB:
3866 gpie |= ixgbe->capab->other_gpie;
3867 break;
3868
3869 case ixgbe_mac_82599EB:
3870 case ixgbe_mac_X540:
3871 gpie |= ixgbe->capab->other_gpie;
3872
3873 /* Enable RSC Delay 8us when LRO enabled */
3874 if (ixgbe->lro_enable) {
3875 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3876 }
3877 break;
3878
3879 default:
3880 break;
3881 }
3882
3883 /* write to interrupt control registers */
3884 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3885 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3886 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3887 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3888 IXGBE_WRITE_FLUSH(hw);
3889 }
3890
3891 /*
3892 * ixgbe_loopback_ioctl - Loopback support.
3893 */
3894 enum ioc_reply
3895 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
3896 {
3897 lb_info_sz_t *lbsp;
3898 lb_property_t *lbpp;
3899 uint32_t *lbmp;
3900 uint32_t size;
3901 uint32_t value;
3902
3903 if (mp->b_cont == NULL)
3904 return (IOC_INVAL);
3905
3906 switch (iocp->ioc_cmd) {
3907 default:
3908 return (IOC_INVAL);
3909
3910 case LB_GET_INFO_SIZE:
3911 size = sizeof (lb_info_sz_t);
3912 if (iocp->ioc_count != size)
3913 return (IOC_INVAL);
3914
3915 value = sizeof (lb_normal);
3916 value += sizeof (lb_mac);
3917 value += sizeof (lb_external);
3918
3919 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3920 *lbsp = value;
3921 break;
3922
3923 case LB_GET_INFO:
3924 value = sizeof (lb_normal);
3925 value += sizeof (lb_mac);
3926 value += sizeof (lb_external);
3927
3928 size = value;
3929 if (iocp->ioc_count != size)
3930 return (IOC_INVAL);
3931
3932 value = 0;
3933 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3934
3935 lbpp[value++] = lb_normal;
3936 lbpp[value++] = lb_mac;
3937 lbpp[value++] = lb_external;
3938 break;
3939
3940 case LB_GET_MODE:
3941 size = sizeof (uint32_t);
3942 if (iocp->ioc_count != size)
3943 return (IOC_INVAL);
3944
3945 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3946 *lbmp = ixgbe->loopback_mode;
3947 break;
3948
3949 case LB_SET_MODE:
3950 size = 0;
3951 if (iocp->ioc_count != sizeof (uint32_t))
3952 return (IOC_INVAL);
3953
3954 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3955 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
3956 return (IOC_INVAL);
3957 break;
3958 }
3959
3960 iocp->ioc_count = size;
3961 iocp->ioc_error = 0;
3962
3963 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3964 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3965 return (IOC_INVAL);
3966 }
3967
3968 return (IOC_REPLY);
3969 }
3970
3971 /*
3972 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
3973 */
3974 static boolean_t
3975 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
3976 {
3977 if (mode == ixgbe->loopback_mode)
3978 return (B_TRUE);
3979
3980 ixgbe->loopback_mode = mode;
3981
3982 if (mode == IXGBE_LB_NONE) {
3983 /*
3984 * Reset the chip
3985 */
3986 (void) ixgbe_reset(ixgbe);
3987 return (B_TRUE);
3988 }
3989
3990 mutex_enter(&ixgbe->gen_lock);
3991
3992 switch (mode) {
3993 default:
3994 mutex_exit(&ixgbe->gen_lock);
3995 return (B_FALSE);
3996
3997 case IXGBE_LB_EXTERNAL:
3998 break;
3999
4000 case IXGBE_LB_INTERNAL_MAC:
4001 ixgbe_set_internal_mac_loopback(ixgbe);
4002 break;
4003 }
4004
4005 mutex_exit(&ixgbe->gen_lock);
4006
4007 return (B_TRUE);
4008 }
4009
4010 /*
4011 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4012 */
4013 static void
4014 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4015 {
4016 struct ixgbe_hw *hw;
4017 uint32_t reg;
4018 uint8_t atlas;
4019
4020 hw = &ixgbe->hw;
4021
4022 /*
4023 * Setup MAC loopback
4024 */
4025 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4026 reg |= IXGBE_HLREG0_LPBK;
4027 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4028
4029 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4030 reg &= ~IXGBE_AUTOC_LMS_MASK;
4031 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4032
4033 /*
4034 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4035 */
4036 switch (hw->mac.type) {
4037 case ixgbe_mac_82598EB:
4038 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4039 &atlas);
4040 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4041 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4042 atlas);
4043
4044 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4045 &atlas);
4046 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4047 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4048 atlas);
4049
4050 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4051 &atlas);
4052 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4053 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4054 atlas);
4055
4056 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4057 &atlas);
4058 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4059 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4060 atlas);
4061 break;
4062
4063 case ixgbe_mac_82599EB:
4064 case ixgbe_mac_X540:
4065 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4066 reg |= (IXGBE_AUTOC_FLU |
4067 IXGBE_AUTOC_10G_KX4);
4068 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4069
4070 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4071 B_FALSE, B_TRUE);
4072 break;
4073
4074 default:
4075 break;
4076 }
4077 }
4078
4079 #pragma inline(ixgbe_intr_rx_work)
4080 /*
4081 * ixgbe_intr_rx_work - RX processing of ISR.
4082 */
4083 static void
4084 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4085 {
4086 mblk_t *mp;
4087
4088 mutex_enter(&rx_ring->rx_lock);
4089
4090 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4091 mutex_exit(&rx_ring->rx_lock);
4092
4093 if (mp != NULL)
4094 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4095 rx_ring->ring_gen_num);
4096 }
4097
4098 #pragma inline(ixgbe_intr_tx_work)
4099 /*
4100 * ixgbe_intr_tx_work - TX processing of ISR.
4101 */
4102 static void
4103 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4104 {
4105 ixgbe_t *ixgbe = tx_ring->ixgbe;
4106
4107 /*
4108 * Recycle the tx descriptors
4109 */
4110 tx_ring->tx_recycle(tx_ring);
4111
4112 /*
4113 * Schedule the re-transmit
4114 */
4115 if (tx_ring->reschedule &&
4116 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4117 tx_ring->reschedule = B_FALSE;
4118 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4119 tx_ring->ring_handle);
4120 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4121 }
4122 }
4123
4124 #pragma inline(ixgbe_intr_other_work)
4125 /*
4126 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4127 */
4128 static void
4129 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4130 {
4131 ASSERT(mutex_owned(&ixgbe->gen_lock));
4132
4133 /*
4134 * handle link status change
4135 */
4136 if (eicr & IXGBE_EICR_LSC) {
4137 ixgbe_driver_link_check(ixgbe);
4138 ixgbe_get_hw_state(ixgbe);
4139 }
4140
4141 /*
4142 * check for fan failure on adapters with fans
4143 */
4144 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4145 (eicr & IXGBE_EICR_GPI_SDP1)) {
4146 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4147
4148 /*
4149 * Disable the adapter interrupts
4150 */
4151 ixgbe_disable_adapter_interrupts(ixgbe);
4152
4153 /*
4154 * Disable Rx/Tx units
4155 */
4156 (void) ixgbe_stop_adapter(&ixgbe->hw);
4157
4158 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4159 ixgbe_error(ixgbe,
4160 "Problem: Network adapter has been stopped "
4161 "because the fan has stopped.\n");
4162 ixgbe_error(ixgbe,
4163 "Action: Replace the adapter.\n");
4164
4165 /* re-enable the interrupt, which was automasked */
4166 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4167 }
4168
4169 /*
4170 * Do SFP check for adapters with hot-plug capability
4171 */
4172 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4173 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) {
4174 ixgbe->eicr = eicr;
4175 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4176 ixgbe_sfp_check, (void *)ixgbe,
4177 DDI_NOSLEEP)) != DDI_SUCCESS) {
4178 ixgbe_log(ixgbe, "No memory available to dispatch "
4179 "taskq for SFP check");
4180 }
4181 }
4182
4183 /*
4184 * Do over-temperature check for adapters with temp sensor
4185 */
4186 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4187 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
4188 ixgbe->eicr = eicr;
4189 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4190 ixgbe_overtemp_check, (void *)ixgbe,
4191 DDI_NOSLEEP)) != DDI_SUCCESS) {
4192 ixgbe_log(ixgbe, "No memory available to dispatch "
4193 "taskq for overtemp check");
4194 }
4195 }
4196 }
4197
4198 /*
4199 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4200 */
4201 static uint_t
4202 ixgbe_intr_legacy(void *arg1, void *arg2)
4203 {
4204 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4205 struct ixgbe_hw *hw = &ixgbe->hw;
4206 ixgbe_tx_ring_t *tx_ring;
4207 ixgbe_rx_ring_t *rx_ring;
4208 uint32_t eicr;
4209 mblk_t *mp;
4210 boolean_t tx_reschedule;
4211 uint_t result;
4212
4213 _NOTE(ARGUNUSED(arg2));
4214
4215 mutex_enter(&ixgbe->gen_lock);
4216 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4217 mutex_exit(&ixgbe->gen_lock);
4218 return (DDI_INTR_UNCLAIMED);
4219 }
4220
4221 mp = NULL;
4222 tx_reschedule = B_FALSE;
4223
4224 /*
4225 * Any bit set in eicr: claim this interrupt
4226 */
4227 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4228
4229 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4230 mutex_exit(&ixgbe->gen_lock);
4231 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4232 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4233 return (DDI_INTR_CLAIMED);
4234 }
4235
4236 if (eicr) {
4237 /*
4238 * For legacy interrupt, we have only one interrupt,
4239 * so we have only one rx ring and one tx ring enabled.
4240 */
4241 ASSERT(ixgbe->num_rx_rings == 1);
4242 ASSERT(ixgbe->num_tx_rings == 1);
4243
4244 /*
4245 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4246 */
4247 if (eicr & 0x1) {
4248 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4249 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4250 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4251 /*
4252 * Clean the rx descriptors
4253 */
4254 rx_ring = &ixgbe->rx_rings[0];
4255 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4256 }
4257
4258 /*
4259 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4260 */
4261 if (eicr & 0x2) {
4262 /*
4263 * Recycle the tx descriptors
4264 */
4265 tx_ring = &ixgbe->tx_rings[0];
4266 tx_ring->tx_recycle(tx_ring);
4267
4268 /*
4269 * Schedule the re-transmit
4270 */
4271 tx_reschedule = (tx_ring->reschedule &&
4272 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4273 }
4274
4275 /* any interrupt type other than tx/rx */
4276 if (eicr & ixgbe->capab->other_intr) {
4277 switch (hw->mac.type) {
4278 case ixgbe_mac_82598EB:
4279 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4280 break;
4281
4282 case ixgbe_mac_82599EB:
4283 case ixgbe_mac_X540:
4284 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4285 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4286 break;
4287
4288 default:
4289 break;
4290 }
4291 ixgbe_intr_other_work(ixgbe, eicr);
4292 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4293 }
4294
4295 mutex_exit(&ixgbe->gen_lock);
4296
4297 result = DDI_INTR_CLAIMED;
4298 } else {
4299 mutex_exit(&ixgbe->gen_lock);
4300
4301 /*
4302 * No interrupt cause bits set: don't claim this interrupt.
4303 */
4304 result = DDI_INTR_UNCLAIMED;
4305 }
4306
4307 /* re-enable the interrupts which were automasked */
4308 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4309
4310 /*
4311 * Do the following work outside of the gen_lock
4312 */
4313 if (mp != NULL) {
4314 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4315 rx_ring->ring_gen_num);
4316 }
4317
4318 if (tx_reschedule) {
4319 tx_ring->reschedule = B_FALSE;
4320 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4321 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4322 }
4323
4324 return (result);
4325 }
4326
4327 /*
4328 * ixgbe_intr_msi - Interrupt handler for MSI.
4329 */
4330 static uint_t
4331 ixgbe_intr_msi(void *arg1, void *arg2)
4332 {
4333 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4334 struct ixgbe_hw *hw = &ixgbe->hw;
4335 uint32_t eicr;
4336
4337 _NOTE(ARGUNUSED(arg2));
4338
4339 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4340
4341 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4342 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4343 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4344 return (DDI_INTR_CLAIMED);
4345 }
4346
4347 /*
4348 * For MSI interrupt, we have only one vector,
4349 * so we have only one rx ring and one tx ring enabled.
4350 */
4351 ASSERT(ixgbe->num_rx_rings == 1);
4352 ASSERT(ixgbe->num_tx_rings == 1);
4353
4354 /*
4355 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4356 */
4357 if (eicr & 0x1) {
4358 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4359 }
4360
4361 /*
4362 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4363 */
4364 if (eicr & 0x2) {
4365 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4366 }
4367
4368 /* any interrupt type other than tx/rx */
4369 if (eicr & ixgbe->capab->other_intr) {
4370 mutex_enter(&ixgbe->gen_lock);
4371 switch (hw->mac.type) {
4372 case ixgbe_mac_82598EB:
4373 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4374 break;
4375
4376 case ixgbe_mac_82599EB:
4377 case ixgbe_mac_X540:
4378 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4379 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4380 break;
4381
4382 default:
4383 break;
4384 }
4385 ixgbe_intr_other_work(ixgbe, eicr);
4386 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4387 mutex_exit(&ixgbe->gen_lock);
4388 }
4389
4390 /* re-enable the interrupts which were automasked */
4391 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4392
4393 return (DDI_INTR_CLAIMED);
4394 }
4395
4396 /*
4397 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4398 */
4399 static uint_t
4400 ixgbe_intr_msix(void *arg1, void *arg2)
4401 {
4402 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4403 ixgbe_t *ixgbe = vect->ixgbe;
4404 struct ixgbe_hw *hw = &ixgbe->hw;
4405 uint32_t eicr;
4406 int r_idx = 0;
4407
4408 _NOTE(ARGUNUSED(arg2));
4409
4410 /*
4411 * Clean each rx ring that has its bit set in the map
4412 */
4413 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4414 while (r_idx >= 0) {
4415 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4416 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4417 (ixgbe->num_rx_rings - 1));
4418 }
4419
4420 /*
4421 * Clean each tx ring that has its bit set in the map
4422 */
4423 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4424 while (r_idx >= 0) {
4425 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4426 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4427 (ixgbe->num_tx_rings - 1));
4428 }
4429
4430
4431 /*
4432 * Clean other interrupt (link change) that has its bit set in the map
4433 */
4434 if (BT_TEST(vect->other_map, 0) == 1) {
4435 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4436
4437 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4438 DDI_FM_OK) {
4439 ddi_fm_service_impact(ixgbe->dip,
4440 DDI_SERVICE_DEGRADED);
4441 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4442 return (DDI_INTR_CLAIMED);
4443 }
4444
4445 /*
4446 * Check "other" cause bits: any interrupt type other than tx/rx
4447 */
4448 if (eicr & ixgbe->capab->other_intr) {
4449 mutex_enter(&ixgbe->gen_lock);
4450 switch (hw->mac.type) {
4451 case ixgbe_mac_82598EB:
4452 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4453 ixgbe_intr_other_work(ixgbe, eicr);
4454 break;
4455
4456 case ixgbe_mac_82599EB:
4457 case ixgbe_mac_X540:
4458 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4459 ixgbe_intr_other_work(ixgbe, eicr);
4460 break;
4461
4462 default:
4463 break;
4464 }
4465 mutex_exit(&ixgbe->gen_lock);
4466 }
4467
4468 /* re-enable the interrupts which were automasked */
4469 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4470 }
4471
4472 return (DDI_INTR_CLAIMED);
4473 }
4474
4475 /*
4476 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4477 *
4478 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4479 * if not successful, try Legacy.
4480 * ixgbe->intr_force can be used to force sequence to start with
4481 * any of the 3 types.
4482 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4483 */
4484 static int
4485 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4486 {
4487 dev_info_t *devinfo;
4488 int intr_types;
4489 int rc;
4490
4491 devinfo = ixgbe->dip;
4492
4493 /*
4494 * Get supported interrupt types
4495 */
4496 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4497
4498 if (rc != DDI_SUCCESS) {
4499 ixgbe_log(ixgbe,
4500 "Get supported interrupt types failed: %d", rc);
4501 return (IXGBE_FAILURE);
4502 }
4503 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4504
4505 ixgbe->intr_type = 0;
4506
4507 /*
4508 * Install MSI-X interrupts
4509 */
4510 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4511 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4512 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4513 if (rc == IXGBE_SUCCESS)
4514 return (IXGBE_SUCCESS);
4515
4516 ixgbe_log(ixgbe,
4517 "Allocate MSI-X failed, trying MSI interrupts...");
4518 }
4519
4520 /*
4521 * MSI-X not used, force rings and groups to 1
4522 */
4523 ixgbe->num_rx_rings = 1;
4524 ixgbe->num_rx_groups = 1;
4525 ixgbe->num_tx_rings = 1;
4526 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4527 ixgbe_log(ixgbe,
4528 "MSI-X not used, force rings and groups number to 1");
4529
4530 /*
4531 * Install MSI interrupts
4532 */
4533 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4534 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4535 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4536 if (rc == IXGBE_SUCCESS)
4537 return (IXGBE_SUCCESS);
4538
4539 ixgbe_log(ixgbe,
4540 "Allocate MSI failed, trying Legacy interrupts...");
4541 }
4542
4543 /*
4544 * Install legacy interrupts
4545 */
4546 if (intr_types & DDI_INTR_TYPE_FIXED) {
4547 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4548 if (rc == IXGBE_SUCCESS)
4549 return (IXGBE_SUCCESS);
4550
4551 ixgbe_log(ixgbe,
4552 "Allocate Legacy interrupts failed");
4553 }
4554
4555 /*
4556 * If none of the 3 types succeeded, return failure
4557 */
4558 return (IXGBE_FAILURE);
4559 }
4560
4561 /*
4562 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4563 *
4564 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4565 * if fewer than 2 handles are available, return failure.
4566 * Upon success, this maps the vectors to rx and tx rings for
4567 * interrupts.
4568 */
4569 static int
4570 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4571 {
4572 dev_info_t *devinfo;
4573 int request, count, actual;
4574 int minimum;
4575 int rc;
4576 uint32_t ring_per_group;
4577
4578 devinfo = ixgbe->dip;
4579
4580 switch (intr_type) {
4581 case DDI_INTR_TYPE_FIXED:
4582 request = 1; /* Request 1 legacy interrupt handle */
4583 minimum = 1;
4584 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4585 break;
4586
4587 case DDI_INTR_TYPE_MSI:
4588 request = 1; /* Request 1 MSI interrupt handle */
4589 minimum = 1;
4590 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4591 break;
4592
4593 case DDI_INTR_TYPE_MSIX:
4594 /*
4595 * Best number of vectors for the adapter is
4596 * (# rx rings + # tx rings), however we will
4597 * limit the request number.
4598 */
4599 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4600 if (request > ixgbe->capab->max_ring_vect)
4601 request = ixgbe->capab->max_ring_vect;
4602 minimum = 1;
4603 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4604 break;
4605
4606 default:
4607 ixgbe_log(ixgbe,
4608 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4609 intr_type);
4610 return (IXGBE_FAILURE);
4611 }
4612 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
4613 request, minimum);
4614
4615 /*
4616 * Get number of supported interrupts
4617 */
4618 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4619 if ((rc != DDI_SUCCESS) || (count < minimum)) {
4620 ixgbe_log(ixgbe,
4621 "Get interrupt number failed. Return: %d, count: %d",
4622 rc, count);
4623 return (IXGBE_FAILURE);
4624 }
4625 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4626
4627 actual = 0;
4628 ixgbe->intr_cnt = 0;
4629 ixgbe->intr_cnt_max = 0;
4630 ixgbe->intr_cnt_min = 0;
4631
4632 /*
4633 * Allocate an array of interrupt handles
4634 */
4635 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
4636 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
4637
4638 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
4639 request, &actual, DDI_INTR_ALLOC_NORMAL);
4640 if (rc != DDI_SUCCESS) {
4641 ixgbe_log(ixgbe, "Allocate interrupts failed. "
4642 "return: %d, request: %d, actual: %d",
4643 rc, request, actual);
4644 goto alloc_handle_fail;
4645 }
4646 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
4647
4648 /*
4649 * upper/lower limit of interrupts
4650 */
4651 ixgbe->intr_cnt = actual;
4652 ixgbe->intr_cnt_max = request;
4653 ixgbe->intr_cnt_min = minimum;
4654
4655 /*
4656 * rss number per group should not exceed the rx interrupt number,
4657 * else need to adjust rx ring number.
4658 */
4659 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4660 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
4661 if (actual < ring_per_group) {
4662 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
4663 ixgbe_setup_vmdq_rss_conf(ixgbe);
4664 }
4665
4666 /*
4667 * Now we know the actual number of vectors. Here we map the vector
4668 * to other, rx rings and tx ring.
4669 */
4670 if (actual < minimum) {
4671 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
4672 actual);
4673 goto alloc_handle_fail;
4674 }
4675
4676 /*
4677 * Get priority for first vector, assume remaining are all the same
4678 */
4679 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
4680 if (rc != DDI_SUCCESS) {
4681 ixgbe_log(ixgbe,
4682 "Get interrupt priority failed: %d", rc);
4683 goto alloc_handle_fail;
4684 }
4685
4686 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
4687 if (rc != DDI_SUCCESS) {
4688 ixgbe_log(ixgbe,
4689 "Get interrupt cap failed: %d", rc);
4690 goto alloc_handle_fail;
4691 }
4692
4693 ixgbe->intr_type = intr_type;
4694
4695 return (IXGBE_SUCCESS);
4696
4697 alloc_handle_fail:
4698 ixgbe_rem_intrs(ixgbe);
4699
4700 return (IXGBE_FAILURE);
4701 }
4702
4703 /*
4704 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
4705 *
4706 * Before adding the interrupt handlers, the interrupt vectors have
4707 * been allocated, and the rx/tx rings have also been allocated.
4708 */
4709 static int
4710 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
4711 {
4712 int vector = 0;
4713 int rc;
4714
4715 switch (ixgbe->intr_type) {
4716 case DDI_INTR_TYPE_MSIX:
4717 /*
4718 * Add interrupt handler for all vectors
4719 */
4720 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
4721 /*
4722 * install pointer to vect_map[vector]
4723 */
4724 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4725 (ddi_intr_handler_t *)ixgbe_intr_msix,
4726 (void *)&ixgbe->vect_map[vector], NULL);
4727
4728 if (rc != DDI_SUCCESS) {
4729 ixgbe_log(ixgbe,
4730 "Add interrupt handler failed. "
4731 "return: %d, vector: %d", rc, vector);
4732 for (vector--; vector >= 0; vector--) {
4733 (void) ddi_intr_remove_handler(
4734 ixgbe->htable[vector]);
4735 }
4736 return (IXGBE_FAILURE);
4737 }
4738 }
4739
4740 break;
4741
4742 case DDI_INTR_TYPE_MSI:
4743 /*
4744 * Add interrupt handlers for the only vector
4745 */
4746 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4747 (ddi_intr_handler_t *)ixgbe_intr_msi,
4748 (void *)ixgbe, NULL);
4749
4750 if (rc != DDI_SUCCESS) {
4751 ixgbe_log(ixgbe,
4752 "Add MSI interrupt handler failed: %d", rc);
4753 return (IXGBE_FAILURE);
4754 }
4755
4756 break;
4757
4758 case DDI_INTR_TYPE_FIXED:
4759 /*
4760 * Add interrupt handlers for the only vector
4761 */
4762 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4763 (ddi_intr_handler_t *)ixgbe_intr_legacy,
4764 (void *)ixgbe, NULL);
4765
4766 if (rc != DDI_SUCCESS) {
4767 ixgbe_log(ixgbe,
4768 "Add legacy interrupt handler failed: %d", rc);
4769 return (IXGBE_FAILURE);
4770 }
4771
4772 break;
4773
4774 default:
4775 return (IXGBE_FAILURE);
4776 }
4777
4778 return (IXGBE_SUCCESS);
4779 }
4780
4781 #pragma inline(ixgbe_map_rxring_to_vector)
4782 /*
4783 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
4784 */
4785 static void
4786 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
4787 {
4788 /*
4789 * Set bit in map
4790 */
4791 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4792
4793 /*
4794 * Count bits set
4795 */
4796 ixgbe->vect_map[v_idx].rxr_cnt++;
4797
4798 /*
4799 * Remember bit position
4800 */
4801 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
4802 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
4803 }
4804
4805 #pragma inline(ixgbe_map_txring_to_vector)
4806 /*
4807 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
4808 */
4809 static void
4810 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
4811 {
4812 /*
4813 * Set bit in map
4814 */
4815 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
4816
4817 /*
4818 * Count bits set
4819 */
4820 ixgbe->vect_map[v_idx].txr_cnt++;
4821
4822 /*
4823 * Remember bit position
4824 */
4825 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
4826 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
4827 }
4828
4829 /*
4830 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
4831 * allocation register (IVAR).
4832 * cause:
4833 * -1 : other cause
4834 * 0 : rx
4835 * 1 : tx
4836 */
4837 static void
4838 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
4839 int8_t cause)
4840 {
4841 struct ixgbe_hw *hw = &ixgbe->hw;
4842 u32 ivar, index;
4843
4844 switch (hw->mac.type) {
4845 case ixgbe_mac_82598EB:
4846 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4847 if (cause == -1) {
4848 cause = 0;
4849 }
4850 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4851 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4852 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4853 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4854 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4855 break;
4856
4857 case ixgbe_mac_82599EB:
4858 case ixgbe_mac_X540:
4859 if (cause == -1) {
4860 /* other causes */
4861 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4862 index = (intr_alloc_entry & 1) * 8;
4863 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4864 ivar &= ~(0xFF << index);
4865 ivar |= (msix_vector << index);
4866 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4867 } else {
4868 /* tx or rx causes */
4869 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4870 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4871 ivar = IXGBE_READ_REG(hw,
4872 IXGBE_IVAR(intr_alloc_entry >> 1));
4873 ivar &= ~(0xFF << index);
4874 ivar |= (msix_vector << index);
4875 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4876 ivar);
4877 }
4878 break;
4879
4880 default:
4881 break;
4882 }
4883 }
4884
4885 /*
4886 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
4887 * given interrupt vector allocation register (IVAR).
4888 * cause:
4889 * -1 : other cause
4890 * 0 : rx
4891 * 1 : tx
4892 */
4893 static void
4894 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4895 {
4896 struct ixgbe_hw *hw = &ixgbe->hw;
4897 u32 ivar, index;
4898
4899 switch (hw->mac.type) {
4900 case ixgbe_mac_82598EB:
4901 if (cause == -1) {
4902 cause = 0;
4903 }
4904 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4905 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4906 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4907 (intr_alloc_entry & 0x3)));
4908 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4909 break;
4910
4911 case ixgbe_mac_82599EB:
4912 case ixgbe_mac_X540:
4913 if (cause == -1) {
4914 /* other causes */
4915 index = (intr_alloc_entry & 1) * 8;
4916 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4917 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4918 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4919 } else {
4920 /* tx or rx causes */
4921 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4922 ivar = IXGBE_READ_REG(hw,
4923 IXGBE_IVAR(intr_alloc_entry >> 1));
4924 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4925 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4926 ivar);
4927 }
4928 break;
4929
4930 default:
4931 break;
4932 }
4933 }
4934
4935 /*
4936 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
4937 * given interrupt vector allocation register (IVAR).
4938 * cause:
4939 * -1 : other cause
4940 * 0 : rx
4941 * 1 : tx
4942 */
4943 static void
4944 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4945 {
4946 struct ixgbe_hw *hw = &ixgbe->hw;
4947 u32 ivar, index;
4948
4949 switch (hw->mac.type) {
4950 case ixgbe_mac_82598EB:
4951 if (cause == -1) {
4952 cause = 0;
4953 }
4954 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4955 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4956 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
4957 (intr_alloc_entry & 0x3)));
4958 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4959 break;
4960
4961 case ixgbe_mac_82599EB:
4962 case ixgbe_mac_X540:
4963 if (cause == -1) {
4964 /* other causes */
4965 index = (intr_alloc_entry & 1) * 8;
4966 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4967 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4968 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4969 } else {
4970 /* tx or rx causes */
4971 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4972 ivar = IXGBE_READ_REG(hw,
4973 IXGBE_IVAR(intr_alloc_entry >> 1));
4974 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4975 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4976 ivar);
4977 }
4978 break;
4979
4980 default:
4981 break;
4982 }
4983 }
4984
4985 /*
4986 * Convert the rx ring index driver maintained to the rx ring index
4987 * in h/w.
4988 */
4989 static uint32_t
4990 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
4991 {
4992
4993 struct ixgbe_hw *hw = &ixgbe->hw;
4994 uint32_t rx_ring_per_group, hw_rx_index;
4995
4996 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
4997 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
4998 return (sw_rx_index);
4999 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5000 switch (hw->mac.type) {
5001 case ixgbe_mac_82598EB:
5002 return (sw_rx_index);
5003
5004 case ixgbe_mac_82599EB:
5005 case ixgbe_mac_X540:
5006 return (sw_rx_index * 2);
5007
5008 default:
5009 break;
5010 }
5011 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5012 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5013
5014 switch (hw->mac.type) {
5015 case ixgbe_mac_82598EB:
5016 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5017 16 + (sw_rx_index % rx_ring_per_group);
5018 return (hw_rx_index);
5019
5020 case ixgbe_mac_82599EB:
5021 case ixgbe_mac_X540:
5022 if (ixgbe->num_rx_groups > 32) {
5023 hw_rx_index = (sw_rx_index /
5024 rx_ring_per_group) * 2 +
5025 (sw_rx_index % rx_ring_per_group);
5026 } else {
5027 hw_rx_index = (sw_rx_index /
5028 rx_ring_per_group) * 4 +
5029 (sw_rx_index % rx_ring_per_group);
5030 }
5031 return (hw_rx_index);
5032
5033 default:
5034 break;
5035 }
5036 }
5037
5038 /*
5039 * Should never reach. Just to make compiler happy.
5040 */
5041 return (sw_rx_index);
5042 }
5043
5044 /*
5045 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5046 *
5047 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5048 * to vector[0 - (intr_cnt -1)].
5049 */
5050 static int
5051 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5052 {
5053 int i, vector = 0;
5054
5055 /* initialize vector map */
5056 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5057 for (i = 0; i < ixgbe->intr_cnt; i++) {
5058 ixgbe->vect_map[i].ixgbe = ixgbe;
5059 }
5060
5061 /*
5062 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5063 * tx rings[0] on RTxQ[1].
5064 */
5065 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5066 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5067 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5068 return (IXGBE_SUCCESS);
5069 }
5070
5071 /*
5072 * Interrupts/vectors mapping for MSI-X
5073 */
5074
5075 /*
5076 * Map other interrupt to vector 0,
5077 * Set bit in map and count the bits set.
5078 */
5079 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5080 ixgbe->vect_map[vector].other_cnt++;
5081
5082 /*
5083 * Map rx ring interrupts to vectors
5084 */
5085 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5086 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5087 vector = (vector +1) % ixgbe->intr_cnt;
5088 }
5089
5090 /*
5091 * Map tx ring interrupts to vectors
5092 */
5093 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5094 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5095 vector = (vector +1) % ixgbe->intr_cnt;
5096 }
5097
5098 return (IXGBE_SUCCESS);
5099 }
5100
5101 /*
5102 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5103 *
5104 * This relies on ring/vector mapping already set up in the
5105 * vect_map[] structures
5106 */
5107 static void
5108 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5109 {
5110 struct ixgbe_hw *hw = &ixgbe->hw;
5111 ixgbe_intr_vector_t *vect; /* vector bitmap */
5112 int r_idx; /* ring index */
5113 int v_idx; /* vector index */
5114 uint32_t hw_index;
5115
5116 /*
5117 * Clear any previous entries
5118 */
5119 switch (hw->mac.type) {
5120 case ixgbe_mac_82598EB:
5121 for (v_idx = 0; v_idx < 25; v_idx++)
5122 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5123 break;
5124
5125 case ixgbe_mac_82599EB:
5126 case ixgbe_mac_X540:
5127 for (v_idx = 0; v_idx < 64; v_idx++)
5128 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5129 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5130 break;
5131
5132 default:
5133 break;
5134 }
5135
5136 /*
5137 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5138 * tx rings[0] will use RTxQ[1].
5139 */
5140 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5141 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5142 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5143 return;
5144 }
5145
5146 /*
5147 * For MSI-X interrupt, "Other" is always on vector[0].
5148 */
5149 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5150
5151 /*
5152 * For each interrupt vector, populate the IVAR table
5153 */
5154 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5155 vect = &ixgbe->vect_map[v_idx];
5156
5157 /*
5158 * For each rx ring bit set
5159 */
5160 r_idx = bt_getlowbit(vect->rx_map, 0,
5161 (ixgbe->num_rx_rings - 1));
5162
5163 while (r_idx >= 0) {
5164 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5165 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5166 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5167 (ixgbe->num_rx_rings - 1));
5168 }
5169
5170 /*
5171 * For each tx ring bit set
5172 */
5173 r_idx = bt_getlowbit(vect->tx_map, 0,
5174 (ixgbe->num_tx_rings - 1));
5175
5176 while (r_idx >= 0) {
5177 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5178 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5179 (ixgbe->num_tx_rings - 1));
5180 }
5181 }
5182 }
5183
5184 /*
5185 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5186 */
5187 static void
5188 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5189 {
5190 int i;
5191 int rc;
5192
5193 for (i = 0; i < ixgbe->intr_cnt; i++) {
5194 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5195 if (rc != DDI_SUCCESS) {
5196 IXGBE_DEBUGLOG_1(ixgbe,
5197 "Remove intr handler failed: %d", rc);
5198 }
5199 }
5200 }
5201
5202 /*
5203 * ixgbe_rem_intrs - Remove the allocated interrupts.
5204 */
5205 static void
5206 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5207 {
5208 int i;
5209 int rc;
5210
5211 for (i = 0; i < ixgbe->intr_cnt; i++) {
5212 rc = ddi_intr_free(ixgbe->htable[i]);
5213 if (rc != DDI_SUCCESS) {
5214 IXGBE_DEBUGLOG_1(ixgbe,
5215 "Free intr failed: %d", rc);
5216 }
5217 }
5218
5219 kmem_free(ixgbe->htable, ixgbe->intr_size);
5220 ixgbe->htable = NULL;
5221 }
5222
5223 /*
5224 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5225 */
5226 static int
5227 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5228 {
5229 int i;
5230 int rc;
5231
5232 /*
5233 * Enable interrupts
5234 */
5235 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5236 /*
5237 * Call ddi_intr_block_enable() for MSI
5238 */
5239 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5240 if (rc != DDI_SUCCESS) {
5241 ixgbe_log(ixgbe,
5242 "Enable block intr failed: %d", rc);
5243 return (IXGBE_FAILURE);
5244 }
5245 } else {
5246 /*
5247 * Call ddi_intr_enable() for Legacy/MSI non block enable
5248 */
5249 for (i = 0; i < ixgbe->intr_cnt; i++) {
5250 rc = ddi_intr_enable(ixgbe->htable[i]);
5251 if (rc != DDI_SUCCESS) {
5252 ixgbe_log(ixgbe,
5253 "Enable intr failed: %d", rc);
5254 return (IXGBE_FAILURE);
5255 }
5256 }
5257 }
5258
5259 return (IXGBE_SUCCESS);
5260 }
5261
5262 /*
5263 * ixgbe_disable_intrs - Disable all the interrupts.
5264 */
5265 static int
5266 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5267 {
5268 int i;
5269 int rc;
5270
5271 /*
5272 * Disable all interrupts
5273 */
5274 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5275 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5276 if (rc != DDI_SUCCESS) {
5277 ixgbe_log(ixgbe,
5278 "Disable block intr failed: %d", rc);
5279 return (IXGBE_FAILURE);
5280 }
5281 } else {
5282 for (i = 0; i < ixgbe->intr_cnt; i++) {
5283 rc = ddi_intr_disable(ixgbe->htable[i]);
5284 if (rc != DDI_SUCCESS) {
5285 ixgbe_log(ixgbe,
5286 "Disable intr failed: %d", rc);
5287 return (IXGBE_FAILURE);
5288 }
5289 }
5290 }
5291
5292 return (IXGBE_SUCCESS);
5293 }
5294
5295 /*
5296 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5297 */
5298 static void
5299 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5300 {
5301 struct ixgbe_hw *hw = &ixgbe->hw;
5302 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5303 boolean_t link_up = B_FALSE;
5304 uint32_t pcs1g_anlp = 0;
5305 uint32_t pcs1g_ana = 0;
5306 boolean_t autoneg = B_FALSE;
5307
5308 ASSERT(mutex_owned(&ixgbe->gen_lock));
5309 ixgbe->param_lp_1000fdx_cap = 0;
5310 ixgbe->param_lp_100fdx_cap = 0;
5311
5312 /* check for link, don't wait */
5313 (void) ixgbe_check_link(hw, &speed, &link_up, false);
5314 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5315
5316 if (link_up) {
5317 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5318
5319 ixgbe->param_lp_1000fdx_cap =
5320 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5321 ixgbe->param_lp_100fdx_cap =
5322 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5323 }
5324
5325 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5326
5327 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5328 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5329 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5330 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5331 }
5332
5333 /*
5334 * ixgbe_get_driver_control - Notify that driver is in control of device.
5335 */
5336 static void
5337 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5338 {
5339 uint32_t ctrl_ext;
5340
5341 /*
5342 * Notify firmware that driver is in control of device
5343 */
5344 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5345 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5346 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5347 }
5348
5349 /*
5350 * ixgbe_release_driver_control - Notify that driver is no longer in control
5351 * of device.
5352 */
5353 static void
5354 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5355 {
5356 uint32_t ctrl_ext;
5357
5358 /*
5359 * Notify firmware that driver is no longer in control of device
5360 */
5361 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5362 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5363 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5364 }
5365
5366 /*
5367 * ixgbe_atomic_reserve - Atomic decrease operation.
5368 */
5369 int
5370 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5371 {
5372 uint32_t oldval;
5373 uint32_t newval;
5374
5375 /*
5376 * ATOMICALLY
5377 */
5378 do {
5379 oldval = *count_p;
5380 if (oldval < n)
5381 return (-1);
5382 newval = oldval - n;
5383 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5384
5385 return (newval);
5386 }
5387
5388 /*
5389 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5390 */
5391 static uint8_t *
5392 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5393 {
5394 uint8_t *addr = *upd_ptr;
5395 uint8_t *new_ptr;
5396
5397 _NOTE(ARGUNUSED(hw));
5398 _NOTE(ARGUNUSED(vmdq));
5399
5400 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5401 *upd_ptr = new_ptr;
5402 return (addr);
5403 }
5404
5405 /*
5406 * FMA support
5407 */
5408 int
5409 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5410 {
5411 ddi_fm_error_t de;
5412
5413 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5414 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5415 return (de.fme_status);
5416 }
5417
5418 int
5419 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5420 {
5421 ddi_fm_error_t de;
5422
5423 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5424 return (de.fme_status);
5425 }
5426
5427 /*
5428 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5429 */
5430 static int
5431 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5432 {
5433 _NOTE(ARGUNUSED(impl_data));
5434 /*
5435 * as the driver can always deal with an error in any dma or
5436 * access handle, we can just return the fme_status value.
5437 */
5438 pci_ereport_post(dip, err, NULL);
5439 return (err->fme_status);
5440 }
5441
5442 static void
5443 ixgbe_fm_init(ixgbe_t *ixgbe)
5444 {
5445 ddi_iblock_cookie_t iblk;
5446 int fma_dma_flag;
5447
5448 /*
5449 * Only register with IO Fault Services if we have some capability
5450 */
5451 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5452 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5453 } else {
5454 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5455 }
5456
5457 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5458 fma_dma_flag = 1;
5459 } else {
5460 fma_dma_flag = 0;
5461 }
5462
5463 ixgbe_set_fma_flags(fma_dma_flag);
5464
5465 if (ixgbe->fm_capabilities) {
5466
5467 /*
5468 * Register capabilities with IO Fault Services
5469 */
5470 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5471
5472 /*
5473 * Initialize pci ereport capabilities if ereport capable
5474 */
5475 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5476 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5477 pci_ereport_setup(ixgbe->dip);
5478
5479 /*
5480 * Register error callback if error callback capable
5481 */
5482 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5483 ddi_fm_handler_register(ixgbe->dip,
5484 ixgbe_fm_error_cb, (void*) ixgbe);
5485 }
5486 }
5487
5488 static void
5489 ixgbe_fm_fini(ixgbe_t *ixgbe)
5490 {
5491 /*
5492 * Only unregister FMA capabilities if they are registered
5493 */
5494 if (ixgbe->fm_capabilities) {
5495
5496 /*
5497 * Release any resources allocated by pci_ereport_setup()
5498 */
5499 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5500 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5501 pci_ereport_teardown(ixgbe->dip);
5502
5503 /*
5504 * Un-register error callback if error callback capable
5505 */
5506 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5507 ddi_fm_handler_unregister(ixgbe->dip);
5508
5509 /*
5510 * Unregister from IO Fault Service
5511 */
5512 ddi_fm_fini(ixgbe->dip);
5513 }
5514 }
5515
5516 void
5517 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5518 {
5519 uint64_t ena;
5520 char buf[FM_MAX_CLASS];
5521
5522 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5523 ena = fm_ena_generate(0, FM_ENA_FMT1);
5524 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5525 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5526 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5527 }
5528 }
5529
5530 static int
5531 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5532 {
5533 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5534
5535 mutex_enter(&rx_ring->rx_lock);
5536 rx_ring->ring_gen_num = mr_gen_num;
5537 mutex_exit(&rx_ring->rx_lock);
5538 return (0);
5539 }
5540
5541 /*
5542 * Get the global ring index by a ring index within a group.
5543 */
5544 static int
5545 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5546 {
5547 ixgbe_rx_ring_t *rx_ring;
5548 int i;
5549
5550 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5551 rx_ring = &ixgbe->rx_rings[i];
5552 if (rx_ring->group_index == gindex)
5553 rindex--;
5554 if (rindex < 0)
5555 return (i);
5556 }
5557
5558 return (-1);
5559 }
5560
5561 /*
5562 * Callback funtion for MAC layer to register all rings.
5563 */
5564 /* ARGSUSED */
5565 void
5566 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5567 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5568 {
5569 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5570 mac_intr_t *mintr = &infop->mri_intr;
5571
5572 switch (rtype) {
5573 case MAC_RING_TYPE_RX: {
5574 /*
5575 * 'index' is the ring index within the group.
5576 * Need to get the global ring index by searching in groups.
5577 */
5578 int global_ring_index = ixgbe_get_rx_ring_index(
5579 ixgbe, group_index, ring_index);
5580
5581 ASSERT(global_ring_index >= 0);
5582
5583 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5584 rx_ring->ring_handle = rh;
5585
5586 infop->mri_driver = (mac_ring_driver_t)rx_ring;
5587 infop->mri_start = ixgbe_ring_start;
5588 infop->mri_stop = NULL;
5589 infop->mri_poll = ixgbe_ring_rx_poll;
5590 infop->mri_stat = ixgbe_rx_ring_stat;
5591
5592 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
5593 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
5594 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
5595 if (ixgbe->intr_type &
5596 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5597 mintr->mi_ddi_handle =
5598 ixgbe->htable[rx_ring->intr_vector];
5599 }
5600
5601 break;
5602 }
5603 case MAC_RING_TYPE_TX: {
5604 ASSERT(group_index == -1);
5605 ASSERT(ring_index < ixgbe->num_tx_rings);
5606
5607 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
5608 tx_ring->ring_handle = rh;
5609
5610 infop->mri_driver = (mac_ring_driver_t)tx_ring;
5611 infop->mri_start = NULL;
5612 infop->mri_stop = NULL;
5613 infop->mri_tx = ixgbe_ring_tx;
5614 infop->mri_stat = ixgbe_tx_ring_stat;
5615 if (ixgbe->intr_type &
5616 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5617 mintr->mi_ddi_handle =
5618 ixgbe->htable[tx_ring->intr_vector];
5619 }
5620 break;
5621 }
5622 default:
5623 break;
5624 }
5625 }
5626
5627 /*
5628 * Callback funtion for MAC layer to register all groups.
5629 */
5630 void
5631 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
5632 mac_group_info_t *infop, mac_group_handle_t gh)
5633 {
5634 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5635
5636 switch (rtype) {
5637 case MAC_RING_TYPE_RX: {
5638 ixgbe_rx_group_t *rx_group;
5639
5640 rx_group = &ixgbe->rx_groups[index];
5641 rx_group->group_handle = gh;
5642
5643 infop->mgi_driver = (mac_group_driver_t)rx_group;
5644 infop->mgi_start = NULL;
5645 infop->mgi_stop = NULL;
5646 infop->mgi_addmac = ixgbe_addmac;
5647 infop->mgi_remmac = ixgbe_remmac;
5648 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
5649
5650 break;
5651 }
5652 case MAC_RING_TYPE_TX:
5653 break;
5654 default:
5655 break;
5656 }
5657 }
5658
5659 /*
5660 * Enable interrupt on the specificed rx ring.
5661 */
5662 int
5663 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
5664 {
5665 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5666 ixgbe_t *ixgbe = rx_ring->ixgbe;
5667 int r_idx = rx_ring->index;
5668 int hw_r_idx = rx_ring->hw_index;
5669 int v_idx = rx_ring->intr_vector;
5670
5671 mutex_enter(&ixgbe->gen_lock);
5672 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5673 mutex_exit(&ixgbe->gen_lock);
5674 /*
5675 * Simply return 0.
5676 * Interrupts are being adjusted. ixgbe_intr_adjust()
5677 * will eventually re-enable the interrupt when it's
5678 * done with the adjustment.
5679 */
5680 return (0);
5681 }
5682
5683 /*
5684 * To enable interrupt by setting the VAL bit of given interrupt
5685 * vector allocation register (IVAR).
5686 */
5687 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
5688
5689 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5690
5691 /*
5692 * Trigger a Rx interrupt on this ring
5693 */
5694 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
5695 IXGBE_WRITE_FLUSH(&ixgbe->hw);
5696
5697 mutex_exit(&ixgbe->gen_lock);
5698
5699 return (0);
5700 }
5701
5702 /*
5703 * Disable interrupt on the specificed rx ring.
5704 */
5705 int
5706 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
5707 {
5708 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5709 ixgbe_t *ixgbe = rx_ring->ixgbe;
5710 int r_idx = rx_ring->index;
5711 int hw_r_idx = rx_ring->hw_index;
5712 int v_idx = rx_ring->intr_vector;
5713
5714 mutex_enter(&ixgbe->gen_lock);
5715 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5716 mutex_exit(&ixgbe->gen_lock);
5717 /*
5718 * Simply return 0.
5719 * In the rare case where an interrupt is being
5720 * disabled while interrupts are being adjusted,
5721 * we don't fail the operation. No interrupts will
5722 * be generated while they are adjusted, and
5723 * ixgbe_intr_adjust() will cause the interrupts
5724 * to be re-enabled once it completes. Note that
5725 * in this case, packets may be delivered to the
5726 * stack via interrupts before xgbe_rx_ring_intr_enable()
5727 * is called again. This is acceptable since interrupt
5728 * adjustment is infrequent, and the stack will be
5729 * able to handle these packets.
5730 */
5731 return (0);
5732 }
5733
5734 /*
5735 * To disable interrupt by clearing the VAL bit of given interrupt
5736 * vector allocation register (IVAR).
5737 */
5738 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
5739
5740 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
5741
5742 mutex_exit(&ixgbe->gen_lock);
5743
5744 return (0);
5745 }
5746
5747 /*
5748 * Add a mac address.
5749 */
5750 static int
5751 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
5752 {
5753 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5754 ixgbe_t *ixgbe = rx_group->ixgbe;
5755 struct ixgbe_hw *hw = &ixgbe->hw;
5756 int slot, i;
5757
5758 mutex_enter(&ixgbe->gen_lock);
5759
5760 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5761 mutex_exit(&ixgbe->gen_lock);
5762 return (ECANCELED);
5763 }
5764
5765 if (ixgbe->unicst_avail == 0) {
5766 /* no slots available */
5767 mutex_exit(&ixgbe->gen_lock);
5768 return (ENOSPC);
5769 }
5770
5771 /*
5772 * The first ixgbe->num_rx_groups slots are reserved for each respective
5773 * group. The rest slots are shared by all groups. While adding a
5774 * MAC address, reserved slots are firstly checked then the shared
5775 * slots are searched.
5776 */
5777 slot = -1;
5778 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
5779 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
5780 if (ixgbe->unicst_addr[i].mac.set == 0) {
5781 slot = i;
5782 break;
5783 }
5784 }
5785 } else {
5786 slot = rx_group->index;
5787 }
5788
5789 if (slot == -1) {
5790 /* no slots available */
5791 mutex_exit(&ixgbe->gen_lock);
5792 return (ENOSPC);
5793 }
5794
5795 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5796 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
5797 rx_group->index, IXGBE_RAH_AV);
5798 ixgbe->unicst_addr[slot].mac.set = 1;
5799 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
5800 ixgbe->unicst_avail--;
5801
5802 mutex_exit(&ixgbe->gen_lock);
5803
5804 return (0);
5805 }
5806
5807 /*
5808 * Remove a mac address.
5809 */
5810 static int
5811 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
5812 {
5813 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5814 ixgbe_t *ixgbe = rx_group->ixgbe;
5815 struct ixgbe_hw *hw = &ixgbe->hw;
5816 int slot;
5817
5818 mutex_enter(&ixgbe->gen_lock);
5819
5820 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5821 mutex_exit(&ixgbe->gen_lock);
5822 return (ECANCELED);
5823 }
5824
5825 slot = ixgbe_unicst_find(ixgbe, mac_addr);
5826 if (slot == -1) {
5827 mutex_exit(&ixgbe->gen_lock);
5828 return (EINVAL);
5829 }
5830
5831 if (ixgbe->unicst_addr[slot].mac.set == 0) {
5832 mutex_exit(&ixgbe->gen_lock);
5833 return (EINVAL);
5834 }
5835
5836 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5837 (void) ixgbe_clear_rar(hw, slot);
5838 ixgbe->unicst_addr[slot].mac.set = 0;
5839 ixgbe->unicst_avail++;
5840
5841 mutex_exit(&ixgbe->gen_lock);
5842
5843 return (0);
5844 }