1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012, Nexenta Systems, Inc. All rights reserved.
30 */
31
32 #include "ixgbe_sw.h"
33
34 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
35 static char ixgbe_version[] = "ixgbe 1.1.7";
36
37 /*
38 * Local function protoypes
39 */
40 static int ixgbe_register_mac(ixgbe_t *);
41 static int ixgbe_identify_hardware(ixgbe_t *);
42 static int ixgbe_regs_map(ixgbe_t *);
43 static void ixgbe_init_properties(ixgbe_t *);
44 static int ixgbe_init_driver_settings(ixgbe_t *);
45 static void ixgbe_init_locks(ixgbe_t *);
46 static void ixgbe_destroy_locks(ixgbe_t *);
47 static int ixgbe_init(ixgbe_t *);
48 static int ixgbe_chip_start(ixgbe_t *);
49 static void ixgbe_chip_stop(ixgbe_t *);
50 static int ixgbe_reset(ixgbe_t *);
51 static void ixgbe_tx_clean(ixgbe_t *);
52 static boolean_t ixgbe_tx_drain(ixgbe_t *);
53 static boolean_t ixgbe_rx_drain(ixgbe_t *);
54 static int ixgbe_alloc_rings(ixgbe_t *);
55 static void ixgbe_free_rings(ixgbe_t *);
56 static int ixgbe_alloc_rx_data(ixgbe_t *);
57 static void ixgbe_free_rx_data(ixgbe_t *);
58 static void ixgbe_setup_rings(ixgbe_t *);
59 static void ixgbe_setup_rx(ixgbe_t *);
60 static void ixgbe_setup_tx(ixgbe_t *);
61 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
62 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
63 static void ixgbe_setup_rss(ixgbe_t *);
64 static void ixgbe_setup_vmdq(ixgbe_t *);
65 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
66 static void ixgbe_init_unicst(ixgbe_t *);
67 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
68 static void ixgbe_setup_multicst(ixgbe_t *);
69 static void ixgbe_get_hw_state(ixgbe_t *);
70 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
71 static void ixgbe_get_conf(ixgbe_t *);
72 static void ixgbe_init_params(ixgbe_t *);
73 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
74 static void ixgbe_driver_link_check(ixgbe_t *);
75 static void ixgbe_sfp_check(void *);
76 static void ixgbe_overtemp_check(void *);
77 static void ixgbe_link_timer(void *);
78 static void ixgbe_local_timer(void *);
79 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
80 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
81 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
82 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
83 static boolean_t is_valid_mac_addr(uint8_t *);
84 static boolean_t ixgbe_stall_check(ixgbe_t *);
85 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
86 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
87 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
88 static int ixgbe_alloc_intrs(ixgbe_t *);
89 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
90 static int ixgbe_add_intr_handlers(ixgbe_t *);
91 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
92 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
93 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
94 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
95 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
96 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
97 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
98 static void ixgbe_setup_adapter_vector(ixgbe_t *);
99 static void ixgbe_rem_intr_handlers(ixgbe_t *);
100 static void ixgbe_rem_intrs(ixgbe_t *);
101 static int ixgbe_enable_intrs(ixgbe_t *);
102 static int ixgbe_disable_intrs(ixgbe_t *);
103 static uint_t ixgbe_intr_legacy(void *, void *);
104 static uint_t ixgbe_intr_msi(void *, void *);
105 static uint_t ixgbe_intr_msix(void *, void *);
106 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
107 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
108 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
109 static void ixgbe_get_driver_control(struct ixgbe_hw *);
110 static int ixgbe_addmac(void *, const uint8_t *);
111 static int ixgbe_remmac(void *, const uint8_t *);
112 static void ixgbe_release_driver_control(struct ixgbe_hw *);
113
114 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
115 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
116 static int ixgbe_resume(dev_info_t *);
117 static int ixgbe_suspend(dev_info_t *);
118 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
119 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
120 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
121 static int ixgbe_intr_cb_register(ixgbe_t *);
122 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
123
124 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
125 const void *impl_data);
126 static void ixgbe_fm_init(ixgbe_t *);
127 static void ixgbe_fm_fini(ixgbe_t *);
128
129 char *ixgbe_priv_props[] = {
130 "_tx_copy_thresh",
131 "_tx_recycle_thresh",
132 "_tx_overload_thresh",
133 "_tx_resched_thresh",
134 "_rx_copy_thresh",
135 "_rx_limit_per_intr",
136 "_intr_throttling",
137 "_adv_pause_cap",
138 "_adv_asym_pause_cap",
139 NULL
140 };
141
142 #define IXGBE_MAX_PRIV_PROPS \
143 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
144
145 static struct cb_ops ixgbe_cb_ops = {
146 nulldev, /* cb_open */
147 nulldev, /* cb_close */
148 nodev, /* cb_strategy */
149 nodev, /* cb_print */
150 nodev, /* cb_dump */
151 nodev, /* cb_read */
152 nodev, /* cb_write */
153 nodev, /* cb_ioctl */
154 nodev, /* cb_devmap */
155 nodev, /* cb_mmap */
156 nodev, /* cb_segmap */
157 nochpoll, /* cb_chpoll */
158 ddi_prop_op, /* cb_prop_op */
159 NULL, /* cb_stream */
160 D_MP | D_HOTPLUG, /* cb_flag */
161 CB_REV, /* cb_rev */
162 nodev, /* cb_aread */
163 nodev /* cb_awrite */
164 };
165
166 static struct dev_ops ixgbe_dev_ops = {
167 DEVO_REV, /* devo_rev */
168 0, /* devo_refcnt */
169 NULL, /* devo_getinfo */
170 nulldev, /* devo_identify */
171 nulldev, /* devo_probe */
172 ixgbe_attach, /* devo_attach */
173 ixgbe_detach, /* devo_detach */
174 nodev, /* devo_reset */
175 &ixgbe_cb_ops, /* devo_cb_ops */
176 NULL, /* devo_bus_ops */
177 ddi_power, /* devo_power */
178 ddi_quiesce_not_supported, /* devo_quiesce */
179 };
180
181 static struct modldrv ixgbe_modldrv = {
182 &mod_driverops, /* Type of module. This one is a driver */
183 ixgbe_ident, /* Discription string */
184 &ixgbe_dev_ops /* driver ops */
185 };
186
187 static struct modlinkage ixgbe_modlinkage = {
188 MODREV_1, &ixgbe_modldrv, NULL
189 };
190
191 /*
192 * Access attributes for register mapping
193 */
194 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
195 DDI_DEVICE_ATTR_V1,
196 DDI_STRUCTURE_LE_ACC,
197 DDI_STRICTORDER_ACC,
198 DDI_FLAGERR_ACC
199 };
200
201 /*
202 * Loopback property
203 */
204 static lb_property_t lb_normal = {
205 normal, "normal", IXGBE_LB_NONE
206 };
207
208 static lb_property_t lb_mac = {
209 internal, "MAC", IXGBE_LB_INTERNAL_MAC
210 };
211
212 static lb_property_t lb_external = {
213 external, "External", IXGBE_LB_EXTERNAL
214 };
215
216 #define IXGBE_M_CALLBACK_FLAGS \
217 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
218
219 static mac_callbacks_t ixgbe_m_callbacks = {
220 IXGBE_M_CALLBACK_FLAGS,
221 ixgbe_m_stat,
222 ixgbe_m_start,
223 ixgbe_m_stop,
224 ixgbe_m_promisc,
225 ixgbe_m_multicst,
226 NULL,
227 NULL,
228 NULL,
229 ixgbe_m_ioctl,
230 ixgbe_m_getcapab,
231 NULL,
232 NULL,
233 ixgbe_m_setprop,
234 ixgbe_m_getprop,
235 ixgbe_m_propinfo
236 };
237
238 /*
239 * Initialize capabilities of each supported adapter type
240 */
241 static adapter_info_t ixgbe_82598eb_cap = {
242 64, /* maximum number of rx queues */
243 1, /* minimum number of rx queues */
244 64, /* default number of rx queues */
245 16, /* maximum number of rx groups */
246 1, /* minimum number of rx groups */
247 1, /* default number of rx groups */
248 32, /* maximum number of tx queues */
249 1, /* minimum number of tx queues */
250 8, /* default number of tx queues */
251 16366, /* maximum MTU size */
252 0xFFFF, /* maximum interrupt throttle rate */
253 0, /* minimum interrupt throttle rate */
254 200, /* default interrupt throttle rate */
255 18, /* maximum total msix vectors */
256 16, /* maximum number of ring vectors */
257 2, /* maximum number of other vectors */
258 IXGBE_EICR_LSC, /* "other" interrupt types handled */
259 0, /* "other" interrupt types enable mask */
260 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
261 | IXGBE_FLAG_RSS_CAPABLE
262 | IXGBE_FLAG_VMDQ_CAPABLE)
263 };
264
265 static adapter_info_t ixgbe_82599eb_cap = {
266 128, /* maximum number of rx queues */
267 1, /* minimum number of rx queues */
268 128, /* default number of rx queues */
269 64, /* maximum number of rx groups */
270 1, /* minimum number of rx groups */
271 1, /* default number of rx groups */
272 128, /* maximum number of tx queues */
273 1, /* minimum number of tx queues */
274 8, /* default number of tx queues */
275 15500, /* maximum MTU size */
276 0xFF8, /* maximum interrupt throttle rate */
277 0, /* minimum interrupt throttle rate */
278 200, /* default interrupt throttle rate */
279 64, /* maximum total msix vectors */
280 16, /* maximum number of ring vectors */
281 2, /* maximum number of other vectors */
282 (IXGBE_EICR_LSC
283 | IXGBE_EICR_GPI_SDP1
284 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
285
286 (IXGBE_SDP1_GPIEN
287 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
288
289 (IXGBE_FLAG_DCA_CAPABLE
290 | IXGBE_FLAG_RSS_CAPABLE
291 | IXGBE_FLAG_VMDQ_CAPABLE
292 | IXGBE_FLAG_RSC_CAPABLE
293 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
294 };
295
296 static adapter_info_t ixgbe_X540_cap = {
297 128, /* maximum number of rx queues */
298 1, /* minimum number of rx queues */
299 128, /* default number of rx queues */
300 64, /* maximum number of rx groups */
301 1, /* minimum number of rx groups */
302 1, /* default number of rx groups */
303 128, /* maximum number of tx queues */
304 1, /* minimum number of tx queues */
305 8, /* default number of tx queues */
306 15500, /* maximum MTU size */
307 0xFF8, /* maximum interrupt throttle rate */
308 0, /* minimum interrupt throttle rate */
309 200, /* default interrupt throttle rate */
310 64, /* maximum total msix vectors */
311 16, /* maximum number of ring vectors */
312 2, /* maximum number of other vectors */
313 /* XXX KEBE ASKS, Do we care about X540's SDP3? */
314 (IXGBE_EICR_LSC
315 | IXGBE_EICR_GPI_SDP0
316 | IXGBE_EICR_GPI_SDP1
317 | IXGBE_EICR_GPI_SDP2
318 /* | IXGBE_EICR_GPI_SDP3 */), /* "other" interrupt types handled */
319
320 (IXGBE_SDP0_GPIEN
321 | IXGBE_SDP1_GPIEN
322 /* | IXGBE_SDP2_GPIEN
323 | IXGBE_SDP3_GPIEN */), /* "other" interrupt types enable mask */
324
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE
329 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
330 /* XXX KEBE ASKS, SFP_PLUG capable?!? */
331 };
332
333 /*
334 * Module Initialization Functions.
335 */
336
337 int
338 _init(void)
339 {
340 int status;
341
342 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
343
344 status = mod_install(&ixgbe_modlinkage);
345
346 if (status != DDI_SUCCESS) {
347 mac_fini_ops(&ixgbe_dev_ops);
348 }
349
350 return (status);
351 }
352
353 int
354 _fini(void)
355 {
356 int status;
357
358 status = mod_remove(&ixgbe_modlinkage);
359
360 if (status == DDI_SUCCESS) {
361 mac_fini_ops(&ixgbe_dev_ops);
362 }
363
364 return (status);
365 }
366
367 int
368 _info(struct modinfo *modinfop)
369 {
370 int status;
371
372 status = mod_info(&ixgbe_modlinkage, modinfop);
373
374 return (status);
375 }
376
377 /*
378 * ixgbe_attach - Driver attach.
379 *
380 * This function is the device specific initialization entry
381 * point. This entry point is required and must be written.
382 * The DDI_ATTACH command must be provided in the attach entry
383 * point. When attach() is called with cmd set to DDI_ATTACH,
384 * all normal kernel services (such as kmem_alloc(9F)) are
385 * available for use by the driver.
386 *
387 * The attach() function will be called once for each instance
388 * of the device on the system with cmd set to DDI_ATTACH.
389 * Until attach() succeeds, the only driver entry points which
390 * may be called are open(9E) and getinfo(9E).
391 */
392 static int
393 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
394 {
395 ixgbe_t *ixgbe;
396 struct ixgbe_osdep *osdep;
397 struct ixgbe_hw *hw;
398 int instance;
399 char taskqname[32];
400
401 /*
402 * Check the command and perform corresponding operations
403 */
404 switch (cmd) {
405 default:
406 return (DDI_FAILURE);
407
408 case DDI_RESUME:
409 return (ixgbe_resume(devinfo));
410
411 case DDI_ATTACH:
412 break;
413 }
414
415 /* Get the device instance */
416 instance = ddi_get_instance(devinfo);
417
418 /* Allocate memory for the instance data structure */
419 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
420
421 ixgbe->dip = devinfo;
422 ixgbe->instance = instance;
423
424 hw = &ixgbe->hw;
425 osdep = &ixgbe->osdep;
426 hw->back = osdep;
427 osdep->ixgbe = ixgbe;
428
429 /* Attach the instance pointer to the dev_info data structure */
430 ddi_set_driver_private(devinfo, ixgbe);
431
432 /*
433 * Initialize for fma support
434 */
435 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
436 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
437 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
438 ixgbe_fm_init(ixgbe);
439 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
440
441 /*
442 * Map PCI config space registers
443 */
444 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
445 ixgbe_error(ixgbe, "Failed to map PCI configurations");
446 goto attach_fail;
447 }
448 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
449
450 /*
451 * Identify the chipset family
452 */
453 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
454 ixgbe_error(ixgbe, "Failed to identify hardware");
455 goto attach_fail;
456 }
457
458 /*
459 * Map device registers
460 */
461 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
462 ixgbe_error(ixgbe, "Failed to map device registers");
463 goto attach_fail;
464 }
465 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
466
467 /*
468 * Initialize driver parameters
469 */
470 ixgbe_init_properties(ixgbe);
471 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
472
473 /*
474 * Register interrupt callback
475 */
476 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
477 ixgbe_error(ixgbe, "Failed to register interrupt callback");
478 goto attach_fail;
479 }
480
481 /*
482 * Allocate interrupts
483 */
484 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
485 ixgbe_error(ixgbe, "Failed to allocate interrupts");
486 goto attach_fail;
487 }
488 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
489
490 /*
491 * Allocate rx/tx rings based on the ring numbers.
492 * The actual numbers of rx/tx rings are decided by the number of
493 * allocated interrupt vectors, so we should allocate the rings after
494 * interrupts are allocated.
495 */
496 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
497 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
498 goto attach_fail;
499 }
500 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
501
502 /*
503 * Map rings to interrupt vectors
504 */
505 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
506 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
507 goto attach_fail;
508 }
509
510 /*
511 * Add interrupt handlers
512 */
513 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
514 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
515 goto attach_fail;
516 }
517 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
518
519 /*
520 * Create a taskq for sfp-change
521 */
522 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
523 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
524 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
525 ixgbe_error(ixgbe, "sfp_taskq create failed");
526 goto attach_fail;
527 }
528 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
529
530 /*
531 * Create a taskq for over-temp
532 */
533 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
534 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
535 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
536 ixgbe_error(ixgbe, "overtemp_taskq create failed");
537 goto attach_fail;
538 }
539 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
540
541 /*
542 * Initialize driver parameters
543 */
544 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
545 ixgbe_error(ixgbe, "Failed to initialize driver settings");
546 goto attach_fail;
547 }
548
549 /*
550 * Initialize mutexes for this device.
551 * Do this before enabling the interrupt handler and
552 * register the softint to avoid the condition where
553 * interrupt handler can try using uninitialized mutex.
554 */
555 ixgbe_init_locks(ixgbe);
556 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
557
558 /*
559 * Initialize chipset hardware
560 */
561 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
562 ixgbe_error(ixgbe, "Failed to initialize adapter");
563 goto attach_fail;
564 }
565 ixgbe->link_check_complete = B_FALSE;
566 ixgbe->link_check_hrtime = gethrtime() +
567 (IXGBE_LINK_UP_TIME * 100000000ULL);
568 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
569
570 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
571 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
572 goto attach_fail;
573 }
574
575 /*
576 * Initialize statistics
577 */
578 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
579 ixgbe_error(ixgbe, "Failed to initialize statistics");
580 goto attach_fail;
581 }
582 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
583
584 /*
585 * Register the driver to the MAC
586 */
587 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
588 ixgbe_error(ixgbe, "Failed to register MAC");
589 goto attach_fail;
590 }
591 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
592 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
593
594 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
595 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
596 if (ixgbe->periodic_id == 0) {
597 ixgbe_error(ixgbe, "Failed to add the link check timer");
598 goto attach_fail;
599 }
600 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
601
602 /*
603 * Now that mutex locks are initialized, and the chip is also
604 * initialized, enable interrupts.
605 */
606 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
607 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
608 goto attach_fail;
609 }
610 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
611
612 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version);
613 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
614
615 return (DDI_SUCCESS);
616
617 attach_fail:
618 ixgbe_unconfigure(devinfo, ixgbe);
619 return (DDI_FAILURE);
620 }
621
622 /*
623 * ixgbe_detach - Driver detach.
624 *
625 * The detach() function is the complement of the attach routine.
626 * If cmd is set to DDI_DETACH, detach() is used to remove the
627 * state associated with a given instance of a device node
628 * prior to the removal of that instance from the system.
629 *
630 * The detach() function will be called once for each instance
631 * of the device for which there has been a successful attach()
632 * once there are no longer any opens on the device.
633 *
634 * Interrupts routine are disabled, All memory allocated by this
635 * driver are freed.
636 */
637 static int
638 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
639 {
640 ixgbe_t *ixgbe;
641
642 /*
643 * Check detach command
644 */
645 switch (cmd) {
646 default:
647 return (DDI_FAILURE);
648
649 case DDI_SUSPEND:
650 return (ixgbe_suspend(devinfo));
651
652 case DDI_DETACH:
653 break;
654 }
655
656 /*
657 * Get the pointer to the driver private data structure
658 */
659 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
660 if (ixgbe == NULL)
661 return (DDI_FAILURE);
662
663 /*
664 * If the device is still running, it needs to be stopped first.
665 * This check is necessary because under some specific circumstances,
666 * the detach routine can be called without stopping the interface
667 * first.
668 */
669 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
670 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
671 mutex_enter(&ixgbe->gen_lock);
672 ixgbe_stop(ixgbe, B_TRUE);
673 mutex_exit(&ixgbe->gen_lock);
674 /* Disable and stop the watchdog timer */
675 ixgbe_disable_watchdog_timer(ixgbe);
676 }
677
678 /*
679 * Check if there are still rx buffers held by the upper layer.
680 * If so, fail the detach.
681 */
682 if (!ixgbe_rx_drain(ixgbe))
683 return (DDI_FAILURE);
684
685 /*
686 * Do the remaining unconfigure routines
687 */
688 ixgbe_unconfigure(devinfo, ixgbe);
689
690 return (DDI_SUCCESS);
691 }
692
693 static void
694 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
695 {
696 /*
697 * Disable interrupt
698 */
699 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
700 (void) ixgbe_disable_intrs(ixgbe);
701 }
702
703 /*
704 * remove the link check timer
705 */
706 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
707 if (ixgbe->periodic_id != NULL) {
708 ddi_periodic_delete(ixgbe->periodic_id);
709 ixgbe->periodic_id = NULL;
710 }
711 }
712
713 /*
714 * Unregister MAC
715 */
716 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
717 (void) mac_unregister(ixgbe->mac_hdl);
718 }
719
720 /*
721 * Free statistics
722 */
723 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
724 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
725 }
726
727 /*
728 * Remove interrupt handlers
729 */
730 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
731 ixgbe_rem_intr_handlers(ixgbe);
732 }
733
734 /*
735 * Remove taskq for sfp-status-change
736 */
737 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
738 ddi_taskq_destroy(ixgbe->sfp_taskq);
739 }
740
741 /*
742 * Remove taskq for over-temp
743 */
744 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
745 ddi_taskq_destroy(ixgbe->overtemp_taskq);
746 }
747
748 /*
749 * Remove interrupts
750 */
751 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
752 ixgbe_rem_intrs(ixgbe);
753 }
754
755 /*
756 * Unregister interrupt callback handler
757 */
758 (void) ddi_cb_unregister(ixgbe->cb_hdl);
759
760 /*
761 * Remove driver properties
762 */
763 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
764 (void) ddi_prop_remove_all(devinfo);
765 }
766
767 /*
768 * Stop the chipset
769 */
770 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
771 mutex_enter(&ixgbe->gen_lock);
772 ixgbe_chip_stop(ixgbe);
773 mutex_exit(&ixgbe->gen_lock);
774 }
775
776 /*
777 * Free register handle
778 */
779 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
780 if (ixgbe->osdep.reg_handle != NULL)
781 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
782 }
783
784 /*
785 * Free PCI config handle
786 */
787 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
788 if (ixgbe->osdep.cfg_handle != NULL)
789 pci_config_teardown(&ixgbe->osdep.cfg_handle);
790 }
791
792 /*
793 * Free locks
794 */
795 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
796 ixgbe_destroy_locks(ixgbe);
797 }
798
799 /*
800 * Free the rx/tx rings
801 */
802 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
803 ixgbe_free_rings(ixgbe);
804 }
805
806 /*
807 * Unregister FMA capabilities
808 */
809 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
810 ixgbe_fm_fini(ixgbe);
811 }
812
813 /*
814 * Free the driver data structure
815 */
816 kmem_free(ixgbe, sizeof (ixgbe_t));
817
818 ddi_set_driver_private(devinfo, NULL);
819 }
820
821 /*
822 * ixgbe_register_mac - Register the driver and its function pointers with
823 * the GLD interface.
824 */
825 static int
826 ixgbe_register_mac(ixgbe_t *ixgbe)
827 {
828 struct ixgbe_hw *hw = &ixgbe->hw;
829 mac_register_t *mac;
830 int status;
831
832 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
833 return (IXGBE_FAILURE);
834
835 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
836 mac->m_driver = ixgbe;
837 mac->m_dip = ixgbe->dip;
838 mac->m_src_addr = hw->mac.addr;
839 mac->m_callbacks = &ixgbe_m_callbacks;
840 mac->m_min_sdu = 0;
841 mac->m_max_sdu = ixgbe->default_mtu;
842 mac->m_margin = VLAN_TAGSZ;
843 mac->m_priv_props = ixgbe_priv_props;
844 mac->m_v12n = MAC_VIRT_LEVEL1;
845
846 status = mac_register(mac, &ixgbe->mac_hdl);
847
848 mac_free(mac);
849
850 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
851 }
852
853 /*
854 * ixgbe_identify_hardware - Identify the type of the chipset.
855 */
856 static int
857 ixgbe_identify_hardware(ixgbe_t *ixgbe)
858 {
859 struct ixgbe_hw *hw = &ixgbe->hw;
860 struct ixgbe_osdep *osdep = &ixgbe->osdep;
861
862 /*
863 * Get the device id
864 */
865 hw->vendor_id =
866 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
867 hw->device_id =
868 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
869 hw->revision_id =
870 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
871 hw->subsystem_device_id =
872 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
873 hw->subsystem_vendor_id =
874 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
875
876 /*
877 * Set the mac type of the adapter based on the device id
878 */
879 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
880 return (IXGBE_FAILURE);
881 }
882
883 /*
884 * Install adapter capabilities
885 */
886 switch (hw->mac.type) {
887 case ixgbe_mac_82598EB:
888 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
889 ixgbe->capab = &ixgbe_82598eb_cap;
890
891 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
892 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
893 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
894 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
895 }
896 break;
897
898 case ixgbe_mac_82599EB:
899 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
900 ixgbe->capab = &ixgbe_82599eb_cap;
901
902 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
903 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
904 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
905 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
906 }
907 break;
908
909 case ixgbe_mac_X540:
910 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
911 ixgbe->capab = &ixgbe_X540_cap;
912 /*
913 * For now, X540 is all set in its capab structure.
914 * As other X540 variants show up, things can change here.
915 */
916 break;
917
918 default:
919 IXGBE_DEBUGLOG_1(ixgbe,
920 "adapter not supported in ixgbe_identify_hardware(): %d\n",
921 hw->mac.type);
922 return (IXGBE_FAILURE);
923 }
924
925 return (IXGBE_SUCCESS);
926 }
927
928 /*
929 * ixgbe_regs_map - Map the device registers.
930 *
931 */
932 static int
933 ixgbe_regs_map(ixgbe_t *ixgbe)
934 {
935 dev_info_t *devinfo = ixgbe->dip;
936 struct ixgbe_hw *hw = &ixgbe->hw;
937 struct ixgbe_osdep *osdep = &ixgbe->osdep;
938 off_t mem_size;
939
940 /*
941 * First get the size of device registers to be mapped.
942 */
943 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
944 != DDI_SUCCESS) {
945 return (IXGBE_FAILURE);
946 }
947
948 /*
949 * Call ddi_regs_map_setup() to map registers
950 */
951 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
952 (caddr_t *)&hw->hw_addr, 0,
953 mem_size, &ixgbe_regs_acc_attr,
954 &osdep->reg_handle)) != DDI_SUCCESS) {
955 return (IXGBE_FAILURE);
956 }
957
958 return (IXGBE_SUCCESS);
959 }
960
961 /*
962 * ixgbe_init_properties - Initialize driver properties.
963 */
964 static void
965 ixgbe_init_properties(ixgbe_t *ixgbe)
966 {
967 /*
968 * Get conf file properties, including link settings
969 * jumbo frames, ring number, descriptor number, etc.
970 */
971 ixgbe_get_conf(ixgbe);
972
973 ixgbe_init_params(ixgbe);
974 }
975
976 /*
977 * ixgbe_init_driver_settings - Initialize driver settings.
978 *
979 * The settings include hardware function pointers, bus information,
980 * rx/tx rings settings, link state, and any other parameters that
981 * need to be setup during driver initialization.
982 */
983 static int
984 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
985 {
986 struct ixgbe_hw *hw = &ixgbe->hw;
987 dev_info_t *devinfo = ixgbe->dip;
988 ixgbe_rx_ring_t *rx_ring;
989 ixgbe_rx_group_t *rx_group;
990 ixgbe_tx_ring_t *tx_ring;
991 uint32_t rx_size;
992 uint32_t tx_size;
993 uint32_t ring_per_group;
994 int i;
995
996 /*
997 * Initialize chipset specific hardware function pointers
998 */
999 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1000 return (IXGBE_FAILURE);
1001 }
1002
1003 /*
1004 * Get the system page size
1005 */
1006 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1007
1008 /*
1009 * Set rx buffer size
1010 *
1011 * The IP header alignment room is counted in the calculation.
1012 * The rx buffer size is in unit of 1K that is required by the
1013 * chipset hardware.
1014 */
1015 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1016 ixgbe->rx_buf_size = ((rx_size >> 10) +
1017 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1018
1019 /*
1020 * Set tx buffer size
1021 */
1022 tx_size = ixgbe->max_frame_size;
1023 ixgbe->tx_buf_size = ((tx_size >> 10) +
1024 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1025
1026 /*
1027 * Initialize rx/tx rings/groups parameters
1028 */
1029 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1030 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1031 rx_ring = &ixgbe->rx_rings[i];
1032 rx_ring->index = i;
1033 rx_ring->ixgbe = ixgbe;
1034 rx_ring->group_index = i / ring_per_group;
1035 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1036 }
1037
1038 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1039 rx_group = &ixgbe->rx_groups[i];
1040 rx_group->index = i;
1041 rx_group->ixgbe = ixgbe;
1042 }
1043
1044 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1045 tx_ring = &ixgbe->tx_rings[i];
1046 tx_ring->index = i;
1047 tx_ring->ixgbe = ixgbe;
1048 if (ixgbe->tx_head_wb_enable)
1049 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1050 else
1051 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1052
1053 tx_ring->ring_size = ixgbe->tx_ring_size;
1054 tx_ring->free_list_size = ixgbe->tx_ring_size +
1055 (ixgbe->tx_ring_size >> 1);
1056 }
1057
1058 /*
1059 * Initialize values of interrupt throttling rate
1060 */
1061 for (i = 1; i < MAX_INTR_VECTOR; i++)
1062 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1063
1064 /*
1065 * The initial link state should be "unknown"
1066 */
1067 ixgbe->link_state = LINK_STATE_UNKNOWN;
1068
1069 return (IXGBE_SUCCESS);
1070 }
1071
1072 /*
1073 * ixgbe_init_locks - Initialize locks.
1074 */
1075 static void
1076 ixgbe_init_locks(ixgbe_t *ixgbe)
1077 {
1078 ixgbe_rx_ring_t *rx_ring;
1079 ixgbe_tx_ring_t *tx_ring;
1080 int i;
1081
1082 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1083 rx_ring = &ixgbe->rx_rings[i];
1084 mutex_init(&rx_ring->rx_lock, NULL,
1085 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1086 }
1087
1088 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1089 tx_ring = &ixgbe->tx_rings[i];
1090 mutex_init(&tx_ring->tx_lock, NULL,
1091 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1092 mutex_init(&tx_ring->recycle_lock, NULL,
1093 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1094 mutex_init(&tx_ring->tcb_head_lock, NULL,
1095 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1096 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1097 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1098 }
1099
1100 mutex_init(&ixgbe->gen_lock, NULL,
1101 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1102
1103 mutex_init(&ixgbe->watchdog_lock, NULL,
1104 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1105 }
1106
1107 /*
1108 * ixgbe_destroy_locks - Destroy locks.
1109 */
1110 static void
1111 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1112 {
1113 ixgbe_rx_ring_t *rx_ring;
1114 ixgbe_tx_ring_t *tx_ring;
1115 int i;
1116
1117 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1118 rx_ring = &ixgbe->rx_rings[i];
1119 mutex_destroy(&rx_ring->rx_lock);
1120 }
1121
1122 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1123 tx_ring = &ixgbe->tx_rings[i];
1124 mutex_destroy(&tx_ring->tx_lock);
1125 mutex_destroy(&tx_ring->recycle_lock);
1126 mutex_destroy(&tx_ring->tcb_head_lock);
1127 mutex_destroy(&tx_ring->tcb_tail_lock);
1128 }
1129
1130 mutex_destroy(&ixgbe->gen_lock);
1131 mutex_destroy(&ixgbe->watchdog_lock);
1132 }
1133
1134 static int
1135 ixgbe_resume(dev_info_t *devinfo)
1136 {
1137 ixgbe_t *ixgbe;
1138 int i;
1139
1140 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1141 if (ixgbe == NULL)
1142 return (DDI_FAILURE);
1143
1144 mutex_enter(&ixgbe->gen_lock);
1145
1146 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1147 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1148 mutex_exit(&ixgbe->gen_lock);
1149 return (DDI_FAILURE);
1150 }
1151
1152 /*
1153 * Enable and start the watchdog timer
1154 */
1155 ixgbe_enable_watchdog_timer(ixgbe);
1156 }
1157
1158 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1159
1160 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1161 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1162 mac_tx_ring_update(ixgbe->mac_hdl,
1163 ixgbe->tx_rings[i].ring_handle);
1164 }
1165 }
1166
1167 mutex_exit(&ixgbe->gen_lock);
1168
1169 return (DDI_SUCCESS);
1170 }
1171
1172 static int
1173 ixgbe_suspend(dev_info_t *devinfo)
1174 {
1175 ixgbe_t *ixgbe;
1176
1177 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1178 if (ixgbe == NULL)
1179 return (DDI_FAILURE);
1180
1181 mutex_enter(&ixgbe->gen_lock);
1182
1183 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1184 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1185 mutex_exit(&ixgbe->gen_lock);
1186 return (DDI_SUCCESS);
1187 }
1188 ixgbe_stop(ixgbe, B_FALSE);
1189
1190 mutex_exit(&ixgbe->gen_lock);
1191
1192 /*
1193 * Disable and stop the watchdog timer
1194 */
1195 ixgbe_disable_watchdog_timer(ixgbe);
1196
1197 return (DDI_SUCCESS);
1198 }
1199
1200 /*
1201 * ixgbe_init - Initialize the device.
1202 */
1203 static int
1204 ixgbe_init(ixgbe_t *ixgbe)
1205 {
1206 struct ixgbe_hw *hw = &ixgbe->hw;
1207
1208 mutex_enter(&ixgbe->gen_lock);
1209
1210 /*
1211 * Reset chipset to put the hardware in a known state
1212 * before we try to do anything with the eeprom.
1213 */
1214 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1215 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1216 goto init_fail;
1217 }
1218
1219 /*
1220 * Need to init eeprom before validating the checksum.
1221 */
1222 if (ixgbe_init_eeprom_params(hw) < 0) {
1223 ixgbe_error(ixgbe,
1224 "Unable to intitialize the eeprom interface.");
1225 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1226 goto init_fail;
1227 }
1228
1229 /*
1230 * NVM validation
1231 */
1232 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1233 /*
1234 * Some PCI-E parts fail the first check due to
1235 * the link being in sleep state. Call it again,
1236 * if it fails a second time it's a real issue.
1237 */
1238 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1239 ixgbe_error(ixgbe,
1240 "Invalid NVM checksum. Please contact "
1241 "the vendor to update the NVM.");
1242 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1243 goto init_fail;
1244 }
1245 }
1246
1247 /*
1248 * Setup default flow control thresholds - enable/disable
1249 * & flow control type is controlled by ixgbe.conf
1250 */
1251 hw->fc.high_water[0] = DEFAULT_FCRTH;
1252 hw->fc.low_water[0] = DEFAULT_FCRTL;
1253 hw->fc.pause_time = DEFAULT_FCPAUSE;
1254 hw->fc.send_xon = B_TRUE;
1255
1256 /*
1257 * Initialize link settings
1258 */
1259 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1260
1261 /*
1262 * Initialize the chipset hardware
1263 */
1264 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1265 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1266 goto init_fail;
1267 }
1268
1269 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1270 goto init_fail;
1271 }
1272
1273 mutex_exit(&ixgbe->gen_lock);
1274 return (IXGBE_SUCCESS);
1275
1276 init_fail:
1277 /*
1278 * Reset PHY
1279 */
1280 (void) ixgbe_reset_phy(hw);
1281
1282 mutex_exit(&ixgbe->gen_lock);
1283 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1284 return (IXGBE_FAILURE);
1285 }
1286
1287 /*
1288 * ixgbe_chip_start - Initialize and start the chipset hardware.
1289 */
1290 static int
1291 ixgbe_chip_start(ixgbe_t *ixgbe)
1292 {
1293 struct ixgbe_hw *hw = &ixgbe->hw;
1294 int ret_val, i;
1295
1296 ASSERT(mutex_owned(&ixgbe->gen_lock));
1297
1298 /*
1299 * Get the mac address
1300 * This function should handle SPARC case correctly.
1301 */
1302 if (!ixgbe_find_mac_address(ixgbe)) {
1303 ixgbe_error(ixgbe, "Failed to get the mac address");
1304 return (IXGBE_FAILURE);
1305 }
1306
1307 /*
1308 * Validate the mac address
1309 */
1310 (void) ixgbe_init_rx_addrs(hw);
1311 if (!is_valid_mac_addr(hw->mac.addr)) {
1312 ixgbe_error(ixgbe, "Invalid mac address");
1313 return (IXGBE_FAILURE);
1314 }
1315
1316 /*
1317 * Configure/Initialize hardware
1318 */
1319 ret_val = ixgbe_init_hw(hw);
1320 if (ret_val != IXGBE_SUCCESS) {
1321 if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1322 ixgbe_error(ixgbe,
1323 "This 82599 device is pre-release and contains"
1324 " outdated firmware, please contact your hardware"
1325 " vendor for a replacement.");
1326 } else {
1327 ixgbe_error(ixgbe, "Failed to initialize hardware");
1328 return (IXGBE_FAILURE);
1329 }
1330 }
1331
1332 /*
1333 * Re-enable relaxed ordering for performance. It is disabled
1334 * by default in the hardware init.
1335 */
1336 if (ixgbe->relax_order_enable == B_TRUE)
1337 ixgbe_enable_relaxed_ordering(hw);
1338
1339 /*
1340 * Setup adapter interrupt vectors
1341 */
1342 ixgbe_setup_adapter_vector(ixgbe);
1343
1344 /*
1345 * Initialize unicast addresses.
1346 */
1347 ixgbe_init_unicst(ixgbe);
1348
1349 /*
1350 * Setup and initialize the mctable structures.
1351 */
1352 ixgbe_setup_multicst(ixgbe);
1353
1354 /*
1355 * Set interrupt throttling rate
1356 */
1357 for (i = 0; i < ixgbe->intr_cnt; i++) {
1358 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1359 }
1360
1361 /*
1362 * Save the state of the phy
1363 */
1364 ixgbe_get_hw_state(ixgbe);
1365
1366 /*
1367 * Make sure driver has control
1368 */
1369 ixgbe_get_driver_control(hw);
1370
1371 return (IXGBE_SUCCESS);
1372 }
1373
1374 /*
1375 * ixgbe_chip_stop - Stop the chipset hardware
1376 */
1377 static void
1378 ixgbe_chip_stop(ixgbe_t *ixgbe)
1379 {
1380 struct ixgbe_hw *hw = &ixgbe->hw;
1381
1382 ASSERT(mutex_owned(&ixgbe->gen_lock));
1383
1384 /*
1385 * Tell firmware driver is no longer in control
1386 */
1387 ixgbe_release_driver_control(hw);
1388
1389 /*
1390 * Reset the chipset
1391 */
1392 (void) ixgbe_reset_hw(hw);
1393
1394 /*
1395 * Reset PHY
1396 */
1397 (void) ixgbe_reset_phy(hw);
1398 }
1399
1400 /*
1401 * ixgbe_reset - Reset the chipset and re-start the driver.
1402 *
1403 * It involves stopping and re-starting the chipset,
1404 * and re-configuring the rx/tx rings.
1405 */
1406 static int
1407 ixgbe_reset(ixgbe_t *ixgbe)
1408 {
1409 int i;
1410
1411 /*
1412 * Disable and stop the watchdog timer
1413 */
1414 ixgbe_disable_watchdog_timer(ixgbe);
1415
1416 mutex_enter(&ixgbe->gen_lock);
1417
1418 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1419 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1420
1421 ixgbe_stop(ixgbe, B_FALSE);
1422
1423 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1424 mutex_exit(&ixgbe->gen_lock);
1425 return (IXGBE_FAILURE);
1426 }
1427
1428 /*
1429 * After resetting, need to recheck the link status.
1430 */
1431 ixgbe->link_check_complete = B_FALSE;
1432 ixgbe->link_check_hrtime = gethrtime() +
1433 (IXGBE_LINK_UP_TIME * 100000000ULL);
1434
1435 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1436
1437 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1438 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1439 mac_tx_ring_update(ixgbe->mac_hdl,
1440 ixgbe->tx_rings[i].ring_handle);
1441 }
1442 }
1443
1444 mutex_exit(&ixgbe->gen_lock);
1445
1446 /*
1447 * Enable and start the watchdog timer
1448 */
1449 ixgbe_enable_watchdog_timer(ixgbe);
1450
1451 return (IXGBE_SUCCESS);
1452 }
1453
1454 /*
1455 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1456 */
1457 static void
1458 ixgbe_tx_clean(ixgbe_t *ixgbe)
1459 {
1460 ixgbe_tx_ring_t *tx_ring;
1461 tx_control_block_t *tcb;
1462 link_list_t pending_list;
1463 uint32_t desc_num;
1464 int i, j;
1465
1466 LINK_LIST_INIT(&pending_list);
1467
1468 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1469 tx_ring = &ixgbe->tx_rings[i];
1470
1471 mutex_enter(&tx_ring->recycle_lock);
1472
1473 /*
1474 * Clean the pending tx data - the pending packets in the
1475 * work_list that have no chances to be transmitted again.
1476 *
1477 * We must ensure the chipset is stopped or the link is down
1478 * before cleaning the transmit packets.
1479 */
1480 desc_num = 0;
1481 for (j = 0; j < tx_ring->ring_size; j++) {
1482 tcb = tx_ring->work_list[j];
1483 if (tcb != NULL) {
1484 desc_num += tcb->desc_num;
1485
1486 tx_ring->work_list[j] = NULL;
1487
1488 ixgbe_free_tcb(tcb);
1489
1490 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1491 }
1492 }
1493
1494 if (desc_num > 0) {
1495 atomic_add_32(&tx_ring->tbd_free, desc_num);
1496 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1497
1498 /*
1499 * Reset the head and tail pointers of the tbd ring;
1500 * Reset the writeback head if it's enable.
1501 */
1502 tx_ring->tbd_head = 0;
1503 tx_ring->tbd_tail = 0;
1504 if (ixgbe->tx_head_wb_enable)
1505 *tx_ring->tbd_head_wb = 0;
1506
1507 IXGBE_WRITE_REG(&ixgbe->hw,
1508 IXGBE_TDH(tx_ring->index), 0);
1509 IXGBE_WRITE_REG(&ixgbe->hw,
1510 IXGBE_TDT(tx_ring->index), 0);
1511 }
1512
1513 mutex_exit(&tx_ring->recycle_lock);
1514
1515 /*
1516 * Add the tx control blocks in the pending list to
1517 * the free list.
1518 */
1519 ixgbe_put_free_list(tx_ring, &pending_list);
1520 }
1521 }
1522
1523 /*
1524 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1525 * transmitted.
1526 */
1527 static boolean_t
1528 ixgbe_tx_drain(ixgbe_t *ixgbe)
1529 {
1530 ixgbe_tx_ring_t *tx_ring;
1531 boolean_t done;
1532 int i, j;
1533
1534 /*
1535 * Wait for a specific time to allow pending tx packets
1536 * to be transmitted.
1537 *
1538 * Check the counter tbd_free to see if transmission is done.
1539 * No lock protection is needed here.
1540 *
1541 * Return B_TRUE if all pending packets have been transmitted;
1542 * Otherwise return B_FALSE;
1543 */
1544 for (i = 0; i < TX_DRAIN_TIME; i++) {
1545
1546 done = B_TRUE;
1547 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1548 tx_ring = &ixgbe->tx_rings[j];
1549 done = done &&
1550 (tx_ring->tbd_free == tx_ring->ring_size);
1551 }
1552
1553 if (done)
1554 break;
1555
1556 msec_delay(1);
1557 }
1558
1559 return (done);
1560 }
1561
1562 /*
1563 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1564 */
1565 static boolean_t
1566 ixgbe_rx_drain(ixgbe_t *ixgbe)
1567 {
1568 boolean_t done = B_TRUE;
1569 int i;
1570
1571 /*
1572 * Polling the rx free list to check if those rx buffers held by
1573 * the upper layer are released.
1574 *
1575 * Check the counter rcb_free to see if all pending buffers are
1576 * released. No lock protection is needed here.
1577 *
1578 * Return B_TRUE if all pending buffers have been released;
1579 * Otherwise return B_FALSE;
1580 */
1581 for (i = 0; i < RX_DRAIN_TIME; i++) {
1582 done = (ixgbe->rcb_pending == 0);
1583
1584 if (done)
1585 break;
1586
1587 msec_delay(1);
1588 }
1589
1590 return (done);
1591 }
1592
1593 /*
1594 * ixgbe_start - Start the driver/chipset.
1595 */
1596 int
1597 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1598 {
1599 int i;
1600
1601 ASSERT(mutex_owned(&ixgbe->gen_lock));
1602
1603 if (alloc_buffer) {
1604 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1605 ixgbe_error(ixgbe,
1606 "Failed to allocate software receive rings");
1607 return (IXGBE_FAILURE);
1608 }
1609
1610 /* Allocate buffers for all the rx/tx rings */
1611 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1612 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1613 return (IXGBE_FAILURE);
1614 }
1615
1616 ixgbe->tx_ring_init = B_TRUE;
1617 } else {
1618 ixgbe->tx_ring_init = B_FALSE;
1619 }
1620
1621 for (i = 0; i < ixgbe->num_rx_rings; i++)
1622 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1623 for (i = 0; i < ixgbe->num_tx_rings; i++)
1624 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1625
1626 /*
1627 * Start the chipset hardware
1628 */
1629 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1630 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1631 goto start_failure;
1632 }
1633
1634 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1635 goto start_failure;
1636 }
1637
1638 /*
1639 * Setup the rx/tx rings
1640 */
1641 ixgbe_setup_rings(ixgbe);
1642
1643 /*
1644 * ixgbe_start() will be called when resetting, however if reset
1645 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1646 * before enabling the interrupts.
1647 */
1648 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1649 | IXGBE_STALL| IXGBE_OVERTEMP));
1650
1651 /*
1652 * Enable adapter interrupts
1653 * The interrupts must be enabled after the driver state is START
1654 */
1655 ixgbe_enable_adapter_interrupts(ixgbe);
1656
1657 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1658 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1659 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1660 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1661
1662 return (IXGBE_SUCCESS);
1663
1664 start_failure:
1665 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1666 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1667 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1668 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1669
1670 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1671
1672 return (IXGBE_FAILURE);
1673 }
1674
1675 /*
1676 * ixgbe_stop - Stop the driver/chipset.
1677 */
1678 void
1679 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1680 {
1681 int i;
1682
1683 ASSERT(mutex_owned(&ixgbe->gen_lock));
1684
1685 /*
1686 * Disable the adapter interrupts
1687 */
1688 ixgbe_disable_adapter_interrupts(ixgbe);
1689
1690 /*
1691 * Drain the pending tx packets
1692 */
1693 (void) ixgbe_tx_drain(ixgbe);
1694
1695 for (i = 0; i < ixgbe->num_rx_rings; i++)
1696 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1697 for (i = 0; i < ixgbe->num_tx_rings; i++)
1698 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1699
1700 /*
1701 * Stop the chipset hardware
1702 */
1703 ixgbe_chip_stop(ixgbe);
1704
1705 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1706 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1707 }
1708
1709 /*
1710 * Clean the pending tx data/resources
1711 */
1712 ixgbe_tx_clean(ixgbe);
1713
1714 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1715 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1716 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1717 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1718
1719 if (ixgbe->link_state == LINK_STATE_UP) {
1720 ixgbe->link_state = LINK_STATE_UNKNOWN;
1721 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1722 }
1723
1724 if (free_buffer) {
1725 /*
1726 * Release the DMA/memory resources of rx/tx rings
1727 */
1728 ixgbe_free_dma(ixgbe);
1729 ixgbe_free_rx_data(ixgbe);
1730 }
1731 }
1732
1733 /*
1734 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1735 */
1736 /* ARGSUSED */
1737 static int
1738 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1739 void *arg1, void *arg2)
1740 {
1741 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1742
1743 switch (cbaction) {
1744 /* IRM callback */
1745 int count;
1746 case DDI_CB_INTR_ADD:
1747 case DDI_CB_INTR_REMOVE:
1748 count = (int)(uintptr_t)cbarg;
1749 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1750 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1751 int, ixgbe->intr_cnt);
1752 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1753 DDI_SUCCESS) {
1754 ixgbe_error(ixgbe,
1755 "IRM CB: Failed to adjust interrupts");
1756 goto cb_fail;
1757 }
1758 break;
1759 default:
1760 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1761 cbaction);
1762 return (DDI_ENOTSUP);
1763 }
1764 return (DDI_SUCCESS);
1765 cb_fail:
1766 return (DDI_FAILURE);
1767 }
1768
1769 /*
1770 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1771 */
1772 static int
1773 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1774 {
1775 int i, rc, actual;
1776
1777 if (count == 0)
1778 return (DDI_SUCCESS);
1779
1780 if ((cbaction == DDI_CB_INTR_ADD &&
1781 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1782 (cbaction == DDI_CB_INTR_REMOVE &&
1783 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
1784 return (DDI_FAILURE);
1785
1786 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1787 return (DDI_FAILURE);
1788 }
1789
1790 for (i = 0; i < ixgbe->num_rx_rings; i++)
1791 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
1792 for (i = 0; i < ixgbe->num_tx_rings; i++)
1793 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
1794
1795 mutex_enter(&ixgbe->gen_lock);
1796 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1797 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
1798 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
1799 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
1800
1801 ixgbe_stop(ixgbe, B_FALSE);
1802 /*
1803 * Disable interrupts
1804 */
1805 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1806 rc = ixgbe_disable_intrs(ixgbe);
1807 ASSERT(rc == IXGBE_SUCCESS);
1808 }
1809 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
1810
1811 /*
1812 * Remove interrupt handlers
1813 */
1814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
1815 ixgbe_rem_intr_handlers(ixgbe);
1816 }
1817 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
1818
1819 /*
1820 * Clear vect_map
1821 */
1822 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
1823 switch (cbaction) {
1824 case DDI_CB_INTR_ADD:
1825 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
1826 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
1827 DDI_INTR_ALLOC_NORMAL);
1828 if (rc != DDI_SUCCESS || actual != count) {
1829 ixgbe_log(ixgbe, "Adjust interrupts failed."
1830 "return: %d, irm cb size: %d, actual: %d",
1831 rc, count, actual);
1832 goto intr_adjust_fail;
1833 }
1834 ixgbe->intr_cnt += count;
1835 break;
1836
1837 case DDI_CB_INTR_REMOVE:
1838 for (i = ixgbe->intr_cnt - count;
1839 i < ixgbe->intr_cnt; i ++) {
1840 rc = ddi_intr_free(ixgbe->htable[i]);
1841 ixgbe->htable[i] = NULL;
1842 if (rc != DDI_SUCCESS) {
1843 ixgbe_log(ixgbe, "Adjust interrupts failed."
1844 "return: %d, irm cb size: %d, actual: %d",
1845 rc, count, actual);
1846 goto intr_adjust_fail;
1847 }
1848 }
1849 ixgbe->intr_cnt -= count;
1850 break;
1851 }
1852
1853 /*
1854 * Get priority for first vector, assume remaining are all the same
1855 */
1856 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
1857 if (rc != DDI_SUCCESS) {
1858 ixgbe_log(ixgbe,
1859 "Get interrupt priority failed: %d", rc);
1860 goto intr_adjust_fail;
1861 }
1862 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
1863 if (rc != DDI_SUCCESS) {
1864 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
1865 goto intr_adjust_fail;
1866 }
1867 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
1868
1869 /*
1870 * Map rings to interrupt vectors
1871 */
1872 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
1873 ixgbe_error(ixgbe,
1874 "IRM CB: Failed to map interrupts to vectors");
1875 goto intr_adjust_fail;
1876 }
1877
1878 /*
1879 * Add interrupt handlers
1880 */
1881 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
1882 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
1883 goto intr_adjust_fail;
1884 }
1885 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
1886
1887 /*
1888 * Now that mutex locks are initialized, and the chip is also
1889 * initialized, enable interrupts.
1890 */
1891 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
1892 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
1893 goto intr_adjust_fail;
1894 }
1895 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
1896 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1897 ixgbe_error(ixgbe, "IRM CB: Failed to start");
1898 goto intr_adjust_fail;
1899 }
1900 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
1901 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
1902 ixgbe->ixgbe_state |= IXGBE_STARTED;
1903 mutex_exit(&ixgbe->gen_lock);
1904
1905 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1906 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
1907 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
1908 }
1909 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1910 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
1911 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
1912 }
1913
1914 /* Wakeup all Tx rings */
1915 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1916 mac_tx_ring_update(ixgbe->mac_hdl,
1917 ixgbe->tx_rings[i].ring_handle);
1918 }
1919
1920 IXGBE_DEBUGLOG_3(ixgbe,
1921 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
1922 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
1923 return (DDI_SUCCESS);
1924
1925 intr_adjust_fail:
1926 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1927 mutex_exit(&ixgbe->gen_lock);
1928 return (DDI_FAILURE);
1929 }
1930
1931 /*
1932 * ixgbe_intr_cb_register - Register interrupt callback function.
1933 */
1934 static int
1935 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
1936 {
1937 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
1938 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
1939 return (IXGBE_FAILURE);
1940 }
1941 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
1942 return (IXGBE_SUCCESS);
1943 }
1944
1945 /*
1946 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
1947 */
1948 static int
1949 ixgbe_alloc_rings(ixgbe_t *ixgbe)
1950 {
1951 /*
1952 * Allocate memory space for rx rings
1953 */
1954 ixgbe->rx_rings = kmem_zalloc(
1955 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
1956 KM_NOSLEEP);
1957
1958 if (ixgbe->rx_rings == NULL) {
1959 return (IXGBE_FAILURE);
1960 }
1961
1962 /*
1963 * Allocate memory space for tx rings
1964 */
1965 ixgbe->tx_rings = kmem_zalloc(
1966 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
1967 KM_NOSLEEP);
1968
1969 if (ixgbe->tx_rings == NULL) {
1970 kmem_free(ixgbe->rx_rings,
1971 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1972 ixgbe->rx_rings = NULL;
1973 return (IXGBE_FAILURE);
1974 }
1975
1976 /*
1977 * Allocate memory space for rx ring groups
1978 */
1979 ixgbe->rx_groups = kmem_zalloc(
1980 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
1981 KM_NOSLEEP);
1982
1983 if (ixgbe->rx_groups == NULL) {
1984 kmem_free(ixgbe->rx_rings,
1985 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1986 kmem_free(ixgbe->tx_rings,
1987 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
1988 ixgbe->rx_rings = NULL;
1989 ixgbe->tx_rings = NULL;
1990 return (IXGBE_FAILURE);
1991 }
1992
1993 return (IXGBE_SUCCESS);
1994 }
1995
1996 /*
1997 * ixgbe_free_rings - Free the memory space of rx/tx rings.
1998 */
1999 static void
2000 ixgbe_free_rings(ixgbe_t *ixgbe)
2001 {
2002 if (ixgbe->rx_rings != NULL) {
2003 kmem_free(ixgbe->rx_rings,
2004 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2005 ixgbe->rx_rings = NULL;
2006 }
2007
2008 if (ixgbe->tx_rings != NULL) {
2009 kmem_free(ixgbe->tx_rings,
2010 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2011 ixgbe->tx_rings = NULL;
2012 }
2013
2014 if (ixgbe->rx_groups != NULL) {
2015 kmem_free(ixgbe->rx_groups,
2016 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2017 ixgbe->rx_groups = NULL;
2018 }
2019 }
2020
2021 static int
2022 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2023 {
2024 ixgbe_rx_ring_t *rx_ring;
2025 int i;
2026
2027 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2028 rx_ring = &ixgbe->rx_rings[i];
2029 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2030 goto alloc_rx_rings_failure;
2031 }
2032 return (IXGBE_SUCCESS);
2033
2034 alloc_rx_rings_failure:
2035 ixgbe_free_rx_data(ixgbe);
2036 return (IXGBE_FAILURE);
2037 }
2038
2039 static void
2040 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2041 {
2042 ixgbe_rx_ring_t *rx_ring;
2043 ixgbe_rx_data_t *rx_data;
2044 int i;
2045
2046 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2047 rx_ring = &ixgbe->rx_rings[i];
2048
2049 mutex_enter(&ixgbe->rx_pending_lock);
2050 rx_data = rx_ring->rx_data;
2051
2052 if (rx_data != NULL) {
2053 rx_data->flag |= IXGBE_RX_STOPPED;
2054
2055 if (rx_data->rcb_pending == 0) {
2056 ixgbe_free_rx_ring_data(rx_data);
2057 rx_ring->rx_data = NULL;
2058 }
2059 }
2060
2061 mutex_exit(&ixgbe->rx_pending_lock);
2062 }
2063 }
2064
2065 /*
2066 * ixgbe_setup_rings - Setup rx/tx rings.
2067 */
2068 static void
2069 ixgbe_setup_rings(ixgbe_t *ixgbe)
2070 {
2071 /*
2072 * Setup the rx/tx rings, including the following:
2073 *
2074 * 1. Setup the descriptor ring and the control block buffers;
2075 * 2. Initialize necessary registers for receive/transmit;
2076 * 3. Initialize software pointers/parameters for receive/transmit;
2077 */
2078 ixgbe_setup_rx(ixgbe);
2079
2080 ixgbe_setup_tx(ixgbe);
2081 }
2082
2083 static void
2084 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2085 {
2086 ixgbe_t *ixgbe = rx_ring->ixgbe;
2087 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2088 struct ixgbe_hw *hw = &ixgbe->hw;
2089 rx_control_block_t *rcb;
2090 union ixgbe_adv_rx_desc *rbd;
2091 uint32_t size;
2092 uint32_t buf_low;
2093 uint32_t buf_high;
2094 uint32_t reg_val;
2095 int i;
2096
2097 ASSERT(mutex_owned(&rx_ring->rx_lock));
2098 ASSERT(mutex_owned(&ixgbe->gen_lock));
2099
2100 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2101 rcb = rx_data->work_list[i];
2102 rbd = &rx_data->rbd_ring[i];
2103
2104 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2105 rbd->read.hdr_addr = NULL;
2106 }
2107
2108 /*
2109 * Initialize the length register
2110 */
2111 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2112 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2113
2114 /*
2115 * Initialize the base address registers
2116 */
2117 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2118 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2119 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2120 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2121
2122 /*
2123 * Setup head & tail pointers
2124 */
2125 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2126 rx_data->ring_size - 1);
2127 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2128
2129 rx_data->rbd_next = 0;
2130 rx_data->lro_first = 0;
2131
2132 /*
2133 * Setup the Receive Descriptor Control Register (RXDCTL)
2134 * PTHRESH=32 descriptors (half the internal cache)
2135 * HTHRESH=0 descriptors (to minimize latency on fetch)
2136 * WTHRESH defaults to 1 (writeback each descriptor)
2137 */
2138 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2139 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2140
2141 /* Not a valid value for 82599 */
2142 if (hw->mac.type < ixgbe_mac_82599EB) {
2143 reg_val |= 0x0020; /* pthresh */
2144 }
2145 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2146
2147 if (hw->mac.type >= ixgbe_mac_82599EB) {
2148 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2149 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2150 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2151 }
2152
2153 /*
2154 * Setup the Split and Replication Receive Control Register.
2155 * Set the rx buffer size and the advanced descriptor type.
2156 */
2157 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2158 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2159 reg_val |= IXGBE_SRRCTL_DROP_EN;
2160 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2161 }
2162
2163 static void
2164 ixgbe_setup_rx(ixgbe_t *ixgbe)
2165 {
2166 ixgbe_rx_ring_t *rx_ring;
2167 struct ixgbe_hw *hw = &ixgbe->hw;
2168 uint32_t reg_val;
2169 uint32_t ring_mapping;
2170 uint32_t i, index;
2171 uint32_t psrtype_rss_bit;
2172
2173 /* PSRTYPE must be configured for 82599 */
2174 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2175 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2176 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2177 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2178 reg_val |= IXGBE_PSRTYPE_L2HDR;
2179 reg_val |= 0x80000000;
2180 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2181 } else {
2182 if (ixgbe->num_rx_groups > 32) {
2183 psrtype_rss_bit = 0x20000000;
2184 } else {
2185 psrtype_rss_bit = 0x40000000;
2186 }
2187 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2188 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2189 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2190 reg_val |= IXGBE_PSRTYPE_L2HDR;
2191 reg_val |= psrtype_rss_bit;
2192 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2193 }
2194 }
2195
2196 /*
2197 * Set filter control in FCTRL to accept broadcast packets and do
2198 * not pass pause frames to host. Flow control settings are already
2199 * in this register, so preserve them.
2200 */
2201 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2202 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */
2203 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */
2204 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2205
2206 /*
2207 * Hardware checksum settings
2208 */
2209 if (ixgbe->rx_hcksum_enable) {
2210 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2211 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2212 }
2213
2214 /*
2215 * Setup VMDq and RSS for multiple receive queues
2216 */
2217 switch (ixgbe->classify_mode) {
2218 case IXGBE_CLASSIFY_RSS:
2219 /*
2220 * One group, only RSS is needed when more than
2221 * one ring enabled.
2222 */
2223 ixgbe_setup_rss(ixgbe);
2224 break;
2225
2226 case IXGBE_CLASSIFY_VMDQ:
2227 /*
2228 * Multiple groups, each group has one ring,
2229 * only VMDq is needed.
2230 */
2231 ixgbe_setup_vmdq(ixgbe);
2232 break;
2233
2234 case IXGBE_CLASSIFY_VMDQ_RSS:
2235 /*
2236 * Multiple groups and multiple rings, both
2237 * VMDq and RSS are needed.
2238 */
2239 ixgbe_setup_vmdq_rss(ixgbe);
2240 break;
2241
2242 default:
2243 break;
2244 }
2245
2246 /*
2247 * Enable the receive unit. This must be done after filter
2248 * control is set in FCTRL.
2249 */
2250 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */
2251 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */
2252 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2253
2254 /*
2255 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2256 */
2257 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2258 rx_ring = &ixgbe->rx_rings[i];
2259 ixgbe_setup_rx_ring(rx_ring);
2260 }
2261
2262 /*
2263 * Setup the per-ring statistics mapping.
2264 */
2265 ring_mapping = 0;
2266 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2267 index = ixgbe->rx_rings[i].hw_index;
2268 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2269 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2270 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2271 }
2272
2273 /*
2274 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2275 * by four bytes if the packet has a VLAN field, so includes MTU,
2276 * ethernet header and frame check sequence.
2277 * Register is MAXFRS in 82599.
2278 */
2279 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2280 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2281 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2282
2283 /*
2284 * Setup Jumbo Frame enable bit
2285 */
2286 if (ixgbe->default_mtu > ETHERMTU) {
2287 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2288 reg_val |= IXGBE_HLREG0_JUMBOEN;
2289 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2290 }
2291
2292 /*
2293 * Setup RSC for multiple receive queues.
2294 */
2295 if (ixgbe->lro_enable) {
2296 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2297 /*
2298 * Make sure rx_buf_size * MAXDESC not greater
2299 * than 65535.
2300 * Intel recommends 4 for MAXDESC field value.
2301 */
2302 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2303 reg_val |= IXGBE_RSCCTL_RSCEN;
2304 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2305 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2306 else
2307 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2308 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2309 }
2310
2311 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2312 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2313 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2314
2315 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2316 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2317 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2318 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2319
2320 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2321 }
2322 }
2323
2324 static void
2325 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2326 {
2327 ixgbe_t *ixgbe = tx_ring->ixgbe;
2328 struct ixgbe_hw *hw = &ixgbe->hw;
2329 uint32_t size;
2330 uint32_t buf_low;
2331 uint32_t buf_high;
2332 uint32_t reg_val;
2333
2334 ASSERT(mutex_owned(&tx_ring->tx_lock));
2335 ASSERT(mutex_owned(&ixgbe->gen_lock));
2336
2337 /*
2338 * Initialize the length register
2339 */
2340 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2341 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2342
2343 /*
2344 * Initialize the base address registers
2345 */
2346 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2347 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2348 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2349 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2350
2351 /*
2352 * Setup head & tail pointers
2353 */
2354 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2355 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2356
2357 /*
2358 * Setup head write-back
2359 */
2360 if (ixgbe->tx_head_wb_enable) {
2361 /*
2362 * The memory of the head write-back is allocated using
2363 * the extra tbd beyond the tail of the tbd ring.
2364 */
2365 tx_ring->tbd_head_wb = (uint32_t *)
2366 ((uintptr_t)tx_ring->tbd_area.address + size);
2367 *tx_ring->tbd_head_wb = 0;
2368
2369 buf_low = (uint32_t)
2370 (tx_ring->tbd_area.dma_address + size);
2371 buf_high = (uint32_t)
2372 ((tx_ring->tbd_area.dma_address + size) >> 32);
2373
2374 /* Set the head write-back enable bit */
2375 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2376
2377 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2378 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2379
2380 /*
2381 * Turn off relaxed ordering for head write back or it will
2382 * cause problems with the tx recycling
2383 */
2384 #if 0
2385 /* XXX KEBE ASKS --> Should we do what FreeBSD does? */
2386 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2387 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2388 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2389 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2390 if (hw->mac.type == ixgbe_mac_82598EB) {
2391 IXGBE_WRITE_REG(hw,
2392 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2393 } else {
2394 IXGBE_WRITE_REG(hw,
2395 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2396 }
2397 #else
2398 /* XXX KEBE ASKS --> Or should we do what we've always done? */
2399 reg_val = IXGBE_READ_REG(hw,
2400 IXGBE_DCA_TXCTRL(tx_ring->index));
2401 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2402 IXGBE_WRITE_REG(hw,
2403 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2404 #endif
2405 } else {
2406 tx_ring->tbd_head_wb = NULL;
2407 #if 0
2408 /*
2409 * XXX KEBE ASKS --> Should we do what FreeBSD does and
2410 * twiddle TXCTRL_DESC_WR0_EN off anyway?
2411 */
2412 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2413 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2414 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2415 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2416 if (hw->mac.type == ixgbe_mac_82598EB) {
2417 IXGBE_WRITE_REG(hw,
2418 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2419 } else {
2420 IXGBE_WRITE_REG(hw,
2421 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2422 }
2423 #endif
2424 }
2425
2426 tx_ring->tbd_head = 0;
2427 tx_ring->tbd_tail = 0;
2428 tx_ring->tbd_free = tx_ring->ring_size;
2429
2430 if (ixgbe->tx_ring_init == B_TRUE) {
2431 tx_ring->tcb_head = 0;
2432 tx_ring->tcb_tail = 0;
2433 tx_ring->tcb_free = tx_ring->free_list_size;
2434 }
2435
2436 /*
2437 * Initialize the s/w context structure
2438 */
2439 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2440 }
2441
2442 static void
2443 ixgbe_setup_tx(ixgbe_t *ixgbe)
2444 {
2445 struct ixgbe_hw *hw = &ixgbe->hw;
2446 ixgbe_tx_ring_t *tx_ring;
2447 uint32_t reg_val;
2448 uint32_t ring_mapping;
2449 int i;
2450
2451 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2452 tx_ring = &ixgbe->tx_rings[i];
2453 ixgbe_setup_tx_ring(tx_ring);
2454 }
2455
2456 /*
2457 * Setup the per-ring statistics mapping.
2458 */
2459 ring_mapping = 0;
2460 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2461 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2462 if ((i & 0x3) == 0x3) {
2463 switch (hw->mac.type) {
2464 case ixgbe_mac_82598EB:
2465 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2466 ring_mapping);
2467 break;
2468
2469 case ixgbe_mac_82599EB:
2470 case ixgbe_mac_X540:
2471 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2472 ring_mapping);
2473 break;
2474
2475 default:
2476 break;
2477 }
2478
2479 ring_mapping = 0;
2480 }
2481 }
2482 if (i & 0x3) {
2483 switch (hw->mac.type) {
2484 case ixgbe_mac_82598EB:
2485 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2486 break;
2487
2488 case ixgbe_mac_82599EB:
2489 case ixgbe_mac_X540:
2490 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2491 break;
2492
2493 default:
2494 break;
2495 }
2496 }
2497
2498 /*
2499 * Enable CRC appending and TX padding (for short tx frames)
2500 */
2501 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2502 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2503 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2504
2505 /*
2506 * enable DMA for 82599 and X540 parts
2507 */
2508 if (hw->mac.type >= ixgbe_mac_82599EB) {
2509 /* DMATXCTL.TE must be set after all Tx config is complete */
2510 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2511 reg_val |= IXGBE_DMATXCTL_TE;
2512 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2513 /* XXX KEBE SAYS - FreeBSD sets up MTQC. Should we? */
2514 }
2515
2516 /*
2517 * Enabling tx queues ..
2518 * For 82599 must be done after DMATXCTL.TE is set
2519 */
2520 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2521 tx_ring = &ixgbe->tx_rings[i];
2522 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2523 reg_val |= IXGBE_TXDCTL_ENABLE;
2524 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2525 }
2526 }
2527
2528 /*
2529 * ixgbe_setup_rss - Setup receive-side scaling feature.
2530 */
2531 static void
2532 ixgbe_setup_rss(ixgbe_t *ixgbe)
2533 {
2534 struct ixgbe_hw *hw = &ixgbe->hw;
2535 uint32_t i, mrqc, rxcsum;
2536 uint32_t random;
2537 uint32_t reta;
2538 uint32_t ring_per_group;
2539
2540 /*
2541 * Fill out redirection table
2542 */
2543 reta = 0;
2544 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2545
2546 for (i = 0; i < 128; i++) {
2547 reta = (reta << 8) | (i % ring_per_group) |
2548 ((i % ring_per_group) << 4);
2549 if ((i & 3) == 3)
2550 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2551 }
2552
2553 /*
2554 * Fill out hash function seeds with a random constant
2555 */
2556 for (i = 0; i < 10; i++) {
2557 (void) random_get_pseudo_bytes((uint8_t *)&random,
2558 sizeof (uint32_t));
2559 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2560 }
2561
2562 /*
2563 * Enable RSS & perform hash on these packet types
2564 */
2565 mrqc = IXGBE_MRQC_RSSEN |
2566 IXGBE_MRQC_RSS_FIELD_IPV4 |
2567 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2568 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2569 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2570 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2571 IXGBE_MRQC_RSS_FIELD_IPV6 |
2572 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2573 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2574 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2575 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2576
2577 /*
2578 * Disable Packet Checksum to enable RSS for multiple receive queues.
2579 * It is an adapter hardware limitation that Packet Checksum is
2580 * mutually exclusive with RSS.
2581 */
2582 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2583 rxcsum |= IXGBE_RXCSUM_PCSD;
2584 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2585 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2586 }
2587
2588 /*
2589 * ixgbe_setup_vmdq - Setup MAC classification feature
2590 */
2591 static void
2592 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2593 {
2594 struct ixgbe_hw *hw = &ixgbe->hw;
2595 uint32_t vmdctl, i, vtctl;
2596
2597 /*
2598 * Setup the VMDq Control register, enable VMDq based on
2599 * packet destination MAC address:
2600 */
2601 switch (hw->mac.type) {
2602 case ixgbe_mac_82598EB:
2603 /*
2604 * VMDq Enable = 1;
2605 * VMDq Filter = 0; MAC filtering
2606 * Default VMDq output index = 0;
2607 */
2608 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2609 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2610 break;
2611
2612 case ixgbe_mac_82599EB:
2613 case ixgbe_mac_X540:
2614 /*
2615 * Enable VMDq-only.
2616 */
2617 vmdctl = IXGBE_MRQC_VMDQEN;
2618 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2619
2620 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2621 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2622 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2623 }
2624
2625 /*
2626 * Enable Virtualization and Replication.
2627 */
2628 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2629 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2630
2631 /*
2632 * Enable receiving packets to all VFs
2633 */
2634 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2635 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2636 break;
2637
2638 default:
2639 break;
2640 }
2641 }
2642
2643 /*
2644 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2645 */
2646 static void
2647 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2648 {
2649 struct ixgbe_hw *hw = &ixgbe->hw;
2650 uint32_t i, mrqc, rxcsum;
2651 uint32_t random;
2652 uint32_t reta;
2653 uint32_t ring_per_group;
2654 uint32_t vmdctl, vtctl;
2655
2656 /*
2657 * Fill out redirection table
2658 */
2659 reta = 0;
2660 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2661 for (i = 0; i < 128; i++) {
2662 reta = (reta << 8) | (i % ring_per_group) |
2663 ((i % ring_per_group) << 4);
2664 if ((i & 3) == 3)
2665 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2666 }
2667
2668 /*
2669 * Fill out hash function seeds with a random constant
2670 */
2671 for (i = 0; i < 10; i++) {
2672 (void) random_get_pseudo_bytes((uint8_t *)&random,
2673 sizeof (uint32_t));
2674 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2675 }
2676
2677 /*
2678 * Enable and setup RSS and VMDq
2679 */
2680 switch (hw->mac.type) {
2681 case ixgbe_mac_82598EB:
2682 /*
2683 * Enable RSS & Setup RSS Hash functions
2684 */
2685 mrqc = IXGBE_MRQC_RSSEN |
2686 IXGBE_MRQC_RSS_FIELD_IPV4 |
2687 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2688 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2689 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2690 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2691 IXGBE_MRQC_RSS_FIELD_IPV6 |
2692 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2693 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2694 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2695 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2696
2697 /*
2698 * Enable and Setup VMDq
2699 * VMDq Filter = 0; MAC filtering
2700 * Default VMDq output index = 0;
2701 */
2702 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2703 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2704 break;
2705
2706 case ixgbe_mac_82599EB:
2707 case ixgbe_mac_X540:
2708 /*
2709 * Enable RSS & Setup RSS Hash functions
2710 */
2711 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2712 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2713 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2714 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2715 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2716 IXGBE_MRQC_RSS_FIELD_IPV6 |
2717 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2718 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2719 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2720
2721 /*
2722 * Enable VMDq+RSS.
2723 */
2724 if (ixgbe->num_rx_groups > 32) {
2725 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2726 } else {
2727 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2728 }
2729
2730 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2731
2732 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2733 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2734 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2735 }
2736 break;
2737
2738 default:
2739 break;
2740
2741 }
2742
2743 /*
2744 * Disable Packet Checksum to enable RSS for multiple receive queues.
2745 * It is an adapter hardware limitation that Packet Checksum is
2746 * mutually exclusive with RSS.
2747 */
2748 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2749 rxcsum |= IXGBE_RXCSUM_PCSD;
2750 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2751 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2752
2753 if (hw->mac.type >= ixgbe_mac_82599EB) {
2754 /*
2755 * Enable Virtualization and Replication.
2756 */
2757 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2758 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2759
2760 /*
2761 * Enable receiving packets to all VFs
2762 */
2763 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2764 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2765 }
2766 }
2767
2768 /*
2769 * ixgbe_init_unicst - Initialize the unicast addresses.
2770 */
2771 static void
2772 ixgbe_init_unicst(ixgbe_t *ixgbe)
2773 {
2774 struct ixgbe_hw *hw = &ixgbe->hw;
2775 uint8_t *mac_addr;
2776 int slot;
2777 /*
2778 * Here we should consider two situations:
2779 *
2780 * 1. Chipset is initialized at the first time,
2781 * Clear all the multiple unicast addresses.
2782 *
2783 * 2. Chipset is reset
2784 * Recover the multiple unicast addresses from the
2785 * software data structure to the RAR registers.
2786 */
2787 if (!ixgbe->unicst_init) {
2788 /*
2789 * Initialize the multiple unicast addresses
2790 */
2791 ixgbe->unicst_total = hw->mac.num_rar_entries;
2792 ixgbe->unicst_avail = ixgbe->unicst_total;
2793 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2794 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2795 bzero(mac_addr, ETHERADDRL);
2796 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2797 ixgbe->unicst_addr[slot].mac.set = 0;
2798 }
2799 ixgbe->unicst_init = B_TRUE;
2800 } else {
2801 /* Re-configure the RAR registers */
2802 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2803 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2804 if (ixgbe->unicst_addr[slot].mac.set == 1) {
2805 (void) ixgbe_set_rar(hw, slot, mac_addr,
2806 ixgbe->unicst_addr[slot].mac.group_index,
2807 IXGBE_RAH_AV);
2808 } else {
2809 bzero(mac_addr, ETHERADDRL);
2810 (void) ixgbe_set_rar(hw, slot, mac_addr,
2811 NULL, NULL);
2812 }
2813 }
2814 }
2815 }
2816
2817 /*
2818 * ixgbe_unicst_find - Find the slot for the specified unicast address
2819 */
2820 int
2821 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2822 {
2823 int slot;
2824
2825 ASSERT(mutex_owned(&ixgbe->gen_lock));
2826
2827 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2828 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2829 mac_addr, ETHERADDRL) == 0)
2830 return (slot);
2831 }
2832
2833 return (-1);
2834 }
2835
2836 /*
2837 * ixgbe_multicst_add - Add a multicst address.
2838 */
2839 int
2840 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2841 {
2842 ASSERT(mutex_owned(&ixgbe->gen_lock));
2843
2844 if ((multiaddr[0] & 01) == 0) {
2845 return (EINVAL);
2846 }
2847
2848 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2849 return (ENOENT);
2850 }
2851
2852 bcopy(multiaddr,
2853 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2854 ixgbe->mcast_count++;
2855
2856 /*
2857 * Update the multicast table in the hardware
2858 */
2859 ixgbe_setup_multicst(ixgbe);
2860
2861 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2862 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2863 return (EIO);
2864 }
2865
2866 return (0);
2867 }
2868
2869 /*
2870 * ixgbe_multicst_remove - Remove a multicst address.
2871 */
2872 int
2873 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2874 {
2875 int i;
2876
2877 ASSERT(mutex_owned(&ixgbe->gen_lock));
2878
2879 for (i = 0; i < ixgbe->mcast_count; i++) {
2880 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2881 ETHERADDRL) == 0) {
2882 for (i++; i < ixgbe->mcast_count; i++) {
2883 ixgbe->mcast_table[i - 1] =
2884 ixgbe->mcast_table[i];
2885 }
2886 ixgbe->mcast_count--;
2887 break;
2888 }
2889 }
2890
2891 /*
2892 * Update the multicast table in the hardware
2893 */
2894 ixgbe_setup_multicst(ixgbe);
2895
2896 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2897 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2898 return (EIO);
2899 }
2900
2901 return (0);
2902 }
2903
2904 /*
2905 * ixgbe_setup_multicast - Setup multicast data structures.
2906 *
2907 * This routine initializes all of the multicast related structures
2908 * and save them in the hardware registers.
2909 */
2910 static void
2911 ixgbe_setup_multicst(ixgbe_t *ixgbe)
2912 {
2913 uint8_t *mc_addr_list;
2914 uint32_t mc_addr_count;
2915 struct ixgbe_hw *hw = &ixgbe->hw;
2916
2917 ASSERT(mutex_owned(&ixgbe->gen_lock));
2918
2919 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
2920
2921 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
2922 mc_addr_count = ixgbe->mcast_count;
2923
2924 /*
2925 * Update the multicast addresses to the MTA registers
2926 */
2927 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
2928 ixgbe_mc_table_itr, TRUE);
2929 }
2930
2931 /*
2932 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
2933 *
2934 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
2935 * Different chipsets may have different allowed configuration of vmdq and rss.
2936 */
2937 static void
2938 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
2939 {
2940 struct ixgbe_hw *hw = &ixgbe->hw;
2941 uint32_t ring_per_group;
2942
2943 switch (hw->mac.type) {
2944 case ixgbe_mac_82598EB:
2945 /*
2946 * 82598 supports the following combination:
2947 * vmdq no. x rss no.
2948 * [5..16] x 1
2949 * [1..4] x [1..16]
2950 * However 8 rss queue per pool (vmdq) is sufficient for
2951 * most cases.
2952 */
2953 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2954 if (ixgbe->num_rx_groups > 4) {
2955 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
2956 } else {
2957 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2958 min(8, ring_per_group);
2959 }
2960
2961 break;
2962
2963 case ixgbe_mac_82599EB:
2964 case ixgbe_mac_X540:
2965 /*
2966 * 82599 supports the following combination:
2967 * vmdq no. x rss no.
2968 * [33..64] x [1..2]
2969 * [2..32] x [1..4]
2970 * 1 x [1..16]
2971 * However 8 rss queue per pool (vmdq) is sufficient for
2972 * most cases.
2973 *
2974 * For now, treat X540 like the 82599.
2975 */
2976 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2977 if (ixgbe->num_rx_groups == 1) {
2978 ixgbe->num_rx_rings = min(8, ring_per_group);
2979 } else if (ixgbe->num_rx_groups <= 32) {
2980 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2981 min(4, ring_per_group);
2982 } else if (ixgbe->num_rx_groups <= 64) {
2983 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2984 min(2, ring_per_group);
2985 }
2986 break;
2987
2988 default:
2989 break;
2990 }
2991
2992 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2993
2994 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
2995 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
2996 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
2997 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
2998 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
2999 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3000 } else {
3001 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3002 }
3003
3004 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3005 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3006 }
3007
3008 /*
3009 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3010 *
3011 * This routine gets user-configured values out of the configuration
3012 * file ixgbe.conf.
3013 *
3014 * For each configurable value, there is a minimum, a maximum, and a
3015 * default.
3016 * If user does not configure a value, use the default.
3017 * If user configures below the minimum, use the minumum.
3018 * If user configures above the maximum, use the maxumum.
3019 */
3020 static void
3021 ixgbe_get_conf(ixgbe_t *ixgbe)
3022 {
3023 struct ixgbe_hw *hw = &ixgbe->hw;
3024 uint32_t flow_control;
3025
3026 /*
3027 * ixgbe driver supports the following user configurations:
3028 *
3029 * Jumbo frame configuration:
3030 * default_mtu
3031 *
3032 * Ethernet flow control configuration:
3033 * flow_control
3034 *
3035 * Multiple rings configurations:
3036 * tx_queue_number
3037 * tx_ring_size
3038 * rx_queue_number
3039 * rx_ring_size
3040 *
3041 * Call ixgbe_get_prop() to get the value for a specific
3042 * configuration parameter.
3043 */
3044
3045 /*
3046 * Jumbo frame configuration - max_frame_size controls host buffer
3047 * allocation, so includes MTU, ethernet header, vlan tag and
3048 * frame check sequence.
3049 */
3050 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3051 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3052
3053 ixgbe->max_frame_size = ixgbe->default_mtu +
3054 sizeof (struct ether_vlan_header) + ETHERFCSL;
3055
3056 /*
3057 * Ethernet flow control configuration
3058 */
3059 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3060 ixgbe_fc_none, 3, ixgbe_fc_none);
3061 if (flow_control == 3)
3062 flow_control = ixgbe_fc_default;
3063
3064 /*
3065 * fc.requested mode is what the user requests. After autoneg,
3066 * fc.current_mode will be the flow_control mode that was negotiated.
3067 */
3068 hw->fc.requested_mode = flow_control;
3069
3070 /*
3071 * Multiple rings configurations
3072 */
3073 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3074 ixgbe->capab->min_tx_que_num,
3075 ixgbe->capab->max_tx_que_num,
3076 ixgbe->capab->def_tx_que_num);
3077 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3078 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3079
3080 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3081 ixgbe->capab->min_rx_que_num,
3082 ixgbe->capab->max_rx_que_num,
3083 ixgbe->capab->def_rx_que_num);
3084 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3085 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3086
3087 /*
3088 * Multiple groups configuration
3089 */
3090 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3091 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3092 ixgbe->capab->def_rx_grp_num);
3093
3094 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3095 0, 1, DEFAULT_MR_ENABLE);
3096
3097 if (ixgbe->mr_enable == B_FALSE) {
3098 ixgbe->num_tx_rings = 1;
3099 ixgbe->num_rx_rings = 1;
3100 ixgbe->num_rx_groups = 1;
3101 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3102 } else {
3103 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3104 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3105 /*
3106 * The combination of num_rx_rings and num_rx_groups
3107 * may be not supported by h/w. We need to adjust
3108 * them to appropriate values.
3109 */
3110 ixgbe_setup_vmdq_rss_conf(ixgbe);
3111 }
3112
3113 /*
3114 * Tunable used to force an interrupt type. The only use is
3115 * for testing of the lesser interrupt types.
3116 * 0 = don't force interrupt type
3117 * 1 = force interrupt type MSI-X
3118 * 2 = force interrupt type MSI
3119 * 3 = force interrupt type Legacy
3120 */
3121 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3122 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3123
3124 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3125 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3126 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3127 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3128 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3129 0, 1, DEFAULT_LSO_ENABLE);
3130 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3131 0, 1, DEFAULT_LRO_ENABLE);
3132 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3133 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3134 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3135 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3136
3137 /* Head Write Back not recommended for 82599 and X540 */
3138 if (hw->mac.type >= ixgbe_mac_82599EB) {
3139 ixgbe->tx_head_wb_enable = B_FALSE;
3140 }
3141
3142 /*
3143 * ixgbe LSO needs the tx h/w checksum support.
3144 * LSO will be disabled if tx h/w checksum is not
3145 * enabled.
3146 */
3147 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3148 ixgbe->lso_enable = B_FALSE;
3149 }
3150
3151 /*
3152 * ixgbe LRO needs the rx h/w checksum support.
3153 * LRO will be disabled if rx h/w checksum is not
3154 * enabled.
3155 */
3156 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3157 ixgbe->lro_enable = B_FALSE;
3158 }
3159
3160 /*
3161 * ixgbe LRO only been supported by 82599 and X540 now
3162 */
3163 if (hw->mac.type < ixgbe_mac_82599EB) {
3164 ixgbe->lro_enable = B_FALSE;
3165 }
3166 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3167 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3168 DEFAULT_TX_COPY_THRESHOLD);
3169 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3170 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3171 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3172 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3173 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3174 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3175 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3176 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3177 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3178
3179 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3180 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3181 DEFAULT_RX_COPY_THRESHOLD);
3182 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3183 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3184 DEFAULT_RX_LIMIT_PER_INTR);
3185
3186 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3187 ixgbe->capab->min_intr_throttle,
3188 ixgbe->capab->max_intr_throttle,
3189 ixgbe->capab->def_intr_throttle);
3190 /*
3191 * 82599 and X540 require the interupt throttling rate is
3192 * a multiple of 8. This is enforced by the register
3193 * definiton.
3194 */
3195 if (hw->mac.type >= ixgbe_mac_82599EB)
3196 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3197 }
3198
3199 static void
3200 ixgbe_init_params(ixgbe_t *ixgbe)
3201 {
3202 ixgbe->param_en_10000fdx_cap = 1;
3203 ixgbe->param_en_1000fdx_cap = 1;
3204 ixgbe->param_en_100fdx_cap = 1;
3205 ixgbe->param_adv_10000fdx_cap = 1;
3206 ixgbe->param_adv_1000fdx_cap = 1;
3207 ixgbe->param_adv_100fdx_cap = 1;
3208
3209 ixgbe->param_pause_cap = 1;
3210 ixgbe->param_asym_pause_cap = 1;
3211 ixgbe->param_rem_fault = 0;
3212
3213 ixgbe->param_adv_autoneg_cap = 1;
3214 ixgbe->param_adv_pause_cap = 1;
3215 ixgbe->param_adv_asym_pause_cap = 1;
3216 ixgbe->param_adv_rem_fault = 0;
3217
3218 ixgbe->param_lp_10000fdx_cap = 0;
3219 ixgbe->param_lp_1000fdx_cap = 0;
3220 ixgbe->param_lp_100fdx_cap = 0;
3221 ixgbe->param_lp_autoneg_cap = 0;
3222 ixgbe->param_lp_pause_cap = 0;
3223 ixgbe->param_lp_asym_pause_cap = 0;
3224 ixgbe->param_lp_rem_fault = 0;
3225 }
3226
3227 /*
3228 * ixgbe_get_prop - Get a property value out of the configuration file
3229 * ixgbe.conf.
3230 *
3231 * Caller provides the name of the property, a default value, a minimum
3232 * value, and a maximum value.
3233 *
3234 * Return configured value of the property, with default, minimum and
3235 * maximum properly applied.
3236 */
3237 static int
3238 ixgbe_get_prop(ixgbe_t *ixgbe,
3239 char *propname, /* name of the property */
3240 int minval, /* minimum acceptable value */
3241 int maxval, /* maximim acceptable value */
3242 int defval) /* default value */
3243 {
3244 int value;
3245
3246 /*
3247 * Call ddi_prop_get_int() to read the conf settings
3248 */
3249 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3250 DDI_PROP_DONTPASS, propname, defval);
3251 if (value > maxval)
3252 value = maxval;
3253
3254 if (value < minval)
3255 value = minval;
3256
3257 return (value);
3258 }
3259
3260 /*
3261 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3262 */
3263 int
3264 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3265 {
3266 u32 autoneg_advertised = 0;
3267
3268 /*
3269 * No half duplex support with 10Gb parts
3270 */
3271 if (ixgbe->param_adv_10000fdx_cap == 1)
3272 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3273
3274 if (ixgbe->param_adv_1000fdx_cap == 1)
3275 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3276
3277 if (ixgbe->param_adv_100fdx_cap == 1)
3278 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3279
3280 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3281 ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3282 "to autonegotiation with full link capabilities.");
3283
3284 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3285 IXGBE_LINK_SPEED_1GB_FULL |
3286 IXGBE_LINK_SPEED_100_FULL;
3287 }
3288
3289 if (setup_hw) {
3290 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3291 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) {
3292 ixgbe_notice(ixgbe, "Setup link failed on this "
3293 "device.");
3294 return (IXGBE_FAILURE);
3295 }
3296 }
3297
3298 return (IXGBE_SUCCESS);
3299 }
3300
3301 /*
3302 * ixgbe_driver_link_check - Link status processing.
3303 *
3304 * This function can be called in both kernel context and interrupt context
3305 */
3306 static void
3307 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3308 {
3309 struct ixgbe_hw *hw = &ixgbe->hw;
3310 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3311 boolean_t link_up = B_FALSE;
3312 boolean_t link_changed = B_FALSE;
3313
3314 ASSERT(mutex_owned(&ixgbe->gen_lock));
3315
3316 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3317 if (link_up) {
3318 ixgbe->link_check_complete = B_TRUE;
3319
3320 /* Link is up, enable flow control settings */
3321 (void) ixgbe_fc_enable(hw);
3322
3323 /*
3324 * The Link is up, check whether it was marked as down earlier
3325 */
3326 if (ixgbe->link_state != LINK_STATE_UP) {
3327 switch (speed) {
3328 case IXGBE_LINK_SPEED_10GB_FULL:
3329 ixgbe->link_speed = SPEED_10GB;
3330 break;
3331 case IXGBE_LINK_SPEED_1GB_FULL:
3332 ixgbe->link_speed = SPEED_1GB;
3333 break;
3334 case IXGBE_LINK_SPEED_100_FULL:
3335 ixgbe->link_speed = SPEED_100;
3336 }
3337 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3338 ixgbe->link_state = LINK_STATE_UP;
3339 link_changed = B_TRUE;
3340 }
3341 } else {
3342 if (ixgbe->link_check_complete == B_TRUE ||
3343 (ixgbe->link_check_complete == B_FALSE &&
3344 gethrtime() >= ixgbe->link_check_hrtime)) {
3345 /*
3346 * The link is really down
3347 */
3348 ixgbe->link_check_complete = B_TRUE;
3349
3350 if (ixgbe->link_state != LINK_STATE_DOWN) {
3351 ixgbe->link_speed = 0;
3352 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3353 ixgbe->link_state = LINK_STATE_DOWN;
3354 link_changed = B_TRUE;
3355 }
3356 }
3357 }
3358
3359 /*
3360 * If we are in an interrupt context, need to re-enable the
3361 * interrupt, which was automasked
3362 */
3363 if (servicing_interrupt() != 0) {
3364 ixgbe->eims |= IXGBE_EICR_LSC;
3365 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3366 }
3367
3368 if (link_changed) {
3369 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3370 }
3371 }
3372
3373 /*
3374 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3375 */
3376 static void
3377 ixgbe_sfp_check(void *arg)
3378 {
3379 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3380 uint32_t eicr = ixgbe->eicr;
3381 struct ixgbe_hw *hw = &ixgbe->hw;
3382
3383 mutex_enter(&ixgbe->gen_lock);
3384 if (eicr & IXGBE_EICR_GPI_SDP1) {
3385 /* clear the interrupt */
3386 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
3387
3388 /* if link up, do multispeed fiber setup */
3389 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3390 B_TRUE, B_TRUE);
3391 ixgbe_driver_link_check(ixgbe);
3392 ixgbe_get_hw_state(ixgbe);
3393 } else if (eicr & IXGBE_EICR_GPI_SDP2) {
3394 /* clear the interrupt */
3395 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
3396
3397 /* if link up, do sfp module setup */
3398 (void) hw->mac.ops.setup_sfp(hw);
3399
3400 /* do multispeed fiber setup */
3401 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3402 B_TRUE, B_TRUE);
3403 ixgbe_driver_link_check(ixgbe);
3404 ixgbe_get_hw_state(ixgbe);
3405 }
3406 mutex_exit(&ixgbe->gen_lock);
3407
3408 /*
3409 * We need to fully re-check the link later.
3410 */
3411 ixgbe->link_check_complete = B_FALSE;
3412 ixgbe->link_check_hrtime = gethrtime() +
3413 (IXGBE_LINK_UP_TIME * 100000000ULL);
3414 }
3415
3416 /*
3417 * ixgbe_overtemp_check - overtemp module processing done in taskq
3418 *
3419 * This routine will only be called on adapters with temperature sensor.
3420 * The indication of over-temperature can be either SDP0 interrupt or the link
3421 * status change interrupt.
3422 */
3423 static void
3424 ixgbe_overtemp_check(void *arg)
3425 {
3426 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3427 struct ixgbe_hw *hw = &ixgbe->hw;
3428 uint32_t eicr = ixgbe->eicr;
3429 ixgbe_link_speed speed;
3430 boolean_t link_up;
3431
3432 mutex_enter(&ixgbe->gen_lock);
3433
3434 /* make sure we know current state of link */
3435 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3436
3437 /* check over-temp condition */
3438 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
3439 (eicr & IXGBE_EICR_LSC)) {
3440 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3441 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3442
3443 /*
3444 * Disable the adapter interrupts
3445 */
3446 ixgbe_disable_adapter_interrupts(ixgbe);
3447
3448 /*
3449 * Disable Rx/Tx units
3450 */
3451 (void) ixgbe_stop_adapter(hw);
3452
3453 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3454 ixgbe_error(ixgbe,
3455 "Problem: Network adapter has been stopped "
3456 "because it has overheated");
3457 ixgbe_error(ixgbe,
3458 "Action: Restart the computer. "
3459 "If the problem persists, power off the system "
3460 "and replace the adapter");
3461 }
3462 }
3463
3464 /* write to clear the interrupt */
3465 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3466
3467 mutex_exit(&ixgbe->gen_lock);
3468 }
3469
3470 /*
3471 * ixgbe_link_timer - timer for link status detection
3472 */
3473 static void
3474 ixgbe_link_timer(void *arg)
3475 {
3476 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3477
3478 mutex_enter(&ixgbe->gen_lock);
3479 ixgbe_driver_link_check(ixgbe);
3480 mutex_exit(&ixgbe->gen_lock);
3481 }
3482
3483 /*
3484 * ixgbe_local_timer - Driver watchdog function.
3485 *
3486 * This function will handle the transmit stall check and other routines.
3487 */
3488 static void
3489 ixgbe_local_timer(void *arg)
3490 {
3491 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3492
3493 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3494 goto out;
3495
3496 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3497 ixgbe->reset_count++;
3498 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3499 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3500 goto out;
3501 }
3502
3503 if (ixgbe_stall_check(ixgbe)) {
3504 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3505 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3506
3507 ixgbe->reset_count++;
3508 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3509 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3510 }
3511
3512 out:
3513 ixgbe_restart_watchdog_timer(ixgbe);
3514 }
3515
3516 /*
3517 * ixgbe_stall_check - Check for transmit stall.
3518 *
3519 * This function checks if the adapter is stalled (in transmit).
3520 *
3521 * It is called each time the watchdog timeout is invoked.
3522 * If the transmit descriptor reclaim continuously fails,
3523 * the watchdog value will increment by 1. If the watchdog
3524 * value exceeds the threshold, the ixgbe is assumed to
3525 * have stalled and need to be reset.
3526 */
3527 static boolean_t
3528 ixgbe_stall_check(ixgbe_t *ixgbe)
3529 {
3530 ixgbe_tx_ring_t *tx_ring;
3531 boolean_t result;
3532 int i;
3533
3534 if (ixgbe->link_state != LINK_STATE_UP)
3535 return (B_FALSE);
3536
3537 /*
3538 * If any tx ring is stalled, we'll reset the chipset
3539 */
3540 result = B_FALSE;
3541 for (i = 0; i < ixgbe->num_tx_rings; i++) {
3542 tx_ring = &ixgbe->tx_rings[i];
3543 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3544 tx_ring->tx_recycle(tx_ring);
3545 }
3546
3547 if (tx_ring->recycle_fail > 0)
3548 tx_ring->stall_watchdog++;
3549 else
3550 tx_ring->stall_watchdog = 0;
3551
3552 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3553 result = B_TRUE;
3554 break;
3555 }
3556 }
3557
3558 if (result) {
3559 tx_ring->stall_watchdog = 0;
3560 tx_ring->recycle_fail = 0;
3561 }
3562
3563 return (result);
3564 }
3565
3566
3567 /*
3568 * is_valid_mac_addr - Check if the mac address is valid.
3569 */
3570 static boolean_t
3571 is_valid_mac_addr(uint8_t *mac_addr)
3572 {
3573 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3574 const uint8_t addr_test2[6] =
3575 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3576
3577 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3578 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3579 return (B_FALSE);
3580
3581 return (B_TRUE);
3582 }
3583
3584 static boolean_t
3585 ixgbe_find_mac_address(ixgbe_t *ixgbe)
3586 {
3587 #ifdef __sparc
3588 struct ixgbe_hw *hw = &ixgbe->hw;
3589 uchar_t *bytes;
3590 struct ether_addr sysaddr;
3591 uint_t nelts;
3592 int err;
3593 boolean_t found = B_FALSE;
3594
3595 /*
3596 * The "vendor's factory-set address" may already have
3597 * been extracted from the chip, but if the property
3598 * "local-mac-address" is set we use that instead.
3599 *
3600 * We check whether it looks like an array of 6
3601 * bytes (which it should, if OBP set it). If we can't
3602 * make sense of it this way, we'll ignore it.
3603 */
3604 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3605 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3606 if (err == DDI_PROP_SUCCESS) {
3607 if (nelts == ETHERADDRL) {
3608 while (nelts--)
3609 hw->mac.addr[nelts] = bytes[nelts];
3610 found = B_TRUE;
3611 }
3612 ddi_prop_free(bytes);
3613 }
3614
3615 /*
3616 * Look up the OBP property "local-mac-address?". If the user has set
3617 * 'local-mac-address? = false', use "the system address" instead.
3618 */
3619 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3620 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3621 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3622 if (localetheraddr(NULL, &sysaddr) != 0) {
3623 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3624 found = B_TRUE;
3625 }
3626 }
3627 ddi_prop_free(bytes);
3628 }
3629
3630 /*
3631 * Finally(!), if there's a valid "mac-address" property (created
3632 * if we netbooted from this interface), we must use this instead
3633 * of any of the above to ensure that the NFS/install server doesn't
3634 * get confused by the address changing as Solaris takes over!
3635 */
3636 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3637 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3638 if (err == DDI_PROP_SUCCESS) {
3639 if (nelts == ETHERADDRL) {
3640 while (nelts--)
3641 hw->mac.addr[nelts] = bytes[nelts];
3642 found = B_TRUE;
3643 }
3644 ddi_prop_free(bytes);
3645 }
3646
3647 if (found) {
3648 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3649 return (B_TRUE);
3650 }
3651 #else
3652 _NOTE(ARGUNUSED(ixgbe));
3653 #endif
3654
3655 return (B_TRUE);
3656 }
3657
3658 #pragma inline(ixgbe_arm_watchdog_timer)
3659 static void
3660 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
3661 {
3662 /*
3663 * Fire a watchdog timer
3664 */
3665 ixgbe->watchdog_tid =
3666 timeout(ixgbe_local_timer,
3667 (void *)ixgbe, 1 * drv_usectohz(1000000));
3668
3669 }
3670
3671 /*
3672 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
3673 */
3674 void
3675 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
3676 {
3677 mutex_enter(&ixgbe->watchdog_lock);
3678
3679 if (!ixgbe->watchdog_enable) {
3680 ixgbe->watchdog_enable = B_TRUE;
3681 ixgbe->watchdog_start = B_TRUE;
3682 ixgbe_arm_watchdog_timer(ixgbe);
3683 }
3684
3685 mutex_exit(&ixgbe->watchdog_lock);
3686 }
3687
3688 /*
3689 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
3690 */
3691 void
3692 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
3693 {
3694 timeout_id_t tid;
3695
3696 mutex_enter(&ixgbe->watchdog_lock);
3697
3698 ixgbe->watchdog_enable = B_FALSE;
3699 ixgbe->watchdog_start = B_FALSE;
3700 tid = ixgbe->watchdog_tid;
3701 ixgbe->watchdog_tid = 0;
3702
3703 mutex_exit(&ixgbe->watchdog_lock);
3704
3705 if (tid != 0)
3706 (void) untimeout(tid);
3707 }
3708
3709 /*
3710 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
3711 */
3712 void
3713 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
3714 {
3715 mutex_enter(&ixgbe->watchdog_lock);
3716
3717 if (ixgbe->watchdog_enable) {
3718 if (!ixgbe->watchdog_start) {
3719 ixgbe->watchdog_start = B_TRUE;
3720 ixgbe_arm_watchdog_timer(ixgbe);
3721 }
3722 }
3723
3724 mutex_exit(&ixgbe->watchdog_lock);
3725 }
3726
3727 /*
3728 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
3729 */
3730 static void
3731 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
3732 {
3733 mutex_enter(&ixgbe->watchdog_lock);
3734
3735 if (ixgbe->watchdog_start)
3736 ixgbe_arm_watchdog_timer(ixgbe);
3737
3738 mutex_exit(&ixgbe->watchdog_lock);
3739 }
3740
3741 /*
3742 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
3743 */
3744 void
3745 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
3746 {
3747 timeout_id_t tid;
3748
3749 mutex_enter(&ixgbe->watchdog_lock);
3750
3751 ixgbe->watchdog_start = B_FALSE;
3752 tid = ixgbe->watchdog_tid;
3753 ixgbe->watchdog_tid = 0;
3754
3755 mutex_exit(&ixgbe->watchdog_lock);
3756
3757 if (tid != 0)
3758 (void) untimeout(tid);
3759 }
3760
3761 /*
3762 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
3763 */
3764 static void
3765 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
3766 {
3767 struct ixgbe_hw *hw = &ixgbe->hw;
3768
3769 /*
3770 * mask all interrupts off
3771 */
3772 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
3773
3774 /*
3775 * for MSI-X, also disable autoclear
3776 */
3777 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3778 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
3779 }
3780
3781 IXGBE_WRITE_FLUSH(hw);
3782 }
3783
3784 /*
3785 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
3786 */
3787 static void
3788 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
3789 {
3790 struct ixgbe_hw *hw = &ixgbe->hw;
3791 uint32_t eiac, eiam;
3792 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3793
3794 /* interrupt types to enable */
3795 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
3796 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
3797 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
3798
3799 /* enable automask on "other" causes that this adapter can generate */
3800 eiam = ixgbe->capab->other_intr;
3801
3802 /*
3803 * msi-x mode
3804 */
3805 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3806 /* enable autoclear but not on bits 29:20 */
3807 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3808
3809 /* general purpose interrupt enable */
3810 gpie |= (IXGBE_GPIE_MSIX_MODE
3811 | IXGBE_GPIE_PBA_SUPPORT
3812 | IXGBE_GPIE_OCD
3813 | IXGBE_GPIE_EIAME);
3814 /*
3815 * non-msi-x mode
3816 */
3817 } else {
3818
3819 /* disable autoclear, leave gpie at default */
3820 eiac = 0;
3821
3822 /*
3823 * General purpose interrupt enable.
3824 * For 82599, extended interrupt automask enable
3825 * only in MSI or MSI-X mode
3826 */
3827 if ((hw->mac.type < ixgbe_mac_82599EB) ||
3828 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3829 gpie |= IXGBE_GPIE_EIAME;
3830 }
3831 }
3832
3833 /* Enable specific "other" interrupt types */
3834 switch (hw->mac.type) {
3835 case ixgbe_mac_82598EB:
3836 gpie |= ixgbe->capab->other_gpie;
3837 break;
3838
3839 case ixgbe_mac_82599EB:
3840 case ixgbe_mac_X540:
3841 gpie |= ixgbe->capab->other_gpie;
3842
3843 /* Enable RSC Delay 8us when LRO enabled */
3844 if (ixgbe->lro_enable) {
3845 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3846 }
3847 break;
3848
3849 default:
3850 break;
3851 }
3852
3853 /* write to interrupt control registers */
3854 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3855 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3856 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3857 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3858 IXGBE_WRITE_FLUSH(hw);
3859 }
3860
3861 /*
3862 * ixgbe_loopback_ioctl - Loopback support.
3863 */
3864 enum ioc_reply
3865 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
3866 {
3867 lb_info_sz_t *lbsp;
3868 lb_property_t *lbpp;
3869 uint32_t *lbmp;
3870 uint32_t size;
3871 uint32_t value;
3872
3873 if (mp->b_cont == NULL)
3874 return (IOC_INVAL);
3875
3876 switch (iocp->ioc_cmd) {
3877 default:
3878 return (IOC_INVAL);
3879
3880 case LB_GET_INFO_SIZE:
3881 size = sizeof (lb_info_sz_t);
3882 if (iocp->ioc_count != size)
3883 return (IOC_INVAL);
3884
3885 value = sizeof (lb_normal);
3886 value += sizeof (lb_mac);
3887 value += sizeof (lb_external);
3888
3889 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3890 *lbsp = value;
3891 break;
3892
3893 case LB_GET_INFO:
3894 value = sizeof (lb_normal);
3895 value += sizeof (lb_mac);
3896 value += sizeof (lb_external);
3897
3898 size = value;
3899 if (iocp->ioc_count != size)
3900 return (IOC_INVAL);
3901
3902 value = 0;
3903 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3904
3905 lbpp[value++] = lb_normal;
3906 lbpp[value++] = lb_mac;
3907 lbpp[value++] = lb_external;
3908 break;
3909
3910 case LB_GET_MODE:
3911 size = sizeof (uint32_t);
3912 if (iocp->ioc_count != size)
3913 return (IOC_INVAL);
3914
3915 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3916 *lbmp = ixgbe->loopback_mode;
3917 break;
3918
3919 case LB_SET_MODE:
3920 size = 0;
3921 if (iocp->ioc_count != sizeof (uint32_t))
3922 return (IOC_INVAL);
3923
3924 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3925 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
3926 return (IOC_INVAL);
3927 break;
3928 }
3929
3930 iocp->ioc_count = size;
3931 iocp->ioc_error = 0;
3932
3933 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3934 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3935 return (IOC_INVAL);
3936 }
3937
3938 return (IOC_REPLY);
3939 }
3940
3941 /*
3942 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
3943 */
3944 static boolean_t
3945 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
3946 {
3947 if (mode == ixgbe->loopback_mode)
3948 return (B_TRUE);
3949
3950 ixgbe->loopback_mode = mode;
3951
3952 if (mode == IXGBE_LB_NONE) {
3953 /*
3954 * Reset the chip
3955 */
3956 (void) ixgbe_reset(ixgbe);
3957 return (B_TRUE);
3958 }
3959
3960 mutex_enter(&ixgbe->gen_lock);
3961
3962 switch (mode) {
3963 default:
3964 mutex_exit(&ixgbe->gen_lock);
3965 return (B_FALSE);
3966
3967 case IXGBE_LB_EXTERNAL:
3968 break;
3969
3970 case IXGBE_LB_INTERNAL_MAC:
3971 ixgbe_set_internal_mac_loopback(ixgbe);
3972 break;
3973 }
3974
3975 mutex_exit(&ixgbe->gen_lock);
3976
3977 return (B_TRUE);
3978 }
3979
3980 /*
3981 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
3982 */
3983 static void
3984 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
3985 {
3986 struct ixgbe_hw *hw;
3987 uint32_t reg;
3988 uint8_t atlas;
3989
3990 hw = &ixgbe->hw;
3991
3992 /*
3993 * Setup MAC loopback
3994 */
3995 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
3996 reg |= IXGBE_HLREG0_LPBK;
3997 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
3998
3999 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4000 reg &= ~IXGBE_AUTOC_LMS_MASK;
4001 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4002
4003 /*
4004 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4005 */
4006 switch (hw->mac.type) {
4007 case ixgbe_mac_82598EB:
4008 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4009 &atlas);
4010 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4011 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4012 atlas);
4013
4014 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4015 &atlas);
4016 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4017 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4018 atlas);
4019
4020 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4021 &atlas);
4022 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4023 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4024 atlas);
4025
4026 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4027 &atlas);
4028 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4029 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4030 atlas);
4031 break;
4032
4033 case ixgbe_mac_82599EB:
4034 case ixgbe_mac_X540:
4035 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4036 reg |= (IXGBE_AUTOC_FLU |
4037 IXGBE_AUTOC_10G_KX4);
4038 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4039
4040 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4041 B_FALSE, B_TRUE);
4042 break;
4043
4044 default:
4045 break;
4046 }
4047 }
4048
4049 #pragma inline(ixgbe_intr_rx_work)
4050 /*
4051 * ixgbe_intr_rx_work - RX processing of ISR.
4052 */
4053 static void
4054 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4055 {
4056 mblk_t *mp;
4057
4058 mutex_enter(&rx_ring->rx_lock);
4059
4060 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4061 mutex_exit(&rx_ring->rx_lock);
4062
4063 if (mp != NULL)
4064 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4065 rx_ring->ring_gen_num);
4066 }
4067
4068 #pragma inline(ixgbe_intr_tx_work)
4069 /*
4070 * ixgbe_intr_tx_work - TX processing of ISR.
4071 */
4072 static void
4073 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4074 {
4075 ixgbe_t *ixgbe = tx_ring->ixgbe;
4076
4077 /*
4078 * Recycle the tx descriptors
4079 */
4080 tx_ring->tx_recycle(tx_ring);
4081
4082 /*
4083 * Schedule the re-transmit
4084 */
4085 if (tx_ring->reschedule &&
4086 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4087 tx_ring->reschedule = B_FALSE;
4088 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4089 tx_ring->ring_handle);
4090 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4091 }
4092 }
4093
4094 #pragma inline(ixgbe_intr_other_work)
4095 /*
4096 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4097 */
4098 static void
4099 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4100 {
4101 ASSERT(mutex_owned(&ixgbe->gen_lock));
4102
4103 /*
4104 * handle link status change
4105 */
4106 if (eicr & IXGBE_EICR_LSC) {
4107 ixgbe_driver_link_check(ixgbe);
4108 ixgbe_get_hw_state(ixgbe);
4109 }
4110
4111 /*
4112 * check for fan failure on adapters with fans
4113 */
4114 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4115 (eicr & IXGBE_EICR_GPI_SDP1)) {
4116 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4117
4118 /*
4119 * Disable the adapter interrupts
4120 */
4121 ixgbe_disable_adapter_interrupts(ixgbe);
4122
4123 /*
4124 * Disable Rx/Tx units
4125 */
4126 (void) ixgbe_stop_adapter(&ixgbe->hw);
4127
4128 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4129 ixgbe_error(ixgbe,
4130 "Problem: Network adapter has been stopped "
4131 "because the fan has stopped.\n");
4132 ixgbe_error(ixgbe,
4133 "Action: Replace the adapter.\n");
4134
4135 /* re-enable the interrupt, which was automasked */
4136 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4137 }
4138
4139 /*
4140 * Do SFP check for adapters with hot-plug capability
4141 */
4142 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4143 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) {
4144 ixgbe->eicr = eicr;
4145 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4146 ixgbe_sfp_check, (void *)ixgbe,
4147 DDI_NOSLEEP)) != DDI_SUCCESS) {
4148 ixgbe_log(ixgbe, "No memory available to dispatch "
4149 "taskq for SFP check");
4150 }
4151 }
4152
4153 /*
4154 * Do over-temperature check for adapters with temp sensor
4155 */
4156 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4157 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
4158 ixgbe->eicr = eicr;
4159 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4160 ixgbe_overtemp_check, (void *)ixgbe,
4161 DDI_NOSLEEP)) != DDI_SUCCESS) {
4162 ixgbe_log(ixgbe, "No memory available to dispatch "
4163 "taskq for overtemp check");
4164 }
4165 }
4166 }
4167
4168 /*
4169 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4170 */
4171 static uint_t
4172 ixgbe_intr_legacy(void *arg1, void *arg2)
4173 {
4174 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4175 struct ixgbe_hw *hw = &ixgbe->hw;
4176 ixgbe_tx_ring_t *tx_ring;
4177 ixgbe_rx_ring_t *rx_ring;
4178 uint32_t eicr;
4179 mblk_t *mp;
4180 boolean_t tx_reschedule;
4181 uint_t result;
4182
4183 _NOTE(ARGUNUSED(arg2));
4184
4185 mutex_enter(&ixgbe->gen_lock);
4186 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4187 mutex_exit(&ixgbe->gen_lock);
4188 return (DDI_INTR_UNCLAIMED);
4189 }
4190
4191 mp = NULL;
4192 tx_reschedule = B_FALSE;
4193
4194 /*
4195 * Any bit set in eicr: claim this interrupt
4196 */
4197 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4198
4199 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4200 mutex_exit(&ixgbe->gen_lock);
4201 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4202 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4203 return (DDI_INTR_CLAIMED);
4204 }
4205
4206 if (eicr) {
4207 /*
4208 * For legacy interrupt, we have only one interrupt,
4209 * so we have only one rx ring and one tx ring enabled.
4210 */
4211 ASSERT(ixgbe->num_rx_rings == 1);
4212 ASSERT(ixgbe->num_tx_rings == 1);
4213
4214 /*
4215 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4216 */
4217 if (eicr & 0x1) {
4218 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4219 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4220 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4221 /*
4222 * Clean the rx descriptors
4223 */
4224 rx_ring = &ixgbe->rx_rings[0];
4225 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4226 }
4227
4228 /*
4229 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4230 */
4231 if (eicr & 0x2) {
4232 /*
4233 * Recycle the tx descriptors
4234 */
4235 tx_ring = &ixgbe->tx_rings[0];
4236 tx_ring->tx_recycle(tx_ring);
4237
4238 /*
4239 * Schedule the re-transmit
4240 */
4241 tx_reschedule = (tx_ring->reschedule &&
4242 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4243 }
4244
4245 /* any interrupt type other than tx/rx */
4246 if (eicr & ixgbe->capab->other_intr) {
4247 switch (hw->mac.type) {
4248 case ixgbe_mac_82598EB:
4249 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4250 break;
4251
4252 case ixgbe_mac_82599EB:
4253 case ixgbe_mac_X540:
4254 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4255 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4256 break;
4257
4258 default:
4259 break;
4260 }
4261 ixgbe_intr_other_work(ixgbe, eicr);
4262 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4263 }
4264
4265 mutex_exit(&ixgbe->gen_lock);
4266
4267 result = DDI_INTR_CLAIMED;
4268 } else {
4269 mutex_exit(&ixgbe->gen_lock);
4270
4271 /*
4272 * No interrupt cause bits set: don't claim this interrupt.
4273 */
4274 result = DDI_INTR_UNCLAIMED;
4275 }
4276
4277 /* re-enable the interrupts which were automasked */
4278 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4279
4280 /*
4281 * Do the following work outside of the gen_lock
4282 */
4283 if (mp != NULL) {
4284 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4285 rx_ring->ring_gen_num);
4286 }
4287
4288 if (tx_reschedule) {
4289 tx_ring->reschedule = B_FALSE;
4290 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4291 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4292 }
4293
4294 return (result);
4295 }
4296
4297 /*
4298 * ixgbe_intr_msi - Interrupt handler for MSI.
4299 */
4300 static uint_t
4301 ixgbe_intr_msi(void *arg1, void *arg2)
4302 {
4303 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4304 struct ixgbe_hw *hw = &ixgbe->hw;
4305 uint32_t eicr;
4306
4307 _NOTE(ARGUNUSED(arg2));
4308
4309 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4310
4311 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4312 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4313 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4314 return (DDI_INTR_CLAIMED);
4315 }
4316
4317 /*
4318 * For MSI interrupt, we have only one vector,
4319 * so we have only one rx ring and one tx ring enabled.
4320 */
4321 ASSERT(ixgbe->num_rx_rings == 1);
4322 ASSERT(ixgbe->num_tx_rings == 1);
4323
4324 /*
4325 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4326 */
4327 if (eicr & 0x1) {
4328 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4329 }
4330
4331 /*
4332 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4333 */
4334 if (eicr & 0x2) {
4335 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4336 }
4337
4338 /* any interrupt type other than tx/rx */
4339 if (eicr & ixgbe->capab->other_intr) {
4340 mutex_enter(&ixgbe->gen_lock);
4341 switch (hw->mac.type) {
4342 case ixgbe_mac_82598EB:
4343 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4344 break;
4345
4346 case ixgbe_mac_82599EB:
4347 case ixgbe_mac_X540:
4348 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4349 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4350 break;
4351
4352 default:
4353 break;
4354 }
4355 ixgbe_intr_other_work(ixgbe, eicr);
4356 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4357 mutex_exit(&ixgbe->gen_lock);
4358 }
4359
4360 /* re-enable the interrupts which were automasked */
4361 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4362
4363 return (DDI_INTR_CLAIMED);
4364 }
4365
4366 /*
4367 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4368 */
4369 static uint_t
4370 ixgbe_intr_msix(void *arg1, void *arg2)
4371 {
4372 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4373 ixgbe_t *ixgbe = vect->ixgbe;
4374 struct ixgbe_hw *hw = &ixgbe->hw;
4375 uint32_t eicr;
4376 int r_idx = 0;
4377
4378 _NOTE(ARGUNUSED(arg2));
4379
4380 /*
4381 * Clean each rx ring that has its bit set in the map
4382 */
4383 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4384 while (r_idx >= 0) {
4385 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4386 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4387 (ixgbe->num_rx_rings - 1));
4388 }
4389
4390 /*
4391 * Clean each tx ring that has its bit set in the map
4392 */
4393 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4394 while (r_idx >= 0) {
4395 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4396 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4397 (ixgbe->num_tx_rings - 1));
4398 }
4399
4400
4401 /*
4402 * Clean other interrupt (link change) that has its bit set in the map
4403 */
4404 if (BT_TEST(vect->other_map, 0) == 1) {
4405 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4406
4407 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4408 DDI_FM_OK) {
4409 ddi_fm_service_impact(ixgbe->dip,
4410 DDI_SERVICE_DEGRADED);
4411 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4412 return (DDI_INTR_CLAIMED);
4413 }
4414
4415 /*
4416 * Check "other" cause bits: any interrupt type other than tx/rx
4417 */
4418 if (eicr & ixgbe->capab->other_intr) {
4419 mutex_enter(&ixgbe->gen_lock);
4420 switch (hw->mac.type) {
4421 case ixgbe_mac_82598EB:
4422 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4423 ixgbe_intr_other_work(ixgbe, eicr);
4424 break;
4425
4426 case ixgbe_mac_82599EB:
4427 case ixgbe_mac_X540:
4428 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4429 ixgbe_intr_other_work(ixgbe, eicr);
4430 break;
4431
4432 default:
4433 break;
4434 }
4435 mutex_exit(&ixgbe->gen_lock);
4436 }
4437
4438 /* re-enable the interrupts which were automasked */
4439 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4440 }
4441
4442 return (DDI_INTR_CLAIMED);
4443 }
4444
4445 /*
4446 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4447 *
4448 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4449 * if not successful, try Legacy.
4450 * ixgbe->intr_force can be used to force sequence to start with
4451 * any of the 3 types.
4452 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4453 */
4454 static int
4455 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4456 {
4457 dev_info_t *devinfo;
4458 int intr_types;
4459 int rc;
4460
4461 devinfo = ixgbe->dip;
4462
4463 /*
4464 * Get supported interrupt types
4465 */
4466 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4467
4468 if (rc != DDI_SUCCESS) {
4469 ixgbe_log(ixgbe,
4470 "Get supported interrupt types failed: %d", rc);
4471 return (IXGBE_FAILURE);
4472 }
4473 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4474
4475 ixgbe->intr_type = 0;
4476
4477 /*
4478 * Install MSI-X interrupts
4479 */
4480 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4481 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4482 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4483 if (rc == IXGBE_SUCCESS)
4484 return (IXGBE_SUCCESS);
4485
4486 ixgbe_log(ixgbe,
4487 "Allocate MSI-X failed, trying MSI interrupts...");
4488 }
4489
4490 /*
4491 * MSI-X not used, force rings and groups to 1
4492 */
4493 ixgbe->num_rx_rings = 1;
4494 ixgbe->num_rx_groups = 1;
4495 ixgbe->num_tx_rings = 1;
4496 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4497 ixgbe_log(ixgbe,
4498 "MSI-X not used, force rings and groups number to 1");
4499
4500 /*
4501 * Install MSI interrupts
4502 */
4503 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4504 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4505 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4506 if (rc == IXGBE_SUCCESS)
4507 return (IXGBE_SUCCESS);
4508
4509 ixgbe_log(ixgbe,
4510 "Allocate MSI failed, trying Legacy interrupts...");
4511 }
4512
4513 /*
4514 * Install legacy interrupts
4515 */
4516 if (intr_types & DDI_INTR_TYPE_FIXED) {
4517 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4518 if (rc == IXGBE_SUCCESS)
4519 return (IXGBE_SUCCESS);
4520
4521 ixgbe_log(ixgbe,
4522 "Allocate Legacy interrupts failed");
4523 }
4524
4525 /*
4526 * If none of the 3 types succeeded, return failure
4527 */
4528 return (IXGBE_FAILURE);
4529 }
4530
4531 /*
4532 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4533 *
4534 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4535 * if fewer than 2 handles are available, return failure.
4536 * Upon success, this maps the vectors to rx and tx rings for
4537 * interrupts.
4538 */
4539 static int
4540 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4541 {
4542 dev_info_t *devinfo;
4543 int request, count, actual;
4544 int minimum;
4545 int rc;
4546 uint32_t ring_per_group;
4547
4548 devinfo = ixgbe->dip;
4549
4550 switch (intr_type) {
4551 case DDI_INTR_TYPE_FIXED:
4552 request = 1; /* Request 1 legacy interrupt handle */
4553 minimum = 1;
4554 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4555 break;
4556
4557 case DDI_INTR_TYPE_MSI:
4558 request = 1; /* Request 1 MSI interrupt handle */
4559 minimum = 1;
4560 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4561 break;
4562
4563 case DDI_INTR_TYPE_MSIX:
4564 /*
4565 * Best number of vectors for the adapter is
4566 * (# rx rings + # tx rings), however we will
4567 * limit the request number.
4568 */
4569 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4570 if (request > ixgbe->capab->max_ring_vect)
4571 request = ixgbe->capab->max_ring_vect;
4572 minimum = 1;
4573 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4574 break;
4575
4576 default:
4577 ixgbe_log(ixgbe,
4578 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4579 intr_type);
4580 return (IXGBE_FAILURE);
4581 }
4582 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
4583 request, minimum);
4584
4585 /*
4586 * Get number of supported interrupts
4587 */
4588 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4589 if ((rc != DDI_SUCCESS) || (count < minimum)) {
4590 ixgbe_log(ixgbe,
4591 "Get interrupt number failed. Return: %d, count: %d",
4592 rc, count);
4593 return (IXGBE_FAILURE);
4594 }
4595 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4596
4597 actual = 0;
4598 ixgbe->intr_cnt = 0;
4599 ixgbe->intr_cnt_max = 0;
4600 ixgbe->intr_cnt_min = 0;
4601
4602 /*
4603 * Allocate an array of interrupt handles
4604 */
4605 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
4606 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
4607
4608 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
4609 request, &actual, DDI_INTR_ALLOC_NORMAL);
4610 if (rc != DDI_SUCCESS) {
4611 ixgbe_log(ixgbe, "Allocate interrupts failed. "
4612 "return: %d, request: %d, actual: %d",
4613 rc, request, actual);
4614 goto alloc_handle_fail;
4615 }
4616 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
4617
4618 /*
4619 * upper/lower limit of interrupts
4620 */
4621 ixgbe->intr_cnt = actual;
4622 ixgbe->intr_cnt_max = request;
4623 ixgbe->intr_cnt_min = minimum;
4624
4625 /*
4626 * rss number per group should not exceed the rx interrupt number,
4627 * else need to adjust rx ring number.
4628 */
4629 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4630 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
4631 if (actual < ring_per_group) {
4632 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
4633 ixgbe_setup_vmdq_rss_conf(ixgbe);
4634 }
4635
4636 /*
4637 * Now we know the actual number of vectors. Here we map the vector
4638 * to other, rx rings and tx ring.
4639 */
4640 if (actual < minimum) {
4641 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
4642 actual);
4643 goto alloc_handle_fail;
4644 }
4645
4646 /*
4647 * Get priority for first vector, assume remaining are all the same
4648 */
4649 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
4650 if (rc != DDI_SUCCESS) {
4651 ixgbe_log(ixgbe,
4652 "Get interrupt priority failed: %d", rc);
4653 goto alloc_handle_fail;
4654 }
4655
4656 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
4657 if (rc != DDI_SUCCESS) {
4658 ixgbe_log(ixgbe,
4659 "Get interrupt cap failed: %d", rc);
4660 goto alloc_handle_fail;
4661 }
4662
4663 ixgbe->intr_type = intr_type;
4664
4665 return (IXGBE_SUCCESS);
4666
4667 alloc_handle_fail:
4668 ixgbe_rem_intrs(ixgbe);
4669
4670 return (IXGBE_FAILURE);
4671 }
4672
4673 /*
4674 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
4675 *
4676 * Before adding the interrupt handlers, the interrupt vectors have
4677 * been allocated, and the rx/tx rings have also been allocated.
4678 */
4679 static int
4680 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
4681 {
4682 int vector = 0;
4683 int rc;
4684
4685 switch (ixgbe->intr_type) {
4686 case DDI_INTR_TYPE_MSIX:
4687 /*
4688 * Add interrupt handler for all vectors
4689 */
4690 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
4691 /*
4692 * install pointer to vect_map[vector]
4693 */
4694 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4695 (ddi_intr_handler_t *)ixgbe_intr_msix,
4696 (void *)&ixgbe->vect_map[vector], NULL);
4697
4698 if (rc != DDI_SUCCESS) {
4699 ixgbe_log(ixgbe,
4700 "Add interrupt handler failed. "
4701 "return: %d, vector: %d", rc, vector);
4702 for (vector--; vector >= 0; vector--) {
4703 (void) ddi_intr_remove_handler(
4704 ixgbe->htable[vector]);
4705 }
4706 return (IXGBE_FAILURE);
4707 }
4708 }
4709
4710 break;
4711
4712 case DDI_INTR_TYPE_MSI:
4713 /*
4714 * Add interrupt handlers for the only vector
4715 */
4716 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4717 (ddi_intr_handler_t *)ixgbe_intr_msi,
4718 (void *)ixgbe, NULL);
4719
4720 if (rc != DDI_SUCCESS) {
4721 ixgbe_log(ixgbe,
4722 "Add MSI interrupt handler failed: %d", rc);
4723 return (IXGBE_FAILURE);
4724 }
4725
4726 break;
4727
4728 case DDI_INTR_TYPE_FIXED:
4729 /*
4730 * Add interrupt handlers for the only vector
4731 */
4732 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4733 (ddi_intr_handler_t *)ixgbe_intr_legacy,
4734 (void *)ixgbe, NULL);
4735
4736 if (rc != DDI_SUCCESS) {
4737 ixgbe_log(ixgbe,
4738 "Add legacy interrupt handler failed: %d", rc);
4739 return (IXGBE_FAILURE);
4740 }
4741
4742 break;
4743
4744 default:
4745 return (IXGBE_FAILURE);
4746 }
4747
4748 return (IXGBE_SUCCESS);
4749 }
4750
4751 #pragma inline(ixgbe_map_rxring_to_vector)
4752 /*
4753 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
4754 */
4755 static void
4756 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
4757 {
4758 /*
4759 * Set bit in map
4760 */
4761 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4762
4763 /*
4764 * Count bits set
4765 */
4766 ixgbe->vect_map[v_idx].rxr_cnt++;
4767
4768 /*
4769 * Remember bit position
4770 */
4771 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
4772 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
4773 }
4774
4775 #pragma inline(ixgbe_map_txring_to_vector)
4776 /*
4777 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
4778 */
4779 static void
4780 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
4781 {
4782 /*
4783 * Set bit in map
4784 */
4785 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
4786
4787 /*
4788 * Count bits set
4789 */
4790 ixgbe->vect_map[v_idx].txr_cnt++;
4791
4792 /*
4793 * Remember bit position
4794 */
4795 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
4796 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
4797 }
4798
4799 /*
4800 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
4801 * allocation register (IVAR).
4802 * cause:
4803 * -1 : other cause
4804 * 0 : rx
4805 * 1 : tx
4806 */
4807 static void
4808 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
4809 int8_t cause)
4810 {
4811 struct ixgbe_hw *hw = &ixgbe->hw;
4812 u32 ivar, index;
4813
4814 switch (hw->mac.type) {
4815 case ixgbe_mac_82598EB:
4816 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4817 if (cause == -1) {
4818 cause = 0;
4819 }
4820 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4821 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4822 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4823 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4824 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4825 break;
4826
4827 case ixgbe_mac_82599EB:
4828 case ixgbe_mac_X540:
4829 if (cause == -1) {
4830 /* other causes */
4831 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4832 index = (intr_alloc_entry & 1) * 8;
4833 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4834 ivar &= ~(0xFF << index);
4835 ivar |= (msix_vector << index);
4836 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4837 } else {
4838 /* tx or rx causes */
4839 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4840 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4841 ivar = IXGBE_READ_REG(hw,
4842 IXGBE_IVAR(intr_alloc_entry >> 1));
4843 ivar &= ~(0xFF << index);
4844 ivar |= (msix_vector << index);
4845 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4846 ivar);
4847 }
4848 break;
4849
4850 default:
4851 break;
4852 }
4853 }
4854
4855 /*
4856 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
4857 * given interrupt vector allocation register (IVAR).
4858 * cause:
4859 * -1 : other cause
4860 * 0 : rx
4861 * 1 : tx
4862 */
4863 static void
4864 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4865 {
4866 struct ixgbe_hw *hw = &ixgbe->hw;
4867 u32 ivar, index;
4868
4869 switch (hw->mac.type) {
4870 case ixgbe_mac_82598EB:
4871 if (cause == -1) {
4872 cause = 0;
4873 }
4874 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4875 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4876 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4877 (intr_alloc_entry & 0x3)));
4878 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4879 break;
4880
4881 case ixgbe_mac_82599EB:
4882 case ixgbe_mac_X540:
4883 if (cause == -1) {
4884 /* other causes */
4885 index = (intr_alloc_entry & 1) * 8;
4886 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4887 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4888 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4889 } else {
4890 /* tx or rx causes */
4891 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4892 ivar = IXGBE_READ_REG(hw,
4893 IXGBE_IVAR(intr_alloc_entry >> 1));
4894 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4895 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4896 ivar);
4897 }
4898 break;
4899
4900 default:
4901 break;
4902 }
4903 }
4904
4905 /*
4906 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
4907 * given interrupt vector allocation register (IVAR).
4908 * cause:
4909 * -1 : other cause
4910 * 0 : rx
4911 * 1 : tx
4912 */
4913 static void
4914 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4915 {
4916 struct ixgbe_hw *hw = &ixgbe->hw;
4917 u32 ivar, index;
4918
4919 switch (hw->mac.type) {
4920 case ixgbe_mac_82598EB:
4921 if (cause == -1) {
4922 cause = 0;
4923 }
4924 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4925 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4926 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
4927 (intr_alloc_entry & 0x3)));
4928 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4929 break;
4930
4931 case ixgbe_mac_82599EB:
4932 case ixgbe_mac_X540:
4933 if (cause == -1) {
4934 /* other causes */
4935 index = (intr_alloc_entry & 1) * 8;
4936 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4937 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4938 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4939 } else {
4940 /* tx or rx causes */
4941 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4942 ivar = IXGBE_READ_REG(hw,
4943 IXGBE_IVAR(intr_alloc_entry >> 1));
4944 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4945 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4946 ivar);
4947 }
4948 break;
4949
4950 default:
4951 break;
4952 }
4953 }
4954
4955 /*
4956 * Convert the rx ring index driver maintained to the rx ring index
4957 * in h/w.
4958 */
4959 static uint32_t
4960 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
4961 {
4962
4963 struct ixgbe_hw *hw = &ixgbe->hw;
4964 uint32_t rx_ring_per_group, hw_rx_index;
4965
4966 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
4967 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
4968 return (sw_rx_index);
4969 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
4970 switch (hw->mac.type) {
4971 case ixgbe_mac_82598EB:
4972 return (sw_rx_index);
4973
4974 case ixgbe_mac_82599EB:
4975 case ixgbe_mac_X540:
4976 return (sw_rx_index * 2);
4977
4978 default:
4979 break;
4980 }
4981 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
4982 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4983
4984 switch (hw->mac.type) {
4985 case ixgbe_mac_82598EB:
4986 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
4987 16 + (sw_rx_index % rx_ring_per_group);
4988 return (hw_rx_index);
4989
4990 case ixgbe_mac_82599EB:
4991 case ixgbe_mac_X540:
4992 if (ixgbe->num_rx_groups > 32) {
4993 hw_rx_index = (sw_rx_index /
4994 rx_ring_per_group) * 2 +
4995 (sw_rx_index % rx_ring_per_group);
4996 } else {
4997 hw_rx_index = (sw_rx_index /
4998 rx_ring_per_group) * 4 +
4999 (sw_rx_index % rx_ring_per_group);
5000 }
5001 return (hw_rx_index);
5002
5003 default:
5004 break;
5005 }
5006 }
5007
5008 /*
5009 * Should never reach. Just to make compiler happy.
5010 */
5011 return (sw_rx_index);
5012 }
5013
5014 /*
5015 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5016 *
5017 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5018 * to vector[0 - (intr_cnt -1)].
5019 */
5020 static int
5021 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5022 {
5023 int i, vector = 0;
5024
5025 /* initialize vector map */
5026 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5027 for (i = 0; i < ixgbe->intr_cnt; i++) {
5028 ixgbe->vect_map[i].ixgbe = ixgbe;
5029 }
5030
5031 /*
5032 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5033 * tx rings[0] on RTxQ[1].
5034 */
5035 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5036 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5037 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5038 return (IXGBE_SUCCESS);
5039 }
5040
5041 /*
5042 * Interrupts/vectors mapping for MSI-X
5043 */
5044
5045 /*
5046 * Map other interrupt to vector 0,
5047 * Set bit in map and count the bits set.
5048 */
5049 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5050 ixgbe->vect_map[vector].other_cnt++;
5051
5052 /*
5053 * Map rx ring interrupts to vectors
5054 */
5055 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5056 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5057 vector = (vector +1) % ixgbe->intr_cnt;
5058 }
5059
5060 /*
5061 * Map tx ring interrupts to vectors
5062 */
5063 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5064 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5065 vector = (vector +1) % ixgbe->intr_cnt;
5066 }
5067
5068 return (IXGBE_SUCCESS);
5069 }
5070
5071 /*
5072 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5073 *
5074 * This relies on ring/vector mapping already set up in the
5075 * vect_map[] structures
5076 */
5077 static void
5078 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5079 {
5080 struct ixgbe_hw *hw = &ixgbe->hw;
5081 ixgbe_intr_vector_t *vect; /* vector bitmap */
5082 int r_idx; /* ring index */
5083 int v_idx; /* vector index */
5084 uint32_t hw_index;
5085
5086 /*
5087 * Clear any previous entries
5088 */
5089 switch (hw->mac.type) {
5090 case ixgbe_mac_82598EB:
5091 for (v_idx = 0; v_idx < 25; v_idx++)
5092 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5093 break;
5094
5095 case ixgbe_mac_82599EB:
5096 case ixgbe_mac_X540:
5097 for (v_idx = 0; v_idx < 64; v_idx++)
5098 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5099 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5100 break;
5101
5102 default:
5103 break;
5104 }
5105
5106 /*
5107 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5108 * tx rings[0] will use RTxQ[1].
5109 */
5110 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5111 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5112 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5113 return;
5114 }
5115
5116 /*
5117 * For MSI-X interrupt, "Other" is always on vector[0].
5118 */
5119 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5120
5121 /*
5122 * For each interrupt vector, populate the IVAR table
5123 */
5124 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5125 vect = &ixgbe->vect_map[v_idx];
5126
5127 /*
5128 * For each rx ring bit set
5129 */
5130 r_idx = bt_getlowbit(vect->rx_map, 0,
5131 (ixgbe->num_rx_rings - 1));
5132
5133 while (r_idx >= 0) {
5134 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5135 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5136 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5137 (ixgbe->num_rx_rings - 1));
5138 }
5139
5140 /*
5141 * For each tx ring bit set
5142 */
5143 r_idx = bt_getlowbit(vect->tx_map, 0,
5144 (ixgbe->num_tx_rings - 1));
5145
5146 while (r_idx >= 0) {
5147 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5148 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5149 (ixgbe->num_tx_rings - 1));
5150 }
5151 }
5152 }
5153
5154 /*
5155 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5156 */
5157 static void
5158 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5159 {
5160 int i;
5161 int rc;
5162
5163 for (i = 0; i < ixgbe->intr_cnt; i++) {
5164 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5165 if (rc != DDI_SUCCESS) {
5166 IXGBE_DEBUGLOG_1(ixgbe,
5167 "Remove intr handler failed: %d", rc);
5168 }
5169 }
5170 }
5171
5172 /*
5173 * ixgbe_rem_intrs - Remove the allocated interrupts.
5174 */
5175 static void
5176 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5177 {
5178 int i;
5179 int rc;
5180
5181 for (i = 0; i < ixgbe->intr_cnt; i++) {
5182 rc = ddi_intr_free(ixgbe->htable[i]);
5183 if (rc != DDI_SUCCESS) {
5184 IXGBE_DEBUGLOG_1(ixgbe,
5185 "Free intr failed: %d", rc);
5186 }
5187 }
5188
5189 kmem_free(ixgbe->htable, ixgbe->intr_size);
5190 ixgbe->htable = NULL;
5191 }
5192
5193 /*
5194 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5195 */
5196 static int
5197 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5198 {
5199 int i;
5200 int rc;
5201
5202 /*
5203 * Enable interrupts
5204 */
5205 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5206 /*
5207 * Call ddi_intr_block_enable() for MSI
5208 */
5209 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5210 if (rc != DDI_SUCCESS) {
5211 ixgbe_log(ixgbe,
5212 "Enable block intr failed: %d", rc);
5213 return (IXGBE_FAILURE);
5214 }
5215 } else {
5216 /*
5217 * Call ddi_intr_enable() for Legacy/MSI non block enable
5218 */
5219 for (i = 0; i < ixgbe->intr_cnt; i++) {
5220 rc = ddi_intr_enable(ixgbe->htable[i]);
5221 if (rc != DDI_SUCCESS) {
5222 ixgbe_log(ixgbe,
5223 "Enable intr failed: %d", rc);
5224 return (IXGBE_FAILURE);
5225 }
5226 }
5227 }
5228
5229 return (IXGBE_SUCCESS);
5230 }
5231
5232 /*
5233 * ixgbe_disable_intrs - Disable all the interrupts.
5234 */
5235 static int
5236 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5237 {
5238 int i;
5239 int rc;
5240
5241 /*
5242 * Disable all interrupts
5243 */
5244 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5245 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5246 if (rc != DDI_SUCCESS) {
5247 ixgbe_log(ixgbe,
5248 "Disable block intr failed: %d", rc);
5249 return (IXGBE_FAILURE);
5250 }
5251 } else {
5252 for (i = 0; i < ixgbe->intr_cnt; i++) {
5253 rc = ddi_intr_disable(ixgbe->htable[i]);
5254 if (rc != DDI_SUCCESS) {
5255 ixgbe_log(ixgbe,
5256 "Disable intr failed: %d", rc);
5257 return (IXGBE_FAILURE);
5258 }
5259 }
5260 }
5261
5262 return (IXGBE_SUCCESS);
5263 }
5264
5265 /*
5266 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5267 */
5268 static void
5269 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5270 {
5271 struct ixgbe_hw *hw = &ixgbe->hw;
5272 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5273 boolean_t link_up = B_FALSE;
5274 uint32_t pcs1g_anlp = 0;
5275 uint32_t pcs1g_ana = 0;
5276 boolean_t autoneg = B_FALSE;
5277
5278 ASSERT(mutex_owned(&ixgbe->gen_lock));
5279 ixgbe->param_lp_1000fdx_cap = 0;
5280 ixgbe->param_lp_100fdx_cap = 0;
5281
5282 /* check for link, don't wait */
5283 (void) ixgbe_check_link(hw, &speed, &link_up, false);
5284 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5285
5286 if (link_up) {
5287 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5288
5289 ixgbe->param_lp_1000fdx_cap =
5290 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5291 ixgbe->param_lp_100fdx_cap =
5292 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5293 }
5294
5295 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5296
5297 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5298 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5299 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5300 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5301 }
5302
5303 /*
5304 * ixgbe_get_driver_control - Notify that driver is in control of device.
5305 */
5306 static void
5307 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5308 {
5309 uint32_t ctrl_ext;
5310
5311 /*
5312 * Notify firmware that driver is in control of device
5313 */
5314 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5315 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5316 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5317 }
5318
5319 /*
5320 * ixgbe_release_driver_control - Notify that driver is no longer in control
5321 * of device.
5322 */
5323 static void
5324 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5325 {
5326 uint32_t ctrl_ext;
5327
5328 /*
5329 * Notify firmware that driver is no longer in control of device
5330 */
5331 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5332 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5333 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5334 }
5335
5336 /*
5337 * ixgbe_atomic_reserve - Atomic decrease operation.
5338 */
5339 int
5340 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5341 {
5342 uint32_t oldval;
5343 uint32_t newval;
5344
5345 /*
5346 * ATOMICALLY
5347 */
5348 do {
5349 oldval = *count_p;
5350 if (oldval < n)
5351 return (-1);
5352 newval = oldval - n;
5353 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5354
5355 return (newval);
5356 }
5357
5358 /*
5359 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5360 */
5361 static uint8_t *
5362 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5363 {
5364 uint8_t *addr = *upd_ptr;
5365 uint8_t *new_ptr;
5366
5367 _NOTE(ARGUNUSED(hw));
5368 _NOTE(ARGUNUSED(vmdq));
5369
5370 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5371 *upd_ptr = new_ptr;
5372 return (addr);
5373 }
5374
5375 /*
5376 * FMA support
5377 */
5378 int
5379 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5380 {
5381 ddi_fm_error_t de;
5382
5383 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5384 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5385 return (de.fme_status);
5386 }
5387
5388 int
5389 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5390 {
5391 ddi_fm_error_t de;
5392
5393 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5394 return (de.fme_status);
5395 }
5396
5397 /*
5398 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5399 */
5400 static int
5401 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5402 {
5403 _NOTE(ARGUNUSED(impl_data));
5404 /*
5405 * as the driver can always deal with an error in any dma or
5406 * access handle, we can just return the fme_status value.
5407 */
5408 pci_ereport_post(dip, err, NULL);
5409 return (err->fme_status);
5410 }
5411
5412 static void
5413 ixgbe_fm_init(ixgbe_t *ixgbe)
5414 {
5415 ddi_iblock_cookie_t iblk;
5416 int fma_dma_flag;
5417
5418 /*
5419 * Only register with IO Fault Services if we have some capability
5420 */
5421 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5422 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5423 } else {
5424 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5425 }
5426
5427 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5428 fma_dma_flag = 1;
5429 } else {
5430 fma_dma_flag = 0;
5431 }
5432
5433 ixgbe_set_fma_flags(fma_dma_flag);
5434
5435 if (ixgbe->fm_capabilities) {
5436
5437 /*
5438 * Register capabilities with IO Fault Services
5439 */
5440 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5441
5442 /*
5443 * Initialize pci ereport capabilities if ereport capable
5444 */
5445 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5446 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5447 pci_ereport_setup(ixgbe->dip);
5448
5449 /*
5450 * Register error callback if error callback capable
5451 */
5452 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5453 ddi_fm_handler_register(ixgbe->dip,
5454 ixgbe_fm_error_cb, (void*) ixgbe);
5455 }
5456 }
5457
5458 static void
5459 ixgbe_fm_fini(ixgbe_t *ixgbe)
5460 {
5461 /*
5462 * Only unregister FMA capabilities if they are registered
5463 */
5464 if (ixgbe->fm_capabilities) {
5465
5466 /*
5467 * Release any resources allocated by pci_ereport_setup()
5468 */
5469 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5470 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5471 pci_ereport_teardown(ixgbe->dip);
5472
5473 /*
5474 * Un-register error callback if error callback capable
5475 */
5476 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5477 ddi_fm_handler_unregister(ixgbe->dip);
5478
5479 /*
5480 * Unregister from IO Fault Service
5481 */
5482 ddi_fm_fini(ixgbe->dip);
5483 }
5484 }
5485
5486 void
5487 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5488 {
5489 uint64_t ena;
5490 char buf[FM_MAX_CLASS];
5491
5492 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5493 ena = fm_ena_generate(0, FM_ENA_FMT1);
5494 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5495 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5496 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5497 }
5498 }
5499
5500 static int
5501 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5502 {
5503 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5504
5505 mutex_enter(&rx_ring->rx_lock);
5506 rx_ring->ring_gen_num = mr_gen_num;
5507 mutex_exit(&rx_ring->rx_lock);
5508 return (0);
5509 }
5510
5511 /*
5512 * Get the global ring index by a ring index within a group.
5513 */
5514 static int
5515 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5516 {
5517 ixgbe_rx_ring_t *rx_ring;
5518 int i;
5519
5520 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5521 rx_ring = &ixgbe->rx_rings[i];
5522 if (rx_ring->group_index == gindex)
5523 rindex--;
5524 if (rindex < 0)
5525 return (i);
5526 }
5527
5528 return (-1);
5529 }
5530
5531 /*
5532 * Callback funtion for MAC layer to register all rings.
5533 */
5534 /* ARGSUSED */
5535 void
5536 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5537 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5538 {
5539 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5540 mac_intr_t *mintr = &infop->mri_intr;
5541
5542 switch (rtype) {
5543 case MAC_RING_TYPE_RX: {
5544 /*
5545 * 'index' is the ring index within the group.
5546 * Need to get the global ring index by searching in groups.
5547 */
5548 int global_ring_index = ixgbe_get_rx_ring_index(
5549 ixgbe, group_index, ring_index);
5550
5551 ASSERT(global_ring_index >= 0);
5552
5553 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5554 rx_ring->ring_handle = rh;
5555
5556 infop->mri_driver = (mac_ring_driver_t)rx_ring;
5557 infop->mri_start = ixgbe_ring_start;
5558 infop->mri_stop = NULL;
5559 infop->mri_poll = ixgbe_ring_rx_poll;
5560 infop->mri_stat = ixgbe_rx_ring_stat;
5561
5562 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
5563 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
5564 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
5565 if (ixgbe->intr_type &
5566 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5567 mintr->mi_ddi_handle =
5568 ixgbe->htable[rx_ring->intr_vector];
5569 }
5570
5571 break;
5572 }
5573 case MAC_RING_TYPE_TX: {
5574 ASSERT(group_index == -1);
5575 ASSERT(ring_index < ixgbe->num_tx_rings);
5576
5577 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
5578 tx_ring->ring_handle = rh;
5579
5580 infop->mri_driver = (mac_ring_driver_t)tx_ring;
5581 infop->mri_start = NULL;
5582 infop->mri_stop = NULL;
5583 infop->mri_tx = ixgbe_ring_tx;
5584 infop->mri_stat = ixgbe_tx_ring_stat;
5585 if (ixgbe->intr_type &
5586 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5587 mintr->mi_ddi_handle =
5588 ixgbe->htable[tx_ring->intr_vector];
5589 }
5590 break;
5591 }
5592 default:
5593 break;
5594 }
5595 }
5596
5597 /*
5598 * Callback funtion for MAC layer to register all groups.
5599 */
5600 void
5601 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
5602 mac_group_info_t *infop, mac_group_handle_t gh)
5603 {
5604 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5605
5606 switch (rtype) {
5607 case MAC_RING_TYPE_RX: {
5608 ixgbe_rx_group_t *rx_group;
5609
5610 rx_group = &ixgbe->rx_groups[index];
5611 rx_group->group_handle = gh;
5612
5613 infop->mgi_driver = (mac_group_driver_t)rx_group;
5614 infop->mgi_start = NULL;
5615 infop->mgi_stop = NULL;
5616 infop->mgi_addmac = ixgbe_addmac;
5617 infop->mgi_remmac = ixgbe_remmac;
5618 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
5619
5620 break;
5621 }
5622 case MAC_RING_TYPE_TX:
5623 break;
5624 default:
5625 break;
5626 }
5627 }
5628
5629 /*
5630 * Enable interrupt on the specificed rx ring.
5631 */
5632 int
5633 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
5634 {
5635 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5636 ixgbe_t *ixgbe = rx_ring->ixgbe;
5637 int r_idx = rx_ring->index;
5638 int hw_r_idx = rx_ring->hw_index;
5639 int v_idx = rx_ring->intr_vector;
5640
5641 mutex_enter(&ixgbe->gen_lock);
5642 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5643 mutex_exit(&ixgbe->gen_lock);
5644 /*
5645 * Simply return 0.
5646 * Interrupts are being adjusted. ixgbe_intr_adjust()
5647 * will eventually re-enable the interrupt when it's
5648 * done with the adjustment.
5649 */
5650 return (0);
5651 }
5652
5653 /*
5654 * To enable interrupt by setting the VAL bit of given interrupt
5655 * vector allocation register (IVAR).
5656 */
5657 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
5658
5659 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5660
5661 /*
5662 * Trigger a Rx interrupt on this ring
5663 */
5664 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
5665 IXGBE_WRITE_FLUSH(&ixgbe->hw);
5666
5667 mutex_exit(&ixgbe->gen_lock);
5668
5669 return (0);
5670 }
5671
5672 /*
5673 * Disable interrupt on the specificed rx ring.
5674 */
5675 int
5676 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
5677 {
5678 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5679 ixgbe_t *ixgbe = rx_ring->ixgbe;
5680 int r_idx = rx_ring->index;
5681 int hw_r_idx = rx_ring->hw_index;
5682 int v_idx = rx_ring->intr_vector;
5683
5684 mutex_enter(&ixgbe->gen_lock);
5685 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5686 mutex_exit(&ixgbe->gen_lock);
5687 /*
5688 * Simply return 0.
5689 * In the rare case where an interrupt is being
5690 * disabled while interrupts are being adjusted,
5691 * we don't fail the operation. No interrupts will
5692 * be generated while they are adjusted, and
5693 * ixgbe_intr_adjust() will cause the interrupts
5694 * to be re-enabled once it completes. Note that
5695 * in this case, packets may be delivered to the
5696 * stack via interrupts before xgbe_rx_ring_intr_enable()
5697 * is called again. This is acceptable since interrupt
5698 * adjustment is infrequent, and the stack will be
5699 * able to handle these packets.
5700 */
5701 return (0);
5702 }
5703
5704 /*
5705 * To disable interrupt by clearing the VAL bit of given interrupt
5706 * vector allocation register (IVAR).
5707 */
5708 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
5709
5710 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
5711
5712 mutex_exit(&ixgbe->gen_lock);
5713
5714 return (0);
5715 }
5716
5717 /*
5718 * Add a mac address.
5719 */
5720 static int
5721 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
5722 {
5723 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5724 ixgbe_t *ixgbe = rx_group->ixgbe;
5725 struct ixgbe_hw *hw = &ixgbe->hw;
5726 int slot, i;
5727
5728 mutex_enter(&ixgbe->gen_lock);
5729
5730 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5731 mutex_exit(&ixgbe->gen_lock);
5732 return (ECANCELED);
5733 }
5734
5735 if (ixgbe->unicst_avail == 0) {
5736 /* no slots available */
5737 mutex_exit(&ixgbe->gen_lock);
5738 return (ENOSPC);
5739 }
5740
5741 /*
5742 * The first ixgbe->num_rx_groups slots are reserved for each respective
5743 * group. The rest slots are shared by all groups. While adding a
5744 * MAC address, reserved slots are firstly checked then the shared
5745 * slots are searched.
5746 */
5747 slot = -1;
5748 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
5749 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
5750 if (ixgbe->unicst_addr[i].mac.set == 0) {
5751 slot = i;
5752 break;
5753 }
5754 }
5755 } else {
5756 slot = rx_group->index;
5757 }
5758
5759 if (slot == -1) {
5760 /* no slots available */
5761 mutex_exit(&ixgbe->gen_lock);
5762 return (ENOSPC);
5763 }
5764
5765 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5766 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
5767 rx_group->index, IXGBE_RAH_AV);
5768 ixgbe->unicst_addr[slot].mac.set = 1;
5769 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
5770 ixgbe->unicst_avail--;
5771
5772 mutex_exit(&ixgbe->gen_lock);
5773
5774 return (0);
5775 }
5776
5777 /*
5778 * Remove a mac address.
5779 */
5780 static int
5781 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
5782 {
5783 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5784 ixgbe_t *ixgbe = rx_group->ixgbe;
5785 struct ixgbe_hw *hw = &ixgbe->hw;
5786 int slot;
5787
5788 mutex_enter(&ixgbe->gen_lock);
5789
5790 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5791 mutex_exit(&ixgbe->gen_lock);
5792 return (ECANCELED);
5793 }
5794
5795 slot = ixgbe_unicst_find(ixgbe, mac_addr);
5796 if (slot == -1) {
5797 mutex_exit(&ixgbe->gen_lock);
5798 return (EINVAL);
5799 }
5800
5801 if (ixgbe->unicst_addr[slot].mac.set == 0) {
5802 mutex_exit(&ixgbe->gen_lock);
5803 return (EINVAL);
5804 }
5805
5806 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5807 (void) ixgbe_clear_rar(hw, slot);
5808 ixgbe->unicst_addr[slot].mac.set = 0;
5809 ixgbe->unicst_avail++;
5810
5811 mutex_exit(&ixgbe->gen_lock);
5812
5813 return (0);
5814 }