1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 */
31
32 #include "ixgbe_sw.h"
33
34 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
35 static char ixgbe_version[] = "ixgbe 1.1.7";
36
37 /*
38 * Local function protoypes
39 */
40 static int ixgbe_register_mac(ixgbe_t *);
41 static int ixgbe_identify_hardware(ixgbe_t *);
42 static int ixgbe_regs_map(ixgbe_t *);
43 static void ixgbe_init_properties(ixgbe_t *);
44 static int ixgbe_init_driver_settings(ixgbe_t *);
45 static void ixgbe_init_locks(ixgbe_t *);
46 static void ixgbe_destroy_locks(ixgbe_t *);
47 static int ixgbe_init(ixgbe_t *);
48 static int ixgbe_chip_start(ixgbe_t *);
49 static void ixgbe_chip_stop(ixgbe_t *);
50 static int ixgbe_reset(ixgbe_t *);
51 static void ixgbe_tx_clean(ixgbe_t *);
52 static boolean_t ixgbe_tx_drain(ixgbe_t *);
53 static boolean_t ixgbe_rx_drain(ixgbe_t *);
54 static int ixgbe_alloc_rings(ixgbe_t *);
55 static void ixgbe_free_rings(ixgbe_t *);
56 static int ixgbe_alloc_rx_data(ixgbe_t *);
57 static void ixgbe_free_rx_data(ixgbe_t *);
58 static void ixgbe_setup_rings(ixgbe_t *);
59 static void ixgbe_setup_rx(ixgbe_t *);
60 static void ixgbe_setup_tx(ixgbe_t *);
61 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
62 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
63 static void ixgbe_setup_rss(ixgbe_t *);
64 static void ixgbe_setup_vmdq(ixgbe_t *);
65 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
66 static void ixgbe_init_unicst(ixgbe_t *);
67 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
68 static void ixgbe_setup_multicst(ixgbe_t *);
69 static void ixgbe_get_hw_state(ixgbe_t *);
70 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
71 static void ixgbe_get_conf(ixgbe_t *);
72 static void ixgbe_init_params(ixgbe_t *);
73 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
74 static void ixgbe_driver_link_check(ixgbe_t *);
75 static void ixgbe_sfp_check(void *);
76 static void ixgbe_overtemp_check(void *);
77 static void ixgbe_link_timer(void *);
78 static void ixgbe_local_timer(void *);
79 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
80 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
81 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
82 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
83 static boolean_t is_valid_mac_addr(uint8_t *);
84 static boolean_t ixgbe_stall_check(ixgbe_t *);
85 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
86 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
87 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
88 static int ixgbe_alloc_intrs(ixgbe_t *);
89 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
90 static int ixgbe_add_intr_handlers(ixgbe_t *);
91 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
92 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
93 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
94 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
95 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
96 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
97 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
98 static void ixgbe_setup_adapter_vector(ixgbe_t *);
99 static void ixgbe_rem_intr_handlers(ixgbe_t *);
100 static void ixgbe_rem_intrs(ixgbe_t *);
101 static int ixgbe_enable_intrs(ixgbe_t *);
102 static int ixgbe_disable_intrs(ixgbe_t *);
103 static uint_t ixgbe_intr_legacy(void *, void *);
104 static uint_t ixgbe_intr_msi(void *, void *);
105 static uint_t ixgbe_intr_msix(void *, void *);
106 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
107 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
108 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
109 static void ixgbe_get_driver_control(struct ixgbe_hw *);
110 static int ixgbe_addmac(void *, const uint8_t *);
111 static int ixgbe_remmac(void *, const uint8_t *);
112 static void ixgbe_release_driver_control(struct ixgbe_hw *);
113
114 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
115 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
116 static int ixgbe_resume(dev_info_t *);
117 static int ixgbe_suspend(dev_info_t *);
118 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
119 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
120 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
121 static int ixgbe_intr_cb_register(ixgbe_t *);
122 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
123
124 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
125 const void *impl_data);
126 static void ixgbe_fm_init(ixgbe_t *);
127 static void ixgbe_fm_fini(ixgbe_t *);
128
129 char *ixgbe_priv_props[] = {
130 "_tx_copy_thresh",
131 "_tx_recycle_thresh",
132 "_tx_overload_thresh",
133 "_tx_resched_thresh",
134 "_rx_copy_thresh",
135 "_rx_limit_per_intr",
136 "_intr_throttling",
137 "_adv_pause_cap",
138 "_adv_asym_pause_cap",
139 NULL
140 };
141
142 #define IXGBE_MAX_PRIV_PROPS \
143 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
144
145 static struct cb_ops ixgbe_cb_ops = {
146 nulldev, /* cb_open */
147 nulldev, /* cb_close */
148 nodev, /* cb_strategy */
149 nodev, /* cb_print */
150 nodev, /* cb_dump */
151 nodev, /* cb_read */
152 nodev, /* cb_write */
153 nodev, /* cb_ioctl */
154 nodev, /* cb_devmap */
155 nodev, /* cb_mmap */
156 nodev, /* cb_segmap */
157 nochpoll, /* cb_chpoll */
158 ddi_prop_op, /* cb_prop_op */
159 NULL, /* cb_stream */
160 D_MP | D_HOTPLUG, /* cb_flag */
161 CB_REV, /* cb_rev */
162 nodev, /* cb_aread */
163 nodev /* cb_awrite */
164 };
165
166 static struct dev_ops ixgbe_dev_ops = {
167 DEVO_REV, /* devo_rev */
168 0, /* devo_refcnt */
169 NULL, /* devo_getinfo */
170 nulldev, /* devo_identify */
171 nulldev, /* devo_probe */
172 ixgbe_attach, /* devo_attach */
173 ixgbe_detach, /* devo_detach */
174 nodev, /* devo_reset */
175 &ixgbe_cb_ops, /* devo_cb_ops */
176 NULL, /* devo_bus_ops */
177 ddi_power, /* devo_power */
178 ddi_quiesce_not_supported, /* devo_quiesce */
179 };
180
181 static struct modldrv ixgbe_modldrv = {
182 &mod_driverops, /* Type of module. This one is a driver */
183 ixgbe_ident, /* Discription string */
184 &ixgbe_dev_ops /* driver ops */
185 };
186
187 static struct modlinkage ixgbe_modlinkage = {
188 MODREV_1, &ixgbe_modldrv, NULL
189 };
190
191 /*
192 * Access attributes for register mapping
193 */
194 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
195 DDI_DEVICE_ATTR_V1,
196 DDI_STRUCTURE_LE_ACC,
197 DDI_STRICTORDER_ACC,
198 DDI_FLAGERR_ACC
199 };
200
201 /*
202 * Loopback property
203 */
204 static lb_property_t lb_normal = {
205 normal, "normal", IXGBE_LB_NONE
206 };
207
208 static lb_property_t lb_mac = {
209 internal, "MAC", IXGBE_LB_INTERNAL_MAC
210 };
211
212 static lb_property_t lb_external = {
213 external, "External", IXGBE_LB_EXTERNAL
214 };
215
216 #define IXGBE_M_CALLBACK_FLAGS \
217 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
218
219 static mac_callbacks_t ixgbe_m_callbacks = {
220 IXGBE_M_CALLBACK_FLAGS,
221 ixgbe_m_stat,
222 ixgbe_m_start,
223 ixgbe_m_stop,
224 ixgbe_m_promisc,
225 ixgbe_m_multicst,
226 NULL,
227 NULL,
228 NULL,
229 ixgbe_m_ioctl,
230 ixgbe_m_getcapab,
231 NULL,
232 NULL,
233 ixgbe_m_setprop,
234 ixgbe_m_getprop,
235 ixgbe_m_propinfo
236 };
237
238 /*
239 * Initialize capabilities of each supported adapter type
240 */
241 static adapter_info_t ixgbe_82598eb_cap = {
242 64, /* maximum number of rx queues */
243 1, /* minimum number of rx queues */
244 64, /* default number of rx queues */
245 16, /* maximum number of rx groups */
246 1, /* minimum number of rx groups */
247 1, /* default number of rx groups */
248 32, /* maximum number of tx queues */
249 1, /* minimum number of tx queues */
250 8, /* default number of tx queues */
251 16366, /* maximum MTU size */
252 0xFFFF, /* maximum interrupt throttle rate */
253 0, /* minimum interrupt throttle rate */
254 200, /* default interrupt throttle rate */
255 18, /* maximum total msix vectors */
256 16, /* maximum number of ring vectors */
257 2, /* maximum number of other vectors */
258 IXGBE_EICR_LSC, /* "other" interrupt types handled */
259 0, /* "other" interrupt types enable mask */
260 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
261 | IXGBE_FLAG_RSS_CAPABLE
262 | IXGBE_FLAG_VMDQ_CAPABLE)
263 };
264
265 static adapter_info_t ixgbe_82599eb_cap = {
266 128, /* maximum number of rx queues */
267 1, /* minimum number of rx queues */
268 128, /* default number of rx queues */
269 64, /* maximum number of rx groups */
270 1, /* minimum number of rx groups */
271 1, /* default number of rx groups */
272 128, /* maximum number of tx queues */
273 1, /* minimum number of tx queues */
274 8, /* default number of tx queues */
275 15500, /* maximum MTU size */
276 0xFF8, /* maximum interrupt throttle rate */
277 0, /* minimum interrupt throttle rate */
278 200, /* default interrupt throttle rate */
279 64, /* maximum total msix vectors */
280 16, /* maximum number of ring vectors */
281 2, /* maximum number of other vectors */
282 (IXGBE_EICR_LSC
283 | IXGBE_EICR_GPI_SDP1
284 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
285
286 (IXGBE_SDP1_GPIEN
287 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
288
289 (IXGBE_FLAG_DCA_CAPABLE
290 | IXGBE_FLAG_RSS_CAPABLE
291 | IXGBE_FLAG_VMDQ_CAPABLE
292 | IXGBE_FLAG_RSC_CAPABLE
293 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
294 };
295
296 static adapter_info_t ixgbe_X540_cap = {
297 128, /* maximum number of rx queues */
298 1, /* minimum number of rx queues */
299 128, /* default number of rx queues */
300 64, /* maximum number of rx groups */
301 1, /* minimum number of rx groups */
302 1, /* default number of rx groups */
303 128, /* maximum number of tx queues */
304 1, /* minimum number of tx queues */
305 8, /* default number of tx queues */
306 15500, /* maximum MTU size */
307 0xFF8, /* maximum interrupt throttle rate */
308 0, /* minimum interrupt throttle rate */
309 200, /* default interrupt throttle rate */
310 64, /* maximum total msix vectors */
311 16, /* maximum number of ring vectors */
312 2, /* maximum number of other vectors */
313 /* XXX KEBE ASKS, Do we care about X540's SDP3? */
314 (IXGBE_EICR_LSC
315 | IXGBE_EICR_GPI_SDP0
316 | IXGBE_EICR_GPI_SDP1
317 | IXGBE_EICR_GPI_SDP2
318 /* | IXGBE_EICR_GPI_SDP3 */), /* "other" interrupt types handled */
319
320 (IXGBE_SDP1_GPIEN
321 | IXGBE_SDP2_GPIEN
322 /* | IXGBE_SDP3_GPIEN */), /* "other" interrupt types enable mask */
323
324 /* XXX KEBE ASKS, SFP_PLUG capable?!? */
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE
329 /* | IXGBE_FLAG_SFP_PLUG_CAPABLE */) /* capability flags */
330 };
331
332 /*
333 * Module Initialization Functions.
334 */
335
336 int
337 _init(void)
338 {
339 int status;
340
341 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
342
343 status = mod_install(&ixgbe_modlinkage);
344
345 if (status != DDI_SUCCESS) {
346 mac_fini_ops(&ixgbe_dev_ops);
347 }
348
349 return (status);
350 }
351
352 int
353 _fini(void)
354 {
355 int status;
356
357 status = mod_remove(&ixgbe_modlinkage);
358
359 if (status == DDI_SUCCESS) {
360 mac_fini_ops(&ixgbe_dev_ops);
361 }
362
363 return (status);
364 }
365
366 int
367 _info(struct modinfo *modinfop)
368 {
369 int status;
370
371 status = mod_info(&ixgbe_modlinkage, modinfop);
372
373 return (status);
374 }
375
376 /*
377 * ixgbe_attach - Driver attach.
378 *
379 * This function is the device specific initialization entry
380 * point. This entry point is required and must be written.
381 * The DDI_ATTACH command must be provided in the attach entry
382 * point. When attach() is called with cmd set to DDI_ATTACH,
383 * all normal kernel services (such as kmem_alloc(9F)) are
384 * available for use by the driver.
385 *
386 * The attach() function will be called once for each instance
387 * of the device on the system with cmd set to DDI_ATTACH.
388 * Until attach() succeeds, the only driver entry points which
389 * may be called are open(9E) and getinfo(9E).
390 */
391 static int
392 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
393 {
394 ixgbe_t *ixgbe;
395 struct ixgbe_osdep *osdep;
396 struct ixgbe_hw *hw;
397 int instance;
398 char taskqname[32];
399
400 /*
401 * Check the command and perform corresponding operations
402 */
403 switch (cmd) {
404 default:
405 return (DDI_FAILURE);
406
407 case DDI_RESUME:
408 return (ixgbe_resume(devinfo));
409
410 case DDI_ATTACH:
411 break;
412 }
413
414 /* Get the device instance */
415 instance = ddi_get_instance(devinfo);
416
417 /* Allocate memory for the instance data structure */
418 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
419
420 ixgbe->dip = devinfo;
421 ixgbe->instance = instance;
422
423 hw = &ixgbe->hw;
424 osdep = &ixgbe->osdep;
425 hw->back = osdep;
426 osdep->ixgbe = ixgbe;
427
428 /* Attach the instance pointer to the dev_info data structure */
429 ddi_set_driver_private(devinfo, ixgbe);
430
431 /*
432 * Initialize for fma support
433 */
434 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
435 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
436 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
437 ixgbe_fm_init(ixgbe);
438 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
439
440 /*
441 * Map PCI config space registers
442 */
443 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
444 ixgbe_error(ixgbe, "Failed to map PCI configurations");
445 goto attach_fail;
446 }
447 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
448
449 /*
450 * Identify the chipset family
451 */
452 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
453 ixgbe_error(ixgbe, "Failed to identify hardware");
454 goto attach_fail;
455 }
456
457 /*
458 * Map device registers
459 */
460 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
461 ixgbe_error(ixgbe, "Failed to map device registers");
462 goto attach_fail;
463 }
464 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
465
466 /*
467 * Initialize driver parameters
468 */
469 ixgbe_init_properties(ixgbe);
470 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
471
472 /*
473 * Register interrupt callback
474 */
475 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
476 ixgbe_error(ixgbe, "Failed to register interrupt callback");
477 goto attach_fail;
478 }
479
480 /*
481 * Allocate interrupts
482 */
483 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
484 ixgbe_error(ixgbe, "Failed to allocate interrupts");
485 goto attach_fail;
486 }
487 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
488
489 /*
490 * Allocate rx/tx rings based on the ring numbers.
491 * The actual numbers of rx/tx rings are decided by the number of
492 * allocated interrupt vectors, so we should allocate the rings after
493 * interrupts are allocated.
494 */
495 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
496 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
497 goto attach_fail;
498 }
499 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
500
501 /*
502 * Map rings to interrupt vectors
503 */
504 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
505 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
506 goto attach_fail;
507 }
508
509 /*
510 * Add interrupt handlers
511 */
512 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
513 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
514 goto attach_fail;
515 }
516 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
517
518 /*
519 * Create a taskq for sfp-change
520 */
521 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
522 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
523 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
524 ixgbe_error(ixgbe, "sfp_taskq create failed");
525 goto attach_fail;
526 }
527 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
528
529 /*
530 * Create a taskq for over-temp
531 */
532 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
533 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
534 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
535 ixgbe_error(ixgbe, "overtemp_taskq create failed");
536 goto attach_fail;
537 }
538 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
539
540 /*
541 * Initialize driver parameters
542 */
543 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
544 ixgbe_error(ixgbe, "Failed to initialize driver settings");
545 goto attach_fail;
546 }
547
548 /*
549 * Initialize mutexes for this device.
550 * Do this before enabling the interrupt handler and
551 * register the softint to avoid the condition where
552 * interrupt handler can try using uninitialized mutex.
553 */
554 ixgbe_init_locks(ixgbe);
555 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
556
557 /*
558 * Initialize chipset hardware
559 */
560 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
561 ixgbe_error(ixgbe, "Failed to initialize adapter");
562 goto attach_fail;
563 }
564 ixgbe->link_check_complete = B_FALSE;
565 ixgbe->link_check_hrtime = gethrtime() +
566 (IXGBE_LINK_UP_TIME * 100000000ULL);
567 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
568
569 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
570 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
571 goto attach_fail;
572 }
573
574 /*
575 * Initialize statistics
576 */
577 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
578 ixgbe_error(ixgbe, "Failed to initialize statistics");
579 goto attach_fail;
580 }
581 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
582
583 /*
584 * Register the driver to the MAC
585 */
586 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
587 ixgbe_error(ixgbe, "Failed to register MAC");
588 goto attach_fail;
589 }
590 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
591 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
592
593 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
594 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
595 if (ixgbe->periodic_id == 0) {
596 ixgbe_error(ixgbe, "Failed to add the link check timer");
597 goto attach_fail;
598 }
599 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
600
601 /*
602 * Now that mutex locks are initialized, and the chip is also
603 * initialized, enable interrupts.
604 */
605 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
606 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
607 goto attach_fail;
608 }
609 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
610
611 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version);
612 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
613
614 return (DDI_SUCCESS);
615
616 attach_fail:
617 ixgbe_unconfigure(devinfo, ixgbe);
618 return (DDI_FAILURE);
619 }
620
621 /*
622 * ixgbe_detach - Driver detach.
623 *
624 * The detach() function is the complement of the attach routine.
625 * If cmd is set to DDI_DETACH, detach() is used to remove the
626 * state associated with a given instance of a device node
627 * prior to the removal of that instance from the system.
628 *
629 * The detach() function will be called once for each instance
630 * of the device for which there has been a successful attach()
631 * once there are no longer any opens on the device.
632 *
633 * Interrupts routine are disabled, All memory allocated by this
634 * driver are freed.
635 */
636 static int
637 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
638 {
639 ixgbe_t *ixgbe;
640
641 /*
642 * Check detach command
643 */
644 switch (cmd) {
645 default:
646 return (DDI_FAILURE);
647
648 case DDI_SUSPEND:
649 return (ixgbe_suspend(devinfo));
650
651 case DDI_DETACH:
652 break;
653 }
654
655 /*
656 * Get the pointer to the driver private data structure
657 */
658 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
659 if (ixgbe == NULL)
660 return (DDI_FAILURE);
661
662 /*
663 * If the device is still running, it needs to be stopped first.
664 * This check is necessary because under some specific circumstances,
665 * the detach routine can be called without stopping the interface
666 * first.
667 */
668 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
669 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
670 mutex_enter(&ixgbe->gen_lock);
671 ixgbe_stop(ixgbe, B_TRUE);
672 mutex_exit(&ixgbe->gen_lock);
673 /* Disable and stop the watchdog timer */
674 ixgbe_disable_watchdog_timer(ixgbe);
675 }
676
677 /*
678 * Check if there are still rx buffers held by the upper layer.
679 * If so, fail the detach.
680 */
681 if (!ixgbe_rx_drain(ixgbe))
682 return (DDI_FAILURE);
683
684 /*
685 * Do the remaining unconfigure routines
686 */
687 ixgbe_unconfigure(devinfo, ixgbe);
688
689 return (DDI_SUCCESS);
690 }
691
692 static void
693 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
694 {
695 /*
696 * Disable interrupt
697 */
698 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
699 (void) ixgbe_disable_intrs(ixgbe);
700 }
701
702 /*
703 * remove the link check timer
704 */
705 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
706 if (ixgbe->periodic_id != NULL) {
707 ddi_periodic_delete(ixgbe->periodic_id);
708 ixgbe->periodic_id = NULL;
709 }
710 }
711
712 /*
713 * Unregister MAC
714 */
715 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
716 (void) mac_unregister(ixgbe->mac_hdl);
717 }
718
719 /*
720 * Free statistics
721 */
722 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
723 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
724 }
725
726 /*
727 * Remove interrupt handlers
728 */
729 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
730 ixgbe_rem_intr_handlers(ixgbe);
731 }
732
733 /*
734 * Remove taskq for sfp-status-change
735 */
736 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
737 ddi_taskq_destroy(ixgbe->sfp_taskq);
738 }
739
740 /*
741 * Remove taskq for over-temp
742 */
743 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
744 ddi_taskq_destroy(ixgbe->overtemp_taskq);
745 }
746
747 /*
748 * Remove interrupts
749 */
750 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
751 ixgbe_rem_intrs(ixgbe);
752 }
753
754 /*
755 * Unregister interrupt callback handler
756 */
757 (void) ddi_cb_unregister(ixgbe->cb_hdl);
758
759 /*
760 * Remove driver properties
761 */
762 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
763 (void) ddi_prop_remove_all(devinfo);
764 }
765
766 /*
767 * Stop the chipset
768 */
769 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
770 mutex_enter(&ixgbe->gen_lock);
771 ixgbe_chip_stop(ixgbe);
772 mutex_exit(&ixgbe->gen_lock);
773 }
774
775 /*
776 * Free register handle
777 */
778 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
779 if (ixgbe->osdep.reg_handle != NULL)
780 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
781 }
782
783 /*
784 * Free PCI config handle
785 */
786 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
787 if (ixgbe->osdep.cfg_handle != NULL)
788 pci_config_teardown(&ixgbe->osdep.cfg_handle);
789 }
790
791 /*
792 * Free locks
793 */
794 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
795 ixgbe_destroy_locks(ixgbe);
796 }
797
798 /*
799 * Free the rx/tx rings
800 */
801 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
802 ixgbe_free_rings(ixgbe);
803 }
804
805 /*
806 * Unregister FMA capabilities
807 */
808 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
809 ixgbe_fm_fini(ixgbe);
810 }
811
812 /*
813 * Free the driver data structure
814 */
815 kmem_free(ixgbe, sizeof (ixgbe_t));
816
817 ddi_set_driver_private(devinfo, NULL);
818 }
819
820 /*
821 * ixgbe_register_mac - Register the driver and its function pointers with
822 * the GLD interface.
823 */
824 static int
825 ixgbe_register_mac(ixgbe_t *ixgbe)
826 {
827 struct ixgbe_hw *hw = &ixgbe->hw;
828 mac_register_t *mac;
829 int status;
830
831 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
832 return (IXGBE_FAILURE);
833
834 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
835 mac->m_driver = ixgbe;
836 mac->m_dip = ixgbe->dip;
837 mac->m_src_addr = hw->mac.addr;
838 mac->m_callbacks = &ixgbe_m_callbacks;
839 mac->m_min_sdu = 0;
840 mac->m_max_sdu = ixgbe->default_mtu;
841 mac->m_margin = VLAN_TAGSZ;
842 mac->m_priv_props = ixgbe_priv_props;
843 mac->m_v12n = MAC_VIRT_LEVEL1;
844
845 status = mac_register(mac, &ixgbe->mac_hdl);
846
847 mac_free(mac);
848
849 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
850 }
851
852 /*
853 * ixgbe_identify_hardware - Identify the type of the chipset.
854 */
855 static int
856 ixgbe_identify_hardware(ixgbe_t *ixgbe)
857 {
858 struct ixgbe_hw *hw = &ixgbe->hw;
859 struct ixgbe_osdep *osdep = &ixgbe->osdep;
860
861 /*
862 * Get the device id
863 */
864 hw->vendor_id =
865 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
866 hw->device_id =
867 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
868 hw->revision_id =
869 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
870 hw->subsystem_device_id =
871 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
872 hw->subsystem_vendor_id =
873 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
874
875 /*
876 * Set the mac type of the adapter based on the device id
877 */
878 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
879 return (IXGBE_FAILURE);
880 }
881
882 /*
883 * Install adapter capabilities
884 */
885 switch (hw->mac.type) {
886 case ixgbe_mac_82598EB:
887 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
888 ixgbe->capab = &ixgbe_82598eb_cap;
889
890 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
891 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
892 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
893 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
894 }
895 break;
896
897 case ixgbe_mac_82599EB:
898 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
899 ixgbe->capab = &ixgbe_82599eb_cap;
900
901 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
902 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
903 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
904 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
905 }
906 break;
907
908 case ixgbe_mac_X540:
909 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
910 ixgbe->capab = &ixgbe_X540_cap;
911 /*
912 * For now, X540 is all set in its capab structure.
913 * As other X540 variants show up, things can change here.
914 */
915 break;
916
917 default:
918 IXGBE_DEBUGLOG_1(ixgbe,
919 "adapter not supported in ixgbe_identify_hardware(): %d\n",
920 hw->mac.type);
921 return (IXGBE_FAILURE);
922 }
923
924 return (IXGBE_SUCCESS);
925 }
926
927 /*
928 * ixgbe_regs_map - Map the device registers.
929 *
930 */
931 static int
932 ixgbe_regs_map(ixgbe_t *ixgbe)
933 {
934 dev_info_t *devinfo = ixgbe->dip;
935 struct ixgbe_hw *hw = &ixgbe->hw;
936 struct ixgbe_osdep *osdep = &ixgbe->osdep;
937 off_t mem_size;
938
939 /*
940 * First get the size of device registers to be mapped.
941 */
942 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
943 != DDI_SUCCESS) {
944 return (IXGBE_FAILURE);
945 }
946
947 /*
948 * Call ddi_regs_map_setup() to map registers
949 */
950 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
951 (caddr_t *)&hw->hw_addr, 0,
952 mem_size, &ixgbe_regs_acc_attr,
953 &osdep->reg_handle)) != DDI_SUCCESS) {
954 return (IXGBE_FAILURE);
955 }
956
957 return (IXGBE_SUCCESS);
958 }
959
960 /*
961 * ixgbe_init_properties - Initialize driver properties.
962 */
963 static void
964 ixgbe_init_properties(ixgbe_t *ixgbe)
965 {
966 /*
967 * Get conf file properties, including link settings
968 * jumbo frames, ring number, descriptor number, etc.
969 */
970 ixgbe_get_conf(ixgbe);
971
972 ixgbe_init_params(ixgbe);
973 }
974
975 /*
976 * ixgbe_init_driver_settings - Initialize driver settings.
977 *
978 * The settings include hardware function pointers, bus information,
979 * rx/tx rings settings, link state, and any other parameters that
980 * need to be setup during driver initialization.
981 */
982 static int
983 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
984 {
985 struct ixgbe_hw *hw = &ixgbe->hw;
986 dev_info_t *devinfo = ixgbe->dip;
987 ixgbe_rx_ring_t *rx_ring;
988 ixgbe_rx_group_t *rx_group;
989 ixgbe_tx_ring_t *tx_ring;
990 uint32_t rx_size;
991 uint32_t tx_size;
992 uint32_t ring_per_group;
993 int i;
994
995 /*
996 * Initialize chipset specific hardware function pointers
997 */
998 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
999 return (IXGBE_FAILURE);
1000 }
1001
1002 /*
1003 * Get the system page size
1004 */
1005 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1006
1007 /*
1008 * Set rx buffer size
1009 *
1010 * The IP header alignment room is counted in the calculation.
1011 * The rx buffer size is in unit of 1K that is required by the
1012 * chipset hardware.
1013 */
1014 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1015 ixgbe->rx_buf_size = ((rx_size >> 10) +
1016 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1017
1018 /*
1019 * Set tx buffer size
1020 */
1021 tx_size = ixgbe->max_frame_size;
1022 ixgbe->tx_buf_size = ((tx_size >> 10) +
1023 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1024
1025 /*
1026 * Initialize rx/tx rings/groups parameters
1027 */
1028 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1029 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1030 rx_ring = &ixgbe->rx_rings[i];
1031 rx_ring->index = i;
1032 rx_ring->ixgbe = ixgbe;
1033 rx_ring->group_index = i / ring_per_group;
1034 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1035 }
1036
1037 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1038 rx_group = &ixgbe->rx_groups[i];
1039 rx_group->index = i;
1040 rx_group->ixgbe = ixgbe;
1041 }
1042
1043 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1044 tx_ring = &ixgbe->tx_rings[i];
1045 tx_ring->index = i;
1046 tx_ring->ixgbe = ixgbe;
1047 if (ixgbe->tx_head_wb_enable)
1048 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1049 else
1050 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1051
1052 tx_ring->ring_size = ixgbe->tx_ring_size;
1053 tx_ring->free_list_size = ixgbe->tx_ring_size +
1054 (ixgbe->tx_ring_size >> 1);
1055 }
1056
1057 /*
1058 * Initialize values of interrupt throttling rate
1059 */
1060 for (i = 1; i < MAX_INTR_VECTOR; i++)
1061 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1062
1063 /*
1064 * The initial link state should be "unknown"
1065 */
1066 ixgbe->link_state = LINK_STATE_UNKNOWN;
1067
1068 return (IXGBE_SUCCESS);
1069 }
1070
1071 /*
1072 * ixgbe_init_locks - Initialize locks.
1073 */
1074 static void
1075 ixgbe_init_locks(ixgbe_t *ixgbe)
1076 {
1077 ixgbe_rx_ring_t *rx_ring;
1078 ixgbe_tx_ring_t *tx_ring;
1079 int i;
1080
1081 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1082 rx_ring = &ixgbe->rx_rings[i];
1083 mutex_init(&rx_ring->rx_lock, NULL,
1084 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1085 }
1086
1087 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1088 tx_ring = &ixgbe->tx_rings[i];
1089 mutex_init(&tx_ring->tx_lock, NULL,
1090 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1091 mutex_init(&tx_ring->recycle_lock, NULL,
1092 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1093 mutex_init(&tx_ring->tcb_head_lock, NULL,
1094 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1095 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1096 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1097 }
1098
1099 mutex_init(&ixgbe->gen_lock, NULL,
1100 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1101
1102 mutex_init(&ixgbe->watchdog_lock, NULL,
1103 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1104 }
1105
1106 /*
1107 * ixgbe_destroy_locks - Destroy locks.
1108 */
1109 static void
1110 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1111 {
1112 ixgbe_rx_ring_t *rx_ring;
1113 ixgbe_tx_ring_t *tx_ring;
1114 int i;
1115
1116 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1117 rx_ring = &ixgbe->rx_rings[i];
1118 mutex_destroy(&rx_ring->rx_lock);
1119 }
1120
1121 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1122 tx_ring = &ixgbe->tx_rings[i];
1123 mutex_destroy(&tx_ring->tx_lock);
1124 mutex_destroy(&tx_ring->recycle_lock);
1125 mutex_destroy(&tx_ring->tcb_head_lock);
1126 mutex_destroy(&tx_ring->tcb_tail_lock);
1127 }
1128
1129 mutex_destroy(&ixgbe->gen_lock);
1130 mutex_destroy(&ixgbe->watchdog_lock);
1131 }
1132
1133 static int
1134 ixgbe_resume(dev_info_t *devinfo)
1135 {
1136 ixgbe_t *ixgbe;
1137 int i;
1138
1139 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1140 if (ixgbe == NULL)
1141 return (DDI_FAILURE);
1142
1143 mutex_enter(&ixgbe->gen_lock);
1144
1145 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1146 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1147 mutex_exit(&ixgbe->gen_lock);
1148 return (DDI_FAILURE);
1149 }
1150
1151 /*
1152 * Enable and start the watchdog timer
1153 */
1154 ixgbe_enable_watchdog_timer(ixgbe);
1155 }
1156
1157 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1158
1159 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1160 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1161 mac_tx_ring_update(ixgbe->mac_hdl,
1162 ixgbe->tx_rings[i].ring_handle);
1163 }
1164 }
1165
1166 mutex_exit(&ixgbe->gen_lock);
1167
1168 return (DDI_SUCCESS);
1169 }
1170
1171 static int
1172 ixgbe_suspend(dev_info_t *devinfo)
1173 {
1174 ixgbe_t *ixgbe;
1175
1176 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1177 if (ixgbe == NULL)
1178 return (DDI_FAILURE);
1179
1180 mutex_enter(&ixgbe->gen_lock);
1181
1182 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1183 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1184 mutex_exit(&ixgbe->gen_lock);
1185 return (DDI_SUCCESS);
1186 }
1187 ixgbe_stop(ixgbe, B_FALSE);
1188
1189 mutex_exit(&ixgbe->gen_lock);
1190
1191 /*
1192 * Disable and stop the watchdog timer
1193 */
1194 ixgbe_disable_watchdog_timer(ixgbe);
1195
1196 return (DDI_SUCCESS);
1197 }
1198
1199 /*
1200 * ixgbe_init - Initialize the device.
1201 */
1202 static int
1203 ixgbe_init(ixgbe_t *ixgbe)
1204 {
1205 struct ixgbe_hw *hw = &ixgbe->hw;
1206
1207 mutex_enter(&ixgbe->gen_lock);
1208
1209 /*
1210 * Reset chipset to put the hardware in a known state
1211 * before we try to do anything with the eeprom.
1212 */
1213 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1214 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1215 goto init_fail;
1216 }
1217
1218 /*
1219 * Need to init eeprom before validating the checksum.
1220 */
1221 if (ixgbe_init_eeprom_params(hw) < 0) {
1222 ixgbe_error(ixgbe,
1223 "Unable to intitialize the eeprom interface.");
1224 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1225 goto init_fail;
1226 }
1227
1228 /*
1229 * NVM validation
1230 */
1231 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1232 /*
1233 * Some PCI-E parts fail the first check due to
1234 * the link being in sleep state. Call it again,
1235 * if it fails a second time it's a real issue.
1236 */
1237 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1238 ixgbe_error(ixgbe,
1239 "Invalid NVM checksum. Please contact "
1240 "the vendor to update the NVM.");
1241 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1242 goto init_fail;
1243 }
1244 }
1245
1246 /*
1247 * Setup default flow control thresholds - enable/disable
1248 * & flow control type is controlled by ixgbe.conf
1249 */
1250 hw->fc.high_water[0] = DEFAULT_FCRTH;
1251 hw->fc.low_water[0] = DEFAULT_FCRTL;
1252 hw->fc.pause_time = DEFAULT_FCPAUSE;
1253 hw->fc.send_xon = B_TRUE;
1254
1255 /*
1256 * Initialize link settings
1257 */
1258 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1259
1260 /*
1261 * Initialize the chipset hardware
1262 */
1263 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1264 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1265 goto init_fail;
1266 }
1267
1268 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1269 goto init_fail;
1270 }
1271
1272 mutex_exit(&ixgbe->gen_lock);
1273 return (IXGBE_SUCCESS);
1274
1275 init_fail:
1276 /*
1277 * Reset PHY
1278 */
1279 (void) ixgbe_reset_phy(hw);
1280
1281 mutex_exit(&ixgbe->gen_lock);
1282 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1283 return (IXGBE_FAILURE);
1284 }
1285
1286 /*
1287 * ixgbe_chip_start - Initialize and start the chipset hardware.
1288 */
1289 static int
1290 ixgbe_chip_start(ixgbe_t *ixgbe)
1291 {
1292 struct ixgbe_hw *hw = &ixgbe->hw;
1293 int ret_val, i;
1294
1295 ASSERT(mutex_owned(&ixgbe->gen_lock));
1296
1297 /*
1298 * Get the mac address
1299 * This function should handle SPARC case correctly.
1300 */
1301 if (!ixgbe_find_mac_address(ixgbe)) {
1302 ixgbe_error(ixgbe, "Failed to get the mac address");
1303 return (IXGBE_FAILURE);
1304 }
1305
1306 /*
1307 * Validate the mac address
1308 */
1309 (void) ixgbe_init_rx_addrs(hw);
1310 if (!is_valid_mac_addr(hw->mac.addr)) {
1311 ixgbe_error(ixgbe, "Invalid mac address");
1312 return (IXGBE_FAILURE);
1313 }
1314
1315 /*
1316 * Configure/Initialize hardware
1317 */
1318 ret_val = ixgbe_init_hw(hw);
1319 if (ret_val != IXGBE_SUCCESS) {
1320 if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1321 ixgbe_error(ixgbe,
1322 "This 82599 device is pre-release and contains"
1323 " outdated firmware, please contact your hardware"
1324 " vendor for a replacement.");
1325 } else {
1326 ixgbe_error(ixgbe, "Failed to initialize hardware");
1327 return (IXGBE_FAILURE);
1328 }
1329 }
1330
1331 /*
1332 * Re-enable relaxed ordering for performance. It is disabled
1333 * by default in the hardware init.
1334 */
1335 if (ixgbe->relax_order_enable == B_TRUE)
1336 ixgbe_enable_relaxed_ordering(hw);
1337
1338 /*
1339 * Setup adapter interrupt vectors
1340 */
1341 ixgbe_setup_adapter_vector(ixgbe);
1342
1343 /*
1344 * Initialize unicast addresses.
1345 */
1346 ixgbe_init_unicst(ixgbe);
1347
1348 /*
1349 * Setup and initialize the mctable structures.
1350 */
1351 ixgbe_setup_multicst(ixgbe);
1352
1353 /*
1354 * Set interrupt throttling rate
1355 */
1356 for (i = 0; i < ixgbe->intr_cnt; i++) {
1357 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1358 }
1359
1360 /*
1361 * Save the state of the phy
1362 */
1363 ixgbe_get_hw_state(ixgbe);
1364
1365 /*
1366 * Make sure driver has control
1367 */
1368 ixgbe_get_driver_control(hw);
1369
1370 return (IXGBE_SUCCESS);
1371 }
1372
1373 /*
1374 * ixgbe_chip_stop - Stop the chipset hardware
1375 */
1376 static void
1377 ixgbe_chip_stop(ixgbe_t *ixgbe)
1378 {
1379 struct ixgbe_hw *hw = &ixgbe->hw;
1380
1381 ASSERT(mutex_owned(&ixgbe->gen_lock));
1382
1383 /*
1384 * Tell firmware driver is no longer in control
1385 */
1386 ixgbe_release_driver_control(hw);
1387
1388 /*
1389 * Reset the chipset
1390 */
1391 (void) ixgbe_reset_hw(hw);
1392
1393 /*
1394 * Reset PHY
1395 */
1396 (void) ixgbe_reset_phy(hw);
1397 }
1398
1399 /*
1400 * ixgbe_reset - Reset the chipset and re-start the driver.
1401 *
1402 * It involves stopping and re-starting the chipset,
1403 * and re-configuring the rx/tx rings.
1404 */
1405 static int
1406 ixgbe_reset(ixgbe_t *ixgbe)
1407 {
1408 int i;
1409
1410 /*
1411 * Disable and stop the watchdog timer
1412 */
1413 ixgbe_disable_watchdog_timer(ixgbe);
1414
1415 mutex_enter(&ixgbe->gen_lock);
1416
1417 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1418 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1419
1420 ixgbe_stop(ixgbe, B_FALSE);
1421
1422 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1423 mutex_exit(&ixgbe->gen_lock);
1424 return (IXGBE_FAILURE);
1425 }
1426
1427 /*
1428 * After resetting, need to recheck the link status.
1429 */
1430 ixgbe->link_check_complete = B_FALSE;
1431 ixgbe->link_check_hrtime = gethrtime() +
1432 (IXGBE_LINK_UP_TIME * 100000000ULL);
1433
1434 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1435
1436 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1437 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1438 mac_tx_ring_update(ixgbe->mac_hdl,
1439 ixgbe->tx_rings[i].ring_handle);
1440 }
1441 }
1442
1443 mutex_exit(&ixgbe->gen_lock);
1444
1445 /*
1446 * Enable and start the watchdog timer
1447 */
1448 ixgbe_enable_watchdog_timer(ixgbe);
1449
1450 return (IXGBE_SUCCESS);
1451 }
1452
1453 /*
1454 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1455 */
1456 static void
1457 ixgbe_tx_clean(ixgbe_t *ixgbe)
1458 {
1459 ixgbe_tx_ring_t *tx_ring;
1460 tx_control_block_t *tcb;
1461 link_list_t pending_list;
1462 uint32_t desc_num;
1463 int i, j;
1464
1465 LINK_LIST_INIT(&pending_list);
1466
1467 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1468 tx_ring = &ixgbe->tx_rings[i];
1469
1470 mutex_enter(&tx_ring->recycle_lock);
1471
1472 /*
1473 * Clean the pending tx data - the pending packets in the
1474 * work_list that have no chances to be transmitted again.
1475 *
1476 * We must ensure the chipset is stopped or the link is down
1477 * before cleaning the transmit packets.
1478 */
1479 desc_num = 0;
1480 for (j = 0; j < tx_ring->ring_size; j++) {
1481 tcb = tx_ring->work_list[j];
1482 if (tcb != NULL) {
1483 desc_num += tcb->desc_num;
1484
1485 tx_ring->work_list[j] = NULL;
1486
1487 ixgbe_free_tcb(tcb);
1488
1489 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1490 }
1491 }
1492
1493 if (desc_num > 0) {
1494 atomic_add_32(&tx_ring->tbd_free, desc_num);
1495 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1496
1497 /*
1498 * Reset the head and tail pointers of the tbd ring;
1499 * Reset the writeback head if it's enable.
1500 */
1501 tx_ring->tbd_head = 0;
1502 tx_ring->tbd_tail = 0;
1503 if (ixgbe->tx_head_wb_enable)
1504 *tx_ring->tbd_head_wb = 0;
1505
1506 IXGBE_WRITE_REG(&ixgbe->hw,
1507 IXGBE_TDH(tx_ring->index), 0);
1508 IXGBE_WRITE_REG(&ixgbe->hw,
1509 IXGBE_TDT(tx_ring->index), 0);
1510 }
1511
1512 mutex_exit(&tx_ring->recycle_lock);
1513
1514 /*
1515 * Add the tx control blocks in the pending list to
1516 * the free list.
1517 */
1518 ixgbe_put_free_list(tx_ring, &pending_list);
1519 }
1520 }
1521
1522 /*
1523 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1524 * transmitted.
1525 */
1526 static boolean_t
1527 ixgbe_tx_drain(ixgbe_t *ixgbe)
1528 {
1529 ixgbe_tx_ring_t *tx_ring;
1530 boolean_t done;
1531 int i, j;
1532
1533 /*
1534 * Wait for a specific time to allow pending tx packets
1535 * to be transmitted.
1536 *
1537 * Check the counter tbd_free to see if transmission is done.
1538 * No lock protection is needed here.
1539 *
1540 * Return B_TRUE if all pending packets have been transmitted;
1541 * Otherwise return B_FALSE;
1542 */
1543 for (i = 0; i < TX_DRAIN_TIME; i++) {
1544
1545 done = B_TRUE;
1546 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1547 tx_ring = &ixgbe->tx_rings[j];
1548 done = done &&
1549 (tx_ring->tbd_free == tx_ring->ring_size);
1550 }
1551
1552 if (done)
1553 break;
1554
1555 msec_delay(1);
1556 }
1557
1558 return (done);
1559 }
1560
1561 /*
1562 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1563 */
1564 static boolean_t
1565 ixgbe_rx_drain(ixgbe_t *ixgbe)
1566 {
1567 boolean_t done = B_TRUE;
1568 int i;
1569
1570 /*
1571 * Polling the rx free list to check if those rx buffers held by
1572 * the upper layer are released.
1573 *
1574 * Check the counter rcb_free to see if all pending buffers are
1575 * released. No lock protection is needed here.
1576 *
1577 * Return B_TRUE if all pending buffers have been released;
1578 * Otherwise return B_FALSE;
1579 */
1580 for (i = 0; i < RX_DRAIN_TIME; i++) {
1581 done = (ixgbe->rcb_pending == 0);
1582
1583 if (done)
1584 break;
1585
1586 msec_delay(1);
1587 }
1588
1589 return (done);
1590 }
1591
1592 /*
1593 * ixgbe_start - Start the driver/chipset.
1594 */
1595 int
1596 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1597 {
1598 int i;
1599
1600 ASSERT(mutex_owned(&ixgbe->gen_lock));
1601
1602 if (alloc_buffer) {
1603 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1604 ixgbe_error(ixgbe,
1605 "Failed to allocate software receive rings");
1606 return (IXGBE_FAILURE);
1607 }
1608
1609 /* Allocate buffers for all the rx/tx rings */
1610 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1611 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1612 return (IXGBE_FAILURE);
1613 }
1614
1615 ixgbe->tx_ring_init = B_TRUE;
1616 } else {
1617 ixgbe->tx_ring_init = B_FALSE;
1618 }
1619
1620 for (i = 0; i < ixgbe->num_rx_rings; i++)
1621 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1622 for (i = 0; i < ixgbe->num_tx_rings; i++)
1623 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1624
1625 /*
1626 * Start the chipset hardware
1627 */
1628 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1629 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1630 goto start_failure;
1631 }
1632
1633 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1634 goto start_failure;
1635 }
1636
1637 /*
1638 * Setup the rx/tx rings
1639 */
1640 ixgbe_setup_rings(ixgbe);
1641
1642 /*
1643 * ixgbe_start() will be called when resetting, however if reset
1644 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1645 * before enabling the interrupts.
1646 */
1647 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1648 | IXGBE_STALL| IXGBE_OVERTEMP));
1649
1650 /*
1651 * Enable adapter interrupts
1652 * The interrupts must be enabled after the driver state is START
1653 */
1654 ixgbe_enable_adapter_interrupts(ixgbe);
1655
1656 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1657 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1658 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1659 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1660
1661 return (IXGBE_SUCCESS);
1662
1663 start_failure:
1664 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1665 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1666 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1667 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1668
1669 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1670
1671 return (IXGBE_FAILURE);
1672 }
1673
1674 /*
1675 * ixgbe_stop - Stop the driver/chipset.
1676 */
1677 void
1678 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1679 {
1680 int i;
1681
1682 ASSERT(mutex_owned(&ixgbe->gen_lock));
1683
1684 /*
1685 * Disable the adapter interrupts
1686 */
1687 ixgbe_disable_adapter_interrupts(ixgbe);
1688
1689 /*
1690 * Drain the pending tx packets
1691 */
1692 (void) ixgbe_tx_drain(ixgbe);
1693
1694 for (i = 0; i < ixgbe->num_rx_rings; i++)
1695 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1696 for (i = 0; i < ixgbe->num_tx_rings; i++)
1697 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1698
1699 /*
1700 * Stop the chipset hardware
1701 */
1702 ixgbe_chip_stop(ixgbe);
1703
1704 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1705 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1706 }
1707
1708 /*
1709 * Clean the pending tx data/resources
1710 */
1711 ixgbe_tx_clean(ixgbe);
1712
1713 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1714 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1715 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1716 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1717
1718 if (ixgbe->link_state == LINK_STATE_UP) {
1719 ixgbe->link_state = LINK_STATE_UNKNOWN;
1720 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1721 }
1722
1723 if (free_buffer) {
1724 /*
1725 * Release the DMA/memory resources of rx/tx rings
1726 */
1727 ixgbe_free_dma(ixgbe);
1728 ixgbe_free_rx_data(ixgbe);
1729 }
1730 }
1731
1732 /*
1733 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1734 */
1735 /* ARGSUSED */
1736 static int
1737 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1738 void *arg1, void *arg2)
1739 {
1740 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1741
1742 switch (cbaction) {
1743 /* IRM callback */
1744 int count;
1745 case DDI_CB_INTR_ADD:
1746 case DDI_CB_INTR_REMOVE:
1747 count = (int)(uintptr_t)cbarg;
1748 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1749 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1750 int, ixgbe->intr_cnt);
1751 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1752 DDI_SUCCESS) {
1753 ixgbe_error(ixgbe,
1754 "IRM CB: Failed to adjust interrupts");
1755 goto cb_fail;
1756 }
1757 break;
1758 default:
1759 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1760 cbaction);
1761 return (DDI_ENOTSUP);
1762 }
1763 return (DDI_SUCCESS);
1764 cb_fail:
1765 return (DDI_FAILURE);
1766 }
1767
1768 /*
1769 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1770 */
1771 static int
1772 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1773 {
1774 int i, rc, actual;
1775
1776 if (count == 0)
1777 return (DDI_SUCCESS);
1778
1779 if ((cbaction == DDI_CB_INTR_ADD &&
1780 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1781 (cbaction == DDI_CB_INTR_REMOVE &&
1782 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
1783 return (DDI_FAILURE);
1784
1785 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1786 return (DDI_FAILURE);
1787 }
1788
1789 for (i = 0; i < ixgbe->num_rx_rings; i++)
1790 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
1791 for (i = 0; i < ixgbe->num_tx_rings; i++)
1792 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
1793
1794 mutex_enter(&ixgbe->gen_lock);
1795 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1796 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
1797 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
1798 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
1799
1800 ixgbe_stop(ixgbe, B_FALSE);
1801 /*
1802 * Disable interrupts
1803 */
1804 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1805 rc = ixgbe_disable_intrs(ixgbe);
1806 ASSERT(rc == IXGBE_SUCCESS);
1807 }
1808 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
1809
1810 /*
1811 * Remove interrupt handlers
1812 */
1813 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
1814 ixgbe_rem_intr_handlers(ixgbe);
1815 }
1816 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
1817
1818 /*
1819 * Clear vect_map
1820 */
1821 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
1822 switch (cbaction) {
1823 case DDI_CB_INTR_ADD:
1824 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
1825 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
1826 DDI_INTR_ALLOC_NORMAL);
1827 if (rc != DDI_SUCCESS || actual != count) {
1828 ixgbe_log(ixgbe, "Adjust interrupts failed."
1829 "return: %d, irm cb size: %d, actual: %d",
1830 rc, count, actual);
1831 goto intr_adjust_fail;
1832 }
1833 ixgbe->intr_cnt += count;
1834 break;
1835
1836 case DDI_CB_INTR_REMOVE:
1837 for (i = ixgbe->intr_cnt - count;
1838 i < ixgbe->intr_cnt; i ++) {
1839 rc = ddi_intr_free(ixgbe->htable[i]);
1840 ixgbe->htable[i] = NULL;
1841 if (rc != DDI_SUCCESS) {
1842 ixgbe_log(ixgbe, "Adjust interrupts failed."
1843 "return: %d, irm cb size: %d, actual: %d",
1844 rc, count, actual);
1845 goto intr_adjust_fail;
1846 }
1847 }
1848 ixgbe->intr_cnt -= count;
1849 break;
1850 }
1851
1852 /*
1853 * Get priority for first vector, assume remaining are all the same
1854 */
1855 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
1856 if (rc != DDI_SUCCESS) {
1857 ixgbe_log(ixgbe,
1858 "Get interrupt priority failed: %d", rc);
1859 goto intr_adjust_fail;
1860 }
1861 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
1862 if (rc != DDI_SUCCESS) {
1863 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
1864 goto intr_adjust_fail;
1865 }
1866 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
1867
1868 /*
1869 * Map rings to interrupt vectors
1870 */
1871 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
1872 ixgbe_error(ixgbe,
1873 "IRM CB: Failed to map interrupts to vectors");
1874 goto intr_adjust_fail;
1875 }
1876
1877 /*
1878 * Add interrupt handlers
1879 */
1880 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
1881 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
1882 goto intr_adjust_fail;
1883 }
1884 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
1885
1886 /*
1887 * Now that mutex locks are initialized, and the chip is also
1888 * initialized, enable interrupts.
1889 */
1890 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
1891 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
1892 goto intr_adjust_fail;
1893 }
1894 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
1895 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1896 ixgbe_error(ixgbe, "IRM CB: Failed to start");
1897 goto intr_adjust_fail;
1898 }
1899 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
1900 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
1901 ixgbe->ixgbe_state |= IXGBE_STARTED;
1902 mutex_exit(&ixgbe->gen_lock);
1903
1904 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1905 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
1906 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
1907 }
1908 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1909 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
1910 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
1911 }
1912
1913 /* Wakeup all Tx rings */
1914 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1915 mac_tx_ring_update(ixgbe->mac_hdl,
1916 ixgbe->tx_rings[i].ring_handle);
1917 }
1918
1919 IXGBE_DEBUGLOG_3(ixgbe,
1920 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
1921 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
1922 return (DDI_SUCCESS);
1923
1924 intr_adjust_fail:
1925 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1926 mutex_exit(&ixgbe->gen_lock);
1927 return (DDI_FAILURE);
1928 }
1929
1930 /*
1931 * ixgbe_intr_cb_register - Register interrupt callback function.
1932 */
1933 static int
1934 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
1935 {
1936 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
1937 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
1938 return (IXGBE_FAILURE);
1939 }
1940 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
1941 return (IXGBE_SUCCESS);
1942 }
1943
1944 /*
1945 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
1946 */
1947 static int
1948 ixgbe_alloc_rings(ixgbe_t *ixgbe)
1949 {
1950 /*
1951 * Allocate memory space for rx rings
1952 */
1953 ixgbe->rx_rings = kmem_zalloc(
1954 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
1955 KM_NOSLEEP);
1956
1957 if (ixgbe->rx_rings == NULL) {
1958 return (IXGBE_FAILURE);
1959 }
1960
1961 /*
1962 * Allocate memory space for tx rings
1963 */
1964 ixgbe->tx_rings = kmem_zalloc(
1965 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
1966 KM_NOSLEEP);
1967
1968 if (ixgbe->tx_rings == NULL) {
1969 kmem_free(ixgbe->rx_rings,
1970 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1971 ixgbe->rx_rings = NULL;
1972 return (IXGBE_FAILURE);
1973 }
1974
1975 /*
1976 * Allocate memory space for rx ring groups
1977 */
1978 ixgbe->rx_groups = kmem_zalloc(
1979 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
1980 KM_NOSLEEP);
1981
1982 if (ixgbe->rx_groups == NULL) {
1983 kmem_free(ixgbe->rx_rings,
1984 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1985 kmem_free(ixgbe->tx_rings,
1986 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
1987 ixgbe->rx_rings = NULL;
1988 ixgbe->tx_rings = NULL;
1989 return (IXGBE_FAILURE);
1990 }
1991
1992 return (IXGBE_SUCCESS);
1993 }
1994
1995 /*
1996 * ixgbe_free_rings - Free the memory space of rx/tx rings.
1997 */
1998 static void
1999 ixgbe_free_rings(ixgbe_t *ixgbe)
2000 {
2001 if (ixgbe->rx_rings != NULL) {
2002 kmem_free(ixgbe->rx_rings,
2003 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2004 ixgbe->rx_rings = NULL;
2005 }
2006
2007 if (ixgbe->tx_rings != NULL) {
2008 kmem_free(ixgbe->tx_rings,
2009 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2010 ixgbe->tx_rings = NULL;
2011 }
2012
2013 if (ixgbe->rx_groups != NULL) {
2014 kmem_free(ixgbe->rx_groups,
2015 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2016 ixgbe->rx_groups = NULL;
2017 }
2018 }
2019
2020 static int
2021 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2022 {
2023 ixgbe_rx_ring_t *rx_ring;
2024 int i;
2025
2026 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2027 rx_ring = &ixgbe->rx_rings[i];
2028 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2029 goto alloc_rx_rings_failure;
2030 }
2031 return (IXGBE_SUCCESS);
2032
2033 alloc_rx_rings_failure:
2034 ixgbe_free_rx_data(ixgbe);
2035 return (IXGBE_FAILURE);
2036 }
2037
2038 static void
2039 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2040 {
2041 ixgbe_rx_ring_t *rx_ring;
2042 ixgbe_rx_data_t *rx_data;
2043 int i;
2044
2045 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2046 rx_ring = &ixgbe->rx_rings[i];
2047
2048 mutex_enter(&ixgbe->rx_pending_lock);
2049 rx_data = rx_ring->rx_data;
2050
2051 if (rx_data != NULL) {
2052 rx_data->flag |= IXGBE_RX_STOPPED;
2053
2054 if (rx_data->rcb_pending == 0) {
2055 ixgbe_free_rx_ring_data(rx_data);
2056 rx_ring->rx_data = NULL;
2057 }
2058 }
2059
2060 mutex_exit(&ixgbe->rx_pending_lock);
2061 }
2062 }
2063
2064 /*
2065 * ixgbe_setup_rings - Setup rx/tx rings.
2066 */
2067 static void
2068 ixgbe_setup_rings(ixgbe_t *ixgbe)
2069 {
2070 /*
2071 * Setup the rx/tx rings, including the following:
2072 *
2073 * 1. Setup the descriptor ring and the control block buffers;
2074 * 2. Initialize necessary registers for receive/transmit;
2075 * 3. Initialize software pointers/parameters for receive/transmit;
2076 */
2077 ixgbe_setup_rx(ixgbe);
2078
2079 ixgbe_setup_tx(ixgbe);
2080 }
2081
2082 static void
2083 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2084 {
2085 ixgbe_t *ixgbe = rx_ring->ixgbe;
2086 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2087 struct ixgbe_hw *hw = &ixgbe->hw;
2088 rx_control_block_t *rcb;
2089 union ixgbe_adv_rx_desc *rbd;
2090 uint32_t size;
2091 uint32_t buf_low;
2092 uint32_t buf_high;
2093 uint32_t reg_val;
2094 int i;
2095
2096 ASSERT(mutex_owned(&rx_ring->rx_lock));
2097 ASSERT(mutex_owned(&ixgbe->gen_lock));
2098
2099 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2100 rcb = rx_data->work_list[i];
2101 rbd = &rx_data->rbd_ring[i];
2102
2103 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2104 rbd->read.hdr_addr = NULL;
2105 }
2106
2107 /*
2108 * Initialize the length register
2109 */
2110 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2111 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2112
2113 /*
2114 * Initialize the base address registers
2115 */
2116 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2117 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2118 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2119 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2120
2121 /*
2122 * Setup head & tail pointers
2123 */
2124 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2125 rx_data->ring_size - 1);
2126 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2127
2128 rx_data->rbd_next = 0;
2129 rx_data->lro_first = 0;
2130
2131 /*
2132 * Setup the Receive Descriptor Control Register (RXDCTL)
2133 * PTHRESH=32 descriptors (half the internal cache)
2134 * HTHRESH=0 descriptors (to minimize latency on fetch)
2135 * WTHRESH defaults to 1 (writeback each descriptor)
2136 */
2137 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2138 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2139
2140 /* Not a valid value for 82599 */
2141 if (hw->mac.type < ixgbe_mac_82599EB) {
2142 reg_val |= 0x0020; /* pthresh */
2143 }
2144 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2145
2146 if (hw->mac.type >= ixgbe_mac_82599EB) {
2147 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2148 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2149 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2150 }
2151
2152 /*
2153 * Setup the Split and Replication Receive Control Register.
2154 * Set the rx buffer size and the advanced descriptor type.
2155 */
2156 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2157 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2158 reg_val |= IXGBE_SRRCTL_DROP_EN;
2159 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2160 }
2161
2162 static void
2163 ixgbe_setup_rx(ixgbe_t *ixgbe)
2164 {
2165 ixgbe_rx_ring_t *rx_ring;
2166 struct ixgbe_hw *hw = &ixgbe->hw;
2167 uint32_t reg_val;
2168 uint32_t ring_mapping;
2169 uint32_t i, index;
2170 uint32_t psrtype_rss_bit;
2171
2172 /* PSRTYPE must be configured for 82599 */
2173 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2174 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2175 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2176 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2177 reg_val |= IXGBE_PSRTYPE_L2HDR;
2178 reg_val |= 0x80000000;
2179 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2180 } else {
2181 if (ixgbe->num_rx_groups > 32) {
2182 psrtype_rss_bit = 0x20000000;
2183 } else {
2184 psrtype_rss_bit = 0x40000000;
2185 }
2186 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2187 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2188 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2189 reg_val |= IXGBE_PSRTYPE_L2HDR;
2190 reg_val |= psrtype_rss_bit;
2191 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2192 }
2193 }
2194
2195 /*
2196 * Set filter control in FCTRL to accept broadcast packets and do
2197 * not pass pause frames to host. Flow control settings are already
2198 * in this register, so preserve them.
2199 */
2200 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2201 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */
2202 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */
2203 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2204
2205 /*
2206 * Hardware checksum settings
2207 */
2208 if (ixgbe->rx_hcksum_enable) {
2209 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2210 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2211 }
2212
2213 /*
2214 * Setup VMDq and RSS for multiple receive queues
2215 */
2216 switch (ixgbe->classify_mode) {
2217 case IXGBE_CLASSIFY_RSS:
2218 /*
2219 * One group, only RSS is needed when more than
2220 * one ring enabled.
2221 */
2222 ixgbe_setup_rss(ixgbe);
2223 break;
2224
2225 case IXGBE_CLASSIFY_VMDQ:
2226 /*
2227 * Multiple groups, each group has one ring,
2228 * only VMDq is needed.
2229 */
2230 ixgbe_setup_vmdq(ixgbe);
2231 break;
2232
2233 case IXGBE_CLASSIFY_VMDQ_RSS:
2234 /*
2235 * Multiple groups and multiple rings, both
2236 * VMDq and RSS are needed.
2237 */
2238 ixgbe_setup_vmdq_rss(ixgbe);
2239 break;
2240
2241 default:
2242 break;
2243 }
2244
2245 /*
2246 * Enable the receive unit. This must be done after filter
2247 * control is set in FCTRL.
2248 */
2249 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */
2250 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */
2251 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2252
2253 /*
2254 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2255 */
2256 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2257 rx_ring = &ixgbe->rx_rings[i];
2258 ixgbe_setup_rx_ring(rx_ring);
2259 }
2260
2261 /*
2262 * Setup the per-ring statistics mapping.
2263 */
2264 ring_mapping = 0;
2265 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2266 index = ixgbe->rx_rings[i].hw_index;
2267 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2268 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2269 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2270 }
2271
2272 /*
2273 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2274 * by four bytes if the packet has a VLAN field, so includes MTU,
2275 * ethernet header and frame check sequence.
2276 * Register is MAXFRS in 82599.
2277 */
2278 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2279 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2280 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2281
2282 /*
2283 * Setup Jumbo Frame enable bit
2284 */
2285 if (ixgbe->default_mtu > ETHERMTU) {
2286 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2287 reg_val |= IXGBE_HLREG0_JUMBOEN;
2288 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2289 }
2290
2291 /*
2292 * Setup RSC for multiple receive queues.
2293 */
2294 if (ixgbe->lro_enable) {
2295 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2296 /*
2297 * Make sure rx_buf_size * MAXDESC not greater
2298 * than 65535.
2299 * Intel recommends 4 for MAXDESC field value.
2300 */
2301 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2302 reg_val |= IXGBE_RSCCTL_RSCEN;
2303 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2304 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2305 else
2306 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2307 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2308 }
2309
2310 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2311 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2312 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2313
2314 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2315 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2316 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2317 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2318
2319 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2320 }
2321 }
2322
2323 static void
2324 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2325 {
2326 ixgbe_t *ixgbe = tx_ring->ixgbe;
2327 struct ixgbe_hw *hw = &ixgbe->hw;
2328 uint32_t size;
2329 uint32_t buf_low;
2330 uint32_t buf_high;
2331 uint32_t reg_val;
2332
2333 ASSERT(mutex_owned(&tx_ring->tx_lock));
2334 ASSERT(mutex_owned(&ixgbe->gen_lock));
2335
2336 /*
2337 * Initialize the length register
2338 */
2339 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2340 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2341
2342 /*
2343 * Initialize the base address registers
2344 */
2345 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2346 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2347 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2348 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2349
2350 /*
2351 * Setup head & tail pointers
2352 */
2353 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2354 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2355
2356 /*
2357 * Setup head write-back
2358 */
2359 if (ixgbe->tx_head_wb_enable) {
2360 /*
2361 * The memory of the head write-back is allocated using
2362 * the extra tbd beyond the tail of the tbd ring.
2363 */
2364 tx_ring->tbd_head_wb = (uint32_t *)
2365 ((uintptr_t)tx_ring->tbd_area.address + size);
2366 *tx_ring->tbd_head_wb = 0;
2367
2368 buf_low = (uint32_t)
2369 (tx_ring->tbd_area.dma_address + size);
2370 buf_high = (uint32_t)
2371 ((tx_ring->tbd_area.dma_address + size) >> 32);
2372
2373 /* Set the head write-back enable bit */
2374 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2375
2376 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2377 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2378
2379 /*
2380 * Turn off relaxed ordering for head write back or it will
2381 * cause problems with the tx recycling
2382 */
2383 #if 1
2384 /* XXX KEBE ASKS --> Should we do what FreeBSD does? */
2385 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2386 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2387 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2388 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2389 if (hw->mac.type == ixgbe_mac_82598EB) {
2390 IXGBE_WRITE_REG(hw,
2391 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2392 } else {
2393 IXGBE_WRITE_REG(hw,
2394 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2395 }
2396 #else
2397 /* XXX KEBE ASKS --> Or should we do what we've always done? */
2398 reg_val = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index));
2399 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2400 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2401 #endif
2402 } else {
2403 tx_ring->tbd_head_wb = NULL;
2404 #if 1
2405 /*
2406 * XXX KEBE ASKS --> Should we do what FreeBSD does and
2407 * twiddle TXCTRL_DESC_WR0_EN off anyway?
2408 */
2409 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2410 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2411 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2412 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2413 if (hw->mac.type == ixgbe_mac_82598EB) {
2414 IXGBE_WRITE_REG(hw,
2415 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2416 } else {
2417 IXGBE_WRITE_REG(hw,
2418 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2419 }
2420 #endif
2421 }
2422
2423 tx_ring->tbd_head = 0;
2424 tx_ring->tbd_tail = 0;
2425 tx_ring->tbd_free = tx_ring->ring_size;
2426
2427 if (ixgbe->tx_ring_init == B_TRUE) {
2428 tx_ring->tcb_head = 0;
2429 tx_ring->tcb_tail = 0;
2430 tx_ring->tcb_free = tx_ring->free_list_size;
2431 }
2432
2433 /*
2434 * Initialize the s/w context structure
2435 */
2436 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2437 }
2438
2439 static void
2440 ixgbe_setup_tx(ixgbe_t *ixgbe)
2441 {
2442 struct ixgbe_hw *hw = &ixgbe->hw;
2443 ixgbe_tx_ring_t *tx_ring;
2444 uint32_t reg_val;
2445 uint32_t ring_mapping;
2446 int i;
2447
2448 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2449 tx_ring = &ixgbe->tx_rings[i];
2450 ixgbe_setup_tx_ring(tx_ring);
2451 }
2452
2453 /*
2454 * Setup the per-ring statistics mapping.
2455 */
2456 ring_mapping = 0;
2457 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2458 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2459 if ((i & 0x3) == 0x3) {
2460 switch (hw->mac.type) {
2461 case ixgbe_mac_82598EB:
2462 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2463 ring_mapping);
2464 break;
2465
2466 case ixgbe_mac_82599EB:
2467 case ixgbe_mac_X540:
2468 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2469 ring_mapping);
2470 break;
2471
2472 default:
2473 break;
2474 }
2475
2476 ring_mapping = 0;
2477 }
2478 }
2479 if (i & 0x3) {
2480 switch (hw->mac.type) {
2481 case ixgbe_mac_82598EB:
2482 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2483 break;
2484
2485 case ixgbe_mac_82599EB:
2486 case ixgbe_mac_X540:
2487 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2488 break;
2489
2490 default:
2491 break;
2492 }
2493 }
2494
2495 /*
2496 * Enable CRC appending and TX padding (for short tx frames)
2497 */
2498 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2499 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2500 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2501
2502 /*
2503 * enable DMA for 82599 and X540 parts
2504 */
2505 if (hw->mac.type >= ixgbe_mac_82599EB) {
2506 /* DMATXCTL.TE must be set after all Tx config is complete */
2507 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2508 reg_val |= IXGBE_DMATXCTL_TE;
2509 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2510 #if 0
2511 /* XXX KEBE SAYS - FreeBSD sets up MTQC. Should we? */
2512 /* Disable arbiter to set MTQC */
2513 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2514 reg_val |= IXGBE_RTTDCS_ARBDIS;
2515 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2516 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2517 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2518 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2519 #endif
2520 }
2521
2522 /*
2523 * Enabling tx queues ..
2524 * For 82599 must be done after DMATXCTL.TE is set
2525 */
2526 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2527 tx_ring = &ixgbe->tx_rings[i];
2528 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2529 reg_val |= IXGBE_TXDCTL_ENABLE;
2530 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2531 }
2532 }
2533
2534 /*
2535 * ixgbe_setup_rss - Setup receive-side scaling feature.
2536 */
2537 static void
2538 ixgbe_setup_rss(ixgbe_t *ixgbe)
2539 {
2540 struct ixgbe_hw *hw = &ixgbe->hw;
2541 uint32_t i, mrqc, rxcsum;
2542 uint32_t random;
2543 uint32_t reta;
2544 uint32_t ring_per_group;
2545
2546 /*
2547 * Fill out redirection table
2548 */
2549 reta = 0;
2550 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2551
2552 for (i = 0; i < 128; i++) {
2553 reta = (reta << 8) | (i % ring_per_group) |
2554 ((i % ring_per_group) << 4);
2555 if ((i & 3) == 3)
2556 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2557 }
2558
2559 /*
2560 * Fill out hash function seeds with a random constant
2561 */
2562 for (i = 0; i < 10; i++) {
2563 (void) random_get_pseudo_bytes((uint8_t *)&random,
2564 sizeof (uint32_t));
2565 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2566 }
2567
2568 /*
2569 * Enable RSS & perform hash on these packet types
2570 */
2571 mrqc = IXGBE_MRQC_RSSEN |
2572 IXGBE_MRQC_RSS_FIELD_IPV4 |
2573 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2574 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2575 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2576 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2577 IXGBE_MRQC_RSS_FIELD_IPV6 |
2578 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2579 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2580 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2581 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2582
2583 /*
2584 * Disable Packet Checksum to enable RSS for multiple receive queues.
2585 * It is an adapter hardware limitation that Packet Checksum is
2586 * mutually exclusive with RSS.
2587 */
2588 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2589 rxcsum |= IXGBE_RXCSUM_PCSD;
2590 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2591 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2592 }
2593
2594 /*
2595 * ixgbe_setup_vmdq - Setup MAC classification feature
2596 */
2597 static void
2598 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2599 {
2600 struct ixgbe_hw *hw = &ixgbe->hw;
2601 uint32_t vmdctl, i, vtctl;
2602
2603 /*
2604 * Setup the VMDq Control register, enable VMDq based on
2605 * packet destination MAC address:
2606 */
2607 switch (hw->mac.type) {
2608 case ixgbe_mac_82598EB:
2609 /*
2610 * VMDq Enable = 1;
2611 * VMDq Filter = 0; MAC filtering
2612 * Default VMDq output index = 0;
2613 */
2614 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2615 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2616 break;
2617
2618 case ixgbe_mac_82599EB:
2619 case ixgbe_mac_X540:
2620 /*
2621 * Enable VMDq-only.
2622 */
2623 vmdctl = IXGBE_MRQC_VMDQEN;
2624 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2625
2626 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2627 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2628 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2629 }
2630
2631 /*
2632 * Enable Virtualization and Replication.
2633 */
2634 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2635 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2636
2637 /*
2638 * Enable receiving packets to all VFs
2639 */
2640 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2641 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2642 break;
2643
2644 default:
2645 break;
2646 }
2647 }
2648
2649 /*
2650 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2651 */
2652 static void
2653 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2654 {
2655 struct ixgbe_hw *hw = &ixgbe->hw;
2656 uint32_t i, mrqc, rxcsum;
2657 uint32_t random;
2658 uint32_t reta;
2659 uint32_t ring_per_group;
2660 uint32_t vmdctl, vtctl;
2661
2662 /*
2663 * Fill out redirection table
2664 */
2665 reta = 0;
2666 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2667 for (i = 0; i < 128; i++) {
2668 reta = (reta << 8) | (i % ring_per_group) |
2669 ((i % ring_per_group) << 4);
2670 if ((i & 3) == 3)
2671 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2672 }
2673
2674 /*
2675 * Fill out hash function seeds with a random constant
2676 */
2677 for (i = 0; i < 10; i++) {
2678 (void) random_get_pseudo_bytes((uint8_t *)&random,
2679 sizeof (uint32_t));
2680 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2681 }
2682
2683 /*
2684 * Enable and setup RSS and VMDq
2685 */
2686 switch (hw->mac.type) {
2687 case ixgbe_mac_82598EB:
2688 /*
2689 * Enable RSS & Setup RSS Hash functions
2690 */
2691 mrqc = IXGBE_MRQC_RSSEN |
2692 IXGBE_MRQC_RSS_FIELD_IPV4 |
2693 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2694 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2695 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2696 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2697 IXGBE_MRQC_RSS_FIELD_IPV6 |
2698 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2699 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2700 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2701 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2702
2703 /*
2704 * Enable and Setup VMDq
2705 * VMDq Filter = 0; MAC filtering
2706 * Default VMDq output index = 0;
2707 */
2708 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2709 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2710 break;
2711
2712 case ixgbe_mac_82599EB:
2713 case ixgbe_mac_X540:
2714 /*
2715 * Enable RSS & Setup RSS Hash functions
2716 */
2717 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2718 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2719 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2720 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2721 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2722 IXGBE_MRQC_RSS_FIELD_IPV6 |
2723 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2724 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2725 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2726
2727 /*
2728 * Enable VMDq+RSS.
2729 */
2730 if (ixgbe->num_rx_groups > 32) {
2731 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2732 } else {
2733 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2734 }
2735
2736 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2737
2738 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2739 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2740 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2741 }
2742 break;
2743
2744 default:
2745 break;
2746
2747 }
2748
2749 /*
2750 * Disable Packet Checksum to enable RSS for multiple receive queues.
2751 * It is an adapter hardware limitation that Packet Checksum is
2752 * mutually exclusive with RSS.
2753 */
2754 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2755 rxcsum |= IXGBE_RXCSUM_PCSD;
2756 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2757 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2758
2759 if (hw->mac.type >= ixgbe_mac_82599EB) {
2760 /*
2761 * Enable Virtualization and Replication.
2762 */
2763 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2764 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2765
2766 /*
2767 * Enable receiving packets to all VFs
2768 */
2769 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2770 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2771 }
2772 }
2773
2774 /*
2775 * ixgbe_init_unicst - Initialize the unicast addresses.
2776 */
2777 static void
2778 ixgbe_init_unicst(ixgbe_t *ixgbe)
2779 {
2780 struct ixgbe_hw *hw = &ixgbe->hw;
2781 uint8_t *mac_addr;
2782 int slot;
2783 /*
2784 * Here we should consider two situations:
2785 *
2786 * 1. Chipset is initialized at the first time,
2787 * Clear all the multiple unicast addresses.
2788 *
2789 * 2. Chipset is reset
2790 * Recover the multiple unicast addresses from the
2791 * software data structure to the RAR registers.
2792 */
2793 if (!ixgbe->unicst_init) {
2794 /*
2795 * Initialize the multiple unicast addresses
2796 */
2797 ixgbe->unicst_total = hw->mac.num_rar_entries;
2798 ixgbe->unicst_avail = ixgbe->unicst_total;
2799 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2800 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2801 bzero(mac_addr, ETHERADDRL);
2802 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2803 ixgbe->unicst_addr[slot].mac.set = 0;
2804 }
2805 ixgbe->unicst_init = B_TRUE;
2806 } else {
2807 /* Re-configure the RAR registers */
2808 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2809 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2810 if (ixgbe->unicst_addr[slot].mac.set == 1) {
2811 (void) ixgbe_set_rar(hw, slot, mac_addr,
2812 ixgbe->unicst_addr[slot].mac.group_index,
2813 IXGBE_RAH_AV);
2814 } else {
2815 bzero(mac_addr, ETHERADDRL);
2816 (void) ixgbe_set_rar(hw, slot, mac_addr,
2817 NULL, NULL);
2818 }
2819 }
2820 }
2821 }
2822
2823 /*
2824 * ixgbe_unicst_find - Find the slot for the specified unicast address
2825 */
2826 int
2827 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2828 {
2829 int slot;
2830
2831 ASSERT(mutex_owned(&ixgbe->gen_lock));
2832
2833 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2834 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2835 mac_addr, ETHERADDRL) == 0)
2836 return (slot);
2837 }
2838
2839 return (-1);
2840 }
2841
2842 /*
2843 * ixgbe_multicst_add - Add a multicst address.
2844 */
2845 int
2846 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2847 {
2848 ASSERT(mutex_owned(&ixgbe->gen_lock));
2849
2850 if ((multiaddr[0] & 01) == 0) {
2851 return (EINVAL);
2852 }
2853
2854 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2855 return (ENOENT);
2856 }
2857
2858 bcopy(multiaddr,
2859 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2860 ixgbe->mcast_count++;
2861
2862 /*
2863 * Update the multicast table in the hardware
2864 */
2865 ixgbe_setup_multicst(ixgbe);
2866
2867 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2868 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2869 return (EIO);
2870 }
2871
2872 return (0);
2873 }
2874
2875 /*
2876 * ixgbe_multicst_remove - Remove a multicst address.
2877 */
2878 int
2879 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2880 {
2881 int i;
2882
2883 ASSERT(mutex_owned(&ixgbe->gen_lock));
2884
2885 for (i = 0; i < ixgbe->mcast_count; i++) {
2886 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2887 ETHERADDRL) == 0) {
2888 for (i++; i < ixgbe->mcast_count; i++) {
2889 ixgbe->mcast_table[i - 1] =
2890 ixgbe->mcast_table[i];
2891 }
2892 ixgbe->mcast_count--;
2893 break;
2894 }
2895 }
2896
2897 /*
2898 * Update the multicast table in the hardware
2899 */
2900 ixgbe_setup_multicst(ixgbe);
2901
2902 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2903 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2904 return (EIO);
2905 }
2906
2907 return (0);
2908 }
2909
2910 /*
2911 * ixgbe_setup_multicast - Setup multicast data structures.
2912 *
2913 * This routine initializes all of the multicast related structures
2914 * and save them in the hardware registers.
2915 */
2916 static void
2917 ixgbe_setup_multicst(ixgbe_t *ixgbe)
2918 {
2919 uint8_t *mc_addr_list;
2920 uint32_t mc_addr_count;
2921 struct ixgbe_hw *hw = &ixgbe->hw;
2922
2923 ASSERT(mutex_owned(&ixgbe->gen_lock));
2924
2925 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
2926
2927 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
2928 mc_addr_count = ixgbe->mcast_count;
2929
2930 /*
2931 * Update the multicast addresses to the MTA registers
2932 */
2933 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
2934 ixgbe_mc_table_itr, TRUE);
2935 }
2936
2937 /*
2938 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
2939 *
2940 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
2941 * Different chipsets may have different allowed configuration of vmdq and rss.
2942 */
2943 static void
2944 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
2945 {
2946 struct ixgbe_hw *hw = &ixgbe->hw;
2947 uint32_t ring_per_group;
2948
2949 switch (hw->mac.type) {
2950 case ixgbe_mac_82598EB:
2951 /*
2952 * 82598 supports the following combination:
2953 * vmdq no. x rss no.
2954 * [5..16] x 1
2955 * [1..4] x [1..16]
2956 * However 8 rss queue per pool (vmdq) is sufficient for
2957 * most cases.
2958 */
2959 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2960 if (ixgbe->num_rx_groups > 4) {
2961 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
2962 } else {
2963 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2964 min(8, ring_per_group);
2965 }
2966
2967 break;
2968
2969 case ixgbe_mac_82599EB:
2970 case ixgbe_mac_X540:
2971 /*
2972 * 82599 supports the following combination:
2973 * vmdq no. x rss no.
2974 * [33..64] x [1..2]
2975 * [2..32] x [1..4]
2976 * 1 x [1..16]
2977 * However 8 rss queue per pool (vmdq) is sufficient for
2978 * most cases.
2979 *
2980 * For now, treat X540 like the 82599.
2981 */
2982 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2983 if (ixgbe->num_rx_groups == 1) {
2984 ixgbe->num_rx_rings = min(8, ring_per_group);
2985 } else if (ixgbe->num_rx_groups <= 32) {
2986 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2987 min(4, ring_per_group);
2988 } else if (ixgbe->num_rx_groups <= 64) {
2989 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2990 min(2, ring_per_group);
2991 }
2992 break;
2993
2994 default:
2995 break;
2996 }
2997
2998 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2999
3000 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3001 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3002 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3003 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3004 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3005 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3006 } else {
3007 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3008 }
3009
3010 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3011 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3012 }
3013
3014 /*
3015 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3016 *
3017 * This routine gets user-configured values out of the configuration
3018 * file ixgbe.conf.
3019 *
3020 * For each configurable value, there is a minimum, a maximum, and a
3021 * default.
3022 * If user does not configure a value, use the default.
3023 * If user configures below the minimum, use the minumum.
3024 * If user configures above the maximum, use the maxumum.
3025 */
3026 static void
3027 ixgbe_get_conf(ixgbe_t *ixgbe)
3028 {
3029 struct ixgbe_hw *hw = &ixgbe->hw;
3030 uint32_t flow_control;
3031
3032 /*
3033 * ixgbe driver supports the following user configurations:
3034 *
3035 * Jumbo frame configuration:
3036 * default_mtu
3037 *
3038 * Ethernet flow control configuration:
3039 * flow_control
3040 *
3041 * Multiple rings configurations:
3042 * tx_queue_number
3043 * tx_ring_size
3044 * rx_queue_number
3045 * rx_ring_size
3046 *
3047 * Call ixgbe_get_prop() to get the value for a specific
3048 * configuration parameter.
3049 */
3050
3051 /*
3052 * Jumbo frame configuration - max_frame_size controls host buffer
3053 * allocation, so includes MTU, ethernet header, vlan tag and
3054 * frame check sequence.
3055 */
3056 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3057 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3058
3059 ixgbe->max_frame_size = ixgbe->default_mtu +
3060 sizeof (struct ether_vlan_header) + ETHERFCSL;
3061
3062 /*
3063 * Ethernet flow control configuration
3064 */
3065 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3066 ixgbe_fc_none, 3, ixgbe_fc_none);
3067 if (flow_control == 3)
3068 flow_control = ixgbe_fc_default;
3069
3070 /*
3071 * fc.requested mode is what the user requests. After autoneg,
3072 * fc.current_mode will be the flow_control mode that was negotiated.
3073 */
3074 hw->fc.requested_mode = flow_control;
3075
3076 /*
3077 * Multiple rings configurations
3078 */
3079 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3080 ixgbe->capab->min_tx_que_num,
3081 ixgbe->capab->max_tx_que_num,
3082 ixgbe->capab->def_tx_que_num);
3083 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3084 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3085
3086 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3087 ixgbe->capab->min_rx_que_num,
3088 ixgbe->capab->max_rx_que_num,
3089 ixgbe->capab->def_rx_que_num);
3090 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3091 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3092
3093 /*
3094 * Multiple groups configuration
3095 */
3096 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3097 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3098 ixgbe->capab->def_rx_grp_num);
3099
3100 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3101 0, 1, DEFAULT_MR_ENABLE);
3102
3103 if (ixgbe->mr_enable == B_FALSE) {
3104 ixgbe->num_tx_rings = 1;
3105 ixgbe->num_rx_rings = 1;
3106 ixgbe->num_rx_groups = 1;
3107 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3108 } else {
3109 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3110 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3111 /*
3112 * The combination of num_rx_rings and num_rx_groups
3113 * may be not supported by h/w. We need to adjust
3114 * them to appropriate values.
3115 */
3116 ixgbe_setup_vmdq_rss_conf(ixgbe);
3117 }
3118
3119 /*
3120 * Tunable used to force an interrupt type. The only use is
3121 * for testing of the lesser interrupt types.
3122 * 0 = don't force interrupt type
3123 * 1 = force interrupt type MSI-X
3124 * 2 = force interrupt type MSI
3125 * 3 = force interrupt type Legacy
3126 */
3127 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3128 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3129
3130 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3131 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3132 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3133 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3134 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3135 0, 1, DEFAULT_LSO_ENABLE);
3136 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3137 0, 1, DEFAULT_LRO_ENABLE);
3138 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3139 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3140 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3141 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3142
3143 /* Head Write Back not recommended for 82599 and X540 */
3144 if (hw->mac.type >= ixgbe_mac_82599EB) {
3145 ixgbe->tx_head_wb_enable = B_FALSE;
3146 }
3147
3148 /*
3149 * ixgbe LSO needs the tx h/w checksum support.
3150 * LSO will be disabled if tx h/w checksum is not
3151 * enabled.
3152 */
3153 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3154 ixgbe->lso_enable = B_FALSE;
3155 }
3156
3157 /*
3158 * ixgbe LRO needs the rx h/w checksum support.
3159 * LRO will be disabled if rx h/w checksum is not
3160 * enabled.
3161 */
3162 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3163 ixgbe->lro_enable = B_FALSE;
3164 }
3165
3166 /*
3167 * ixgbe LRO only been supported by 82599 and X540 now
3168 */
3169 if (hw->mac.type < ixgbe_mac_82599EB) {
3170 ixgbe->lro_enable = B_FALSE;
3171 }
3172 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3173 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3174 DEFAULT_TX_COPY_THRESHOLD);
3175 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3176 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3177 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3178 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3179 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3180 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3181 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3182 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3183 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3184
3185 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3186 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3187 DEFAULT_RX_COPY_THRESHOLD);
3188 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3189 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3190 DEFAULT_RX_LIMIT_PER_INTR);
3191
3192 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3193 ixgbe->capab->min_intr_throttle,
3194 ixgbe->capab->max_intr_throttle,
3195 ixgbe->capab->def_intr_throttle);
3196 /*
3197 * 82599 and X540 require the interupt throttling rate is
3198 * a multiple of 8. This is enforced by the register
3199 * definiton.
3200 */
3201 if (hw->mac.type >= ixgbe_mac_82599EB)
3202 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3203 }
3204
3205 static void
3206 ixgbe_init_params(ixgbe_t *ixgbe)
3207 {
3208 ixgbe->param_en_10000fdx_cap = 1;
3209 ixgbe->param_en_1000fdx_cap = 1;
3210 ixgbe->param_en_100fdx_cap = 1;
3211 ixgbe->param_adv_10000fdx_cap = 1;
3212 ixgbe->param_adv_1000fdx_cap = 1;
3213 ixgbe->param_adv_100fdx_cap = 1;
3214
3215 ixgbe->param_pause_cap = 1;
3216 ixgbe->param_asym_pause_cap = 1;
3217 ixgbe->param_rem_fault = 0;
3218
3219 ixgbe->param_adv_autoneg_cap = 1;
3220 ixgbe->param_adv_pause_cap = 1;
3221 ixgbe->param_adv_asym_pause_cap = 1;
3222 ixgbe->param_adv_rem_fault = 0;
3223
3224 ixgbe->param_lp_10000fdx_cap = 0;
3225 ixgbe->param_lp_1000fdx_cap = 0;
3226 ixgbe->param_lp_100fdx_cap = 0;
3227 ixgbe->param_lp_autoneg_cap = 0;
3228 ixgbe->param_lp_pause_cap = 0;
3229 ixgbe->param_lp_asym_pause_cap = 0;
3230 ixgbe->param_lp_rem_fault = 0;
3231 }
3232
3233 /*
3234 * ixgbe_get_prop - Get a property value out of the configuration file
3235 * ixgbe.conf.
3236 *
3237 * Caller provides the name of the property, a default value, a minimum
3238 * value, and a maximum value.
3239 *
3240 * Return configured value of the property, with default, minimum and
3241 * maximum properly applied.
3242 */
3243 static int
3244 ixgbe_get_prop(ixgbe_t *ixgbe,
3245 char *propname, /* name of the property */
3246 int minval, /* minimum acceptable value */
3247 int maxval, /* maximim acceptable value */
3248 int defval) /* default value */
3249 {
3250 int value;
3251
3252 /*
3253 * Call ddi_prop_get_int() to read the conf settings
3254 */
3255 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3256 DDI_PROP_DONTPASS, propname, defval);
3257 if (value > maxval)
3258 value = maxval;
3259
3260 if (value < minval)
3261 value = minval;
3262
3263 return (value);
3264 }
3265
3266 /*
3267 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3268 */
3269 int
3270 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3271 {
3272 u32 autoneg_advertised = 0;
3273
3274 /*
3275 * No half duplex support with 10Gb parts
3276 */
3277 if (ixgbe->param_adv_10000fdx_cap == 1)
3278 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3279
3280 if (ixgbe->param_adv_1000fdx_cap == 1)
3281 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3282
3283 if (ixgbe->param_adv_100fdx_cap == 1)
3284 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3285
3286 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3287 ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3288 "to autonegotiation with full link capabilities.");
3289
3290 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3291 IXGBE_LINK_SPEED_1GB_FULL |
3292 IXGBE_LINK_SPEED_100_FULL;
3293 }
3294
3295 if (setup_hw) {
3296 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3297 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) {
3298 ixgbe_notice(ixgbe, "Setup link failed on this "
3299 "device.");
3300 return (IXGBE_FAILURE);
3301 }
3302 }
3303
3304 return (IXGBE_SUCCESS);
3305 }
3306
3307 /*
3308 * ixgbe_driver_link_check - Link status processing.
3309 *
3310 * This function can be called in both kernel context and interrupt context
3311 */
3312 static void
3313 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3314 {
3315 struct ixgbe_hw *hw = &ixgbe->hw;
3316 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3317 boolean_t link_up = B_FALSE;
3318 boolean_t link_changed = B_FALSE;
3319
3320 ASSERT(mutex_owned(&ixgbe->gen_lock));
3321
3322 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3323 if (link_up) {
3324 ixgbe->link_check_complete = B_TRUE;
3325
3326 /* Link is up, enable flow control settings */
3327 (void) ixgbe_fc_enable(hw);
3328
3329 /*
3330 * The Link is up, check whether it was marked as down earlier
3331 */
3332 if (ixgbe->link_state != LINK_STATE_UP) {
3333 switch (speed) {
3334 case IXGBE_LINK_SPEED_10GB_FULL:
3335 ixgbe->link_speed = SPEED_10GB;
3336 break;
3337 case IXGBE_LINK_SPEED_1GB_FULL:
3338 ixgbe->link_speed = SPEED_1GB;
3339 break;
3340 case IXGBE_LINK_SPEED_100_FULL:
3341 ixgbe->link_speed = SPEED_100;
3342 }
3343 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3344 ixgbe->link_state = LINK_STATE_UP;
3345 link_changed = B_TRUE;
3346 }
3347 } else {
3348 if (ixgbe->link_check_complete == B_TRUE ||
3349 (ixgbe->link_check_complete == B_FALSE &&
3350 gethrtime() >= ixgbe->link_check_hrtime)) {
3351 /*
3352 * The link is really down
3353 */
3354 ixgbe->link_check_complete = B_TRUE;
3355
3356 if (ixgbe->link_state != LINK_STATE_DOWN) {
3357 ixgbe->link_speed = 0;
3358 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3359 ixgbe->link_state = LINK_STATE_DOWN;
3360 link_changed = B_TRUE;
3361 }
3362 }
3363 }
3364
3365 /*
3366 * If we are in an interrupt context, need to re-enable the
3367 * interrupt, which was automasked
3368 */
3369 if (servicing_interrupt() != 0) {
3370 ixgbe->eims |= IXGBE_EICR_LSC;
3371 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3372 }
3373
3374 if (link_changed) {
3375 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3376 }
3377 }
3378
3379 /*
3380 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3381 */
3382 static void
3383 ixgbe_sfp_check(void *arg)
3384 {
3385 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3386 uint32_t eicr = ixgbe->eicr;
3387 struct ixgbe_hw *hw = &ixgbe->hw;
3388
3389 mutex_enter(&ixgbe->gen_lock);
3390 if (eicr & IXGBE_EICR_GPI_SDP1) {
3391 /* clear the interrupt */
3392 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
3393
3394 /* if link up, do multispeed fiber setup */
3395 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3396 B_TRUE, B_TRUE);
3397 ixgbe_driver_link_check(ixgbe);
3398 ixgbe_get_hw_state(ixgbe);
3399 } else if (eicr & IXGBE_EICR_GPI_SDP2) {
3400 /* clear the interrupt */
3401 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
3402
3403 /* if link up, do sfp module setup */
3404 (void) hw->mac.ops.setup_sfp(hw);
3405
3406 /* do multispeed fiber setup */
3407 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3408 B_TRUE, B_TRUE);
3409 ixgbe_driver_link_check(ixgbe);
3410 ixgbe_get_hw_state(ixgbe);
3411 }
3412 mutex_exit(&ixgbe->gen_lock);
3413
3414 /*
3415 * We need to fully re-check the link later.
3416 */
3417 ixgbe->link_check_complete = B_FALSE;
3418 ixgbe->link_check_hrtime = gethrtime() +
3419 (IXGBE_LINK_UP_TIME * 100000000ULL);
3420 }
3421
3422 /*
3423 * ixgbe_overtemp_check - overtemp module processing done in taskq
3424 *
3425 * This routine will only be called on adapters with temperature sensor.
3426 * The indication of over-temperature can be either SDP0 interrupt or the link
3427 * status change interrupt.
3428 */
3429 static void
3430 ixgbe_overtemp_check(void *arg)
3431 {
3432 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3433 struct ixgbe_hw *hw = &ixgbe->hw;
3434 uint32_t eicr = ixgbe->eicr;
3435 ixgbe_link_speed speed;
3436 boolean_t link_up;
3437
3438 mutex_enter(&ixgbe->gen_lock);
3439
3440 /* make sure we know current state of link */
3441 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3442
3443 /* check over-temp condition */
3444 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
3445 (eicr & IXGBE_EICR_LSC)) {
3446 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3447 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3448
3449 /*
3450 * Disable the adapter interrupts
3451 */
3452 ixgbe_disable_adapter_interrupts(ixgbe);
3453
3454 /*
3455 * Disable Rx/Tx units
3456 */
3457 (void) ixgbe_stop_adapter(hw);
3458
3459 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3460 ixgbe_error(ixgbe,
3461 "Problem: Network adapter has been stopped "
3462 "because it has overheated");
3463 ixgbe_error(ixgbe,
3464 "Action: Restart the computer. "
3465 "If the problem persists, power off the system "
3466 "and replace the adapter");
3467 }
3468 }
3469
3470 /* write to clear the interrupt */
3471 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3472
3473 mutex_exit(&ixgbe->gen_lock);
3474 }
3475
3476 /*
3477 * ixgbe_link_timer - timer for link status detection
3478 */
3479 static void
3480 ixgbe_link_timer(void *arg)
3481 {
3482 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3483
3484 mutex_enter(&ixgbe->gen_lock);
3485 ixgbe_driver_link_check(ixgbe);
3486 mutex_exit(&ixgbe->gen_lock);
3487 }
3488
3489 /*
3490 * ixgbe_local_timer - Driver watchdog function.
3491 *
3492 * This function will handle the transmit stall check and other routines.
3493 */
3494 static void
3495 ixgbe_local_timer(void *arg)
3496 {
3497 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3498
3499 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3500 goto out;
3501
3502 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3503 ixgbe->reset_count++;
3504 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3505 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3506 goto out;
3507 }
3508
3509 if (ixgbe_stall_check(ixgbe)) {
3510 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3511 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3512
3513 ixgbe->reset_count++;
3514 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3515 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3516 }
3517
3518 out:
3519 ixgbe_restart_watchdog_timer(ixgbe);
3520 }
3521
3522 /*
3523 * ixgbe_stall_check - Check for transmit stall.
3524 *
3525 * This function checks if the adapter is stalled (in transmit).
3526 *
3527 * It is called each time the watchdog timeout is invoked.
3528 * If the transmit descriptor reclaim continuously fails,
3529 * the watchdog value will increment by 1. If the watchdog
3530 * value exceeds the threshold, the ixgbe is assumed to
3531 * have stalled and need to be reset.
3532 */
3533 static boolean_t
3534 ixgbe_stall_check(ixgbe_t *ixgbe)
3535 {
3536 ixgbe_tx_ring_t *tx_ring;
3537 boolean_t result;
3538 int i;
3539
3540 if (ixgbe->link_state != LINK_STATE_UP)
3541 return (B_FALSE);
3542
3543 /*
3544 * If any tx ring is stalled, we'll reset the chipset
3545 */
3546 result = B_FALSE;
3547 for (i = 0; i < ixgbe->num_tx_rings; i++) {
3548 tx_ring = &ixgbe->tx_rings[i];
3549 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3550 tx_ring->tx_recycle(tx_ring);
3551 }
3552
3553 if (tx_ring->recycle_fail > 0)
3554 tx_ring->stall_watchdog++;
3555 else
3556 tx_ring->stall_watchdog = 0;
3557
3558 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3559 result = B_TRUE;
3560 break;
3561 }
3562 }
3563
3564 if (result) {
3565 tx_ring->stall_watchdog = 0;
3566 tx_ring->recycle_fail = 0;
3567 }
3568
3569 return (result);
3570 }
3571
3572
3573 /*
3574 * is_valid_mac_addr - Check if the mac address is valid.
3575 */
3576 static boolean_t
3577 is_valid_mac_addr(uint8_t *mac_addr)
3578 {
3579 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3580 const uint8_t addr_test2[6] =
3581 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3582
3583 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3584 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3585 return (B_FALSE);
3586
3587 return (B_TRUE);
3588 }
3589
3590 static boolean_t
3591 ixgbe_find_mac_address(ixgbe_t *ixgbe)
3592 {
3593 #ifdef __sparc
3594 struct ixgbe_hw *hw = &ixgbe->hw;
3595 uchar_t *bytes;
3596 struct ether_addr sysaddr;
3597 uint_t nelts;
3598 int err;
3599 boolean_t found = B_FALSE;
3600
3601 /*
3602 * The "vendor's factory-set address" may already have
3603 * been extracted from the chip, but if the property
3604 * "local-mac-address" is set we use that instead.
3605 *
3606 * We check whether it looks like an array of 6
3607 * bytes (which it should, if OBP set it). If we can't
3608 * make sense of it this way, we'll ignore it.
3609 */
3610 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3611 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3612 if (err == DDI_PROP_SUCCESS) {
3613 if (nelts == ETHERADDRL) {
3614 while (nelts--)
3615 hw->mac.addr[nelts] = bytes[nelts];
3616 found = B_TRUE;
3617 }
3618 ddi_prop_free(bytes);
3619 }
3620
3621 /*
3622 * Look up the OBP property "local-mac-address?". If the user has set
3623 * 'local-mac-address? = false', use "the system address" instead.
3624 */
3625 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3626 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3627 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3628 if (localetheraddr(NULL, &sysaddr) != 0) {
3629 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3630 found = B_TRUE;
3631 }
3632 }
3633 ddi_prop_free(bytes);
3634 }
3635
3636 /*
3637 * Finally(!), if there's a valid "mac-address" property (created
3638 * if we netbooted from this interface), we must use this instead
3639 * of any of the above to ensure that the NFS/install server doesn't
3640 * get confused by the address changing as Solaris takes over!
3641 */
3642 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3643 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3644 if (err == DDI_PROP_SUCCESS) {
3645 if (nelts == ETHERADDRL) {
3646 while (nelts--)
3647 hw->mac.addr[nelts] = bytes[nelts];
3648 found = B_TRUE;
3649 }
3650 ddi_prop_free(bytes);
3651 }
3652
3653 if (found) {
3654 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3655 return (B_TRUE);
3656 }
3657 #else
3658 _NOTE(ARGUNUSED(ixgbe));
3659 #endif
3660
3661 return (B_TRUE);
3662 }
3663
3664 #pragma inline(ixgbe_arm_watchdog_timer)
3665 static void
3666 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
3667 {
3668 /*
3669 * Fire a watchdog timer
3670 */
3671 ixgbe->watchdog_tid =
3672 timeout(ixgbe_local_timer,
3673 (void *)ixgbe, 1 * drv_usectohz(1000000));
3674
3675 }
3676
3677 /*
3678 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
3679 */
3680 void
3681 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
3682 {
3683 mutex_enter(&ixgbe->watchdog_lock);
3684
3685 if (!ixgbe->watchdog_enable) {
3686 ixgbe->watchdog_enable = B_TRUE;
3687 ixgbe->watchdog_start = B_TRUE;
3688 ixgbe_arm_watchdog_timer(ixgbe);
3689 }
3690
3691 mutex_exit(&ixgbe->watchdog_lock);
3692 }
3693
3694 /*
3695 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
3696 */
3697 void
3698 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
3699 {
3700 timeout_id_t tid;
3701
3702 mutex_enter(&ixgbe->watchdog_lock);
3703
3704 ixgbe->watchdog_enable = B_FALSE;
3705 ixgbe->watchdog_start = B_FALSE;
3706 tid = ixgbe->watchdog_tid;
3707 ixgbe->watchdog_tid = 0;
3708
3709 mutex_exit(&ixgbe->watchdog_lock);
3710
3711 if (tid != 0)
3712 (void) untimeout(tid);
3713 }
3714
3715 /*
3716 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
3717 */
3718 void
3719 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
3720 {
3721 mutex_enter(&ixgbe->watchdog_lock);
3722
3723 if (ixgbe->watchdog_enable) {
3724 if (!ixgbe->watchdog_start) {
3725 ixgbe->watchdog_start = B_TRUE;
3726 ixgbe_arm_watchdog_timer(ixgbe);
3727 }
3728 }
3729
3730 mutex_exit(&ixgbe->watchdog_lock);
3731 }
3732
3733 /*
3734 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
3735 */
3736 static void
3737 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
3738 {
3739 mutex_enter(&ixgbe->watchdog_lock);
3740
3741 if (ixgbe->watchdog_start)
3742 ixgbe_arm_watchdog_timer(ixgbe);
3743
3744 mutex_exit(&ixgbe->watchdog_lock);
3745 }
3746
3747 /*
3748 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
3749 */
3750 void
3751 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
3752 {
3753 timeout_id_t tid;
3754
3755 mutex_enter(&ixgbe->watchdog_lock);
3756
3757 ixgbe->watchdog_start = B_FALSE;
3758 tid = ixgbe->watchdog_tid;
3759 ixgbe->watchdog_tid = 0;
3760
3761 mutex_exit(&ixgbe->watchdog_lock);
3762
3763 if (tid != 0)
3764 (void) untimeout(tid);
3765 }
3766
3767 /*
3768 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
3769 */
3770 static void
3771 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
3772 {
3773 struct ixgbe_hw *hw = &ixgbe->hw;
3774
3775 /*
3776 * mask all interrupts off
3777 */
3778 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
3779
3780 /*
3781 * for MSI-X, also disable autoclear
3782 */
3783 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3784 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
3785 }
3786
3787 IXGBE_WRITE_FLUSH(hw);
3788 }
3789
3790 /*
3791 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
3792 */
3793 static void
3794 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
3795 {
3796 struct ixgbe_hw *hw = &ixgbe->hw;
3797 uint32_t eiac, eiam;
3798 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3799
3800 /* interrupt types to enable */
3801 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
3802 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
3803 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
3804
3805 /* enable automask on "other" causes that this adapter can generate */
3806 eiam = ixgbe->capab->other_intr;
3807
3808 /*
3809 * msi-x mode
3810 */
3811 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3812 /* enable autoclear but not on bits 29:20 */
3813 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3814
3815 /* general purpose interrupt enable */
3816 gpie |= (IXGBE_GPIE_MSIX_MODE
3817 | IXGBE_GPIE_PBA_SUPPORT
3818 | IXGBE_GPIE_OCD
3819 | IXGBE_GPIE_EIAME);
3820 /*
3821 * non-msi-x mode
3822 */
3823 } else {
3824
3825 /* disable autoclear, leave gpie at default */
3826 eiac = 0;
3827
3828 /*
3829 * General purpose interrupt enable.
3830 * For 82599, extended interrupt automask enable
3831 * only in MSI or MSI-X mode
3832 */
3833 if ((hw->mac.type < ixgbe_mac_82599EB) ||
3834 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3835 gpie |= IXGBE_GPIE_EIAME;
3836 }
3837 }
3838
3839 /* Enable specific "other" interrupt types */
3840 switch (hw->mac.type) {
3841 case ixgbe_mac_82598EB:
3842 gpie |= ixgbe->capab->other_gpie;
3843 break;
3844
3845 case ixgbe_mac_82599EB:
3846 case ixgbe_mac_X540:
3847 gpie |= ixgbe->capab->other_gpie;
3848
3849 /* Enable RSC Delay 8us when LRO enabled */
3850 if (ixgbe->lro_enable) {
3851 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3852 }
3853 break;
3854
3855 default:
3856 break;
3857 }
3858
3859 /* write to interrupt control registers */
3860 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3861 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3862 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3863 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3864 IXGBE_WRITE_FLUSH(hw);
3865 }
3866
3867 /*
3868 * ixgbe_loopback_ioctl - Loopback support.
3869 */
3870 enum ioc_reply
3871 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
3872 {
3873 lb_info_sz_t *lbsp;
3874 lb_property_t *lbpp;
3875 uint32_t *lbmp;
3876 uint32_t size;
3877 uint32_t value;
3878
3879 if (mp->b_cont == NULL)
3880 return (IOC_INVAL);
3881
3882 switch (iocp->ioc_cmd) {
3883 default:
3884 return (IOC_INVAL);
3885
3886 case LB_GET_INFO_SIZE:
3887 size = sizeof (lb_info_sz_t);
3888 if (iocp->ioc_count != size)
3889 return (IOC_INVAL);
3890
3891 value = sizeof (lb_normal);
3892 value += sizeof (lb_mac);
3893 value += sizeof (lb_external);
3894
3895 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3896 *lbsp = value;
3897 break;
3898
3899 case LB_GET_INFO:
3900 value = sizeof (lb_normal);
3901 value += sizeof (lb_mac);
3902 value += sizeof (lb_external);
3903
3904 size = value;
3905 if (iocp->ioc_count != size)
3906 return (IOC_INVAL);
3907
3908 value = 0;
3909 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3910
3911 lbpp[value++] = lb_normal;
3912 lbpp[value++] = lb_mac;
3913 lbpp[value++] = lb_external;
3914 break;
3915
3916 case LB_GET_MODE:
3917 size = sizeof (uint32_t);
3918 if (iocp->ioc_count != size)
3919 return (IOC_INVAL);
3920
3921 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3922 *lbmp = ixgbe->loopback_mode;
3923 break;
3924
3925 case LB_SET_MODE:
3926 size = 0;
3927 if (iocp->ioc_count != sizeof (uint32_t))
3928 return (IOC_INVAL);
3929
3930 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3931 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
3932 return (IOC_INVAL);
3933 break;
3934 }
3935
3936 iocp->ioc_count = size;
3937 iocp->ioc_error = 0;
3938
3939 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3940 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3941 return (IOC_INVAL);
3942 }
3943
3944 return (IOC_REPLY);
3945 }
3946
3947 /*
3948 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
3949 */
3950 static boolean_t
3951 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
3952 {
3953 if (mode == ixgbe->loopback_mode)
3954 return (B_TRUE);
3955
3956 ixgbe->loopback_mode = mode;
3957
3958 if (mode == IXGBE_LB_NONE) {
3959 /*
3960 * Reset the chip
3961 */
3962 (void) ixgbe_reset(ixgbe);
3963 return (B_TRUE);
3964 }
3965
3966 mutex_enter(&ixgbe->gen_lock);
3967
3968 switch (mode) {
3969 default:
3970 mutex_exit(&ixgbe->gen_lock);
3971 return (B_FALSE);
3972
3973 case IXGBE_LB_EXTERNAL:
3974 break;
3975
3976 case IXGBE_LB_INTERNAL_MAC:
3977 ixgbe_set_internal_mac_loopback(ixgbe);
3978 break;
3979 }
3980
3981 mutex_exit(&ixgbe->gen_lock);
3982
3983 return (B_TRUE);
3984 }
3985
3986 /*
3987 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
3988 */
3989 static void
3990 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
3991 {
3992 struct ixgbe_hw *hw;
3993 uint32_t reg;
3994 uint8_t atlas;
3995
3996 hw = &ixgbe->hw;
3997
3998 /*
3999 * Setup MAC loopback
4000 */
4001 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4002 reg |= IXGBE_HLREG0_LPBK;
4003 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4004
4005 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4006 reg &= ~IXGBE_AUTOC_LMS_MASK;
4007 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4008
4009 /*
4010 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4011 */
4012 switch (hw->mac.type) {
4013 case ixgbe_mac_82598EB:
4014 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4015 &atlas);
4016 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4017 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4018 atlas);
4019
4020 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4021 &atlas);
4022 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4023 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4024 atlas);
4025
4026 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4027 &atlas);
4028 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4029 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4030 atlas);
4031
4032 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4033 &atlas);
4034 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4035 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4036 atlas);
4037 break;
4038
4039 case ixgbe_mac_82599EB:
4040 case ixgbe_mac_X540:
4041 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4042 reg |= (IXGBE_AUTOC_FLU |
4043 IXGBE_AUTOC_10G_KX4);
4044 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4045
4046 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4047 B_FALSE, B_TRUE);
4048 break;
4049
4050 default:
4051 break;
4052 }
4053 }
4054
4055 #pragma inline(ixgbe_intr_rx_work)
4056 /*
4057 * ixgbe_intr_rx_work - RX processing of ISR.
4058 */
4059 static void
4060 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4061 {
4062 mblk_t *mp;
4063
4064 mutex_enter(&rx_ring->rx_lock);
4065
4066 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4067 mutex_exit(&rx_ring->rx_lock);
4068
4069 if (mp != NULL)
4070 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4071 rx_ring->ring_gen_num);
4072 }
4073
4074 #pragma inline(ixgbe_intr_tx_work)
4075 /*
4076 * ixgbe_intr_tx_work - TX processing of ISR.
4077 */
4078 static void
4079 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4080 {
4081 ixgbe_t *ixgbe = tx_ring->ixgbe;
4082
4083 /*
4084 * Recycle the tx descriptors
4085 */
4086 tx_ring->tx_recycle(tx_ring);
4087
4088 /*
4089 * Schedule the re-transmit
4090 */
4091 if (tx_ring->reschedule &&
4092 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4093 tx_ring->reschedule = B_FALSE;
4094 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4095 tx_ring->ring_handle);
4096 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4097 }
4098 }
4099
4100 #pragma inline(ixgbe_intr_other_work)
4101 /*
4102 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4103 */
4104 static void
4105 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4106 {
4107 ASSERT(mutex_owned(&ixgbe->gen_lock));
4108
4109 /*
4110 * handle link status change
4111 */
4112 if (eicr & IXGBE_EICR_LSC) {
4113 ixgbe_driver_link_check(ixgbe);
4114 ixgbe_get_hw_state(ixgbe);
4115 }
4116
4117 /*
4118 * check for fan failure on adapters with fans
4119 */
4120 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4121 (eicr & IXGBE_EICR_GPI_SDP1)) {
4122 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4123
4124 /*
4125 * Disable the adapter interrupts
4126 */
4127 ixgbe_disable_adapter_interrupts(ixgbe);
4128
4129 /*
4130 * Disable Rx/Tx units
4131 */
4132 (void) ixgbe_stop_adapter(&ixgbe->hw);
4133
4134 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4135 ixgbe_error(ixgbe,
4136 "Problem: Network adapter has been stopped "
4137 "because the fan has stopped.\n");
4138 ixgbe_error(ixgbe,
4139 "Action: Replace the adapter.\n");
4140
4141 /* re-enable the interrupt, which was automasked */
4142 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4143 }
4144
4145 /*
4146 * Do SFP check for adapters with hot-plug capability
4147 */
4148 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4149 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) {
4150 ixgbe->eicr = eicr;
4151 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4152 ixgbe_sfp_check, (void *)ixgbe,
4153 DDI_NOSLEEP)) != DDI_SUCCESS) {
4154 ixgbe_log(ixgbe, "No memory available to dispatch "
4155 "taskq for SFP check");
4156 }
4157 }
4158
4159 /*
4160 * Do over-temperature check for adapters with temp sensor
4161 */
4162 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4163 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
4164 ixgbe->eicr = eicr;
4165 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4166 ixgbe_overtemp_check, (void *)ixgbe,
4167 DDI_NOSLEEP)) != DDI_SUCCESS) {
4168 ixgbe_log(ixgbe, "No memory available to dispatch "
4169 "taskq for overtemp check");
4170 }
4171 }
4172 }
4173
4174 /*
4175 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4176 */
4177 static uint_t
4178 ixgbe_intr_legacy(void *arg1, void *arg2)
4179 {
4180 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4181 struct ixgbe_hw *hw = &ixgbe->hw;
4182 ixgbe_tx_ring_t *tx_ring;
4183 ixgbe_rx_ring_t *rx_ring;
4184 uint32_t eicr;
4185 mblk_t *mp;
4186 boolean_t tx_reschedule;
4187 uint_t result;
4188
4189 _NOTE(ARGUNUSED(arg2));
4190
4191 mutex_enter(&ixgbe->gen_lock);
4192 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4193 mutex_exit(&ixgbe->gen_lock);
4194 return (DDI_INTR_UNCLAIMED);
4195 }
4196
4197 mp = NULL;
4198 tx_reschedule = B_FALSE;
4199
4200 /*
4201 * Any bit set in eicr: claim this interrupt
4202 */
4203 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4204
4205 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4206 mutex_exit(&ixgbe->gen_lock);
4207 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4208 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4209 return (DDI_INTR_CLAIMED);
4210 }
4211
4212 if (eicr) {
4213 /*
4214 * For legacy interrupt, we have only one interrupt,
4215 * so we have only one rx ring and one tx ring enabled.
4216 */
4217 ASSERT(ixgbe->num_rx_rings == 1);
4218 ASSERT(ixgbe->num_tx_rings == 1);
4219
4220 /*
4221 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4222 */
4223 if (eicr & 0x1) {
4224 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4225 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4226 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4227 /*
4228 * Clean the rx descriptors
4229 */
4230 rx_ring = &ixgbe->rx_rings[0];
4231 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4232 }
4233
4234 /*
4235 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4236 */
4237 if (eicr & 0x2) {
4238 /*
4239 * Recycle the tx descriptors
4240 */
4241 tx_ring = &ixgbe->tx_rings[0];
4242 tx_ring->tx_recycle(tx_ring);
4243
4244 /*
4245 * Schedule the re-transmit
4246 */
4247 tx_reschedule = (tx_ring->reschedule &&
4248 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4249 }
4250
4251 /* any interrupt type other than tx/rx */
4252 if (eicr & ixgbe->capab->other_intr) {
4253 switch (hw->mac.type) {
4254 case ixgbe_mac_82598EB:
4255 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4256 break;
4257
4258 case ixgbe_mac_82599EB:
4259 case ixgbe_mac_X540:
4260 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4261 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4262 break;
4263
4264 default:
4265 break;
4266 }
4267 ixgbe_intr_other_work(ixgbe, eicr);
4268 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4269 }
4270
4271 mutex_exit(&ixgbe->gen_lock);
4272
4273 result = DDI_INTR_CLAIMED;
4274 } else {
4275 mutex_exit(&ixgbe->gen_lock);
4276
4277 /*
4278 * No interrupt cause bits set: don't claim this interrupt.
4279 */
4280 result = DDI_INTR_UNCLAIMED;
4281 }
4282
4283 /* re-enable the interrupts which were automasked */
4284 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4285
4286 /*
4287 * Do the following work outside of the gen_lock
4288 */
4289 if (mp != NULL) {
4290 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4291 rx_ring->ring_gen_num);
4292 }
4293
4294 if (tx_reschedule) {
4295 tx_ring->reschedule = B_FALSE;
4296 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4297 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4298 }
4299
4300 return (result);
4301 }
4302
4303 /*
4304 * ixgbe_intr_msi - Interrupt handler for MSI.
4305 */
4306 static uint_t
4307 ixgbe_intr_msi(void *arg1, void *arg2)
4308 {
4309 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4310 struct ixgbe_hw *hw = &ixgbe->hw;
4311 uint32_t eicr;
4312
4313 _NOTE(ARGUNUSED(arg2));
4314
4315 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4316
4317 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4318 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4319 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4320 return (DDI_INTR_CLAIMED);
4321 }
4322
4323 /*
4324 * For MSI interrupt, we have only one vector,
4325 * so we have only one rx ring and one tx ring enabled.
4326 */
4327 ASSERT(ixgbe->num_rx_rings == 1);
4328 ASSERT(ixgbe->num_tx_rings == 1);
4329
4330 /*
4331 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4332 */
4333 if (eicr & 0x1) {
4334 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4335 }
4336
4337 /*
4338 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4339 */
4340 if (eicr & 0x2) {
4341 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4342 }
4343
4344 /* any interrupt type other than tx/rx */
4345 if (eicr & ixgbe->capab->other_intr) {
4346 mutex_enter(&ixgbe->gen_lock);
4347 switch (hw->mac.type) {
4348 case ixgbe_mac_82598EB:
4349 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4350 break;
4351
4352 case ixgbe_mac_82599EB:
4353 case ixgbe_mac_X540:
4354 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4355 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4356 break;
4357
4358 default:
4359 break;
4360 }
4361 ixgbe_intr_other_work(ixgbe, eicr);
4362 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4363 mutex_exit(&ixgbe->gen_lock);
4364 }
4365
4366 /* re-enable the interrupts which were automasked */
4367 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4368
4369 return (DDI_INTR_CLAIMED);
4370 }
4371
4372 /*
4373 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4374 */
4375 static uint_t
4376 ixgbe_intr_msix(void *arg1, void *arg2)
4377 {
4378 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4379 ixgbe_t *ixgbe = vect->ixgbe;
4380 struct ixgbe_hw *hw = &ixgbe->hw;
4381 uint32_t eicr;
4382 int r_idx = 0;
4383
4384 _NOTE(ARGUNUSED(arg2));
4385
4386 /*
4387 * Clean each rx ring that has its bit set in the map
4388 */
4389 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4390 while (r_idx >= 0) {
4391 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4392 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4393 (ixgbe->num_rx_rings - 1));
4394 }
4395
4396 /*
4397 * Clean each tx ring that has its bit set in the map
4398 */
4399 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4400 while (r_idx >= 0) {
4401 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4402 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4403 (ixgbe->num_tx_rings - 1));
4404 }
4405
4406
4407 /*
4408 * Clean other interrupt (link change) that has its bit set in the map
4409 */
4410 if (BT_TEST(vect->other_map, 0) == 1) {
4411 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4412
4413 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4414 DDI_FM_OK) {
4415 ddi_fm_service_impact(ixgbe->dip,
4416 DDI_SERVICE_DEGRADED);
4417 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4418 return (DDI_INTR_CLAIMED);
4419 }
4420
4421 /*
4422 * Check "other" cause bits: any interrupt type other than tx/rx
4423 */
4424 if (eicr & ixgbe->capab->other_intr) {
4425 mutex_enter(&ixgbe->gen_lock);
4426 switch (hw->mac.type) {
4427 case ixgbe_mac_82598EB:
4428 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4429 ixgbe_intr_other_work(ixgbe, eicr);
4430 break;
4431
4432 case ixgbe_mac_82599EB:
4433 case ixgbe_mac_X540:
4434 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4435 ixgbe_intr_other_work(ixgbe, eicr);
4436 break;
4437
4438 default:
4439 break;
4440 }
4441 mutex_exit(&ixgbe->gen_lock);
4442 }
4443
4444 /* re-enable the interrupts which were automasked */
4445 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4446 }
4447
4448 return (DDI_INTR_CLAIMED);
4449 }
4450
4451 /*
4452 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4453 *
4454 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4455 * if not successful, try Legacy.
4456 * ixgbe->intr_force can be used to force sequence to start with
4457 * any of the 3 types.
4458 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4459 */
4460 static int
4461 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4462 {
4463 dev_info_t *devinfo;
4464 int intr_types;
4465 int rc;
4466
4467 devinfo = ixgbe->dip;
4468
4469 /*
4470 * Get supported interrupt types
4471 */
4472 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4473
4474 if (rc != DDI_SUCCESS) {
4475 ixgbe_log(ixgbe,
4476 "Get supported interrupt types failed: %d", rc);
4477 return (IXGBE_FAILURE);
4478 }
4479 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4480
4481 ixgbe->intr_type = 0;
4482
4483 /*
4484 * Install MSI-X interrupts
4485 */
4486 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4487 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4488 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4489 if (rc == IXGBE_SUCCESS)
4490 return (IXGBE_SUCCESS);
4491
4492 ixgbe_log(ixgbe,
4493 "Allocate MSI-X failed, trying MSI interrupts...");
4494 }
4495
4496 /*
4497 * MSI-X not used, force rings and groups to 1
4498 */
4499 ixgbe->num_rx_rings = 1;
4500 ixgbe->num_rx_groups = 1;
4501 ixgbe->num_tx_rings = 1;
4502 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4503 ixgbe_log(ixgbe,
4504 "MSI-X not used, force rings and groups number to 1");
4505
4506 /*
4507 * Install MSI interrupts
4508 */
4509 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4510 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4511 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4512 if (rc == IXGBE_SUCCESS)
4513 return (IXGBE_SUCCESS);
4514
4515 ixgbe_log(ixgbe,
4516 "Allocate MSI failed, trying Legacy interrupts...");
4517 }
4518
4519 /*
4520 * Install legacy interrupts
4521 */
4522 if (intr_types & DDI_INTR_TYPE_FIXED) {
4523 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4524 if (rc == IXGBE_SUCCESS)
4525 return (IXGBE_SUCCESS);
4526
4527 ixgbe_log(ixgbe,
4528 "Allocate Legacy interrupts failed");
4529 }
4530
4531 /*
4532 * If none of the 3 types succeeded, return failure
4533 */
4534 return (IXGBE_FAILURE);
4535 }
4536
4537 /*
4538 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4539 *
4540 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4541 * if fewer than 2 handles are available, return failure.
4542 * Upon success, this maps the vectors to rx and tx rings for
4543 * interrupts.
4544 */
4545 static int
4546 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4547 {
4548 dev_info_t *devinfo;
4549 int request, count, actual;
4550 int minimum;
4551 int rc;
4552 uint32_t ring_per_group;
4553
4554 devinfo = ixgbe->dip;
4555
4556 switch (intr_type) {
4557 case DDI_INTR_TYPE_FIXED:
4558 request = 1; /* Request 1 legacy interrupt handle */
4559 minimum = 1;
4560 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4561 break;
4562
4563 case DDI_INTR_TYPE_MSI:
4564 request = 1; /* Request 1 MSI interrupt handle */
4565 minimum = 1;
4566 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4567 break;
4568
4569 case DDI_INTR_TYPE_MSIX:
4570 /*
4571 * Best number of vectors for the adapter is
4572 * (# rx rings + # tx rings), however we will
4573 * limit the request number.
4574 */
4575 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4576 if (request > ixgbe->capab->max_ring_vect)
4577 request = ixgbe->capab->max_ring_vect;
4578 minimum = 1;
4579 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4580 break;
4581
4582 default:
4583 ixgbe_log(ixgbe,
4584 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4585 intr_type);
4586 return (IXGBE_FAILURE);
4587 }
4588 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
4589 request, minimum);
4590
4591 /*
4592 * Get number of supported interrupts
4593 */
4594 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4595 if ((rc != DDI_SUCCESS) || (count < minimum)) {
4596 ixgbe_log(ixgbe,
4597 "Get interrupt number failed. Return: %d, count: %d",
4598 rc, count);
4599 return (IXGBE_FAILURE);
4600 }
4601 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4602
4603 actual = 0;
4604 ixgbe->intr_cnt = 0;
4605 ixgbe->intr_cnt_max = 0;
4606 ixgbe->intr_cnt_min = 0;
4607
4608 /*
4609 * Allocate an array of interrupt handles
4610 */
4611 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
4612 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
4613
4614 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
4615 request, &actual, DDI_INTR_ALLOC_NORMAL);
4616 if (rc != DDI_SUCCESS) {
4617 ixgbe_log(ixgbe, "Allocate interrupts failed. "
4618 "return: %d, request: %d, actual: %d",
4619 rc, request, actual);
4620 goto alloc_handle_fail;
4621 }
4622 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
4623
4624 /*
4625 * upper/lower limit of interrupts
4626 */
4627 ixgbe->intr_cnt = actual;
4628 ixgbe->intr_cnt_max = request;
4629 ixgbe->intr_cnt_min = minimum;
4630
4631 /*
4632 * rss number per group should not exceed the rx interrupt number,
4633 * else need to adjust rx ring number.
4634 */
4635 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4636 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
4637 if (actual < ring_per_group) {
4638 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
4639 ixgbe_setup_vmdq_rss_conf(ixgbe);
4640 }
4641
4642 /*
4643 * Now we know the actual number of vectors. Here we map the vector
4644 * to other, rx rings and tx ring.
4645 */
4646 if (actual < minimum) {
4647 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
4648 actual);
4649 goto alloc_handle_fail;
4650 }
4651
4652 /*
4653 * Get priority for first vector, assume remaining are all the same
4654 */
4655 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
4656 if (rc != DDI_SUCCESS) {
4657 ixgbe_log(ixgbe,
4658 "Get interrupt priority failed: %d", rc);
4659 goto alloc_handle_fail;
4660 }
4661
4662 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
4663 if (rc != DDI_SUCCESS) {
4664 ixgbe_log(ixgbe,
4665 "Get interrupt cap failed: %d", rc);
4666 goto alloc_handle_fail;
4667 }
4668
4669 ixgbe->intr_type = intr_type;
4670
4671 return (IXGBE_SUCCESS);
4672
4673 alloc_handle_fail:
4674 ixgbe_rem_intrs(ixgbe);
4675
4676 return (IXGBE_FAILURE);
4677 }
4678
4679 /*
4680 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
4681 *
4682 * Before adding the interrupt handlers, the interrupt vectors have
4683 * been allocated, and the rx/tx rings have also been allocated.
4684 */
4685 static int
4686 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
4687 {
4688 int vector = 0;
4689 int rc;
4690
4691 switch (ixgbe->intr_type) {
4692 case DDI_INTR_TYPE_MSIX:
4693 /*
4694 * Add interrupt handler for all vectors
4695 */
4696 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
4697 /*
4698 * install pointer to vect_map[vector]
4699 */
4700 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4701 (ddi_intr_handler_t *)ixgbe_intr_msix,
4702 (void *)&ixgbe->vect_map[vector], NULL);
4703
4704 if (rc != DDI_SUCCESS) {
4705 ixgbe_log(ixgbe,
4706 "Add interrupt handler failed. "
4707 "return: %d, vector: %d", rc, vector);
4708 for (vector--; vector >= 0; vector--) {
4709 (void) ddi_intr_remove_handler(
4710 ixgbe->htable[vector]);
4711 }
4712 return (IXGBE_FAILURE);
4713 }
4714 }
4715
4716 break;
4717
4718 case DDI_INTR_TYPE_MSI:
4719 /*
4720 * Add interrupt handlers for the only vector
4721 */
4722 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4723 (ddi_intr_handler_t *)ixgbe_intr_msi,
4724 (void *)ixgbe, NULL);
4725
4726 if (rc != DDI_SUCCESS) {
4727 ixgbe_log(ixgbe,
4728 "Add MSI interrupt handler failed: %d", rc);
4729 return (IXGBE_FAILURE);
4730 }
4731
4732 break;
4733
4734 case DDI_INTR_TYPE_FIXED:
4735 /*
4736 * Add interrupt handlers for the only vector
4737 */
4738 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4739 (ddi_intr_handler_t *)ixgbe_intr_legacy,
4740 (void *)ixgbe, NULL);
4741
4742 if (rc != DDI_SUCCESS) {
4743 ixgbe_log(ixgbe,
4744 "Add legacy interrupt handler failed: %d", rc);
4745 return (IXGBE_FAILURE);
4746 }
4747
4748 break;
4749
4750 default:
4751 return (IXGBE_FAILURE);
4752 }
4753
4754 return (IXGBE_SUCCESS);
4755 }
4756
4757 #pragma inline(ixgbe_map_rxring_to_vector)
4758 /*
4759 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
4760 */
4761 static void
4762 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
4763 {
4764 /*
4765 * Set bit in map
4766 */
4767 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4768
4769 /*
4770 * Count bits set
4771 */
4772 ixgbe->vect_map[v_idx].rxr_cnt++;
4773
4774 /*
4775 * Remember bit position
4776 */
4777 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
4778 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
4779 }
4780
4781 #pragma inline(ixgbe_map_txring_to_vector)
4782 /*
4783 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
4784 */
4785 static void
4786 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
4787 {
4788 /*
4789 * Set bit in map
4790 */
4791 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
4792
4793 /*
4794 * Count bits set
4795 */
4796 ixgbe->vect_map[v_idx].txr_cnt++;
4797
4798 /*
4799 * Remember bit position
4800 */
4801 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
4802 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
4803 }
4804
4805 /*
4806 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
4807 * allocation register (IVAR).
4808 * cause:
4809 * -1 : other cause
4810 * 0 : rx
4811 * 1 : tx
4812 */
4813 static void
4814 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
4815 int8_t cause)
4816 {
4817 struct ixgbe_hw *hw = &ixgbe->hw;
4818 u32 ivar, index;
4819
4820 switch (hw->mac.type) {
4821 case ixgbe_mac_82598EB:
4822 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4823 if (cause == -1) {
4824 cause = 0;
4825 }
4826 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4827 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4828 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4829 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4830 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4831 break;
4832
4833 case ixgbe_mac_82599EB:
4834 case ixgbe_mac_X540:
4835 if (cause == -1) {
4836 /* other causes */
4837 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4838 index = (intr_alloc_entry & 1) * 8;
4839 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4840 ivar &= ~(0xFF << index);
4841 ivar |= (msix_vector << index);
4842 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4843 } else {
4844 /* tx or rx causes */
4845 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4846 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4847 ivar = IXGBE_READ_REG(hw,
4848 IXGBE_IVAR(intr_alloc_entry >> 1));
4849 ivar &= ~(0xFF << index);
4850 ivar |= (msix_vector << index);
4851 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4852 ivar);
4853 }
4854 break;
4855
4856 default:
4857 break;
4858 }
4859 }
4860
4861 /*
4862 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
4863 * given interrupt vector allocation register (IVAR).
4864 * cause:
4865 * -1 : other cause
4866 * 0 : rx
4867 * 1 : tx
4868 */
4869 static void
4870 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4871 {
4872 struct ixgbe_hw *hw = &ixgbe->hw;
4873 u32 ivar, index;
4874
4875 switch (hw->mac.type) {
4876 case ixgbe_mac_82598EB:
4877 if (cause == -1) {
4878 cause = 0;
4879 }
4880 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4881 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4882 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4883 (intr_alloc_entry & 0x3)));
4884 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4885 break;
4886
4887 case ixgbe_mac_82599EB:
4888 case ixgbe_mac_X540:
4889 if (cause == -1) {
4890 /* other causes */
4891 index = (intr_alloc_entry & 1) * 8;
4892 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4893 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4894 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4895 } else {
4896 /* tx or rx causes */
4897 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4898 ivar = IXGBE_READ_REG(hw,
4899 IXGBE_IVAR(intr_alloc_entry >> 1));
4900 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4901 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4902 ivar);
4903 }
4904 break;
4905
4906 default:
4907 break;
4908 }
4909 }
4910
4911 /*
4912 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
4913 * given interrupt vector allocation register (IVAR).
4914 * cause:
4915 * -1 : other cause
4916 * 0 : rx
4917 * 1 : tx
4918 */
4919 static void
4920 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4921 {
4922 struct ixgbe_hw *hw = &ixgbe->hw;
4923 u32 ivar, index;
4924
4925 switch (hw->mac.type) {
4926 case ixgbe_mac_82598EB:
4927 if (cause == -1) {
4928 cause = 0;
4929 }
4930 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4931 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4932 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
4933 (intr_alloc_entry & 0x3)));
4934 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4935 break;
4936
4937 case ixgbe_mac_82599EB:
4938 case ixgbe_mac_X540:
4939 if (cause == -1) {
4940 /* other causes */
4941 index = (intr_alloc_entry & 1) * 8;
4942 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4943 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4944 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4945 } else {
4946 /* tx or rx causes */
4947 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4948 ivar = IXGBE_READ_REG(hw,
4949 IXGBE_IVAR(intr_alloc_entry >> 1));
4950 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4951 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4952 ivar);
4953 }
4954 break;
4955
4956 default:
4957 break;
4958 }
4959 }
4960
4961 /*
4962 * Convert the rx ring index driver maintained to the rx ring index
4963 * in h/w.
4964 */
4965 static uint32_t
4966 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
4967 {
4968
4969 struct ixgbe_hw *hw = &ixgbe->hw;
4970 uint32_t rx_ring_per_group, hw_rx_index;
4971
4972 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
4973 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
4974 return (sw_rx_index);
4975 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
4976 switch (hw->mac.type) {
4977 case ixgbe_mac_82598EB:
4978 return (sw_rx_index);
4979
4980 case ixgbe_mac_82599EB:
4981 case ixgbe_mac_X540:
4982 return (sw_rx_index * 2);
4983
4984 default:
4985 break;
4986 }
4987 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
4988 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4989
4990 switch (hw->mac.type) {
4991 case ixgbe_mac_82598EB:
4992 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
4993 16 + (sw_rx_index % rx_ring_per_group);
4994 return (hw_rx_index);
4995
4996 case ixgbe_mac_82599EB:
4997 case ixgbe_mac_X540:
4998 if (ixgbe->num_rx_groups > 32) {
4999 hw_rx_index = (sw_rx_index /
5000 rx_ring_per_group) * 2 +
5001 (sw_rx_index % rx_ring_per_group);
5002 } else {
5003 hw_rx_index = (sw_rx_index /
5004 rx_ring_per_group) * 4 +
5005 (sw_rx_index % rx_ring_per_group);
5006 }
5007 return (hw_rx_index);
5008
5009 default:
5010 break;
5011 }
5012 }
5013
5014 /*
5015 * Should never reach. Just to make compiler happy.
5016 */
5017 return (sw_rx_index);
5018 }
5019
5020 /*
5021 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5022 *
5023 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5024 * to vector[0 - (intr_cnt -1)].
5025 */
5026 static int
5027 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5028 {
5029 int i, vector = 0;
5030
5031 /* initialize vector map */
5032 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5033 for (i = 0; i < ixgbe->intr_cnt; i++) {
5034 ixgbe->vect_map[i].ixgbe = ixgbe;
5035 }
5036
5037 /*
5038 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5039 * tx rings[0] on RTxQ[1].
5040 */
5041 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5042 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5043 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5044 return (IXGBE_SUCCESS);
5045 }
5046
5047 /*
5048 * Interrupts/vectors mapping for MSI-X
5049 */
5050
5051 /*
5052 * Map other interrupt to vector 0,
5053 * Set bit in map and count the bits set.
5054 */
5055 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5056 ixgbe->vect_map[vector].other_cnt++;
5057
5058 /*
5059 * Map rx ring interrupts to vectors
5060 */
5061 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5062 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5063 vector = (vector +1) % ixgbe->intr_cnt;
5064 }
5065
5066 /*
5067 * Map tx ring interrupts to vectors
5068 */
5069 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5070 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5071 vector = (vector +1) % ixgbe->intr_cnt;
5072 }
5073
5074 return (IXGBE_SUCCESS);
5075 }
5076
5077 /*
5078 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5079 *
5080 * This relies on ring/vector mapping already set up in the
5081 * vect_map[] structures
5082 */
5083 static void
5084 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5085 {
5086 struct ixgbe_hw *hw = &ixgbe->hw;
5087 ixgbe_intr_vector_t *vect; /* vector bitmap */
5088 int r_idx; /* ring index */
5089 int v_idx; /* vector index */
5090 uint32_t hw_index;
5091
5092 /*
5093 * Clear any previous entries
5094 */
5095 switch (hw->mac.type) {
5096 case ixgbe_mac_82598EB:
5097 for (v_idx = 0; v_idx < 25; v_idx++)
5098 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5099 break;
5100
5101 case ixgbe_mac_82599EB:
5102 case ixgbe_mac_X540:
5103 for (v_idx = 0; v_idx < 64; v_idx++)
5104 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5105 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5106 break;
5107
5108 default:
5109 break;
5110 }
5111
5112 /*
5113 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5114 * tx rings[0] will use RTxQ[1].
5115 */
5116 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5117 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5118 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5119 return;
5120 }
5121
5122 /*
5123 * For MSI-X interrupt, "Other" is always on vector[0].
5124 */
5125 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5126
5127 /*
5128 * For each interrupt vector, populate the IVAR table
5129 */
5130 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5131 vect = &ixgbe->vect_map[v_idx];
5132
5133 /*
5134 * For each rx ring bit set
5135 */
5136 r_idx = bt_getlowbit(vect->rx_map, 0,
5137 (ixgbe->num_rx_rings - 1));
5138
5139 while (r_idx >= 0) {
5140 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5141 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5142 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5143 (ixgbe->num_rx_rings - 1));
5144 }
5145
5146 /*
5147 * For each tx ring bit set
5148 */
5149 r_idx = bt_getlowbit(vect->tx_map, 0,
5150 (ixgbe->num_tx_rings - 1));
5151
5152 while (r_idx >= 0) {
5153 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5154 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5155 (ixgbe->num_tx_rings - 1));
5156 }
5157 }
5158 }
5159
5160 /*
5161 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5162 */
5163 static void
5164 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5165 {
5166 int i;
5167 int rc;
5168
5169 for (i = 0; i < ixgbe->intr_cnt; i++) {
5170 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5171 if (rc != DDI_SUCCESS) {
5172 IXGBE_DEBUGLOG_1(ixgbe,
5173 "Remove intr handler failed: %d", rc);
5174 }
5175 }
5176 }
5177
5178 /*
5179 * ixgbe_rem_intrs - Remove the allocated interrupts.
5180 */
5181 static void
5182 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5183 {
5184 int i;
5185 int rc;
5186
5187 for (i = 0; i < ixgbe->intr_cnt; i++) {
5188 rc = ddi_intr_free(ixgbe->htable[i]);
5189 if (rc != DDI_SUCCESS) {
5190 IXGBE_DEBUGLOG_1(ixgbe,
5191 "Free intr failed: %d", rc);
5192 }
5193 }
5194
5195 kmem_free(ixgbe->htable, ixgbe->intr_size);
5196 ixgbe->htable = NULL;
5197 }
5198
5199 /*
5200 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5201 */
5202 static int
5203 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5204 {
5205 int i;
5206 int rc;
5207
5208 /*
5209 * Enable interrupts
5210 */
5211 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5212 /*
5213 * Call ddi_intr_block_enable() for MSI
5214 */
5215 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5216 if (rc != DDI_SUCCESS) {
5217 ixgbe_log(ixgbe,
5218 "Enable block intr failed: %d", rc);
5219 return (IXGBE_FAILURE);
5220 }
5221 } else {
5222 /*
5223 * Call ddi_intr_enable() for Legacy/MSI non block enable
5224 */
5225 for (i = 0; i < ixgbe->intr_cnt; i++) {
5226 rc = ddi_intr_enable(ixgbe->htable[i]);
5227 if (rc != DDI_SUCCESS) {
5228 ixgbe_log(ixgbe,
5229 "Enable intr failed: %d", rc);
5230 return (IXGBE_FAILURE);
5231 }
5232 }
5233 }
5234
5235 return (IXGBE_SUCCESS);
5236 }
5237
5238 /*
5239 * ixgbe_disable_intrs - Disable all the interrupts.
5240 */
5241 static int
5242 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5243 {
5244 int i;
5245 int rc;
5246
5247 /*
5248 * Disable all interrupts
5249 */
5250 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5251 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5252 if (rc != DDI_SUCCESS) {
5253 ixgbe_log(ixgbe,
5254 "Disable block intr failed: %d", rc);
5255 return (IXGBE_FAILURE);
5256 }
5257 } else {
5258 for (i = 0; i < ixgbe->intr_cnt; i++) {
5259 rc = ddi_intr_disable(ixgbe->htable[i]);
5260 if (rc != DDI_SUCCESS) {
5261 ixgbe_log(ixgbe,
5262 "Disable intr failed: %d", rc);
5263 return (IXGBE_FAILURE);
5264 }
5265 }
5266 }
5267
5268 return (IXGBE_SUCCESS);
5269 }
5270
5271 /*
5272 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5273 */
5274 static void
5275 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5276 {
5277 struct ixgbe_hw *hw = &ixgbe->hw;
5278 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5279 boolean_t link_up = B_FALSE;
5280 uint32_t pcs1g_anlp = 0;
5281 uint32_t pcs1g_ana = 0;
5282 boolean_t autoneg = B_FALSE;
5283
5284 ASSERT(mutex_owned(&ixgbe->gen_lock));
5285 ixgbe->param_lp_1000fdx_cap = 0;
5286 ixgbe->param_lp_100fdx_cap = 0;
5287
5288 /* check for link, don't wait */
5289 (void) ixgbe_check_link(hw, &speed, &link_up, false);
5290 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5291
5292 if (link_up) {
5293 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5294
5295 ixgbe->param_lp_1000fdx_cap =
5296 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5297 ixgbe->param_lp_100fdx_cap =
5298 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5299 }
5300
5301 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5302
5303 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5304 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5305 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5306 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5307 }
5308
5309 /*
5310 * ixgbe_get_driver_control - Notify that driver is in control of device.
5311 */
5312 static void
5313 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5314 {
5315 uint32_t ctrl_ext;
5316
5317 /*
5318 * Notify firmware that driver is in control of device
5319 */
5320 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5321 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5322 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5323 }
5324
5325 /*
5326 * ixgbe_release_driver_control - Notify that driver is no longer in control
5327 * of device.
5328 */
5329 static void
5330 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5331 {
5332 uint32_t ctrl_ext;
5333
5334 /*
5335 * Notify firmware that driver is no longer in control of device
5336 */
5337 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5338 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5339 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5340 }
5341
5342 /*
5343 * ixgbe_atomic_reserve - Atomic decrease operation.
5344 */
5345 int
5346 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5347 {
5348 uint32_t oldval;
5349 uint32_t newval;
5350
5351 /*
5352 * ATOMICALLY
5353 */
5354 do {
5355 oldval = *count_p;
5356 if (oldval < n)
5357 return (-1);
5358 newval = oldval - n;
5359 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5360
5361 return (newval);
5362 }
5363
5364 /*
5365 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5366 */
5367 static uint8_t *
5368 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5369 {
5370 uint8_t *addr = *upd_ptr;
5371 uint8_t *new_ptr;
5372
5373 _NOTE(ARGUNUSED(hw));
5374 _NOTE(ARGUNUSED(vmdq));
5375
5376 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5377 *upd_ptr = new_ptr;
5378 return (addr);
5379 }
5380
5381 /*
5382 * FMA support
5383 */
5384 int
5385 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5386 {
5387 ddi_fm_error_t de;
5388
5389 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5390 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5391 return (de.fme_status);
5392 }
5393
5394 int
5395 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5396 {
5397 ddi_fm_error_t de;
5398
5399 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5400 return (de.fme_status);
5401 }
5402
5403 /*
5404 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5405 */
5406 static int
5407 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5408 {
5409 _NOTE(ARGUNUSED(impl_data));
5410 /*
5411 * as the driver can always deal with an error in any dma or
5412 * access handle, we can just return the fme_status value.
5413 */
5414 pci_ereport_post(dip, err, NULL);
5415 return (err->fme_status);
5416 }
5417
5418 static void
5419 ixgbe_fm_init(ixgbe_t *ixgbe)
5420 {
5421 ddi_iblock_cookie_t iblk;
5422 int fma_dma_flag;
5423
5424 /*
5425 * Only register with IO Fault Services if we have some capability
5426 */
5427 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5428 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5429 } else {
5430 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5431 }
5432
5433 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5434 fma_dma_flag = 1;
5435 } else {
5436 fma_dma_flag = 0;
5437 }
5438
5439 ixgbe_set_fma_flags(fma_dma_flag);
5440
5441 if (ixgbe->fm_capabilities) {
5442
5443 /*
5444 * Register capabilities with IO Fault Services
5445 */
5446 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5447
5448 /*
5449 * Initialize pci ereport capabilities if ereport capable
5450 */
5451 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5452 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5453 pci_ereport_setup(ixgbe->dip);
5454
5455 /*
5456 * Register error callback if error callback capable
5457 */
5458 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5459 ddi_fm_handler_register(ixgbe->dip,
5460 ixgbe_fm_error_cb, (void*) ixgbe);
5461 }
5462 }
5463
5464 static void
5465 ixgbe_fm_fini(ixgbe_t *ixgbe)
5466 {
5467 /*
5468 * Only unregister FMA capabilities if they are registered
5469 */
5470 if (ixgbe->fm_capabilities) {
5471
5472 /*
5473 * Release any resources allocated by pci_ereport_setup()
5474 */
5475 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5476 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5477 pci_ereport_teardown(ixgbe->dip);
5478
5479 /*
5480 * Un-register error callback if error callback capable
5481 */
5482 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5483 ddi_fm_handler_unregister(ixgbe->dip);
5484
5485 /*
5486 * Unregister from IO Fault Service
5487 */
5488 ddi_fm_fini(ixgbe->dip);
5489 }
5490 }
5491
5492 void
5493 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5494 {
5495 uint64_t ena;
5496 char buf[FM_MAX_CLASS];
5497
5498 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5499 ena = fm_ena_generate(0, FM_ENA_FMT1);
5500 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5501 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5502 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5503 }
5504 }
5505
5506 static int
5507 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5508 {
5509 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5510
5511 mutex_enter(&rx_ring->rx_lock);
5512 rx_ring->ring_gen_num = mr_gen_num;
5513 mutex_exit(&rx_ring->rx_lock);
5514 return (0);
5515 }
5516
5517 /*
5518 * Get the global ring index by a ring index within a group.
5519 */
5520 static int
5521 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5522 {
5523 ixgbe_rx_ring_t *rx_ring;
5524 int i;
5525
5526 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5527 rx_ring = &ixgbe->rx_rings[i];
5528 if (rx_ring->group_index == gindex)
5529 rindex--;
5530 if (rindex < 0)
5531 return (i);
5532 }
5533
5534 return (-1);
5535 }
5536
5537 /*
5538 * Callback funtion for MAC layer to register all rings.
5539 */
5540 /* ARGSUSED */
5541 void
5542 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5543 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5544 {
5545 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5546 mac_intr_t *mintr = &infop->mri_intr;
5547
5548 switch (rtype) {
5549 case MAC_RING_TYPE_RX: {
5550 /*
5551 * 'index' is the ring index within the group.
5552 * Need to get the global ring index by searching in groups.
5553 */
5554 int global_ring_index = ixgbe_get_rx_ring_index(
5555 ixgbe, group_index, ring_index);
5556
5557 ASSERT(global_ring_index >= 0);
5558
5559 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5560 rx_ring->ring_handle = rh;
5561
5562 infop->mri_driver = (mac_ring_driver_t)rx_ring;
5563 infop->mri_start = ixgbe_ring_start;
5564 infop->mri_stop = NULL;
5565 infop->mri_poll = ixgbe_ring_rx_poll;
5566 infop->mri_stat = ixgbe_rx_ring_stat;
5567
5568 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
5569 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
5570 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
5571 if (ixgbe->intr_type &
5572 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5573 mintr->mi_ddi_handle =
5574 ixgbe->htable[rx_ring->intr_vector];
5575 }
5576
5577 break;
5578 }
5579 case MAC_RING_TYPE_TX: {
5580 ASSERT(group_index == -1);
5581 ASSERT(ring_index < ixgbe->num_tx_rings);
5582
5583 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
5584 tx_ring->ring_handle = rh;
5585
5586 infop->mri_driver = (mac_ring_driver_t)tx_ring;
5587 infop->mri_start = NULL;
5588 infop->mri_stop = NULL;
5589 infop->mri_tx = ixgbe_ring_tx;
5590 infop->mri_stat = ixgbe_tx_ring_stat;
5591 if (ixgbe->intr_type &
5592 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5593 mintr->mi_ddi_handle =
5594 ixgbe->htable[tx_ring->intr_vector];
5595 }
5596 break;
5597 }
5598 default:
5599 break;
5600 }
5601 }
5602
5603 /*
5604 * Callback funtion for MAC layer to register all groups.
5605 */
5606 void
5607 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
5608 mac_group_info_t *infop, mac_group_handle_t gh)
5609 {
5610 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5611
5612 switch (rtype) {
5613 case MAC_RING_TYPE_RX: {
5614 ixgbe_rx_group_t *rx_group;
5615
5616 rx_group = &ixgbe->rx_groups[index];
5617 rx_group->group_handle = gh;
5618
5619 infop->mgi_driver = (mac_group_driver_t)rx_group;
5620 infop->mgi_start = NULL;
5621 infop->mgi_stop = NULL;
5622 infop->mgi_addmac = ixgbe_addmac;
5623 infop->mgi_remmac = ixgbe_remmac;
5624 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
5625
5626 break;
5627 }
5628 case MAC_RING_TYPE_TX:
5629 break;
5630 default:
5631 break;
5632 }
5633 }
5634
5635 /*
5636 * Enable interrupt on the specificed rx ring.
5637 */
5638 int
5639 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
5640 {
5641 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5642 ixgbe_t *ixgbe = rx_ring->ixgbe;
5643 int r_idx = rx_ring->index;
5644 int hw_r_idx = rx_ring->hw_index;
5645 int v_idx = rx_ring->intr_vector;
5646
5647 mutex_enter(&ixgbe->gen_lock);
5648 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5649 mutex_exit(&ixgbe->gen_lock);
5650 /*
5651 * Simply return 0.
5652 * Interrupts are being adjusted. ixgbe_intr_adjust()
5653 * will eventually re-enable the interrupt when it's
5654 * done with the adjustment.
5655 */
5656 return (0);
5657 }
5658
5659 /*
5660 * To enable interrupt by setting the VAL bit of given interrupt
5661 * vector allocation register (IVAR).
5662 */
5663 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
5664
5665 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5666
5667 /*
5668 * Trigger a Rx interrupt on this ring
5669 */
5670 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
5671 IXGBE_WRITE_FLUSH(&ixgbe->hw);
5672
5673 mutex_exit(&ixgbe->gen_lock);
5674
5675 return (0);
5676 }
5677
5678 /*
5679 * Disable interrupt on the specificed rx ring.
5680 */
5681 int
5682 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
5683 {
5684 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5685 ixgbe_t *ixgbe = rx_ring->ixgbe;
5686 int r_idx = rx_ring->index;
5687 int hw_r_idx = rx_ring->hw_index;
5688 int v_idx = rx_ring->intr_vector;
5689
5690 mutex_enter(&ixgbe->gen_lock);
5691 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5692 mutex_exit(&ixgbe->gen_lock);
5693 /*
5694 * Simply return 0.
5695 * In the rare case where an interrupt is being
5696 * disabled while interrupts are being adjusted,
5697 * we don't fail the operation. No interrupts will
5698 * be generated while they are adjusted, and
5699 * ixgbe_intr_adjust() will cause the interrupts
5700 * to be re-enabled once it completes. Note that
5701 * in this case, packets may be delivered to the
5702 * stack via interrupts before xgbe_rx_ring_intr_enable()
5703 * is called again. This is acceptable since interrupt
5704 * adjustment is infrequent, and the stack will be
5705 * able to handle these packets.
5706 */
5707 return (0);
5708 }
5709
5710 /*
5711 * To disable interrupt by clearing the VAL bit of given interrupt
5712 * vector allocation register (IVAR).
5713 */
5714 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
5715
5716 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
5717
5718 mutex_exit(&ixgbe->gen_lock);
5719
5720 return (0);
5721 }
5722
5723 /*
5724 * Add a mac address.
5725 */
5726 static int
5727 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
5728 {
5729 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5730 ixgbe_t *ixgbe = rx_group->ixgbe;
5731 struct ixgbe_hw *hw = &ixgbe->hw;
5732 int slot, i;
5733
5734 mutex_enter(&ixgbe->gen_lock);
5735
5736 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5737 mutex_exit(&ixgbe->gen_lock);
5738 return (ECANCELED);
5739 }
5740
5741 if (ixgbe->unicst_avail == 0) {
5742 /* no slots available */
5743 mutex_exit(&ixgbe->gen_lock);
5744 return (ENOSPC);
5745 }
5746
5747 /*
5748 * The first ixgbe->num_rx_groups slots are reserved for each respective
5749 * group. The rest slots are shared by all groups. While adding a
5750 * MAC address, reserved slots are firstly checked then the shared
5751 * slots are searched.
5752 */
5753 slot = -1;
5754 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
5755 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
5756 if (ixgbe->unicst_addr[i].mac.set == 0) {
5757 slot = i;
5758 break;
5759 }
5760 }
5761 } else {
5762 slot = rx_group->index;
5763 }
5764
5765 if (slot == -1) {
5766 /* no slots available */
5767 mutex_exit(&ixgbe->gen_lock);
5768 return (ENOSPC);
5769 }
5770
5771 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5772 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
5773 rx_group->index, IXGBE_RAH_AV);
5774 ixgbe->unicst_addr[slot].mac.set = 1;
5775 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
5776 ixgbe->unicst_avail--;
5777
5778 mutex_exit(&ixgbe->gen_lock);
5779
5780 return (0);
5781 }
5782
5783 /*
5784 * Remove a mac address.
5785 */
5786 static int
5787 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
5788 {
5789 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5790 ixgbe_t *ixgbe = rx_group->ixgbe;
5791 struct ixgbe_hw *hw = &ixgbe->hw;
5792 int slot;
5793
5794 mutex_enter(&ixgbe->gen_lock);
5795
5796 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5797 mutex_exit(&ixgbe->gen_lock);
5798 return (ECANCELED);
5799 }
5800
5801 slot = ixgbe_unicst_find(ixgbe, mac_addr);
5802 if (slot == -1) {
5803 mutex_exit(&ixgbe->gen_lock);
5804 return (EINVAL);
5805 }
5806
5807 if (ixgbe->unicst_addr[slot].mac.set == 0) {
5808 mutex_exit(&ixgbe->gen_lock);
5809 return (EINVAL);
5810 }
5811
5812 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5813 (void) ixgbe_clear_rar(hw, slot);
5814 ixgbe->unicst_addr[slot].mac.set = 0;
5815 ixgbe->unicst_avail++;
5816
5817 mutex_exit(&ixgbe->gen_lock);
5818
5819 return (0);
5820 }