1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/modctl.h>
27 #include <sys/conf.h>
28 #include <sys/devops.h>
29 #include <sys/pci.h>
30 #include <sys/atomic.h>
31 #include <sys/types.h>
32 #include <sys/file.h>
33 #include <sys/errno.h>
34 #include <sys/open.h>
35 #include <sys/cred.h>
36 #include <sys/stat.h>
37 #include <sys/mkdev.h>
38 #include <sys/queue.h>
39
40 #include "version.h"
41 #include "common/common.h"
42 #include "common/t4_msg.h"
43 #include "common/t4_regs.h"
44 #include "firmware/t4_fw.h"
45 #include "firmware/t4_cfg.h"
46 #include "firmware/t5_fw.h"
47 #include "firmware/t5_cfg.h"
48 #include "firmware/t6_fw.h"
49 #include "firmware/t6_cfg.h"
50 #include "t4_l2t.h"
51
52 static int t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp);
53 static int t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp);
54 static int t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp,
55 int *rp);
56 struct cb_ops t4_cb_ops = {
57 .cb_open = t4_cb_open,
58 .cb_close = t4_cb_close,
59 .cb_strategy = nodev,
60 .cb_print = nodev,
61 .cb_dump = nodev,
62 .cb_read = nodev,
63 .cb_write = nodev,
64 .cb_ioctl = t4_cb_ioctl,
65 .cb_devmap = nodev,
66 .cb_mmap = nodev,
67 .cb_segmap = nodev,
68 .cb_chpoll = nochpoll,
69 .cb_prop_op = ddi_prop_op,
70 .cb_flag = D_MP,
71 .cb_rev = CB_REV,
72 .cb_aread = nodev,
73 .cb_awrite = nodev
74 };
75
76 static int t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op,
77 void *arg, void *result);
78 static int t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
79 void *arg, dev_info_t **cdipp);
80 static int t4_bus_unconfig(dev_info_t *dip, uint_t flags,
81 ddi_bus_config_op_t op, void *arg);
82 struct bus_ops t4_bus_ops = {
83 .busops_rev = BUSO_REV,
84 .bus_ctl = t4_bus_ctl,
85 .bus_prop_op = ddi_bus_prop_op,
86 .bus_config = t4_bus_config,
87 .bus_unconfig = t4_bus_unconfig,
88 };
89
90 static int t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
91 void **rp);
92 static int t4_devo_probe(dev_info_t *dip);
93 static int t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
94 static int t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
95 static int t4_devo_quiesce(dev_info_t *dip);
96 struct dev_ops t4_dev_ops = {
97 .devo_rev = DEVO_REV,
98 .devo_getinfo = t4_devo_getinfo,
99 .devo_identify = nulldev,
100 .devo_probe = t4_devo_probe,
101 .devo_attach = t4_devo_attach,
102 .devo_detach = t4_devo_detach,
103 .devo_reset = nodev,
104 .devo_cb_ops = &t4_cb_ops,
105 .devo_bus_ops = &t4_bus_ops,
106 .devo_quiesce = &t4_devo_quiesce,
107 };
108
109 static struct modldrv modldrv = {
110 .drv_modops = &mod_driverops,
111 .drv_linkinfo = "Chelsio T4 nexus " DRV_VERSION,
112 .drv_dev_ops = &t4_dev_ops
113 };
114
115 static struct modlinkage modlinkage = {
116 .ml_rev = MODREV_1,
117 .ml_linkage = {&modldrv, NULL},
118 };
119
120 void *t4_list;
121
122 struct intrs_and_queues {
123 int intr_type; /* DDI_INTR_TYPE_* */
124 int nirq; /* Number of vectors */
125 int intr_fwd; /* Interrupts forwarded */
126 int ntxq10g; /* # of NIC txq's for each 10G port */
127 int nrxq10g; /* # of NIC rxq's for each 10G port */
128 int ntxq1g; /* # of NIC txq's for each 1G port */
129 int nrxq1g; /* # of NIC rxq's for each 1G port */
130 #ifdef TCP_OFFLOAD_ENABLE
131 int nofldtxq10g; /* # of TOE txq's for each 10G port */
132 int nofldrxq10g; /* # of TOE rxq's for each 10G port */
133 int nofldtxq1g; /* # of TOE txq's for each 1G port */
134 int nofldrxq1g; /* # of TOE rxq's for each 1G port */
135 #endif
136 };
137
138 struct fw_info fi[3];
139
140 static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss,
141 mblk_t *m);
142 static int fw_msg_not_handled(struct adapter *, const __be64 *);
143 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
144 static unsigned int getpf(struct adapter *sc);
145 static int prep_firmware(struct adapter *sc);
146 static int upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma);
147 static int partition_resources(struct adapter *sc);
148 static int adap__pre_init_tweaks(struct adapter *sc);
149 static int get_params__pre_init(struct adapter *sc);
150 static int get_params__post_init(struct adapter *sc);
151 static int set_params__post_init(struct adapter *);
152 static void setup_memwin(struct adapter *sc);
153 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
154 uint32_t *);
155 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
156 uint32_t position_memwin(struct adapter *, int, uint32_t);
157 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
158 uint_t count);
159 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
160 uint_t count);
161 static int init_driver_props(struct adapter *sc, struct driver_properties *p);
162 static int remove_extra_props(struct adapter *sc, int n10g, int n1g);
163 static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
164 struct intrs_and_queues *iaq);
165 static int add_child_node(struct adapter *sc, int idx);
166 static int remove_child_node(struct adapter *sc, int idx);
167 static kstat_t *setup_kstats(struct adapter *sc);
168 static kstat_t *setup_wc_kstats(struct adapter *);
169 static int update_wc_kstats(kstat_t *, int);
170 #ifdef TCP_OFFLOAD_ENABLE
171 static int toe_capability(struct port_info *pi, int enable);
172 static int activate_uld(struct adapter *sc, int id, struct uld_softc *usc);
173 static int deactivate_uld(struct uld_softc *usc);
174 #endif
175 static kmutex_t t4_adapter_list_lock;
176 static SLIST_HEAD(, adapter) t4_adapter_list;
177 #ifdef TCP_OFFLOAD_ENABLE
178 static kmutex_t t4_uld_list_lock;
179 static SLIST_HEAD(, uld_info) t4_uld_list;
180 #endif
181
182 int
183 _init(void)
184 {
185 int rc;
186
187 rc = ddi_soft_state_init(&t4_list, sizeof (struct adapter), 0);
188 if (rc != 0)
189 return (rc);
190
191 rc = mod_install(&modlinkage);
192 if (rc != 0)
193 ddi_soft_state_fini(&t4_list);
194
195 mutex_init(&t4_adapter_list_lock, NULL, MUTEX_DRIVER, NULL);
196 SLIST_INIT(&t4_adapter_list);
197
198 #ifdef TCP_OFFLOAD_ENABLE
199 mutex_init(&t4_uld_list_lock, NULL, MUTEX_DRIVER, NULL);
200 SLIST_INIT(&t4_uld_list);
201 #endif
202
203 return (rc);
204 }
205
206 int
207 _fini(void)
208 {
209 int rc;
210
211 rc = mod_remove(&modlinkage);
212 if (rc != 0)
213 return (rc);
214
215 ddi_soft_state_fini(&t4_list);
216 return (0);
217 }
218
219 int
220 _info(struct modinfo *mi)
221 {
222 return (mod_info(&modlinkage, mi));
223 }
224
225 /* ARGSUSED */
226 static int
227 t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp)
228 {
229 struct adapter *sc;
230 minor_t minor;
231
232 minor = getminor((dev_t)arg); /* same as instance# in our case */
233
234 if (cmd == DDI_INFO_DEVT2DEVINFO) {
235 sc = ddi_get_soft_state(t4_list, minor);
236 if (sc == NULL)
237 return (DDI_FAILURE);
238
239 ASSERT(sc->dev == (dev_t)arg);
240 *rp = (void *)sc->dip;
241 } else if (cmd == DDI_INFO_DEVT2INSTANCE)
242 *rp = (void *) (unsigned long) minor;
243 else
244 ASSERT(0);
245
246 return (DDI_SUCCESS);
247 }
248
249 static int
250 t4_devo_probe(dev_info_t *dip)
251 {
252 int rc, id, *reg;
253 uint_t n, pf;
254
255 id = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
256 "device-id", 0xffff);
257 if (id == 0xffff)
258 return (DDI_PROBE_DONTCARE);
259
260 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
261 "reg", ®, &n);
262 if (rc != DDI_SUCCESS)
263 return (DDI_PROBE_DONTCARE);
264
265 pf = PCI_REG_FUNC_G(reg[0]);
266 ddi_prop_free(reg);
267
268 /* Prevent driver attachment on any PF except 0 on the FPGA */
269 if (id == 0xa000 && pf != 0)
270 return (DDI_PROBE_FAILURE);
271
272 return (DDI_PROBE_DONTCARE);
273 }
274
275 static int
276 t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
277 {
278 struct adapter *sc = NULL;
279 struct sge *s;
280 int i, instance, rc = DDI_SUCCESS, rqidx, tqidx, q;
281 int irq = 0, nxg, n100g, n40g, n25g, n10g, n1g;
282 #ifdef TCP_OFFLOAD_ENABLE
283 int ofld_rqidx, ofld_tqidx;
284 #endif
285 char name[16];
286 struct driver_properties *prp;
287 struct intrs_and_queues iaq;
288 ddi_device_acc_attr_t da = {
289 .devacc_attr_version = DDI_DEVICE_ATTR_V0,
290 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
291 .devacc_attr_dataorder = DDI_UNORDERED_OK_ACC
292 };
293 ddi_device_acc_attr_t da1 = {
294 .devacc_attr_version = DDI_DEVICE_ATTR_V0,
295 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
296 .devacc_attr_dataorder = DDI_MERGING_OK_ACC
297 };
298
299 if (cmd != DDI_ATTACH)
300 return (DDI_FAILURE);
301
302 /*
303 * Allocate space for soft state.
304 */
305 instance = ddi_get_instance(dip);
306 rc = ddi_soft_state_zalloc(t4_list, instance);
307 if (rc != DDI_SUCCESS) {
308 cxgb_printf(dip, CE_WARN,
309 "failed to allocate soft state: %d", rc);
310 return (DDI_FAILURE);
311 }
312
313 sc = ddi_get_soft_state(t4_list, instance);
314 sc->dip = dip;
315 sc->dev = makedevice(ddi_driver_major(dip), instance);
316 mutex_init(&sc->lock, NULL, MUTEX_DRIVER, NULL);
317 cv_init(&sc->cv, NULL, CV_DRIVER, NULL);
318 mutex_init(&sc->sfl_lock, NULL, MUTEX_DRIVER, NULL);
319
320 mutex_enter(&t4_adapter_list_lock);
321 SLIST_INSERT_HEAD(&t4_adapter_list, sc, link);
322 mutex_exit(&t4_adapter_list_lock);
323
324 sc->pf = getpf(sc);
325 if (sc->pf > 8) {
326 rc = EINVAL;
327 cxgb_printf(dip, CE_WARN,
328 "failed to determine PCI PF# of device");
329 goto done;
330 }
331 sc->mbox = sc->pf;
332
333 /* Initialize the driver properties */
334 prp = &sc->props;
335 (void)init_driver_props(sc, prp);
336
337 /*
338 * Enable access to the PCI config space.
339 */
340 rc = pci_config_setup(dip, &sc->pci_regh);
341 if (rc != DDI_SUCCESS) {
342 cxgb_printf(dip, CE_WARN,
343 "failed to enable PCI config space access: %d", rc);
344 goto done;
345 }
346
347 /* TODO: Set max read request to 4K */
348
349 /*
350 * Enable MMIO access.
351 */
352 rc = ddi_regs_map_setup(dip, 1, &sc->regp, 0, 0, &da, &sc->regh);
353 if (rc != DDI_SUCCESS) {
354 cxgb_printf(dip, CE_WARN,
355 "failed to map device registers: %d", rc);
356 goto done;
357 }
358
359 (void) memset(sc->chan_map, 0xff, sizeof (sc->chan_map));
360
361 /*
362 * Initialize cpl handler.
363 */
364 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) {
365 sc->cpl_handler[i] = cpl_not_handled;
366 }
367
368 for (i = 0; i < ARRAY_SIZE(sc->fw_msg_handler); i++) {
369 sc->fw_msg_handler[i] = fw_msg_not_handled;
370 }
371
372 /*
373 * Prepare the adapter for operation.
374 */
375 rc = -t4_prep_adapter(sc, false);
376 if (rc != 0) {
377 cxgb_printf(dip, CE_WARN, "failed to prepare adapter: %d", rc);
378 goto done;
379 }
380
381 /*
382 * Enable BAR1 access.
383 */
384 sc->doorbells |= DOORBELL_KDB;
385 rc = ddi_regs_map_setup(dip, 2, &sc->reg1p, 0, 0, &da1, &sc->reg1h);
386 if (rc != DDI_SUCCESS) {
387 cxgb_printf(dip, CE_WARN,
388 "failed to map BAR1 device registers: %d", rc);
389 goto done;
390 } else {
391 if (is_t5(sc->params.chip)) {
392 sc->doorbells |= DOORBELL_UDB;
393 if (prp->wc) {
394 /*
395 * Enable write combining on BAR2. This is the
396 * userspace doorbell BAR and is split into 128B
397 * (UDBS_SEG_SIZE) doorbell regions, each associated
398 * with an egress queue. The first 64B has the doorbell
399 * and the second 64B can be used to submit a tx work
400 * request with an implicit doorbell.
401 */
402 sc->doorbells &= ~DOORBELL_UDB;
403 sc->doorbells |= (DOORBELL_WCWR |
404 DOORBELL_UDBWC);
405 t4_write_reg(sc, A_SGE_STAT_CFG,
406 V_STATSOURCE_T5(7) | V_STATMODE(0));
407 }
408 }
409 }
410
411 /*
412 * Do this really early. Note that minor number = instance.
413 */
414 (void) snprintf(name, sizeof (name), "%s,%d", T4_NEXUS_NAME, instance);
415 rc = ddi_create_minor_node(dip, name, S_IFCHR, instance,
416 DDI_NT_NEXUS, 0);
417 if (rc != DDI_SUCCESS) {
418 cxgb_printf(dip, CE_WARN,
419 "failed to create device node: %d", rc);
420 rc = DDI_SUCCESS; /* carry on */
421 }
422
423 /* Do this early. Memory window is required for loading config file. */
424 setup_memwin(sc);
425
426 /* Prepare the firmware for operation */
427 rc = prep_firmware(sc);
428 if (rc != 0)
429 goto done; /* error message displayed already */
430
431 rc = adap__pre_init_tweaks(sc);
432 if (rc != 0)
433 goto done;
434
435 rc = get_params__pre_init(sc);
436 if (rc != 0)
437 goto done; /* error message displayed already */
438
439 t4_sge_init(sc);
440
441 if (sc->flags & MASTER_PF) {
442 /* get basic stuff going */
443 rc = -t4_fw_initialize(sc, sc->mbox);
444 if (rc != 0) {
445 cxgb_printf(sc->dip, CE_WARN,
446 "early init failed: %d.\n", rc);
447 goto done;
448 }
449 }
450
451 rc = get_params__post_init(sc);
452 if (rc != 0)
453 goto done; /* error message displayed already */
454
455 rc = set_params__post_init(sc);
456 if (rc != 0)
457 goto done; /* error message displayed already */
458
459 /*
460 * TODO: This is the place to call t4_set_filter_mode()
461 */
462
463 /* tweak some settings */
464 t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) |
465 V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) |
466 V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9));
467 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
468
469 /*
470 * Work-around for bug 2619
471 * Set DisableVlan field in TP_RSS_CONFIG_VRT register so that the
472 * VLAN tag extraction is disabled.
473 */
474 t4_set_reg_field(sc, A_TP_RSS_CONFIG_VRT, F_DISABLEVLAN, F_DISABLEVLAN);
475
476 /* Store filter mode */
477 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
478 A_TP_VLAN_PRI_MAP);
479
480 /*
481 * First pass over all the ports - allocate VIs and initialize some
482 * basic parameters like mac address, port type, etc. We also figure
483 * out whether a port is 10G or 1G and use that information when
484 * calculating how many interrupts to attempt to allocate.
485 */
486 n100g = n40g = n25g = n10g = n1g = 0;
487 for_each_port(sc, i) {
488 struct port_info *pi;
489
490 pi = kmem_zalloc(sizeof (*pi), KM_SLEEP);
491 sc->port[i] = pi;
492
493 /* These must be set before t4_port_init */
494 pi->adapter = sc;
495 /* LINTED: E_ASSIGN_NARROW_CONV */
496 pi->port_id = i;
497 }
498
499 /* Allocate the vi and initialize parameters like mac addr */
500 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0);
501 if (rc) {
502 cxgb_printf(dip, CE_WARN,
503 "unable to initialize port: %d", rc);
504 goto done;
505 }
506
507 for_each_port(sc, i) {
508 struct port_info *pi = sc->port[i];
509
510 mutex_init(&pi->lock, NULL, MUTEX_DRIVER, NULL);
511 pi->mtu = ETHERMTU;
512
513 if (is_100G_port(pi)) {
514 n100g++;
515 pi->tmr_idx = prp->tmr_idx_10g;
516 pi->pktc_idx = prp->pktc_idx_10g;
517 } else if (is_40G_port(pi)) {
518 n40g++;
519 pi->tmr_idx = prp->tmr_idx_10g;
520 pi->pktc_idx = prp->pktc_idx_10g;
521 } else if (is_25G_port(pi)) {
522 n25g++;
523 pi->tmr_idx = prp->tmr_idx_10g;
524 pi->pktc_idx = prp->pktc_idx_10g;
525 } else if (is_10G_port(pi)) {
526 n10g++;
527 pi->tmr_idx = prp->tmr_idx_10g;
528 pi->pktc_idx = prp->pktc_idx_10g;
529 } else {
530 n1g++;
531 pi->tmr_idx = prp->tmr_idx_1g;
532 pi->pktc_idx = prp->pktc_idx_1g;
533 }
534
535 pi->xact_addr_filt = -1;
536 t4_mc_init(pi);
537
538 setbit(&sc->registered_device_map, i);
539 }
540
541 nxg = n10g + n25g + n40g + n100g;
542 (void) remove_extra_props(sc, nxg, n1g);
543
544 if (sc->registered_device_map == 0) {
545 cxgb_printf(dip, CE_WARN, "no usable ports");
546 rc = DDI_FAILURE;
547 goto done;
548 }
549
550 rc = cfg_itype_and_nqueues(sc, nxg, n1g, &iaq);
551 if (rc != 0)
552 goto done; /* error message displayed already */
553
554 sc->intr_type = iaq.intr_type;
555 sc->intr_count = iaq.nirq;
556
557 if (sc->props.multi_rings && (sc->intr_type != DDI_INTR_TYPE_MSIX)) {
558 sc->props.multi_rings = 0;
559 cxgb_printf(dip, CE_WARN,
560 "Multiple rings disabled as interrupt type is not MSI-X");
561 }
562
563 if (sc->props.multi_rings && iaq.intr_fwd) {
564 sc->props.multi_rings = 0;
565 cxgb_printf(dip, CE_WARN,
566 "Multiple rings disabled as interrupts are forwarded");
567 }
568
569 if (!sc->props.multi_rings) {
570 iaq.ntxq10g = 1;
571 iaq.ntxq1g = 1;
572 }
573 s = &sc->sge;
574 s->nrxq = nxg * iaq.nrxq10g + n1g * iaq.nrxq1g;
575 s->ntxq = nxg * iaq.ntxq10g + n1g * iaq.ntxq1g;
576 s->neq = s->ntxq + s->nrxq; /* the fl in an rxq is an eq */
577 #ifdef TCP_OFFLOAD_ENABLE
578 /* control queues, 1 per port + 1 mgmtq */
579 s->neq += sc->params.nports + 1;
580 #endif
581 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
582 if (iaq.intr_fwd != 0)
583 sc->flags |= INTR_FWD;
584 #ifdef TCP_OFFLOAD_ENABLE
585 if (is_offload(sc) != 0) {
586
587 s->nofldrxq = nxg * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
588 s->nofldtxq = nxg * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
589 s->neq += s->nofldtxq + s->nofldrxq;
590 s->niq += s->nofldrxq;
591
592 s->ofld_rxq = kmem_zalloc(s->nofldrxq *
593 sizeof (struct sge_ofld_rxq), KM_SLEEP);
594 s->ofld_txq = kmem_zalloc(s->nofldtxq *
595 sizeof (struct sge_wrq), KM_SLEEP);
596 s->ctrlq = kmem_zalloc(sc->params.nports *
597 sizeof (struct sge_wrq), KM_SLEEP);
598
599 }
600 #endif
601 s->rxq = kmem_zalloc(s->nrxq * sizeof (struct sge_rxq), KM_SLEEP);
602 s->txq = kmem_zalloc(s->ntxq * sizeof (struct sge_txq), KM_SLEEP);
603 s->iqmap = kmem_zalloc(s->niq * sizeof (struct sge_iq *), KM_SLEEP);
604 s->eqmap = kmem_zalloc(s->neq * sizeof (struct sge_eq *), KM_SLEEP);
605
606 sc->intr_handle = kmem_zalloc(sc->intr_count *
607 sizeof (ddi_intr_handle_t), KM_SLEEP);
608
609 /*
610 * Second pass over the ports. This time we know the number of rx and
611 * tx queues that each port should get.
612 */
613 rqidx = tqidx = 0;
614 #ifdef TCP_OFFLOAD_ENABLE
615 ofld_rqidx = ofld_tqidx = 0;
616 #endif
617 for_each_port(sc, i) {
618 struct port_info *pi = sc->port[i];
619
620 if (pi == NULL)
621 continue;
622
623 t4_mc_cb_init(pi);
624 /* LINTED: E_ASSIGN_NARROW_CONV */
625 pi->first_rxq = rqidx;
626 /* LINTED: E_ASSIGN_NARROW_CONV */
627 pi->nrxq = (is_10XG_port(pi)) ? iaq.nrxq10g
628 : iaq.nrxq1g;
629 /* LINTED: E_ASSIGN_NARROW_CONV */
630 pi->first_txq = tqidx;
631 /* LINTED: E_ASSIGN_NARROW_CONV */
632 pi->ntxq = (is_10XG_port(pi)) ? iaq.ntxq10g
633 : iaq.ntxq1g;
634
635 rqidx += pi->nrxq;
636 tqidx += pi->ntxq;
637
638 #ifdef TCP_OFFLOAD_ENABLE
639 if (is_offload(sc) != 0) {
640 /* LINTED: E_ASSIGN_NARROW_CONV */
641 pi->first_ofld_rxq = ofld_rqidx;
642 pi->nofldrxq = max(1, pi->nrxq / 4);
643
644 /* LINTED: E_ASSIGN_NARROW_CONV */
645 pi->first_ofld_txq = ofld_tqidx;
646 pi->nofldtxq = max(1, pi->ntxq / 2);
647
648 ofld_rqidx += pi->nofldrxq;
649 ofld_tqidx += pi->nofldtxq;
650 }
651 #endif
652
653 /*
654 * Enable hw checksumming and LSO for all ports by default.
655 * They can be disabled using ndd (hw_csum and hw_lso).
656 */
657 pi->features |= (CXGBE_HW_CSUM | CXGBE_HW_LSO);
658 }
659
660 #ifdef TCP_OFFLOAD_ENABLE
661 sc->l2t = t4_init_l2t(sc);
662 #endif
663
664 /*
665 * Setup Interrupts.
666 */
667
668 i = 0;
669 rc = ddi_intr_alloc(dip, sc->intr_handle, sc->intr_type, 0,
670 sc->intr_count, &i, DDI_INTR_ALLOC_STRICT);
671 if (rc != DDI_SUCCESS) {
672 cxgb_printf(dip, CE_WARN,
673 "failed to allocate %d interrupt(s) of type %d: %d, %d",
674 sc->intr_count, sc->intr_type, rc, i);
675 goto done;
676 }
677 ASSERT(sc->intr_count == i); /* allocation was STRICT */
678 (void) ddi_intr_get_cap(sc->intr_handle[0], &sc->intr_cap);
679 (void) ddi_intr_get_pri(sc->intr_handle[0], &sc->intr_pri);
680 if (sc->intr_count == 1) {
681 ASSERT(sc->flags & INTR_FWD);
682 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_all, sc,
683 &s->fwq);
684 } else {
685 /* Multiple interrupts. The first one is always error intr */
686 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_err, sc,
687 NULL);
688 irq++;
689
690 /* The second one is always the firmware event queue */
691 (void) ddi_intr_add_handler(sc->intr_handle[1], t4_intr, sc,
692 &s->fwq);
693 irq++;
694 /*
695 * Note that if INTR_FWD is set then either the NIC rx
696 * queues or (exclusive or) the TOE rx queueus will be taking
697 * direct interrupts.
698 *
699 * There is no need to check for is_offload(sc) as nofldrxq
700 * will be 0 if offload is disabled.
701 */
702 for_each_port(sc, i) {
703 struct port_info *pi = sc->port[i];
704 struct sge_rxq *rxq;
705 #ifdef TCP_OFFLOAD_ENABLE
706 struct sge_ofld_rxq *ofld_rxq;
707
708 /*
709 * Skip over the NIC queues if they aren't taking direct
710 * interrupts.
711 */
712 if ((sc->flags & INTR_FWD) &&
713 pi->nofldrxq > pi->nrxq)
714 goto ofld_queues;
715 #endif
716 rxq = &s->rxq[pi->first_rxq];
717 for (q = 0; q < pi->nrxq; q++, rxq++) {
718 (void) ddi_intr_add_handler(
719 sc->intr_handle[irq], t4_intr, sc,
720 &rxq->iq);
721 irq++;
722 }
723
724 #ifdef TCP_OFFLOAD_ENABLE
725 /*
726 * Skip over the offload queues if they aren't taking
727 * direct interrupts.
728 */
729 if ((sc->flags & INTR_FWD))
730 continue;
731 ofld_queues:
732 ofld_rxq = &s->ofld_rxq[pi->first_ofld_rxq];
733 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
734 (void) ddi_intr_add_handler(
735 sc->intr_handle[irq], t4_intr, sc,
736 &ofld_rxq->iq);
737 irq++;
738 }
739 #endif
740 }
741
742 }
743 sc->flags |= INTR_ALLOCATED;
744
745 ASSERT(rc == DDI_SUCCESS);
746 ddi_report_dev(dip);
747
748 /*
749 * Hardware/Firmware/etc. Version/Revision IDs.
750 */
751 t4_dump_version_info(sc);
752
753 if (n100g) {
754 cxgb_printf(dip, CE_NOTE,
755 "%dx100G (%d rxq, %d txq total) %d %s.",
756 n100g, rqidx, tqidx, sc->intr_count,
757 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
758 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
759 "fixed interrupt");
760 } else if (n40g) {
761 cxgb_printf(dip, CE_NOTE,
762 "%dx40G (%d rxq, %d txq total) %d %s.",
763 n40g, rqidx, tqidx, sc->intr_count,
764 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
765 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
766 "fixed interrupt");
767 } else if (n25g) {
768 cxgb_printf(dip, CE_NOTE,
769 "%dx25G (%d rxq, %d txq total) %d %s.",
770 n25g, rqidx, tqidx, sc->intr_count,
771 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
772 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
773 "fixed interrupt");
774 } else if (n10g && n1g) {
775 cxgb_printf(dip, CE_NOTE,
776 "%dx10G %dx1G (%d rxq, %d txq total) %d %s.",
777 n10g, n1g, rqidx, tqidx, sc->intr_count,
778 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
779 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
780 "fixed interrupt");
781 } else {
782 cxgb_printf(dip, CE_NOTE,
783 "%dx%sG (%d rxq, %d txq per port) %d %s.",
784 n10g ? n10g : n1g,
785 n10g ? "10" : "1",
786 n10g ? iaq.nrxq10g : iaq.nrxq1g,
787 n10g ? iaq.ntxq10g : iaq.ntxq1g,
788 sc->intr_count,
789 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
790 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
791 "fixed interrupt");
792 }
793
794 sc->ksp = setup_kstats(sc);
795 sc->ksp_stat = setup_wc_kstats(sc);
796 sc->params.drv_memwin = MEMWIN_NIC;
797
798 done:
799 if (rc != DDI_SUCCESS) {
800 (void) t4_devo_detach(dip, DDI_DETACH);
801
802 /* rc may have errno style errors or DDI errors */
803 rc = DDI_FAILURE;
804 }
805
806 return (rc);
807 }
808
809 static int
810 t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
811 {
812 int instance, i;
813 struct adapter *sc;
814 struct port_info *pi;
815 struct sge *s;
816
817 if (cmd != DDI_DETACH)
818 return (DDI_FAILURE);
819
820 instance = ddi_get_instance(dip);
821 sc = ddi_get_soft_state(t4_list, instance);
822 if (sc == NULL)
823 return (DDI_SUCCESS);
824
825 if (sc->flags & FULL_INIT_DONE) {
826 t4_intr_disable(sc);
827 for_each_port(sc, i) {
828 pi = sc->port[i];
829 if (pi && pi->flags & PORT_INIT_DONE)
830 (void) port_full_uninit(pi);
831 }
832 (void) adapter_full_uninit(sc);
833 }
834
835 /* Safe to call no matter what */
836 ddi_prop_remove_all(dip);
837 ddi_remove_minor_node(dip, NULL);
838
839 if (sc->ksp != NULL)
840 kstat_delete(sc->ksp);
841 if (sc->ksp_stat != NULL)
842 kstat_delete(sc->ksp_stat);
843
844 s = &sc->sge;
845 if (s->rxq != NULL)
846 kmem_free(s->rxq, s->nrxq * sizeof (struct sge_rxq));
847 #ifdef TCP_OFFLOAD_ENABLE
848 if (s->ofld_txq != NULL)
849 kmem_free(s->ofld_txq, s->nofldtxq * sizeof (struct sge_wrq));
850 if (s->ofld_rxq != NULL)
851 kmem_free(s->ofld_rxq,
852 s->nofldrxq * sizeof (struct sge_ofld_rxq));
853 if (s->ctrlq != NULL)
854 kmem_free(s->ctrlq,
855 sc->params.nports * sizeof (struct sge_wrq));
856 #endif
857 if (s->txq != NULL)
858 kmem_free(s->txq, s->ntxq * sizeof (struct sge_txq));
859 if (s->iqmap != NULL)
860 kmem_free(s->iqmap, s->niq * sizeof (struct sge_iq *));
861 if (s->eqmap != NULL)
862 kmem_free(s->eqmap, s->neq * sizeof (struct sge_eq *));
863
864 if (s->rxbuf_cache != NULL)
865 rxbuf_cache_destroy(s->rxbuf_cache);
866
867 if (sc->flags & INTR_ALLOCATED) {
868 for (i = 0; i < sc->intr_count; i++) {
869 (void) ddi_intr_remove_handler(sc->intr_handle[i]);
870 (void) ddi_intr_free(sc->intr_handle[i]);
871 }
872 sc->flags &= ~INTR_ALLOCATED;
873 }
874
875 if (sc->intr_handle != NULL) {
876 kmem_free(sc->intr_handle,
877 sc->intr_count * sizeof (*sc->intr_handle));
878 }
879
880 for_each_port(sc, i) {
881 pi = sc->port[i];
882 if (pi != NULL) {
883 mutex_destroy(&pi->lock);
884 kmem_free(pi, sizeof (*pi));
885 clrbit(&sc->registered_device_map, i);
886 }
887 }
888
889 if (sc->flags & FW_OK)
890 (void) t4_fw_bye(sc, sc->mbox);
891
892 if (sc->reg1h != NULL)
893 ddi_regs_map_free(&sc->reg1h);
894
895 if (sc->regh != NULL)
896 ddi_regs_map_free(&sc->regh);
897
898 if (sc->pci_regh != NULL)
899 pci_config_teardown(&sc->pci_regh);
900
901 mutex_enter(&t4_adapter_list_lock);
902 SLIST_REMOVE_HEAD(&t4_adapter_list, link);
903 mutex_exit(&t4_adapter_list_lock);
904
905 mutex_destroy(&sc->lock);
906 cv_destroy(&sc->cv);
907 mutex_destroy(&sc->sfl_lock);
908
909 #ifdef DEBUG
910 bzero(sc, sizeof (*sc));
911 #endif
912 ddi_soft_state_free(t4_list, instance);
913
914 return (DDI_SUCCESS);
915 }
916
917 static int
918 t4_devo_quiesce(dev_info_t *dip)
919 {
920 int instance;
921 struct adapter *sc;
922
923 instance = ddi_get_instance(dip);
924 sc = ddi_get_soft_state(t4_list, instance);
925 if (sc == NULL)
926 return (DDI_SUCCESS);
927
928 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
929 t4_intr_disable(sc);
930 t4_write_reg(sc, A_PL_RST, F_PIORSTMODE | F_PIORST);
931
932 return (DDI_SUCCESS);
933 }
934
935 static int
936 t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg,
937 void *result)
938 {
939 char s[4];
940 struct port_info *pi;
941 dev_info_t *child = (dev_info_t *)arg;
942
943 switch (op) {
944 case DDI_CTLOPS_REPORTDEV:
945 pi = ddi_get_parent_data(rdip);
946 pi->instance = ddi_get_instance(dip);
947 pi->child_inst = ddi_get_instance(rdip);
948 cmn_err(CE_CONT, "?%s%d is port %s on %s%d\n",
949 ddi_node_name(rdip), ddi_get_instance(rdip),
950 ddi_get_name_addr(rdip), ddi_driver_name(dip),
951 ddi_get_instance(dip));
952 return (DDI_SUCCESS);
953
954 case DDI_CTLOPS_INITCHILD:
955 pi = ddi_get_parent_data(child);
956 if (pi == NULL)
957 return (DDI_NOT_WELL_FORMED);
958 (void) snprintf(s, sizeof (s), "%d", pi->port_id);
959 ddi_set_name_addr(child, s);
960 return (DDI_SUCCESS);
961
962 case DDI_CTLOPS_UNINITCHILD:
963 ddi_set_name_addr(child, NULL);
964 return (DDI_SUCCESS);
965
966 case DDI_CTLOPS_ATTACH:
967 case DDI_CTLOPS_DETACH:
968 return (DDI_SUCCESS);
969
970 default:
971 return (ddi_ctlops(dip, rdip, op, arg, result));
972 }
973 }
974
975 static int
976 t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, void *arg,
977 dev_info_t **cdipp)
978 {
979 int instance, i;
980 struct adapter *sc;
981
982 instance = ddi_get_instance(dip);
983 sc = ddi_get_soft_state(t4_list, instance);
984
985 if (op == BUS_CONFIG_ONE) {
986 char *c;
987
988 /*
989 * arg is something like "cxgb@0" where 0 is the port_id hanging
990 * off this nexus.
991 */
992
993 c = arg;
994 while (*(c + 1))
995 c++;
996
997 /* There should be exactly 1 digit after '@' */
998 if (*(c - 1) != '@')
999 return (NDI_FAILURE);
1000
1001 i = *c - '0';
1002
1003 if (add_child_node(sc, i) != 0)
1004 return (NDI_FAILURE);
1005
1006 flags |= NDI_ONLINE_ATTACH;
1007
1008 } else if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) {
1009 /* Allocate and bind all child device nodes */
1010 for_each_port(sc, i)
1011 (void) add_child_node(sc, i);
1012 flags |= NDI_ONLINE_ATTACH;
1013 }
1014
1015 return (ndi_busop_bus_config(dip, flags, op, arg, cdipp, 0));
1016 }
1017
1018 static int
1019 t4_bus_unconfig(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
1020 void *arg)
1021 {
1022 int instance, i, rc;
1023 struct adapter *sc;
1024
1025 instance = ddi_get_instance(dip);
1026 sc = ddi_get_soft_state(t4_list, instance);
1027
1028 if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ALL ||
1029 op == BUS_UNCONFIG_DRIVER)
1030 flags |= NDI_UNCONFIG;
1031
1032 rc = ndi_busop_bus_unconfig(dip, flags, op, arg);
1033 if (rc != 0)
1034 return (rc);
1035
1036 if (op == BUS_UNCONFIG_ONE) {
1037 char *c;
1038
1039 c = arg;
1040 while (*(c + 1))
1041 c++;
1042
1043 if (*(c - 1) != '@')
1044 return (NDI_SUCCESS);
1045
1046 i = *c - '0';
1047
1048 rc = remove_child_node(sc, i);
1049
1050 } else if (op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) {
1051
1052 for_each_port(sc, i)
1053 (void) remove_child_node(sc, i);
1054 }
1055
1056 return (rc);
1057 }
1058
1059 /* ARGSUSED */
1060 static int
1061 t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp)
1062 {
1063 struct adapter *sc;
1064
1065 if (otyp != OTYP_CHR)
1066 return (EINVAL);
1067
1068 sc = ddi_get_soft_state(t4_list, getminor(*devp));
1069 if (sc == NULL)
1070 return (ENXIO);
1071
1072 return (atomic_cas_uint(&sc->open, 0, EBUSY));
1073 }
1074
1075 /* ARGSUSED */
1076 static int
1077 t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp)
1078 {
1079 struct adapter *sc;
1080
1081 sc = ddi_get_soft_state(t4_list, getminor(dev));
1082 if (sc == NULL)
1083 return (EINVAL);
1084
1085 (void) atomic_swap_uint(&sc->open, 0);
1086 return (0);
1087 }
1088
1089 /* ARGSUSED */
1090 static int
1091 t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, int *rp)
1092 {
1093 int instance;
1094 struct adapter *sc;
1095 void *data = (void *)d;
1096
1097 if (crgetuid(credp) != 0)
1098 return (EPERM);
1099
1100 instance = getminor(dev);
1101 sc = ddi_get_soft_state(t4_list, instance);
1102 if (sc == NULL)
1103 return (EINVAL);
1104
1105 return (t4_ioctl(sc, cmd, data, mode));
1106 }
1107
1108 static unsigned int
1109 getpf(struct adapter *sc)
1110 {
1111 int rc, *data;
1112 uint_t n, pf;
1113
1114 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1115 DDI_PROP_DONTPASS, "reg", &data, &n);
1116 if (rc != DDI_SUCCESS) {
1117 cxgb_printf(sc->dip, CE_WARN,
1118 "failed to lookup \"reg\" property: %d", rc);
1119 return (0xff);
1120 }
1121
1122 pf = PCI_REG_FUNC_G(data[0]);
1123 ddi_prop_free(data);
1124
1125 return (pf);
1126 }
1127
1128
1129 static struct fw_info *
1130 find_fw_info(int chip)
1131 {
1132 u32 i;
1133
1134 fi[0].chip = CHELSIO_T4;
1135 fi[0].fw_hdr.chip = FW_HDR_CHIP_T4;
1136 fi[0].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T4));
1137 fi[0].fw_hdr.intfver_nic = FW_INTFVER(T4, NIC);
1138 fi[0].fw_hdr.intfver_vnic = FW_INTFVER(T4, VNIC);
1139 fi[0].fw_hdr.intfver_ofld = FW_INTFVER(T4, OFLD);
1140 fi[0].fw_hdr.intfver_ri = FW_INTFVER(T4, RI);
1141 fi[0].fw_hdr.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU);
1142 fi[0].fw_hdr.intfver_iscsi = FW_INTFVER(T4, ISCSI);
1143 fi[0].fw_hdr.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU);
1144 fi[0].fw_hdr.intfver_fcoe = FW_INTFVER(T4, FCOE);
1145
1146 fi[1].chip = CHELSIO_T5;
1147 fi[1].fw_hdr.chip = FW_HDR_CHIP_T5;
1148 fi[1].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T5));
1149 fi[1].fw_hdr.intfver_nic = FW_INTFVER(T5, NIC);
1150 fi[1].fw_hdr.intfver_vnic = FW_INTFVER(T5, VNIC);
1151 fi[1].fw_hdr.intfver_ofld = FW_INTFVER(T5, OFLD);
1152 fi[1].fw_hdr.intfver_ri = FW_INTFVER(T5, RI);
1153 fi[1].fw_hdr.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU);
1154 fi[1].fw_hdr.intfver_iscsi = FW_INTFVER(T5, ISCSI);
1155 fi[1].fw_hdr.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU);
1156 fi[1].fw_hdr.intfver_fcoe = FW_INTFVER(T5, FCOE);
1157
1158 fi[2].chip = CHELSIO_T6;
1159 fi[2].fw_hdr.chip = FW_HDR_CHIP_T6;
1160 fi[2].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T6));
1161 fi[2].fw_hdr.intfver_nic = FW_INTFVER(T6, NIC);
1162 fi[2].fw_hdr.intfver_vnic = FW_INTFVER(T6, VNIC);
1163 fi[2].fw_hdr.intfver_ofld = FW_INTFVER(T6, OFLD);
1164 fi[2].fw_hdr.intfver_ri = FW_INTFVER(T6, RI);
1165 fi[2].fw_hdr.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU);
1166 fi[2].fw_hdr.intfver_iscsi = FW_INTFVER(T6, ISCSI);
1167 fi[2].fw_hdr.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU);
1168 fi[2].fw_hdr.intfver_fcoe = FW_INTFVER(T6, FCOE);
1169
1170 for (i = 0; i < ARRAY_SIZE(fi); i++) {
1171 if (fi[i].chip == chip)
1172 return &fi[i];
1173 }
1174
1175 return NULL;
1176 }
1177
1178 /*
1179 * Install a compatible firmware (if required), establish contact with it,
1180 * become the master, and reset the device.
1181 */
1182 static int
1183 prep_firmware(struct adapter *sc)
1184 {
1185 int rc;
1186 int fw_size;
1187 int reset = 1;
1188 enum dev_state state;
1189 unsigned char *fw_data;
1190 struct fw_info *fw_info;
1191 struct fw_hdr *card_fw;
1192
1193 struct driver_properties *p = &sc->props;
1194
1195 /* Contact firmware, request master */
1196 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state);
1197 if (rc < 0) {
1198 rc = -rc;
1199 cxgb_printf(sc->dip, CE_WARN,
1200 "failed to connect to the firmware: %d.", rc);
1201 return (rc);
1202 }
1203
1204 if (rc == sc->mbox)
1205 sc->flags |= MASTER_PF;
1206
1207 /* We may need FW version info for later reporting */
1208 t4_get_version_info(sc);
1209 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(sc->params.chip));
1210 /* allocate memory to read the header of the firmware on the
1211 * card
1212 */
1213 if (!fw_info) {
1214 cxgb_printf(sc->dip, CE_WARN,
1215 "unable to look up firmware information for chip %d.\n",
1216 CHELSIO_CHIP_VERSION(sc->params.chip));
1217 return EINVAL;
1218 }
1219 card_fw = kmem_zalloc(sizeof(*card_fw), KM_SLEEP);
1220 if(!card_fw) {
1221 cxgb_printf(sc->dip, CE_WARN,
1222 "Memory allocation for card FW header failed\n");
1223 return ENOMEM;
1224 }
1225 switch(CHELSIO_CHIP_VERSION(sc->params.chip)) {
1226 case CHELSIO_T4:
1227 fw_data = t4fw_data;
1228 fw_size = t4fw_size;
1229 break;
1230 case CHELSIO_T5:
1231 fw_data = t5fw_data;
1232 fw_size = t5fw_size;
1233 break;
1234 case CHELSIO_T6:
1235 fw_data = t6fw_data;
1236 fw_size = t6fw_size;
1237 break;
1238 default:
1239 cxgb_printf(sc->dip, CE_WARN, "Adapter type not supported\n");
1240 kmem_free(card_fw, sizeof(*card_fw));
1241 return EINVAL;
1242 }
1243
1244 rc = -t4_prep_fw(sc, fw_info, fw_data, fw_size, card_fw,
1245 p->t4_fw_install, state, &reset);
1246
1247 kmem_free(card_fw, sizeof(*card_fw));
1248
1249 if (rc != 0) {
1250 cxgb_printf(sc->dip, CE_WARN,
1251 "failed to install firmware: %d", rc);
1252 return (rc);
1253 } else {
1254 /* refresh */
1255 (void) t4_check_fw_version(sc);
1256 }
1257
1258 /* Reset device */
1259 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1260 if (rc != 0) {
1261 cxgb_printf(sc->dip, CE_WARN,
1262 "firmware reset failed: %d.", rc);
1263 if (rc != ETIMEDOUT && rc != EIO)
1264 (void) t4_fw_bye(sc, sc->mbox);
1265 return (rc);
1266 }
1267
1268 /* Partition adapter resources as specified in the config file. */
1269 if (sc->flags & MASTER_PF) {
1270 /* Handle default vs special T4 config file */
1271
1272 rc = partition_resources(sc);
1273 if (rc != 0)
1274 goto err; /* error message displayed already */
1275 }
1276
1277 sc->flags |= FW_OK;
1278 return (0);
1279 err:
1280 return (rc);
1281
1282 }
1283
1284 static const struct memwin t4_memwin[] = {
1285 { MEMWIN0_BASE, MEMWIN0_APERTURE },
1286 { MEMWIN1_BASE, MEMWIN1_APERTURE },
1287 { MEMWIN2_BASE, MEMWIN2_APERTURE }
1288 };
1289
1290 static const struct memwin t5_memwin[] = {
1291 { MEMWIN0_BASE, MEMWIN0_APERTURE },
1292 { MEMWIN1_BASE, MEMWIN1_APERTURE },
1293 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1294 };
1295
1296 #define FW_PARAM_DEV(param) \
1297 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1298 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1299 #define FW_PARAM_PFVF(param) \
1300 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1301 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1302
1303 /*
1304 * Verify that the memory range specified by the memtype/offset/len pair is
1305 * valid and lies entirely within the memtype specified. The global address of
1306 * the start of the range is returned in addr.
1307 */
1308 int
1309 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1310 uint32_t *addr)
1311 {
1312 uint32_t em, addr_len, maddr, mlen;
1313
1314 /* Memory can only be accessed in naturally aligned 4 byte units */
1315 if (off & 3 || len & 3 || len == 0)
1316 return (EINVAL);
1317
1318 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1319 switch (mtype) {
1320 case MEM_EDC0:
1321 if (!(em & F_EDRAM0_ENABLE))
1322 return (EINVAL);
1323 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1324 maddr = G_EDRAM0_BASE(addr_len) << 20;
1325 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1326 break;
1327 case MEM_EDC1:
1328 if (!(em & F_EDRAM1_ENABLE))
1329 return (EINVAL);
1330 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1331 maddr = G_EDRAM1_BASE(addr_len) << 20;
1332 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1333 break;
1334 case MEM_MC:
1335 if (!(em & F_EXT_MEM_ENABLE))
1336 return (EINVAL);
1337 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1338 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1339 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1340 break;
1341 case MEM_MC1:
1342 if (is_t4(sc->params.chip) || !(em & F_EXT_MEM1_ENABLE))
1343 return (EINVAL);
1344 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1345 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1346 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1347 break;
1348 default:
1349 return (EINVAL);
1350 }
1351
1352 if (mlen > 0 && off < mlen && off + len <= mlen) {
1353 *addr = maddr + off; /* global address */
1354 return (0);
1355 }
1356
1357 return (EFAULT);
1358 }
1359
1360 void
1361 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1362 {
1363 const struct memwin *mw;
1364
1365 if (is_t4(sc->params.chip)) {
1366 mw = &t4_memwin[win];
1367 } else {
1368 mw = &t5_memwin[win];
1369 }
1370
1371 if (base != NULL)
1372 *base = mw->base;
1373 if (aperture != NULL)
1374 *aperture = mw->aperture;
1375 }
1376
1377 /*
1378 * Upload configuration file to card's memory.
1379 */
1380 static int
1381 upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma)
1382 {
1383 int rc = 0, cflen;
1384 u_int i, n;
1385 uint32_t param, val, addr, mtype, maddr;
1386 uint32_t off, mw_base, mw_aperture;
1387 const uint32_t *cfdata;
1388
1389 /* Figure out where the firmware wants us to upload it. */
1390 param = FW_PARAM_DEV(CF);
1391 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1392 if (rc != 0) {
1393 /* Firmwares without config file support will fail this way */
1394 cxgb_printf(sc->dip, CE_WARN,
1395 "failed to query config file location: %d.\n", rc);
1396 return (rc);
1397 }
1398 *mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1399 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1400
1401 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
1402 case CHELSIO_T4:
1403 cflen = t4cfg_size & ~3;
1404 /* LINTED: E_BAD_PTR_CAST_ALIGN */
1405 cfdata = (const uint32_t *)t4cfg_data;
1406 break;
1407 case CHELSIO_T5:
1408 cflen = t5cfg_size & ~3;
1409 /* LINTED: E_BAD_PTR_CAST_ALIGN */
1410 cfdata = (const uint32_t *)t5cfg_data;
1411 break;
1412 case CHELSIO_T6:
1413 cflen = t6cfg_size & ~3;
1414 /* LINTED: E_BAD_PTR_CAST_ALIGN */
1415 cfdata = (const uint32_t *)t6cfg_data;
1416 break;
1417 default:
1418 cxgb_printf(sc->dip, CE_WARN,
1419 "Invalid Adapter detected\n");
1420 return EINVAL;
1421 }
1422
1423 if (cflen > FLASH_CFG_MAX_SIZE) {
1424 cxgb_printf(sc->dip, CE_WARN,
1425 "config file too long (%d, max allowed is %d). ",
1426 cflen, FLASH_CFG_MAX_SIZE);
1427 return (EFBIG);
1428 }
1429
1430 rc = validate_mt_off_len(sc, mtype, maddr, cflen, &addr);
1431 if (rc != 0) {
1432
1433 cxgb_printf(sc->dip, CE_WARN,
1434 "%s: addr (%d/0x%x) or len %d is not valid: %d. "
1435 "Will try to use the config on the card, if any.\n",
1436 __func__, mtype, maddr, cflen, rc);
1437 return (EFAULT);
1438 }
1439
1440 memwin_info(sc, 2, &mw_base, &mw_aperture);
1441 while (cflen) {
1442 off = position_memwin(sc, 2, addr);
1443 n = min(cflen, mw_aperture - off);
1444 for (i = 0; i < n; i += 4)
1445 t4_write_reg(sc, mw_base + off + i, *cfdata++);
1446 cflen -= n;
1447 addr += n;
1448 }
1449
1450 return (rc);
1451 }
1452
1453 /*
1454 * Partition chip resources for use between various PFs, VFs, etc. This is done
1455 * by uploading the firmware configuration file to the adapter and instructing
1456 * the firmware to process it.
1457 */
1458 static int
1459 partition_resources(struct adapter *sc)
1460 {
1461 int rc;
1462 struct fw_caps_config_cmd caps;
1463 uint32_t mtype, maddr, finicsum, cfcsum;
1464
1465 rc = upload_config_file(sc, &mtype, &maddr);
1466 if (rc != 0) {
1467 mtype = FW_MEMTYPE_CF_FLASH;
1468 maddr = t4_flash_cfg_addr(sc);
1469 }
1470
1471 bzero(&caps, sizeof (caps));
1472 caps.op_to_write = BE_32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1473 F_FW_CMD_REQUEST | F_FW_CMD_READ);
1474 caps.cfvalid_to_len16 = BE_32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1475 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1476 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1477 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1478 if (rc != 0) {
1479 cxgb_printf(sc->dip, CE_WARN,
1480 "failed to pre-process config file: %d.\n", rc);
1481 return (rc);
1482 }
1483
1484 finicsum = ntohl(caps.finicsum);
1485 cfcsum = ntohl(caps.cfcsum);
1486 if (finicsum != cfcsum) {
1487 cxgb_printf(sc->dip, CE_WARN,
1488 "WARNING: config file checksum mismatch: %08x %08x\n",
1489 finicsum, cfcsum);
1490 }
1491 sc->cfcsum = cfcsum;
1492
1493 /* TODO: Need to configure this correctly */
1494 caps.toecaps = htons(FW_CAPS_CONFIG_TOE);
1495 caps.iscsicaps = 0;
1496 caps.rdmacaps = 0;
1497 caps.fcoecaps = 0;
1498 /* TODO: Disable VNIC cap for now */
1499 caps.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
1500
1501 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1502 F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1503 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1504 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), NULL);
1505 if (rc != 0) {
1506 cxgb_printf(sc->dip, CE_WARN,
1507 "failed to process config file: %d.\n", rc);
1508 return (rc);
1509 }
1510
1511 return (0);
1512 }
1513
1514 /*
1515 * Tweak configuration based on module parameters, etc. Most of these have
1516 * defaults assigned to them by Firmware Configuration Files (if we're using
1517 * them) but need to be explicitly set if we're using hard-coded
1518 * initialization. But even in the case of using Firmware Configuration
1519 * Files, we'd like to expose the ability to change these via module
1520 * parameters so these are essentially common tweaks/settings for
1521 * Configuration Files and hard-coded initialization ...
1522 */
1523 static int
1524 adap__pre_init_tweaks(struct adapter *sc)
1525 {
1526 int rx_dma_offset = 2; /* Offset of RX packets into DMA buffers */
1527
1528 /*
1529 * Fix up various Host-Dependent Parameters like Page Size, Cache
1530 * Line Size, etc. The firmware default is for a 4KB Page Size and
1531 * 64B Cache Line Size ...
1532 */
1533 (void) t4_fixup_host_params_compat(sc, PAGE_SIZE, CACHE_LINE, T5_LAST_REV);
1534
1535 t4_set_reg_field(sc, A_SGE_CONTROL,
1536 V_PKTSHIFT(M_PKTSHIFT), V_PKTSHIFT(rx_dma_offset));
1537
1538 return 0;
1539 }
1540 /*
1541 * Retrieve parameters that are needed (or nice to have) prior to calling
1542 * t4_sge_init and t4_fw_initialize.
1543 */
1544 static int
1545 get_params__pre_init(struct adapter *sc)
1546 {
1547 int rc;
1548 uint32_t param[2], val[2];
1549 struct fw_devlog_cmd cmd;
1550 struct devlog_params *dlog = &sc->params.devlog;
1551
1552 /*
1553 * Grab the raw VPD parameters.
1554 */
1555 rc = -t4_get_raw_vpd_params(sc, &sc->params.vpd);
1556 if (rc != 0) {
1557 cxgb_printf(sc->dip, CE_WARN,
1558 "failed to query VPD parameters (pre_init): %d.\n", rc);
1559 return (rc);
1560 }
1561
1562 param[0] = FW_PARAM_DEV(PORTVEC);
1563 param[1] = FW_PARAM_DEV(CCLK);
1564 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1565 if (rc != 0) {
1566 cxgb_printf(sc->dip, CE_WARN,
1567 "failed to query parameters (pre_init): %d.\n", rc);
1568 return (rc);
1569 }
1570
1571 sc->params.portvec = val[0];
1572 sc->params.nports = 0;
1573 while (val[0]) {
1574 sc->params.nports++;
1575 val[0] &= val[0] - 1;
1576 }
1577
1578 sc->params.vpd.cclk = val[1];
1579
1580 /* Read device log parameters. */
1581 bzero(&cmd, sizeof (cmd));
1582 cmd.op_to_write = htonl(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1583 F_FW_CMD_REQUEST | F_FW_CMD_READ);
1584 cmd.retval_len16 = htonl(FW_LEN16(cmd));
1585 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof (cmd), &cmd);
1586 if (rc != 0) {
1587 cxgb_printf(sc->dip, CE_WARN,
1588 "failed to get devlog parameters: %d.\n", rc);
1589 bzero(dlog, sizeof (*dlog));
1590 rc = 0; /* devlog isn't critical for device operation */
1591 } else {
1592 val[0] = ntohl(cmd.memtype_devlog_memaddr16_devlog);
1593 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1594 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1595 dlog->size = ntohl(cmd.memsize_devlog);
1596 }
1597
1598 return (rc);
1599 }
1600
1601 /*
1602 * Retrieve various parameters that are of interest to the driver. The device
1603 * has been initialized by the firmware at this point.
1604 */
1605 static int
1606 get_params__post_init(struct adapter *sc)
1607 {
1608 int rc;
1609 uint32_t param[7], val[7];
1610 struct fw_caps_config_cmd caps;
1611
1612 param[0] = FW_PARAM_PFVF(IQFLINT_START);
1613 param[1] = FW_PARAM_PFVF(EQ_START);
1614 param[2] = FW_PARAM_PFVF(FILTER_START);
1615 param[3] = FW_PARAM_PFVF(FILTER_END);
1616 param[4] = FW_PARAM_PFVF(L2T_START);
1617 param[5] = FW_PARAM_PFVF(L2T_END);
1618 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1619 if (rc != 0) {
1620 cxgb_printf(sc->dip, CE_WARN,
1621 "failed to query parameters (post_init): %d.\n", rc);
1622 return (rc);
1623 }
1624
1625 /* LINTED: E_ASSIGN_NARROW_CONV */
1626 sc->sge.iq_start = val[0];
1627 sc->sge.eq_start = val[1];
1628 sc->tids.ftid_base = val[2];
1629 sc->tids.nftids = val[3] - val[2] + 1;
1630 sc->vres.l2t.start = val[4];
1631 sc->vres.l2t.size = val[5] - val[4] + 1;
1632
1633 /* get capabilites */
1634 bzero(&caps, sizeof (caps));
1635 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1636 F_FW_CMD_REQUEST | F_FW_CMD_READ);
1637 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1638 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1639 if (rc != 0) {
1640 cxgb_printf(sc->dip, CE_WARN,
1641 "failed to get card capabilities: %d.\n", rc);
1642 return (rc);
1643 }
1644
1645 if (caps.toecaps != 0) {
1646 /* query offload-related parameters */
1647 param[0] = FW_PARAM_DEV(NTID);
1648 param[1] = FW_PARAM_PFVF(SERVER_START);
1649 param[2] = FW_PARAM_PFVF(SERVER_END);
1650 param[3] = FW_PARAM_PFVF(TDDP_START);
1651 param[4] = FW_PARAM_PFVF(TDDP_END);
1652 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1653 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1654 if (rc != 0) {
1655 cxgb_printf(sc->dip, CE_WARN,
1656 "failed to query TOE parameters: %d.\n", rc);
1657 return (rc);
1658 }
1659 sc->tids.ntids = val[0];
1660 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1661 sc->tids.stid_base = val[1];
1662 sc->tids.nstids = val[2] - val[1] + 1;
1663 sc->vres.ddp.start = val[3];
1664 sc->vres.ddp.size = val[4] - val[3] + 1;
1665 sc->params.ofldq_wr_cred = val[5];
1666 sc->params.offload = 1;
1667 }
1668
1669 /* These are finalized by FW initialization, load their values now */
1670 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1671 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1672 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1673 t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1674
1675 return (rc);
1676 }
1677
1678 static int
1679 set_params__post_init(struct adapter *sc)
1680 {
1681 uint32_t param, val;
1682
1683 /* ask for encapsulated CPLs */
1684 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
1685 val = 1;
1686 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1687
1688 return (0);
1689 }
1690
1691 /* TODO: verify */
1692 static void
1693 setup_memwin(struct adapter *sc)
1694 {
1695 pci_regspec_t *data;
1696 int rc;
1697 uint_t n;
1698 uintptr_t bar0;
1699 uintptr_t mem_win0_base, mem_win1_base, mem_win2_base;
1700 uintptr_t mem_win2_aperture;
1701
1702 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1703 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&data, &n);
1704 if (rc != DDI_SUCCESS) {
1705 cxgb_printf(sc->dip, CE_WARN,
1706 "failed to lookup \"assigned-addresses\" property: %d", rc);
1707 return;
1708 }
1709 n /= sizeof (*data);
1710
1711 bar0 = ((uint64_t)data[0].pci_phys_mid << 32) | data[0].pci_phys_low;
1712 ddi_prop_free(data);
1713
1714 if (is_t4(sc->params.chip)) {
1715 mem_win0_base = bar0 + MEMWIN0_BASE;
1716 mem_win1_base = bar0 + MEMWIN1_BASE;
1717 mem_win2_base = bar0 + MEMWIN2_BASE;
1718 mem_win2_aperture = MEMWIN2_APERTURE;
1719 } else {
1720 /* For T5, only relative offset inside the PCIe BAR is passed */
1721 mem_win0_base = MEMWIN0_BASE;
1722 mem_win1_base = MEMWIN1_BASE;
1723 mem_win2_base = MEMWIN2_BASE_T5;
1724 mem_win2_aperture = MEMWIN2_APERTURE_T5;
1725 }
1726
1727 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1728 mem_win0_base | V_BIR(0) |
1729 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1730
1731 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1732 mem_win1_base | V_BIR(0) |
1733 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1734
1735 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1736 mem_win2_base | V_BIR(0) |
1737 V_WINDOW(ilog2(mem_win2_aperture) - 10));
1738
1739 /* flush */
1740 (void)t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1741 }
1742
1743 /*
1744 * Positions the memory window such that it can be used to access the specified
1745 * address in the chip's address space. The return value is the offset of addr
1746 * from the start of the window.
1747 */
1748 uint32_t
1749 position_memwin(struct adapter *sc, int n, uint32_t addr)
1750 {
1751 uint32_t start, pf;
1752 uint32_t reg;
1753
1754 if (addr & 3) {
1755 cxgb_printf(sc->dip, CE_WARN,
1756 "addr (0x%x) is not at a 4B boundary.\n", addr);
1757 return (EFAULT);
1758 }
1759
1760 if (is_t4(sc->params.chip)) {
1761 pf = 0;
1762 start = addr & ~0xf; /* start must be 16B aligned */
1763 } else {
1764 pf = V_PFNUM(sc->pf);
1765 start = addr & ~0x7f; /* start must be 128B aligned */
1766 }
1767 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1768
1769 t4_write_reg(sc, reg, start | pf);
1770 (void) t4_read_reg(sc, reg);
1771
1772 return (addr - start);
1773 }
1774
1775
1776 /*
1777 * Reads the named property and fills up the "data" array (which has at least
1778 * "count" elements). We first try and lookup the property for our dev_t and
1779 * then retry with DDI_DEV_T_ANY if it's not found.
1780 *
1781 * Returns non-zero if the property was found and "data" has been updated.
1782 */
1783 static int
1784 prop_lookup_int_array(struct adapter *sc, char *name, int *data, uint_t count)
1785 {
1786 dev_info_t *dip = sc->dip;
1787 dev_t dev = sc->dev;
1788 int rc, *d;
1789 uint_t i, n;
1790
1791 rc = ddi_prop_lookup_int_array(dev, dip, DDI_PROP_DONTPASS,
1792 name, &d, &n);
1793 if (rc == DDI_PROP_SUCCESS)
1794 goto found;
1795
1796 if (rc != DDI_PROP_NOT_FOUND) {
1797 cxgb_printf(dip, CE_WARN,
1798 "failed to lookup property %s for minor %d: %d.",
1799 name, getminor(dev), rc);
1800 return (0);
1801 }
1802
1803 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1804 name, &d, &n);
1805 if (rc == DDI_PROP_SUCCESS)
1806 goto found;
1807
1808 if (rc != DDI_PROP_NOT_FOUND) {
1809 cxgb_printf(dip, CE_WARN,
1810 "failed to lookup property %s: %d.", name, rc);
1811 return (0);
1812 }
1813
1814 return (0);
1815
1816 found:
1817 if (n > count) {
1818 cxgb_printf(dip, CE_NOTE,
1819 "property %s has too many elements (%d), ignoring extras",
1820 name, n);
1821 }
1822
1823 for (i = 0; i < n && i < count; i++)
1824 data[i] = d[i];
1825 ddi_prop_free(d);
1826
1827 return (1);
1828 }
1829
1830 static int
1831 prop_lookup_int(struct adapter *sc, char *name, int defval)
1832 {
1833 int rc;
1834
1835 rc = ddi_prop_get_int(sc->dev, sc->dip, DDI_PROP_DONTPASS, name, -1);
1836 if (rc != -1)
1837 return (rc);
1838
1839 return (ddi_prop_get_int(DDI_DEV_T_ANY, sc->dip, DDI_PROP_DONTPASS,
1840 name, defval));
1841 }
1842
1843 static int
1844 init_driver_props(struct adapter *sc, struct driver_properties *p)
1845 {
1846 dev_t dev = sc->dev;
1847 dev_info_t *dip = sc->dip;
1848 int i, *data;
1849 uint_t tmr[SGE_NTIMERS] = {5, 10, 20, 50, 100, 200};
1850 uint_t cnt[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
1851
1852 /*
1853 * Holdoff timer
1854 */
1855 data = &p->timer_val[0];
1856 for (i = 0; i < SGE_NTIMERS; i++)
1857 data[i] = tmr[i];
1858 (void) prop_lookup_int_array(sc, "holdoff-timer-values", data,
1859 SGE_NTIMERS);
1860 for (i = 0; i < SGE_NTIMERS; i++) {
1861 int limit = 200U;
1862 if (data[i] > limit) {
1863 cxgb_printf(dip, CE_WARN,
1864 "holdoff timer %d is too high (%d), lowered to %d.",
1865 i, data[i], limit);
1866 data[i] = limit;
1867 }
1868 }
1869 (void) ddi_prop_update_int_array(dev, dip, "holdoff-timer-values",
1870 data, SGE_NTIMERS);
1871
1872 /*
1873 * Holdoff packet counter
1874 */
1875 data = &p->counter_val[0];
1876 for (i = 0; i < SGE_NCOUNTERS; i++)
1877 data[i] = cnt[i];
1878 (void) prop_lookup_int_array(sc, "holdoff-pkt-counter-values", data,
1879 SGE_NCOUNTERS);
1880 for (i = 0; i < SGE_NCOUNTERS; i++) {
1881 int limit = M_THRESHOLD_0;
1882 if (data[i] > limit) {
1883 cxgb_printf(dip, CE_WARN,
1884 "holdoff pkt-counter %d is too high (%d), "
1885 "lowered to %d.", i, data[i], limit);
1886 data[i] = limit;
1887 }
1888 }
1889 (void) ddi_prop_update_int_array(dev, dip, "holdoff-pkt-counter-values",
1890 data, SGE_NCOUNTERS);
1891
1892 /*
1893 * Maximum # of tx and rx queues to use for each
1894 * 100G, 40G, 25G, 10G and 1G port.
1895 */
1896 p->max_ntxq_10g = prop_lookup_int(sc, "max-ntxq-10G-port", 8);
1897 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1898 p->max_ntxq_10g);
1899
1900 p->max_nrxq_10g = prop_lookup_int(sc, "max-nrxq-10G-port", 8);
1901 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1902 p->max_nrxq_10g);
1903
1904 p->max_ntxq_1g = prop_lookup_int(sc, "max-ntxq-1G-port", 2);
1905 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1906 p->max_ntxq_1g);
1907
1908 p->max_nrxq_1g = prop_lookup_int(sc, "max-nrxq-1G-port", 2);
1909 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1910 p->max_nrxq_1g);
1911
1912 #ifdef TCP_OFFLOAD_ENABLE
1913 p->max_nofldtxq_10g = prop_lookup_int(sc, "max-nofldtxq-10G-port", 8);
1914 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1915 p->max_nofldtxq_10g);
1916
1917 p->max_nofldrxq_10g = prop_lookup_int(sc, "max-nofldrxq-10G-port", 2);
1918 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1919 p->max_nofldrxq_10g);
1920
1921 p->max_nofldtxq_1g = prop_lookup_int(sc, "max-nofldtxq-1G-port", 2);
1922 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1923 p->max_nofldtxq_1g);
1924
1925 p->max_nofldrxq_1g = prop_lookup_int(sc, "max-nofldrxq-1G-port", 1);
1926 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1927 p->max_nofldrxq_1g);
1928 #endif
1929
1930 /*
1931 * Holdoff parameters for 10G and 1G ports.
1932 */
1933 p->tmr_idx_10g = prop_lookup_int(sc, "holdoff-timer-idx-10G", 0);
1934 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-10G",
1935 p->tmr_idx_10g);
1936
1937 p->pktc_idx_10g = prop_lookup_int(sc, "holdoff-pktc-idx-10G", 2);
1938 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-10G",
1939 p->pktc_idx_10g);
1940
1941 p->tmr_idx_1g = prop_lookup_int(sc, "holdoff-timer-idx-1G", 0);
1942 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-1G",
1943 p->tmr_idx_1g);
1944
1945 p->pktc_idx_1g = prop_lookup_int(sc, "holdoff-pktc-idx-1G", 2);
1946 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-1G",
1947 p->pktc_idx_1g);
1948
1949 /*
1950 * Size (number of entries) of each tx and rx queue.
1951 */
1952 i = prop_lookup_int(sc, "qsize-txq", TX_EQ_QSIZE);
1953 p->qsize_txq = max(i, 128);
1954 if (p->qsize_txq != i) {
1955 cxgb_printf(dip, CE_WARN,
1956 "using %d instead of %d as the tx queue size",
1957 p->qsize_txq, i);
1958 }
1959 (void) ddi_prop_update_int(dev, dip, "qsize-txq", p->qsize_txq);
1960
1961 i = prop_lookup_int(sc, "qsize-rxq", RX_IQ_QSIZE);
1962 p->qsize_rxq = max(i, 128);
1963 while (p->qsize_rxq & 7)
1964 p->qsize_rxq--;
1965 if (p->qsize_rxq != i) {
1966 cxgb_printf(dip, CE_WARN,
1967 "using %d instead of %d as the rx queue size",
1968 p->qsize_rxq, i);
1969 }
1970 (void) ddi_prop_update_int(dev, dip, "qsize-rxq", p->qsize_rxq);
1971
1972 /*
1973 * Interrupt types allowed.
1974 * Bits 0, 1, 2 = INTx, MSI, MSI-X respectively. See sys/ddi_intr.h
1975 */
1976 p->intr_types = prop_lookup_int(sc, "interrupt-types",
1977 DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_FIXED);
1978 (void) ddi_prop_update_int(dev, dip, "interrupt-types", p->intr_types);
1979
1980 /*
1981 * Forwarded interrupt queues. Create this property to force the driver
1982 * to use forwarded interrupt queues.
1983 */
1984 if (ddi_prop_exists(dev, dip, DDI_PROP_DONTPASS,
1985 "interrupt-forwarding") != 0 ||
1986 ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1987 "interrupt-forwarding") != 0) {
1988 UNIMPLEMENTED();
1989 (void) ddi_prop_create(dev, dip, DDI_PROP_CANSLEEP,
1990 "interrupt-forwarding", NULL, 0);
1991 }
1992
1993 /*
1994 * Write combining
1995 * 0 to disable, 1 to enable
1996 */
1997 p->wc = prop_lookup_int(sc, "write-combine", 1);
1998 cxgb_printf(dip, CE_WARN, "write-combine: using of %d", p->wc);
1999 if (p->wc != 0 && p->wc != 1) {
2000 cxgb_printf(dip, CE_WARN,
2001 "write-combine: using 1 instead of %d", p->wc);
2002 p->wc = 1;
2003 }
2004 (void) ddi_prop_update_int(dev, dip, "write-combine", p->wc);
2005
2006 p->t4_fw_install = prop_lookup_int(sc, "t4_fw_install", 1);
2007 if (p->t4_fw_install != 0 && p->t4_fw_install != 2)
2008 p->t4_fw_install = 1;
2009 (void) ddi_prop_update_int(dev, dip, "t4_fw_install", p->t4_fw_install);
2010
2011 /* Multiple Rings */
2012 p->multi_rings = prop_lookup_int(sc, "multi-rings", 1);
2013 if (p->multi_rings != 0 && p->multi_rings != 1) {
2014 cxgb_printf(dip, CE_NOTE,
2015 "multi-rings: using value 1 instead of %d", p->multi_rings);
2016 p->multi_rings = 1;
2017 }
2018
2019 (void) ddi_prop_update_int(dev, dip, "multi-rings", p->multi_rings);
2020
2021 return (0);
2022 }
2023
2024 static int
2025 remove_extra_props(struct adapter *sc, int n10g, int n1g)
2026 {
2027 if (n10g == 0) {
2028 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-10G-port");
2029 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-10G-port");
2030 (void) ddi_prop_remove(sc->dev, sc->dip,
2031 "holdoff-timer-idx-10G");
2032 (void) ddi_prop_remove(sc->dev, sc->dip,
2033 "holdoff-pktc-idx-10G");
2034 }
2035
2036 if (n1g == 0) {
2037 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-1G-port");
2038 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-1G-port");
2039 (void) ddi_prop_remove(sc->dev, sc->dip,
2040 "holdoff-timer-idx-1G");
2041 (void) ddi_prop_remove(sc->dev, sc->dip, "holdoff-pktc-idx-1G");
2042 }
2043
2044 return (0);
2045 }
2046
2047 static int
2048 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
2049 struct intrs_and_queues *iaq)
2050 {
2051 struct driver_properties *p = &sc->props;
2052 int rc, itype, itypes, navail, nc, nrxq10g, nrxq1g, n;
2053 int nofldrxq10g = 0, nofldrxq1g = 0;
2054
2055 bzero(iaq, sizeof (*iaq));
2056 nc = ncpus; /* our snapshot of the number of CPUs */
2057 iaq->ntxq10g = min(nc, p->max_ntxq_10g);
2058 iaq->ntxq1g = min(nc, p->max_ntxq_1g);
2059 iaq->nrxq10g = nrxq10g = min(nc, p->max_nrxq_10g);
2060 iaq->nrxq1g = nrxq1g = min(nc, p->max_nrxq_1g);
2061 #ifdef TCP_OFFLOAD_ENABLE
2062 iaq->nofldtxq10g = min(nc, p->max_nofldtxq_10g);
2063 iaq->nofldtxq1g = min(nc, p->max_nofldtxq_1g);
2064 iaq->nofldrxq10g = nofldrxq10g = min(nc, p->max_nofldrxq_10g);
2065 iaq->nofldrxq1g = nofldrxq1g = min(nc, p->max_nofldrxq_1g);
2066 #endif
2067
2068 rc = ddi_intr_get_supported_types(sc->dip, &itypes);
2069 if (rc != DDI_SUCCESS) {
2070 cxgb_printf(sc->dip, CE_WARN,
2071 "failed to determine supported interrupt types: %d", rc);
2072 return (rc);
2073 }
2074
2075 for (itype = DDI_INTR_TYPE_MSIX; itype; itype >>= 1) {
2076 ASSERT(itype == DDI_INTR_TYPE_MSIX ||
2077 itype == DDI_INTR_TYPE_MSI ||
2078 itype == DDI_INTR_TYPE_FIXED);
2079
2080 if ((itype & itypes & p->intr_types) == 0)
2081 continue; /* not supported or not allowed */
2082
2083 navail = 0;
2084 rc = ddi_intr_get_navail(sc->dip, itype, &navail);
2085 if (rc != DDI_SUCCESS || navail == 0) {
2086 cxgb_printf(sc->dip, CE_WARN,
2087 "failed to get # of interrupts for type %d: %d",
2088 itype, rc);
2089 continue; /* carry on */
2090 }
2091
2092 iaq->intr_type = itype;
2093 if (navail == 0)
2094 continue;
2095
2096 /*
2097 * Best option: an interrupt vector for errors, one for the
2098 * firmware event queue, and one each for each rxq (NIC as well
2099 * as offload).
2100 */
2101 iaq->nirq = T4_EXTRA_INTR;
2102 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
2103 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
2104
2105 if (iaq->nirq <= navail &&
2106 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2107 iaq->intr_fwd = 0;
2108 goto allocate;
2109 }
2110
2111 /*
2112 * Second best option: an interrupt vector for errors, one for
2113 * the firmware event queue, and one each for either NIC or
2114 * offload rxq's.
2115 */
2116 iaq->nirq = T4_EXTRA_INTR;
2117 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
2118 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
2119 if (iaq->nirq <= navail &&
2120 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2121 iaq->intr_fwd = 1;
2122 goto allocate;
2123 }
2124
2125 /*
2126 * Next best option: an interrupt vector for errors, one for the
2127 * firmware event queue, and at least one per port. At this
2128 * point we know we'll have to downsize nrxq or nofldrxq to fit
2129 * what's available to us.
2130 */
2131 iaq->nirq = T4_EXTRA_INTR;
2132 iaq->nirq += n10g + n1g;
2133 if (iaq->nirq <= navail) {
2134 int leftover = navail - iaq->nirq;
2135
2136 if (n10g > 0) {
2137 int target = max(nrxq10g, nofldrxq10g);
2138
2139 n = 1;
2140 while (n < target && leftover >= n10g) {
2141 leftover -= n10g;
2142 iaq->nirq += n10g;
2143 n++;
2144 }
2145 iaq->nrxq10g = min(n, nrxq10g);
2146 #ifdef TCP_OFFLOAD_ENABLE
2147 iaq->nofldrxq10g = min(n, nofldrxq10g);
2148 #endif
2149 }
2150
2151 if (n1g > 0) {
2152 int target = max(nrxq1g, nofldrxq1g);
2153
2154 n = 1;
2155 while (n < target && leftover >= n1g) {
2156 leftover -= n1g;
2157 iaq->nirq += n1g;
2158 n++;
2159 }
2160 iaq->nrxq1g = min(n, nrxq1g);
2161 #ifdef TCP_OFFLOAD_ENABLE
2162 iaq->nofldrxq1g = min(n, nofldrxq1g);
2163 #endif
2164 }
2165
2166 /* We have arrived at a minimum value required to enable
2167 * per queue irq(either NIC or offload). Thus for non-
2168 * offload case, we will get a vector per queue, while
2169 * offload case, we will get a vector per offload/NIC q.
2170 * Hence enable Interrupt forwarding only for offload
2171 * case.
2172 */
2173 #ifdef TCP_OFFLOAD_ENABLE
2174 if (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq)) {
2175 iaq->intr_fwd = 1;
2176 #else
2177 if (itype != DDI_INTR_TYPE_MSI) {
2178 #endif
2179 goto allocate;
2180 }
2181 }
2182
2183 /*
2184 * Least desirable option: one interrupt vector for everything.
2185 */
2186 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2187 #ifdef TCP_OFFLOAD_ENABLE
2188 iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2189 #endif
2190 iaq->intr_fwd = 1;
2191
2192 allocate:
2193 return (0);
2194 }
2195
2196 cxgb_printf(sc->dip, CE_WARN,
2197 "failed to find a usable interrupt type. supported=%d, allowed=%d",
2198 itypes, p->intr_types);
2199 return (DDI_FAILURE);
2200 }
2201
2202 static int
2203 add_child_node(struct adapter *sc, int idx)
2204 {
2205 int rc;
2206 struct port_info *pi;
2207
2208 if (idx < 0 || idx >= sc->params.nports)
2209 return (EINVAL);
2210
2211 pi = sc->port[idx];
2212 if (pi == NULL)
2213 return (ENODEV); /* t4_port_init failed earlier */
2214
2215 PORT_LOCK(pi);
2216 if (pi->dip != NULL) {
2217 rc = 0; /* EEXIST really, but then bus_config fails */
2218 goto done;
2219 }
2220
2221 rc = ndi_devi_alloc(sc->dip, T4_PORT_NAME, DEVI_SID_NODEID, &pi->dip);
2222 if (rc != DDI_SUCCESS || pi->dip == NULL) {
2223 rc = ENOMEM;
2224 goto done;
2225 }
2226
2227 (void) ddi_set_parent_data(pi->dip, pi);
2228 (void) ndi_devi_bind_driver(pi->dip, 0);
2229 rc = 0;
2230 done:
2231 PORT_UNLOCK(pi);
2232 return (rc);
2233 }
2234
2235 static int
2236 remove_child_node(struct adapter *sc, int idx)
2237 {
2238 int rc;
2239 struct port_info *pi;
2240
2241 if (idx < 0 || idx >= sc->params.nports)
2242 return (EINVAL);
2243
2244 pi = sc->port[idx];
2245 if (pi == NULL)
2246 return (ENODEV);
2247
2248 PORT_LOCK(pi);
2249 if (pi->dip == NULL) {
2250 rc = ENODEV;
2251 goto done;
2252 }
2253
2254 rc = ndi_devi_free(pi->dip);
2255 if (rc == 0)
2256 pi->dip = NULL;
2257 done:
2258 PORT_UNLOCK(pi);
2259 return (rc);
2260 }
2261
2262 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
2263 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
2264 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
2265 #define KS_C_SET(x, ...) \
2266 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
2267
2268 /*
2269 * t4nex:X:config
2270 */
2271 struct t4_kstats {
2272 kstat_named_t chip_ver;
2273 kstat_named_t fw_vers;
2274 kstat_named_t tp_vers;
2275 kstat_named_t driver_version;
2276 kstat_named_t serial_number;
2277 kstat_named_t ec_level;
2278 kstat_named_t id;
2279 kstat_named_t bus_type;
2280 kstat_named_t bus_width;
2281 kstat_named_t bus_speed;
2282 kstat_named_t core_clock;
2283 kstat_named_t port_cnt;
2284 kstat_named_t port_type;
2285 kstat_named_t pci_vendor_id;
2286 kstat_named_t pci_device_id;
2287 };
2288 static kstat_t *
2289 setup_kstats(struct adapter *sc)
2290 {
2291 kstat_t *ksp;
2292 struct t4_kstats *kstatp;
2293 int ndata;
2294 struct pci_params *p = &sc->params.pci;
2295 struct vpd_params *v = &sc->params.vpd;
2296 uint16_t pci_vendor, pci_device;
2297
2298 ndata = sizeof (struct t4_kstats) / sizeof (kstat_named_t);
2299
2300 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "config",
2301 "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2302 if (ksp == NULL) {
2303 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2304 return (NULL);
2305 }
2306
2307 kstatp = (struct t4_kstats *)ksp->ks_data;
2308
2309 KS_UINIT(chip_ver);
2310 KS_CINIT(fw_vers);
2311 KS_CINIT(tp_vers);
2312 KS_CINIT(driver_version);
2313 KS_CINIT(serial_number);
2314 KS_CINIT(ec_level);
2315 KS_CINIT(id);
2316 KS_CINIT(bus_type);
2317 KS_CINIT(bus_width);
2318 KS_CINIT(bus_speed);
2319 KS_UINIT(core_clock);
2320 KS_UINIT(port_cnt);
2321 KS_CINIT(port_type);
2322 KS_CINIT(pci_vendor_id);
2323 KS_CINIT(pci_device_id);
2324
2325 KS_U_SET(chip_ver, sc->params.chip);
2326 KS_C_SET(fw_vers, "%d.%d.%d.%d",
2327 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2328 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2329 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2330 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2331 KS_C_SET(tp_vers, "%d.%d.%d.%d",
2332 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
2333 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
2334 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
2335 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
2336 KS_C_SET(driver_version, DRV_VERSION);
2337 KS_C_SET(serial_number, "%s", v->sn);
2338 KS_C_SET(ec_level, "%s", v->ec);
2339 KS_C_SET(id, "%s", v->id);
2340 KS_C_SET(bus_type, "pci-express");
2341 KS_C_SET(bus_width, "x%d lanes", p->width);
2342 KS_C_SET(bus_speed, "%d", p->speed);
2343 KS_U_SET(core_clock, v->cclk);
2344 KS_U_SET(port_cnt, sc->params.nports);
2345
2346 t4_os_pci_read_cfg2(sc, PCI_CONF_VENID, &pci_vendor);
2347 KS_C_SET(pci_vendor_id, "0x%x", pci_vendor);
2348
2349 t4_os_pci_read_cfg2(sc, PCI_CONF_DEVID, &pci_device);
2350 KS_C_SET(pci_device_id, "0x%x", pci_device);
2351
2352 KS_C_SET(port_type, "%s/%s/%s/%s",
2353 print_port_speed(sc->port[0]),
2354 print_port_speed(sc->port[1]),
2355 print_port_speed(sc->port[2]),
2356 print_port_speed(sc->port[3]));
2357
2358 /* Do NOT set ksp->ks_update. These kstats do not change. */
2359
2360 /* Install the kstat */
2361 ksp->ks_private = (void *)sc;
2362 kstat_install(ksp);
2363
2364 return (ksp);
2365 }
2366
2367 /*
2368 * t4nex:X:stat
2369 */
2370 struct t4_wc_kstats {
2371 kstat_named_t write_coal_success;
2372 kstat_named_t write_coal_failure;
2373 };
2374 static kstat_t *
2375 setup_wc_kstats(struct adapter *sc)
2376 {
2377 kstat_t *ksp;
2378 struct t4_wc_kstats *kstatp;
2379 int ndata;
2380
2381 ndata = sizeof(struct t4_wc_kstats) / sizeof(kstat_named_t);
2382 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "stats",
2383 "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2384 if (ksp == NULL) {
2385 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2386 return (NULL);
2387 }
2388
2389 kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2390
2391 KS_UINIT(write_coal_success);
2392 KS_UINIT(write_coal_failure);
2393
2394 ksp->ks_update = update_wc_kstats;
2395 /* Install the kstat */
2396 ksp->ks_private = (void *)sc;
2397 kstat_install(ksp);
2398
2399 return (ksp);
2400 }
2401
2402 static int
2403 update_wc_kstats(kstat_t *ksp, int rw)
2404 {
2405 struct t4_wc_kstats *kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2406 struct adapter *sc = ksp->ks_private;
2407 uint32_t wc_total, wc_success, wc_failure;
2408
2409 if (rw == KSTAT_WRITE)
2410 return (0);
2411
2412 if (is_t5(sc->params.chip)) {
2413 wc_total = t4_read_reg(sc, A_SGE_STAT_TOTAL);
2414 wc_failure = t4_read_reg(sc, A_SGE_STAT_MATCH);
2415 wc_success = wc_total - wc_failure;
2416 } else {
2417 wc_success = 0;
2418 wc_failure = 0;
2419 }
2420
2421 KS_U_SET(write_coal_success, wc_success);
2422 KS_U_SET(write_coal_failure, wc_failure);
2423
2424 return (0);
2425 }
2426
2427 int
2428 adapter_full_init(struct adapter *sc)
2429 {
2430 int i, rc = 0;
2431
2432 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2433
2434 rc = t4_setup_adapter_queues(sc);
2435 if (rc != 0)
2436 goto done;
2437
2438 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2439 (void) ddi_intr_block_enable(sc->intr_handle, sc->intr_count);
2440 else {
2441 for (i = 0; i < sc->intr_count; i++)
2442 (void) ddi_intr_enable(sc->intr_handle[i]);
2443 }
2444 t4_intr_enable(sc);
2445 sc->flags |= FULL_INIT_DONE;
2446
2447 #ifdef TCP_OFFLOAD_ENABLE
2448 /* TODO: wrong place to enable TOE capability */
2449 if (is_offload(sc) != 0) {
2450 for_each_port(sc, i) {
2451 struct port_info *pi = sc->port[i];
2452 rc = toe_capability(pi, 1);
2453 if (rc != 0) {
2454 cxgb_printf(pi->dip, CE_WARN,
2455 "Failed to activate toe capability: %d",
2456 rc);
2457 rc = 0; /* not a fatal error */
2458 }
2459 }
2460 }
2461 #endif
2462
2463 done:
2464 if (rc != 0)
2465 (void) adapter_full_uninit(sc);
2466
2467 return (rc);
2468 }
2469
2470 int
2471 adapter_full_uninit(struct adapter *sc)
2472 {
2473 int i, rc = 0;
2474
2475 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2476
2477 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2478 (void) ddi_intr_block_disable(sc->intr_handle, sc->intr_count);
2479 else {
2480 for (i = 0; i < sc->intr_count; i++)
2481 (void) ddi_intr_disable(sc->intr_handle[i]);
2482 }
2483
2484 rc = t4_teardown_adapter_queues(sc);
2485 if (rc != 0)
2486 return (rc);
2487
2488 sc->flags &= ~FULL_INIT_DONE;
2489
2490 return (0);
2491 }
2492
2493 int
2494 port_full_init(struct port_info *pi)
2495 {
2496 struct adapter *sc = pi->adapter;
2497 uint16_t *rss;
2498 struct sge_rxq *rxq;
2499 int rc, i;
2500
2501 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2502 ASSERT((pi->flags & PORT_INIT_DONE) == 0);
2503
2504 /*
2505 * Allocate tx/rx/fl queues for this port.
2506 */
2507 rc = t4_setup_port_queues(pi);
2508 if (rc != 0)
2509 goto done; /* error message displayed already */
2510
2511 /*
2512 * Setup RSS for this port.
2513 */
2514 rss = kmem_zalloc(pi->nrxq * sizeof (*rss), KM_SLEEP);
2515 for_each_rxq(pi, i, rxq) {
2516 rss[i] = rxq->iq.abs_id;
2517 }
2518 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2519 pi->rss_size, rss, pi->nrxq);
2520 kmem_free(rss, pi->nrxq * sizeof (*rss));
2521 if (rc != 0) {
2522 cxgb_printf(pi->dip, CE_WARN, "rss_config failed: %d", rc);
2523 goto done;
2524 }
2525
2526 pi->flags |= PORT_INIT_DONE;
2527 done:
2528 if (rc != 0)
2529 (void) port_full_uninit(pi);
2530
2531 return (rc);
2532 }
2533
2534 /*
2535 * Idempotent.
2536 */
2537 int
2538 port_full_uninit(struct port_info *pi)
2539 {
2540
2541 ASSERT(pi->flags & PORT_INIT_DONE);
2542
2543 (void) t4_teardown_port_queues(pi);
2544 pi->flags &= ~PORT_INIT_DONE;
2545
2546 return (0);
2547 }
2548
2549 void
2550 enable_port_queues(struct port_info *pi)
2551 {
2552 struct adapter *sc = pi->adapter;
2553 int i;
2554 struct sge_iq *iq;
2555 struct sge_rxq *rxq;
2556 #ifdef TCP_OFFLOAD_ENABLE
2557 struct sge_ofld_rxq *ofld_rxq;
2558 #endif
2559
2560 ASSERT(pi->flags & PORT_INIT_DONE);
2561
2562 /*
2563 * TODO: whatever was queued up after we set iq->state to IQS_DISABLED
2564 * back in disable_port_queues will be processed now, after an unbounded
2565 * delay. This can't be good.
2566 */
2567
2568 #ifdef TCP_OFFLOAD_ENABLE
2569 for_each_ofld_rxq(pi, i, ofld_rxq) {
2570 iq = &ofld_rxq->iq;
2571 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2572 IQS_DISABLED)
2573 panic("%s: iq %p wasn't disabled", __func__,
2574 (void *)iq);
2575 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2576 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2577 }
2578 #endif
2579
2580 for_each_rxq(pi, i, rxq) {
2581 iq = &rxq->iq;
2582 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2583 IQS_DISABLED)
2584 panic("%s: iq %p wasn't disabled", __func__,
2585 (void *) iq);
2586 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2587 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2588 }
2589 }
2590
2591 void
2592 disable_port_queues(struct port_info *pi)
2593 {
2594 int i;
2595 struct adapter *sc = pi->adapter;
2596 struct sge_rxq *rxq;
2597 #ifdef TCP_OFFLOAD_ENABLE
2598 struct sge_ofld_rxq *ofld_rxq;
2599 #endif
2600
2601 ASSERT(pi->flags & PORT_INIT_DONE);
2602
2603 /*
2604 * TODO: need proper implementation for all tx queues (ctrl, eth, ofld).
2605 */
2606
2607 #ifdef TCP_OFFLOAD_ENABLE
2608 for_each_ofld_rxq(pi, i, ofld_rxq) {
2609 while (atomic_cas_uint(&ofld_rxq->iq.state, IQS_IDLE,
2610 IQS_DISABLED) != IQS_IDLE)
2611 msleep(1);
2612 }
2613 #endif
2614
2615 for_each_rxq(pi, i, rxq) {
2616 while (atomic_cas_uint(&rxq->iq.state, IQS_IDLE,
2617 IQS_DISABLED) != IQS_IDLE)
2618 msleep(1);
2619 }
2620
2621 mutex_enter(&sc->sfl_lock);
2622 #ifdef TCP_OFFLOAD_ENABLE
2623 for_each_ofld_rxq(pi, i, ofld_rxq)
2624 ofld_rxq->fl.flags |= FL_DOOMED;
2625 #endif
2626 for_each_rxq(pi, i, rxq)
2627 rxq->fl.flags |= FL_DOOMED;
2628 mutex_exit(&sc->sfl_lock);
2629 /* TODO: need to wait for all fl's to be removed from sc->sfl */
2630 }
2631
2632 void
2633 t4_fatal_err(struct adapter *sc)
2634 {
2635 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
2636 t4_intr_disable(sc);
2637 cxgb_printf(sc->dip, CE_WARN,
2638 "encountered fatal error, adapter stopped.");
2639 }
2640
2641 int
2642 t4_os_find_pci_capability(struct adapter *sc, int cap)
2643 {
2644 uint16_t stat;
2645 uint8_t cap_ptr, cap_id;
2646
2647 t4_os_pci_read_cfg2(sc, PCI_CONF_STAT, &stat);
2648 if ((stat & PCI_STAT_CAP) == 0)
2649 return (0); /* does not implement capabilities */
2650
2651 t4_os_pci_read_cfg1(sc, PCI_CONF_CAP_PTR, &cap_ptr);
2652 while (cap_ptr) {
2653 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_ID, &cap_id);
2654 if (cap_id == cap)
2655 return (cap_ptr); /* found */
2656 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_NEXT_PTR, &cap_ptr);
2657 }
2658
2659 return (0); /* not found */
2660 }
2661
2662 void
2663 t4_os_portmod_changed(const struct adapter *sc, int idx)
2664 {
2665 static const char *mod_str[] = {
2666 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
2667 };
2668 const struct port_info *pi = sc->port[idx];
2669
2670 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
2671 cxgb_printf(pi->dip, CE_NOTE, "transceiver unplugged.");
2672 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
2673 cxgb_printf(pi->dip, CE_NOTE,
2674 "unknown transceiver inserted.\n");
2675 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
2676 cxgb_printf(pi->dip, CE_NOTE,
2677 "unsupported transceiver inserted.\n");
2678 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
2679 cxgb_printf(pi->dip, CE_NOTE, "%s transceiver inserted.\n",
2680 mod_str[pi->mod_type]);
2681 else
2682 cxgb_printf(pi->dip, CE_NOTE, "transceiver (type %d) inserted.",
2683 pi->mod_type);
2684 }
2685
2686 /* ARGSUSED */
2687 static int
2688 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
2689 {
2690 if (m != NULL)
2691 freemsg(m);
2692 return (0);
2693 }
2694
2695 int
2696 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
2697 {
2698 uint_t *loc, new;
2699
2700 if (opcode >= ARRAY_SIZE(sc->cpl_handler))
2701 return (EINVAL);
2702
2703 new = (uint_t)(unsigned long) (h ? h : cpl_not_handled);
2704 loc = (uint_t *)&sc->cpl_handler[opcode];
2705 (void) atomic_swap_uint(loc, new);
2706
2707 return (0);
2708 }
2709
2710 static int
2711 fw_msg_not_handled(struct adapter *sc, const __be64 *data)
2712 {
2713 struct cpl_fw6_msg *cpl = container_of(data, struct cpl_fw6_msg, data);
2714
2715 cxgb_printf(sc->dip, CE_WARN, "%s fw_msg type %d", __func__, cpl->type);
2716 return (0);
2717 }
2718
2719 int
2720 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
2721 {
2722 fw_msg_handler_t *loc, new;
2723
2724 if (type >= ARRAY_SIZE(sc->fw_msg_handler))
2725 return (EINVAL);
2726
2727 /*
2728 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
2729 * handler dispatch table. Reject any attempt to install a handler for
2730 * this subtype.
2731 */
2732 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
2733 return (EINVAL);
2734
2735 new = h ? h : fw_msg_not_handled;
2736 loc = &sc->fw_msg_handler[type];
2737 (void)atomic_swap_ptr(loc, (void *)new);
2738
2739 return (0);
2740 }
2741
2742 #ifdef TCP_OFFLOAD_ENABLE
2743 static int
2744 toe_capability(struct port_info *pi, int enable)
2745 {
2746 int rc;
2747 struct adapter *sc = pi->adapter;
2748
2749 if (!is_offload(sc))
2750 return (ENODEV);
2751
2752 if (enable != 0) {
2753 if (isset(&sc->offload_map, pi->port_id) != 0)
2754 return (0);
2755
2756 if (sc->offload_map == 0) {
2757 rc = activate_uld(sc, ULD_TOM, &sc->tom);
2758 if (rc != 0)
2759 return (rc);
2760 }
2761
2762 setbit(&sc->offload_map, pi->port_id);
2763 } else {
2764 if (!isset(&sc->offload_map, pi->port_id))
2765 return (0);
2766
2767 clrbit(&sc->offload_map, pi->port_id);
2768
2769 if (sc->offload_map == 0) {
2770 rc = deactivate_uld(&sc->tom);
2771 if (rc != 0) {
2772 setbit(&sc->offload_map, pi->port_id);
2773 return (rc);
2774 }
2775 }
2776 }
2777
2778 return (0);
2779 }
2780
2781 /*
2782 * Add an upper layer driver to the global list.
2783 */
2784 int
2785 t4_register_uld(struct uld_info *ui)
2786 {
2787 int rc = 0;
2788 struct uld_info *u;
2789
2790 mutex_enter(&t4_uld_list_lock);
2791 SLIST_FOREACH(u, &t4_uld_list, link) {
2792 if (u->uld_id == ui->uld_id) {
2793 rc = EEXIST;
2794 goto done;
2795 }
2796 }
2797
2798 SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
2799 ui->refcount = 0;
2800 done:
2801 mutex_exit(&t4_uld_list_lock);
2802 return (rc);
2803 }
2804
2805 int
2806 t4_unregister_uld(struct uld_info *ui)
2807 {
2808 int rc = EINVAL;
2809 struct uld_info *u;
2810
2811 mutex_enter(&t4_uld_list_lock);
2812
2813 SLIST_FOREACH(u, &t4_uld_list, link) {
2814 if (u == ui) {
2815 if (ui->refcount > 0) {
2816 rc = EBUSY;
2817 goto done;
2818 }
2819
2820 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
2821 rc = 0;
2822 goto done;
2823 }
2824 }
2825 done:
2826 mutex_exit(&t4_uld_list_lock);
2827 return (rc);
2828 }
2829
2830 static int
2831 activate_uld(struct adapter *sc, int id, struct uld_softc *usc)
2832 {
2833 int rc = EAGAIN;
2834 struct uld_info *ui;
2835
2836 mutex_enter(&t4_uld_list_lock);
2837
2838 SLIST_FOREACH(ui, &t4_uld_list, link) {
2839 if (ui->uld_id == id) {
2840 rc = ui->attach(sc, &usc->softc);
2841 if (rc == 0) {
2842 ASSERT(usc->softc != NULL);
2843 ui->refcount++;
2844 usc->uld = ui;
2845 }
2846 goto done;
2847 }
2848 }
2849 done:
2850 mutex_exit(&t4_uld_list_lock);
2851
2852 return (rc);
2853 }
2854
2855 static int
2856 deactivate_uld(struct uld_softc *usc)
2857 {
2858 int rc;
2859
2860 mutex_enter(&t4_uld_list_lock);
2861
2862 if (usc->uld == NULL || usc->softc == NULL) {
2863 rc = EINVAL;
2864 goto done;
2865 }
2866
2867 rc = usc->uld->detach(usc->softc);
2868 if (rc == 0) {
2869 ASSERT(usc->uld->refcount > 0);
2870 usc->uld->refcount--;
2871 usc->uld = NULL;
2872 usc->softc = NULL;
2873 }
2874 done:
2875 mutex_exit(&t4_uld_list_lock);
2876
2877 return (rc);
2878 }
2879
2880 void
2881 t4_iterate(void (*func)(int, void *), void *arg)
2882 {
2883 struct adapter *sc;
2884
2885 mutex_enter(&t4_adapter_list_lock);
2886 SLIST_FOREACH(sc, &t4_adapter_list, link) {
2887 /*
2888 * func should not make any assumptions about what state sc is
2889 * in - the only guarantee is that sc->sc_lock is a valid lock.
2890 */
2891 func(ddi_get_instance(sc->dip), arg);
2892 }
2893 mutex_exit(&t4_adapter_list_lock);
2894 }
2895
2896 #endif