Print this page
MFV: illumos-gate@bbb9d5d65bf8372aae4b8821c80e218b8b832846
9994 cxgbe t4nex: Handle get_fl_payload() alloc failures
9995 cxgbe t4_devo_attach() should initialize ->sfl
Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Garrett D'Amore <garrett@damore.org>
Approved by: Dan McDonald <danmcd@joyent.com>
Author: John Levon <john.levon@joyent.com>
9484 cxgbe should clean TX descriptors in timely manner
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Ryan Zezeski <rpz@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Toomas Soome <tsoome@me.com>
Approved by: Dan McDonald <danmcd@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/cxgbe/t4nex/t4_sge.c
+++ new/usr/src/uts/common/io/cxgbe/t4nex/t4_sge.c
1 1 /*
2 2 * This file and its contents are supplied under the terms of the
3 3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 4 * You may only use this file in accordance with the terms of version
5 5 * 1.0 of the CDDL.
6 6 *
7 7 * A full copy of the text of the CDDL should have accompanied this
8 8 * source. A copy of the CDDL is also available via the Internet at
9 9 * http://www.illumos.org/license/CDDL.
10 10 */
11 11
12 12 /*
13 13 * This file is part of the Chelsio T4 support code.
14 14 *
15 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 16 *
17 17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 20 * release for licensing terms and conditions.
21 21 */
22 22
23 23 #include <sys/ddi.h>
24 24 #include <sys/sunddi.h>
25 25 #include <sys/sunndi.h>
26 26 #include <sys/atomic.h>
27 27 #include <sys/dlpi.h>
28 28 #include <sys/pattr.h>
29 29 #include <sys/strsubr.h>
30 30 #include <sys/stream.h>
31 31 #include <sys/strsun.h>
32 32 #include <inet/ip.h>
33 33 #include <inet/tcp.h>
34 34
35 35 #include "version.h"
36 36 #include "common/common.h"
37 37 #include "common/t4_msg.h"
38 38 #include "common/t4_regs.h"
39 39 #include "common/t4_regs_values.h"
40 40
41 41 /* TODO: Tune. */
42 42 int rx_buf_size = 8192;
43 43 int tx_copy_threshold = 256;
44 44 uint16_t rx_copy_threshold = 256;
45 45
46 46 /* Used to track coalesced tx work request */
47 47 struct txpkts {
48 48 mblk_t *tail; /* head is in the software descriptor */
49 49 uint64_t *flitp; /* ptr to flit where next pkt should start */
50 50 uint8_t npkt; /* # of packets in this work request */
51 51 uint8_t nflits; /* # of flits used by this work request */
52 52 uint16_t plen; /* total payload (sum of all packets) */
53 53 };
54 54
55 55 /* All information needed to tx a frame */
56 56 struct txinfo {
57 57 uint32_t len; /* Total length of frame */
58 58 uint32_t flags; /* Checksum and LSO flags */
59 59 uint32_t mss; /* MSS for LSO */
60 60 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */
61 61 uint8_t nflits; /* # of flits needed for the SGL */
62 62 uint8_t hdls_used; /* # of DMA handles used */
63 63 uint32_t txb_used; /* txb_space used */
64 64 struct ulptx_sgl sgl __attribute__((aligned(8)));
65 65 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
66 66 };
67 67
68 68 static int service_iq(struct sge_iq *iq, int budget);
69 69 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
70 70 int8_t pktc_idx, int qsize, uint8_t esize);
71 71 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
72 72 static inline void init_eq(struct adapter *sc, struct sge_eq *eq,
73 73 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid);
74 74 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
75 75 struct sge_fl *fl, int intr_idx, int cong);
76 76 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
77 77 struct sge_fl *fl);
78 78 static int alloc_fwq(struct adapter *sc);
79 79 static int free_fwq(struct adapter *sc);
80 80 #ifdef TCP_OFFLOAD_ENABLE
81 81 static int alloc_mgmtq(struct adapter *sc);
82 82 #endif
83 83 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
84 84 int i);
85 85 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
86 86 #ifdef TCP_OFFLOAD_ENABLE
87 87 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
88 88 int intr_idx);
89 89 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq);
90 90 #endif
91 91 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq);
92 92 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
93 93 struct sge_eq *eq);
94 94 #ifdef TCP_OFFLOAD_ENABLE
95 95 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi,
96 96 struct sge_eq *eq);
97 97 #endif
98 98 static int alloc_eq(struct adapter *sc, struct port_info *pi,
99 99 struct sge_eq *eq);
100 100 static int free_eq(struct adapter *sc, struct sge_eq *eq);
101 101 #ifdef TCP_OFFLOAD_ENABLE
102 102 static int alloc_wrq(struct adapter *sc, struct port_info *pi,
103 103 struct sge_wrq *wrq, int idx);
104 104 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq);
105 105 #endif
106 106 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
107 107 static int free_txq(struct port_info *pi, struct sge_txq *txq);
108 108 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
109 109 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
110 110 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
111 111 caddr_t *pva);
112 112 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
113 113 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
114 114 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
115 115 caddr_t *pva);
116 116 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
117 117 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
118 118 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
119 119 caddr_t *pva);
120 120 static inline bool is_new_response(const struct sge_iq *iq,
121 121 struct rsp_ctrl **ctrl);
122 122 static inline void iq_next(struct sge_iq *iq);
123 123 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
124 124 static void refill_sfl(void *arg);
125 125 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
126 126 static void free_fl_bufs(struct sge_fl *fl);
127 127 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl,
128 128 uint32_t len_newbuf, int *fl_bufs_used);
129 129 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
130 130 struct txinfo *txinfo, int sgl_only);
131 131 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
132 132 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
133 133 struct txinfo *txinfo);
134 134 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
135 135 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
136 136 mblk_t *m, int len);
137 137 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
138 138 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
139 139 struct txinfo *txinfo);
140 140 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
141 141 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
142 142 struct txinfo *txinfo);
143 143 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
144 144 struct txpkts *txpkts, struct txinfo *txinfo);
145 145 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
146 146 int len);
147 147 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
148 148 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
149 149 static void write_txqflush_wr(struct sge_txq *txq);
150 150 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
151 151 mblk_t *m);
152 152 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
153 153 static kstat_t *setup_port_config_kstats(struct port_info *pi);
154 154 static kstat_t *setup_port_info_kstats(struct port_info *pi);
155 155 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
156 156 int idx);
157 157 static int update_rxq_kstats(kstat_t *ksp, int rw);
158 158 static int update_port_info_kstats(kstat_t *ksp, int rw);
159 159 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
160 160 int idx);
161 161 static int update_txq_kstats(kstat_t *ksp, int rw);
162 162 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
163 163 mblk_t *);
164 164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss,
165 165 mblk_t *m);
166 166
167 167 static inline int
168 168 reclaimable(struct sge_eq *eq)
169 169 {
170 170 unsigned int cidx;
171 171
172 172 cidx = eq->spg->cidx; /* stable snapshot */
173 173 cidx = be16_to_cpu(cidx);
174 174
175 175 if (cidx >= eq->cidx)
176 176 return (cidx - eq->cidx);
177 177 else
178 178 return (cidx + eq->cap - eq->cidx);
179 179 }
180 180
181 181 void
182 182 t4_sge_init(struct adapter *sc)
183 183 {
184 184 struct driver_properties *p = &sc->props;
185 185 ddi_dma_attr_t *dma_attr;
186 186 ddi_device_acc_attr_t *acc_attr;
187 187 uint32_t sge_control, sge_conm_ctrl;
188 188 int egress_threshold;
189 189
190 190 /*
191 191 * Device access and DMA attributes for descriptor rings
192 192 */
193 193 acc_attr = &sc->sge.acc_attr_desc;
194 194 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
195 195 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
196 196 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
197 197
198 198 dma_attr = &sc->sge.dma_attr_desc;
199 199 dma_attr->dma_attr_version = DMA_ATTR_V0;
200 200 dma_attr->dma_attr_addr_lo = 0;
201 201 dma_attr->dma_attr_addr_hi = UINT64_MAX;
202 202 dma_attr->dma_attr_count_max = UINT64_MAX;
203 203 dma_attr->dma_attr_align = 512;
204 204 dma_attr->dma_attr_burstsizes = 0xfff;
205 205 dma_attr->dma_attr_minxfer = 1;
206 206 dma_attr->dma_attr_maxxfer = UINT64_MAX;
207 207 dma_attr->dma_attr_seg = UINT64_MAX;
208 208 dma_attr->dma_attr_sgllen = 1;
209 209 dma_attr->dma_attr_granular = 1;
210 210 dma_attr->dma_attr_flags = 0;
211 211
212 212 /*
213 213 * Device access and DMA attributes for tx buffers
214 214 */
215 215 acc_attr = &sc->sge.acc_attr_tx;
216 216 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
217 217 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
218 218
219 219 dma_attr = &sc->sge.dma_attr_tx;
220 220 dma_attr->dma_attr_version = DMA_ATTR_V0;
221 221 dma_attr->dma_attr_addr_lo = 0;
222 222 dma_attr->dma_attr_addr_hi = UINT64_MAX;
223 223 dma_attr->dma_attr_count_max = UINT64_MAX;
224 224 dma_attr->dma_attr_align = 1;
225 225 dma_attr->dma_attr_burstsizes = 0xfff;
226 226 dma_attr->dma_attr_minxfer = 1;
227 227 dma_attr->dma_attr_maxxfer = UINT64_MAX;
228 228 dma_attr->dma_attr_seg = UINT64_MAX;
229 229 dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
230 230 dma_attr->dma_attr_granular = 1;
231 231 dma_attr->dma_attr_flags = 0;
232 232
233 233 /*
234 234 * Ingress Padding Boundary and Egress Status Page Size are set up by
235 235 * t4_fixup_host_params().
236 236 */
237 237 sge_control = t4_read_reg(sc, A_SGE_CONTROL);
238 238 sc->sge.pktshift = G_PKTSHIFT(sge_control);
239 239 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
240 240
241 241 /* t4_nex uses FLM packed mode */
242 242 sc->sge.fl_align = t4_fl_pkt_align(sc, true);
243 243
244 244 /*
245 245 * Device access and DMA attributes for rx buffers
246 246 */
247 247 sc->sge.rxb_params.dip = sc->dip;
248 248 sc->sge.rxb_params.buf_size = rx_buf_size;
249 249
250 250 acc_attr = &sc->sge.rxb_params.acc_attr_rx;
251 251 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
252 252 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
253 253
254 254 dma_attr = &sc->sge.rxb_params.dma_attr_rx;
255 255 dma_attr->dma_attr_version = DMA_ATTR_V0;
256 256 dma_attr->dma_attr_addr_lo = 0;
257 257 dma_attr->dma_attr_addr_hi = UINT64_MAX;
258 258 dma_attr->dma_attr_count_max = UINT64_MAX;
259 259 /*
260 260 * Low 4 bits of an rx buffer address have a special meaning to the SGE
261 261 * and an rx buf cannot have an address with any of these bits set.
262 262 * FL_ALIGN is >= 32 so we're sure things are ok.
263 263 */
264 264 dma_attr->dma_attr_align = sc->sge.fl_align;
265 265 dma_attr->dma_attr_burstsizes = 0xfff;
266 266 dma_attr->dma_attr_minxfer = 1;
267 267 dma_attr->dma_attr_maxxfer = UINT64_MAX;
268 268 dma_attr->dma_attr_seg = UINT64_MAX;
269 269 dma_attr->dma_attr_sgllen = 1;
270 270 dma_attr->dma_attr_granular = 1;
271 271 dma_attr->dma_attr_flags = 0;
272 272
273 273 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
274 274
275 275 /*
276 276 * A FL with <= fl_starve_thres buffers is starving and a periodic
277 277 * timer will attempt to refill it. This needs to be larger than the
278 278 * SGE's Egress Congestion Threshold. If it isn't, then we can get
279 279 * stuck waiting for new packets while the SGE is waiting for us to
280 280 * give it more Free List entries. (Note that the SGE's Egress
281 281 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
282 282 * there was only a single field to control this. For T5 there's the
283 283 * original field which now only applies to Unpacked Mode Free List
284 284 * buffers and a new field which only applies to Packed Mode Free List
285 285 * buffers.
286 286 */
287 287
288 288 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL);
289 289 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
290 290 case CHELSIO_T4:
291 291 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
292 292 break;
293 293 case CHELSIO_T5:
294 294 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
295 295 break;
296 296 case CHELSIO_T6:
297 297 default:
298 298 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl);
299 299 }
300 300 sc->sge.fl_starve_threshold = 2*egress_threshold + 1;
301 301
302 302 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
303 303
304 304 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
305 305 V_THRESHOLD_0(p->counter_val[0]) |
306 306 V_THRESHOLD_1(p->counter_val[1]) |
307 307 V_THRESHOLD_2(p->counter_val[2]) |
308 308 V_THRESHOLD_3(p->counter_val[3]));
309 309
310 310 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
311 311 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
312 312 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
313 313 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
314 314 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
315 315 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
316 316 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
317 317 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
318 318 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
319 319
320 320 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
321 321 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
322 322 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
323 323 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
324 324 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL,
325 325 t4_handle_fw_rpl);
326 326 }
327 327
328 328 /*
329 329 * Allocate and initialize the firmware event queue and the forwarded interrupt
330 330 * queues, if any. The adapter owns all these queues as they are not associated
331 331 * with any particular port.
332 332 *
333 333 * Returns errno on failure. Resources allocated up to that point may still be
334 334 * allocated. Caller is responsible for cleanup in case this function fails.
335 335 */
336 336 int
337 337 t4_setup_adapter_queues(struct adapter *sc)
338 338 {
339 339 int rc;
340 340
341 341 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
342 342
343 343 /*
344 344 * Firmware event queue
345 345 */
346 346 rc = alloc_fwq(sc);
347 347 if (rc != 0)
348 348 return (rc);
349 349
350 350 #ifdef TCP_OFFLOAD_ENABLE
351 351 /*
352 352 * Management queue. This is just a control queue that uses the fwq as
353 353 * its associated iq.
354 354 */
355 355 rc = alloc_mgmtq(sc);
356 356 #endif
357 357
358 358 return (rc);
359 359 }
360 360
361 361 /*
362 362 * Idempotent
363 363 */
364 364 int
365 365 t4_teardown_adapter_queues(struct adapter *sc)
366 366 {
367 367
368 368 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
369 369
370 370 (void) free_fwq(sc);
371 371
372 372 return (0);
373 373 }
374 374
375 375 static inline int
376 376 first_vector(struct port_info *pi)
377 377 {
378 378 struct adapter *sc = pi->adapter;
379 379 int rc = T4_EXTRA_INTR, i;
380 380
381 381 if (sc->intr_count == 1)
382 382 return (0);
383 383
384 384 for_each_port(sc, i) {
385 385 struct port_info *p = sc->port[i];
386 386
387 387 if (i == pi->port_id)
388 388 break;
389 389
390 390 #ifdef TCP_OFFLOAD_ENABLE
391 391 if (!(sc->flags & INTR_FWD))
392 392 rc += p->nrxq + p->nofldrxq;
393 393 else
394 394 rc += max(p->nrxq, p->nofldrxq);
395 395 #else
396 396 /*
397 397 * Not compiled with offload support and intr_count > 1. Only
398 398 * NIC queues exist and they'd better be taking direct
399 399 * interrupts.
400 400 */
401 401 ASSERT(!(sc->flags & INTR_FWD));
402 402 rc += p->nrxq;
403 403 #endif
404 404 }
405 405 return (rc);
406 406 }
407 407
408 408 /*
409 409 * Given an arbitrary "index," come up with an iq that can be used by other
410 410 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
411 411 * The iq returned is guaranteed to be something that takes direct interrupts.
412 412 */
413 413 static struct sge_iq *
414 414 port_intr_iq(struct port_info *pi, int idx)
415 415 {
416 416 struct adapter *sc = pi->adapter;
417 417 struct sge *s = &sc->sge;
418 418 struct sge_iq *iq = NULL;
419 419
420 420 if (sc->intr_count == 1)
421 421 return (&sc->sge.fwq);
422 422
423 423 #ifdef TCP_OFFLOAD_ENABLE
424 424 if (!(sc->flags & INTR_FWD)) {
425 425 idx %= pi->nrxq + pi->nofldrxq;
426 426
427 427 if (idx >= pi->nrxq) {
428 428 idx -= pi->nrxq;
429 429 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
430 430 } else
431 431 iq = &s->rxq[pi->first_rxq + idx].iq;
432 432
433 433 } else {
434 434 idx %= max(pi->nrxq, pi->nofldrxq);
435 435
436 436 if (pi->nrxq >= pi->nofldrxq)
437 437 iq = &s->rxq[pi->first_rxq + idx].iq;
438 438 else
439 439 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
440 440 }
441 441 #else
442 442 /*
443 443 * Not compiled with offload support and intr_count > 1. Only NIC
444 444 * queues exist and they'd better be taking direct interrupts.
445 445 */
446 446 ASSERT(!(sc->flags & INTR_FWD));
447 447
448 448 idx %= pi->nrxq;
449 449 iq = &s->rxq[pi->first_rxq + idx].iq;
450 450 #endif
451 451
452 452 return (iq);
453 453 }
454 454
455 455 int
456 456 t4_setup_port_queues(struct port_info *pi)
457 457 {
458 458 int rc = 0, i, intr_idx, j;
459 459 struct sge_rxq *rxq;
460 460 struct sge_txq *txq;
461 461 #ifdef TCP_OFFLOAD_ENABLE
462 462 int iqid;
463 463 struct sge_wrq *ctrlq;
464 464 struct sge_ofld_rxq *ofld_rxq;
465 465 struct sge_wrq *ofld_txq;
466 466 #endif
467 467 struct adapter *sc = pi->adapter;
468 468 struct driver_properties *p = &sc->props;
469 469
470 470 pi->ksp_config = setup_port_config_kstats(pi);
471 471 pi->ksp_info = setup_port_info_kstats(pi);
472 472
473 473 /* Interrupt vector to start from (when using multiple vectors) */
474 474 intr_idx = first_vector(pi);
475 475
476 476 /*
477 477 * First pass over all rx queues (NIC and TOE):
478 478 * a) initialize iq and fl
479 479 * b) allocate queue iff it will take direct interrupts.
480 480 */
481 481
482 482 for_each_rxq(pi, i, rxq) {
483 483
484 484 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
485 485 RX_IQ_ESIZE);
486 486
487 487 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
488 488
489 489 if ((!(sc->flags & INTR_FWD))
490 490 #ifdef TCP_OFFLOAD_ENABLE
491 491 || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
492 492 #else
493 493 || (sc->intr_count > 1 && pi->nrxq)
494 494 #endif
495 495 ) {
496 496 rxq->iq.flags |= IQ_INTR;
497 497 rc = alloc_rxq(pi, rxq, intr_idx, i);
498 498 if (rc != 0)
499 499 goto done;
500 500 intr_idx++;
501 501 }
502 502
503 503 }
504 504
505 505 #ifdef TCP_OFFLOAD_ENABLE
506 506 for_each_ofld_rxq(pi, i, ofld_rxq) {
507 507
508 508 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
509 509 p->qsize_rxq, RX_IQ_ESIZE);
510 510
511 511 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8);
512 512
513 513 if (!(sc->flags & INTR_FWD) ||
514 514 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
515 515 ofld_rxq->iq.flags = IQ_INTR;
516 516 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
517 517 if (rc != 0)
518 518 goto done;
519 519
520 520 intr_idx++;
521 521 }
522 522 }
523 523 #endif
524 524
525 525 /*
526 526 * Second pass over all rx queues (NIC and TOE). The queues forwarding
527 527 * their interrupts are allocated now.
528 528 */
529 529 j = 0;
530 530 for_each_rxq(pi, i, rxq) {
531 531 if (rxq->iq.flags & IQ_INTR)
532 532 continue;
533 533
534 534 intr_idx = port_intr_iq(pi, j)->abs_id;
535 535
536 536 rc = alloc_rxq(pi, rxq, intr_idx, i);
537 537 if (rc != 0)
538 538 goto done;
539 539 j++;
540 540 }
541 541
542 542 #ifdef TCP_OFFLOAD_ENABLE
543 543 for_each_ofld_rxq(pi, i, ofld_rxq) {
544 544 if (ofld_rxq->iq.flags & IQ_INTR)
545 545 continue;
546 546
547 547 intr_idx = port_intr_iq(pi, j)->abs_id;
548 548 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
549 549 if (rc != 0)
550 550 goto done;
551 551 j++;
552 552 }
553 553 #endif
554 554 /*
555 555 * Now the tx queues. Only one pass needed.
556 556 */
557 557 j = 0;
558 558 for_each_txq(pi, i, txq) {
559 559 uint16_t iqid;
560 560
561 561 iqid = port_intr_iq(pi, j)->cntxt_id;
562 562 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid);
563 563 rc = alloc_txq(pi, txq, i);
564 564 if (rc != 0)
565 565 goto done;
566 566 }
567 567
568 568 #ifdef TCP_OFFLOAD_ENABLE
569 569 for_each_ofld_txq(pi, i, ofld_txq) {
570 570 uint16_t iqid;
571 571
572 572 iqid = port_intr_iq(pi, j)->cntxt_id;
573 573 init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan,
574 574 iqid);
575 575 rc = alloc_wrq(sc, pi, ofld_txq, i);
576 576 if (rc != 0)
577 577 goto done;
578 578 }
579 579
580 580 /*
581 581 * Finally, the control queue.
582 582 */
583 583 ctrlq = &sc->sge.ctrlq[pi->port_id];
584 584 iqid = port_intr_iq(pi, 0)->cntxt_id;
585 585 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid);
586 586 rc = alloc_wrq(sc, pi, ctrlq, 0);
587 587 #endif
588 588
589 589 done:
590 590 if (rc != 0)
591 591 (void) t4_teardown_port_queues(pi);
592 592
593 593 return (rc);
594 594 }
595 595
596 596 /*
597 597 * Idempotent
598 598 */
599 599 int
600 600 t4_teardown_port_queues(struct port_info *pi)
601 601 {
602 602 int i;
603 603 struct sge_rxq *rxq;
604 604 struct sge_txq *txq;
605 605 #ifdef TCP_OFFLOAD_ENABLE
606 606 struct adapter *sc = pi->adapter;
607 607 struct sge_ofld_rxq *ofld_rxq;
608 608 struct sge_wrq *ofld_txq;
609 609 #endif
610 610
611 611 if (pi->ksp_config != NULL) {
612 612 kstat_delete(pi->ksp_config);
613 613 pi->ksp_config = NULL;
614 614 }
615 615 if (pi->ksp_info != NULL) {
616 616 kstat_delete(pi->ksp_info);
617 617 pi->ksp_info = NULL;
618 618 }
619 619
620 620 #ifdef TCP_OFFLOAD_ENABLE
621 621 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
622 622 #endif
623 623
624 624 for_each_txq(pi, i, txq) {
625 625 (void) free_txq(pi, txq);
626 626 }
627 627
628 628 #ifdef TCP_OFFLOAD_ENABLE
629 629 for_each_ofld_txq(pi, i, ofld_txq) {
630 630 (void) free_wrq(sc, ofld_txq);
631 631 }
632 632
633 633 for_each_ofld_rxq(pi, i, ofld_rxq) {
634 634 if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
635 635 (void) free_ofld_rxq(pi, ofld_rxq);
636 636 }
637 637 #endif
638 638
639 639 for_each_rxq(pi, i, rxq) {
640 640 if ((rxq->iq.flags & IQ_INTR) == 0)
641 641 (void) free_rxq(pi, rxq);
642 642 }
643 643
644 644 /*
645 645 * Then take down the rx queues that take direct interrupts.
646 646 */
647 647
648 648 for_each_rxq(pi, i, rxq) {
649 649 if (rxq->iq.flags & IQ_INTR)
650 650 (void) free_rxq(pi, rxq);
651 651 }
652 652
653 653 #ifdef TCP_OFFLOAD_ENABLE
654 654 for_each_ofld_rxq(pi, i, ofld_rxq) {
655 655 if (ofld_rxq->iq.flags & IQ_INTR)
656 656 (void) free_ofld_rxq(pi, ofld_rxq);
657 657 }
658 658 #endif
659 659
660 660 return (0);
661 661 }
662 662
663 663 /* Deals with errors and forwarded interrupts */
664 664 uint_t
665 665 t4_intr_all(caddr_t arg1, caddr_t arg2)
666 666 {
667 667
668 668 (void) t4_intr_err(arg1, arg2);
669 669 (void) t4_intr(arg1, arg2);
670 670
671 671 return (DDI_INTR_CLAIMED);
672 672 }
673 673
674 674 static void
675 675 t4_intr_rx_work(struct sge_iq *iq)
676 676 {
677 677 mblk_t *mp = NULL;
678 678 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
679 679 RXQ_LOCK(rxq);
680 680 if (!iq->polling) {
681 681 mp = t4_ring_rx(rxq, iq->qsize/8);
682 682 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
683 683 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
684 684 }
685 685 RXQ_UNLOCK(rxq);
|
↓ open down ↓ |
685 lines elided |
↑ open up ↑ |
686 686 if (mp != NULL)
687 687 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
688 688 rxq->ring_gen_num);
689 689 }
690 690
691 691 /* Deals with interrupts on the given ingress queue */
692 692 /* ARGSUSED */
693 693 uint_t
694 694 t4_intr(caddr_t arg1, caddr_t arg2)
695 695 {
696 - /* LINTED: E_BAD_PTR_CAST_ALIGN */
697 696 struct sge_iq *iq = (struct sge_iq *)arg2;
698 697 int state;
699 698
700 699 /* Right now receive polling is only enabled for MSI-X and
701 700 * when we have enough msi-x vectors i.e no interrupt forwarding.
702 701 */
703 702 if (iq->adapter->props.multi_rings) {
704 703 t4_intr_rx_work(iq);
705 704 } else {
706 705 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
707 706 if (state == IQS_IDLE) {
708 707 (void) service_iq(iq, 0);
709 708 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
710 709 }
711 710 }
712 711 return (DDI_INTR_CLAIMED);
713 712 }
714 713
715 714 /* Deals with error interrupts */
716 715 /* ARGSUSED */
717 716 uint_t
718 717 t4_intr_err(caddr_t arg1, caddr_t arg2)
719 718 {
720 719 /* LINTED: E_BAD_PTR_CAST_ALIGN */
721 720 struct adapter *sc = (struct adapter *)arg1;
722 721
723 722 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
724 723 (void) t4_slow_intr_handler(sc);
725 724
726 725 return (DDI_INTR_CLAIMED);
727 726 }
728 727
729 728 /*
730 729 * t4_ring_rx - Process responses from an SGE response queue.
731 730 *
732 731 * This function processes responses from an SGE response queue up to the supplied budget.
733 732 * Responses include received packets as well as control messages from FW
734 733 * or HW.
735 734 * It returns a chain of mblks containing the received data, to be
736 735 * passed up to mac_ring_rx().
737 736 */
738 737 mblk_t *
739 738 t4_ring_rx(struct sge_rxq *rxq, int budget)
740 739 {
741 740 struct sge_iq *iq = &rxq->iq;
742 741 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
743 742 struct adapter *sc = iq->adapter;
744 743 struct rsp_ctrl *ctrl;
745 744 const struct rss_header *rss;
746 745 int ndescs = 0, fl_bufs_used = 0;
747 746 int rsp_type;
748 747 uint32_t lq;
749 748 mblk_t *mblk_head = NULL, **mblk_tail, *m;
750 749 struct cpl_rx_pkt *cpl;
751 750 uint32_t received_bytes = 0, pkt_len = 0;
752 751 bool csum_ok;
753 752 uint16_t err_vec;
754 753
755 754 mblk_tail = &mblk_head;
756 755
757 756 while (is_new_response(iq, &ctrl)) {
758 757
759 758 membar_consumer();
760 759
761 760 m = NULL;
762 761 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
763 762 lq = be32_to_cpu(ctrl->pldbuflen_qid);
764 763 rss = (const void *)iq->cdesc;
765 764
766 765 switch (rsp_type) {
767 766 case X_RSPD_TYPE_FLBUF:
768 767
|
↓ open down ↓ |
62 lines elided |
↑ open up ↑ |
769 768 ASSERT(iq->flags & IQ_HAS_FL);
770 769
771 770 if (CPL_RX_PKT == rss->opcode) {
772 771 cpl = (void *)(rss + 1);
773 772 pkt_len = be16_to_cpu(cpl->len);
774 773
775 774 if (iq->polling && ((received_bytes + pkt_len) > budget))
776 775 goto done;
777 776
778 777 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
779 - if (m == NULL) {
780 - panic("%s: line %d.", __func__,
781 - __LINE__);
782 - }
778 + if (m == NULL)
779 + goto done;
783 780
784 781 iq->intr_next = iq->intr_params;
785 782 m->b_rptr += sc->sge.pktshift;
786 783 if (sc->params.tp.rx_pkt_encap)
787 784 /* It is enabled only in T6 config file */
788 785 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
789 786 else
790 787 err_vec = ntohs(cpl->err_vec);
791 788
792 789 csum_ok = cpl->csum_calc && !err_vec;
793 790
794 791 /* TODO: what about cpl->ip_frag? */
795 792 if (csum_ok && !cpl->ip_frag) {
796 793 mac_hcksum_set(m, 0, 0, 0, 0xffff,
797 794 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
798 795 HCK_IPV4_HDRCKSUM_OK);
799 796 rxq->rxcsum++;
800 797 }
801 798 rxq->rxpkts++;
|
↓ open down ↓ |
9 lines elided |
↑ open up ↑ |
802 799 rxq->rxbytes += pkt_len;
803 800 received_bytes += pkt_len;
804 801
805 802 *mblk_tail = m;
806 803 mblk_tail = &m->b_next;
807 804
808 805 break;
809 806 }
810 807
811 808 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
812 - if (m == NULL) {
813 - panic("%s: line %d.", __func__,
814 - __LINE__);
815 - }
809 + if (m == NULL)
810 + goto done;
811 + /* FALLTHROUGH */
816 812
817 813 case X_RSPD_TYPE_CPL:
818 814 ASSERT(rss->opcode < NUM_CPL_CMDS);
819 815 sc->cpl_handler[rss->opcode](iq, rss, m);
820 816 break;
821 817
822 818 default:
823 819 break;
824 820 }
825 821 iq_next(iq);
826 822 ++ndescs;
827 823 if (!iq->polling && (ndescs == budget))
828 824 break;
829 825 }
830 826
831 827 done:
832 828
833 829 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
834 830 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
835 831 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
836 832
837 833 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) {
838 834 int starved;
839 835 FL_LOCK(fl);
840 836 fl->needed += fl_bufs_used;
841 837 starved = refill_fl(sc, fl, fl->cap / 8);
842 838 FL_UNLOCK(fl);
843 839 if (starved)
844 840 add_fl_to_sfl(sc, fl);
845 841 }
846 842 return (mblk_head);
847 843 }
848 844
849 845 /*
850 846 * Deals with anything and everything on the given ingress queue.
851 847 */
852 848 static int
853 849 service_iq(struct sge_iq *iq, int budget)
|
↓ open down ↓ |
28 lines elided |
↑ open up ↑ |
854 850 {
855 851 struct sge_iq *q;
856 852 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
857 853 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
858 854 struct adapter *sc = iq->adapter;
859 855 struct rsp_ctrl *ctrl;
860 856 const struct rss_header *rss;
861 857 int ndescs = 0, limit, fl_bufs_used = 0;
862 858 int rsp_type;
863 859 uint32_t lq;
860 + int starved;
864 861 mblk_t *m;
865 862 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
866 863
867 864 limit = budget ? budget : iq->qsize / 8;
868 865
869 866 /*
870 867 * We always come back and check the descriptor ring for new indirect
871 868 * interrupts and other responses after running a single handler.
872 869 */
873 870 for (;;) {
874 871 while (is_new_response(iq, &ctrl)) {
875 872
876 873 membar_consumer();
877 874
878 875 m = NULL;
879 876 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
|
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
880 877 lq = be32_to_cpu(ctrl->pldbuflen_qid);
881 878 rss = (const void *)iq->cdesc;
882 879
883 880 switch (rsp_type) {
884 881 case X_RSPD_TYPE_FLBUF:
885 882
886 883 ASSERT(iq->flags & IQ_HAS_FL);
887 884
888 885 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
889 886 if (m == NULL) {
890 - panic("%s: line %d.", __func__,
891 - __LINE__);
887 + /*
888 + * Rearm the iq with a
889 + * longer-than-default timer
890 + */
891 + t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
892 + V_INGRESSQID((u32)iq->cntxt_id) |
893 + V_SEINTARM(V_QINTR_TIMER_IDX(SGE_NTIMERS-1)));
894 + if (fl_bufs_used > 0) {
895 + ASSERT(iq->flags & IQ_HAS_FL);
896 + FL_LOCK(fl);
897 + fl->needed += fl_bufs_used;
898 + starved = refill_fl(sc, fl, fl->cap / 8);
899 + FL_UNLOCK(fl);
900 + if (starved)
901 + add_fl_to_sfl(sc, fl);
902 + }
903 + return (0);
892 904 }
893 905
894 906 /* FALLTHRU */
895 907 case X_RSPD_TYPE_CPL:
896 908
897 909 ASSERT(rss->opcode < NUM_CPL_CMDS);
898 910 sc->cpl_handler[rss->opcode](iq, rss, m);
899 911 break;
900 912
901 913 case X_RSPD_TYPE_INTR:
902 914
903 915 /*
904 916 * Interrupts should be forwarded only to queues
905 917 * that are not forwarding their interrupts.
906 918 * This means service_iq can recurse but only 1
907 919 * level deep.
908 920 */
909 921 ASSERT(budget == 0);
910 922
911 923 q = sc->sge.iqmap[lq - sc->sge.iq_start];
912 924 if (atomic_cas_uint(&q->state, IQS_IDLE,
913 925 IQS_BUSY) == IQS_IDLE) {
914 926 if (service_iq(q, q->qsize / 8) == 0) {
915 927 (void) atomic_cas_uint(
916 928 &q->state, IQS_BUSY,
917 929 IQS_IDLE);
918 930 } else {
919 931 STAILQ_INSERT_TAIL(&iql, q,
920 932 link);
921 933 }
922 934 }
923 935 break;
924 936
925 937 default:
926 938 break;
927 939 }
928 940
929 941 iq_next(iq);
930 942 if (++ndescs == limit) {
931 943 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
932 944 V_CIDXINC(ndescs) |
933 945 V_INGRESSQID(iq->cntxt_id) |
934 946 V_SEINTARM(V_QINTR_TIMER_IDX(
935 947 X_TIMERREG_UPDATE_CIDX)));
936 948 ndescs = 0;
937 949
938 950 if (fl_bufs_used > 0) {
939 951 ASSERT(iq->flags & IQ_HAS_FL);
940 952 FL_LOCK(fl);
941 953 fl->needed += fl_bufs_used;
942 954 (void) refill_fl(sc, fl, fl->cap / 8);
943 955 FL_UNLOCK(fl);
944 956 fl_bufs_used = 0;
945 957 }
946 958
947 959 if (budget != 0)
948 960 return (EINPROGRESS);
949 961 }
950 962 }
951 963
952 964 if (STAILQ_EMPTY(&iql) != 0)
953 965 break;
954 966
955 967 /*
956 968 * Process the head only, and send it to the back of the list if
957 969 * it's still not done.
958 970 */
959 971 q = STAILQ_FIRST(&iql);
960 972 STAILQ_REMOVE_HEAD(&iql, link);
|
↓ open down ↓ |
59 lines elided |
↑ open up ↑ |
961 973 if (service_iq(q, q->qsize / 8) == 0)
962 974 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
963 975 else
964 976 STAILQ_INSERT_TAIL(&iql, q, link);
965 977 }
966 978
967 979 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
968 980 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
969 981
970 982 if (iq->flags & IQ_HAS_FL) {
971 - int starved;
972 983
973 984 FL_LOCK(fl);
974 985 fl->needed += fl_bufs_used;
975 986 starved = refill_fl(sc, fl, fl->cap / 4);
976 987 FL_UNLOCK(fl);
977 988 if (starved != 0)
978 989 add_fl_to_sfl(sc, fl);
979 990 }
980 991
981 992 return (0);
982 993 }
983 994
984 995 #ifdef TCP_OFFLOAD_ENABLE
985 996 int
986 997 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
987 998 {
988 999 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
989 1000 }
990 1001
991 1002 /*
992 1003 * Doesn't fail. Holds on to work requests it can't send right away.
993 1004 */
994 1005 int
995 1006 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0)
996 1007 {
997 1008 struct sge_eq *eq = &wrq->eq;
998 1009 struct mblk_pair *wr_list = &wrq->wr_list;
999 1010 int can_reclaim;
1000 1011 caddr_t dst;
1001 1012 mblk_t *wr, *next;
1002 1013
1003 1014 TXQ_LOCK_ASSERT_OWNED(wrq);
1004 1015 #ifdef TCP_OFFLOAD_ENABLE
1005 1016 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
1006 1017 (eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1007 1018 #else
1008 1019 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1009 1020 #endif
1010 1021
1011 1022 if (m0 != NULL) {
1012 1023 if (wr_list->head != NULL)
1013 1024 wr_list->tail->b_next = m0;
1014 1025 else
1015 1026 wr_list->head = m0;
1016 1027 while (m0->b_next)
1017 1028 m0 = m0->b_next;
1018 1029 wr_list->tail = m0;
1019 1030 }
1020 1031
1021 1032 can_reclaim = reclaimable(eq);
1022 1033 eq->cidx += can_reclaim;
1023 1034 eq->avail += can_reclaim;
1024 1035 if (eq->cidx >= eq->cap)
1025 1036 eq->cidx -= eq->cap;
1026 1037
1027 1038 for (wr = wr_list->head; wr; wr = next) {
1028 1039 int ndesc, len = 0;
1029 1040 mblk_t *m;
1030 1041
1031 1042 next = wr->b_next;
1032 1043 wr->b_next = NULL;
1033 1044
1034 1045 for (m = wr; m; m = m->b_cont)
1035 1046 len += MBLKL(m);
1036 1047
1037 1048 ASSERT(len > 0 && (len & 0x7) == 0);
1038 1049 ASSERT(len <= SGE_MAX_WR_LEN);
1039 1050
1040 1051 ndesc = howmany(len, EQ_ESIZE);
1041 1052 if (eq->avail < ndesc) {
1042 1053 wr->b_next = next;
1043 1054 wrq->no_desc++;
1044 1055 break;
1045 1056 }
1046 1057
1047 1058 dst = (void *)&eq->desc[eq->pidx];
1048 1059 for (m = wr; m; m = m->b_cont)
1049 1060 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
1050 1061
1051 1062 eq->pidx += ndesc;
1052 1063 eq->avail -= ndesc;
1053 1064 if (eq->pidx >= eq->cap)
1054 1065 eq->pidx -= eq->cap;
1055 1066
1056 1067 eq->pending += ndesc;
1057 1068 if (eq->pending > 16)
1058 1069 ring_tx_db(sc, eq);
1059 1070
1060 1071 wrq->tx_wrs++;
1061 1072 freemsg(wr);
1062 1073
1063 1074 if (eq->avail < 8) {
1064 1075 can_reclaim = reclaimable(eq);
1065 1076 eq->cidx += can_reclaim;
1066 1077 eq->avail += can_reclaim;
1067 1078 if (eq->cidx >= eq->cap)
1068 1079 eq->cidx -= eq->cap;
1069 1080 }
1070 1081 }
1071 1082
1072 1083 if (eq->pending != 0)
1073 1084 ring_tx_db(sc, eq);
1074 1085
1075 1086 if (wr == NULL)
1076 1087 wr_list->head = wr_list->tail = NULL;
1077 1088 else {
1078 1089 wr_list->head = wr;
1079 1090
1080 1091 ASSERT(wr_list->tail->b_next == NULL);
1081 1092 }
1082 1093
1083 1094 return (0);
1084 1095 }
1085 1096 #endif
1086 1097
1087 1098 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
1088 1099 #define TXPKTS_PKT_HDR ((\
1089 1100 sizeof (struct ulp_txpkt) + \
1090 1101 sizeof (struct ulptx_idata) + \
1091 1102 sizeof (struct cpl_tx_pkt_core)) / 8)
1092 1103
1093 1104 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
1094 1105 #define TXPKTS_WR_HDR (\
1095 1106 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
1096 1107 TXPKTS_PKT_HDR)
1097 1108
1098 1109 /* Header of a tx WR, before SGL of first packet (in flits) */
1099 1110 #define TXPKT_WR_HDR ((\
1100 1111 sizeof (struct fw_eth_tx_pkt_wr) + \
1101 1112 sizeof (struct cpl_tx_pkt_core)) / 8)
1102 1113
1103 1114 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
1104 1115 #define TXPKT_LSO_WR_HDR ((\
1105 1116 sizeof (struct fw_eth_tx_pkt_wr) + \
1106 1117 sizeof(struct cpl_tx_pkt_lso_core) + \
1107 1118 sizeof (struct cpl_tx_pkt_core)) / 8)
1108 1119
1109 1120 mblk_t *
1110 1121 t4_eth_tx(void *arg, mblk_t *frame)
1111 1122 {
1112 1123 struct sge_txq *txq = (struct sge_txq *) arg;
1113 1124 struct port_info *pi = txq->port;
1114 1125 struct adapter *sc = pi->adapter;
1115 1126 struct sge_eq *eq = &txq->eq;
1116 1127 mblk_t *next_frame;
1117 1128 int rc, coalescing;
1118 1129 struct txpkts txpkts;
1119 1130 struct txinfo txinfo;
1120 1131
1121 1132 txpkts.npkt = 0; /* indicates there's nothing in txpkts */
1122 1133 coalescing = 0;
1123 1134
1124 1135 TXQ_LOCK(txq);
1125 1136 if (eq->avail < 8)
1126 1137 (void) reclaim_tx_descs(txq, 8);
1127 1138 for (; frame; frame = next_frame) {
1128 1139
1129 1140 if (eq->avail < 8)
1130 1141 break;
1131 1142
1132 1143 next_frame = frame->b_next;
1133 1144 frame->b_next = NULL;
1134 1145
1135 1146 if (next_frame != NULL)
1136 1147 coalescing = 1;
1137 1148
1138 1149 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
1139 1150 if (rc != 0) {
1140 1151 if (rc == ENOMEM) {
1141 1152
1142 1153 /* Short of resources, suspend tx */
1143 1154
1144 1155 frame->b_next = next_frame;
1145 1156 break;
1146 1157 }
1147 1158
1148 1159 /*
1149 1160 * Unrecoverable error for this frame, throw it
1150 1161 * away and move on to the next.
1151 1162 */
1152 1163
1153 1164 freemsg(frame);
1154 1165 continue;
1155 1166 }
1156 1167
1157 1168 if (coalescing != 0 &&
1158 1169 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
1159 1170
1160 1171 /* Successfully absorbed into txpkts */
1161 1172
1162 1173 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
1163 1174 goto doorbell;
1164 1175 }
1165 1176
1166 1177 /*
1167 1178 * We weren't coalescing to begin with, or current frame could
1168 1179 * not be coalesced (add_to_txpkts flushes txpkts if a frame
1169 1180 * given to it can't be coalesced). Either way there should be
1170 1181 * nothing in txpkts.
1171 1182 */
1172 1183 ASSERT(txpkts.npkt == 0);
1173 1184
1174 1185 /* We're sending out individual frames now */
1175 1186 coalescing = 0;
1176 1187
1177 1188 if (eq->avail < 8)
1178 1189 (void) reclaim_tx_descs(txq, 8);
1179 1190 rc = write_txpkt_wr(pi, txq, frame, &txinfo);
1180 1191 if (rc != 0) {
1181 1192
1182 1193 /* Short of hardware descriptors, suspend tx */
1183 1194
1184 1195 /*
1185 1196 * This is an unlikely but expensive failure. We've
1186 1197 * done all the hard work (DMA bindings etc.) and now we
1187 1198 * can't send out the frame. What's worse, we have to
1188 1199 * spend even more time freeing up everything in txinfo.
1189 1200 */
1190 1201 txq->qfull++;
1191 1202 free_txinfo_resources(txq, &txinfo);
1192 1203
1193 1204 frame->b_next = next_frame;
1194 1205 break;
1195 1206 }
1196 1207
1197 1208 doorbell:
1198 1209 /* Fewer and fewer doorbells as the queue fills up */
1199 1210 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) {
1200 1211 txq->txbytes += txinfo.len;
1201 1212 txq->txpkts++;
1202 1213 ring_tx_db(sc, eq);
1203 1214 }
1204 1215 (void) reclaim_tx_descs(txq, 32);
1205 1216 }
1206 1217
1207 1218 if (txpkts.npkt > 0)
1208 1219 write_txpkts_wr(txq, &txpkts);
1209 1220
1210 1221 /*
1211 1222 * frame not NULL means there was an error but we haven't thrown it
1212 1223 * away. This can happen when we're short of tx descriptors (qfull) or
1213 1224 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush
1214 1225 * and reclaim will get things going again.
1215 1226 *
1216 1227 * If eq->avail is already 0 we know a credit flush was requested in the
1217 1228 * WR that reduced it to 0 so we don't need another flush (we don't have
1218 1229 * any descriptor for a flush WR anyway, duh).
1219 1230 */
1220 1231 if (frame && eq->avail > 0)
1221 1232 write_txqflush_wr(txq);
1222 1233
1223 1234 if (eq->pending != 0)
1224 1235 ring_tx_db(sc, eq);
1225 1236
1226 1237 (void) reclaim_tx_descs(txq, eq->qsize);
1227 1238 TXQ_UNLOCK(txq);
1228 1239
1229 1240 return (frame);
1230 1241 }
1231 1242
1232 1243 static inline void
1233 1244 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx,
1234 1245 int qsize, uint8_t esize)
1235 1246 {
1236 1247 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1237 1248 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1238 1249
1239 1250 iq->flags = 0;
1240 1251 iq->adapter = sc;
1241 1252 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1242 1253 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1243 1254 if (pktc_idx >= 0) {
1244 1255 iq->intr_params |= F_QINTR_CNT_EN;
1245 1256 iq->intr_pktc_idx = pktc_idx;
|
↓ open down ↓ |
264 lines elided |
↑ open up ↑ |
1246 1257 }
1247 1258 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1248 1259 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1249 1260 }
1250 1261
1251 1262 static inline void
1252 1263 init_fl(struct sge_fl *fl, uint16_t qsize)
1253 1264 {
1254 1265
1255 1266 fl->qsize = qsize;
1267 + fl->allocb_fail = 0;
1256 1268 }
1257 1269
1258 1270 static inline void
1259 1271 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1260 1272 uint8_t tx_chan, uint16_t iqid)
1261 1273 {
1262 1274 struct sge *s = &sc->sge;
1263 1275 uint32_t r;
1264 1276
1265 1277 ASSERT(tx_chan < NCHAN);
1266 1278 ASSERT(eqtype <= EQ_TYPEMASK);
1267 1279
1268 1280 if (is_t5(sc->params.chip)) {
1269 1281 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1270 1282 r >>= S_QUEUESPERPAGEPF0 +
1271 1283 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1272 1284 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1273 1285 }
1274 1286
1275 1287 eq->flags = eqtype & EQ_TYPEMASK;
1276 1288 eq->tx_chan = tx_chan;
1277 1289 eq->iqid = iqid;
1278 1290 eq->qsize = qsize;
1279 1291 }
1280 1292
1281 1293 /*
1282 1294 * Allocates the ring for an ingress queue and an optional freelist. If the
1283 1295 * freelist is specified it will be allocated and then associated with the
1284 1296 * ingress queue.
1285 1297 *
1286 1298 * Returns errno on failure. Resources allocated up to that point may still be
1287 1299 * allocated. Caller is responsible for cleanup in case this function fails.
1288 1300 *
1289 1301 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1290 1302 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies
1291 1303 * the index of the queue to which its interrupts will be forwarded.
1292 1304 */
1293 1305 static int
1294 1306 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1295 1307 int intr_idx, int cong)
1296 1308 {
1297 1309 int rc, i, cntxt_id;
1298 1310 size_t len;
1299 1311 struct fw_iq_cmd c;
1300 1312 struct adapter *sc = iq->adapter;
1301 1313 uint32_t v = 0;
1302 1314
1303 1315 len = iq->qsize * iq->esize;
1304 1316 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1305 1317 &iq->ba, (caddr_t *)&iq->desc);
1306 1318 if (rc != 0)
1307 1319 return (rc);
1308 1320
1309 1321 bzero(&c, sizeof (c));
1310 1322 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1311 1323 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1312 1324 V_FW_IQ_CMD_VFN(0));
1313 1325
1314 1326 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1315 1327 FW_LEN16(c));
1316 1328
1317 1329 /* Special handling for firmware event queue */
1318 1330 if (iq == &sc->sge.fwq)
1319 1331 v |= F_FW_IQ_CMD_IQASYNCH;
1320 1332
1321 1333 if (iq->flags & IQ_INTR)
1322 1334 ASSERT(intr_idx < sc->intr_count);
1323 1335 else
1324 1336 v |= F_FW_IQ_CMD_IQANDST;
1325 1337 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1326 1338
1327 1339 c.type_to_iqandstindex = cpu_to_be32(v |
1328 1340 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1329 1341 V_FW_IQ_CMD_VIID(pi->viid) |
1330 1342 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1331 1343 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1332 1344 F_FW_IQ_CMD_IQGTSMODE |
1333 1345 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1334 1346 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1335 1347 c.iqsize = cpu_to_be16(iq->qsize);
1336 1348 c.iqaddr = cpu_to_be64(iq->ba);
1337 1349 if (cong >= 0)
1338 1350 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN);
1339 1351
1340 1352 if (fl != NULL) {
1341 1353 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip);
1342 1354
1343 1355 mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1344 1356 DDI_INTR_PRI(sc->intr_pri));
1345 1357 fl->flags |= FL_MTX;
1346 1358
1347 1359 len = fl->qsize * RX_FL_ESIZE;
1348 1360 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1349 1361 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1350 1362 if (rc != 0)
1351 1363 return (rc);
1352 1364
1353 1365 /* Allocate space for one software descriptor per buffer. */
1354 1366 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8;
1355 1367 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1356 1368 KM_SLEEP);
1357 1369 fl->needed = fl->cap;
1358 1370 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1359 1371
1360 1372 c.iqns_to_fl0congen |=
1361 1373 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1362 1374 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1363 1375 if (cong >= 0) {
1364 1376 c.iqns_to_fl0congen |=
1365 1377 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1366 1378 F_FW_IQ_CMD_FL0CONGCIF |
1367 1379 F_FW_IQ_CMD_FL0CONGEN);
1368 1380 }
1369 1381
1370 1382 /* In T6, for egress queue type FL there is internal overhead
1371 1383 * of 16B for header going into FLM module. Hence the maximum
1372 1384 * allowed burst size is 448 bytes. For T4/T5, the hardware
1373 1385 * doesn't coalesce fetch requests if more than 64 bytes of
1374 1386 * Free List pointers are provided, so we use a 128-byte Fetch
1375 1387 * Burst Minimum there (T6 implements coalescing so we can use
1376 1388 * the smaller 64-byte value there).
1377 1389 */
1378 1390
1379 1391 c.fl0dcaen_to_fl0cidxfthresh =
1380 1392 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5
1381 1393 ? X_FETCHBURSTMIN_128B
1382 1394 : X_FETCHBURSTMIN_64B) |
1383 1395 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5
1384 1396 ? X_FETCHBURSTMAX_512B
1385 1397 : X_FETCHBURSTMAX_256B));
1386 1398 c.fl0size = cpu_to_be16(fl->qsize);
1387 1399 c.fl0addr = cpu_to_be64(fl->ba);
1388 1400 }
1389 1401
1390 1402 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1391 1403 if (rc != 0) {
1392 1404 cxgb_printf(sc->dip, CE_WARN,
1393 1405 "failed to create ingress queue: %d", rc);
1394 1406 return (rc);
1395 1407 }
1396 1408
1397 1409 iq->cdesc = iq->desc;
1398 1410 iq->cidx = 0;
1399 1411 iq->gen = 1;
1400 1412 iq->intr_next = iq->intr_params;
1401 1413 iq->adapter = sc;
1402 1414 iq->cntxt_id = be16_to_cpu(c.iqid);
1403 1415 iq->abs_id = be16_to_cpu(c.physiqid);
1404 1416 iq->flags |= IQ_ALLOCATED;
1405 1417 mutex_init(&iq->lock, NULL,
1406 1418 MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri)));
1407 1419 iq->polling = 0;
1408 1420
1409 1421 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1410 1422 if (cntxt_id >= sc->sge.niq) {
1411 1423 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1412 1424 cntxt_id, sc->sge.niq - 1);
1413 1425 }
1414 1426 sc->sge.iqmap[cntxt_id] = iq;
1415 1427
1416 1428 if (fl != NULL) {
1417 1429 fl->cntxt_id = be16_to_cpu(c.fl0id);
1418 1430 fl->pidx = fl->cidx = 0;
1419 1431 fl->copy_threshold = rx_copy_threshold;
1420 1432
1421 1433 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1422 1434 if (cntxt_id >= sc->sge.neq) {
1423 1435 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1424 1436 __func__, cntxt_id, sc->sge.neq - 1);
1425 1437 }
1426 1438 sc->sge.eqmap[cntxt_id] = (void *)fl;
1427 1439
1428 1440 FL_LOCK(fl);
1429 1441 (void) refill_fl(sc, fl, fl->lowat);
1430 1442 FL_UNLOCK(fl);
1431 1443
1432 1444 iq->flags |= IQ_HAS_FL;
1433 1445 }
1434 1446
1435 1447 if (is_t5(sc->params.chip) && cong >= 0) {
1436 1448 uint32_t param, val;
1437 1449
1438 1450 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1439 1451 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
1440 1452 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
1441 1453 if (cong == 0)
1442 1454 val = 1 << 19;
1443 1455 else {
1444 1456 val = 2 << 19;
1445 1457 for (i = 0; i < 4; i++) {
1446 1458 if (cong & (1 << i))
1447 1459 val |= 1 << (i << 2);
1448 1460 }
1449 1461 }
1450 1462
1451 1463 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1452 1464 if (rc != 0) {
1453 1465 /* report error but carry on */
1454 1466 cxgb_printf(sc->dip, CE_WARN,
1455 1467 "failed to set congestion manager context for "
1456 1468 "ingress queue %d: %d", iq->cntxt_id, rc);
1457 1469 }
1458 1470 }
1459 1471
1460 1472 /* Enable IQ interrupts */
1461 1473 iq->state = IQS_IDLE;
1462 1474 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1463 1475 V_INGRESSQID(iq->cntxt_id));
1464 1476
1465 1477 return (0);
1466 1478 }
1467 1479
1468 1480 static int
1469 1481 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1470 1482 {
1471 1483 int rc;
1472 1484 struct adapter *sc = iq->adapter;
1473 1485 dev_info_t *dip;
1474 1486
1475 1487 dip = pi ? pi->dip : sc->dip;
1476 1488
1477 1489 if (iq != NULL) {
1478 1490 if (iq->flags & IQ_ALLOCATED) {
1479 1491 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1480 1492 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1481 1493 fl ? fl->cntxt_id : 0xffff, 0xffff);
1482 1494 if (rc != 0) {
1483 1495 cxgb_printf(dip, CE_WARN,
1484 1496 "failed to free queue %p: %d", iq, rc);
1485 1497 return (rc);
1486 1498 }
1487 1499 mutex_destroy(&iq->lock);
1488 1500 iq->flags &= ~IQ_ALLOCATED;
1489 1501 }
1490 1502
1491 1503 if (iq->desc != NULL) {
1492 1504 (void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1493 1505 iq->desc = NULL;
1494 1506 }
1495 1507
1496 1508 bzero(iq, sizeof (*iq));
1497 1509 }
1498 1510
1499 1511 if (fl != NULL) {
1500 1512 if (fl->sdesc != NULL) {
1501 1513 FL_LOCK(fl);
1502 1514 free_fl_bufs(fl);
1503 1515 FL_UNLOCK(fl);
1504 1516
1505 1517 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1506 1518 fl->cap);
1507 1519 fl->sdesc = NULL;
1508 1520 }
1509 1521
1510 1522 if (fl->desc != NULL) {
1511 1523 (void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1512 1524 fl->desc = NULL;
1513 1525 }
1514 1526
1515 1527 if (fl->flags & FL_MTX) {
1516 1528 mutex_destroy(&fl->lock);
1517 1529 fl->flags &= ~FL_MTX;
1518 1530 }
1519 1531
1520 1532 bzero(fl, sizeof (struct sge_fl));
1521 1533 }
1522 1534
1523 1535 return (0);
1524 1536 }
1525 1537
1526 1538 static int
1527 1539 alloc_fwq(struct adapter *sc)
1528 1540 {
1529 1541 int rc, intr_idx;
1530 1542 struct sge_iq *fwq = &sc->sge.fwq;
1531 1543
1532 1544 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1533 1545 fwq->flags |= IQ_INTR; /* always */
1534 1546 intr_idx = sc->intr_count > 1 ? 1 : 0;
1535 1547 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1536 1548 if (rc != 0) {
1537 1549 cxgb_printf(sc->dip, CE_WARN,
1538 1550 "failed to create firmware event queue: %d.", rc);
1539 1551 return (rc);
1540 1552 }
1541 1553
1542 1554 return (0);
1543 1555 }
1544 1556
1545 1557 static int
1546 1558 free_fwq(struct adapter *sc)
1547 1559 {
1548 1560
1549 1561 return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1550 1562 }
1551 1563
1552 1564 #ifdef TCP_OFFLOAD_ENABLE
1553 1565 static int
1554 1566 alloc_mgmtq(struct adapter *sc)
1555 1567 {
1556 1568 int rc;
1557 1569 struct sge_wrq *mgmtq = &sc->sge.mgmtq;
1558 1570
1559 1571 init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
1560 1572 sc->sge.fwq.cntxt_id);
1561 1573 rc = alloc_wrq(sc, NULL, mgmtq, 0);
1562 1574 if (rc != 0) {
1563 1575 cxgb_printf(sc->dip, CE_WARN,
1564 1576 "failed to create management queue: %d\n", rc);
1565 1577 return (rc);
1566 1578 }
1567 1579
1568 1580 return (0);
1569 1581 }
1570 1582 #endif
1571 1583
1572 1584 static int
1573 1585 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1574 1586 {
1575 1587 int rc;
1576 1588
1577 1589 rxq->port = pi;
1578 1590 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1 << pi->tx_chan);
1579 1591 if (rc != 0)
1580 1592 return (rc);
1581 1593
1582 1594 rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1583 1595
1584 1596 return (rc);
1585 1597 }
1586 1598
1587 1599 static int
1588 1600 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1589 1601 {
1590 1602 int rc;
1591 1603
1592 1604 if (rxq->ksp != NULL) {
1593 1605 kstat_delete(rxq->ksp);
1594 1606 rxq->ksp = NULL;
1595 1607 }
1596 1608
1597 1609 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1598 1610 if (rc == 0)
1599 1611 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1600 1612
1601 1613 return (rc);
1602 1614 }
1603 1615
1604 1616 #ifdef TCP_OFFLOAD_ENABLE
1605 1617 static int
1606 1618 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
1607 1619 int intr_idx)
1608 1620 {
1609 1621 int rc;
1610 1622
1611 1623 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
1612 1624 1 << pi->tx_chan);
1613 1625 if (rc != 0)
1614 1626 return (rc);
1615 1627
1616 1628 return (rc);
1617 1629 }
1618 1630
1619 1631 static int
1620 1632 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
1621 1633 {
1622 1634 int rc;
1623 1635
1624 1636 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
1625 1637 if (rc == 0)
1626 1638 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) -
1627 1639 offsetof(struct sge_ofld_rxq, fl));
1628 1640
1629 1641 return (rc);
1630 1642 }
1631 1643 #endif
1632 1644
1633 1645 static int
1634 1646 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
1635 1647 {
1636 1648 int rc, cntxt_id;
1637 1649 struct fw_eq_ctrl_cmd c;
1638 1650
1639 1651 bzero(&c, sizeof (c));
1640 1652
1641 1653 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
1642 1654 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
1643 1655 V_FW_EQ_CTRL_CMD_VFN(0));
1644 1656 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC |
1645 1657 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
1646 1658 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */
1647 1659 c.physeqid_pkd = BE_32(0);
1648 1660 c.fetchszm_to_iqid =
1649 1661 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1650 1662 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
1651 1663 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
1652 1664 c.dcaen_to_eqsize =
1653 1665 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1654 1666 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1655 1667 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1656 1668 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
1657 1669 c.eqaddr = BE_64(eq->ba);
1658 1670
1659 1671 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1660 1672 if (rc != 0) {
1661 1673 cxgb_printf(sc->dip, CE_WARN,
1662 1674 "failed to create control queue %d: %d", eq->tx_chan, rc);
1663 1675 return (rc);
1664 1676 }
1665 1677 eq->flags |= EQ_ALLOCATED;
1666 1678
1667 1679 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid));
1668 1680 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1669 1681 if (cntxt_id >= sc->sge.neq)
1670 1682 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1671 1683 cntxt_id, sc->sge.neq - 1);
1672 1684 sc->sge.eqmap[cntxt_id] = eq;
1673 1685
1674 1686 return (rc);
1675 1687 }
1676 1688
1677 1689 static int
1678 1690 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1679 1691 {
|
↓ open down ↓ |
414 lines elided |
↑ open up ↑ |
1680 1692 int rc, cntxt_id;
1681 1693 struct fw_eq_eth_cmd c;
1682 1694
1683 1695 bzero(&c, sizeof (c));
1684 1696
1685 1697 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1686 1698 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1687 1699 V_FW_EQ_ETH_CMD_VFN(0));
1688 1700 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1689 1701 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1690 - c.autoequiqe_to_viid = BE_32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
1702 + c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1703 + F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1691 1704 c.fetchszm_to_iqid =
1692 1705 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1693 1706 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1694 1707 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1695 1708 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1696 1709 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1697 1710 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1698 1711 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1699 1712 c.eqaddr = BE_64(eq->ba);
1700 1713
1701 1714 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1702 1715 if (rc != 0) {
1703 1716 cxgb_printf(pi->dip, CE_WARN,
1704 1717 "failed to create Ethernet egress queue: %d", rc);
1705 1718 return (rc);
1706 1719 }
1707 1720 eq->flags |= EQ_ALLOCATED;
1708 1721
1709 1722 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1710 1723 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1711 1724 if (cntxt_id >= sc->sge.neq)
1712 1725 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1713 1726 cntxt_id, sc->sge.neq - 1);
1714 1727 sc->sge.eqmap[cntxt_id] = eq;
1715 1728
1716 1729 return (rc);
1717 1730 }
1718 1731
1719 1732 #ifdef TCP_OFFLOAD_ENABLE
1720 1733 static int
1721 1734 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1722 1735 {
1723 1736 int rc, cntxt_id;
1724 1737 struct fw_eq_ofld_cmd c;
1725 1738
1726 1739 bzero(&c, sizeof (c));
1727 1740
1728 1741 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
1729 1742 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
1730 1743 V_FW_EQ_OFLD_CMD_VFN(0));
1731 1744 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
1732 1745 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
1733 1746 c.fetchszm_to_iqid =
1734 1747 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1735 1748 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
1736 1749 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
1737 1750 c.dcaen_to_eqsize =
1738 1751 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1739 1752 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1740 1753 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1741 1754 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
1742 1755 c.eqaddr = BE_64(eq->ba);
1743 1756
1744 1757 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1745 1758 if (rc != 0) {
1746 1759 cxgb_printf(pi->dip, CE_WARN,
1747 1760 "failed to create egress queue for TCP offload: %d", rc);
1748 1761 return (rc);
1749 1762 }
1750 1763 eq->flags |= EQ_ALLOCATED;
1751 1764
1752 1765 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd));
1753 1766 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1754 1767 if (cntxt_id >= sc->sge.neq)
1755 1768 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1756 1769 cntxt_id, sc->sge.neq - 1);
1757 1770 sc->sge.eqmap[cntxt_id] = eq;
1758 1771
1759 1772 return (rc);
1760 1773 }
1761 1774 #endif
1762 1775
1763 1776 static int
1764 1777 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1765 1778 {
1766 1779 int rc;
1767 1780 size_t len;
1768 1781
1769 1782 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1770 1783 eq->flags |= EQ_MTX;
1771 1784
1772 1785 len = eq->qsize * EQ_ESIZE;
1773 1786 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1774 1787 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1775 1788 if (rc != 0)
1776 1789 return (rc);
1777 1790
1778 1791 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE;
1779 1792 eq->spg = (void *)&eq->desc[eq->cap];
1780 1793 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
1781 1794 eq->pidx = eq->cidx = 0;
1782 1795 eq->doorbells = sc->doorbells;
1783 1796
1784 1797 switch (eq->flags & EQ_TYPEMASK) {
1785 1798 case EQ_CTRL:
1786 1799 rc = ctrl_eq_alloc(sc, eq);
1787 1800 break;
1788 1801
1789 1802 case EQ_ETH:
1790 1803 rc = eth_eq_alloc(sc, pi, eq);
1791 1804 break;
1792 1805
1793 1806 #ifdef TCP_OFFLOAD_ENABLE
1794 1807 case EQ_OFLD:
1795 1808 rc = ofld_eq_alloc(sc, pi, eq);
1796 1809 break;
1797 1810 #endif
1798 1811
1799 1812 default:
1800 1813 panic("%s: invalid eq type %d.", __func__,
1801 1814 eq->flags & EQ_TYPEMASK);
1802 1815 }
1803 1816
1804 1817 if (eq->doorbells &
1805 1818 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) {
1806 1819 uint32_t s_qpp = sc->sge.s_qpp;
1807 1820 uint32_t mask = (1 << s_qpp) - 1;
1808 1821 volatile uint8_t *udb;
1809 1822
1810 1823 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET;
1811 1824 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
1812 1825 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
1813 1826 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
1814 1827 eq->doorbells &= ~DOORBELL_WCWR;
1815 1828 else {
1816 1829 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
1817 1830 eq->udb_qid = 0;
1818 1831 }
1819 1832 eq->udb = (volatile void *)udb;
1820 1833 }
1821 1834
1822 1835 if (rc != 0) {
1823 1836 cxgb_printf(sc->dip, CE_WARN,
1824 1837 "failed to allocate egress queue(%d): %d",
1825 1838 eq->flags & EQ_TYPEMASK, rc);
1826 1839 }
1827 1840
1828 1841 return (rc);
1829 1842 }
1830 1843
1831 1844 static int
1832 1845 free_eq(struct adapter *sc, struct sge_eq *eq)
1833 1846 {
1834 1847 int rc;
1835 1848
1836 1849 if (eq->flags & EQ_ALLOCATED) {
1837 1850 switch (eq->flags & EQ_TYPEMASK) {
1838 1851 case EQ_CTRL:
1839 1852 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
1840 1853 eq->cntxt_id);
1841 1854 break;
1842 1855
1843 1856 case EQ_ETH:
1844 1857 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
1845 1858 eq->cntxt_id);
1846 1859 break;
1847 1860
1848 1861 #ifdef TCP_OFFLOAD_ENABLE
1849 1862 case EQ_OFLD:
1850 1863 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
1851 1864 eq->cntxt_id);
1852 1865 break;
1853 1866 #endif
1854 1867
1855 1868 default:
1856 1869 panic("%s: invalid eq type %d.", __func__,
1857 1870 eq->flags & EQ_TYPEMASK);
1858 1871 }
1859 1872 if (rc != 0) {
1860 1873 cxgb_printf(sc->dip, CE_WARN,
1861 1874 "failed to free egress queue (%d): %d",
1862 1875 eq->flags & EQ_TYPEMASK, rc);
1863 1876 return (rc);
1864 1877 }
1865 1878 eq->flags &= ~EQ_ALLOCATED;
1866 1879 }
1867 1880
1868 1881 if (eq->desc != NULL) {
1869 1882 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1870 1883 eq->desc = NULL;
1871 1884 }
1872 1885
1873 1886 if (eq->flags & EQ_MTX)
1874 1887 mutex_destroy(&eq->lock);
1875 1888
1876 1889 bzero(eq, sizeof (*eq));
1877 1890 return (0);
1878 1891 }
1879 1892
1880 1893 #ifdef TCP_OFFLOAD_ENABLE
1881 1894 /* ARGSUSED */
1882 1895 static int
1883 1896 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
1884 1897 int idx)
1885 1898 {
1886 1899 int rc;
1887 1900
1888 1901 rc = alloc_eq(sc, pi, &wrq->eq);
1889 1902 if (rc != 0)
1890 1903 return (rc);
1891 1904
1892 1905 wrq->adapter = sc;
1893 1906 wrq->wr_list.head = NULL;
1894 1907 wrq->wr_list.tail = NULL;
1895 1908
1896 1909 /*
1897 1910 * TODO: use idx to figure out what kind of wrq this is and install
1898 1911 * useful kstats for it.
1899 1912 */
1900 1913
1901 1914 return (rc);
1902 1915 }
1903 1916
1904 1917 static int
1905 1918 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
1906 1919 {
1907 1920 int rc;
1908 1921
1909 1922 rc = free_eq(sc, &wrq->eq);
1910 1923 if (rc != 0)
1911 1924 return (rc);
1912 1925
1913 1926 bzero(wrq, sizeof (*wrq));
1914 1927 return (0);
1915 1928 }
1916 1929 #endif
1917 1930
1918 1931 static int
1919 1932 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1920 1933 {
1921 1934 int rc, i;
1922 1935 struct adapter *sc = pi->adapter;
1923 1936 struct sge_eq *eq = &txq->eq;
1924 1937
1925 1938 rc = alloc_eq(sc, pi, eq);
1926 1939 if (rc != 0)
1927 1940 return (rc);
1928 1941
1929 1942 txq->port = pi;
1930 1943 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1931 1944 txq->txb_size = eq->qsize * tx_copy_threshold;
1932 1945 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1933 1946 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1934 1947 if (rc == 0)
1935 1948 txq->txb_avail = txq->txb_size;
1936 1949 else
1937 1950 txq->txb_avail = txq->txb_size = 0;
1938 1951
1939 1952 /*
1940 1953 * TODO: is this too low? Worst case would need around 4 times qsize
1941 1954 * (all tx descriptors filled to the brim with SGLs, with each entry in
1942 1955 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total
1943 1956 * if you see too many dma_hdl_failed.
1944 1957 */
1945 1958 txq->tx_dhdl_total = eq->qsize * 2;
1946 1959 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1947 1960 txq->tx_dhdl_total, KM_SLEEP);
1948 1961 for (i = 0; i < txq->tx_dhdl_total; i++) {
1949 1962 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1950 1963 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1951 1964 if (rc != DDI_SUCCESS) {
1952 1965 cxgb_printf(sc->dip, CE_WARN,
1953 1966 "%s: failed to allocate DMA handle (%d)",
1954 1967 __func__, rc);
1955 1968 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1956 1969 }
1957 1970 txq->tx_dhdl_avail++;
1958 1971 }
1959 1972
1960 1973 txq->ksp = setup_txq_kstats(pi, txq, idx);
1961 1974
1962 1975 return (rc);
1963 1976 }
1964 1977
1965 1978 static int
1966 1979 free_txq(struct port_info *pi, struct sge_txq *txq)
1967 1980 {
1968 1981 int i;
1969 1982 struct adapter *sc = pi->adapter;
1970 1983 struct sge_eq *eq = &txq->eq;
1971 1984
1972 1985 if (txq->ksp != NULL) {
1973 1986 kstat_delete(txq->ksp);
1974 1987 txq->ksp = NULL;
1975 1988 }
1976 1989
1977 1990 if (txq->txb_va != NULL) {
1978 1991 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1979 1992 txq->txb_va = NULL;
1980 1993 }
1981 1994
1982 1995 if (txq->sdesc != NULL) {
1983 1996 struct tx_sdesc *sd;
1984 1997 ddi_dma_handle_t hdl;
1985 1998
1986 1999 TXQ_LOCK(txq);
1987 2000 while (eq->cidx != eq->pidx) {
1988 2001 sd = &txq->sdesc[eq->cidx];
1989 2002
1990 2003 for (i = sd->hdls_used; i; i--) {
1991 2004 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
1992 2005 (void) ddi_dma_unbind_handle(hdl);
1993 2006 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
1994 2007 txq->tx_dhdl_cidx = 0;
1995 2008 }
1996 2009
1997 2010 ASSERT(sd->m);
1998 2011 freemsgchain(sd->m);
1999 2012
2000 2013 eq->cidx += sd->desc_used;
2001 2014 if (eq->cidx >= eq->cap)
2002 2015 eq->cidx -= eq->cap;
2003 2016
2004 2017 txq->txb_avail += txq->txb_used;
2005 2018 }
2006 2019 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
2007 2020 ASSERT(txq->txb_avail == txq->txb_size);
2008 2021 TXQ_UNLOCK(txq);
2009 2022
2010 2023 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
2011 2024 txq->sdesc = NULL;
2012 2025 }
2013 2026
2014 2027 if (txq->tx_dhdl != NULL) {
2015 2028 for (i = 0; i < txq->tx_dhdl_total; i++) {
2016 2029 if (txq->tx_dhdl[i] != NULL)
2017 2030 ddi_dma_free_handle(&txq->tx_dhdl[i]);
2018 2031 }
2019 2032 }
2020 2033
2021 2034 (void) free_eq(sc, &txq->eq);
2022 2035
2023 2036 bzero(txq, sizeof (*txq));
2024 2037 return (0);
2025 2038 }
2026 2039
2027 2040 /*
2028 2041 * Allocates a block of contiguous memory for DMA. Can be used to allocate
2029 2042 * memory for descriptor rings or for tx/rx copy buffers.
2030 2043 *
2031 2044 * Caller does not have to clean up anything if this function fails, it cleans
2032 2045 * up after itself.
2033 2046 *
2034 2047 * Caller provides the following:
2035 2048 * len length of the block of memory to allocate.
2036 2049 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
2037 2050 * acc_attr device access attributes for the allocation.
2038 2051 * dma_attr DMA attributes for the allocation
2039 2052 *
2040 2053 * If the function is successful it fills up this information:
2041 2054 * dma_hdl DMA handle for the allocated memory
2042 2055 * acc_hdl access handle for the allocated memory
2043 2056 * ba bus address of the allocated memory
2044 2057 * va KVA of the allocated memory.
2045 2058 */
2046 2059 static int
2047 2060 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
2048 2061 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
2049 2062 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2050 2063 uint64_t *pba, caddr_t *pva)
2051 2064 {
2052 2065 int rc;
2053 2066 ddi_dma_handle_t dhdl;
2054 2067 ddi_acc_handle_t ahdl;
2055 2068 ddi_dma_cookie_t cookie;
2056 2069 uint_t ccount;
2057 2070 caddr_t va;
2058 2071 size_t real_len;
2059 2072
2060 2073 *pva = NULL;
2061 2074
2062 2075 /*
2063 2076 * DMA handle.
2064 2077 */
2065 2078 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
2066 2079 if (rc != DDI_SUCCESS) {
2067 2080 cxgb_printf(sc->dip, CE_WARN,
2068 2081 "failed to allocate DMA handle: %d", rc);
2069 2082
2070 2083 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
2071 2084 }
2072 2085
2073 2086 /*
2074 2087 * Memory suitable for DMA.
2075 2088 */
2076 2089 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
2077 2090 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
2078 2091 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
2079 2092 if (rc != DDI_SUCCESS) {
2080 2093 cxgb_printf(sc->dip, CE_WARN,
2081 2094 "failed to allocate DMA memory: %d", rc);
2082 2095
2083 2096 ddi_dma_free_handle(&dhdl);
2084 2097 return (ENOMEM);
2085 2098 }
2086 2099
2087 2100 if (len != real_len) {
2088 2101 cxgb_printf(sc->dip, CE_WARN,
2089 2102 "%s: len (%u) != real_len (%u)\n", len, real_len);
2090 2103 }
2091 2104
2092 2105 /*
2093 2106 * DMA bindings.
2094 2107 */
2095 2108 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
2096 2109 NULL, &cookie, &ccount);
2097 2110 if (rc != DDI_DMA_MAPPED) {
2098 2111 cxgb_printf(sc->dip, CE_WARN,
2099 2112 "failed to map DMA memory: %d", rc);
2100 2113
2101 2114 ddi_dma_mem_free(&ahdl);
2102 2115 ddi_dma_free_handle(&dhdl);
2103 2116 return (ENOMEM);
2104 2117 }
2105 2118 if (ccount != 1) {
2106 2119 cxgb_printf(sc->dip, CE_WARN,
2107 2120 "unusable DMA mapping (%d segments)", ccount);
2108 2121 (void) free_desc_ring(&dhdl, &ahdl);
2109 2122 }
2110 2123
2111 2124 bzero(va, real_len);
2112 2125 *dma_hdl = dhdl;
2113 2126 *acc_hdl = ahdl;
2114 2127 *pba = cookie.dmac_laddress;
2115 2128 *pva = va;
2116 2129
2117 2130 return (0);
2118 2131 }
2119 2132
2120 2133 static int
2121 2134 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2122 2135 {
2123 2136 (void) ddi_dma_unbind_handle(*dhdl);
2124 2137 ddi_dma_mem_free(ahdl);
2125 2138 ddi_dma_free_handle(dhdl);
2126 2139
2127 2140 return (0);
2128 2141 }
2129 2142
2130 2143 static int
2131 2144 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
2132 2145 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2133 2146 uint64_t *pba, caddr_t *pva)
2134 2147 {
2135 2148 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
2136 2149 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
2137 2150
2138 2151 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
2139 2152 dma_attr, dma_hdl, acc_hdl, pba, pva));
2140 2153 }
2141 2154
2142 2155 static int
2143 2156 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2144 2157 {
2145 2158 return (free_dma_memory(dhdl, ahdl));
2146 2159 }
2147 2160
2148 2161 static int
2149 2162 alloc_tx_copybuffer(struct adapter *sc, size_t len,
2150 2163 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2151 2164 uint64_t *pba, caddr_t *pva)
2152 2165 {
2153 2166 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
2154 2167 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
2155 2168
2156 2169 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
2157 2170 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
2158 2171 }
2159 2172
2160 2173 static inline bool
2161 2174 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
2162 2175 {
2163 2176 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
2164 2177 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
2165 2178
2166 2179 *ctrl = (void *)((uintptr_t)iq->cdesc +
2167 2180 (iq->esize - sizeof (struct rsp_ctrl)));
2168 2181
2169 2182 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
2170 2183 }
2171 2184
2172 2185 static inline void
2173 2186 iq_next(struct sge_iq *iq)
2174 2187 {
2175 2188 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
2176 2189 if (++iq->cidx == iq->qsize - 1) {
2177 2190 iq->cidx = 0;
2178 2191 iq->gen ^= 1;
2179 2192 iq->cdesc = iq->desc;
2180 2193 }
2181 2194 }
2182 2195
2183 2196 /*
2184 2197 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
2185 2198 *
2186 2199 * Returns non-zero to indicate that it should be added to the list of starving
2187 2200 * freelists.
2188 2201 */
2189 2202 static int
2190 2203 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
2191 2204 {
2192 2205 uint64_t *d = &fl->desc[fl->pidx];
2193 2206 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
2194 2207
2195 2208 FL_LOCK_ASSERT_OWNED(fl);
2196 2209 ASSERT(nbufs >= 0);
2197 2210
2198 2211 if (nbufs > fl->needed)
2199 2212 nbufs = fl->needed;
2200 2213
2201 2214 while (nbufs--) {
2202 2215 if (sd->rxb != NULL) {
2203 2216 if (sd->rxb->ref_cnt == 1) {
2204 2217 /*
2205 2218 * Buffer is available for recycling. Two ways
2206 2219 * this can happen:
2207 2220 *
2208 2221 * a) All the packets DMA'd into it last time
2209 2222 * around were within the rx_copy_threshold
2210 2223 * and no part of the buffer was ever passed
2211 2224 * up (ref_cnt never went over 1).
2212 2225 *
2213 2226 * b) Packets DMA'd into the buffer were passed
2214 2227 * up but have all been freed by the upper
2215 2228 * layers by now (ref_cnt went over 1 but is
2216 2229 * now back to 1).
2217 2230 *
2218 2231 * Either way the bus address in the descriptor
2219 2232 * ring is already valid.
2220 2233 */
2221 2234 ASSERT(*d == cpu_to_be64(sd->rxb->ba));
2222 2235 d++;
2223 2236 goto recycled;
2224 2237 } else {
2225 2238 /*
2226 2239 * Buffer still in use and we need a
2227 2240 * replacement. But first release our reference
2228 2241 * on the existing buffer.
2229 2242 */
2230 2243 rxbuf_free(sd->rxb);
2231 2244 }
2232 2245 }
2233 2246
2234 2247 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
2235 2248 if (sd->rxb == NULL)
2236 2249 break;
2237 2250 *d++ = cpu_to_be64(sd->rxb->ba);
2238 2251
2239 2252 recycled: fl->pending++;
2240 2253 sd++;
2241 2254 fl->needed--;
2242 2255 if (++fl->pidx == fl->cap) {
2243 2256 fl->pidx = 0;
2244 2257 sd = fl->sdesc;
2245 2258 d = fl->desc;
2246 2259 }
2247 2260 }
2248 2261
2249 2262 if (fl->pending >= 8)
2250 2263 ring_fl_db(sc, fl);
2251 2264
2252 2265 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
2253 2266 }
2254 2267
2255 2268 #ifndef TAILQ_FOREACH_SAFE
2256 2269 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
2257 2270 for ((var) = TAILQ_FIRST((head)); \
2258 2271 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
2259 2272 (var) = (tvar))
2260 2273 #endif
2261 2274
2262 2275 /*
2263 2276 * Attempt to refill all starving freelists.
2264 2277 */
2265 2278 static void
2266 2279 refill_sfl(void *arg)
2267 2280 {
2268 2281 struct adapter *sc = arg;
2269 2282 struct sge_fl *fl, *fl_temp;
2270 2283
2271 2284 mutex_enter(&sc->sfl_lock);
2272 2285 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
2273 2286 FL_LOCK(fl);
2274 2287 (void) refill_fl(sc, fl, 64);
2275 2288 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
2276 2289 TAILQ_REMOVE(&sc->sfl, fl, link);
2277 2290 fl->flags &= ~FL_STARVING;
2278 2291 }
2279 2292 FL_UNLOCK(fl);
2280 2293 }
2281 2294
2282 2295 if (!TAILQ_EMPTY(&sc->sfl) != 0)
2283 2296 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000));
2284 2297 mutex_exit(&sc->sfl_lock);
2285 2298 }
2286 2299
2287 2300 static void
2288 2301 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
2289 2302 {
2290 2303 mutex_enter(&sc->sfl_lock);
2291 2304 FL_LOCK(fl);
2292 2305 if ((fl->flags & FL_DOOMED) == 0) {
2293 2306 if (TAILQ_EMPTY(&sc->sfl) != 0) {
2294 2307 sc->sfl_timer = timeout(refill_sfl, sc,
2295 2308 drv_usectohz(100000));
2296 2309 }
2297 2310 fl->flags |= FL_STARVING;
2298 2311 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
2299 2312 }
2300 2313 FL_UNLOCK(fl);
2301 2314 mutex_exit(&sc->sfl_lock);
2302 2315 }
2303 2316
2304 2317 static void
2305 2318 free_fl_bufs(struct sge_fl *fl)
2306 2319 {
2307 2320 struct fl_sdesc *sd;
2308 2321 unsigned int i;
2309 2322
2310 2323 FL_LOCK_ASSERT_OWNED(fl);
2311 2324
2312 2325 for (i = 0; i < fl->cap; i++) {
2313 2326 sd = &fl->sdesc[i];
2314 2327
2315 2328 if (sd->rxb != NULL) {
2316 2329 rxbuf_free(sd->rxb);
2317 2330 sd->rxb = NULL;
2318 2331 }
2319 2332 }
2320 2333 }
2321 2334
2322 2335 /*
|
↓ open down ↓ |
622 lines elided |
↑ open up ↑ |
2323 2336 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2324 2337 */
2325 2338 static mblk_t *
2326 2339 get_fl_payload(struct adapter *sc, struct sge_fl *fl,
2327 2340 uint32_t len_newbuf, int *fl_bufs_used)
2328 2341 {
2329 2342 struct mblk_pair frame = {0};
2330 2343 struct rxbuf *rxb;
2331 2344 mblk_t *m = NULL;
2332 2345 uint_t nbuf = 0, len, copy, n;
2333 - uint32_t cidx, offset;
2346 + uint32_t cidx, offset, rcidx, roffset;
2334 2347
2335 2348 /*
2336 2349 * The SGE won't pack a new frame into the current buffer if the entire
2337 2350 * payload doesn't fit in the remaining space. Move on to the next buf
2338 2351 * in that case.
2339 2352 */
2353 + rcidx = fl->cidx;
2354 + roffset = fl->offset;
2340 2355 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2341 2356 fl->offset = 0;
2342 2357 if (++fl->cidx == fl->cap)
2343 2358 fl->cidx = 0;
2344 2359 nbuf++;
2345 2360 }
2346 2361 cidx = fl->cidx;
2347 2362 offset = fl->offset;
2348 2363
2349 2364 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2350 2365 copy = (len <= fl->copy_threshold);
2351 2366 if (copy != 0) {
2352 2367 frame.head = m = allocb(len, BPRI_HI);
2353 - if (m == NULL)
2368 + if (m == NULL) {
2369 + fl->allocb_fail++;
2370 + cmn_err(CE_WARN,"%s: mbuf allocation failure "
2371 + "count = %llu", __func__,
2372 + (unsigned long long)fl->allocb_fail);
2373 + fl->cidx = rcidx;
2374 + fl->offset = roffset;
2354 2375 return (NULL);
2376 + }
2355 2377 }
2356 2378
2357 2379 while (len) {
2358 2380 rxb = fl->sdesc[cidx].rxb;
2359 2381 n = min(len, rxb->buf_size - offset);
2360 2382
2361 2383 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2362 2384 DDI_DMA_SYNC_FORKERNEL);
2363 2385
2364 2386 if (copy != 0)
2365 2387 bcopy(rxb->va + offset, m->b_wptr, n);
2366 2388 else {
2367 2389 m = desballoc((unsigned char *)rxb->va + offset, n,
2368 2390 BPRI_HI, &rxb->freefunc);
2369 2391 if (m == NULL) {
2370 - freemsg(frame.head);
2392 + fl->allocb_fail++;
2393 + cmn_err(CE_WARN,
2394 + "%s: mbuf allocation failure "
2395 + "count = %llu", __func__,
2396 + (unsigned long long)fl->allocb_fail);
2397 + if (frame.head)
2398 + freemsgchain(frame.head);
2399 + fl->cidx = rcidx;
2400 + fl->offset = roffset;
2371 2401 return (NULL);
2372 2402 }
2373 2403 atomic_inc_uint(&rxb->ref_cnt);
2374 2404 if (frame.head != NULL)
2375 2405 frame.tail->b_cont = m;
2376 2406 else
2377 2407 frame.head = m;
2378 2408 frame.tail = m;
2379 2409 }
2380 2410 m->b_wptr += n;
2381 2411 len -= n;
2382 2412 offset += roundup(n, sc->sge.fl_align);
2383 2413 ASSERT(offset <= rxb->buf_size);
2384 2414 if (offset == rxb->buf_size) {
2385 2415 offset = 0;
2386 2416 if (++cidx == fl->cap)
2387 2417 cidx = 0;
2388 2418 nbuf++;
2389 2419 }
2390 2420 }
2391 2421
2392 2422 fl->cidx = cidx;
2393 2423 fl->offset = offset;
2394 2424 (*fl_bufs_used) += nbuf;
2395 2425
2396 2426 ASSERT(frame.head != NULL);
2397 2427 return (frame.head);
2398 2428 }
2399 2429
2400 2430 /*
2401 2431 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're
2402 2432 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2403 2433 * of immediate data.
2404 2434 */
2405 2435 #define IMM_LEN ( \
2406 2436 2 * EQ_ESIZE \
2407 2437 - sizeof (struct fw_eth_tx_pkt_wr) \
2408 2438 - sizeof (struct cpl_tx_pkt_core))
2409 2439
2410 2440 /*
2411 2441 * Returns non-zero on failure, no need to cleanup anything in that case.
2412 2442 *
2413 2443 * Note 1: We always try to pull up the mblk if required and return E2BIG only
2414 2444 * if this fails.
2415 2445 *
2416 2446 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
2417 2447 * does not have the TCP header in it.
2418 2448 */
2419 2449 static int
2420 2450 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2421 2451 int sgl_only)
2422 2452 {
2423 2453 uint32_t flags = 0, len, n;
2424 2454 mblk_t *m = *fp;
2425 2455 int rc;
2426 2456
2427 2457 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */
2428 2458
2429 2459 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2430 2460 txinfo->flags = flags;
2431 2461
2432 2462 mac_lso_get(m, &txinfo->mss, &flags);
2433 2463 txinfo->flags |= flags;
2434 2464
2435 2465 if (flags & HW_LSO)
2436 2466 sgl_only = 1; /* Do not allow immediate data with LSO */
2437 2467
2438 2468 start: txinfo->nsegs = 0;
2439 2469 txinfo->hdls_used = 0;
2440 2470 txinfo->txb_used = 0;
2441 2471 txinfo->len = 0;
2442 2472
2443 2473 /* total length and a rough estimate of # of segments */
2444 2474 n = 0;
2445 2475 for (; m; m = m->b_cont) {
2446 2476 len = MBLKL(m);
2447 2477 n += (len / PAGE_SIZE) + 1;
2448 2478 txinfo->len += len;
2449 2479 }
2450 2480 m = *fp;
2451 2481
2452 2482 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2453 2483 txq->pullup_early++;
2454 2484 m = msgpullup(*fp, -1);
2455 2485 if (m == NULL) {
2456 2486 txq->pullup_failed++;
2457 2487 return (E2BIG); /* (*fp) left as it was */
2458 2488 }
2459 2489 freemsg(*fp);
2460 2490 *fp = m;
2461 2491 mac_hcksum_set(m, NULL, NULL, NULL, NULL, txinfo->flags);
2462 2492 }
2463 2493
2464 2494 if (txinfo->len <= IMM_LEN && !sgl_only)
2465 2495 return (0); /* nsegs = 0 tells caller to use imm. tx */
2466 2496
2467 2497 if (txinfo->len <= txq->copy_threshold &&
2468 2498 copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2469 2499 goto done;
2470 2500
2471 2501 for (; m; m = m->b_cont) {
2472 2502
2473 2503 len = MBLKL(m);
2474 2504
2475 2505 /* Use tx copy buffer if this mblk is small enough */
2476 2506 if (len <= txq->copy_threshold &&
2477 2507 copy_into_txb(txq, m, len, txinfo) == 0)
2478 2508 continue;
2479 2509
2480 2510 /* Add DMA bindings for this mblk to the SGL */
2481 2511 rc = add_mblk(txq, txinfo, m, len);
2482 2512
2483 2513 if (rc == E2BIG ||
2484 2514 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2485 2515
2486 2516 txq->pullup_late++;
2487 2517 m = msgpullup(*fp, -1);
2488 2518 if (m != NULL) {
2489 2519 free_txinfo_resources(txq, txinfo);
2490 2520 freemsg(*fp);
2491 2521 *fp = m;
2492 2522 mac_hcksum_set(m, NULL, NULL, NULL, NULL,
2493 2523 txinfo->flags);
2494 2524 goto start;
2495 2525 }
2496 2526
2497 2527 txq->pullup_failed++;
2498 2528 rc = E2BIG;
2499 2529 }
2500 2530
2501 2531 if (rc != 0) {
2502 2532 free_txinfo_resources(txq, txinfo);
2503 2533 return (rc);
2504 2534 }
2505 2535 }
2506 2536
2507 2537 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2508 2538
2509 2539 done:
2510 2540
2511 2541 /*
2512 2542 * Store the # of flits required to hold this frame's SGL in nflits. An
2513 2543 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2514 2544 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
2515 2545 * then len1 must be set to 0.
2516 2546 */
2517 2547 n = txinfo->nsegs - 1;
2518 2548 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2519 2549 if (n & 1)
2520 2550 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2521 2551
2522 2552 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2523 2553 V_ULPTX_NSGE(txinfo->nsegs));
2524 2554
2525 2555 return (0);
2526 2556 }
2527 2557
2528 2558 static inline int
2529 2559 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2530 2560 {
2531 2561 if (txq->txb_avail < len)
2532 2562 return (0);
2533 2563
2534 2564 if (txq->txb_next + len <= txq->txb_size) {
2535 2565 *waste = 0;
2536 2566 return (1);
2537 2567 }
2538 2568
2539 2569 *waste = txq->txb_size - txq->txb_next;
2540 2570
2541 2571 return (txq->txb_avail - *waste < len ? 0 : 1);
2542 2572 }
2543 2573
2544 2574 #define TXB_CHUNK 64
2545 2575
2546 2576 /*
2547 2577 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2548 2578 * and txq to indicate resources used. Caller has to make sure that those many
2549 2579 * bytes are available in the mblk chain (b_cont linked).
2550 2580 */
2551 2581 static inline int
2552 2582 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2553 2583 {
2554 2584 int waste, n;
2555 2585
2556 2586 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */
2557 2587
2558 2588 if (!fits_in_txb(txq, len, &waste)) {
2559 2589 txq->txb_full++;
2560 2590 return (ENOMEM);
2561 2591 }
2562 2592
2563 2593 if (waste != 0) {
2564 2594 ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2565 2595 txinfo->txb_used += waste;
2566 2596 txq->txb_avail -= waste;
2567 2597 txq->txb_next = 0;
2568 2598 }
2569 2599
2570 2600 for (n = 0; n < len; m = m->b_cont) {
2571 2601 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2572 2602 n += MBLKL(m);
2573 2603 }
2574 2604
2575 2605 add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2576 2606
2577 2607 n = roundup(len, TXB_CHUNK);
2578 2608 txinfo->txb_used += n;
2579 2609 txq->txb_avail -= n;
2580 2610 txq->txb_next += n;
2581 2611 ASSERT(txq->txb_next <= txq->txb_size);
2582 2612 if (txq->txb_next == txq->txb_size)
2583 2613 txq->txb_next = 0;
2584 2614
2585 2615 return (0);
2586 2616 }
2587 2617
2588 2618 static inline void
2589 2619 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2590 2620 {
2591 2621 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */
2592 2622
2593 2623 if (txinfo->nsegs != 0) {
2594 2624 int idx = txinfo->nsegs - 1;
2595 2625 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2596 2626 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2597 2627 } else {
2598 2628 txinfo->sgl.len0 = cpu_to_be32(len);
2599 2629 txinfo->sgl.addr0 = cpu_to_be64(ba);
2600 2630 }
2601 2631 txinfo->nsegs++;
2602 2632 }
2603 2633
2604 2634 /*
2605 2635 * This function cleans up any partially allocated resources when it fails so
2606 2636 * there's nothing for the caller to clean up in that case.
2607 2637 *
2608 2638 * EIO indicates permanent failure. Caller should drop the frame containing
2609 2639 * this mblk and continue.
2610 2640 *
2611 2641 * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2612 2642 * limit. Caller should pull up the frame before trying to send it out.
2613 2643 * (This error means our pullup_early heuristic did not work for this frame)
2614 2644 *
2615 2645 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2616 2646 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to
2617 2647 * free up resources.
2618 2648 */
2619 2649 static inline int
2620 2650 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2621 2651 {
2622 2652 ddi_dma_handle_t dhdl;
2623 2653 ddi_dma_cookie_t cookie;
2624 2654 uint_t ccount = 0;
2625 2655 int rc;
2626 2656
2627 2657 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */
2628 2658
2629 2659 if (txq->tx_dhdl_avail == 0) {
2630 2660 txq->dma_hdl_failed++;
2631 2661 return (ENOMEM);
2632 2662 }
2633 2663
2634 2664 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2635 2665 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2636 2666 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2637 2667 &ccount);
2638 2668 if (rc != DDI_DMA_MAPPED) {
2639 2669 txq->dma_map_failed++;
2640 2670
2641 2671 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2642 2672
2643 2673 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2644 2674 }
2645 2675
2646 2676 if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2647 2677 (void) ddi_dma_unbind_handle(dhdl);
2648 2678 return (E2BIG);
2649 2679 }
2650 2680
2651 2681 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2652 2682 while (--ccount) {
2653 2683 ddi_dma_nextcookie(dhdl, &cookie);
2654 2684 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2655 2685 }
2656 2686
2657 2687 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2658 2688 txq->tx_dhdl_pidx = 0;
2659 2689 txq->tx_dhdl_avail--;
2660 2690 txinfo->hdls_used++;
2661 2691
2662 2692 return (0);
2663 2693 }
2664 2694
2665 2695 /*
2666 2696 * Releases all the txq resources used up in the specified txinfo.
2667 2697 */
2668 2698 static void
2669 2699 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2670 2700 {
2671 2701 int n;
2672 2702
2673 2703 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */
2674 2704
2675 2705 n = txinfo->txb_used;
2676 2706 if (n > 0) {
2677 2707 txq->txb_avail += n;
2678 2708 if (n <= txq->txb_next)
2679 2709 txq->txb_next -= n;
2680 2710 else {
2681 2711 n -= txq->txb_next;
2682 2712 txq->txb_next = txq->txb_size - n;
2683 2713 }
2684 2714 }
2685 2715
2686 2716 for (n = txinfo->hdls_used; n > 0; n--) {
2687 2717 if (txq->tx_dhdl_pidx > 0)
2688 2718 txq->tx_dhdl_pidx--;
2689 2719 else
2690 2720 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2691 2721 txq->tx_dhdl_avail++;
2692 2722 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2693 2723 }
2694 2724 }
2695 2725
2696 2726 /*
2697 2727 * Returns 0 to indicate that m has been accepted into a coalesced tx work
2698 2728 * request. It has either been folded into txpkts or txpkts was flushed and m
2699 2729 * has started a new coalesced work request (as the first frame in a fresh
2700 2730 * txpkts).
2701 2731 *
2702 2732 * Returns non-zero to indicate a failure - caller is responsible for
2703 2733 * transmitting m, if there was anything in txpkts it has been flushed.
2704 2734 */
2705 2735 static int
2706 2736 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2707 2737 struct txinfo *txinfo)
2708 2738 {
2709 2739 struct sge_eq *eq = &txq->eq;
2710 2740 int can_coalesce;
2711 2741 struct tx_sdesc *txsd;
2712 2742 uint8_t flits;
2713 2743
2714 2744 TXQ_LOCK_ASSERT_OWNED(txq);
2715 2745
2716 2746 if (txpkts->npkt > 0) {
2717 2747 flits = TXPKTS_PKT_HDR + txinfo->nflits;
2718 2748 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2719 2749 txpkts->nflits + flits <= TX_WR_FLITS &&
2720 2750 txpkts->nflits + flits <= eq->avail * 8 &&
2721 2751 txpkts->plen + txinfo->len < 65536;
2722 2752
2723 2753 if (can_coalesce != 0) {
2724 2754 txpkts->tail->b_next = m;
2725 2755 txpkts->tail = m;
2726 2756 txpkts->npkt++;
2727 2757 txpkts->nflits += flits;
2728 2758 txpkts->plen += txinfo->len;
2729 2759
2730 2760 txsd = &txq->sdesc[eq->pidx];
2731 2761 txsd->txb_used += txinfo->txb_used;
2732 2762 txsd->hdls_used += txinfo->hdls_used;
2733 2763
2734 2764 return (0);
2735 2765 }
2736 2766
2737 2767 /*
2738 2768 * Couldn't coalesce m into txpkts. The first order of business
2739 2769 * is to send txpkts on its way. Then we'll revisit m.
2740 2770 */
2741 2771 write_txpkts_wr(txq, txpkts);
2742 2772 }
2743 2773
2744 2774 /*
2745 2775 * Check if we can start a new coalesced tx work request with m as
2746 2776 * the first packet in it.
2747 2777 */
2748 2778
2749 2779 ASSERT(txpkts->npkt == 0);
2750 2780 ASSERT(txinfo->len < 65536);
2751 2781
2752 2782 flits = TXPKTS_WR_HDR + txinfo->nflits;
2753 2783 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2754 2784 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2755 2785
2756 2786 if (can_coalesce == 0)
2757 2787 return (EINVAL);
2758 2788
2759 2789 /*
2760 2790 * Start a fresh coalesced tx WR with m as the first frame in it.
2761 2791 */
2762 2792 txpkts->tail = m;
2763 2793 txpkts->npkt = 1;
2764 2794 txpkts->nflits = flits;
2765 2795 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2766 2796 txpkts->plen = txinfo->len;
2767 2797
2768 2798 txsd = &txq->sdesc[eq->pidx];
2769 2799 txsd->m = m;
2770 2800 txsd->txb_used = txinfo->txb_used;
2771 2801 txsd->hdls_used = txinfo->hdls_used;
2772 2802
2773 2803 return (0);
2774 2804 }
2775 2805
2776 2806 /*
2777 2807 * Note that write_txpkts_wr can never run out of hardware descriptors (but
2778 2808 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
2779 2809 * coalescing only if sufficient hardware descriptors are available.
2780 2810 */
2781 2811 static void
2782 2812 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2783 2813 {
2784 2814 struct sge_eq *eq = &txq->eq;
2785 2815 struct fw_eth_tx_pkts_wr *wr;
2786 2816 struct tx_sdesc *txsd;
2787 2817 uint32_t ctrl;
2788 2818 uint16_t ndesc;
2789 2819
2790 2820 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2791 2821
2792 2822 ndesc = howmany(txpkts->nflits, 8);
2793 2823
2794 2824 wr = (void *)&eq->desc[eq->pidx];
2795 2825 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2796 2826 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2797 2827 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2798 2828 if (eq->avail == ndesc)
2799 2829 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2800 2830 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2801 2831 wr->plen = cpu_to_be16(txpkts->plen);
2802 2832 wr->npkt = txpkts->npkt;
2803 2833 wr->r3 = wr->type = 0;
2804 2834
2805 2835 /* Everything else already written */
2806 2836
2807 2837 txsd = &txq->sdesc[eq->pidx];
2808 2838 txsd->desc_used = ndesc;
2809 2839
2810 2840 txq->txb_used += txsd->txb_used / TXB_CHUNK;
2811 2841 txq->hdl_used += txsd->hdls_used;
2812 2842
2813 2843 ASSERT(eq->avail >= ndesc);
2814 2844
2815 2845 eq->pending += ndesc;
2816 2846 eq->avail -= ndesc;
2817 2847 eq->pidx += ndesc;
2818 2848 if (eq->pidx >= eq->cap)
2819 2849 eq->pidx -= eq->cap;
2820 2850
2821 2851 txq->txpkts_pkts += txpkts->npkt;
2822 2852 txq->txpkts_wrs++;
2823 2853 txpkts->npkt = 0; /* emptied */
2824 2854 }
2825 2855
2826 2856 static int
2827 2857 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2828 2858 struct txinfo *txinfo)
2829 2859 {
2830 2860 struct sge_eq *eq = &txq->eq;
2831 2861 struct fw_eth_tx_pkt_wr *wr;
2832 2862 struct cpl_tx_pkt_core *cpl;
2833 2863 uint32_t ctrl; /* used in many unrelated places */
2834 2864 uint64_t ctrl1;
2835 2865 int nflits, ndesc;
2836 2866 struct tx_sdesc *txsd;
2837 2867 caddr_t dst;
2838 2868
2839 2869 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2840 2870
2841 2871 /*
2842 2872 * Do we have enough flits to send this frame out?
2843 2873 */
2844 2874 ctrl = sizeof (struct cpl_tx_pkt_core);
2845 2875 if (txinfo->flags & HW_LSO) {
2846 2876 nflits = TXPKT_LSO_WR_HDR;
2847 2877 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
2848 2878 } else
2849 2879 nflits = TXPKT_WR_HDR;
2850 2880 if (txinfo->nsegs > 0)
2851 2881 nflits += txinfo->nflits;
2852 2882 else {
2853 2883 nflits += howmany(txinfo->len, 8);
2854 2884 ctrl += txinfo->len;
2855 2885 }
2856 2886 ndesc = howmany(nflits, 8);
2857 2887 if (ndesc > eq->avail)
2858 2888 return (ENOMEM);
2859 2889
2860 2890 /* Firmware work request header */
2861 2891 wr = (void *)&eq->desc[eq->pidx];
2862 2892 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2863 2893 V_FW_WR_IMMDLEN(ctrl));
2864 2894 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2865 2895 if (eq->avail == ndesc)
2866 2896 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2867 2897 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2868 2898 wr->r3 = 0;
2869 2899
2870 2900 if (txinfo->flags & HW_LSO) {
2871 2901 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
2872 2902 char *p = (void *)m->b_rptr;
2873 2903 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2874 2904 F_LSO_LAST_SLICE;
2875 2905
2876 2906 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2877 2907 if (((struct ether_header *)p)->ether_type ==
2878 2908 htons(ETHERTYPE_VLAN)) {
2879 2909 ctrl |= V_LSO_ETHHDR_LEN(1);
2880 2910 p += sizeof (struct ether_vlan_header);
2881 2911 } else
2882 2912 p += sizeof (struct ether_header);
2883 2913
2884 2914 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2885 2915 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4);
2886 2916 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2887 2917 p += IPH_HDR_LENGTH(p);
2888 2918 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4);
2889 2919
2890 2920 lso->lso_ctrl = cpu_to_be32(ctrl);
2891 2921 lso->ipid_ofst = cpu_to_be16(0);
2892 2922 lso->mss = cpu_to_be16(txinfo->mss);
2893 2923 lso->seqno_offset = cpu_to_be32(0);
2894 2924 if (is_t4(pi->adapter->params.chip))
2895 2925 lso->len = cpu_to_be32(txinfo->len);
2896 2926 else
2897 2927 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len));
2898 2928
2899 2929 cpl = (void *)(lso + 1);
2900 2930
2901 2931 txq->tso_wrs++;
2902 2932 } else
2903 2933 cpl = (void *)(wr + 1);
2904 2934
2905 2935 /* Checksum offload */
2906 2936 ctrl1 = 0;
2907 2937 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2908 2938 ctrl1 |= F_TXPKT_IPCSUM_DIS;
2909 2939 if (!(txinfo->flags & HCK_FULLCKSUM))
2910 2940 ctrl1 |= F_TXPKT_L4CSUM_DIS;
2911 2941 if (ctrl1 == 0)
2912 2942 txq->txcsum++; /* some hardware assistance provided */
2913 2943
2914 2944 /* CPL header */
2915 2945 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2916 2946 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2917 2947 cpl->pack = 0;
2918 2948 cpl->len = cpu_to_be16(txinfo->len);
2919 2949 cpl->ctrl1 = cpu_to_be64(ctrl1);
2920 2950
2921 2951 /* Software descriptor */
2922 2952 txsd = &txq->sdesc[eq->pidx];
2923 2953 txsd->m = m;
2924 2954 txsd->txb_used = txinfo->txb_used;
2925 2955 txsd->hdls_used = txinfo->hdls_used;
2926 2956 /* LINTED: E_ASSIGN_NARROW_CONV */
2927 2957 txsd->desc_used = ndesc;
2928 2958
2929 2959 txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2930 2960 txq->hdl_used += txinfo->hdls_used;
2931 2961
2932 2962 eq->pending += ndesc;
2933 2963 eq->avail -= ndesc;
2934 2964 eq->pidx += ndesc;
2935 2965 if (eq->pidx >= eq->cap)
2936 2966 eq->pidx -= eq->cap;
2937 2967
2938 2968 /* SGL */
2939 2969 dst = (void *)(cpl + 1);
2940 2970 if (txinfo->nsegs > 0) {
2941 2971 txq->sgl_wrs++;
2942 2972 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2943 2973
2944 2974 /* Need to zero-pad to a 16 byte boundary if not on one */
2945 2975 if ((uintptr_t)dst & 0xf)
2946 2976 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2947 2977 *(uint64_t *)dst = 0;
2948 2978
2949 2979 } else {
2950 2980 txq->imm_wrs++;
2951 2981 #ifdef DEBUG
2952 2982 ctrl = txinfo->len;
2953 2983 #endif
2954 2984 for (; m; m = m->b_cont) {
2955 2985 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
2956 2986 #ifdef DEBUG
2957 2987 ctrl -= MBLKL(m);
2958 2988 #endif
2959 2989 }
2960 2990 ASSERT(ctrl == 0);
2961 2991 }
2962 2992
2963 2993 txq->txpkt_wrs++;
2964 2994 return (0);
2965 2995 }
2966 2996
2967 2997 static inline void
2968 2998 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
2969 2999 struct txpkts *txpkts, struct txinfo *txinfo)
2970 3000 {
2971 3001 struct ulp_txpkt *ulpmc;
2972 3002 struct ulptx_idata *ulpsc;
2973 3003 struct cpl_tx_pkt_core *cpl;
2974 3004 uintptr_t flitp, start, end;
2975 3005 uint64_t ctrl;
2976 3006 caddr_t dst;
2977 3007
2978 3008 ASSERT(txpkts->npkt > 0);
2979 3009
2980 3010 start = (uintptr_t)txq->eq.desc;
2981 3011 end = (uintptr_t)txq->eq.spg;
2982 3012
2983 3013 /* Checksum offload */
2984 3014 ctrl = 0;
2985 3015 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2986 3016 ctrl |= F_TXPKT_IPCSUM_DIS;
2987 3017 if (!(txinfo->flags & HCK_FULLCKSUM))
2988 3018 ctrl |= F_TXPKT_L4CSUM_DIS;
2989 3019 if (ctrl == 0)
2990 3020 txq->txcsum++; /* some hardware assistance provided */
2991 3021
2992 3022 /*
2993 3023 * The previous packet's SGL must have ended at a 16 byte boundary (this
2994 3024 * is required by the firmware/hardware). It follows that flitp cannot
2995 3025 * wrap around between the ULPTX master command and ULPTX subcommand (8
2996 3026 * bytes each), and that it can not wrap around in the middle of the
2997 3027 * cpl_tx_pkt_core either.
2998 3028 */
2999 3029 flitp = (uintptr_t)txpkts->flitp;
3000 3030 ASSERT((flitp & 0xf) == 0);
3001 3031
3002 3032 /* ULP master command */
3003 3033 ulpmc = (void *)flitp;
3004 3034 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
3005 3035 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
3006 3036 sizeof (*cpl) + 8 * txinfo->nflits, 16));
3007 3037
3008 3038 /* ULP subcommand */
3009 3039 ulpsc = (void *)(ulpmc + 1);
3010 3040 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
3011 3041 F_ULP_TX_SC_MORE);
3012 3042 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
3013 3043
3014 3044 flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
3015 3045 if (flitp == end)
3016 3046 flitp = start;
3017 3047
3018 3048 /* CPL_TX_PKT */
3019 3049 cpl = (void *)flitp;
3020 3050 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3021 3051 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
3022 3052 cpl->pack = 0;
3023 3053 cpl->len = cpu_to_be16(txinfo->len);
3024 3054 cpl->ctrl1 = cpu_to_be64(ctrl);
3025 3055
3026 3056 flitp += sizeof (*cpl);
3027 3057 if (flitp == end)
3028 3058 flitp = start;
3029 3059
3030 3060 /* SGL for this frame */
3031 3061 dst = (caddr_t)flitp;
3032 3062 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
3033 3063 flitp = (uintptr_t)dst;
3034 3064
3035 3065 /* Zero pad and advance to a 16 byte boundary if not already at one. */
3036 3066 if (flitp & 0xf) {
3037 3067
3038 3068 /* no matter what, flitp should be on an 8 byte boundary */
3039 3069 ASSERT((flitp & 0x7) == 0);
3040 3070
3041 3071 *(uint64_t *)flitp = 0;
3042 3072 flitp += sizeof (uint64_t);
3043 3073 txpkts->nflits++;
3044 3074 }
3045 3075
3046 3076 if (flitp == end)
3047 3077 flitp = start;
3048 3078
3049 3079 txpkts->flitp = (void *)flitp;
3050 3080 }
3051 3081
3052 3082 static inline void
3053 3083 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
3054 3084 {
3055 3085 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
3056 3086 bcopy(from, *to, len);
3057 3087 (*to) += len;
3058 3088 } else {
3059 3089 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
3060 3090
3061 3091 bcopy(from, *to, portion);
3062 3092 from += portion;
3063 3093 portion = len - portion; /* remaining */
3064 3094 bcopy(from, (void *)eq->desc, portion);
3065 3095 (*to) = (caddr_t)eq->desc + portion;
3066 3096 }
3067 3097 }
3068 3098
3069 3099 static inline void
3070 3100 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
3071 3101 {
3072 3102 int val, db_mode;
3073 3103 u_int db = eq->doorbells;
3074 3104
3075 3105 if (eq->pending > 1)
3076 3106 db &= ~DOORBELL_WCWR;
3077 3107
3078 3108 if (eq->pending > eq->pidx) {
3079 3109 int offset = eq->cap - (eq->pending - eq->pidx);
3080 3110
3081 3111 /* pidx has wrapped around since last doorbell */
3082 3112
3083 3113 (void) ddi_dma_sync(eq->desc_dhdl,
3084 3114 offset * sizeof (struct tx_desc), 0,
3085 3115 DDI_DMA_SYNC_FORDEV);
3086 3116 (void) ddi_dma_sync(eq->desc_dhdl,
3087 3117 0, eq->pidx * sizeof (struct tx_desc),
3088 3118 DDI_DMA_SYNC_FORDEV);
3089 3119 } else if (eq->pending > 0) {
3090 3120 (void) ddi_dma_sync(eq->desc_dhdl,
3091 3121 (eq->pidx - eq->pending) * sizeof (struct tx_desc),
3092 3122 eq->pending * sizeof (struct tx_desc),
3093 3123 DDI_DMA_SYNC_FORDEV);
3094 3124 }
3095 3125
3096 3126 membar_producer();
3097 3127
3098 3128 if (is_t4(sc->params.chip))
3099 3129 val = V_PIDX(eq->pending);
3100 3130 else
3101 3131 val = V_PIDX_T5(eq->pending);
3102 3132
3103 3133 db_mode = (1 << (ffs(db) - 1));
3104 3134 switch (db_mode) {
3105 3135 case DOORBELL_UDB:
3106 3136 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3107 3137 break;
3108 3138
3109 3139 case DOORBELL_WCWR:
3110 3140 {
3111 3141 volatile uint64_t *dst, *src;
3112 3142 int i;
3113 3143 /*
3114 3144 * Queues whose 128B doorbell segment fits in
3115 3145 * the page do not use relative qid
3116 3146 * (udb_qid is always 0). Only queues with
3117 3147 * doorbell segments can do WCWR.
3118 3148 */
3119 3149 ASSERT(eq->udb_qid == 0 && eq->pending == 1);
3120 3150
3121 3151 dst = (volatile void *)((uintptr_t)eq->udb +
3122 3152 UDBS_WR_OFFSET - UDBS_DB_OFFSET);
3123 3153 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
3124 3154 src = (void *)&eq->desc[i];
3125 3155 while (src != (void *)&eq->desc[i + 1])
3126 3156 *dst++ = *src++;
3127 3157 membar_producer();
3128 3158 break;
3129 3159 }
3130 3160
3131 3161 case DOORBELL_UDBWC:
3132 3162 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3133 3163 membar_producer();
3134 3164 break;
3135 3165
3136 3166 case DOORBELL_KDB:
3137 3167 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
3138 3168 V_QID(eq->cntxt_id) | val);
3139 3169 break;
3140 3170 }
3141 3171
3142 3172 eq->pending = 0;
3143 3173 }
3144 3174
3145 3175 static int
3146 3176 reclaim_tx_descs(struct sge_txq *txq, int howmany)
3147 3177 {
3148 3178 struct tx_sdesc *txsd;
3149 3179 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
3150 3180 struct sge_eq *eq = &txq->eq;
3151 3181
3152 3182 EQ_LOCK_ASSERT_OWNED(eq);
3153 3183
3154 3184 cidx = eq->spg->cidx; /* stable snapshot */
3155 3185 cidx = be16_to_cpu(cidx);
3156 3186
3157 3187 if (cidx >= eq->cidx)
3158 3188 can_reclaim = cidx - eq->cidx;
3159 3189 else
3160 3190 can_reclaim = cidx + eq->cap - eq->cidx;
3161 3191
3162 3192 if (can_reclaim == 0)
3163 3193 return (0);
3164 3194
3165 3195 txb_freed = hdls_freed = reclaimed = 0;
3166 3196 do {
3167 3197 int ndesc;
3168 3198
3169 3199 txsd = &txq->sdesc[eq->cidx];
3170 3200 ndesc = txsd->desc_used;
3171 3201
3172 3202 /* Firmware doesn't return "partial" credits. */
3173 3203 ASSERT(can_reclaim >= ndesc);
3174 3204
3175 3205 /*
3176 3206 * We always keep mblk around, even for immediate data. If mblk
3177 3207 * is NULL, this has to be the software descriptor for a credit
3178 3208 * flush work request.
3179 3209 */
3180 3210 if (txsd->m != NULL)
3181 3211 freemsgchain(txsd->m);
3182 3212 #ifdef DEBUG
3183 3213 else {
3184 3214 ASSERT(txsd->txb_used == 0);
3185 3215 ASSERT(txsd->hdls_used == 0);
3186 3216 ASSERT(ndesc == 1);
3187 3217 }
3188 3218 #endif
3189 3219
3190 3220 txb_freed += txsd->txb_used;
3191 3221 hdls_freed += txsd->hdls_used;
3192 3222 reclaimed += ndesc;
3193 3223
3194 3224 eq->cidx += ndesc;
3195 3225 if (eq->cidx >= eq->cap)
3196 3226 eq->cidx -= eq->cap;
3197 3227
3198 3228 can_reclaim -= ndesc;
3199 3229
3200 3230 } while (can_reclaim && reclaimed < howmany);
3201 3231
3202 3232 eq->avail += reclaimed;
3203 3233 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */
3204 3234
3205 3235 txq->txb_avail += txb_freed;
3206 3236
3207 3237 txq->tx_dhdl_avail += hdls_freed;
3208 3238 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
3209 3239 for (; hdls_freed; hdls_freed--) {
3210 3240 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
3211 3241 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
3212 3242 txq->tx_dhdl_cidx = 0;
3213 3243 }
3214 3244
3215 3245 return (reclaimed);
3216 3246 }
3217 3247
3218 3248 static void
3219 3249 write_txqflush_wr(struct sge_txq *txq)
3220 3250 {
3221 3251 struct sge_eq *eq = &txq->eq;
3222 3252 struct fw_eq_flush_wr *wr;
3223 3253 struct tx_sdesc *txsd;
3224 3254
3225 3255 EQ_LOCK_ASSERT_OWNED(eq);
3226 3256 ASSERT(eq->avail > 0);
3227 3257
3228 3258 wr = (void *)&eq->desc[eq->pidx];
3229 3259 bzero(wr, sizeof (*wr));
3230 3260 wr->opcode = FW_EQ_FLUSH_WR;
3231 3261 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
3232 3262 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3233 3263
3234 3264 txsd = &txq->sdesc[eq->pidx];
3235 3265 txsd->m = NULL;
3236 3266 txsd->txb_used = 0;
3237 3267 txsd->hdls_used = 0;
3238 3268 txsd->desc_used = 1;
3239 3269
3240 3270 eq->pending++;
3241 3271 eq->avail--;
3242 3272 if (++eq->pidx == eq->cap)
3243 3273 eq->pidx = 0;
3244 3274 }
3245 3275
3246 3276 static int
3247 3277 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3248 3278 {
3249 3279 bool csum_ok;
3250 3280 uint16_t err_vec;
3251 3281 struct sge_rxq *rxq = (void *)iq;
3252 3282 struct mblk_pair chain = {0};
3253 3283 struct adapter *sc = iq->adapter;
3254 3284 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
3255 3285
3256 3286 iq->intr_next = iq->intr_params;
3257 3287
3258 3288 m->b_rptr += sc->sge.pktshift;
3259 3289
3260 3290 /* Compressed error vector is enabled for T6 only */
3261 3291 if (sc->params.tp.rx_pkt_encap)
3262 3292 /* It is enabled only in T6 config file */
3263 3293 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
3264 3294 else
3265 3295 err_vec = ntohs(cpl->err_vec);
3266 3296
3267 3297 csum_ok = cpl->csum_calc && !err_vec;
3268 3298 /* TODO: what about cpl->ip_frag? */
3269 3299 if (csum_ok && !cpl->ip_frag) {
3270 3300 mac_hcksum_set(m, 0, 0, 0, 0xffff,
3271 3301 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
3272 3302 HCK_IPV4_HDRCKSUM_OK);
3273 3303 rxq->rxcsum++;
3274 3304 }
3275 3305
3276 3306 /* Add to the chain that we'll send up */
3277 3307 if (chain.head != NULL)
3278 3308 chain.tail->b_next = m;
3279 3309 else
3280 3310 chain.head = m;
3281 3311 chain.tail = m;
3282 3312
3283 3313 t4_mac_rx(rxq->port, rxq, chain.head);
3284 3314
3285 3315 rxq->rxpkts++;
3286 3316 rxq->rxbytes += be16_to_cpu(cpl->len);
3287 3317 return (0);
3288 3318 }
3289 3319
3290 3320 #define FL_HW_IDX(idx) ((idx) >> 3)
3291 3321
3292 3322 static inline void
3293 3323 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3294 3324 {
3295 3325 int desc_start, desc_last, ndesc;
3296 3326 uint32_t v = sc->params.arch.sge_fl_db ;
3297 3327
3298 3328 ndesc = FL_HW_IDX(fl->pending);
3299 3329
3300 3330 /* Hold back one credit if pidx = cidx */
3301 3331 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3302 3332 ndesc--;
3303 3333
3304 3334 /*
3305 3335 * There are chances of ndesc modified above (to avoid pidx = cidx).
3306 3336 * If there is nothing to post, return.
3307 3337 */
3308 3338 if (ndesc <= 0)
3309 3339 return;
3310 3340
3311 3341 desc_last = FL_HW_IDX(fl->pidx);
3312 3342
3313 3343 if (fl->pidx < fl->pending) {
3314 3344 /* There was a wrap */
3315 3345 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
3316 3346
3317 3347 /* From desc_start to the end of list */
3318 3348 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
3319 3349 DDI_DMA_SYNC_FORDEV);
3320 3350
3321 3351 /* From start of list to the desc_last */
3322 3352 if (desc_last != 0)
3323 3353 (void) ddi_dma_sync(fl->dhdl, 0, desc_last *
3324 3354 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3325 3355 } else {
3326 3356 /* There was no wrap, sync from start_desc to last_desc */
3327 3357 desc_start = FL_HW_IDX(fl->pidx - fl->pending);
3328 3358 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
3329 3359 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3330 3360 }
3331 3361
3332 3362 if (is_t4(sc->params.chip))
3333 3363 v |= V_PIDX(ndesc);
3334 3364 else
3335 3365 v |= V_PIDX_T5(ndesc);
3336 3366 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3337 3367
3338 3368 membar_producer();
|
↓ open down ↓ |
958 lines elided |
↑ open up ↑ |
3339 3369
3340 3370 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3341 3371
3342 3372 /*
3343 3373 * Update pending count:
3344 3374 * Deduct the number of descriptors posted
3345 3375 */
3346 3376 fl->pending -= ndesc * 8;
3347 3377 }
3348 3378
3379 +static void
3380 +tx_reclaim_task(void *arg)
3381 +{
3382 + struct sge_txq *txq = arg;
3383 +
3384 + TXQ_LOCK(txq);
3385 + reclaim_tx_descs(txq, txq->eq.qsize);
3386 + TXQ_UNLOCK(txq);
3387 +}
3388 +
3349 3389 /* ARGSUSED */
3350 3390 static int
3351 3391 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3352 3392 mblk_t *m)
3353 3393 {
3354 3394 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3355 3395 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3356 3396 struct adapter *sc = iq->adapter;
3357 3397 struct sge *s = &sc->sge;
3398 + struct sge_eq *eq;
3358 3399 struct sge_txq *txq;
3359 3400
3360 3401 txq = (void *)s->eqmap[qid - s->eq_start];
3402 + eq = &txq->eq;
3361 3403 txq->qflush++;
3362 3404 t4_mac_tx_update(txq->port, txq);
3363 3405
3406 + ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3407 + (void *)txq, DDI_NOSLEEP);
3408 +
3364 3409 return (0);
3365 3410 }
3366 3411
3367 3412 static int
3368 3413 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3369 3414 {
3370 3415 struct adapter *sc = iq->adapter;
3371 3416 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3372 3417
3373 3418 ASSERT(m == NULL);
3374 3419
3375 3420 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3376 3421 const struct rss_header *rss2;
3377 3422
3378 3423 rss2 = (const struct rss_header *)&cpl->data[0];
3379 3424 return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3380 3425 }
3381 3426 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3382 3427 }
3383 3428
3384 3429 int
3385 3430 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3386 3431 int flags)
3387 3432 {
3388 3433 int i, rc;
3389 3434
3390 3435 txmaps->map_total = count;
3391 3436 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3392 3437
3393 3438 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) *
3394 3439 txmaps->map_total, flags);
3395 3440
3396 3441 for (i = 0; i < count; i++) {
3397 3442 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3398 3443 DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3399 3444 if (rc != DDI_SUCCESS) {
3400 3445 cxgb_printf(sc->dip, CE_WARN,
3401 3446 "%s: failed to allocate DMA handle (%d)",
3402 3447 __func__, rc);
3403 3448 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3404 3449 }
3405 3450 txmaps->map_avail++;
3406 3451 }
3407 3452
3408 3453 return (0);
3409 3454 }
3410 3455
3411 3456 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3412 3457 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3413 3458 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
3414 3459 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x
3415 3460 #define KS_C_SET(x, ...) \
3416 3461 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
3417 3462
3418 3463 /*
3419 3464 * cxgbe:X:config
3420 3465 */
3421 3466 struct cxgbe_port_config_kstats {
3422 3467 kstat_named_t idx;
3423 3468 kstat_named_t nrxq;
3424 3469 kstat_named_t ntxq;
3425 3470 kstat_named_t first_rxq;
3426 3471 kstat_named_t first_txq;
3427 3472 kstat_named_t controller;
3428 3473 kstat_named_t factory_mac_address;
3429 3474 };
3430 3475
3431 3476 /*
3432 3477 * cxgbe:X:info
3433 3478 */
3434 3479 struct cxgbe_port_info_kstats {
3435 3480 kstat_named_t transceiver;
3436 3481 kstat_named_t rx_ovflow0;
3437 3482 kstat_named_t rx_ovflow1;
3438 3483 kstat_named_t rx_ovflow2;
3439 3484 kstat_named_t rx_ovflow3;
3440 3485 kstat_named_t rx_trunc0;
3441 3486 kstat_named_t rx_trunc1;
3442 3487 kstat_named_t rx_trunc2;
3443 3488 kstat_named_t rx_trunc3;
3444 3489 kstat_named_t tx_pause;
3445 3490 kstat_named_t rx_pause;
3446 3491 };
3447 3492
3448 3493 static kstat_t *
3449 3494 setup_port_config_kstats(struct port_info *pi)
3450 3495 {
3451 3496 kstat_t *ksp;
3452 3497 struct cxgbe_port_config_kstats *kstatp;
3453 3498 int ndata;
3454 3499 dev_info_t *pdip = ddi_get_parent(pi->dip);
3455 3500 uint8_t *ma = &pi->hw_addr[0];
3456 3501
3457 3502 ndata = sizeof (struct cxgbe_port_config_kstats) /
3458 3503 sizeof (kstat_named_t);
3459 3504
3460 3505 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3461 3506 "net", KSTAT_TYPE_NAMED, ndata, 0);
3462 3507 if (ksp == NULL) {
3463 3508 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3464 3509 return (NULL);
3465 3510 }
3466 3511
3467 3512 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3468 3513
3469 3514 KS_UINIT(idx);
3470 3515 KS_UINIT(nrxq);
3471 3516 KS_UINIT(ntxq);
3472 3517 KS_UINIT(first_rxq);
3473 3518 KS_UINIT(first_txq);
3474 3519 KS_CINIT(controller);
3475 3520 KS_CINIT(factory_mac_address);
3476 3521
3477 3522 KS_U_SET(idx, pi->port_id);
3478 3523 KS_U_SET(nrxq, pi->nrxq);
3479 3524 KS_U_SET(ntxq, pi->ntxq);
3480 3525 KS_U_SET(first_rxq, pi->first_rxq);
3481 3526 KS_U_SET(first_txq, pi->first_txq);
3482 3527 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3483 3528 ddi_get_instance(pdip));
3484 3529 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3485 3530 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3486 3531
3487 3532 /* Do NOT set ksp->ks_update. These kstats do not change. */
3488 3533
3489 3534 /* Install the kstat */
3490 3535 ksp->ks_private = (void *)pi;
3491 3536 kstat_install(ksp);
3492 3537
3493 3538 return (ksp);
3494 3539 }
3495 3540
3496 3541 static kstat_t *
3497 3542 setup_port_info_kstats(struct port_info *pi)
3498 3543 {
3499 3544 kstat_t *ksp;
3500 3545 struct cxgbe_port_info_kstats *kstatp;
3501 3546 int ndata;
3502 3547
3503 3548 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3504 3549
3505 3550 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3506 3551 "net", KSTAT_TYPE_NAMED, ndata, 0);
3507 3552 if (ksp == NULL) {
3508 3553 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3509 3554 return (NULL);
3510 3555 }
3511 3556
3512 3557 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3513 3558
3514 3559 KS_CINIT(transceiver);
3515 3560 KS_UINIT(rx_ovflow0);
3516 3561 KS_UINIT(rx_ovflow1);
3517 3562 KS_UINIT(rx_ovflow2);
3518 3563 KS_UINIT(rx_ovflow3);
3519 3564 KS_UINIT(rx_trunc0);
3520 3565 KS_UINIT(rx_trunc1);
3521 3566 KS_UINIT(rx_trunc2);
3522 3567 KS_UINIT(rx_trunc3);
3523 3568 KS_UINIT(tx_pause);
3524 3569 KS_UINIT(rx_pause);
3525 3570
3526 3571 /* Install the kstat */
3527 3572 ksp->ks_update = update_port_info_kstats;
3528 3573 ksp->ks_private = (void *)pi;
3529 3574 kstat_install(ksp);
3530 3575
3531 3576 return (ksp);
3532 3577 }
3533 3578
3534 3579 static int
3535 3580 update_port_info_kstats(kstat_t *ksp, int rw)
3536 3581 {
3537 3582 struct cxgbe_port_info_kstats *kstatp =
3538 3583 (struct cxgbe_port_info_kstats *)ksp->ks_data;
3539 3584 struct port_info *pi = ksp->ks_private;
3540 3585 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3541 3586 "active TWINAX", "LRM" };
3542 3587 uint32_t bgmap;
3543 3588
3544 3589 if (rw == KSTAT_WRITE)
3545 3590 return (0);
3546 3591
3547 3592 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3548 3593 KS_C_SET(transceiver, "unplugged");
3549 3594 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3550 3595 KS_C_SET(transceiver, "unknown");
3551 3596 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3552 3597 KS_C_SET(transceiver, "unsupported");
3553 3598 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3554 3599 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3555 3600 else
3556 3601 KS_C_SET(transceiver, "type %d", pi->mod_type);
3557 3602
3558 3603 #define GET_STAT(name) t4_read_reg64(pi->adapter, \
3559 3604 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3560 3605 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3561 3606 A_MPS_STAT_##name##_L)
3562 3607
3563 3608 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3564 3609 if (bgmap == 0)
3565 3610 bgmap = (pi->port_id == 0) ? 0xf : 0;
3566 3611 else if (bgmap == 1)
3567 3612 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3568 3613 else
3569 3614 bgmap = 1;
3570 3615
3571 3616 KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3572 3617 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3573 3618 KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3574 3619 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3575 3620 KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3576 3621 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3577 3622 KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3578 3623 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3579 3624 KS_U_SET(rx_trunc0, (bgmap & 1) ?
3580 3625 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3581 3626 KS_U_SET(rx_trunc1, (bgmap & 2) ?
3582 3627 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3583 3628 KS_U_SET(rx_trunc2, (bgmap & 4) ?
3584 3629 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3585 3630 KS_U_SET(rx_trunc3, (bgmap & 8) ?
3586 3631 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3587 3632
3588 3633 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3589 3634 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3590 3635
3591 3636 return (0);
3592 3637
3593 3638 }
3594 3639
3595 3640 /*
3596 3641 * cxgbe:X:rxqY
3597 3642 */
3598 3643 struct rxq_kstats {
3599 3644 kstat_named_t rxcsum;
3600 3645 kstat_named_t rxpkts;
3601 3646 kstat_named_t rxbytes;
3602 3647 kstat_named_t nomem;
3603 3648 };
3604 3649
3605 3650 static kstat_t *
3606 3651 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3607 3652 {
3608 3653 struct kstat *ksp;
3609 3654 struct rxq_kstats *kstatp;
3610 3655 int ndata;
3611 3656 char str[16];
3612 3657
3613 3658 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3614 3659 (void) snprintf(str, sizeof (str), "rxq%u", idx);
3615 3660
3616 3661 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3617 3662 KSTAT_TYPE_NAMED, ndata, 0);
3618 3663 if (ksp == NULL) {
3619 3664 cxgb_printf(pi->dip, CE_WARN,
3620 3665 "%s: failed to initialize rxq kstats for queue %d.",
3621 3666 __func__, idx);
3622 3667 return (NULL);
3623 3668 }
3624 3669
3625 3670 kstatp = (struct rxq_kstats *)ksp->ks_data;
3626 3671
3627 3672 KS_UINIT(rxcsum);
3628 3673 KS_UINIT(rxpkts);
3629 3674 KS_UINIT(rxbytes);
3630 3675 KS_UINIT(nomem);
3631 3676
3632 3677 ksp->ks_update = update_rxq_kstats;
3633 3678 ksp->ks_private = (void *)rxq;
3634 3679 kstat_install(ksp);
3635 3680
3636 3681 return (ksp);
3637 3682 }
3638 3683
3639 3684 static int
3640 3685 update_rxq_kstats(kstat_t *ksp, int rw)
3641 3686 {
3642 3687 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3643 3688 struct sge_rxq *rxq = ksp->ks_private;
3644 3689
3645 3690 if (rw == KSTAT_WRITE)
3646 3691 return (0);
3647 3692
3648 3693 KS_U_FROM(rxcsum, rxq);
3649 3694 KS_U_FROM(rxpkts, rxq);
3650 3695 KS_U_FROM(rxbytes, rxq);
3651 3696 KS_U_FROM(nomem, rxq);
3652 3697
3653 3698 return (0);
3654 3699 }
3655 3700
3656 3701 /*
3657 3702 * cxgbe:X:txqY
3658 3703 */
3659 3704 struct txq_kstats {
3660 3705 kstat_named_t txcsum;
3661 3706 kstat_named_t tso_wrs;
3662 3707 kstat_named_t imm_wrs;
3663 3708 kstat_named_t sgl_wrs;
3664 3709 kstat_named_t txpkt_wrs;
3665 3710 kstat_named_t txpkts_wrs;
3666 3711 kstat_named_t txpkts_pkts;
3667 3712 kstat_named_t txb_used;
3668 3713 kstat_named_t hdl_used;
3669 3714 kstat_named_t txb_full;
3670 3715 kstat_named_t dma_hdl_failed;
3671 3716 kstat_named_t dma_map_failed;
3672 3717 kstat_named_t qfull;
3673 3718 kstat_named_t qflush;
3674 3719 kstat_named_t pullup_early;
3675 3720 kstat_named_t pullup_late;
3676 3721 kstat_named_t pullup_failed;
3677 3722 };
3678 3723
3679 3724 static kstat_t *
3680 3725 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3681 3726 {
3682 3727 struct kstat *ksp;
3683 3728 struct txq_kstats *kstatp;
3684 3729 int ndata;
3685 3730 char str[16];
3686 3731
3687 3732 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3688 3733 (void) snprintf(str, sizeof (str), "txq%u", idx);
3689 3734
3690 3735 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3691 3736 KSTAT_TYPE_NAMED, ndata, 0);
3692 3737 if (ksp == NULL) {
3693 3738 cxgb_printf(pi->dip, CE_WARN,
3694 3739 "%s: failed to initialize txq kstats for queue %d.",
3695 3740 __func__, idx);
3696 3741 return (NULL);
3697 3742 }
3698 3743
3699 3744 kstatp = (struct txq_kstats *)ksp->ks_data;
3700 3745
3701 3746 KS_UINIT(txcsum);
3702 3747 KS_UINIT(tso_wrs);
3703 3748 KS_UINIT(imm_wrs);
3704 3749 KS_UINIT(sgl_wrs);
3705 3750 KS_UINIT(txpkt_wrs);
3706 3751 KS_UINIT(txpkts_wrs);
3707 3752 KS_UINIT(txpkts_pkts);
3708 3753 KS_UINIT(txb_used);
3709 3754 KS_UINIT(hdl_used);
3710 3755 KS_UINIT(txb_full);
3711 3756 KS_UINIT(dma_hdl_failed);
3712 3757 KS_UINIT(dma_map_failed);
3713 3758 KS_UINIT(qfull);
3714 3759 KS_UINIT(qflush);
3715 3760 KS_UINIT(pullup_early);
3716 3761 KS_UINIT(pullup_late);
3717 3762 KS_UINIT(pullup_failed);
3718 3763
3719 3764 ksp->ks_update = update_txq_kstats;
3720 3765 ksp->ks_private = (void *)txq;
3721 3766 kstat_install(ksp);
3722 3767
3723 3768 return (ksp);
3724 3769 }
3725 3770
3726 3771 static int
3727 3772 update_txq_kstats(kstat_t *ksp, int rw)
3728 3773 {
3729 3774 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3730 3775 struct sge_txq *txq = ksp->ks_private;
3731 3776
3732 3777 if (rw == KSTAT_WRITE)
3733 3778 return (0);
3734 3779
3735 3780 KS_U_FROM(txcsum, txq);
3736 3781 KS_U_FROM(tso_wrs, txq);
3737 3782 KS_U_FROM(imm_wrs, txq);
3738 3783 KS_U_FROM(sgl_wrs, txq);
3739 3784 KS_U_FROM(txpkt_wrs, txq);
3740 3785 KS_U_FROM(txpkts_wrs, txq);
3741 3786 KS_U_FROM(txpkts_pkts, txq);
3742 3787 KS_U_FROM(txb_used, txq);
3743 3788 KS_U_FROM(hdl_used, txq);
3744 3789 KS_U_FROM(txb_full, txq);
3745 3790 KS_U_FROM(dma_hdl_failed, txq);
3746 3791 KS_U_FROM(dma_map_failed, txq);
3747 3792 KS_U_FROM(qfull, txq);
3748 3793 KS_U_FROM(qflush, txq);
3749 3794 KS_U_FROM(pullup_early, txq);
3750 3795 KS_U_FROM(pullup_late, txq);
3751 3796 KS_U_FROM(pullup_failed, txq);
3752 3797
3753 3798 return (0);
3754 3799 }
|
↓ open down ↓ |
381 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX