1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009-2012 Emulex. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27
28
29 /*
30 * Source file containing the implementation of the Transmit
31 * Path
32 */
33
34 #include <oce_impl.h>
35
36 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed);
37 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed,
38 mblk_t *mp, uint32_t pkt_len);
39 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
40 uint32_t pkt_len);
41 static inline oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq);
42 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd);
43
44 static void oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
45 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
46 static inline oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq);
47 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq);
48 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
49 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed);
50 static inline int oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm);
51
52
53 /*
54 * WQ map handle destructor
55 *
56 * wq - Pointer to WQ structure
57 * wqmd - pointer to WQE mapping handle descriptor
58 *
59 * return none
60 */
61
62 static void
63 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
64 {
65 _NOTE(ARGUNUSED(wq));
66 /* Free the DMA handle */
67 if (wqmd->dma_handle != NULL)
68 (void) ddi_dma_free_handle(&(wqmd->dma_handle));
69 wqmd->dma_handle = NULL;
70 } /* oce_wqm_dtor */
71
72 /*
73 * WQ map handles contructor
74 *
75 * wqmd - pointer to WQE mapping handle descriptor
76 * wq - Pointer to WQ structure
77 *
78 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
79 */
80 static int
81 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq)
82 {
83 struct oce_dev *dev;
84 int ret;
85 ddi_dma_attr_t tx_map_attr = {0};
86
87 dev = wq->parent;
88 /* Populate the DMA attributes structure */
89 tx_map_attr.dma_attr_version = DMA_ATTR_V0;
90 tx_map_attr.dma_attr_addr_lo = 0x0000000000000000ull;
91 tx_map_attr.dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull;
92 tx_map_attr.dma_attr_count_max = 0x00000000FFFFFFFFull;
93 tx_map_attr.dma_attr_align = OCE_TXMAP_ALIGN;
94 tx_map_attr.dma_attr_burstsizes = 0x000007FF;
95 tx_map_attr.dma_attr_minxfer = 0x00000001;
96 tx_map_attr.dma_attr_maxxfer = 0x00000000FFFFFFFFull;
97 tx_map_attr.dma_attr_seg = 0xFFFFFFFFFFFFFFFFull;
98 tx_map_attr.dma_attr_sgllen = OCE_MAX_TXDMA_COOKIES;
99 tx_map_attr.dma_attr_granular = 0x00000001;
100
101 if (DDI_FM_DMA_ERR_CAP(dev->fm_caps)) {
102 tx_map_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
103 }
104 /* Allocate DMA handle */
105 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_attr,
106 DDI_DMA_DONTWAIT, NULL, &wqmd->dma_handle);
107
108 return (ret);
109 } /* oce_wqm_ctor */
110
111 /*
112 * function to create WQ mapping handles cache
113 *
114 * wq - pointer to WQ structure
115 *
116 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
117 */
118 int
119 oce_wqm_cache_create(struct oce_wq *wq)
120 {
121 struct oce_dev *dev = wq->parent;
122 int size;
123 int cnt;
124 int ret;
125
126 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t);
127 wq->wq_mdesc_array = kmem_zalloc(size, KM_NOSLEEP);
128 if (wq->wq_mdesc_array == NULL) {
129 return (DDI_FAILURE);
130 }
131
132 wq->wqm_freelist =
133 kmem_zalloc(wq->cfg.nhdl * sizeof (oce_wq_mdesc_t *), KM_NOSLEEP);
134 if (wq->wqm_freelist == NULL) {
135 kmem_free(wq->wq_mdesc_array, size);
136 return (DDI_FAILURE);
137 }
138
139 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) {
140 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq);
141 if (ret != DDI_SUCCESS) {
142 goto wqm_fail;
143 }
144 wq->wqm_freelist[cnt] = &wq->wq_mdesc_array[cnt];
145 atomic_inc_32(&wq->wqm_free);
146 }
147
148 wq->wqmd_next_free = 0;
149 wq->wqmd_rc_head = 0;
150
151 mutex_init(&wq->wqm_alloc_lock, NULL, MUTEX_DRIVER,
152 DDI_INTR_PRI(dev->intr_pri));
153 mutex_init(&wq->wqm_free_lock, NULL, MUTEX_DRIVER,
154 DDI_INTR_PRI(dev->intr_pri));
155 return (DDI_SUCCESS);
156
157 wqm_fail:
158 oce_wqm_cache_destroy(wq);
159 return (DDI_FAILURE);
160 }
161
162 /*
163 * function to destroy WQ mapping handles cache
164 *
165 * wq - pointer to WQ structure
166 *
167 * return none
168 */
169 void
170 oce_wqm_cache_destroy(struct oce_wq *wq)
171 {
172 oce_wq_mdesc_t *wqmd;
173
174 while ((wqmd = oce_wqm_alloc(wq)) != NULL) {
175 oce_wqm_dtor(wq, wqmd);
176 }
177
178 mutex_destroy(&wq->wqm_alloc_lock);
179 mutex_destroy(&wq->wqm_free_lock);
180 kmem_free(wq->wqm_freelist,
181 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t *));
182 kmem_free(wq->wq_mdesc_array,
183 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t));
184 }
185
186 /*
187 * function to create WQ buffer cache
188 *
189 * wq - pointer to WQ structure
190 * buf_size - size of the buffer
191 *
192 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
193 */
194 int
195 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size)
196 {
197 struct oce_dev *dev = wq->parent;
198 oce_wq_bdesc_t *wqbd;
199 uint64_t paddr;
200 caddr_t vaddr;
201 int size;
202 int bufs_per_cookie = 0;
203 int tidx = 0;
204 int ncookies = 0;
205 int i = 0;
206 ddi_dma_cookie_t cookie;
207 ddi_dma_attr_t tx_buf_attr = {0};
208 int ret;
209
210 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t);
211 wq->wq_bdesc_array = kmem_zalloc(size, KM_NOSLEEP);
212 if (wq->wq_bdesc_array == NULL) {
213 return (DDI_FAILURE);
214 }
215
216 wq->wqb_freelist =
217 kmem_zalloc(wq->cfg.nbufs * sizeof (oce_wq_bdesc_t *), KM_NOSLEEP);
218 if (wq->wqb_freelist == NULL) {
219 kmem_free(wq->wq_bdesc_array,
220 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t));
221 return (DDI_FAILURE);
222 }
223
224 size = wq->cfg.nbufs * wq->cfg.buf_size;
225
226 /* Populate dma attributes */
227 tx_buf_attr.dma_attr_version = DMA_ATTR_V0;
228 tx_buf_attr.dma_attr_addr_lo = 0x0000000000000000ull;
229 tx_buf_attr.dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull;
230 tx_buf_attr.dma_attr_count_max = 0x00000000FFFFFFFFull;
231 tx_buf_attr.dma_attr_align = OCE_DMA_ALIGNMENT;
232 tx_buf_attr.dma_attr_burstsizes = 0x000007FF;
233 tx_buf_attr.dma_attr_minxfer = 0x00000001;
234 tx_buf_attr.dma_attr_maxxfer = 0x00000000FFFFFFFFull;
235 tx_buf_attr.dma_attr_seg = 0xFFFFFFFFFFFFFFFFull;
236 tx_buf_attr.dma_attr_granular = (uint32_t)buf_size;
237
238 if (DDI_FM_DMA_ERR_CAP(dev->fm_caps)) {
239 tx_buf_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
240 }
241
242 tx_buf_attr.dma_attr_sgllen = 1;
243
244 ret = oce_alloc_dma_buffer(dev, &wq->wqb, size, &tx_buf_attr,
245 DDI_DMA_STREAMING|DDI_DMA_WRITE);
246 if (ret != DDI_SUCCESS) {
247 tx_buf_attr.dma_attr_sgllen =
248 size/ddi_ptob(dev->dip, (ulong_t)1) + 2;
249 ret = oce_alloc_dma_buffer(dev, &wq->wqb, size, &tx_buf_attr,
250 DDI_DMA_STREAMING|DDI_DMA_WRITE);
251 if (ret != DDI_SUCCESS) {
252 kmem_free(wq->wq_bdesc_array,
253 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t));
254 kmem_free(wq->wqb_freelist,
255 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t *));
256 return (DDI_FAILURE);
257 }
258 }
259
260 wqbd = wq->wq_bdesc_array;
261 vaddr = wq->wqb.base;
262 cookie = wq->wqb.cookie;
263 ncookies = wq->wqb.ncookies;
264 do {
265 paddr = cookie.dmac_laddress;
266 bufs_per_cookie = cookie.dmac_size/buf_size;
267 for (i = 0; i < bufs_per_cookie; i++, wqbd++) {
268 wqbd->wqb.acc_handle = wq->wqb.acc_handle;
269 wqbd->wqb.dma_handle = wq->wqb.dma_handle;
270 wqbd->wqb.base = vaddr;
271 wqbd->wqb.addr = paddr;
272 wqbd->wqb.len = buf_size;
273 wqbd->wqb.size = buf_size;
274 wqbd->wqb.off = tidx * buf_size;
275 wqbd->frag_addr.dw.addr_lo = ADDR_LO(paddr);
276 wqbd->frag_addr.dw.addr_hi = ADDR_HI(paddr);
277 wq->wqb_freelist[tidx] = wqbd;
278 /* increment the addresses */
279 paddr += buf_size;
280 vaddr += buf_size;
281 atomic_inc_32(&wq->wqb_free);
282 tidx++;
283 if (tidx >= wq->cfg.nbufs)
284 break;
285 }
286 if (--ncookies > 0) {
287 (void) ddi_dma_nextcookie(wq->wqb.dma_handle, &cookie);
288 }
289 } while (ncookies > 0);
290
291 wq->wqbd_next_free = 0;
292 wq->wqbd_rc_head = 0;
293
294 mutex_init(&wq->wqb_alloc_lock, NULL, MUTEX_DRIVER,
295 DDI_INTR_PRI(dev->intr_pri));
296 mutex_init(&wq->wqb_free_lock, NULL, MUTEX_DRIVER,
297 DDI_INTR_PRI(dev->intr_pri));
298 return (DDI_SUCCESS);
299 }
300
301 /*
302 * function to destroy WQ buffer cache
303 *
304 * wq - pointer to WQ structure
305 *
306 * return none
307 */
308 void
309 oce_wqb_cache_destroy(struct oce_wq *wq)
310 {
311 /* Free Tx buffer dma memory */
312 oce_free_dma_buffer(wq->parent, &wq->wqb);
313
314 mutex_destroy(&wq->wqb_alloc_lock);
315 mutex_destroy(&wq->wqb_free_lock);
316 kmem_free(wq->wqb_freelist,
317 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t *));
318 wq->wqb_freelist = NULL;
319 kmem_free(wq->wq_bdesc_array,
320 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t));
321 }
322
323 /*
324 * function to alloc WQE buffer descriptor
325 *
326 * wq - pointer to WQ structure
327 *
328 * return pointer to WQE buffer descriptor
329 */
330 static inline oce_wq_bdesc_t *
331 oce_wqb_alloc(struct oce_wq *wq)
332 {
333 oce_wq_bdesc_t *wqbd;
334 if (oce_atomic_reserve(&wq->wqb_free, 1) < 0) {
335 return (NULL);
336 }
337
338 mutex_enter(&wq->wqb_alloc_lock);
339 wqbd = wq->wqb_freelist[wq->wqbd_next_free];
340 wq->wqb_freelist[wq->wqbd_next_free] = NULL;
341 wq->wqbd_next_free = GET_Q_NEXT(wq->wqbd_next_free, 1, wq->cfg.nbufs);
342 mutex_exit(&wq->wqb_alloc_lock);
343
344 return (wqbd);
345 }
346
347 /*
348 * function to free WQE buffer descriptor
349 *
350 * wq - pointer to WQ structure
351 * wqbd - pointer to WQ buffer descriptor
352 *
353 * return none
354 */
355 static inline void
356 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd)
357 {
358 mutex_enter(&wq->wqb_free_lock);
359 wq->wqb_freelist[wq->wqbd_rc_head] = wqbd;
360 wq->wqbd_rc_head = GET_Q_NEXT(wq->wqbd_rc_head, 1, wq->cfg.nbufs);
361 atomic_inc_32(&wq->wqb_free);
362 mutex_exit(&wq->wqb_free_lock);
363 } /* oce_wqb_free */
364
365 /*
366 * function to allocate WQE mapping descriptor
367 *
368 * wq - pointer to WQ structure
369 *
370 * return pointer to WQE mapping descriptor
371 */
372 static inline oce_wq_mdesc_t *
373 oce_wqm_alloc(struct oce_wq *wq)
374 {
375 oce_wq_mdesc_t *wqmd;
376
377 if (oce_atomic_reserve(&wq->wqm_free, 1) < 0) {
378 return (NULL);
379 }
380
381 mutex_enter(&wq->wqm_alloc_lock);
382 wqmd = wq->wqm_freelist[wq->wqmd_next_free];
383 wq->wqm_freelist[wq->wqmd_next_free] = NULL;
384 wq->wqmd_next_free = GET_Q_NEXT(wq->wqmd_next_free, 1, wq->cfg.nhdl);
385 mutex_exit(&wq->wqm_alloc_lock);
386
387 return (wqmd);
388 } /* oce_wqm_alloc */
389
390 /*
391 * function to insert WQE mapping descriptor to the list
392 *
393 * wq - pointer to WQ structure
394 * wqmd - Pointer to WQ mapping descriptor
395 *
396 * return none
397 */
398 static inline void
399 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
400 {
401 mutex_enter(&wq->wqm_free_lock);
402 wq->wqm_freelist[wq->wqmd_rc_head] = wqmd;
403 wq->wqmd_rc_head = GET_Q_NEXT(wq->wqmd_rc_head, 1, wq->cfg.nhdl);
404 atomic_inc_32(&wq->wqm_free);
405 mutex_exit(&wq->wqm_free_lock);
406 }
407
408 /*
409 * function to free WQE mapping descriptor
410 *
411 * wq - pointer to WQ structure
412 * wqmd - Pointer to WQ mapping descriptor
413 *
414 * return none
415 */
416 static void
417 oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
418 {
419 if (wqmd == NULL) {
420 return;
421 }
422 (void) ddi_dma_unbind_handle(wqmd->dma_handle);
423 oce_wqm_free(wq, wqmd);
424 }
425
426 /*
427 * WQED kmem_cache constructor
428 *
429 * buf - pointer to WQE descriptor
430 *
431 * return DDI_SUCCESS
432 */
433 int
434 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags)
435 {
436 _NOTE(ARGUNUSED(buf));
437 _NOTE(ARGUNUSED(arg));
438 _NOTE(ARGUNUSED(kmflags));
439
440 return (DDI_SUCCESS);
441 }
442
443 /*
444 * WQED kmem_cache destructor
445 *
446 * buf - pointer to WQE descriptor
447 *
448 * return none
449 */
450 void
451 oce_wqe_desc_dtor(void *buf, void *arg)
452 {
453 _NOTE(ARGUNUSED(buf));
454 _NOTE(ARGUNUSED(arg));
455 }
456
457 /*
458 * function to populate the single WQE
459 *
460 * wq - pointer to wq
461 * wqed - pointer to WQ entry descriptor
462 *
463 * return none
464 */
465 #pragma inline(oce_fill_ring_descs)
466 static void
467 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed)
468 {
469
470 struct oce_nic_frag_wqe *wqe;
471 int i;
472 /* Copy the precreate WQE descs to the ring desc */
473 for (i = 0; i < wqed->wqe_cnt; i++) {
474 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring,
475 struct oce_nic_frag_wqe);
476
477 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE);
478 RING_PUT(wq->ring, 1);
479 }
480 } /* oce_fill_ring_descs */
481
482 /*
483 * function to copy the packet to preallocated Tx buffer
484 *
485 * wq - pointer to WQ
486 * wqed - Pointer to WQE descriptor
487 * mp - Pointer to packet chain
488 * pktlen - Size of the packet
489 *
490 * return 0=>success, error code otherwise
491 */
492 static int
493 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
494 uint32_t pkt_len)
495 {
496 oce_wq_bdesc_t *wqbd;
497 caddr_t buf_va;
498 struct oce_dev *dev = wq->parent;
499 int len = 0;
500
501 wqbd = oce_wqb_alloc(wq);
502 if (wqbd == NULL) {
503 atomic_inc_32(&dev->tx_noxmtbuf);
504 oce_log(dev, CE_WARN, MOD_TX, "%s",
505 "wqb pool empty");
506 return (ENOMEM);
507 }
508
509 /* create a fragment wqe for the packet */
510 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi;
511 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo;
512 buf_va = DBUF_VA(wqbd->wqb);
513
514 /* copy pkt into buffer */
515 for (len = 0; mp != NULL && len < pkt_len; mp = mp->b_cont) {
516 bcopy(mp->b_rptr, buf_va, MBLKL(mp));
517 buf_va += MBLKL(mp);
518 len += MBLKL(mp);
519 }
520
521 DBUF_SYNC(wqbd->wqb, wqbd->wqb.off, pkt_len, DDI_DMA_SYNC_FORDEV);
522
523 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) {
524 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED);
525 /* Free the buffer */
526 oce_wqb_free(wq, wqbd);
527 return (EIO);
528 }
529 wqed->frag[wqed->frag_idx].u0.s.frag_len = pkt_len;
530 wqed->frag[wqed->frag_idx].u0.s.rsvd0 = 0;
531 wqed->hdesc[wqed->nhdl].hdl = (void *)(wqbd);
532 wqed->hdesc[wqed->nhdl].type = COPY_WQE;
533 wqed->frag_cnt++;
534 wqed->frag_idx++;
535 wqed->nhdl++;
536 return (0);
537 } /* oce_bcopy_wqe */
538
539 /*
540 * function to copy the packet or dma map on the fly depending on size
541 *
542 * wq - pointer to WQ
543 * wqed - Pointer to WQE descriptor
544 * mp - Pointer to packet chain
545 *
546 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
547 */
548 static int
549 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
550 uint32_t pkt_len)
551 {
552 ddi_dma_cookie_t cookie;
553 oce_wq_mdesc_t *wqmd;
554 uint32_t ncookies;
555 int ret;
556 struct oce_dev *dev = wq->parent;
557
558 wqmd = oce_wqm_alloc(wq);
559 if (wqmd == NULL) {
560 oce_log(dev, CE_WARN, MOD_TX, "%s",
561 "wqm pool empty");
562 return (ENOMEM);
563 }
564
565 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle,
566 (struct as *)0, (caddr_t)mp->b_rptr,
567 pkt_len, DDI_DMA_WRITE | DDI_DMA_STREAMING,
568 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies);
569 if (ret != DDI_DMA_MAPPED) {
570 oce_log(dev, CE_WARN, MOD_TX, "MAP FAILED %d",
571 ret);
572 /* free the last one */
573 oce_wqm_free(wq, wqmd);
574 return (ENOMEM);
575 }
576 do {
577 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi =
578 ADDR_HI(cookie.dmac_laddress);
579 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo =
580 ADDR_LO(cookie.dmac_laddress);
581 wqed->frag[wqed->frag_idx].u0.s.frag_len =
582 (uint32_t)cookie.dmac_size;
583 wqed->frag[wqed->frag_idx].u0.s.rsvd0 = 0;
584 wqed->frag_cnt++;
585 wqed->frag_idx++;
586 if (--ncookies > 0)
587 ddi_dma_nextcookie(wqmd->dma_handle, &cookie);
588 else
589 break;
590 } while (ncookies > 0);
591
592 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd;
593 wqed->hdesc[wqed->nhdl].type = MAPPED_WQE;
594 wqed->nhdl++;
595 return (0);
596 } /* oce_map_wqe */
597
598 static inline int
599 oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm)
600 {
601 struct oce_nic_tx_cqe *cqe;
602 uint16_t num_cqe = 0;
603 struct oce_cq *cq;
604 oce_wqe_desc_t *wqed;
605 int wqe_freed = 0;
606 struct oce_dev *dev;
607 list_t wqe_desc_list;
608
609 cq = wq->cq;
610 dev = wq->parent;
611
612 DBUF_SYNC(cq->ring->dbuf, 0, 0, DDI_DMA_SYNC_FORKERNEL);
613
614 list_create(&wqe_desc_list, sizeof (oce_wqe_desc_t),
615 offsetof(oce_wqe_desc_t, link));
616
617 mutex_enter(&wq->txc_lock);
618 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
619 while (WQ_CQE_VALID(cqe)) {
620
621 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe));
622
623 /* update stats */
624 if (cqe->u0.s.status != 0) {
625 atomic_inc_32(&dev->tx_errors);
626 }
627
628 mutex_enter(&wq->wqed_list_lock);
629 wqed = list_remove_head(&wq->wqe_desc_list);
630 mutex_exit(&wq->wqed_list_lock);
631 if (wqed == NULL) {
632 oce_log(dev, CE_NOTE, MOD_CONFIG, "%s",
633 "oce_process_tx_compl: wqed list empty");
634 break;
635 }
636 atomic_dec_32(&wq->wqe_pending);
637
638 wqe_freed += wqed->wqe_cnt;
639 list_insert_tail(&wqe_desc_list, wqed);
640 /* clear the valid bit and progress cqe */
641 WQ_CQE_INVALIDATE(cqe);
642 RING_GET(cq->ring, 1);
643
644 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring,
645 struct oce_nic_tx_cqe);
646 num_cqe++;
647 } /* for all valid CQE */
648
649 if (num_cqe == 0 && wq->wqe_pending > 0) {
650 mutex_exit(&wq->txc_lock);
651 return (0);
652 }
653
654 DBUF_SYNC(cq->ring->dbuf, 0, 0, DDI_DMA_SYNC_FORDEV);
655 oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm);
656
657 RING_GET(wq->ring, wqe_freed);
658 atomic_add_32(&wq->wq_free, wqe_freed);
659 if (wq->resched && wq->wq_free >= OCE_MAX_TX_HDL) {
660 wq->resched = B_FALSE;
661 mac_tx_ring_update(dev->mac_handle, wq->handle);
662 }
663 wq->last_compl = ddi_get_lbolt();
664 mutex_exit(&wq->txc_lock);
665 while (wqed = list_remove_head(&wqe_desc_list)) {
666 oce_free_wqed(wq, wqed);
667 }
668 list_destroy(&wqe_desc_list);
669 return (num_cqe);
670 } /* oce_process_tx_completion */
671
672 /*
673 * function to drain a TxCQ and process its CQEs
674 *
675 * dev - software handle to the device
676 * cq - pointer to the cq to drain
677 *
678 * return the number of CQEs processed
679 */
680 void *
681 oce_drain_wq_cq(void *arg, int arg2, int arg3)
682 {
683 struct oce_dev *dev;
684 struct oce_wq *wq;
685
686 _NOTE(ARGUNUSED(arg2));
687 _NOTE(ARGUNUSED(arg3));
688
689 wq = (struct oce_wq *)arg;
690 dev = wq->parent;
691 wq->last_intr = ddi_get_lbolt();
692 /* do while we do not reach a cqe that is not valid */
693 (void) oce_process_tx_compl(wq, B_FALSE);
694 (void) atomic_cas_uint(&wq->qmode, OCE_MODE_INTR, OCE_MODE_POLL);
695 if ((wq->wq_free > OCE_MAX_TX_HDL) && wq->resched) {
696 wq->resched = B_FALSE;
697 mac_tx_ring_update(dev->mac_handle, wq->handle);
698 }
699 return (NULL);
700
701 } /* oce_process_wq_cqe */
702
703
704 boolean_t
705 oce_tx_stall_check(struct oce_dev *dev)
706 {
707 struct oce_wq *wq;
708 int ring = 0;
709 boolean_t is_stalled = B_FALSE;
710
711 if (!(dev->state & STATE_MAC_STARTED) ||
712 (dev->link_status != LINK_STATE_UP)) {
713 return (B_FALSE);
714 }
715
716 for (ring = 0; ring < dev->tx_rings; ring++) {
717 wq = &dev->wq[ring];
718
719 if (wq->resched) {
720 if (wq->wq_free > OCE_MAX_TX_HDL) {
721 mac_tx_ring_update(dev->mac_handle, wq->handle);
722 } else {
723 /* enable the interrupts only once */
724 if (atomic_cas_uint(&wq->qmode, OCE_MODE_POLL,
725 OCE_MODE_INTR) == OCE_MODE_POLL) {
726 oce_arm_cq(dev, wq->cq->cq_id, 0,
727 B_TRUE);
728 }
729 }
730 }
731 }
732 return (is_stalled);
733 }
734 /*
735 * Function to check whether TX stall
736 * can occur for an IPV6 packet for
737 * some versions of BE cards
738 *
739 * dev - software handle to the device
740 * mp - Pointer to packet chain
741 * ipoffset - ip header offset in mp chain
742 *
743 * return B_TRUE or B_FALSE
744 */
745
746 static inline int
747 oce_check_ipv6_tx_stall(struct oce_dev *dev,
748 mblk_t *mp, uint32_t ip_offset) {
749
750 _NOTE(ARGUNUSED(dev));
751 ip6_t *ipv6_hdr;
752 struct ip6_opt *v6_op;
753 ipv6_hdr = (ip6_t *)(void *)
754 (mp->b_rptr + ip_offset);
755 v6_op = (struct ip6_opt *)(ipv6_hdr+1);
756 if (ipv6_hdr->ip6_nxt != IPPROTO_TCP &&
757 ipv6_hdr->ip6_nxt != IPPROTO_UDP &&
758 v6_op->ip6o_len == 0xFF) {
759 return (B_TRUE);
760 } else {
761 return (B_FALSE);
762 }
763 }
764 /*
765 * Function to insert VLAN Tag to
766 * mp cahin
767 *
768 * dev - software handle to the device
769 * mblk_haad - Pointer holding packet chain
770 *
771 * return DDI_FAILURE or DDI_SUCCESS
772 */
773
774 static int
775 oce_ipv6_tx_stall_workaround(struct oce_dev *dev,
776 mblk_t **mblk_head) {
777
778 mblk_t *mp;
779 struct ether_vlan_header *evh;
780 mp = allocb(OCE_HDR_LEN, BPRI_HI);
781 if (mp == NULL) {
782 return (DDI_FAILURE);
783 }
784 /* copy ether header */
785 (void) memcpy(mp->b_rptr, (*mblk_head)->b_rptr, 2 * ETHERADDRL);
786 evh = (struct ether_vlan_header *)(void *)mp->b_rptr;
787 evh->ether_tpid = htons(VLAN_TPID);
788 evh->ether_tci = ((dev->pvid > 0) ? LE_16(dev->pvid) :
789 htons(dev->QnQ_tag));
790 mp->b_wptr = mp->b_rptr + (2 * ETHERADDRL) + VTAG_SIZE;
791 (*mblk_head)->b_rptr += 2 * ETHERADDRL;
792
793 if (MBLKL(*mblk_head) > 0) {
794 mp->b_cont = *mblk_head;
795 } else {
796 mp->b_cont = (*mblk_head)->b_cont;
797 freeb(*mblk_head);
798 }
799 *mblk_head = mp;
800 return (DDI_SUCCESS);
801 }
802
803 /*
804 * function to xmit Single packet over the wire
805 *
806 * wq - pointer to WQ
807 * mp - Pointer to packet chain
808 *
809 * return pointer to the packet
810 */
811 mblk_t *
812 oce_send_packet(struct oce_wq *wq, mblk_t *mp)
813 {
814 struct oce_nic_hdr_wqe *wqeh;
815 struct oce_dev *dev;
816 struct ether_header *eh;
817 struct ether_vlan_header *evh;
818 int32_t num_wqes;
819 uint16_t etype;
820 uint32_t ip_offset;
821 uint32_t csum_flags = 0;
822 boolean_t use_copy = B_FALSE;
823 boolean_t tagged = B_FALSE;
824 boolean_t ipv6_stall = B_FALSE;
825 uint32_t reg_value = 0;
826 oce_wqe_desc_t *wqed = NULL;
827 mblk_t *nmp = NULL;
828 mblk_t *tmp = NULL;
829 uint32_t pkt_len = 0;
830 int num_mblks = 0;
831 int ret = 0;
832 uint32_t mss = 0;
833 uint32_t flags = 0;
834 int len = 0;
835
836 /* retrieve the adap priv struct ptr */
837 dev = wq->parent;
838
839 /* check if we have enough free slots */
840 if (wq->wq_free < dev->tx_reclaim_threshold) {
841 (void) oce_process_tx_compl(wq, B_FALSE);
842 }
843 if (wq->wq_free < OCE_MAX_TX_HDL) {
844 wq->resched = B_TRUE;
845 wq->last_defered = ddi_get_lbolt();
846 atomic_inc_32(&wq->tx_deferd);
847 return (mp);
848 }
849
850 /* check if we should copy */
851 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) {
852 pkt_len += MBLKL(tmp);
853 num_mblks++;
854 }
855
856 if (pkt_len == 0 || num_mblks == 0) {
857 freemsg(mp);
858 return (NULL);
859 }
860
861 /* retrieve LSO information */
862 mac_lso_get(mp, &mss, &flags);
863
864 /* get the offload flags */
865 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags);
866
867 /* restrict the mapped segment to wat we support */
868 if (num_mblks > OCE_MAX_TX_HDL) {
869 nmp = msgpullup(mp, -1);
870 if (nmp == NULL) {
871 atomic_inc_32(&wq->pkt_drops);
872 freemsg(mp);
873 return (NULL);
874 }
875 /* Reset it to new collapsed mp */
876 freemsg(mp);
877 mp = nmp;
878 /* restore the flags on new mp */
879 if (flags & HW_LSO) {
880 DB_CKSUMFLAGS(mp) |= HW_LSO;
881 DB_LSOMSS(mp) = (uint16_t)mss;
882 }
883 if (csum_flags != 0) {
884 DB_CKSUMFLAGS(mp) |= csum_flags;
885 }
886 }
887
888 /* Get the packet descriptor for Tx */
889 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP);
890 if (wqed == NULL) {
891 atomic_inc_32(&wq->pkt_drops);
892 freemsg(mp);
893 return (NULL);
894 }
895
896 /* Save the WQ pointer */
897 wqed->wq = wq;
898 wqed->frag_idx = 1; /* index zero is always header */
899 wqed->frag_cnt = 0;
900 wqed->nhdl = 0;
901 wqed->mp = NULL;
902
903 eh = (struct ether_header *)(void *)mp->b_rptr;
904 if (ntohs(eh->ether_type) == VLAN_TPID) {
905 evh = (struct ether_vlan_header *)(void *)mp->b_rptr;
906 tagged = B_TRUE;
907 etype = ntohs(evh->ether_type);
908 ip_offset = sizeof (struct ether_vlan_header);
909
910 } else {
911 etype = ntohs(eh->ether_type);
912 ip_offset = sizeof (struct ether_header);
913 }
914 /* Check Workaround required for IPV6 TX stall */
915 if (BE3_A1(dev) && (etype == ETHERTYPE_IPV6) &&
916 ((dev->QnQ_valid) || (!tagged && dev->pvid != 0))) {
917 len = ip_offset + sizeof (ip6_t) + sizeof (struct ip6_opt);
918 if (MBLKL(mp) < len) {
919 nmp = msgpullup(mp, len);
920 if (nmp == NULL) {
921 oce_free_wqed(wq, wqed);
922 atomic_inc_32(&wq->pkt_drops);
923 freemsg(mp);
924 return (NULL);
925 }
926 freemsg(mp);
927 mp = nmp;
928 }
929 ipv6_stall = oce_check_ipv6_tx_stall(dev, mp, ip_offset);
930 if (ipv6_stall) {
931 if (dev->QnQ_queried)
932 ret = oce_ipv6_tx_stall_workaround(dev, &mp);
933 else {
934 /* FW Workaround not available */
935 ret = DDI_FAILURE;
936 }
937 if (ret) {
938 oce_free_wqed(wq, wqed);
939 atomic_inc_32(&wq->pkt_drops);
940 freemsg(mp);
941 return (NULL);
942 }
943 pkt_len += VTAG_SIZE;
944 }
945 }
946
947 /* If entire packet is less than the copy limit just do copy */
948 if (pkt_len < dev->tx_bcopy_limit) {
949 use_copy = B_TRUE;
950 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len);
951 } else {
952 /* copy or dma map the individual fragments */
953 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
954 len = MBLKL(nmp);
955 if (len == 0) {
956 continue;
957 }
958 if (len < dev->tx_bcopy_limit) {
959 ret = oce_bcopy_wqe(wq, wqed, nmp, len);
960 } else {
961 ret = oce_map_wqe(wq, wqed, nmp, len);
962 }
963 if (ret != 0)
964 break;
965 }
966 }
967
968 /*
969 * Any failure other than insufficient Q entries
970 * drop the packet
971 */
972 if (ret != 0) {
973 oce_free_wqed(wq, wqed);
974 atomic_inc_32(&wq->pkt_drops);
975 freemsg(mp);
976 return (NULL);
977 }
978
979 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0];
980 bzero(wqeh, sizeof (struct oce_nic_hdr_wqe));
981
982 /* fill rest of wqe header fields based on packet */
983 if (flags & HW_LSO) {
984 wqeh->u0.s.lso = B_TRUE;
985 wqeh->u0.s.lso_mss = mss;
986 }
987 if (csum_flags & HCK_FULLCKSUM) {
988 uint8_t *proto;
989 if (etype == ETHERTYPE_IP) {
990 proto = (uint8_t *)(void *)
991 (mp->b_rptr + ip_offset);
992 if (proto[9] == 6)
993 /* IPPROTO_TCP */
994 wqeh->u0.s.tcpcs = B_TRUE;
995 else if (proto[9] == 17)
996 /* IPPROTO_UDP */
997 wqeh->u0.s.udpcs = B_TRUE;
998 }
999 }
1000
1001 if (csum_flags & HCK_IPV4_HDRCKSUM)
1002 wqeh->u0.s.ipcs = B_TRUE;
1003 if (ipv6_stall) {
1004 wqeh->u0.s.complete = B_FALSE;
1005 wqeh->u0.s.event = B_TRUE;
1006 } else {
1007
1008 wqeh->u0.s.complete = B_TRUE;
1009 wqeh->u0.s.event = B_TRUE;
1010 }
1011 wqeh->u0.s.crc = B_TRUE;
1012 wqeh->u0.s.total_length = pkt_len;
1013
1014 num_wqes = wqed->frag_cnt + 1;
1015
1016 /* h/w expects even no. of WQEs */
1017 if ((num_wqes & 0x1) && !(LANCER_CHIP(dev))) {
1018 bzero(&wqed->frag[num_wqes], sizeof (struct oce_nic_frag_wqe));
1019 num_wqes++;
1020 }
1021 wqed->wqe_cnt = (uint16_t)num_wqes;
1022 wqeh->u0.s.num_wqe = num_wqes;
1023 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE));
1024
1025 mutex_enter(&wq->tx_lock);
1026 if (num_wqes > wq->wq_free - 2) {
1027 atomic_inc_32(&wq->tx_deferd);
1028 mutex_exit(&wq->tx_lock);
1029 goto wqe_fail;
1030 }
1031 atomic_add_32(&wq->wq_free, -num_wqes);
1032
1033 /* fill the wq for adapter */
1034 oce_fill_ring_descs(wq, wqed);
1035
1036 /* Set the mp pointer in the wqe descriptor */
1037 if (use_copy == B_FALSE) {
1038 wqed->mp = mp;
1039 }
1040 /* Add the packet desc to list to be retrieved during cmpl */
1041 mutex_enter(&wq->wqed_list_lock);
1042 list_insert_tail(&wq->wqe_desc_list, wqed);
1043 mutex_exit(&wq->wqed_list_lock);
1044 atomic_inc_32(&wq->wqe_pending);
1045 DBUF_SYNC(wq->ring->dbuf, 0, 0, DDI_DMA_SYNC_FORDEV);
1046
1047 /* ring tx doorbell */
1048 reg_value = (num_wqes << 16) | wq->wq_id;
1049 /* Ring the door bell */
1050 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value);
1051
1052 /* update the ring stats */
1053 wq->stat_bytes += pkt_len;
1054 wq->stat_pkts++;
1055
1056 mutex_exit(&wq->tx_lock);
1057 if (oce_fm_check_acc_handle(dev, dev->db_handle) != DDI_FM_OK) {
1058 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED);
1059 }
1060
1061 /* free mp if copied or packet chain collapsed */
1062 if (use_copy == B_TRUE) {
1063 freemsg(mp);
1064 }
1065 return (NULL);
1066
1067 wqe_fail:
1068
1069 oce_free_wqed(wq, wqed);
1070 wq->resched = B_TRUE;
1071 wq->last_defered = ddi_get_lbolt();
1072 return (mp);
1073 } /* oce_send_packet */
1074
1075 /*
1076 * function to free the WQE descriptor
1077 *
1078 * wq - pointer to WQ
1079 * wqed - Pointer to WQE descriptor
1080 *
1081 * return none
1082 */
1083 #pragma inline(oce_free_wqed)
1084 static void
1085 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed)
1086 {
1087 int i = 0;
1088 if (wqed == NULL) {
1089 return;
1090 }
1091
1092 for (i = 0; i < wqed->nhdl; i++) {
1093 if (wqed->hdesc[i].type == COPY_WQE) {
1094 oce_wqb_free(wq, wqed->hdesc[i].hdl);
1095 } else if (wqed->hdesc[i].type == MAPPED_WQE) {
1096 oce_wqmd_free(wq, wqed->hdesc[i].hdl);
1097 }
1098 }
1099 if (wqed->mp)
1100 freemsg(wqed->mp);
1101 kmem_cache_free(wq->wqed_cache, wqed);
1102 } /* oce_free_wqed */
1103
1104 /*
1105 * function to start the WQ
1106 *
1107 * wq - pointer to WQ
1108 *
1109 * return DDI_SUCCESS
1110 */
1111
1112 int
1113 oce_start_wq(struct oce_wq *wq)
1114 {
1115 _NOTE(ARGUNUSED(wq));
1116 return (DDI_SUCCESS);
1117 } /* oce_start_wq */
1118
1119 /*
1120 * function to stop the WQ
1121 *
1122 * wq - pointer to WQ
1123 *
1124 * return none
1125 */
1126 void
1127 oce_clean_wq(struct oce_wq *wq)
1128 {
1129 oce_wqe_desc_t *wqed;
1130 int ti;
1131
1132 /* Wait for already posted Tx to complete */
1133
1134 for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) {
1135 (void) oce_process_tx_compl(wq, B_FALSE);
1136 OCE_MSDELAY(1);
1137 }
1138
1139 /* Free the remaining descriptors */
1140 mutex_enter(&wq->wqed_list_lock);
1141 while ((wqed = list_remove_head(&wq->wqe_desc_list)) != NULL) {
1142 atomic_add_32(&wq->wq_free, wqed->wqe_cnt);
1143 oce_free_wqed(wq, wqed);
1144 }
1145 mutex_exit(&wq->wqed_list_lock);
1146 oce_drain_eq(wq->cq->eq);
1147 } /* oce_stop_wq */