Print this page
3014 Intel X540 Support
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ixgbe/ixgbe_tx.c
+++ new/usr/src/uts/common/io/ixgbe/ixgbe_tx.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
|
↓ open down ↓ |
17 lines elided |
↑ open up ↑ |
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 24 */
25 25
26 26 /*
27 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 + * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
28 29 */
29 30
30 31 #include "ixgbe_sw.h"
31 32
32 33 static int ixgbe_tx_copy(ixgbe_tx_ring_t *, tx_control_block_t *, mblk_t *,
33 34 uint32_t, boolean_t);
34 35 static int ixgbe_tx_bind(ixgbe_tx_ring_t *, tx_control_block_t *, mblk_t *,
35 36 uint32_t);
36 37 static int ixgbe_tx_fill_ring(ixgbe_tx_ring_t *, link_list_t *,
37 38 ixgbe_tx_context_t *, size_t);
38 39 static void ixgbe_save_desc(tx_control_block_t *, uint64_t, size_t);
39 40 static tx_control_block_t *ixgbe_get_free_list(ixgbe_tx_ring_t *);
40 41
41 42 static int ixgbe_get_context(mblk_t *, ixgbe_tx_context_t *);
42 43 static boolean_t ixgbe_check_context(ixgbe_tx_ring_t *,
43 44 ixgbe_tx_context_t *);
44 45 static void ixgbe_fill_context(struct ixgbe_adv_tx_context_desc *,
45 46 ixgbe_tx_context_t *);
46 47
47 48 #ifndef IXGBE_DEBUG
48 49 #pragma inline(ixgbe_save_desc)
49 50 #pragma inline(ixgbe_get_context)
50 51 #pragma inline(ixgbe_check_context)
51 52 #pragma inline(ixgbe_fill_context)
52 53 #endif
53 54
54 55 /*
55 56 * ixgbe_ring_tx
56 57 *
57 58 * To transmit one mblk through one specified ring.
58 59 *
59 60 * One mblk can consist of several fragments, each fragment
60 61 * will be processed with different methods based on the size.
61 62 * For the fragments with size less than the bcopy threshold,
62 63 * they will be processed by using bcopy; otherwise, they will
63 64 * be processed by using DMA binding.
64 65 *
65 66 * To process the mblk, a tx control block is got from the
66 67 * free list. One tx control block contains one tx buffer, which
67 68 * is used to copy mblk fragments' data; and one tx DMA handle,
68 69 * which is used to bind a mblk fragment with DMA resource.
69 70 *
70 71 * Several small mblk fragments can be copied into one tx control
71 72 * block's buffer, and then the buffer will be transmitted with
72 73 * one tx descriptor.
73 74 *
74 75 * A large fragment only binds with one tx control block's DMA
75 76 * handle, and it can span several tx descriptors for transmitting.
76 77 *
77 78 * So to transmit a packet (mblk), several tx control blocks can
78 79 * be used. After the processing, those tx control blocks will
79 80 * be put to the work list.
80 81 */
81 82 mblk_t *
82 83 ixgbe_ring_tx(void *arg, mblk_t *mp)
83 84 {
84 85 ixgbe_tx_ring_t *tx_ring = (ixgbe_tx_ring_t *)arg;
85 86 ixgbe_t *ixgbe = tx_ring->ixgbe;
86 87 tx_type_t current_flag, next_flag;
87 88 uint32_t current_len, next_len;
88 89 uint32_t desc_total;
89 90 size_t mbsize;
90 91 int desc_num;
91 92 boolean_t copy_done, eop;
92 93 mblk_t *current_mp, *next_mp, *nmp, *pull_mp = NULL;
93 94 tx_control_block_t *tcb;
94 95 ixgbe_tx_context_t tx_context, *ctx;
95 96 link_list_t pending_list;
96 97 uint32_t len, hdr_frag_len, hdr_len;
97 98 uint32_t copy_thresh;
98 99 mblk_t *hdr_new_mp = NULL;
99 100 mblk_t *hdr_pre_mp = NULL;
100 101 mblk_t *hdr_nmp = NULL;
101 102
102 103 ASSERT(mp->b_next == NULL);
103 104
104 105 if ((ixgbe->ixgbe_state & IXGBE_SUSPENDED) ||
105 106 (ixgbe->ixgbe_state & IXGBE_ERROR) ||
106 107 (ixgbe->ixgbe_state & IXGBE_OVERTEMP) ||
107 108 !(ixgbe->ixgbe_state & IXGBE_STARTED)) {
108 109 return (mp);
109 110 }
110 111
111 112 copy_thresh = ixgbe->tx_copy_thresh;
112 113
113 114 /* Get the mblk size */
114 115 mbsize = 0;
115 116 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
116 117 mbsize += MBLKL(nmp);
117 118 }
118 119
119 120 if (ixgbe->tx_hcksum_enable) {
120 121 /*
121 122 * Retrieve checksum context information from the mblk
122 123 * that will be used to decide whether/how to fill the
123 124 * context descriptor.
124 125 */
125 126 ctx = &tx_context;
126 127 if (ixgbe_get_context(mp, ctx) < 0) {
127 128 freemsg(mp);
128 129 return (NULL);
129 130 }
130 131
131 132 /*
132 133 * If the mblk size exceeds the max size ixgbe could
133 134 * process, then discard this mblk, and return NULL.
134 135 */
135 136 if ((ctx->lso_flag &&
136 137 ((mbsize - ctx->mac_hdr_len) > IXGBE_LSO_MAXLEN)) ||
137 138 (!ctx->lso_flag &&
138 139 (mbsize > (ixgbe->max_frame_size - ETHERFCSL)))) {
139 140 freemsg(mp);
140 141 IXGBE_DEBUGLOG_0(ixgbe, "ixgbe_tx: packet oversize");
141 142 return (NULL);
142 143 }
143 144 } else {
144 145 ctx = NULL;
145 146 }
146 147
147 148 /*
148 149 * Check and recycle tx descriptors.
149 150 * The recycle threshold here should be selected carefully
150 151 */
151 152 if (tx_ring->tbd_free < ixgbe->tx_recycle_thresh) {
152 153 tx_ring->tx_recycle(tx_ring);
153 154 }
154 155
155 156 /*
156 157 * After the recycling, if the tbd_free is less than the
157 158 * overload_threshold, assert overload, return mp;
158 159 * and we need to re-schedule the tx again.
159 160 */
160 161 if (tx_ring->tbd_free < ixgbe->tx_overload_thresh) {
161 162 tx_ring->reschedule = B_TRUE;
162 163 IXGBE_DEBUG_STAT(tx_ring->stat_overload);
163 164 return (mp);
164 165 }
165 166
166 167 /*
167 168 * The pending_list is a linked list that is used to save
168 169 * the tx control blocks that have packet data processed
169 170 * but have not put the data to the tx descriptor ring.
170 171 * It is used to reduce the lock contention of the tx_lock.
171 172 */
172 173 LINK_LIST_INIT(&pending_list);
173 174 desc_num = 0;
174 175 desc_total = 0;
175 176
176 177 /*
177 178 * The software should guarantee LSO packet header(MAC+IP+TCP)
178 179 * to be within one descriptor. Here we reallocate and refill the
179 180 * the header if it's physical memory non-contiguous.
180 181 */
181 182 if ((ctx != NULL) && ctx->lso_flag) {
182 183 /* find the last fragment of the header */
183 184 len = MBLKL(mp);
184 185 ASSERT(len > 0);
185 186 hdr_nmp = mp;
186 187 hdr_len = ctx->ip_hdr_len + ctx->mac_hdr_len + ctx->l4_hdr_len;
187 188 while (len < hdr_len) {
188 189 hdr_pre_mp = hdr_nmp;
189 190 hdr_nmp = hdr_nmp->b_cont;
190 191 len += MBLKL(hdr_nmp);
191 192 }
192 193 /*
193 194 * If the header and the payload are in different mblks,
194 195 * we simply force the header to be copied into pre-allocated
195 196 * page-aligned buffer.
196 197 */
197 198 if (len == hdr_len)
198 199 goto adjust_threshold;
199 200
200 201 hdr_frag_len = hdr_len - (len - MBLKL(hdr_nmp));
201 202 /*
202 203 * There are two cases we need to reallocate a mblk for the
203 204 * last header fragment:
204 205 * 1. the header is in multiple mblks and the last fragment
205 206 * share the same mblk with the payload
206 207 * 2. the header is in a single mblk shared with the payload
207 208 * and the header is physical memory non-contiguous
208 209 */
209 210 if ((hdr_nmp != mp) ||
210 211 (P2NPHASE((uintptr_t)hdr_nmp->b_rptr, ixgbe->sys_page_size)
211 212 < hdr_len)) {
212 213 IXGBE_DEBUG_STAT(tx_ring->stat_lso_header_fail);
213 214 /*
214 215 * reallocate the mblk for the last header fragment,
215 216 * expect to bcopy into pre-allocated page-aligned
216 217 * buffer
217 218 */
218 219 hdr_new_mp = allocb(hdr_frag_len, NULL);
219 220 if (!hdr_new_mp)
220 221 return (mp);
221 222 bcopy(hdr_nmp->b_rptr, hdr_new_mp->b_rptr,
222 223 hdr_frag_len);
223 224 /* link the new header fragment with the other parts */
224 225 hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len;
225 226 hdr_new_mp->b_cont = hdr_nmp;
226 227 if (hdr_pre_mp)
227 228 hdr_pre_mp->b_cont = hdr_new_mp;
228 229 else
229 230 mp = hdr_new_mp;
230 231 hdr_nmp->b_rptr += hdr_frag_len;
231 232 }
232 233 adjust_threshold:
233 234 /*
234 235 * adjust the bcopy threshhold to guarantee
235 236 * the header to use bcopy way
236 237 */
237 238 if (copy_thresh < hdr_len)
238 239 copy_thresh = hdr_len;
239 240 }
240 241
241 242 current_mp = mp;
242 243 current_len = MBLKL(current_mp);
243 244 /*
244 245 * Decide which method to use for the first fragment
245 246 */
246 247 current_flag = (current_len <= copy_thresh) ?
247 248 USE_COPY : USE_DMA;
248 249 /*
249 250 * If the mblk includes several contiguous small fragments,
250 251 * they may be copied into one buffer. This flag is used to
251 252 * indicate whether there are pending fragments that need to
252 253 * be copied to the current tx buffer.
253 254 *
254 255 * If this flag is B_TRUE, it indicates that a new tx control
255 256 * block is needed to process the next fragment using either
256 257 * copy or DMA binding.
257 258 *
258 259 * Otherwise, it indicates that the next fragment will be
259 260 * copied to the current tx buffer that is maintained by the
260 261 * current tx control block. No new tx control block is needed.
261 262 */
262 263 copy_done = B_TRUE;
263 264 while (current_mp) {
264 265 next_mp = current_mp->b_cont;
265 266 eop = (next_mp == NULL); /* Last fragment of the packet? */
266 267 next_len = eop ? 0: MBLKL(next_mp);
267 268
268 269 /*
269 270 * When the current fragment is an empty fragment, if
270 271 * the next fragment will still be copied to the current
271 272 * tx buffer, we cannot skip this fragment here. Because
272 273 * the copy processing is pending for completion. We have
273 274 * to process this empty fragment in the tx_copy routine.
274 275 *
275 276 * If the copy processing is completed or a DMA binding
276 277 * processing is just completed, we can just skip this
277 278 * empty fragment.
278 279 */
279 280 if ((current_len == 0) && (copy_done)) {
280 281 current_mp = next_mp;
281 282 current_len = next_len;
282 283 current_flag = (current_len <= copy_thresh) ?
283 284 USE_COPY : USE_DMA;
284 285 continue;
285 286 }
286 287
287 288 if (copy_done) {
288 289 /*
289 290 * Get a new tx control block from the free list
290 291 */
291 292 tcb = ixgbe_get_free_list(tx_ring);
292 293
293 294 if (tcb == NULL) {
294 295 IXGBE_DEBUG_STAT(tx_ring->stat_fail_no_tcb);
295 296 goto tx_failure;
296 297 }
297 298
298 299 /*
299 300 * Push the tx control block to the pending list
300 301 * to avoid using lock too early
301 302 */
302 303 LIST_PUSH_TAIL(&pending_list, &tcb->link);
303 304 }
304 305
305 306 if (current_flag == USE_COPY) {
306 307 /*
307 308 * Check whether to use bcopy or DMA binding to process
308 309 * the next fragment, and if using bcopy, whether we
309 310 * need to continue copying the next fragment into the
310 311 * current tx buffer.
311 312 */
312 313 ASSERT((tcb->tx_buf.len + current_len) <=
313 314 tcb->tx_buf.size);
314 315
315 316 if (eop) {
316 317 /*
317 318 * This is the last fragment of the packet, so
318 319 * the copy processing will be completed with
319 320 * this fragment.
320 321 */
321 322 next_flag = USE_NONE;
322 323 copy_done = B_TRUE;
323 324 } else if ((tcb->tx_buf.len + current_len + next_len) >
324 325 tcb->tx_buf.size) {
325 326 /*
326 327 * If the next fragment is too large to be
327 328 * copied to the current tx buffer, we need
328 329 * to complete the current copy processing.
329 330 */
330 331 next_flag = (next_len > copy_thresh) ?
331 332 USE_DMA: USE_COPY;
332 333 copy_done = B_TRUE;
333 334 } else if (next_len > copy_thresh) {
334 335 /*
335 336 * The next fragment needs to be processed with
336 337 * DMA binding. So the copy prcessing will be
337 338 * completed with the current fragment.
338 339 */
339 340 next_flag = USE_DMA;
340 341 copy_done = B_TRUE;
341 342 } else {
342 343 /*
343 344 * Continue to copy the next fragment to the
344 345 * current tx buffer.
345 346 */
346 347 next_flag = USE_COPY;
347 348 copy_done = B_FALSE;
348 349 }
349 350
350 351 desc_num = ixgbe_tx_copy(tx_ring, tcb, current_mp,
351 352 current_len, copy_done);
352 353 } else {
353 354 /*
354 355 * Check whether to use bcopy or DMA binding to process
355 356 * the next fragment.
356 357 */
357 358 next_flag = (next_len > copy_thresh) ?
358 359 USE_DMA: USE_COPY;
359 360 ASSERT(copy_done == B_TRUE);
360 361
361 362 desc_num = ixgbe_tx_bind(tx_ring, tcb, current_mp,
362 363 current_len);
363 364 }
364 365
365 366 if (desc_num > 0)
366 367 desc_total += desc_num;
367 368 else if (desc_num < 0)
368 369 goto tx_failure;
369 370
370 371 current_mp = next_mp;
371 372 current_len = next_len;
372 373 current_flag = next_flag;
373 374 }
374 375
375 376 /*
376 377 * Attach the mblk to the last tx control block
377 378 */
378 379 ASSERT(tcb);
379 380 ASSERT(tcb->mp == NULL);
380 381 tcb->mp = mp;
381 382
382 383 /*
383 384 * 82598/82599 chipset has a limitation that no more than 32 tx
384 385 * descriptors can be transmited out at one time.
385 386 *
386 387 * Here is a workaround for it: pull up the mblk then send it
387 388 * out with bind way. By doing so, no more than MAX_COOKIE (18)
388 389 * descriptors is needed.
389 390 */
390 391 if (desc_total + 1 > IXGBE_TX_DESC_LIMIT) {
391 392 IXGBE_DEBUG_STAT(tx_ring->stat_break_tbd_limit);
392 393
393 394 /*
394 395 * Discard the mblk and free the used resources
395 396 */
396 397 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
397 398 while (tcb) {
398 399 tcb->mp = NULL;
399 400 ixgbe_free_tcb(tcb);
400 401 tcb = (tx_control_block_t *)
401 402 LIST_GET_NEXT(&pending_list, &tcb->link);
402 403 }
403 404
404 405 /*
405 406 * Return the tx control blocks in the pending list to
406 407 * the free list.
407 408 */
408 409 ixgbe_put_free_list(tx_ring, &pending_list);
409 410
410 411 /*
411 412 * pull up the mblk and send it out with bind way
412 413 */
413 414 if ((pull_mp = msgpullup(mp, -1)) == NULL) {
414 415 tx_ring->reschedule = B_TRUE;
415 416
416 417 /*
417 418 * If new mblk has been allocted for the last header
418 419 * fragment of a LSO packet, we should restore the
419 420 * modified mp.
420 421 */
421 422 if (hdr_new_mp) {
422 423 hdr_new_mp->b_cont = NULL;
423 424 freeb(hdr_new_mp);
424 425 hdr_nmp->b_rptr -= hdr_frag_len;
425 426 if (hdr_pre_mp)
426 427 hdr_pre_mp->b_cont = hdr_nmp;
427 428 else
428 429 mp = hdr_nmp;
429 430 }
430 431 return (mp);
431 432 }
432 433
433 434 LINK_LIST_INIT(&pending_list);
434 435 desc_total = 0;
435 436
436 437 /*
437 438 * if the packet is a LSO packet, we simply
438 439 * transmit the header in one descriptor using the copy way
439 440 */
440 441 if ((ctx != NULL) && ctx->lso_flag) {
441 442 hdr_len = ctx->ip_hdr_len + ctx->mac_hdr_len +
442 443 ctx->l4_hdr_len;
443 444
444 445 tcb = ixgbe_get_free_list(tx_ring);
445 446 if (tcb == NULL) {
446 447 IXGBE_DEBUG_STAT(tx_ring->stat_fail_no_tcb);
447 448 goto tx_failure;
448 449 }
449 450 desc_num = ixgbe_tx_copy(tx_ring, tcb, pull_mp,
450 451 hdr_len, B_TRUE);
451 452 LIST_PUSH_TAIL(&pending_list, &tcb->link);
452 453 desc_total += desc_num;
453 454
454 455 pull_mp->b_rptr += hdr_len;
455 456 }
456 457
457 458 tcb = ixgbe_get_free_list(tx_ring);
458 459 if (tcb == NULL) {
459 460 IXGBE_DEBUG_STAT(tx_ring->stat_fail_no_tcb);
460 461 goto tx_failure;
461 462 }
462 463 if ((ctx != NULL) && ctx->lso_flag) {
463 464 desc_num = ixgbe_tx_bind(tx_ring, tcb, pull_mp,
464 465 mbsize - hdr_len);
465 466 } else {
466 467 desc_num = ixgbe_tx_bind(tx_ring, tcb, pull_mp,
467 468 mbsize);
468 469 }
469 470 if (desc_num < 0) {
470 471 goto tx_failure;
471 472 }
472 473 LIST_PUSH_TAIL(&pending_list, &tcb->link);
473 474
474 475 desc_total += desc_num;
475 476 tcb->mp = pull_mp;
476 477 }
477 478
478 479 /*
479 480 * Before fill the tx descriptor ring with the data, we need to
480 481 * ensure there are adequate free descriptors for transmit
481 482 * (including one context descriptor).
482 483 * Do not use up all the tx descriptors.
483 484 * Otherwise tx recycle will fail and cause false hang.
484 485 */
485 486 if (tx_ring->tbd_free <= (desc_total + 1)) {
486 487 tx_ring->tx_recycle(tx_ring);
487 488 }
488 489
489 490 mutex_enter(&tx_ring->tx_lock);
490 491 /*
491 492 * If the number of free tx descriptors is not enough for transmit
492 493 * then return mp.
493 494 *
494 495 * Note: we must put this check under the mutex protection to
495 496 * ensure the correctness when multiple threads access it in
496 497 * parallel.
497 498 */
498 499 if (tx_ring->tbd_free <= (desc_total + 1)) {
499 500 IXGBE_DEBUG_STAT(tx_ring->stat_fail_no_tbd);
500 501 mutex_exit(&tx_ring->tx_lock);
501 502 goto tx_failure;
502 503 }
503 504
504 505 desc_num = ixgbe_tx_fill_ring(tx_ring, &pending_list, ctx,
505 506 mbsize);
506 507
507 508 ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1)));
508 509
509 510 tx_ring->stat_obytes += mbsize;
510 511 tx_ring->stat_opackets ++;
511 512
512 513 mutex_exit(&tx_ring->tx_lock);
513 514
514 515 /*
515 516 * now that the transmission succeeds, need to free the original
516 517 * mp if we used the pulling up mblk for transmission.
517 518 */
518 519 if (pull_mp) {
519 520 freemsg(mp);
520 521 }
521 522
522 523 return (NULL);
523 524
524 525 tx_failure:
525 526 /*
526 527 * If transmission fails, need to free the pulling up mblk.
527 528 */
528 529 if (pull_mp) {
529 530 freemsg(pull_mp);
530 531 }
531 532
532 533 /*
533 534 * If new mblk has been allocted for the last header
534 535 * fragment of a LSO packet, we should restore the
535 536 * modified mp.
536 537 */
537 538 if (hdr_new_mp) {
538 539 hdr_new_mp->b_cont = NULL;
539 540 freeb(hdr_new_mp);
540 541 hdr_nmp->b_rptr -= hdr_frag_len;
541 542 if (hdr_pre_mp)
542 543 hdr_pre_mp->b_cont = hdr_nmp;
543 544 else
544 545 mp = hdr_nmp;
545 546 }
546 547 /*
547 548 * Discard the mblk and free the used resources
548 549 */
549 550 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
550 551 while (tcb) {
551 552 tcb->mp = NULL;
552 553
553 554 ixgbe_free_tcb(tcb);
554 555
555 556 tcb = (tx_control_block_t *)
556 557 LIST_GET_NEXT(&pending_list, &tcb->link);
557 558 }
558 559
559 560 /*
560 561 * Return the tx control blocks in the pending list to the free list.
561 562 */
562 563 ixgbe_put_free_list(tx_ring, &pending_list);
563 564
564 565 /* Transmit failed, do not drop the mblk, rechedule the transmit */
565 566 tx_ring->reschedule = B_TRUE;
566 567
567 568 return (mp);
568 569 }
569 570
570 571 /*
571 572 * ixgbe_tx_copy
572 573 *
573 574 * Copy the mblk fragment to the pre-allocated tx buffer
574 575 */
575 576 static int
576 577 ixgbe_tx_copy(ixgbe_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
577 578 uint32_t len, boolean_t copy_done)
578 579 {
579 580 dma_buffer_t *tx_buf;
580 581 uint32_t desc_num;
581 582 _NOTE(ARGUNUSED(tx_ring));
582 583
583 584 tx_buf = &tcb->tx_buf;
584 585
585 586 /*
586 587 * Copy the packet data of the mblk fragment into the
587 588 * pre-allocated tx buffer, which is maintained by the
588 589 * tx control block.
589 590 *
590 591 * Several mblk fragments can be copied into one tx buffer.
591 592 * The destination address of the current copied fragment in
592 593 * the tx buffer is next to the end of the previous copied
593 594 * fragment.
594 595 */
595 596 if (len > 0) {
596 597 bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len);
597 598
598 599 tx_buf->len += len;
599 600 tcb->frag_num++;
600 601 }
601 602
602 603 desc_num = 0;
603 604
604 605 /*
605 606 * If it is the last fragment copied to the current tx buffer,
606 607 * in other words, if there's no remaining fragment or the remaining
607 608 * fragment requires a new tx control block to process, we need to
608 609 * complete the current copy processing by syncing up the current
609 610 * DMA buffer and saving the descriptor data.
610 611 */
611 612 if (copy_done) {
612 613 /*
613 614 * Sync the DMA buffer of the packet data
614 615 */
615 616 DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV);
616 617
617 618 tcb->tx_type = USE_COPY;
618 619
619 620 /*
620 621 * Save the address and length to the private data structure
621 622 * of the tx control block, which will be used to fill the
622 623 * tx descriptor ring after all the fragments are processed.
623 624 */
624 625 ixgbe_save_desc(tcb, tx_buf->dma_address, tx_buf->len);
625 626 desc_num++;
626 627 }
627 628
628 629 return (desc_num);
629 630 }
630 631
631 632 /*
632 633 * ixgbe_tx_bind
633 634 *
634 635 * Bind the mblk fragment with DMA
635 636 */
636 637 static int
637 638 ixgbe_tx_bind(ixgbe_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
638 639 uint32_t len)
639 640 {
640 641 int status, i;
641 642 ddi_dma_cookie_t dma_cookie;
642 643 uint_t ncookies;
643 644 int desc_num;
644 645
645 646 /*
646 647 * Use DMA binding to process the mblk fragment
647 648 */
648 649 status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL,
649 650 (caddr_t)mp->b_rptr, len,
650 651 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
651 652 0, &dma_cookie, &ncookies);
652 653
653 654 if (status != DDI_DMA_MAPPED) {
654 655 IXGBE_DEBUG_STAT(tx_ring->stat_fail_dma_bind);
655 656 return (-1);
656 657 }
657 658
658 659 tcb->frag_num++;
659 660 tcb->tx_type = USE_DMA;
660 661 /*
661 662 * Each fragment can span several cookies. One cookie will have
662 663 * one tx descriptor to transmit.
663 664 */
664 665 desc_num = 0;
665 666 for (i = ncookies; i > 0; i--) {
666 667 /*
667 668 * Save the address and length to the private data structure
668 669 * of the tx control block, which will be used to fill the
669 670 * tx descriptor ring after all the fragments are processed.
670 671 */
671 672 ixgbe_save_desc(tcb,
672 673 dma_cookie.dmac_laddress,
673 674 dma_cookie.dmac_size);
674 675
675 676 desc_num++;
676 677
677 678 if (i > 1)
678 679 ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie);
679 680 }
680 681
681 682 return (desc_num);
682 683 }
683 684
684 685 /*
685 686 * ixgbe_get_context
686 687 *
687 688 * Get the context information from the mblk
688 689 */
689 690 static int
690 691 ixgbe_get_context(mblk_t *mp, ixgbe_tx_context_t *ctx)
691 692 {
692 693 uint32_t start;
693 694 uint32_t hckflags;
694 695 uint32_t lsoflags;
695 696 uint32_t mss;
696 697 uint32_t len;
697 698 uint32_t size;
698 699 uint32_t offset;
699 700 unsigned char *pos;
700 701 ushort_t etype;
701 702 uint32_t mac_hdr_len;
702 703 uint32_t l4_proto;
703 704 uint32_t l4_hdr_len;
704 705
705 706 ASSERT(mp != NULL);
706 707
707 708 mac_hcksum_get(mp, &start, NULL, NULL, NULL, &hckflags);
708 709 bzero(ctx, sizeof (ixgbe_tx_context_t));
709 710
710 711 if (hckflags == 0) {
711 712 return (0);
712 713 }
713 714
714 715 ctx->hcksum_flags = hckflags;
715 716
716 717 mac_lso_get(mp, &mss, &lsoflags);
717 718 ctx->mss = mss;
718 719 ctx->lso_flag = (lsoflags == HW_LSO);
719 720
720 721 /*
721 722 * LSO relies on tx h/w checksum, so here will drop the package
722 723 * if h/w checksum flag is not declared.
723 724 */
724 725 if (ctx->lso_flag) {
725 726 if (!((ctx->hcksum_flags & HCK_PARTIALCKSUM) &&
726 727 (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM))) {
727 728 IXGBE_DEBUGLOG_0(NULL, "ixgbe_tx: h/w "
728 729 "checksum flags are not specified when doing LSO");
729 730 return (-1);
730 731 }
731 732 }
732 733
733 734 etype = 0;
734 735 mac_hdr_len = 0;
735 736 l4_proto = 0;
736 737
737 738 /*
738 739 * Firstly get the position of the ether_type/ether_tpid.
739 740 * Here we don't assume the ether (VLAN) header is fully included
740 741 * in one mblk fragment, so we go thourgh the fragments to parse
741 742 * the ether type.
742 743 */
743 744 size = len = MBLKL(mp);
744 745 offset = offsetof(struct ether_header, ether_type);
745 746 while (size <= offset) {
746 747 mp = mp->b_cont;
747 748 ASSERT(mp != NULL);
748 749 len = MBLKL(mp);
749 750 size += len;
750 751 }
751 752 pos = mp->b_rptr + offset + len - size;
752 753
753 754 etype = ntohs(*(ushort_t *)(uintptr_t)pos);
754 755 if (etype == ETHERTYPE_VLAN) {
755 756 /*
756 757 * Get the position of the ether_type in VLAN header
757 758 */
758 759 offset = offsetof(struct ether_vlan_header, ether_type);
759 760 while (size <= offset) {
760 761 mp = mp->b_cont;
761 762 ASSERT(mp != NULL);
762 763 len = MBLKL(mp);
763 764 size += len;
764 765 }
765 766 pos = mp->b_rptr + offset + len - size;
766 767
767 768 etype = ntohs(*(ushort_t *)(uintptr_t)pos);
768 769 mac_hdr_len = sizeof (struct ether_vlan_header);
769 770 } else {
770 771 mac_hdr_len = sizeof (struct ether_header);
771 772 }
772 773
773 774 /*
774 775 * Here we don't assume the IP(V6) header is fully included in
775 776 * one mblk fragment.
776 777 */
777 778 switch (etype) {
778 779 case ETHERTYPE_IP:
779 780 if (ctx->lso_flag) {
780 781 offset = offsetof(ipha_t, ipha_length) + mac_hdr_len;
781 782 while (size <= offset) {
782 783 mp = mp->b_cont;
783 784 ASSERT(mp != NULL);
784 785 len = MBLKL(mp);
785 786 size += len;
786 787 }
787 788 pos = mp->b_rptr + offset + len - size;
788 789 *((uint16_t *)(uintptr_t)(pos)) = 0;
789 790
790 791 offset = offsetof(ipha_t, ipha_hdr_checksum) +
791 792 mac_hdr_len;
792 793 while (size <= offset) {
793 794 mp = mp->b_cont;
794 795 ASSERT(mp != NULL);
795 796 len = MBLKL(mp);
796 797 size += len;
797 798 }
798 799 pos = mp->b_rptr + offset + len - size;
799 800 *((uint16_t *)(uintptr_t)(pos)) = 0;
800 801
801 802 /*
802 803 * To perform ixgbe LSO, here also need to fill
803 804 * the tcp checksum field of the packet with the
804 805 * following pseudo-header checksum:
805 806 * (ip_source_addr, ip_destination_addr, l4_proto)
806 807 * Currently the tcp/ip stack has done it.
807 808 */
808 809 }
809 810
810 811 offset = offsetof(ipha_t, ipha_protocol) + mac_hdr_len;
811 812 while (size <= offset) {
812 813 mp = mp->b_cont;
813 814 ASSERT(mp != NULL);
814 815 len = MBLKL(mp);
815 816 size += len;
816 817 }
817 818 pos = mp->b_rptr + offset + len - size;
818 819
819 820 l4_proto = *(uint8_t *)pos;
820 821 break;
821 822 case ETHERTYPE_IPV6:
822 823 offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len;
823 824 while (size <= offset) {
824 825 mp = mp->b_cont;
825 826 ASSERT(mp != NULL);
826 827 len = MBLKL(mp);
827 828 size += len;
828 829 }
829 830 pos = mp->b_rptr + offset + len - size;
830 831
831 832 l4_proto = *(uint8_t *)pos;
832 833 break;
833 834 default:
834 835 /* Unrecoverable error */
835 836 IXGBE_DEBUGLOG_0(NULL, "Ether type error with tx hcksum");
836 837 return (-2);
837 838 }
838 839
839 840 if (ctx->lso_flag) {
840 841 offset = mac_hdr_len + start;
841 842 while (size <= offset) {
842 843 mp = mp->b_cont;
843 844 ASSERT(mp != NULL);
844 845 len = MBLKL(mp);
845 846 size += len;
846 847 }
847 848 pos = mp->b_rptr + offset + len - size;
848 849
849 850 l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos);
850 851 } else {
851 852 /*
852 853 * l4 header length is only required for LSO
853 854 */
854 855 l4_hdr_len = 0;
855 856 }
856 857
857 858 ctx->mac_hdr_len = mac_hdr_len;
858 859 ctx->ip_hdr_len = start;
859 860 ctx->l4_proto = l4_proto;
860 861 ctx->l4_hdr_len = l4_hdr_len;
861 862
862 863 return (0);
863 864 }
864 865
865 866 /*
866 867 * ixgbe_check_context
867 868 *
868 869 * Check if a new context descriptor is needed
869 870 */
870 871 static boolean_t
871 872 ixgbe_check_context(ixgbe_tx_ring_t *tx_ring, ixgbe_tx_context_t *ctx)
872 873 {
873 874 ixgbe_tx_context_t *last;
874 875
875 876 if (ctx == NULL)
876 877 return (B_FALSE);
877 878
878 879 /*
879 880 * Compare the context data retrieved from the mblk and the
880 881 * stored data of the last context descriptor. The data need
881 882 * to be checked are:
882 883 * hcksum_flags
883 884 * l4_proto
884 885 * mac_hdr_len
885 886 * ip_hdr_len
886 887 * lso_flag
887 888 * mss (only checked for LSO)
888 889 * l4_hr_len (only checked for LSO)
889 890 * Either one of the above data is changed, a new context descriptor
890 891 * will be needed.
891 892 */
892 893 last = &tx_ring->tx_context;
893 894
894 895 if ((ctx->hcksum_flags != last->hcksum_flags) ||
895 896 (ctx->l4_proto != last->l4_proto) ||
896 897 (ctx->mac_hdr_len != last->mac_hdr_len) ||
897 898 (ctx->ip_hdr_len != last->ip_hdr_len) ||
898 899 (ctx->lso_flag != last->lso_flag) ||
899 900 (ctx->lso_flag && ((ctx->mss != last->mss) ||
900 901 (ctx->l4_hdr_len != last->l4_hdr_len)))) {
901 902 return (B_TRUE);
902 903 }
903 904
904 905 return (B_FALSE);
905 906 }
906 907
907 908 /*
908 909 * ixgbe_fill_context
909 910 *
910 911 * Fill the context descriptor with hardware checksum informations
911 912 */
912 913 static void
913 914 ixgbe_fill_context(struct ixgbe_adv_tx_context_desc *ctx_tbd,
914 915 ixgbe_tx_context_t *ctx)
915 916 {
916 917 /*
917 918 * Fill the context descriptor with the checksum
918 919 * context information we've got.
919 920 */
920 921 ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len;
921 922 ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len <<
922 923 IXGBE_ADVTXD_MACLEN_SHIFT;
923 924
924 925 ctx_tbd->type_tucmd_mlhl =
925 926 IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
926 927
927 928 if (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM)
928 929 ctx_tbd->type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
929 930
930 931 if (ctx->hcksum_flags & HCK_PARTIALCKSUM) {
931 932 switch (ctx->l4_proto) {
932 933 case IPPROTO_TCP:
933 934 ctx_tbd->type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
934 935 break;
935 936 case IPPROTO_UDP:
936 937 /*
937 938 * We don't have to explicitly set:
938 939 * ctx_tbd->type_tucmd_mlhl |=
939 940 * IXGBE_ADVTXD_TUCMD_L4T_UDP;
940 941 * Because IXGBE_ADVTXD_TUCMD_L4T_UDP == 0b
941 942 */
942 943 break;
943 944 default:
944 945 /* Unrecoverable error */
945 946 IXGBE_DEBUGLOG_0(NULL, "L4 type error with tx hcksum");
946 947 break;
947 948 }
948 949 }
949 950
950 951 ctx_tbd->seqnum_seed = 0;
951 952
952 953 if (ctx->lso_flag) {
953 954 ctx_tbd->mss_l4len_idx =
954 955 (ctx->l4_hdr_len << IXGBE_ADVTXD_L4LEN_SHIFT) |
955 956 (ctx->mss << IXGBE_ADVTXD_MSS_SHIFT);
956 957 } else {
957 958 ctx_tbd->mss_l4len_idx = 0;
958 959 }
959 960 }
960 961
961 962 /*
962 963 * ixgbe_tx_fill_ring
963 964 *
964 965 * Fill the tx descriptor ring with the data
965 966 */
966 967 static int
967 968 ixgbe_tx_fill_ring(ixgbe_tx_ring_t *tx_ring, link_list_t *pending_list,
968 969 ixgbe_tx_context_t *ctx, size_t mbsize)
969 970 {
970 971 struct ixgbe_hw *hw = &tx_ring->ixgbe->hw;
971 972 boolean_t load_context;
972 973 uint32_t index, tcb_index, desc_num;
973 974 union ixgbe_adv_tx_desc *tbd, *first_tbd;
974 975 tx_control_block_t *tcb, *first_tcb;
975 976 uint32_t hcksum_flags;
976 977 int i;
977 978
978 979 ASSERT(mutex_owned(&tx_ring->tx_lock));
979 980
980 981 tbd = NULL;
981 982 first_tbd = NULL;
982 983 first_tcb = NULL;
983 984 desc_num = 0;
984 985 hcksum_flags = 0;
985 986 load_context = B_FALSE;
986 987
987 988 /*
988 989 * Get the index of the first tx descriptor that will be filled,
989 990 * and the index of the first work list item that will be attached
990 991 * with the first used tx control block in the pending list.
991 992 * Note: the two indexes are the same.
992 993 */
993 994 index = tx_ring->tbd_tail;
994 995 tcb_index = tx_ring->tbd_tail;
995 996
996 997 if (ctx != NULL) {
997 998 hcksum_flags = ctx->hcksum_flags;
998 999
999 1000 /*
1000 1001 * Check if a new context descriptor is needed for this packet
1001 1002 */
1002 1003 load_context = ixgbe_check_context(tx_ring, ctx);
1003 1004
1004 1005 if (load_context) {
1005 1006 tbd = &tx_ring->tbd_ring[index];
1006 1007
1007 1008 /*
1008 1009 * Fill the context descriptor with the
1009 1010 * hardware checksum offload informations.
1010 1011 */
1011 1012 ixgbe_fill_context(
1012 1013 (struct ixgbe_adv_tx_context_desc *)tbd, ctx);
1013 1014
1014 1015 index = NEXT_INDEX(index, 1, tx_ring->ring_size);
1015 1016 desc_num++;
1016 1017
1017 1018 /*
1018 1019 * Store the checksum context data if
1019 1020 * a new context descriptor is added
1020 1021 */
1021 1022 tx_ring->tx_context = *ctx;
1022 1023 }
1023 1024 }
1024 1025
1025 1026 first_tbd = &tx_ring->tbd_ring[index];
1026 1027
1027 1028 /*
1028 1029 * Fill tx data descriptors with the data saved in the pending list.
1029 1030 * The tx control blocks in the pending list are added to the work list
1030 1031 * at the same time.
1031 1032 *
1032 1033 * The work list is strictly 1:1 corresponding to the descriptor ring.
1033 1034 * One item of the work list corresponds to one tx descriptor. Because
1034 1035 * one tx control block can span multiple tx descriptors, the tx
1035 1036 * control block will be added to the first work list item that
1036 1037 * corresponds to the first tx descriptor generated from that tx
1037 1038 * control block.
1038 1039 */
1039 1040 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1040 1041 first_tcb = tcb;
1041 1042 while (tcb != NULL) {
1042 1043
1043 1044 for (i = 0; i < tcb->desc_num; i++) {
1044 1045 tbd = &tx_ring->tbd_ring[index];
1045 1046
1046 1047 tbd->read.buffer_addr = tcb->desc[i].address;
1047 1048 tbd->read.cmd_type_len = tcb->desc[i].length;
1048 1049
1049 1050 tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_DEXT
1050 1051 | IXGBE_ADVTXD_DTYP_DATA;
1051 1052
1052 1053 tbd->read.olinfo_status = 0;
1053 1054
1054 1055 index = NEXT_INDEX(index, 1, tx_ring->ring_size);
1055 1056 desc_num++;
1056 1057 }
1057 1058
1058 1059 /*
1059 1060 * Add the tx control block to the work list
1060 1061 */
1061 1062 ASSERT(tx_ring->work_list[tcb_index] == NULL);
1062 1063 tx_ring->work_list[tcb_index] = tcb;
1063 1064
1064 1065 tcb_index = index;
1065 1066 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1066 1067 }
1067 1068
1068 1069 if (load_context) {
1069 1070 /*
1070 1071 * Count the context descriptor for
|
↓ open down ↓ |
1033 lines elided |
↑ open up ↑ |
1071 1072 * the first tx control block.
1072 1073 */
1073 1074 first_tcb->desc_num++;
1074 1075 }
1075 1076 first_tcb->last_index = PREV_INDEX(index, 1, tx_ring->ring_size);
1076 1077
1077 1078 /*
1078 1079 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only
1079 1080 * valid in the first descriptor of the packet.
1080 1081 * Setting paylen in every first_tbd for all parts.
1081 - * 82599 requires the packet length in paylen field with or without
1082 - * LSO and 82598 will ignore it in non-LSO mode.
1082 + * 82599 and X540 require the packet length in paylen field with or
1083 + * without LSO and 82598 will ignore it in non-LSO mode.
1083 1084 */
1084 1085 ASSERT(first_tbd != NULL);
1085 1086 first_tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS;
1086 1087
1087 1088 switch (hw->mac.type) {
1088 1089 case ixgbe_mac_82598EB:
1089 1090 if (ctx != NULL && ctx->lso_flag) {
1090 1091 first_tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1091 1092 first_tbd->read.olinfo_status |=
1092 1093 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len
1093 1094 - ctx->l4_hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT;
1094 1095 }
1095 1096 break;
1096 1097
1097 1098 case ixgbe_mac_82599EB:
1099 + case ixgbe_mac_X540:
1098 1100 if (ctx != NULL && ctx->lso_flag) {
1099 1101 first_tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1100 1102 first_tbd->read.olinfo_status |=
1101 1103 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len
1102 1104 - ctx->l4_hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT;
1103 1105 } else {
1104 1106 first_tbd->read.olinfo_status |=
1105 1107 (mbsize << IXGBE_ADVTXD_PAYLEN_SHIFT);
1106 1108 }
1107 1109 break;
1108 1110
1109 1111 default:
1110 1112 break;
1111 1113 }
1112 1114
1113 1115 /* Set hardware checksum bits */
1114 1116 if (hcksum_flags != 0) {
1115 1117 if (hcksum_flags & HCK_IPV4_HDRCKSUM)
1116 1118 first_tbd->read.olinfo_status |=
1117 1119 IXGBE_ADVTXD_POPTS_IXSM;
1118 1120 if (hcksum_flags & HCK_PARTIALCKSUM)
1119 1121 first_tbd->read.olinfo_status |=
1120 1122 IXGBE_ADVTXD_POPTS_TXSM;
1121 1123 }
1122 1124
1123 1125 /*
1124 1126 * The last descriptor of packet needs End Of Packet (EOP),
1125 1127 * and Report Status (RS) bits set
1126 1128 */
1127 1129 ASSERT(tbd != NULL);
1128 1130 tbd->read.cmd_type_len |=
1129 1131 IXGBE_ADVTXD_DCMD_EOP | IXGBE_ADVTXD_DCMD_RS;
1130 1132
1131 1133 /*
1132 1134 * Sync the DMA buffer of the tx descriptor ring
1133 1135 */
1134 1136 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV);
1135 1137
1136 1138 /*
1137 1139 * Update the number of the free tx descriptors.
1138 1140 * The mutual exclusion between the transmission and the recycling
1139 1141 * (for the tx descriptor ring and the work list) is implemented
1140 1142 * with the atomic operation on the number of the free tx descriptors.
1141 1143 *
1142 1144 * Note: we should always decrement the counter tbd_free before
1143 1145 * advancing the hardware TDT pointer to avoid the race condition -
1144 1146 * before the counter tbd_free is decremented, the transmit of the
1145 1147 * tx descriptors has done and the counter tbd_free is increased by
1146 1148 * the tx recycling.
1147 1149 */
1148 1150 i = ixgbe_atomic_reserve(&tx_ring->tbd_free, desc_num);
1149 1151 ASSERT(i >= 0);
1150 1152
1151 1153 tx_ring->tbd_tail = index;
1152 1154
1153 1155 /*
1154 1156 * Advance the hardware TDT pointer of the tx descriptor ring
1155 1157 */
1156 1158 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), index);
1157 1159
1158 1160 if (ixgbe_check_acc_handle(tx_ring->ixgbe->osdep.reg_handle) !=
1159 1161 DDI_FM_OK) {
1160 1162 ddi_fm_service_impact(tx_ring->ixgbe->dip,
1161 1163 DDI_SERVICE_DEGRADED);
1162 1164 atomic_or_32(&tx_ring->ixgbe->ixgbe_state, IXGBE_ERROR);
1163 1165 }
1164 1166
1165 1167 return (desc_num);
1166 1168 }
1167 1169
1168 1170 /*
1169 1171 * ixgbe_save_desc
1170 1172 *
1171 1173 * Save the address/length pair to the private array
1172 1174 * of the tx control block. The address/length pairs
1173 1175 * will be filled into the tx descriptor ring later.
1174 1176 */
1175 1177 static void
1176 1178 ixgbe_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length)
1177 1179 {
1178 1180 sw_desc_t *desc;
1179 1181
1180 1182 desc = &tcb->desc[tcb->desc_num];
1181 1183 desc->address = address;
1182 1184 desc->length = length;
1183 1185
1184 1186 tcb->desc_num++;
1185 1187 }
1186 1188
1187 1189 /*
1188 1190 * ixgbe_tx_recycle_legacy
1189 1191 *
1190 1192 * Recycle the tx descriptors and tx control blocks.
1191 1193 *
1192 1194 * The work list is traversed to check if the corresponding
1193 1195 * tx descriptors have been transmitted. If so, the resources
1194 1196 * bound to the tx control blocks will be freed, and those
1195 1197 * tx control blocks will be returned to the free list.
1196 1198 */
1197 1199 uint32_t
1198 1200 ixgbe_tx_recycle_legacy(ixgbe_tx_ring_t *tx_ring)
1199 1201 {
1200 1202 uint32_t index, last_index, prev_index;
1201 1203 int desc_num;
1202 1204 boolean_t desc_done;
1203 1205 tx_control_block_t *tcb;
1204 1206 link_list_t pending_list;
1205 1207 ixgbe_t *ixgbe = tx_ring->ixgbe;
1206 1208
1207 1209 mutex_enter(&tx_ring->recycle_lock);
1208 1210
1209 1211 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1210 1212
1211 1213 if (tx_ring->tbd_free == tx_ring->ring_size) {
1212 1214 tx_ring->recycle_fail = 0;
1213 1215 tx_ring->stall_watchdog = 0;
1214 1216 if (tx_ring->reschedule) {
1215 1217 tx_ring->reschedule = B_FALSE;
1216 1218 mac_tx_ring_update(ixgbe->mac_hdl,
1217 1219 tx_ring->ring_handle);
1218 1220 }
1219 1221 mutex_exit(&tx_ring->recycle_lock);
1220 1222 return (0);
1221 1223 }
1222 1224
1223 1225 /*
1224 1226 * Sync the DMA buffer of the tx descriptor ring
1225 1227 */
1226 1228 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1227 1229
1228 1230 if (ixgbe_check_dma_handle(tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1229 1231 mutex_exit(&tx_ring->recycle_lock);
1230 1232 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
1231 1233 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
1232 1234 return (0);
1233 1235 }
1234 1236
1235 1237 LINK_LIST_INIT(&pending_list);
1236 1238 desc_num = 0;
1237 1239 index = tx_ring->tbd_head; /* Index of next tbd/tcb to recycle */
1238 1240
1239 1241 tcb = tx_ring->work_list[index];
1240 1242 ASSERT(tcb != NULL);
1241 1243
1242 1244 while (tcb != NULL) {
1243 1245 /*
1244 1246 * Get the last tx descriptor of this packet.
1245 1247 * If the last tx descriptor is done, then
1246 1248 * we can recycle all descriptors of a packet
1247 1249 * which usually includes several tx control blocks.
1248 1250 * For 82599, LSO descriptors can not be recycled
1249 1251 * unless the whole packet's transmission is done.
1250 1252 * That's why packet level recycling is used here.
1251 1253 * For 82598, there's not such limit.
1252 1254 */
1253 1255 last_index = tcb->last_index;
1254 1256 /*
1255 1257 * MAX_TX_RING_SIZE is used to judge whether
1256 1258 * the index is a valid value or not.
1257 1259 */
1258 1260 if (last_index == MAX_TX_RING_SIZE)
1259 1261 break;
1260 1262
1261 1263 /*
1262 1264 * Check if the Descriptor Done bit is set
1263 1265 */
1264 1266 desc_done = tx_ring->tbd_ring[last_index].wb.status &
1265 1267 IXGBE_TXD_STAT_DD;
1266 1268 if (desc_done) {
1267 1269 /*
1268 1270 * recycle all descriptors of the packet
1269 1271 */
1270 1272 while (tcb != NULL) {
1271 1273 /*
1272 1274 * Strip off the tx control block from
1273 1275 * the work list, and add it to the
1274 1276 * pending list.
1275 1277 */
1276 1278 tx_ring->work_list[index] = NULL;
1277 1279 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1278 1280
1279 1281 /*
1280 1282 * Count the total number of the tx
1281 1283 * descriptors recycled
1282 1284 */
1283 1285 desc_num += tcb->desc_num;
1284 1286
1285 1287 index = NEXT_INDEX(index, tcb->desc_num,
1286 1288 tx_ring->ring_size);
1287 1289
1288 1290 tcb = tx_ring->work_list[index];
1289 1291
1290 1292 prev_index = PREV_INDEX(index, 1,
1291 1293 tx_ring->ring_size);
1292 1294 if (prev_index == last_index)
1293 1295 break;
1294 1296 }
1295 1297 } else {
1296 1298 break;
1297 1299 }
1298 1300 }
1299 1301
1300 1302 /*
1301 1303 * If no tx descriptors are recycled, no need to do more processing
1302 1304 */
1303 1305 if (desc_num == 0) {
1304 1306 tx_ring->recycle_fail++;
1305 1307 mutex_exit(&tx_ring->recycle_lock);
1306 1308 return (0);
1307 1309 }
1308 1310
1309 1311 tx_ring->recycle_fail = 0;
1310 1312 tx_ring->stall_watchdog = 0;
1311 1313
1312 1314 /*
1313 1315 * Update the head index of the tx descriptor ring
1314 1316 */
1315 1317 tx_ring->tbd_head = index;
1316 1318
1317 1319 /*
1318 1320 * Update the number of the free tx descriptors with atomic operations
1319 1321 */
1320 1322 atomic_add_32(&tx_ring->tbd_free, desc_num);
1321 1323
1322 1324 if ((tx_ring->tbd_free >= ixgbe->tx_resched_thresh) &&
1323 1325 (tx_ring->reschedule)) {
1324 1326 tx_ring->reschedule = B_FALSE;
1325 1327 mac_tx_ring_update(ixgbe->mac_hdl,
1326 1328 tx_ring->ring_handle);
1327 1329 }
1328 1330 mutex_exit(&tx_ring->recycle_lock);
1329 1331
1330 1332 /*
1331 1333 * Free the resources used by the tx control blocks
1332 1334 * in the pending list
1333 1335 */
1334 1336 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1335 1337 while (tcb != NULL) {
1336 1338 /*
1337 1339 * Release the resources occupied by the tx control block
1338 1340 */
1339 1341 ixgbe_free_tcb(tcb);
1340 1342
1341 1343 tcb = (tx_control_block_t *)
1342 1344 LIST_GET_NEXT(&pending_list, &tcb->link);
1343 1345 }
1344 1346
1345 1347 /*
1346 1348 * Add the tx control blocks in the pending list to the free list.
1347 1349 */
1348 1350 ixgbe_put_free_list(tx_ring, &pending_list);
1349 1351
1350 1352 return (desc_num);
1351 1353 }
1352 1354
1353 1355 /*
1354 1356 * ixgbe_tx_recycle_head_wb
1355 1357 *
1356 1358 * Check the head write-back, and recycle all the transmitted
1357 1359 * tx descriptors and tx control blocks.
1358 1360 */
1359 1361 uint32_t
1360 1362 ixgbe_tx_recycle_head_wb(ixgbe_tx_ring_t *tx_ring)
1361 1363 {
1362 1364 uint32_t index;
1363 1365 uint32_t head_wb;
1364 1366 int desc_num;
1365 1367 tx_control_block_t *tcb;
1366 1368 link_list_t pending_list;
1367 1369 ixgbe_t *ixgbe = tx_ring->ixgbe;
1368 1370
1369 1371 mutex_enter(&tx_ring->recycle_lock);
1370 1372
1371 1373 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1372 1374
1373 1375 if (tx_ring->tbd_free == tx_ring->ring_size) {
1374 1376 tx_ring->recycle_fail = 0;
1375 1377 tx_ring->stall_watchdog = 0;
1376 1378 if (tx_ring->reschedule) {
1377 1379 tx_ring->reschedule = B_FALSE;
1378 1380 mac_tx_ring_update(ixgbe->mac_hdl,
1379 1381 tx_ring->ring_handle);
1380 1382 }
1381 1383 mutex_exit(&tx_ring->recycle_lock);
1382 1384 return (0);
1383 1385 }
1384 1386
1385 1387 /*
1386 1388 * Sync the DMA buffer of the tx descriptor ring
1387 1389 *
1388 1390 * Note: For head write-back mode, the tx descriptors will not
1389 1391 * be written back, but the head write-back value is stored at
1390 1392 * the last extra tbd at the end of the DMA area, we still need
1391 1393 * to sync the head write-back value for kernel.
1392 1394 *
1393 1395 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1394 1396 */
1395 1397 (void) ddi_dma_sync(tx_ring->tbd_area.dma_handle,
1396 1398 sizeof (union ixgbe_adv_tx_desc) * tx_ring->ring_size,
1397 1399 sizeof (uint32_t),
1398 1400 DDI_DMA_SYNC_FORKERNEL);
1399 1401
1400 1402 if (ixgbe_check_dma_handle(tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1401 1403 mutex_exit(&tx_ring->recycle_lock);
1402 1404 ddi_fm_service_impact(ixgbe->dip,
1403 1405 DDI_SERVICE_DEGRADED);
1404 1406 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
1405 1407 return (0);
1406 1408 }
1407 1409
1408 1410 LINK_LIST_INIT(&pending_list);
1409 1411 desc_num = 0;
1410 1412 index = tx_ring->tbd_head; /* Next index to clean */
1411 1413
1412 1414 /*
1413 1415 * Get the value of head write-back
1414 1416 */
1415 1417 head_wb = *tx_ring->tbd_head_wb;
1416 1418 while (index != head_wb) {
1417 1419 tcb = tx_ring->work_list[index];
1418 1420 ASSERT(tcb != NULL);
1419 1421
1420 1422 if (OFFSET(index, head_wb, tx_ring->ring_size) <
1421 1423 tcb->desc_num) {
1422 1424 /*
1423 1425 * The current tx control block is not
1424 1426 * completely transmitted, stop recycling
1425 1427 */
1426 1428 break;
1427 1429 }
1428 1430
1429 1431 /*
1430 1432 * Strip off the tx control block from the work list,
1431 1433 * and add it to the pending list.
1432 1434 */
1433 1435 tx_ring->work_list[index] = NULL;
1434 1436 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1435 1437
1436 1438 /*
1437 1439 * Advance the index of the tx descriptor ring
1438 1440 */
1439 1441 index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size);
1440 1442
1441 1443 /*
1442 1444 * Count the total number of the tx descriptors recycled
1443 1445 */
1444 1446 desc_num += tcb->desc_num;
1445 1447 }
1446 1448
1447 1449 /*
1448 1450 * If no tx descriptors are recycled, no need to do more processing
1449 1451 */
1450 1452 if (desc_num == 0) {
1451 1453 tx_ring->recycle_fail++;
1452 1454 mutex_exit(&tx_ring->recycle_lock);
1453 1455 return (0);
1454 1456 }
1455 1457
1456 1458 tx_ring->recycle_fail = 0;
1457 1459 tx_ring->stall_watchdog = 0;
1458 1460
1459 1461 /*
1460 1462 * Update the head index of the tx descriptor ring
1461 1463 */
1462 1464 tx_ring->tbd_head = index;
1463 1465
1464 1466 /*
1465 1467 * Update the number of the free tx descriptors with atomic operations
1466 1468 */
1467 1469 atomic_add_32(&tx_ring->tbd_free, desc_num);
1468 1470
1469 1471 if ((tx_ring->tbd_free >= ixgbe->tx_resched_thresh) &&
1470 1472 (tx_ring->reschedule)) {
1471 1473 tx_ring->reschedule = B_FALSE;
1472 1474 mac_tx_ring_update(ixgbe->mac_hdl,
1473 1475 tx_ring->ring_handle);
1474 1476 }
1475 1477 mutex_exit(&tx_ring->recycle_lock);
1476 1478
1477 1479 /*
1478 1480 * Free the resources used by the tx control blocks
1479 1481 * in the pending list
1480 1482 */
1481 1483 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1482 1484 while (tcb) {
1483 1485 /*
1484 1486 * Release the resources occupied by the tx control block
1485 1487 */
1486 1488 ixgbe_free_tcb(tcb);
1487 1489
1488 1490 tcb = (tx_control_block_t *)
1489 1491 LIST_GET_NEXT(&pending_list, &tcb->link);
1490 1492 }
1491 1493
1492 1494 /*
1493 1495 * Add the tx control blocks in the pending list to the free list.
1494 1496 */
1495 1497 ixgbe_put_free_list(tx_ring, &pending_list);
1496 1498
1497 1499 return (desc_num);
1498 1500 }
1499 1501
1500 1502 /*
1501 1503 * ixgbe_free_tcb - free up the tx control block
1502 1504 *
1503 1505 * Free the resources of the tx control block, including
1504 1506 * unbind the previously bound DMA handle, and reset other
1505 1507 * control fields.
1506 1508 */
1507 1509 void
1508 1510 ixgbe_free_tcb(tx_control_block_t *tcb)
1509 1511 {
1510 1512 switch (tcb->tx_type) {
1511 1513 case USE_COPY:
1512 1514 /*
1513 1515 * Reset the buffer length that is used for copy
1514 1516 */
1515 1517 tcb->tx_buf.len = 0;
1516 1518 break;
1517 1519 case USE_DMA:
1518 1520 /*
1519 1521 * Release the DMA resource that is used for
1520 1522 * DMA binding.
1521 1523 */
1522 1524 (void) ddi_dma_unbind_handle(tcb->tx_dma_handle);
1523 1525 break;
1524 1526 default:
1525 1527 break;
1526 1528 }
1527 1529
1528 1530 /*
1529 1531 * Free the mblk
1530 1532 */
1531 1533 if (tcb->mp != NULL) {
1532 1534 freemsg(tcb->mp);
1533 1535 tcb->mp = NULL;
1534 1536 }
1535 1537
1536 1538 tcb->tx_type = USE_NONE;
1537 1539 tcb->last_index = MAX_TX_RING_SIZE;
1538 1540 tcb->frag_num = 0;
1539 1541 tcb->desc_num = 0;
1540 1542 }
1541 1543
1542 1544 /*
1543 1545 * ixgbe_get_free_list - Get a free tx control block from the free list
1544 1546 *
1545 1547 * The atomic operation on the number of the available tx control block
1546 1548 * in the free list is used to keep this routine mutual exclusive with
1547 1549 * the routine ixgbe_put_check_list.
1548 1550 */
1549 1551 static tx_control_block_t *
1550 1552 ixgbe_get_free_list(ixgbe_tx_ring_t *tx_ring)
1551 1553 {
1552 1554 tx_control_block_t *tcb;
1553 1555
1554 1556 /*
1555 1557 * Check and update the number of the free tx control block
1556 1558 * in the free list.
1557 1559 */
1558 1560 if (ixgbe_atomic_reserve(&tx_ring->tcb_free, 1) < 0)
1559 1561 return (NULL);
1560 1562
1561 1563 mutex_enter(&tx_ring->tcb_head_lock);
1562 1564
1563 1565 tcb = tx_ring->free_list[tx_ring->tcb_head];
1564 1566 ASSERT(tcb != NULL);
1565 1567 tx_ring->free_list[tx_ring->tcb_head] = NULL;
1566 1568 tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1,
1567 1569 tx_ring->free_list_size);
1568 1570
1569 1571 mutex_exit(&tx_ring->tcb_head_lock);
1570 1572
1571 1573 return (tcb);
1572 1574 }
1573 1575
1574 1576 /*
1575 1577 * ixgbe_put_free_list
1576 1578 *
1577 1579 * Put a list of used tx control blocks back to the free list
1578 1580 *
1579 1581 * A mutex is used here to ensure the serialization. The mutual exclusion
1580 1582 * between ixgbe_get_free_list and ixgbe_put_free_list is implemented with
1581 1583 * the atomic operation on the counter tcb_free.
1582 1584 */
1583 1585 void
1584 1586 ixgbe_put_free_list(ixgbe_tx_ring_t *tx_ring, link_list_t *pending_list)
1585 1587 {
1586 1588 uint32_t index;
1587 1589 int tcb_num;
1588 1590 tx_control_block_t *tcb;
1589 1591
1590 1592 mutex_enter(&tx_ring->tcb_tail_lock);
1591 1593
1592 1594 index = tx_ring->tcb_tail;
1593 1595
1594 1596 tcb_num = 0;
1595 1597 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1596 1598 while (tcb != NULL) {
1597 1599 ASSERT(tx_ring->free_list[index] == NULL);
1598 1600 tx_ring->free_list[index] = tcb;
1599 1601
1600 1602 tcb_num++;
1601 1603
1602 1604 index = NEXT_INDEX(index, 1, tx_ring->free_list_size);
1603 1605
1604 1606 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1605 1607 }
1606 1608
1607 1609 tx_ring->tcb_tail = index;
1608 1610
1609 1611 /*
1610 1612 * Update the number of the free tx control block
1611 1613 * in the free list. This operation must be placed
1612 1614 * under the protection of the lock.
1613 1615 */
1614 1616 atomic_add_32(&tx_ring->tcb_free, tcb_num);
1615 1617
1616 1618 mutex_exit(&tx_ring->tcb_tail_lock);
1617 1619 }
|
↓ open down ↓ |
510 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX