Print this page
DLPX-25998 TCP congestion control is inadequate
Reviewed at: http://reviews.delphix.com/r/34808/
DLPX-43064 include high-resolution round-trip times in connstat (EP-652)
DLPX-37540 TCP per-connection kernel statistics DLPX-37544 connstat command to display per-connection TCP statistics
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/inet/tcp/tcp_output.c
+++ new/usr/src/uts/common/inet/tcp/tcp_output.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 - * Copyright (c) 2014 by Delphix. All rights reserved.
24 + * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
25 25 */
26 26
27 27 /* This file contains all TCP output processing functions. */
28 28
29 29 #include <sys/types.h>
30 30 #include <sys/stream.h>
31 31 #include <sys/strsun.h>
32 32 #include <sys/strsubr.h>
33 33 #include <sys/stropts.h>
34 34 #include <sys/strlog.h>
35 35 #define _SUN_TPI_VERSION 2
36 36 #include <sys/tihdr.h>
37 37 #include <sys/suntpi.h>
38 38 #include <sys/xti_inet.h>
39 39 #include <sys/timod.h>
40 40 #include <sys/pattr.h>
41 41 #include <sys/squeue_impl.h>
42 42 #include <sys/squeue.h>
43 43 #include <sys/sockio.h>
44 44 #include <sys/tsol/tnet.h>
45 45
46 46 #include <inet/common.h>
47 47 #include <inet/ip.h>
48 48 #include <inet/tcp.h>
49 49 #include <inet/tcp_impl.h>
50 50 #include <inet/snmpcom.h>
51 51 #include <inet/proto_set.h>
52 52 #include <inet/ipsec_impl.h>
53 53 #include <inet/ip_ndp.h>
54 54
55 55 static mblk_t *tcp_get_seg_mp(tcp_t *, uint32_t, int32_t *);
|
↓ open down ↓ |
21 lines elided |
↑ open up ↑ |
56 56 static void tcp_wput_cmdblk(queue_t *, mblk_t *);
57 57 static void tcp_wput_flush(tcp_t *, mblk_t *);
58 58 static void tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp);
59 59 static int tcp_xmit_end(tcp_t *);
60 60 static int tcp_send(tcp_t *, const int, const int, const int,
61 61 const int, int *, uint_t *, int *, mblk_t **, mblk_t *);
62 62 static void tcp_xmit_early_reset(char *, mblk_t *, uint32_t, uint32_t,
63 63 int, ip_recv_attr_t *, ip_stack_t *, conn_t *);
64 64 static boolean_t tcp_send_rst_chk(tcp_stack_t *);
65 65 static void tcp_process_shrunk_swnd(tcp_t *, uint32_t);
66 -static void tcp_fill_header(tcp_t *, uchar_t *, clock_t, int);
66 +static void tcp_fill_header(tcp_t *, uchar_t *, int);
67 67
68 68 /*
69 69 * Functions called directly via squeue having a prototype of edesc_t.
70 70 */
71 71 static void tcp_wput_nondata(void *, mblk_t *, void *, ip_recv_attr_t *);
72 72 static void tcp_wput_ioctl(void *, mblk_t *, void *, ip_recv_attr_t *);
73 73 static void tcp_wput_proto(void *, mblk_t *, void *, ip_recv_attr_t *);
74 74
75 75 /*
76 76 * This controls how tiny a write must be before we try to copy it
77 77 * into the mblk on the tail of the transmit queue. Not much
78 78 * speedup is observed for values larger than sixteen. Zero will
79 79 * disable the optimisation.
80 80 */
81 81 static int tcp_tx_pull_len = 16;
82 82
83 +static void
84 +cc_after_idle(tcp_t *tcp)
85 +{
86 + uint32_t old_cwnd = tcp->tcp_cwnd;
87 +
88 + if (CC_ALGO(tcp)->after_idle != NULL)
89 + CC_ALGO(tcp)->after_idle(&tcp->tcp_ccv);
90 +
91 + DTRACE_PROBE3(cwnd__cc__after__idle, tcp_t *, tcp, uint32_t, old_cwnd,
92 + uint32_t, tcp->tcp_cwnd);
93 +}
94 +
83 95 void
84 96 tcp_wput(queue_t *q, mblk_t *mp)
85 97 {
86 98 conn_t *connp = Q_TO_CONN(q);
87 99 tcp_t *tcp;
88 100 void (*output_proc)();
89 101 t_scalar_t type;
90 102 uchar_t *rptr;
91 103 struct iocblk *iocp;
92 104 size_t size;
93 105
94 106 ASSERT(connp->conn_ref >= 2);
95 107
96 108 switch (DB_TYPE(mp)) {
97 109 case M_DATA:
98 110 tcp = connp->conn_tcp;
99 111 ASSERT(tcp != NULL);
100 112
101 113 size = msgdsize(mp);
102 114
103 115 mutex_enter(&tcp->tcp_non_sq_lock);
104 116 tcp->tcp_squeue_bytes += size;
105 117 if (TCP_UNSENT_BYTES(tcp) > connp->conn_sndbuf) {
106 118 tcp_setqfull(tcp);
107 119 }
108 120 mutex_exit(&tcp->tcp_non_sq_lock);
109 121
110 122 CONN_INC_REF(connp);
111 123 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, tcp_output, connp,
112 124 NULL, tcp_squeue_flag, SQTAG_TCP_OUTPUT);
113 125 return;
114 126
115 127 case M_CMD:
116 128 tcp_wput_cmdblk(q, mp);
117 129 return;
118 130
119 131 case M_PROTO:
120 132 case M_PCPROTO:
121 133 /*
122 134 * if it is a snmp message, don't get behind the squeue
123 135 */
124 136 tcp = connp->conn_tcp;
125 137 rptr = mp->b_rptr;
126 138 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
127 139 type = ((union T_primitives *)rptr)->type;
128 140 } else {
129 141 if (connp->conn_debug) {
130 142 (void) strlog(TCP_MOD_ID, 0, 1,
131 143 SL_ERROR|SL_TRACE,
132 144 "tcp_wput_proto, dropping one...");
133 145 }
134 146 freemsg(mp);
135 147 return;
136 148 }
137 149 if (type == T_SVR4_OPTMGMT_REQ) {
138 150 /*
139 151 * All Solaris components should pass a db_credp
140 152 * for this TPI message, hence we ASSERT.
141 153 * But in case there is some other M_PROTO that looks
142 154 * like a TPI message sent by some other kernel
143 155 * component, we check and return an error.
144 156 */
145 157 cred_t *cr = msg_getcred(mp, NULL);
146 158
147 159 ASSERT(cr != NULL);
148 160 if (cr == NULL) {
149 161 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
150 162 return;
151 163 }
152 164 if (snmpcom_req(q, mp, tcp_snmp_set, ip_snmp_get,
153 165 cr)) {
154 166 /*
155 167 * This was a SNMP request
156 168 */
157 169 return;
158 170 } else {
159 171 output_proc = tcp_wput_proto;
160 172 }
161 173 } else {
162 174 output_proc = tcp_wput_proto;
163 175 }
164 176 break;
165 177 case M_IOCTL:
166 178 /*
167 179 * Most ioctls can be processed right away without going via
168 180 * squeues - process them right here. Those that do require
169 181 * squeue (currently _SIOCSOCKFALLBACK)
170 182 * are processed by tcp_wput_ioctl().
171 183 */
172 184 iocp = (struct iocblk *)mp->b_rptr;
173 185 tcp = connp->conn_tcp;
174 186
175 187 switch (iocp->ioc_cmd) {
176 188 case TCP_IOC_ABORT_CONN:
177 189 tcp_ioctl_abort_conn(q, mp);
178 190 return;
179 191 case TI_GETPEERNAME:
180 192 case TI_GETMYNAME:
181 193 mi_copyin(q, mp, NULL,
182 194 SIZEOF_STRUCT(strbuf, iocp->ioc_flag));
183 195 return;
184 196
185 197 default:
186 198 output_proc = tcp_wput_ioctl;
187 199 break;
188 200 }
189 201 break;
190 202 default:
191 203 output_proc = tcp_wput_nondata;
192 204 break;
193 205 }
194 206
195 207 CONN_INC_REF(connp);
196 208 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, output_proc, connp,
197 209 NULL, tcp_squeue_flag, SQTAG_TCP_WPUT_OTHER);
198 210 }
199 211
200 212 /*
201 213 * The TCP normal data output path.
202 214 * NOTE: the logic of the fast path is duplicated from this function.
203 215 */
204 216 void
205 217 tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent)
206 218 {
207 219 int len;
208 220 mblk_t *local_time;
209 221 mblk_t *mp1;
|
↓ open down ↓ |
117 lines elided |
↑ open up ↑ |
210 222 uint32_t snxt;
211 223 int tail_unsent;
212 224 int tcpstate;
213 225 int usable = 0;
214 226 mblk_t *xmit_tail;
215 227 int32_t mss;
216 228 int32_t num_sack_blk = 0;
217 229 int32_t total_hdr_len;
218 230 int32_t tcp_hdr_len;
219 231 int rc;
220 - tcp_stack_t *tcps = tcp->tcp_tcps;
221 232 conn_t *connp = tcp->tcp_connp;
222 233 clock_t now = LBOLT_FASTPATH;
223 234
224 235 tcpstate = tcp->tcp_state;
225 236 if (mp == NULL) {
226 237 /*
227 238 * tcp_wput_data() with NULL mp should only be called when
228 239 * there is unsent data.
229 240 */
230 241 ASSERT(tcp->tcp_unsent > 0);
231 242 /* Really tacky... but we need this for detached closes. */
232 243 len = tcp->tcp_unsent;
233 244 goto data_null;
234 245 }
235 246
236 247 ASSERT(mp->b_datap->db_type == M_DATA);
237 248 /*
238 249 * Don't allow data after T_ORDREL_REQ or T_DISCON_REQ,
239 250 * or before a connection attempt has begun.
240 251 */
241 252 if (tcpstate < TCPS_SYN_SENT || tcpstate > TCPS_CLOSE_WAIT ||
242 253 (tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
243 254 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
244 255 #ifdef DEBUG
245 256 cmn_err(CE_WARN,
246 257 "tcp_wput_data: data after ordrel, %s",
247 258 tcp_display(tcp, NULL,
248 259 DISP_ADDR_AND_PORT));
249 260 #else
250 261 if (connp->conn_debug) {
251 262 (void) strlog(TCP_MOD_ID, 0, 1,
252 263 SL_TRACE|SL_ERROR,
253 264 "tcp_wput_data: data after ordrel, %s\n",
254 265 tcp_display(tcp, NULL,
255 266 DISP_ADDR_AND_PORT));
256 267 }
257 268 #endif /* DEBUG */
258 269 }
259 270 if (tcp->tcp_snd_zcopy_aware &&
260 271 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
261 272 tcp_zcopy_notify(tcp);
262 273 freemsg(mp);
263 274 mutex_enter(&tcp->tcp_non_sq_lock);
264 275 if (tcp->tcp_flow_stopped &&
265 276 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
266 277 tcp_clrqfull(tcp);
267 278 }
268 279 mutex_exit(&tcp->tcp_non_sq_lock);
269 280 return;
270 281 }
271 282
272 283 /* Strip empties */
273 284 for (;;) {
274 285 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <=
275 286 (uintptr_t)INT_MAX);
276 287 len = (int)(mp->b_wptr - mp->b_rptr);
277 288 if (len > 0)
278 289 break;
279 290 mp1 = mp;
280 291 mp = mp->b_cont;
281 292 freeb(mp1);
282 293 if (mp == NULL) {
283 294 return;
284 295 }
285 296 }
286 297
287 298 /* If we are the first on the list ... */
288 299 if (tcp->tcp_xmit_head == NULL) {
289 300 tcp->tcp_xmit_head = mp;
290 301 tcp->tcp_xmit_tail = mp;
291 302 tcp->tcp_xmit_tail_unsent = len;
292 303 } else {
293 304 /* If tiny tx and room in txq tail, pullup to save mblks. */
294 305 struct datab *dp;
295 306
296 307 mp1 = tcp->tcp_xmit_last;
297 308 if (len < tcp_tx_pull_len &&
298 309 (dp = mp1->b_datap)->db_ref == 1 &&
299 310 dp->db_lim - mp1->b_wptr >= len) {
300 311 ASSERT(len > 0);
301 312 ASSERT(!mp1->b_cont);
302 313 if (len == 1) {
303 314 *mp1->b_wptr++ = *mp->b_rptr;
304 315 } else {
305 316 bcopy(mp->b_rptr, mp1->b_wptr, len);
306 317 mp1->b_wptr += len;
307 318 }
308 319 if (mp1 == tcp->tcp_xmit_tail)
309 320 tcp->tcp_xmit_tail_unsent += len;
310 321 mp1->b_cont = mp->b_cont;
311 322 if (tcp->tcp_snd_zcopy_aware &&
312 323 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
313 324 mp1->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
314 325 freeb(mp);
315 326 mp = mp1;
316 327 } else {
317 328 tcp->tcp_xmit_last->b_cont = mp;
318 329 }
319 330 len += tcp->tcp_unsent;
320 331 }
321 332
322 333 /* Tack on however many more positive length mblks we have */
323 334 if ((mp1 = mp->b_cont) != NULL) {
324 335 do {
325 336 int tlen;
326 337 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <=
327 338 (uintptr_t)INT_MAX);
328 339 tlen = (int)(mp1->b_wptr - mp1->b_rptr);
329 340 if (tlen <= 0) {
330 341 mp->b_cont = mp1->b_cont;
331 342 freeb(mp1);
332 343 } else {
333 344 len += tlen;
334 345 mp = mp1;
335 346 }
336 347 } while ((mp1 = mp->b_cont) != NULL);
337 348 }
338 349 tcp->tcp_xmit_last = mp;
339 350 tcp->tcp_unsent = len;
340 351
341 352 if (urgent)
342 353 usable = 1;
343 354
344 355 data_null:
345 356 snxt = tcp->tcp_snxt;
346 357 xmit_tail = tcp->tcp_xmit_tail;
347 358 tail_unsent = tcp->tcp_xmit_tail_unsent;
348 359
349 360 /*
350 361 * Note that tcp_mss has been adjusted to take into account the
351 362 * timestamp option if applicable. Because SACK options do not
352 363 * appear in every TCP segments and they are of variable lengths,
353 364 * they cannot be included in tcp_mss. Thus we need to calculate
354 365 * the actual segment length when we need to send a segment which
355 366 * includes SACK options.
356 367 */
357 368 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
358 369 int32_t opt_len;
359 370
360 371 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
361 372 tcp->tcp_num_sack_blk);
362 373 opt_len = num_sack_blk * sizeof (sack_blk_t) + TCPOPT_NOP_LEN *
363 374 2 + TCPOPT_HEADER_LEN;
364 375 mss = tcp->tcp_mss - opt_len;
|
↓ open down ↓ |
134 lines elided |
↑ open up ↑ |
365 376 total_hdr_len = connp->conn_ht_iphc_len + opt_len;
366 377 tcp_hdr_len = connp->conn_ht_ulp_len + opt_len;
367 378 } else {
368 379 mss = tcp->tcp_mss;
369 380 total_hdr_len = connp->conn_ht_iphc_len;
370 381 tcp_hdr_len = connp->conn_ht_ulp_len;
371 382 }
372 383
373 384 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
374 385 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
375 - TCP_SET_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle);
386 + cc_after_idle(tcp);
376 387 }
377 388 if (tcpstate == TCPS_SYN_RCVD) {
378 389 /*
379 390 * The three-way connection establishment handshake is not
380 391 * complete yet. We want to queue the data for transmission
381 392 * after entering ESTABLISHED state (RFC793). A jump to
382 393 * "done" label effectively leaves data on the queue.
383 394 */
384 395 goto done;
385 396 } else {
386 397 int usable_r;
387 398
388 399 /*
389 400 * In the special case when cwnd is zero, which can only
390 401 * happen if the connection is ECN capable, return now.
391 402 * New segments is sent using tcp_timer(). The timer
392 403 * is set in tcp_input_data().
393 404 */
394 405 if (tcp->tcp_cwnd == 0) {
395 406 /*
396 407 * Note that tcp_cwnd is 0 before 3-way handshake is
397 408 * finished.
398 409 */
399 410 ASSERT(tcp->tcp_ecn_ok ||
400 411 tcp->tcp_state < TCPS_ESTABLISHED);
401 412 return;
402 413 }
403 414
404 415 /* NOTE: trouble if xmitting while SYN not acked? */
405 416 usable_r = snxt - tcp->tcp_suna;
406 417 usable_r = tcp->tcp_swnd - usable_r;
407 418
408 419 /*
409 420 * Check if the receiver has shrunk the window. If
410 421 * tcp_wput_data() with NULL mp is called, tcp_fin_sent
411 422 * cannot be set as there is unsent data, so FIN cannot
412 423 * be sent out. Otherwise, we need to take into account
413 424 * of FIN as it consumes an "invisible" sequence number.
414 425 */
415 426 ASSERT(tcp->tcp_fin_sent == 0);
416 427 if (usable_r < 0) {
417 428 /*
418 429 * The receiver has shrunk the window and we have sent
419 430 * -usable_r date beyond the window, re-adjust.
420 431 *
421 432 * If TCP window scaling is enabled, there can be
422 433 * round down error as the advertised receive window
423 434 * is actually right shifted n bits. This means that
424 435 * the lower n bits info is wiped out. It will look
425 436 * like the window is shrunk. Do a check here to
426 437 * see if the shrunk amount is actually within the
427 438 * error in window calculation. If it is, just
428 439 * return. Note that this check is inside the
429 440 * shrunk window check. This makes sure that even
430 441 * though tcp_process_shrunk_swnd() is not called,
431 442 * we will stop further processing.
432 443 */
433 444 if ((-usable_r >> tcp->tcp_snd_ws) > 0) {
434 445 tcp_process_shrunk_swnd(tcp, -usable_r);
435 446 }
436 447 return;
437 448 }
438 449
439 450 /* usable = MIN(swnd, cwnd) - unacked_bytes */
440 451 if (tcp->tcp_swnd > tcp->tcp_cwnd)
441 452 usable_r -= tcp->tcp_swnd - tcp->tcp_cwnd;
442 453
443 454 /* usable = MIN(usable, unsent) */
444 455 if (usable_r > len)
445 456 usable_r = len;
|
↓ open down ↓ |
60 lines elided |
↑ open up ↑ |
446 457
447 458 /* usable = MAX(usable, {1 for urgent, 0 for data}) */
448 459 if (usable_r > 0) {
449 460 usable = usable_r;
450 461 } else {
451 462 /* Bypass all other unnecessary processing. */
452 463 goto done;
453 464 }
454 465 }
455 466
467 +#ifdef KERNEL_32
456 468 local_time = (mblk_t *)now;
469 +#else
470 + local_time = (mblk_t *)(intptr_t)gethrtime();
471 +#endif
457 472
458 473 /*
459 474 * "Our" Nagle Algorithm. This is not the same as in the old
460 475 * BSD. This is more in line with the true intent of Nagle.
461 476 *
462 477 * The conditions are:
463 478 * 1. The amount of unsent data (or amount of data which can be
464 479 * sent, whichever is smaller) is less than Nagle limit.
465 480 * 2. The last sent size is also less than Nagle limit.
466 481 * 3. There is unack'ed data.
467 482 * 4. Urgent pointer is not set. Send urgent data ignoring the
468 483 * Nagle algorithm. This reduces the probability that urgent
469 484 * bytes get "merged" together.
470 485 * 5. The app has not closed the connection. This eliminates the
471 486 * wait time of the receiving side waiting for the last piece of
472 487 * (small) data.
473 488 *
474 489 * If all are satisified, exit without sending anything. Note
475 490 * that Nagle limit can be smaller than 1 MSS. Nagle limit is
476 491 * the smaller of 1 MSS and global tcp_naglim_def (default to be
477 492 * 4095).
478 493 */
479 494 if (usable < (int)tcp->tcp_naglim &&
480 495 tcp->tcp_naglim > tcp->tcp_last_sent_len &&
481 496 snxt != tcp->tcp_suna &&
482 497 !(tcp->tcp_valid_bits & TCP_URG_VALID) &&
483 498 !(tcp->tcp_valid_bits & TCP_FSS_VALID)) {
484 499 goto done;
485 500 }
486 501
487 502 /*
488 503 * If tcp_zero_win_probe is not set and the tcp->tcp_cork option
489 504 * is set, then we have to force TCP not to send partial segment
490 505 * (smaller than MSS bytes). We are calculating the usable now
491 506 * based on full mss and will save the rest of remaining data for
492 507 * later. When tcp_zero_win_probe is set, TCP needs to send out
493 508 * something to do zero window probe.
494 509 */
495 510 if (tcp->tcp_cork && !tcp->tcp_zero_win_probe) {
496 511 if (usable < mss)
497 512 goto done;
498 513 usable = (usable / mss) * mss;
499 514 }
500 515
501 516 /* Update the latest receive window size in TCP header. */
502 517 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
503 518
504 519 /* Send the packet. */
505 520 rc = tcp_send(tcp, mss, total_hdr_len, tcp_hdr_len,
506 521 num_sack_blk, &usable, &snxt, &tail_unsent, &xmit_tail,
507 522 local_time);
508 523
509 524 /* Pretend that all we were trying to send really got sent */
510 525 if (rc < 0 && tail_unsent < 0) {
511 526 do {
512 527 xmit_tail = xmit_tail->b_cont;
513 528 xmit_tail->b_prev = local_time;
514 529 ASSERT((uintptr_t)(xmit_tail->b_wptr -
515 530 xmit_tail->b_rptr) <= (uintptr_t)INT_MAX);
516 531 tail_unsent += (int)(xmit_tail->b_wptr -
517 532 xmit_tail->b_rptr);
518 533 } while (tail_unsent < 0);
519 534 }
520 535 done:;
521 536 tcp->tcp_xmit_tail = xmit_tail;
522 537 tcp->tcp_xmit_tail_unsent = tail_unsent;
523 538 len = tcp->tcp_snxt - snxt;
524 539 if (len) {
525 540 /*
526 541 * If new data was sent, need to update the notsack
527 542 * list, which is, afterall, data blocks that have
528 543 * not been sack'ed by the receiver. New data is
529 544 * not sack'ed.
530 545 */
531 546 if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) {
532 547 /* len is a negative value. */
533 548 tcp->tcp_pipe -= len;
534 549 tcp_notsack_update(&(tcp->tcp_notsack_list),
535 550 tcp->tcp_snxt, snxt,
536 551 &(tcp->tcp_num_notsack_blk),
537 552 &(tcp->tcp_cnt_notsack_list));
538 553 }
539 554 tcp->tcp_snxt = snxt + tcp->tcp_fin_sent;
540 555 tcp->tcp_rack = tcp->tcp_rnxt;
541 556 tcp->tcp_rack_cnt = 0;
542 557 if ((snxt + len) == tcp->tcp_suna) {
543 558 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
544 559 }
545 560 } else if (snxt == tcp->tcp_suna && tcp->tcp_swnd == 0) {
546 561 /*
547 562 * Didn't send anything. Make sure the timer is running
548 563 * so that we will probe a zero window.
549 564 */
550 565 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
551 566 }
552 567 /* Note that len is the amount we just sent but with a negative sign */
553 568 tcp->tcp_unsent += len;
554 569 mutex_enter(&tcp->tcp_non_sq_lock);
555 570 if (tcp->tcp_flow_stopped) {
556 571 if (TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
557 572 tcp_clrqfull(tcp);
558 573 }
559 574 } else if (TCP_UNSENT_BYTES(tcp) >= connp->conn_sndbuf) {
560 575 if (!(tcp->tcp_detached))
561 576 tcp_setqfull(tcp);
562 577 }
563 578 mutex_exit(&tcp->tcp_non_sq_lock);
564 579 }
565 580
566 581 /*
567 582 * Initial STREAMS write side put() procedure for sockets. It tries to
568 583 * handle the T_CAPABILITY_REQ which sockfs sends down while setting
569 584 * up the socket without using the squeue. Non T_CAPABILITY_REQ messages
570 585 * are handled by tcp_wput() as usual.
571 586 *
572 587 * All further messages will also be handled by tcp_wput() because we cannot
573 588 * be sure that the above short cut is safe later.
574 589 */
575 590 void
576 591 tcp_wput_sock(queue_t *wq, mblk_t *mp)
577 592 {
578 593 conn_t *connp = Q_TO_CONN(wq);
579 594 tcp_t *tcp = connp->conn_tcp;
580 595 struct T_capability_req *car = (struct T_capability_req *)mp->b_rptr;
581 596
582 597 ASSERT(wq->q_qinfo == &tcp_sock_winit);
583 598 wq->q_qinfo = &tcp_winit;
584 599
585 600 ASSERT(IPCL_IS_TCP(connp));
586 601 ASSERT(TCP_IS_SOCKET(tcp));
587 602
588 603 if (DB_TYPE(mp) == M_PCPROTO &&
589 604 MBLKL(mp) == sizeof (struct T_capability_req) &&
590 605 car->PRIM_type == T_CAPABILITY_REQ) {
591 606 tcp_capability_req(tcp, mp);
592 607 return;
593 608 }
594 609
595 610 tcp_wput(wq, mp);
596 611 }
597 612
598 613 /* ARGSUSED */
599 614 void
600 615 tcp_wput_fallback(queue_t *wq, mblk_t *mp)
601 616 {
602 617 #ifdef DEBUG
603 618 cmn_err(CE_CONT, "tcp_wput_fallback: Message during fallback \n");
604 619 #endif
605 620 freemsg(mp);
606 621 }
607 622
608 623 /*
609 624 * Call by tcp_wput() to handle misc non M_DATA messages.
610 625 */
611 626 /* ARGSUSED */
612 627 static void
613 628 tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
614 629 {
615 630 conn_t *connp = (conn_t *)arg;
616 631 tcp_t *tcp = connp->conn_tcp;
617 632
618 633 ASSERT(DB_TYPE(mp) != M_IOCTL);
619 634 /*
620 635 * TCP is D_MP and qprocsoff() is done towards the end of the tcp_close.
621 636 * Once the close starts, streamhead and sockfs will not let any data
622 637 * packets come down (close ensures that there are no threads using the
623 638 * queue and no new threads will come down) but since qprocsoff()
624 639 * hasn't happened yet, a M_FLUSH or some non data message might
625 640 * get reflected back (in response to our own FLUSHRW) and get
626 641 * processed after tcp_close() is done. The conn would still be valid
627 642 * because a ref would have added but we need to check the state
628 643 * before actually processing the packet.
629 644 */
630 645 if (TCP_IS_DETACHED(tcp) || (tcp->tcp_state == TCPS_CLOSED)) {
631 646 freemsg(mp);
632 647 return;
633 648 }
634 649
635 650 switch (DB_TYPE(mp)) {
636 651 case M_IOCDATA:
637 652 tcp_wput_iocdata(tcp, mp);
638 653 break;
639 654 case M_FLUSH:
640 655 tcp_wput_flush(tcp, mp);
641 656 break;
642 657 default:
643 658 ip_wput_nondata(connp->conn_wq, mp);
644 659 break;
645 660 }
646 661 }
647 662
648 663 /* tcp_wput_flush is called by tcp_wput_nondata to handle M_FLUSH messages. */
649 664 static void
650 665 tcp_wput_flush(tcp_t *tcp, mblk_t *mp)
651 666 {
652 667 uchar_t fval = *mp->b_rptr;
653 668 mblk_t *tail;
654 669 conn_t *connp = tcp->tcp_connp;
655 670 queue_t *q = connp->conn_wq;
656 671
657 672 /* TODO: How should flush interact with urgent data? */
658 673 if ((fval & FLUSHW) && tcp->tcp_xmit_head != NULL &&
659 674 !(tcp->tcp_valid_bits & TCP_URG_VALID)) {
660 675 /*
661 676 * Flush only data that has not yet been put on the wire. If
662 677 * we flush data that we have already transmitted, life, as we
663 678 * know it, may come to an end.
664 679 */
665 680 tail = tcp->tcp_xmit_tail;
666 681 tail->b_wptr -= tcp->tcp_xmit_tail_unsent;
667 682 tcp->tcp_xmit_tail_unsent = 0;
668 683 tcp->tcp_unsent = 0;
669 684 if (tail->b_wptr != tail->b_rptr)
670 685 tail = tail->b_cont;
671 686 if (tail) {
672 687 mblk_t **excess = &tcp->tcp_xmit_head;
673 688 for (;;) {
674 689 mblk_t *mp1 = *excess;
675 690 if (mp1 == tail)
676 691 break;
677 692 tcp->tcp_xmit_tail = mp1;
678 693 tcp->tcp_xmit_last = mp1;
679 694 excess = &mp1->b_cont;
680 695 }
681 696 *excess = NULL;
682 697 tcp_close_mpp(&tail);
683 698 if (tcp->tcp_snd_zcopy_aware)
684 699 tcp_zcopy_notify(tcp);
685 700 }
686 701 /*
687 702 * We have no unsent data, so unsent must be less than
688 703 * conn_sndlowat, so re-enable flow.
689 704 */
690 705 mutex_enter(&tcp->tcp_non_sq_lock);
691 706 if (tcp->tcp_flow_stopped) {
692 707 tcp_clrqfull(tcp);
693 708 }
694 709 mutex_exit(&tcp->tcp_non_sq_lock);
695 710 }
696 711 /*
697 712 * TODO: you can't just flush these, you have to increase rwnd for one
698 713 * thing. For another, how should urgent data interact?
699 714 */
700 715 if (fval & FLUSHR) {
701 716 *mp->b_rptr = fval & ~FLUSHW;
702 717 /* XXX */
703 718 qreply(q, mp);
704 719 return;
705 720 }
706 721 freemsg(mp);
707 722 }
708 723
709 724 /*
710 725 * tcp_wput_iocdata is called by tcp_wput_nondata to handle all M_IOCDATA
711 726 * messages.
712 727 */
713 728 static void
714 729 tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp)
715 730 {
716 731 mblk_t *mp1;
717 732 struct iocblk *iocp = (struct iocblk *)mp->b_rptr;
718 733 STRUCT_HANDLE(strbuf, sb);
719 734 uint_t addrlen;
720 735 conn_t *connp = tcp->tcp_connp;
721 736 queue_t *q = connp->conn_wq;
722 737
723 738 /* Make sure it is one of ours. */
724 739 switch (iocp->ioc_cmd) {
725 740 case TI_GETMYNAME:
726 741 case TI_GETPEERNAME:
727 742 break;
728 743 default:
729 744 /*
730 745 * If the conn is closing, then error the ioctl here. Otherwise
731 746 * use the CONN_IOCTLREF_* macros to hold off tcp_close until
732 747 * we're done here.
733 748 */
734 749 mutex_enter(&connp->conn_lock);
735 750 if (connp->conn_state_flags & CONN_CLOSING) {
736 751 mutex_exit(&connp->conn_lock);
737 752 iocp->ioc_error = EINVAL;
738 753 mp->b_datap->db_type = M_IOCNAK;
739 754 iocp->ioc_count = 0;
740 755 qreply(q, mp);
741 756 return;
742 757 }
743 758
744 759 CONN_INC_IOCTLREF_LOCKED(connp);
745 760 ip_wput_nondata(q, mp);
746 761 CONN_DEC_IOCTLREF(connp);
747 762 return;
748 763 }
749 764 switch (mi_copy_state(q, mp, &mp1)) {
750 765 case -1:
751 766 return;
752 767 case MI_COPY_CASE(MI_COPY_IN, 1):
753 768 break;
754 769 case MI_COPY_CASE(MI_COPY_OUT, 1):
755 770 /* Copy out the strbuf. */
756 771 mi_copyout(q, mp);
757 772 return;
758 773 case MI_COPY_CASE(MI_COPY_OUT, 2):
759 774 /* All done. */
760 775 mi_copy_done(q, mp, 0);
761 776 return;
762 777 default:
763 778 mi_copy_done(q, mp, EPROTO);
764 779 return;
765 780 }
766 781 /* Check alignment of the strbuf */
767 782 if (!OK_32PTR(mp1->b_rptr)) {
768 783 mi_copy_done(q, mp, EINVAL);
769 784 return;
770 785 }
771 786
772 787 STRUCT_SET_HANDLE(sb, iocp->ioc_flag, (void *)mp1->b_rptr);
773 788
774 789 if (connp->conn_family == AF_INET)
775 790 addrlen = sizeof (sin_t);
776 791 else
777 792 addrlen = sizeof (sin6_t);
778 793
779 794 if (STRUCT_FGET(sb, maxlen) < addrlen) {
780 795 mi_copy_done(q, mp, EINVAL);
781 796 return;
782 797 }
783 798
784 799 switch (iocp->ioc_cmd) {
785 800 case TI_GETMYNAME:
786 801 break;
787 802 case TI_GETPEERNAME:
788 803 if (tcp->tcp_state < TCPS_SYN_RCVD) {
789 804 mi_copy_done(q, mp, ENOTCONN);
790 805 return;
791 806 }
792 807 break;
793 808 }
794 809 mp1 = mi_copyout_alloc(q, mp, STRUCT_FGETP(sb, buf), addrlen, B_TRUE);
795 810 if (!mp1)
796 811 return;
797 812
798 813 STRUCT_FSET(sb, len, addrlen);
799 814 switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) {
800 815 case TI_GETMYNAME:
801 816 (void) conn_getsockname(connp, (struct sockaddr *)mp1->b_wptr,
802 817 &addrlen);
803 818 break;
804 819 case TI_GETPEERNAME:
805 820 (void) conn_getpeername(connp, (struct sockaddr *)mp1->b_wptr,
806 821 &addrlen);
807 822 break;
808 823 }
809 824 mp1->b_wptr += addrlen;
810 825 /* Copy out the address */
811 826 mi_copyout(q, mp);
812 827 }
813 828
814 829 /*
815 830 * tcp_wput_ioctl is called by tcp_wput_nondata() to handle all M_IOCTL
816 831 * messages.
817 832 */
818 833 /* ARGSUSED */
819 834 static void
820 835 tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
821 836 {
822 837 conn_t *connp = (conn_t *)arg;
823 838 tcp_t *tcp = connp->conn_tcp;
824 839 queue_t *q = connp->conn_wq;
825 840 struct iocblk *iocp;
826 841
827 842 ASSERT(DB_TYPE(mp) == M_IOCTL);
828 843 /*
829 844 * Try and ASSERT the minimum possible references on the
830 845 * conn early enough. Since we are executing on write side,
831 846 * the connection is obviously not detached and that means
832 847 * there is a ref each for TCP and IP. Since we are behind
833 848 * the squeue, the minimum references needed are 3. If the
834 849 * conn is in classifier hash list, there should be an
835 850 * extra ref for that (we check both the possibilities).
836 851 */
837 852 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
838 853 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
839 854
840 855 iocp = (struct iocblk *)mp->b_rptr;
841 856 switch (iocp->ioc_cmd) {
842 857 case _SIOCSOCKFALLBACK:
843 858 /*
844 859 * Either sockmod is about to be popped and the socket
845 860 * would now be treated as a plain stream, or a module
846 861 * is about to be pushed so we could no longer use read-
847 862 * side synchronous streams for fused loopback tcp.
848 863 * Drain any queued data and disable direct sockfs
849 864 * interface from now on.
850 865 */
851 866 if (!tcp->tcp_issocket) {
852 867 DB_TYPE(mp) = M_IOCNAK;
853 868 iocp->ioc_error = EINVAL;
854 869 } else {
855 870 tcp_use_pure_tpi(tcp);
856 871 DB_TYPE(mp) = M_IOCACK;
857 872 iocp->ioc_error = 0;
858 873 }
859 874 iocp->ioc_count = 0;
860 875 iocp->ioc_rval = 0;
861 876 qreply(q, mp);
862 877 return;
863 878 }
864 879
865 880 /*
866 881 * If the conn is closing, then error the ioctl here. Otherwise bump the
867 882 * conn_ioctlref to hold off tcp_close until we're done here.
868 883 */
869 884 mutex_enter(&(connp)->conn_lock);
870 885 if ((connp)->conn_state_flags & CONN_CLOSING) {
871 886 mutex_exit(&(connp)->conn_lock);
872 887 iocp->ioc_error = EINVAL;
873 888 mp->b_datap->db_type = M_IOCNAK;
874 889 iocp->ioc_count = 0;
875 890 qreply(q, mp);
876 891 return;
877 892 }
878 893
879 894 CONN_INC_IOCTLREF_LOCKED(connp);
880 895 ip_wput_nondata(q, mp);
881 896 CONN_DEC_IOCTLREF(connp);
882 897 }
883 898
884 899 /*
885 900 * This routine is called by tcp_wput() to handle all TPI requests.
886 901 */
887 902 /* ARGSUSED */
888 903 static void
889 904 tcp_wput_proto(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
890 905 {
891 906 conn_t *connp = (conn_t *)arg;
892 907 tcp_t *tcp = connp->conn_tcp;
893 908 union T_primitives *tprim = (union T_primitives *)mp->b_rptr;
894 909 uchar_t *rptr;
895 910 t_scalar_t type;
896 911 cred_t *cr;
897 912
898 913 /*
899 914 * Try and ASSERT the minimum possible references on the
900 915 * conn early enough. Since we are executing on write side,
901 916 * the connection is obviously not detached and that means
902 917 * there is a ref each for TCP and IP. Since we are behind
903 918 * the squeue, the minimum references needed are 3. If the
904 919 * conn is in classifier hash list, there should be an
905 920 * extra ref for that (we check both the possibilities).
906 921 */
907 922 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
908 923 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
909 924
910 925 rptr = mp->b_rptr;
911 926 ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX);
912 927 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
913 928 type = ((union T_primitives *)rptr)->type;
914 929 if (type == T_EXDATA_REQ) {
915 930 tcp_output_urgent(connp, mp, arg2, NULL);
916 931 } else if (type != T_DATA_REQ) {
917 932 goto non_urgent_data;
918 933 } else {
919 934 /* TODO: options, flags, ... from user */
920 935 /* Set length to zero for reclamation below */
921 936 tcp_wput_data(tcp, mp->b_cont, B_TRUE);
922 937 freeb(mp);
923 938 }
924 939 return;
925 940 } else {
926 941 if (connp->conn_debug) {
927 942 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
928 943 "tcp_wput_proto, dropping one...");
929 944 }
930 945 freemsg(mp);
931 946 return;
932 947 }
933 948
934 949 non_urgent_data:
935 950
936 951 switch ((int)tprim->type) {
937 952 case O_T_BIND_REQ: /* bind request */
938 953 case T_BIND_REQ: /* new semantics bind request */
939 954 tcp_tpi_bind(tcp, mp);
940 955 break;
941 956 case T_UNBIND_REQ: /* unbind request */
942 957 tcp_tpi_unbind(tcp, mp);
943 958 break;
944 959 case O_T_CONN_RES: /* old connection response XXX */
945 960 case T_CONN_RES: /* connection response */
946 961 tcp_tli_accept(tcp, mp);
947 962 break;
948 963 case T_CONN_REQ: /* connection request */
949 964 tcp_tpi_connect(tcp, mp);
950 965 break;
951 966 case T_DISCON_REQ: /* disconnect request */
952 967 tcp_disconnect(tcp, mp);
953 968 break;
954 969 case T_CAPABILITY_REQ:
955 970 tcp_capability_req(tcp, mp); /* capability request */
956 971 break;
957 972 case T_INFO_REQ: /* information request */
958 973 tcp_info_req(tcp, mp);
959 974 break;
960 975 case T_SVR4_OPTMGMT_REQ: /* manage options req */
961 976 case T_OPTMGMT_REQ:
962 977 /*
963 978 * Note: no support for snmpcom_req() through new
964 979 * T_OPTMGMT_REQ. See comments in ip.c
965 980 */
966 981
967 982 /*
968 983 * All Solaris components should pass a db_credp
969 984 * for this TPI message, hence we ASSERT.
970 985 * But in case there is some other M_PROTO that looks
971 986 * like a TPI message sent by some other kernel
972 987 * component, we check and return an error.
973 988 */
974 989 cr = msg_getcred(mp, NULL);
975 990 ASSERT(cr != NULL);
976 991 if (cr == NULL) {
977 992 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
978 993 return;
979 994 }
980 995 /*
981 996 * If EINPROGRESS is returned, the request has been queued
982 997 * for subsequent processing by ip_restart_optmgmt(), which
983 998 * will do the CONN_DEC_REF().
984 999 */
985 1000 if ((int)tprim->type == T_SVR4_OPTMGMT_REQ) {
986 1001 svr4_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
987 1002 } else {
988 1003 tpi_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
989 1004 }
990 1005 break;
991 1006
992 1007 case T_UNITDATA_REQ: /* unitdata request */
993 1008 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
994 1009 break;
995 1010 case T_ORDREL_REQ: /* orderly release req */
996 1011 freemsg(mp);
997 1012
998 1013 if (tcp->tcp_fused)
999 1014 tcp_unfuse(tcp);
1000 1015
1001 1016 if (tcp_xmit_end(tcp) != 0) {
1002 1017 /*
1003 1018 * We were crossing FINs and got a reset from
1004 1019 * the other side. Just ignore it.
1005 1020 */
1006 1021 if (connp->conn_debug) {
1007 1022 (void) strlog(TCP_MOD_ID, 0, 1,
1008 1023 SL_ERROR|SL_TRACE,
1009 1024 "tcp_wput_proto, T_ORDREL_REQ out of "
1010 1025 "state %s",
1011 1026 tcp_display(tcp, NULL,
1012 1027 DISP_ADDR_AND_PORT));
1013 1028 }
1014 1029 }
1015 1030 break;
1016 1031 case T_ADDR_REQ:
1017 1032 tcp_addr_req(tcp, mp);
1018 1033 break;
1019 1034 default:
1020 1035 if (connp->conn_debug) {
1021 1036 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
1022 1037 "tcp_wput_proto, bogus TPI msg, type %d",
1023 1038 tprim->type);
1024 1039 }
1025 1040 /*
1026 1041 * We used to M_ERROR. Sending TNOTSUPPORT gives the user
1027 1042 * to recover.
1028 1043 */
1029 1044 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
1030 1045 break;
1031 1046 }
1032 1047 }
1033 1048
1034 1049 /*
1035 1050 * Handle special out-of-band ioctl requests (see PSARC/2008/265).
1036 1051 */
1037 1052 static void
1038 1053 tcp_wput_cmdblk(queue_t *q, mblk_t *mp)
1039 1054 {
1040 1055 void *data;
1041 1056 mblk_t *datamp = mp->b_cont;
1042 1057 conn_t *connp = Q_TO_CONN(q);
1043 1058 tcp_t *tcp = connp->conn_tcp;
1044 1059 cmdblk_t *cmdp = (cmdblk_t *)mp->b_rptr;
1045 1060
1046 1061 if (datamp == NULL || MBLKL(datamp) < cmdp->cb_len) {
1047 1062 cmdp->cb_error = EPROTO;
1048 1063 qreply(q, mp);
1049 1064 return;
1050 1065 }
1051 1066
1052 1067 data = datamp->b_rptr;
1053 1068
1054 1069 switch (cmdp->cb_cmd) {
1055 1070 case TI_GETPEERNAME:
1056 1071 if (tcp->tcp_state < TCPS_SYN_RCVD)
1057 1072 cmdp->cb_error = ENOTCONN;
1058 1073 else
1059 1074 cmdp->cb_error = conn_getpeername(connp, data,
1060 1075 &cmdp->cb_len);
1061 1076 break;
1062 1077 case TI_GETMYNAME:
1063 1078 cmdp->cb_error = conn_getsockname(connp, data, &cmdp->cb_len);
1064 1079 break;
1065 1080 default:
1066 1081 cmdp->cb_error = EINVAL;
1067 1082 break;
1068 1083 }
1069 1084
1070 1085 qreply(q, mp);
1071 1086 }
1072 1087
1073 1088 /*
1074 1089 * The TCP fast path write put procedure.
1075 1090 * NOTE: the logic of the fast path is duplicated from tcp_wput_data()
1076 1091 */
1077 1092 /* ARGSUSED */
1078 1093 void
1079 1094 tcp_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1080 1095 {
1081 1096 int len;
1082 1097 int hdrlen;
1083 1098 int plen;
1084 1099 mblk_t *mp1;
1085 1100 uchar_t *rptr;
1086 1101 uint32_t snxt;
1087 1102 tcpha_t *tcpha;
1088 1103 struct datab *db;
1089 1104 uint32_t suna;
1090 1105 uint32_t mss;
1091 1106 ipaddr_t *dst;
1092 1107 ipaddr_t *src;
1093 1108 uint32_t sum;
1094 1109 int usable;
1095 1110 conn_t *connp = (conn_t *)arg;
1096 1111 tcp_t *tcp = connp->conn_tcp;
1097 1112 uint32_t msize;
1098 1113 tcp_stack_t *tcps = tcp->tcp_tcps;
1099 1114 ip_xmit_attr_t *ixa;
1100 1115 clock_t now;
1101 1116
1102 1117 /*
1103 1118 * Try and ASSERT the minimum possible references on the
1104 1119 * conn early enough. Since we are executing on write side,
1105 1120 * the connection is obviously not detached and that means
1106 1121 * there is a ref each for TCP and IP. Since we are behind
1107 1122 * the squeue, the minimum references needed are 3. If the
1108 1123 * conn is in classifier hash list, there should be an
1109 1124 * extra ref for that (we check both the possibilities).
1110 1125 */
1111 1126 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
1112 1127 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
1113 1128
1114 1129 ASSERT(DB_TYPE(mp) == M_DATA);
1115 1130 msize = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1116 1131
1117 1132 mutex_enter(&tcp->tcp_non_sq_lock);
1118 1133 tcp->tcp_squeue_bytes -= msize;
1119 1134 mutex_exit(&tcp->tcp_non_sq_lock);
1120 1135
1121 1136 /* Bypass tcp protocol for fused tcp loopback */
1122 1137 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1123 1138 return;
1124 1139
1125 1140 mss = tcp->tcp_mss;
1126 1141 /*
1127 1142 * If ZEROCOPY has turned off, try not to send any zero-copy message
1128 1143 * down. Do backoff, now.
1129 1144 */
1130 1145 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on)
1131 1146 mp = tcp_zcopy_backoff(tcp, mp, B_FALSE);
1132 1147
1133 1148
1134 1149 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX);
1135 1150 len = (int)(mp->b_wptr - mp->b_rptr);
1136 1151
1137 1152 /*
1138 1153 * Criteria for fast path:
1139 1154 *
1140 1155 * 1. no unsent data
1141 1156 * 2. single mblk in request
1142 1157 * 3. connection established
1143 1158 * 4. data in mblk
1144 1159 * 5. len <= mss
1145 1160 * 6. no tcp_valid bits
1146 1161 */
1147 1162 if ((tcp->tcp_unsent != 0) ||
1148 1163 (tcp->tcp_cork) ||
1149 1164 (mp->b_cont != NULL) ||
1150 1165 (tcp->tcp_state != TCPS_ESTABLISHED) ||
1151 1166 (len == 0) ||
1152 1167 (len > mss) ||
1153 1168 (tcp->tcp_valid_bits != 0)) {
1154 1169 tcp_wput_data(tcp, mp, B_FALSE);
1155 1170 return;
1156 1171 }
1157 1172
1158 1173 ASSERT(tcp->tcp_xmit_tail_unsent == 0);
1159 1174 ASSERT(tcp->tcp_fin_sent == 0);
1160 1175
1161 1176 /* queue new packet onto retransmission queue */
1162 1177 if (tcp->tcp_xmit_head == NULL) {
1163 1178 tcp->tcp_xmit_head = mp;
1164 1179 } else {
1165 1180 tcp->tcp_xmit_last->b_cont = mp;
1166 1181 }
1167 1182 tcp->tcp_xmit_last = mp;
1168 1183 tcp->tcp_xmit_tail = mp;
1169 1184
1170 1185 /* find out how much we can send */
1171 1186 /* BEGIN CSTYLED */
1172 1187 /*
1173 1188 * un-acked usable
1174 1189 * |--------------|-----------------|
1175 1190 * tcp_suna tcp_snxt tcp_suna+tcp_swnd
1176 1191 */
1177 1192 /* END CSTYLED */
1178 1193
1179 1194 /* start sending from tcp_snxt */
1180 1195 snxt = tcp->tcp_snxt;
1181 1196
1182 1197 /*
|
↓ open down ↓ |
716 lines elided |
↑ open up ↑ |
1183 1198 * Check to see if this connection has been idled for some
1184 1199 * time and no ACK is expected. If it is, we need to slow
1185 1200 * start again to get back the connection's "self-clock" as
1186 1201 * described in VJ's paper.
1187 1202 *
1188 1203 * Reinitialize tcp_cwnd after idle.
1189 1204 */
1190 1205 now = LBOLT_FASTPATH;
1191 1206 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
1192 1207 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
1193 - TCP_SET_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle);
1208 + cc_after_idle(tcp);
1194 1209 }
1195 1210
1196 1211 usable = tcp->tcp_swnd; /* tcp window size */
1197 1212 if (usable > tcp->tcp_cwnd)
1198 1213 usable = tcp->tcp_cwnd; /* congestion window smaller */
1199 1214 usable -= snxt; /* subtract stuff already sent */
1200 1215 suna = tcp->tcp_suna;
1201 1216 usable += suna;
1202 1217 /* usable can be < 0 if the congestion window is smaller */
1203 1218 if (len > usable) {
1204 1219 /* Can't send complete M_DATA in one shot */
1205 1220 goto slow;
1206 1221 }
1207 1222
1208 1223 mutex_enter(&tcp->tcp_non_sq_lock);
1209 1224 if (tcp->tcp_flow_stopped &&
1210 1225 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
1211 1226 tcp_clrqfull(tcp);
1212 1227 }
1213 1228 mutex_exit(&tcp->tcp_non_sq_lock);
1214 1229
1215 1230 /*
1216 1231 * determine if anything to send (Nagle).
1217 1232 *
1218 1233 * 1. len < tcp_mss (i.e. small)
1219 1234 * 2. unacknowledged data present
1220 1235 * 3. len < nagle limit
1221 1236 * 4. last packet sent < nagle limit (previous packet sent)
1222 1237 */
1223 1238 if ((len < mss) && (snxt != suna) &&
1224 1239 (len < (int)tcp->tcp_naglim) &&
1225 1240 (tcp->tcp_last_sent_len < tcp->tcp_naglim)) {
1226 1241 /*
1227 1242 * This was the first unsent packet and normally
1228 1243 * mss < xmit_hiwater so there is no need to worry
1229 1244 * about flow control. The next packet will go
1230 1245 * through the flow control check in tcp_wput_data().
1231 1246 */
1232 1247 /* leftover work from above */
1233 1248 tcp->tcp_unsent = len;
1234 1249 tcp->tcp_xmit_tail_unsent = len;
1235 1250
1236 1251 return;
1237 1252 }
1238 1253
1239 1254 /*
1240 1255 * len <= tcp->tcp_mss && len == unsent so no sender silly window. Can
1241 1256 * send now.
1242 1257 */
1243 1258
1244 1259 if (snxt == suna) {
1245 1260 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
|
↓ open down ↓ |
42 lines elided |
↑ open up ↑ |
1246 1261 }
1247 1262
1248 1263 /* we have always sent something */
1249 1264 tcp->tcp_rack_cnt = 0;
1250 1265
1251 1266 tcp->tcp_snxt = snxt + len;
1252 1267 tcp->tcp_rack = tcp->tcp_rnxt;
1253 1268
1254 1269 if ((mp1 = dupb(mp)) == 0)
1255 1270 goto no_memory;
1271 +#ifdef KERNEL_32
1256 1272 mp->b_prev = (mblk_t *)(uintptr_t)now;
1273 +#else
1274 + mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
1275 +#endif
1257 1276 mp->b_next = (mblk_t *)(uintptr_t)snxt;
1258 1277
1259 1278 /* adjust tcp header information */
1260 1279 tcpha = tcp->tcp_tcpha;
1261 1280 tcpha->tha_flags = (TH_ACK|TH_PUSH);
1262 1281
1263 1282 sum = len + connp->conn_ht_ulp_len + connp->conn_sum;
1264 1283 sum = (sum >> 16) + (sum & 0xFFFF);
1265 1284 tcpha->tha_sum = htons(sum);
1266 1285
1267 1286 tcpha->tha_seq = htonl(snxt);
1268 1287
1269 1288 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1270 1289 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1271 - BUMP_LOCAL(tcp->tcp_obsegs);
1290 + TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1291 + tcp->tcp_cs.tcp_out_data_segs++;
1292 + tcp->tcp_cs.tcp_out_data_bytes += len;
1272 1293
1273 1294 /* Update the latest receive window size in TCP header. */
1274 1295 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
1275 1296
1276 1297 tcp->tcp_last_sent_len = (ushort_t)len;
1277 1298
1278 1299 plen = len + connp->conn_ht_iphc_len;
1279 1300
1280 1301 ixa = connp->conn_ixa;
1281 1302 ixa->ixa_pktlen = plen;
1282 1303
1283 1304 if (ixa->ixa_flags & IXAF_IS_IPV4) {
1284 1305 tcp->tcp_ipha->ipha_length = htons(plen);
1285 1306 } else {
1286 1307 tcp->tcp_ip6h->ip6_plen = htons(plen - IPV6_HDR_LEN);
1287 1308 }
1288 1309
1289 1310 /* see if we need to allocate a mblk for the headers */
1290 1311 hdrlen = connp->conn_ht_iphc_len;
1291 1312 rptr = mp1->b_rptr - hdrlen;
1292 1313 db = mp1->b_datap;
1293 1314 if ((db->db_ref != 2) || rptr < db->db_base ||
1294 1315 (!OK_32PTR(rptr))) {
1295 1316 /* NOTE: we assume allocb returns an OK_32PTR */
1296 1317 mp = allocb(hdrlen + tcps->tcps_wroff_xtra, BPRI_MED);
1297 1318 if (!mp) {
1298 1319 freemsg(mp1);
1299 1320 goto no_memory;
1300 1321 }
|
↓ open down ↓ |
19 lines elided |
↑ open up ↑ |
1301 1322 mp->b_cont = mp1;
1302 1323 mp1 = mp;
1303 1324 /* Leave room for Link Level header */
1304 1325 rptr = &mp1->b_rptr[tcps->tcps_wroff_xtra];
1305 1326 mp1->b_wptr = &rptr[hdrlen];
1306 1327 }
1307 1328 mp1->b_rptr = rptr;
1308 1329
1309 1330 /* Fill in the timestamp option. */
1310 1331 if (tcp->tcp_snd_ts_ok) {
1311 - uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
1312 -
1313 - U32_TO_BE32(llbolt,
1314 - (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
1332 + U32_TO_BE32(now,
1333 + (char *)tcpha + TCP_MIN_HEADER_LENGTH + 4);
1315 1334 U32_TO_BE32(tcp->tcp_ts_recent,
1316 - (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
1335 + (char *)tcpha + TCP_MIN_HEADER_LENGTH + 8);
1317 1336 } else {
1318 1337 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
1319 1338 }
1320 1339
1321 1340 /* copy header into outgoing packet */
1322 1341 dst = (ipaddr_t *)rptr;
1323 1342 src = (ipaddr_t *)connp->conn_ht_iphc;
1324 1343 dst[0] = src[0];
1325 1344 dst[1] = src[1];
1326 1345 dst[2] = src[2];
1327 1346 dst[3] = src[3];
1328 1347 dst[4] = src[4];
1329 1348 dst[5] = src[5];
1330 1349 dst[6] = src[6];
1331 1350 dst[7] = src[7];
1332 1351 dst[8] = src[8];
1333 1352 dst[9] = src[9];
1334 1353 if (hdrlen -= 40) {
1335 1354 hdrlen >>= 2;
1336 1355 dst += 10;
1337 1356 src += 10;
1338 1357 do {
1339 1358 *dst++ = *src++;
1340 1359 } while (--hdrlen);
1341 1360 }
1342 1361
1343 1362 /*
1344 1363 * Set the ECN info in the TCP header. Note that this
1345 1364 * is not the template header.
1346 1365 */
1347 1366 if (tcp->tcp_ecn_ok) {
1348 1367 TCP_SET_ECT(tcp, rptr);
1349 1368
1350 1369 tcpha = (tcpha_t *)(rptr + ixa->ixa_ip_hdr_length);
1351 1370 if (tcp->tcp_ecn_echo_on)
1352 1371 tcpha->tha_flags |= TH_ECE;
1353 1372 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
1354 1373 tcpha->tha_flags |= TH_CWR;
1355 1374 tcp->tcp_ecn_cwr_sent = B_TRUE;
1356 1375 }
1357 1376 }
1358 1377
1359 1378 if (tcp->tcp_ip_forward_progress) {
1360 1379 tcp->tcp_ip_forward_progress = B_FALSE;
1361 1380 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
1362 1381 } else {
1363 1382 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
1364 1383 }
1365 1384 tcp_send_data(tcp, mp1);
1366 1385 return;
1367 1386
1368 1387 /*
1369 1388 * If we ran out of memory, we pretend to have sent the packet
1370 1389 * and that it was lost on the wire.
1371 1390 */
1372 1391 no_memory:
1373 1392 return;
1374 1393
1375 1394 slow:
1376 1395 /* leftover work from above */
1377 1396 tcp->tcp_unsent = len;
1378 1397 tcp->tcp_xmit_tail_unsent = len;
1379 1398 tcp_wput_data(tcp, NULL, B_FALSE);
1380 1399 }
1381 1400
1382 1401 /* ARGSUSED2 */
1383 1402 void
1384 1403 tcp_output_urgent(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1385 1404 {
1386 1405 int len;
1387 1406 uint32_t msize;
1388 1407 conn_t *connp = (conn_t *)arg;
1389 1408 tcp_t *tcp = connp->conn_tcp;
1390 1409
1391 1410 msize = msgdsize(mp);
1392 1411
1393 1412 len = msize - 1;
1394 1413 if (len < 0) {
1395 1414 freemsg(mp);
1396 1415 return;
1397 1416 }
1398 1417
1399 1418 /*
1400 1419 * Try to force urgent data out on the wire. Even if we have unsent
1401 1420 * data this will at least send the urgent flag.
1402 1421 * XXX does not handle more flag correctly.
1403 1422 */
1404 1423 len += tcp->tcp_unsent;
1405 1424 len += tcp->tcp_snxt;
1406 1425 tcp->tcp_urg = len;
1407 1426 tcp->tcp_valid_bits |= TCP_URG_VALID;
1408 1427
1409 1428 /* Bypass tcp protocol for fused tcp loopback */
1410 1429 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1411 1430 return;
1412 1431
1413 1432 /* Strip off the T_EXDATA_REQ if the data is from TPI */
1414 1433 if (DB_TYPE(mp) != M_DATA) {
1415 1434 mblk_t *mp1 = mp;
1416 1435 ASSERT(!IPCL_IS_NONSTR(connp));
1417 1436 mp = mp->b_cont;
1418 1437 freeb(mp1);
1419 1438 }
1420 1439 tcp_wput_data(tcp, mp, B_TRUE);
1421 1440 }
1422 1441
1423 1442 /*
1424 1443 * Called by streams close routine via squeues when our client blows off its
1425 1444 * descriptor, we take this to mean: "close the stream state NOW, close the tcp
1426 1445 * connection politely" When SO_LINGER is set (with a non-zero linger time and
1427 1446 * it is not a nonblocking socket) then this routine sleeps until the FIN is
1428 1447 * acked.
1429 1448 *
1430 1449 * NOTE: tcp_close potentially returns error when lingering.
1431 1450 * However, the stream head currently does not pass these errors
1432 1451 * to the application. 4.4BSD only returns EINTR and EWOULDBLOCK
1433 1452 * errors to the application (from tsleep()) and not errors
1434 1453 * like ECONNRESET caused by receiving a reset packet.
1435 1454 */
1436 1455
1437 1456 /* ARGSUSED */
1438 1457 void
1439 1458 tcp_close_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1440 1459 {
1441 1460 char *msg;
1442 1461 conn_t *connp = (conn_t *)arg;
1443 1462 tcp_t *tcp = connp->conn_tcp;
1444 1463 clock_t delta = 0;
1445 1464 tcp_stack_t *tcps = tcp->tcp_tcps;
1446 1465
1447 1466 /*
1448 1467 * When a non-STREAMS socket is being closed, it does not always
1449 1468 * stick around waiting for tcp_close_output to run and can therefore
1450 1469 * have dropped a reference already. So adjust the asserts accordingly.
1451 1470 */
1452 1471 ASSERT((connp->conn_fanout != NULL &&
1453 1472 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 3 : 4)) ||
1454 1473 (connp->conn_fanout == NULL &&
1455 1474 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3)));
1456 1475
1457 1476 mutex_enter(&tcp->tcp_eager_lock);
1458 1477 if (tcp->tcp_conn_req_cnt_q0 != 0 || tcp->tcp_conn_req_cnt_q != 0) {
1459 1478 /*
1460 1479 * Cleanup for listener. For non-STREAM sockets sockfs will
1461 1480 * close all the eagers on 'q', so in that case only deal
1462 1481 * with 'q0'.
1463 1482 */
1464 1483 tcp_eager_cleanup(tcp, IPCL_IS_NONSTR(connp) ? 1 : 0);
1465 1484 tcp->tcp_wait_for_eagers = 1;
1466 1485 }
1467 1486 mutex_exit(&tcp->tcp_eager_lock);
1468 1487
1469 1488 tcp->tcp_lso = B_FALSE;
1470 1489
1471 1490 msg = NULL;
1472 1491 switch (tcp->tcp_state) {
1473 1492 case TCPS_CLOSED:
1474 1493 case TCPS_IDLE:
1475 1494 break;
1476 1495 case TCPS_BOUND:
1477 1496 if (tcp->tcp_listener != NULL) {
1478 1497 ASSERT(IPCL_IS_NONSTR(connp));
1479 1498 /*
1480 1499 * Unlink from the listener and drop the reference
1481 1500 * put on it by the eager. tcp_closei_local will not
1482 1501 * do it because tcp_tconnind_started is TRUE.
1483 1502 */
1484 1503 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1485 1504 tcp_eager_unlink(tcp);
1486 1505 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1487 1506 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1488 1507 }
1489 1508 break;
1490 1509 case TCPS_LISTEN:
1491 1510 break;
1492 1511 case TCPS_SYN_SENT:
1493 1512 msg = "tcp_close, during connect";
1494 1513 break;
1495 1514 case TCPS_SYN_RCVD:
1496 1515 /*
1497 1516 * Close during the connect 3-way handshake
1498 1517 * but here there may or may not be pending data
1499 1518 * already on queue. Process almost same as in
1500 1519 * the ESTABLISHED state.
1501 1520 */
1502 1521 /* FALLTHRU */
1503 1522 default:
1504 1523 if (tcp->tcp_fused)
1505 1524 tcp_unfuse(tcp);
1506 1525
1507 1526 /*
1508 1527 * If SO_LINGER has set a zero linger time, abort the
1509 1528 * connection with a reset.
1510 1529 */
1511 1530 if (connp->conn_linger && connp->conn_lingertime == 0) {
1512 1531 msg = "tcp_close, zero lingertime";
1513 1532 break;
1514 1533 }
1515 1534
1516 1535 /*
1517 1536 * Abort connection if there is unread data queued.
1518 1537 */
1519 1538 if (tcp->tcp_rcv_list || tcp->tcp_reass_head) {
1520 1539 msg = "tcp_close, unread data";
1521 1540 break;
1522 1541 }
1523 1542
1524 1543 /*
1525 1544 * Abort connection if it is being closed without first
1526 1545 * being accepted. This can happen if a listening non-STREAM
1527 1546 * socket wants to get rid of the socket, for example, if the
1528 1547 * listener is closing.
1529 1548 */
1530 1549 if (tcp->tcp_listener != NULL) {
1531 1550 ASSERT(IPCL_IS_NONSTR(connp));
1532 1551 msg = "tcp_close, close before accept";
1533 1552
1534 1553 /*
1535 1554 * Unlink from the listener and drop the reference
1536 1555 * put on it by the eager. tcp_closei_local will not
1537 1556 * do it because tcp_tconnind_started is TRUE.
1538 1557 */
1539 1558 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1540 1559 tcp_eager_unlink(tcp);
1541 1560 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1542 1561 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1543 1562 break;
1544 1563 }
1545 1564
1546 1565 /*
1547 1566 * Transmit the FIN before detaching the tcp_t.
1548 1567 * After tcp_detach returns this queue/perimeter
1549 1568 * no longer owns the tcp_t thus others can modify it.
1550 1569 */
1551 1570 (void) tcp_xmit_end(tcp);
1552 1571
1553 1572 /*
1554 1573 * If lingering on close then wait until the fin is acked,
1555 1574 * the SO_LINGER time passes, or a reset is sent/received.
1556 1575 */
1557 1576 if (connp->conn_linger && connp->conn_lingertime > 0 &&
1558 1577 !(tcp->tcp_fin_acked) &&
1559 1578 tcp->tcp_state >= TCPS_ESTABLISHED) {
1560 1579 if (tcp->tcp_closeflags & (FNDELAY|FNONBLOCK)) {
1561 1580 tcp->tcp_client_errno = EWOULDBLOCK;
1562 1581 } else if (tcp->tcp_client_errno == 0) {
1563 1582
1564 1583 ASSERT(tcp->tcp_linger_tid == 0);
1565 1584
1566 1585 /* conn_lingertime is in sec. */
1567 1586 tcp->tcp_linger_tid = TCP_TIMER(tcp,
1568 1587 tcp_close_linger_timeout,
1569 1588 connp->conn_lingertime * MILLISEC);
1570 1589
1571 1590 /* tcp_close_linger_timeout will finish close */
1572 1591 if (tcp->tcp_linger_tid == 0)
1573 1592 tcp->tcp_client_errno = ENOSR;
1574 1593 else
1575 1594 return;
1576 1595 }
1577 1596
1578 1597 /*
1579 1598 * Check if we need to detach or just close
1580 1599 * the instance.
1581 1600 */
1582 1601 if (tcp->tcp_state <= TCPS_LISTEN)
1583 1602 break;
1584 1603 }
1585 1604
1586 1605 /*
1587 1606 * Make sure that no other thread will access the conn_rq of
1588 1607 * this instance (through lookups etc.) as conn_rq will go
1589 1608 * away shortly.
1590 1609 */
1591 1610 tcp_acceptor_hash_remove(tcp);
1592 1611
1593 1612 mutex_enter(&tcp->tcp_non_sq_lock);
1594 1613 if (tcp->tcp_flow_stopped) {
1595 1614 tcp_clrqfull(tcp);
1596 1615 }
1597 1616 mutex_exit(&tcp->tcp_non_sq_lock);
1598 1617
1599 1618 if (tcp->tcp_timer_tid != 0) {
1600 1619 delta = TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid);
1601 1620 tcp->tcp_timer_tid = 0;
1602 1621 }
1603 1622 /*
1604 1623 * Need to cancel those timers which will not be used when
1605 1624 * TCP is detached. This has to be done before the conn_wq
1606 1625 * is set to NULL.
1607 1626 */
1608 1627 tcp_timers_stop(tcp);
1609 1628
1610 1629 tcp->tcp_detached = B_TRUE;
1611 1630 if (tcp->tcp_state == TCPS_TIME_WAIT) {
1612 1631 tcp_time_wait_append(tcp);
1613 1632 TCP_DBGSTAT(tcps, tcp_detach_time_wait);
1614 1633 ASSERT(connp->conn_ref >=
1615 1634 (IPCL_IS_NONSTR(connp) ? 2 : 3));
1616 1635 goto finish;
1617 1636 }
1618 1637
1619 1638 /*
1620 1639 * If delta is zero the timer event wasn't executed and was
1621 1640 * successfully canceled. In this case we need to restart it
1622 1641 * with the minimal delta possible.
1623 1642 */
1624 1643 if (delta >= 0)
1625 1644 tcp->tcp_timer_tid = TCP_TIMER(tcp, tcp_timer,
1626 1645 delta ? delta : 1);
1627 1646
1628 1647 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3));
1629 1648 goto finish;
1630 1649 }
1631 1650
1632 1651 /* Detach did not complete. Still need to remove q from stream. */
1633 1652 if (msg) {
1634 1653 if (tcp->tcp_state == TCPS_ESTABLISHED ||
1635 1654 tcp->tcp_state == TCPS_CLOSE_WAIT)
1636 1655 TCPS_BUMP_MIB(tcps, tcpEstabResets);
1637 1656 if (tcp->tcp_state == TCPS_SYN_SENT ||
1638 1657 tcp->tcp_state == TCPS_SYN_RCVD)
1639 1658 TCPS_BUMP_MIB(tcps, tcpAttemptFails);
1640 1659 tcp_xmit_ctl(msg, tcp, tcp->tcp_snxt, 0, TH_RST);
1641 1660 }
1642 1661
1643 1662 tcp_closei_local(tcp);
1644 1663 CONN_DEC_REF(connp);
1645 1664 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 1 : 2));
1646 1665
1647 1666 finish:
1648 1667 /*
1649 1668 * Don't change the queues in the case of a listener that has
1650 1669 * eagers in its q or q0. It could surprise the eagers.
1651 1670 * Instead wait for the eagers outside the squeue.
1652 1671 *
1653 1672 * For non-STREAMS sockets tcp_wait_for_eagers implies that
1654 1673 * we should delay the su_closed upcall until all eagers have
1655 1674 * dropped their references.
1656 1675 */
1657 1676 if (!tcp->tcp_wait_for_eagers) {
1658 1677 tcp->tcp_detached = B_TRUE;
1659 1678 connp->conn_rq = NULL;
1660 1679 connp->conn_wq = NULL;
1661 1680
1662 1681 /* non-STREAM socket, release the upper handle */
1663 1682 if (IPCL_IS_NONSTR(connp)) {
1664 1683 ASSERT(connp->conn_upper_handle != NULL);
1665 1684 (*connp->conn_upcalls->su_closed)
1666 1685 (connp->conn_upper_handle);
1667 1686 connp->conn_upper_handle = NULL;
1668 1687 connp->conn_upcalls = NULL;
1669 1688 }
1670 1689 }
1671 1690
1672 1691 /* Signal tcp_close() to finish closing. */
1673 1692 mutex_enter(&tcp->tcp_closelock);
1674 1693 tcp->tcp_closed = 1;
1675 1694 cv_signal(&tcp->tcp_closecv);
1676 1695 mutex_exit(&tcp->tcp_closelock);
1677 1696 }
1678 1697
1679 1698 /* ARGSUSED */
1680 1699 void
1681 1700 tcp_shutdown_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1682 1701 {
1683 1702 conn_t *connp = (conn_t *)arg;
1684 1703 tcp_t *tcp = connp->conn_tcp;
1685 1704
1686 1705 freemsg(mp);
1687 1706
1688 1707 if (tcp->tcp_fused)
1689 1708 tcp_unfuse(tcp);
1690 1709
1691 1710 if (tcp_xmit_end(tcp) != 0) {
1692 1711 /*
1693 1712 * We were crossing FINs and got a reset from
1694 1713 * the other side. Just ignore it.
1695 1714 */
1696 1715 if (connp->conn_debug) {
1697 1716 (void) strlog(TCP_MOD_ID, 0, 1,
1698 1717 SL_ERROR|SL_TRACE,
1699 1718 "tcp_shutdown_output() out of state %s",
1700 1719 tcp_display(tcp, NULL, DISP_ADDR_AND_PORT));
1701 1720 }
1702 1721 }
1703 1722 }
1704 1723
1705 1724 #pragma inline(tcp_send_data)
1706 1725
1707 1726 void
1708 1727 tcp_send_data(tcp_t *tcp, mblk_t *mp)
1709 1728 {
1710 1729 conn_t *connp = tcp->tcp_connp;
1711 1730
1712 1731 /*
1713 1732 * Check here to avoid sending zero-copy message down to IP when
1714 1733 * ZEROCOPY capability has turned off. We only need to deal with
1715 1734 * the race condition between sockfs and the notification here.
1716 1735 * Since we have tried to backoff the tcp_xmit_head when turning
1717 1736 * zero-copy off and new messages in tcp_output(), we simply drop
1718 1737 * the dup'ed packet here and let tcp retransmit, if tcp_xmit_zc_clean
1719 1738 * is not true.
1720 1739 */
1721 1740 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on &&
1722 1741 !tcp->tcp_xmit_zc_clean) {
1723 1742 ip_drop_output("TCP ZC was disabled but not clean", mp, NULL);
1724 1743 freemsg(mp);
1725 1744 return;
1726 1745 }
1727 1746
1728 1747 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, connp->conn_ixa,
1729 1748 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, tcp,
1730 1749 __dtrace_tcp_tcph_t *,
1731 1750 &mp->b_rptr[connp->conn_ixa->ixa_ip_hdr_length]);
1732 1751
1733 1752 ASSERT(connp->conn_ixa->ixa_notify_cookie == connp->conn_tcp);
1734 1753 (void) conn_ip_output(mp, connp->conn_ixa);
1735 1754 }
1736 1755
1737 1756 /* ARGSUSED2 */
1738 1757 void
1739 1758 tcp_send_synack(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1740 1759 {
1741 1760 conn_t *econnp = (conn_t *)arg;
1742 1761 tcp_t *tcp = econnp->conn_tcp;
1743 1762 ip_xmit_attr_t *ixa = econnp->conn_ixa;
1744 1763
1745 1764 /* Guard against a RST having blown it away while on the squeue */
1746 1765 if (tcp->tcp_state == TCPS_CLOSED) {
1747 1766 freemsg(mp);
1748 1767 return;
1749 1768 }
1750 1769
1751 1770 /*
1752 1771 * In the off-chance that the eager received and responded to
1753 1772 * some other packet while the SYN|ACK was queued, we recalculate
1754 1773 * the ixa_pktlen. It would be better to fix the SYN/accept
1755 1774 * multithreading scheme to avoid this complexity.
1756 1775 */
1757 1776 ixa->ixa_pktlen = msgdsize(mp);
1758 1777 (void) conn_ip_output(mp, ixa);
1759 1778 }
1760 1779
1761 1780 /*
1762 1781 * tcp_send() is called by tcp_wput_data() and returns one of the following:
1763 1782 *
1764 1783 * -1 = failed allocation.
1765 1784 * 0 = We've either successfully sent data, or our usable send window is too
1766 1785 * small and we'd rather wait until later before sending again.
1767 1786 */
1768 1787 static int
1769 1788 tcp_send(tcp_t *tcp, const int mss, const int total_hdr_len,
1770 1789 const int tcp_hdr_len, const int num_sack_blk, int *usable,
1771 1790 uint_t *snxt, int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time)
1772 1791 {
1773 1792 int num_lso_seg = 1;
1774 1793 uint_t lso_usable;
1775 1794 boolean_t do_lso_send = B_FALSE;
1776 1795 tcp_stack_t *tcps = tcp->tcp_tcps;
1777 1796 conn_t *connp = tcp->tcp_connp;
1778 1797 ip_xmit_attr_t *ixa = connp->conn_ixa;
1779 1798
1780 1799 /*
1781 1800 * Check LSO possibility. The value of tcp->tcp_lso indicates whether
1782 1801 * the underlying connection is LSO capable. Will check whether having
1783 1802 * enough available data to initiate LSO transmission in the for(){}
1784 1803 * loops.
1785 1804 */
1786 1805 if (tcp->tcp_lso && (tcp->tcp_valid_bits & ~TCP_FSS_VALID) == 0)
1787 1806 do_lso_send = B_TRUE;
1788 1807
1789 1808 for (;;) {
1790 1809 struct datab *db;
1791 1810 tcpha_t *tcpha;
1792 1811 uint32_t sum;
1793 1812 mblk_t *mp, *mp1;
1794 1813 uchar_t *rptr;
1795 1814 int len;
1796 1815
1797 1816 /*
1798 1817 * Calculate the maximum payload length we can send at one
1799 1818 * time.
1800 1819 */
1801 1820 if (do_lso_send) {
1802 1821 /*
1803 1822 * Determine whether or not it's possible to do LSO,
1804 1823 * and if so, how much data we can send.
1805 1824 */
1806 1825 if ((*usable - 1) / mss >= 1) {
1807 1826 lso_usable = MIN(tcp->tcp_lso_max, *usable);
1808 1827 num_lso_seg = lso_usable / mss;
1809 1828 if (lso_usable % mss) {
1810 1829 num_lso_seg++;
1811 1830 tcp->tcp_last_sent_len = (ushort_t)
1812 1831 (lso_usable % mss);
1813 1832 } else {
1814 1833 tcp->tcp_last_sent_len = (ushort_t)mss;
1815 1834 }
1816 1835 } else {
1817 1836 do_lso_send = B_FALSE;
1818 1837 num_lso_seg = 1;
1819 1838 lso_usable = mss;
1820 1839 }
1821 1840 }
1822 1841
1823 1842 ASSERT(num_lso_seg <= IP_MAXPACKET / mss + 1);
1824 1843
1825 1844 len = mss;
1826 1845 if (len > *usable) {
1827 1846 ASSERT(do_lso_send == B_FALSE);
1828 1847
1829 1848 len = *usable;
1830 1849 if (len <= 0) {
1831 1850 /* Terminate the loop */
1832 1851 break; /* success; too small */
1833 1852 }
1834 1853 /*
1835 1854 * Sender silly-window avoidance.
1836 1855 * Ignore this if we are going to send a
1837 1856 * zero window probe out.
1838 1857 *
1839 1858 * TODO: force data into microscopic window?
1840 1859 * ==> (!pushed || (unsent > usable))
1841 1860 */
1842 1861 if (len < (tcp->tcp_max_swnd >> 1) &&
1843 1862 (tcp->tcp_unsent - (*snxt - tcp->tcp_snxt)) > len &&
1844 1863 !((tcp->tcp_valid_bits & TCP_URG_VALID) &&
1845 1864 len == 1) && (! tcp->tcp_zero_win_probe)) {
1846 1865 /*
1847 1866 * If the retransmit timer is not running
1848 1867 * we start it so that we will retransmit
1849 1868 * in the case when the receiver has
1850 1869 * decremented the window.
1851 1870 */
1852 1871 if (*snxt == tcp->tcp_snxt &&
1853 1872 *snxt == tcp->tcp_suna) {
1854 1873 /*
1855 1874 * We are not supposed to send
1856 1875 * anything. So let's wait a little
1857 1876 * bit longer before breaking SWS
1858 1877 * avoidance.
1859 1878 *
1860 1879 * What should the value be?
1861 1880 * Suggestion: MAX(init rexmit time,
1862 1881 * tcp->tcp_rto)
1863 1882 */
1864 1883 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1865 1884 }
1866 1885 break; /* success; too small */
1867 1886 }
1868 1887 }
1869 1888
1870 1889 tcpha = tcp->tcp_tcpha;
1871 1890
1872 1891 /*
1873 1892 * The reason to adjust len here is that we need to set flags
1874 1893 * and calculate checksum.
1875 1894 */
1876 1895 if (do_lso_send)
1877 1896 len = lso_usable;
1878 1897
1879 1898 *usable -= len; /* Approximate - can be adjusted later */
1880 1899 if (*usable > 0)
1881 1900 tcpha->tha_flags = TH_ACK;
1882 1901 else
1883 1902 tcpha->tha_flags = (TH_ACK | TH_PUSH);
1884 1903
1885 1904 /*
1886 1905 * Prime pump for IP's checksumming on our behalf.
1887 1906 * Include the adjustment for a source route if any.
1888 1907 * In case of LSO, the partial pseudo-header checksum should
1889 1908 * exclusive TCP length, so zero tha_sum before IP calculate
1890 1909 * pseudo-header checksum for partial checksum offload.
1891 1910 */
1892 1911 if (do_lso_send) {
1893 1912 sum = 0;
1894 1913 } else {
1895 1914 sum = len + tcp_hdr_len + connp->conn_sum;
1896 1915 sum = (sum >> 16) + (sum & 0xFFFF);
1897 1916 }
1898 1917 tcpha->tha_sum = htons(sum);
1899 1918 tcpha->tha_seq = htonl(*snxt);
1900 1919
1901 1920 /*
1902 1921 * Branch off to tcp_xmit_mp() if any of the VALID bits is
1903 1922 * set. For the case when TCP_FSS_VALID is the only valid
1904 1923 * bit (normal active close), branch off only when we think
1905 1924 * that the FIN flag needs to be set. Note for this case,
1906 1925 * that (snxt + len) may not reflect the actual seg_len,
1907 1926 * as len may be further reduced in tcp_xmit_mp(). If len
1908 1927 * gets modified, we will end up here again.
1909 1928 */
1910 1929 if (tcp->tcp_valid_bits != 0 &&
1911 1930 (tcp->tcp_valid_bits != TCP_FSS_VALID ||
1912 1931 ((*snxt + len) == tcp->tcp_fss))) {
1913 1932 uchar_t *prev_rptr;
1914 1933 uint32_t prev_snxt = tcp->tcp_snxt;
1915 1934
1916 1935 if (*tail_unsent == 0) {
1917 1936 ASSERT((*xmit_tail)->b_cont != NULL);
1918 1937 *xmit_tail = (*xmit_tail)->b_cont;
1919 1938 prev_rptr = (*xmit_tail)->b_rptr;
1920 1939 *tail_unsent = (int)((*xmit_tail)->b_wptr -
1921 1940 (*xmit_tail)->b_rptr);
1922 1941 } else {
1923 1942 prev_rptr = (*xmit_tail)->b_rptr;
1924 1943 (*xmit_tail)->b_rptr = (*xmit_tail)->b_wptr -
1925 1944 *tail_unsent;
1926 1945 }
1927 1946 mp = tcp_xmit_mp(tcp, *xmit_tail, len, NULL, NULL,
1928 1947 *snxt, B_FALSE, (uint32_t *)&len, B_FALSE);
1929 1948 /* Restore tcp_snxt so we get amount sent right. */
1930 1949 tcp->tcp_snxt = prev_snxt;
1931 1950 if (prev_rptr == (*xmit_tail)->b_rptr) {
1932 1951 /*
1933 1952 * If the previous timestamp is still in use,
1934 1953 * don't stomp on it.
1935 1954 */
1936 1955 if ((*xmit_tail)->b_next == NULL) {
1937 1956 (*xmit_tail)->b_prev = local_time;
1938 1957 (*xmit_tail)->b_next =
1939 1958 (mblk_t *)(uintptr_t)(*snxt);
1940 1959 }
1941 1960 } else
1942 1961 (*xmit_tail)->b_rptr = prev_rptr;
1943 1962
1944 1963 if (mp == NULL) {
1945 1964 return (-1);
1946 1965 }
1947 1966 mp1 = mp->b_cont;
1948 1967
1949 1968 if (len <= mss) /* LSO is unusable (!do_lso_send) */
|
↓ open down ↓ |
623 lines elided |
↑ open up ↑ |
1950 1969 tcp->tcp_last_sent_len = (ushort_t)len;
1951 1970 while (mp1->b_cont) {
1952 1971 *xmit_tail = (*xmit_tail)->b_cont;
1953 1972 (*xmit_tail)->b_prev = local_time;
1954 1973 (*xmit_tail)->b_next =
1955 1974 (mblk_t *)(uintptr_t)(*snxt);
1956 1975 mp1 = mp1->b_cont;
1957 1976 }
1958 1977 *snxt += len;
1959 1978 *tail_unsent = (*xmit_tail)->b_wptr - mp1->b_wptr;
1960 - BUMP_LOCAL(tcp->tcp_obsegs);
1979 + TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1961 1980 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1962 1981 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1982 + tcp->tcp_cs.tcp_out_data_segs++;
1983 + tcp->tcp_cs.tcp_out_data_bytes += len;
1963 1984 tcp_send_data(tcp, mp);
1964 1985 continue;
1965 1986 }
1966 1987
1967 1988 *snxt += len; /* Adjust later if we don't send all of len */
1989 + TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1968 1990 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1969 1991 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1992 + tcp->tcp_cs.tcp_out_data_segs++;
1993 + tcp->tcp_cs.tcp_out_data_bytes += len;
1970 1994
1971 1995 if (*tail_unsent) {
1972 1996 /* Are the bytes above us in flight? */
1973 1997 rptr = (*xmit_tail)->b_wptr - *tail_unsent;
1974 1998 if (rptr != (*xmit_tail)->b_rptr) {
1975 1999 *tail_unsent -= len;
1976 2000 if (len <= mss) /* LSO is unusable */
1977 2001 tcp->tcp_last_sent_len = (ushort_t)len;
1978 2002 len += total_hdr_len;
1979 2003 ixa->ixa_pktlen = len;
1980 2004
1981 2005 if (ixa->ixa_flags & IXAF_IS_IPV4) {
1982 2006 tcp->tcp_ipha->ipha_length = htons(len);
1983 2007 } else {
1984 2008 tcp->tcp_ip6h->ip6_plen =
1985 2009 htons(len - IPV6_HDR_LEN);
1986 2010 }
1987 2011
1988 2012 mp = dupb(*xmit_tail);
1989 2013 if (mp == NULL) {
1990 2014 return (-1); /* out_of_mem */
1991 2015 }
1992 2016 mp->b_rptr = rptr;
1993 2017 /*
1994 2018 * If the old timestamp is no longer in use,
1995 2019 * sample a new timestamp now.
1996 2020 */
1997 2021 if ((*xmit_tail)->b_next == NULL) {
1998 2022 (*xmit_tail)->b_prev = local_time;
1999 2023 (*xmit_tail)->b_next =
2000 2024 (mblk_t *)(uintptr_t)(*snxt-len);
2001 2025 }
2002 2026 goto must_alloc;
2003 2027 }
2004 2028 } else {
2005 2029 *xmit_tail = (*xmit_tail)->b_cont;
2006 2030 ASSERT((uintptr_t)((*xmit_tail)->b_wptr -
2007 2031 (*xmit_tail)->b_rptr) <= (uintptr_t)INT_MAX);
2008 2032 *tail_unsent = (int)((*xmit_tail)->b_wptr -
2009 2033 (*xmit_tail)->b_rptr);
2010 2034 }
2011 2035
2012 2036 (*xmit_tail)->b_prev = local_time;
2013 2037 (*xmit_tail)->b_next = (mblk_t *)(uintptr_t)(*snxt - len);
2014 2038
2015 2039 *tail_unsent -= len;
2016 2040 if (len <= mss) /* LSO is unusable (!do_lso_send) */
2017 2041 tcp->tcp_last_sent_len = (ushort_t)len;
2018 2042
2019 2043 len += total_hdr_len;
2020 2044 ixa->ixa_pktlen = len;
2021 2045
2022 2046 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2023 2047 tcp->tcp_ipha->ipha_length = htons(len);
2024 2048 } else {
2025 2049 tcp->tcp_ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2026 2050 }
2027 2051
2028 2052 mp = dupb(*xmit_tail);
2029 2053 if (mp == NULL) {
2030 2054 return (-1); /* out_of_mem */
2031 2055 }
2032 2056
2033 2057 len = total_hdr_len;
2034 2058 /*
2035 2059 * There are four reasons to allocate a new hdr mblk:
2036 2060 * 1) The bytes above us are in use by another packet
2037 2061 * 2) We don't have good alignment
2038 2062 * 3) The mblk is being shared
2039 2063 * 4) We don't have enough room for a header
2040 2064 */
2041 2065 rptr = mp->b_rptr - len;
2042 2066 if (!OK_32PTR(rptr) ||
2043 2067 ((db = mp->b_datap), db->db_ref != 2) ||
2044 2068 rptr < db->db_base) {
2045 2069 /* NOTE: we assume allocb returns an OK_32PTR */
2046 2070
2047 2071 must_alloc:;
2048 2072 mp1 = allocb(connp->conn_ht_iphc_allocated +
2049 2073 tcps->tcps_wroff_xtra, BPRI_MED);
2050 2074 if (mp1 == NULL) {
2051 2075 freemsg(mp);
2052 2076 return (-1); /* out_of_mem */
2053 2077 }
2054 2078 mp1->b_cont = mp;
2055 2079 mp = mp1;
|
↓ open down ↓ |
76 lines elided |
↑ open up ↑ |
2056 2080 /* Leave room for Link Level header */
2057 2081 len = total_hdr_len;
2058 2082 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2059 2083 mp->b_wptr = &rptr[len];
2060 2084 }
2061 2085
2062 2086 /*
2063 2087 * Fill in the header using the template header, and add
2064 2088 * options such as time-stamp, ECN and/or SACK, as needed.
2065 2089 */
2066 - tcp_fill_header(tcp, rptr, (clock_t)local_time, num_sack_blk);
2090 + tcp_fill_header(tcp, rptr, num_sack_blk);
2067 2091
2068 2092 mp->b_rptr = rptr;
2069 2093
2070 2094 if (*tail_unsent) {
2071 2095 int spill = *tail_unsent;
2072 2096
2073 2097 mp1 = mp->b_cont;
2074 2098 if (mp1 == NULL)
2075 2099 mp1 = mp;
2076 2100
2077 2101 /*
2078 2102 * If we're a little short, tack on more mblks until
2079 2103 * there is no more spillover.
2080 2104 */
2081 2105 while (spill < 0) {
2082 2106 mblk_t *nmp;
2083 2107 int nmpsz;
2084 2108
2085 2109 nmp = (*xmit_tail)->b_cont;
2086 2110 nmpsz = MBLKL(nmp);
2087 2111
2088 2112 /*
2089 2113 * Excess data in mblk; can we split it?
2090 2114 * If LSO is enabled for the connection,
2091 2115 * keep on splitting as this is a transient
2092 2116 * send path.
2093 2117 */
2094 2118 if (!do_lso_send && (spill + nmpsz > 0)) {
2095 2119 /*
2096 2120 * Don't split if stream head was
2097 2121 * told to break up larger writes
2098 2122 * into smaller ones.
2099 2123 */
2100 2124 if (tcp->tcp_maxpsz_multiplier > 0)
2101 2125 break;
2102 2126
2103 2127 /*
2104 2128 * Next mblk is less than SMSS/2
2105 2129 * rounded up to nearest 64-byte;
2106 2130 * let it get sent as part of the
2107 2131 * next segment.
2108 2132 */
2109 2133 if (tcp->tcp_localnet &&
2110 2134 !tcp->tcp_cork &&
2111 2135 (nmpsz < roundup((mss >> 1), 64)))
2112 2136 break;
2113 2137 }
2114 2138
2115 2139 *xmit_tail = nmp;
2116 2140 ASSERT((uintptr_t)nmpsz <= (uintptr_t)INT_MAX);
2117 2141 /* Stash for rtt use later */
2118 2142 (*xmit_tail)->b_prev = local_time;
2119 2143 (*xmit_tail)->b_next =
2120 2144 (mblk_t *)(uintptr_t)(*snxt - len);
2121 2145 mp1->b_cont = dupb(*xmit_tail);
2122 2146 mp1 = mp1->b_cont;
2123 2147
2124 2148 spill += nmpsz;
2125 2149 if (mp1 == NULL) {
2126 2150 *tail_unsent = spill;
2127 2151 freemsg(mp);
2128 2152 return (-1); /* out_of_mem */
2129 2153 }
2130 2154 }
2131 2155
2132 2156 /* Trim back any surplus on the last mblk */
2133 2157 if (spill >= 0) {
2134 2158 mp1->b_wptr -= spill;
|
↓ open down ↓ |
58 lines elided |
↑ open up ↑ |
2135 2159 *tail_unsent = spill;
2136 2160 } else {
2137 2161 /*
2138 2162 * We did not send everything we could in
2139 2163 * order to remain within the b_cont limit.
2140 2164 */
2141 2165 *usable -= spill;
2142 2166 *snxt += spill;
2143 2167 tcp->tcp_last_sent_len += spill;
2144 2168 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, spill);
2169 + tcp->tcp_cs.tcp_out_data_bytes += spill;
2145 2170 /*
2146 2171 * Adjust the checksum
2147 2172 */
2148 2173 tcpha = (tcpha_t *)(rptr +
2149 2174 ixa->ixa_ip_hdr_length);
2150 2175 sum += spill;
2151 2176 sum = (sum >> 16) + (sum & 0xFFFF);
2152 2177 tcpha->tha_sum = htons(sum);
2153 2178 if (connp->conn_ipversion == IPV4_VERSION) {
2154 2179 sum = ntohs(
2155 2180 ((ipha_t *)rptr)->ipha_length) +
2156 2181 spill;
2157 2182 ((ipha_t *)rptr)->ipha_length =
2158 2183 htons(sum);
2159 2184 } else {
2160 2185 sum = ntohs(
2161 2186 ((ip6_t *)rptr)->ip6_plen) +
2162 2187 spill;
2163 2188 ((ip6_t *)rptr)->ip6_plen =
2164 2189 htons(sum);
2165 2190 }
2166 2191 ixa->ixa_pktlen += spill;
2167 2192 *tail_unsent = 0;
2168 2193 }
2169 2194 }
2170 2195 if (tcp->tcp_ip_forward_progress) {
2171 2196 tcp->tcp_ip_forward_progress = B_FALSE;
2172 2197 ixa->ixa_flags |= IXAF_REACH_CONF;
2173 2198 } else {
2174 2199 ixa->ixa_flags &= ~IXAF_REACH_CONF;
2175 2200 }
2176 2201
2177 2202 if (do_lso_send) {
2178 2203 /* Append LSO information to the mp. */
2179 2204 lso_info_set(mp, mss, HW_LSO);
2180 2205 ixa->ixa_fragsize = IP_MAXPACKET;
2181 2206 ixa->ixa_extra_ident = num_lso_seg - 1;
2182 2207
|
↓ open down ↓ |
28 lines elided |
↑ open up ↑ |
2183 2208 DTRACE_PROBE2(tcp_send_lso, int, num_lso_seg,
2184 2209 boolean_t, B_TRUE);
2185 2210
2186 2211 tcp_send_data(tcp, mp);
2187 2212
2188 2213 /*
2189 2214 * Restore values of ixa_fragsize and ixa_extra_ident.
2190 2215 */
2191 2216 ixa->ixa_fragsize = ixa->ixa_pmtu;
2192 2217 ixa->ixa_extra_ident = 0;
2193 - tcp->tcp_obsegs += num_lso_seg;
2218 + TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2194 2219 TCP_STAT(tcps, tcp_lso_times);
2195 2220 TCP_STAT_UPDATE(tcps, tcp_lso_pkt_out, num_lso_seg);
2196 2221 } else {
2197 2222 /*
2198 2223 * Make sure to clean up LSO information. Wherever a
2199 2224 * new mp uses the prepended header room after dupb(),
2200 2225 * lso_info_cleanup() should be called.
2201 2226 */
2202 2227 lso_info_cleanup(mp);
2203 2228 tcp_send_data(tcp, mp);
2204 - BUMP_LOCAL(tcp->tcp_obsegs);
2229 + TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2205 2230 }
2206 2231 }
2207 2232
2208 2233 return (0);
2209 2234 }
2210 2235
2211 2236 /*
2212 2237 * Initiate closedown sequence on an active connection. (May be called as
2213 2238 * writer.) Return value zero for OK return, non-zero for error return.
2214 2239 */
2215 2240 static int
2216 2241 tcp_xmit_end(tcp_t *tcp)
2217 2242 {
2218 2243 mblk_t *mp;
2219 2244 tcp_stack_t *tcps = tcp->tcp_tcps;
2220 2245 iulp_t uinfo;
2221 2246 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip;
2222 2247 conn_t *connp = tcp->tcp_connp;
2223 2248
2224 2249 if (tcp->tcp_state < TCPS_SYN_RCVD ||
2225 2250 tcp->tcp_state > TCPS_CLOSE_WAIT) {
2226 2251 /*
2227 2252 * Invalid state, only states TCPS_SYN_RCVD,
2228 2253 * TCPS_ESTABLISHED and TCPS_CLOSE_WAIT are valid
2229 2254 */
2230 2255 return (-1);
2231 2256 }
2232 2257
2233 2258 tcp->tcp_fss = tcp->tcp_snxt + tcp->tcp_unsent;
2234 2259 tcp->tcp_valid_bits |= TCP_FSS_VALID;
2235 2260 /*
2236 2261 * If there is nothing more unsent, send the FIN now.
2237 2262 * Otherwise, it will go out with the last segment.
2238 2263 */
2239 2264 if (tcp->tcp_unsent == 0) {
2240 2265 mp = tcp_xmit_mp(tcp, NULL, 0, NULL, NULL,
2241 2266 tcp->tcp_fss, B_FALSE, NULL, B_FALSE);
2242 2267
2243 2268 if (mp) {
2244 2269 tcp_send_data(tcp, mp);
2245 2270 } else {
2246 2271 /*
2247 2272 * Couldn't allocate msg. Pretend we got it out.
2248 2273 * Wait for rexmit timeout.
2249 2274 */
2250 2275 tcp->tcp_snxt = tcp->tcp_fss + 1;
2251 2276 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2252 2277 }
2253 2278
2254 2279 /*
2255 2280 * If needed, update tcp_rexmit_snxt as tcp_snxt is
2256 2281 * changed.
2257 2282 */
2258 2283 if (tcp->tcp_rexmit && tcp->tcp_rexmit_nxt == tcp->tcp_fss) {
2259 2284 tcp->tcp_rexmit_nxt = tcp->tcp_snxt;
2260 2285 }
2261 2286 } else {
2262 2287 /*
2263 2288 * If tcp->tcp_cork is set, then the data will not get sent,
2264 2289 * so we have to check that and unset it first.
2265 2290 */
2266 2291 if (tcp->tcp_cork)
2267 2292 tcp->tcp_cork = B_FALSE;
2268 2293 tcp_wput_data(tcp, NULL, B_FALSE);
2269 2294 }
2270 2295
2271 2296 /*
2272 2297 * If TCP does not get enough samples of RTT or tcp_rtt_updates
2273 2298 * is 0, don't update the cache.
|
↓ open down ↓ |
59 lines elided |
↑ open up ↑ |
2274 2299 */
2275 2300 if (tcps->tcps_rtt_updates == 0 ||
2276 2301 tcp->tcp_rtt_update < tcps->tcps_rtt_updates)
2277 2302 return (0);
2278 2303
2279 2304 /*
2280 2305 * We do not have a good algorithm to update ssthresh at this time.
2281 2306 * So don't do any update.
2282 2307 */
2283 2308 bzero(&uinfo, sizeof (uinfo));
2284 - uinfo.iulp_rtt = tcp->tcp_rtt_sa;
2285 - uinfo.iulp_rtt_sd = tcp->tcp_rtt_sd;
2309 + uinfo.iulp_rtt = NSEC2MSEC(tcp->tcp_rtt_sa);
2310 + uinfo.iulp_rtt_sd = NSEC2MSEC(tcp->tcp_rtt_sd);
2286 2311
2287 2312 /*
2288 2313 * Note that uinfo is kept for conn_faddr in the DCE. Could update even
2289 2314 * if source routed but we don't.
2290 2315 */
2291 2316 if (connp->conn_ipversion == IPV4_VERSION) {
2292 2317 if (connp->conn_faddr_v4 != tcp->tcp_ipha->ipha_dst) {
2293 2318 return (0);
2294 2319 }
2295 2320 (void) dce_update_uinfo_v4(connp->conn_faddr_v4, &uinfo, ipst);
2296 2321 } else {
2297 2322 uint_t ifindex;
2298 2323
2299 2324 if (!(IN6_ARE_ADDR_EQUAL(&connp->conn_faddr_v6,
2300 2325 &tcp->tcp_ip6h->ip6_dst))) {
2301 2326 return (0);
2302 2327 }
2303 2328 ifindex = 0;
2304 2329 if (IN6_IS_ADDR_LINKSCOPE(&connp->conn_faddr_v6)) {
2305 2330 ip_xmit_attr_t *ixa = connp->conn_ixa;
2306 2331
2307 2332 /*
2308 2333 * If we are going to create a DCE we'd better have
2309 2334 * an ifindex
2310 2335 */
2311 2336 if (ixa->ixa_nce != NULL) {
2312 2337 ifindex = ixa->ixa_nce->nce_common->ncec_ill->
2313 2338 ill_phyint->phyint_ifindex;
2314 2339 } else {
2315 2340 return (0);
2316 2341 }
2317 2342 }
2318 2343
2319 2344 (void) dce_update_uinfo(&connp->conn_faddr_v6, ifindex, &uinfo,
2320 2345 ipst);
2321 2346 }
2322 2347 return (0);
2323 2348 }
2324 2349
2325 2350 /*
2326 2351 * Send out a control packet on the tcp connection specified. This routine
2327 2352 * is typically called where we need a simple ACK or RST generated.
2328 2353 */
2329 2354 void
2330 2355 tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq, uint32_t ack, int ctl)
2331 2356 {
2332 2357 uchar_t *rptr;
2333 2358 tcpha_t *tcpha;
2334 2359 ipha_t *ipha = NULL;
2335 2360 ip6_t *ip6h = NULL;
2336 2361 uint32_t sum;
2337 2362 int total_hdr_len;
2338 2363 int ip_hdr_len;
2339 2364 mblk_t *mp;
2340 2365 tcp_stack_t *tcps = tcp->tcp_tcps;
2341 2366 conn_t *connp = tcp->tcp_connp;
2342 2367 ip_xmit_attr_t *ixa = connp->conn_ixa;
2343 2368
2344 2369 /*
2345 2370 * Save sum for use in source route later.
2346 2371 */
2347 2372 sum = connp->conn_ht_ulp_len + connp->conn_sum;
2348 2373 total_hdr_len = connp->conn_ht_iphc_len;
2349 2374 ip_hdr_len = ixa->ixa_ip_hdr_length;
2350 2375
2351 2376 /* If a text string is passed in with the request, pass it to strlog. */
2352 2377 if (str != NULL && connp->conn_debug) {
2353 2378 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2354 2379 "tcp_xmit_ctl: '%s', seq 0x%x, ack 0x%x, ctl 0x%x",
2355 2380 str, seq, ack, ctl);
2356 2381 }
2357 2382 mp = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
2358 2383 BPRI_MED);
2359 2384 if (mp == NULL) {
2360 2385 return;
2361 2386 }
2362 2387 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2363 2388 mp->b_rptr = rptr;
2364 2389 mp->b_wptr = &rptr[total_hdr_len];
2365 2390 bcopy(connp->conn_ht_iphc, rptr, total_hdr_len);
2366 2391
2367 2392 ixa->ixa_pktlen = total_hdr_len;
2368 2393
2369 2394 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2370 2395 ipha = (ipha_t *)rptr;
2371 2396 ipha->ipha_length = htons(total_hdr_len);
2372 2397 } else {
2373 2398 ip6h = (ip6_t *)rptr;
2374 2399 ip6h->ip6_plen = htons(total_hdr_len - IPV6_HDR_LEN);
2375 2400 }
2376 2401 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2377 2402 tcpha->tha_flags = (uint8_t)ctl;
2378 2403 if (ctl & TH_RST) {
2379 2404 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2380 2405 TCPS_BUMP_MIB(tcps, tcpOutControl);
2381 2406 /*
2382 2407 * Don't send TSopt w/ TH_RST packets per RFC 1323.
2383 2408 */
2384 2409 if (tcp->tcp_snd_ts_ok &&
2385 2410 tcp->tcp_state > TCPS_SYN_SENT) {
2386 2411 mp->b_wptr = &rptr[total_hdr_len - TCPOPT_REAL_TS_LEN];
2387 2412 *(mp->b_wptr) = TCPOPT_EOL;
2388 2413
2389 2414 ixa->ixa_pktlen = total_hdr_len - TCPOPT_REAL_TS_LEN;
2390 2415
2391 2416 if (connp->conn_ipversion == IPV4_VERSION) {
2392 2417 ipha->ipha_length = htons(total_hdr_len -
2393 2418 TCPOPT_REAL_TS_LEN);
2394 2419 } else {
2395 2420 ip6h->ip6_plen = htons(total_hdr_len -
2396 2421 IPV6_HDR_LEN - TCPOPT_REAL_TS_LEN);
2397 2422 }
2398 2423 tcpha->tha_offset_and_reserved -= (3 << 4);
2399 2424 sum -= TCPOPT_REAL_TS_LEN;
2400 2425 }
2401 2426 }
2402 2427 if (ctl & TH_ACK) {
2403 2428 if (tcp->tcp_snd_ts_ok) {
2404 2429 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2405 2430
2406 2431 U32_TO_BE32(llbolt,
2407 2432 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
2408 2433 U32_TO_BE32(tcp->tcp_ts_recent,
2409 2434 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
|
↓ open down ↓ |
114 lines elided |
↑ open up ↑ |
2410 2435 }
2411 2436
2412 2437 /* Update the latest receive window size in TCP header. */
2413 2438 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
2414 2439 /* Track what we sent to the peer */
2415 2440 tcp->tcp_tcpha->tha_win = tcpha->tha_win;
2416 2441 tcp->tcp_rack = ack;
2417 2442 tcp->tcp_rack_cnt = 0;
2418 2443 TCPS_BUMP_MIB(tcps, tcpOutAck);
2419 2444 }
2420 - BUMP_LOCAL(tcp->tcp_obsegs);
2445 + TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2421 2446 tcpha->tha_seq = htonl(seq);
2422 2447 tcpha->tha_ack = htonl(ack);
2423 2448 /*
2424 2449 * Include the adjustment for a source route if any.
2425 2450 */
2426 2451 sum = (sum >> 16) + (sum & 0xFFFF);
2427 2452 tcpha->tha_sum = htons(sum);
2428 2453 tcp_send_data(tcp, mp);
2429 2454 }
2430 2455
2431 2456 /*
2432 2457 * Generate a reset based on an inbound packet, connp is set by caller
2433 2458 * when RST is in response to an unexpected inbound packet for which
2434 2459 * there is active tcp state in the system.
2435 2460 *
2436 2461 * IPSEC NOTE : Try to send the reply with the same protection as it came
2437 2462 * in. We have the ip_recv_attr_t which is reversed to form the ip_xmit_attr_t.
2438 2463 * That way the packet will go out at the same level of protection as it
2439 2464 * came in with.
2440 2465 */
2441 2466 static void
2442 2467 tcp_xmit_early_reset(char *str, mblk_t *mp, uint32_t seq, uint32_t ack, int ctl,
2443 2468 ip_recv_attr_t *ira, ip_stack_t *ipst, conn_t *connp)
2444 2469 {
2445 2470 ipha_t *ipha = NULL;
2446 2471 ip6_t *ip6h = NULL;
2447 2472 ushort_t len;
2448 2473 tcpha_t *tcpha;
2449 2474 int i;
2450 2475 ipaddr_t v4addr;
2451 2476 in6_addr_t v6addr;
2452 2477 netstack_t *ns = ipst->ips_netstack;
2453 2478 tcp_stack_t *tcps = ns->netstack_tcp;
2454 2479 ip_xmit_attr_t ixas, *ixa;
2455 2480 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2456 2481 boolean_t need_refrele = B_FALSE; /* ixa_refrele(ixa) */
2457 2482 ushort_t port;
2458 2483
2459 2484 if (!tcp_send_rst_chk(tcps)) {
2460 2485 TCP_STAT(tcps, tcp_rst_unsent);
2461 2486 freemsg(mp);
2462 2487 return;
2463 2488 }
2464 2489
2465 2490 /*
2466 2491 * If connp != NULL we use conn_ixa to keep IP_NEXTHOP and other
2467 2492 * options from the listener. In that case the caller must ensure that
2468 2493 * we are running on the listener = connp squeue.
2469 2494 *
2470 2495 * We get a safe copy of conn_ixa so we don't need to restore anything
2471 2496 * we or ip_output_simple might change in the ixa.
2472 2497 */
2473 2498 if (connp != NULL) {
2474 2499 ASSERT(connp->conn_on_sqp);
2475 2500
2476 2501 ixa = conn_get_ixa_exclusive(connp);
2477 2502 if (ixa == NULL) {
2478 2503 TCP_STAT(tcps, tcp_rst_unsent);
2479 2504 freemsg(mp);
2480 2505 return;
2481 2506 }
2482 2507 need_refrele = B_TRUE;
2483 2508 } else {
2484 2509 bzero(&ixas, sizeof (ixas));
2485 2510 ixa = &ixas;
2486 2511 /*
2487 2512 * IXAF_VERIFY_SOURCE is overkill since we know the
2488 2513 * packet was for us.
2489 2514 */
2490 2515 ixa->ixa_flags |= IXAF_SET_ULP_CKSUM | IXAF_VERIFY_SOURCE;
2491 2516 ixa->ixa_protocol = IPPROTO_TCP;
2492 2517 ixa->ixa_zoneid = ira->ira_zoneid;
2493 2518 ixa->ixa_ifindex = 0;
2494 2519 ixa->ixa_ipst = ipst;
2495 2520 ixa->ixa_cred = kcred;
2496 2521 ixa->ixa_cpid = NOPID;
2497 2522 }
2498 2523
2499 2524 if (str && tcps->tcps_dbg) {
2500 2525 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2501 2526 "tcp_xmit_early_reset: '%s', seq 0x%x, ack 0x%x, "
2502 2527 "flags 0x%x",
2503 2528 str, seq, ack, ctl);
2504 2529 }
2505 2530 if (mp->b_datap->db_ref != 1) {
2506 2531 mblk_t *mp1 = copyb(mp);
2507 2532 freemsg(mp);
2508 2533 mp = mp1;
2509 2534 if (mp == NULL)
2510 2535 goto done;
2511 2536 } else if (mp->b_cont) {
2512 2537 freemsg(mp->b_cont);
2513 2538 mp->b_cont = NULL;
2514 2539 DB_CKSUMFLAGS(mp) = 0;
2515 2540 }
2516 2541 /*
2517 2542 * We skip reversing source route here.
2518 2543 * (for now we replace all IP options with EOL)
2519 2544 */
2520 2545 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2521 2546 ipha = (ipha_t *)mp->b_rptr;
2522 2547 for (i = IP_SIMPLE_HDR_LENGTH; i < (int)ip_hdr_len; i++)
2523 2548 mp->b_rptr[i] = IPOPT_EOL;
2524 2549 /*
2525 2550 * Make sure that src address isn't flagrantly invalid.
2526 2551 * Not all broadcast address checking for the src address
2527 2552 * is possible, since we don't know the netmask of the src
2528 2553 * addr. No check for destination address is done, since
2529 2554 * IP will not pass up a packet with a broadcast dest
2530 2555 * address to TCP. Similar checks are done below for IPv6.
2531 2556 */
2532 2557 if (ipha->ipha_src == 0 || ipha->ipha_src == INADDR_BROADCAST ||
2533 2558 CLASSD(ipha->ipha_src)) {
2534 2559 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsInDiscards);
2535 2560 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2536 2561 freemsg(mp);
2537 2562 goto done;
2538 2563 }
2539 2564 } else {
2540 2565 ip6h = (ip6_t *)mp->b_rptr;
2541 2566
2542 2567 if (IN6_IS_ADDR_UNSPECIFIED(&ip6h->ip6_src) ||
2543 2568 IN6_IS_ADDR_MULTICAST(&ip6h->ip6_src)) {
2544 2569 BUMP_MIB(&ipst->ips_ip6_mib, ipIfStatsInDiscards);
2545 2570 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2546 2571 freemsg(mp);
2547 2572 goto done;
2548 2573 }
2549 2574
2550 2575 /* Remove any extension headers assuming partial overlay */
2551 2576 if (ip_hdr_len > IPV6_HDR_LEN) {
2552 2577 uint8_t *to;
2553 2578
2554 2579 to = mp->b_rptr + ip_hdr_len - IPV6_HDR_LEN;
2555 2580 ovbcopy(ip6h, to, IPV6_HDR_LEN);
2556 2581 mp->b_rptr += ip_hdr_len - IPV6_HDR_LEN;
2557 2582 ip_hdr_len = IPV6_HDR_LEN;
2558 2583 ip6h = (ip6_t *)mp->b_rptr;
2559 2584 ip6h->ip6_nxt = IPPROTO_TCP;
2560 2585 }
2561 2586 }
2562 2587 tcpha = (tcpha_t *)&mp->b_rptr[ip_hdr_len];
2563 2588 if (tcpha->tha_flags & TH_RST) {
2564 2589 freemsg(mp);
2565 2590 goto done;
2566 2591 }
2567 2592 tcpha->tha_offset_and_reserved = (5 << 4);
2568 2593 len = ip_hdr_len + sizeof (tcpha_t);
2569 2594 mp->b_wptr = &mp->b_rptr[len];
2570 2595 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2571 2596 ipha->ipha_length = htons(len);
2572 2597 /* Swap addresses */
2573 2598 v4addr = ipha->ipha_src;
2574 2599 ipha->ipha_src = ipha->ipha_dst;
2575 2600 ipha->ipha_dst = v4addr;
2576 2601 ipha->ipha_ident = 0;
2577 2602 ipha->ipha_ttl = (uchar_t)tcps->tcps_ipv4_ttl;
2578 2603 ixa->ixa_flags |= IXAF_IS_IPV4;
2579 2604 ixa->ixa_ip_hdr_length = ip_hdr_len;
2580 2605 } else {
2581 2606 ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2582 2607 /* Swap addresses */
2583 2608 v6addr = ip6h->ip6_src;
2584 2609 ip6h->ip6_src = ip6h->ip6_dst;
2585 2610 ip6h->ip6_dst = v6addr;
2586 2611 ip6h->ip6_hops = (uchar_t)tcps->tcps_ipv6_hoplimit;
2587 2612 ixa->ixa_flags &= ~IXAF_IS_IPV4;
2588 2613
2589 2614 if (IN6_IS_ADDR_LINKSCOPE(&ip6h->ip6_dst)) {
2590 2615 ixa->ixa_flags |= IXAF_SCOPEID_SET;
2591 2616 ixa->ixa_scopeid = ira->ira_ruifindex;
2592 2617 }
2593 2618 ixa->ixa_ip_hdr_length = IPV6_HDR_LEN;
2594 2619 }
2595 2620 ixa->ixa_pktlen = len;
2596 2621
2597 2622 /* Swap the ports */
2598 2623 port = tcpha->tha_fport;
2599 2624 tcpha->tha_fport = tcpha->tha_lport;
2600 2625 tcpha->tha_lport = port;
2601 2626
2602 2627 tcpha->tha_ack = htonl(ack);
2603 2628 tcpha->tha_seq = htonl(seq);
2604 2629 tcpha->tha_win = 0;
2605 2630 tcpha->tha_sum = htons(sizeof (tcpha_t));
2606 2631 tcpha->tha_flags = (uint8_t)ctl;
2607 2632 if (ctl & TH_RST) {
2608 2633 if (ctl & TH_ACK) {
2609 2634 /*
2610 2635 * Probe connection rejection here.
2611 2636 * tcp_xmit_listeners_reset() drops non-SYN segments
2612 2637 * that do not specify TH_ACK in their flags without
2613 2638 * calling this function. As a consequence, if this
2614 2639 * function is called with a TH_RST|TH_ACK ctl argument,
2615 2640 * it is being called in response to a SYN segment
2616 2641 * and thus the tcp:::accept-refused probe point
2617 2642 * is valid here.
2618 2643 */
2619 2644 DTRACE_TCP5(accept__refused, mblk_t *, NULL,
2620 2645 void, NULL, void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2621 2646 tcph_t *, tcpha);
2622 2647 }
2623 2648 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2624 2649 TCPS_BUMP_MIB(tcps, tcpOutControl);
2625 2650 }
2626 2651
2627 2652 /* Discard any old label */
2628 2653 if (ixa->ixa_free_flags & IXA_FREE_TSL) {
2629 2654 ASSERT(ixa->ixa_tsl != NULL);
2630 2655 label_rele(ixa->ixa_tsl);
2631 2656 ixa->ixa_free_flags &= ~IXA_FREE_TSL;
2632 2657 }
2633 2658 ixa->ixa_tsl = ira->ira_tsl; /* Behave as a multi-level responder */
2634 2659
2635 2660 if (ira->ira_flags & IRAF_IPSEC_SECURE) {
2636 2661 /*
2637 2662 * Apply IPsec based on how IPsec was applied to
2638 2663 * the packet that caused the RST.
2639 2664 */
2640 2665 if (!ipsec_in_to_out(ira, ixa, mp, ipha, ip6h)) {
2641 2666 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsOutDiscards);
2642 2667 /* Note: mp already consumed and ip_drop_packet done */
2643 2668 goto done;
2644 2669 }
2645 2670 } else {
2646 2671 /*
2647 2672 * This is in clear. The RST message we are building
2648 2673 * here should go out in clear, independent of our policy.
2649 2674 */
2650 2675 ixa->ixa_flags |= IXAF_NO_IPSEC;
2651 2676 }
2652 2677
2653 2678 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, ixa,
2654 2679 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2655 2680 __dtrace_tcp_tcph_t *, tcpha);
2656 2681
2657 2682 /*
2658 2683 * NOTE: one might consider tracing a TCP packet here, but
2659 2684 * this function has no active TCP state and no tcp structure
2660 2685 * that has a trace buffer. If we traced here, we would have
2661 2686 * to keep a local trace buffer in tcp_record_trace().
2662 2687 */
2663 2688
2664 2689 (void) ip_output_simple(mp, ixa);
2665 2690 done:
2666 2691 ixa_cleanup(ixa);
2667 2692 if (need_refrele) {
2668 2693 ASSERT(ixa != &ixas);
2669 2694 ixa_refrele(ixa);
2670 2695 }
2671 2696 }
2672 2697
2673 2698 /*
2674 2699 * Generate a "no listener here" RST in response to an "unknown" segment.
2675 2700 * connp is set by caller when RST is in response to an unexpected
2676 2701 * inbound packet for which there is active tcp state in the system.
2677 2702 * Note that we are reusing the incoming mp to construct the outgoing RST.
2678 2703 */
2679 2704 void
2680 2705 tcp_xmit_listeners_reset(mblk_t *mp, ip_recv_attr_t *ira, ip_stack_t *ipst,
2681 2706 conn_t *connp)
2682 2707 {
2683 2708 uchar_t *rptr;
2684 2709 uint32_t seg_len;
2685 2710 tcpha_t *tcpha;
2686 2711 uint32_t seg_seq;
2687 2712 uint32_t seg_ack;
2688 2713 uint_t flags;
2689 2714 ipha_t *ipha;
2690 2715 ip6_t *ip6h;
2691 2716 boolean_t policy_present;
2692 2717 netstack_t *ns = ipst->ips_netstack;
2693 2718 tcp_stack_t *tcps = ns->netstack_tcp;
2694 2719 ipsec_stack_t *ipss = tcps->tcps_netstack->netstack_ipsec;
2695 2720 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2696 2721
2697 2722 TCP_STAT(tcps, tcp_no_listener);
2698 2723
2699 2724 /*
2700 2725 * DTrace this "unknown" segment as a tcp:::receive, as we did
2701 2726 * just receive something that was TCP.
2702 2727 */
2703 2728 DTRACE_TCP5(receive, mblk_t *, NULL, ip_xmit_attr_t *, NULL,
2704 2729 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2705 2730 __dtrace_tcp_tcph_t *, &mp->b_rptr[ip_hdr_len]);
2706 2731
2707 2732 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2708 2733 policy_present = ipss->ipsec_inbound_v4_policy_present;
2709 2734 ipha = (ipha_t *)mp->b_rptr;
2710 2735 ip6h = NULL;
2711 2736 } else {
2712 2737 policy_present = ipss->ipsec_inbound_v6_policy_present;
2713 2738 ipha = NULL;
2714 2739 ip6h = (ip6_t *)mp->b_rptr;
2715 2740 }
2716 2741
2717 2742 if (policy_present) {
2718 2743 /*
2719 2744 * The conn_t parameter is NULL because we already know
2720 2745 * nobody's home.
2721 2746 */
2722 2747 mp = ipsec_check_global_policy(mp, (conn_t *)NULL, ipha, ip6h,
2723 2748 ira, ns);
2724 2749 if (mp == NULL)
2725 2750 return;
2726 2751 }
2727 2752 if (is_system_labeled() && !tsol_can_reply_error(mp, ira)) {
2728 2753 DTRACE_PROBE2(
2729 2754 tx__ip__log__error__nolistener__tcp,
2730 2755 char *, "Could not reply with RST to mp(1)",
2731 2756 mblk_t *, mp);
2732 2757 ip2dbg(("tcp_xmit_listeners_reset: not permitted to reply\n"));
2733 2758 freemsg(mp);
2734 2759 return;
2735 2760 }
2736 2761
2737 2762 rptr = mp->b_rptr;
2738 2763
2739 2764 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2740 2765 seg_seq = ntohl(tcpha->tha_seq);
2741 2766 seg_ack = ntohl(tcpha->tha_ack);
2742 2767 flags = tcpha->tha_flags;
2743 2768
2744 2769 seg_len = msgdsize(mp) - (TCP_HDR_LENGTH(tcpha) + ip_hdr_len);
2745 2770 if (flags & TH_RST) {
2746 2771 freemsg(mp);
2747 2772 } else if (flags & TH_ACK) {
2748 2773 tcp_xmit_early_reset("no tcp, reset", mp, seg_ack, 0, TH_RST,
2749 2774 ira, ipst, connp);
2750 2775 } else {
2751 2776 if (flags & TH_SYN) {
2752 2777 seg_len++;
2753 2778 } else {
2754 2779 /*
2755 2780 * Here we violate the RFC. Note that a normal
2756 2781 * TCP will never send a segment without the ACK
2757 2782 * flag, except for RST or SYN segment. This
2758 2783 * segment is neither. Just drop it on the
2759 2784 * floor.
2760 2785 */
2761 2786 freemsg(mp);
2762 2787 TCP_STAT(tcps, tcp_rst_unsent);
2763 2788 return;
2764 2789 }
2765 2790
2766 2791 tcp_xmit_early_reset("no tcp, reset/ack", mp, 0,
2767 2792 seg_seq + seg_len, TH_RST | TH_ACK, ira, ipst, connp);
2768 2793 }
2769 2794 }
2770 2795
2771 2796 /*
2772 2797 * Helper function for tcp_xmit_mp() in handling connection set up flag
2773 2798 * options setting.
2774 2799 */
2775 2800 static void
2776 2801 tcp_xmit_mp_aux_iss(tcp_t *tcp, conn_t *connp, tcpha_t *tcpha, mblk_t *mp,
2777 2802 uint_t *flags)
2778 2803 {
2779 2804 uint32_t u1;
2780 2805 uint8_t *wptr = mp->b_wptr;
2781 2806 tcp_stack_t *tcps = tcp->tcp_tcps;
2782 2807 boolean_t add_sack = B_FALSE;
2783 2808
2784 2809 /*
2785 2810 * If TCP_ISS_VALID and the seq number is tcp_iss,
2786 2811 * TCP can only be in SYN-SENT, SYN-RCVD or
2787 2812 * FIN-WAIT-1 state. It can be FIN-WAIT-1 if
2788 2813 * our SYN is not ack'ed but the app closes this
2789 2814 * TCP connection.
2790 2815 */
2791 2816 ASSERT(tcp->tcp_state == TCPS_SYN_SENT ||
2792 2817 tcp->tcp_state == TCPS_SYN_RCVD ||
2793 2818 tcp->tcp_state == TCPS_FIN_WAIT_1);
2794 2819
2795 2820 /*
2796 2821 * Tack on the MSS option. It is always needed
2797 2822 * for both active and passive open.
2798 2823 *
2799 2824 * MSS option value should be interface MTU - MIN
2800 2825 * TCP/IP header according to RFC 793 as it means
2801 2826 * the maximum segment size TCP can receive. But
2802 2827 * to get around some broken middle boxes/end hosts
2803 2828 * out there, we allow the option value to be the
2804 2829 * same as the MSS option size on the peer side.
2805 2830 * In this way, the other side will not send
2806 2831 * anything larger than they can receive.
2807 2832 *
2808 2833 * Note that for SYN_SENT state, the ndd param
2809 2834 * tcp_use_smss_as_mss_opt has no effect as we
2810 2835 * don't know the peer's MSS option value. So
2811 2836 * the only case we need to take care of is in
2812 2837 * SYN_RCVD state, which is done later.
2813 2838 */
2814 2839 wptr[0] = TCPOPT_MAXSEG;
2815 2840 wptr[1] = TCPOPT_MAXSEG_LEN;
2816 2841 wptr += 2;
2817 2842 u1 = tcp->tcp_initial_pmtu - (connp->conn_ipversion == IPV4_VERSION ?
2818 2843 IP_SIMPLE_HDR_LENGTH : IPV6_HDR_LEN) - TCP_MIN_HEADER_LENGTH;
2819 2844 U16_TO_BE16(u1, wptr);
2820 2845 wptr += 2;
2821 2846
2822 2847 /* Update the offset to cover the additional word */
2823 2848 tcpha->tha_offset_and_reserved += (1 << 4);
2824 2849
2825 2850 switch (tcp->tcp_state) {
2826 2851 case TCPS_SYN_SENT:
2827 2852 *flags = TH_SYN;
2828 2853
2829 2854 if (tcp->tcp_snd_sack_ok)
2830 2855 add_sack = B_TRUE;
2831 2856
2832 2857 if (tcp->tcp_snd_ts_ok) {
2833 2858 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2834 2859
2835 2860 if (add_sack) {
2836 2861 wptr[0] = TCPOPT_SACK_PERMITTED;
2837 2862 wptr[1] = TCPOPT_SACK_OK_LEN;
2838 2863 add_sack = B_FALSE;
2839 2864 } else {
2840 2865 wptr[0] = TCPOPT_NOP;
2841 2866 wptr[1] = TCPOPT_NOP;
2842 2867 }
2843 2868 wptr[2] = TCPOPT_TSTAMP;
2844 2869 wptr[3] = TCPOPT_TSTAMP_LEN;
2845 2870 wptr += 4;
2846 2871 U32_TO_BE32(llbolt, wptr);
2847 2872 wptr += 4;
2848 2873 ASSERT(tcp->tcp_ts_recent == 0);
2849 2874 U32_TO_BE32(0L, wptr);
2850 2875 wptr += 4;
2851 2876 tcpha->tha_offset_and_reserved += (3 << 4);
2852 2877 }
2853 2878
2854 2879 /*
2855 2880 * Set up all the bits to tell other side
2856 2881 * we are ECN capable.
2857 2882 */
2858 2883 if (tcp->tcp_ecn_ok)
2859 2884 *flags |= (TH_ECE | TH_CWR);
2860 2885
2861 2886 break;
2862 2887
2863 2888 case TCPS_SYN_RCVD:
2864 2889 *flags |= TH_SYN;
2865 2890
2866 2891 /*
2867 2892 * Reset the MSS option value to be SMSS
2868 2893 * We should probably add back the bytes
2869 2894 * for timestamp option and IPsec. We
2870 2895 * don't do that as this is a workaround
2871 2896 * for broken middle boxes/end hosts, it
2872 2897 * is better for us to be more cautious.
2873 2898 * They may not take these things into
2874 2899 * account in their SMSS calculation. Thus
2875 2900 * the peer's calculated SMSS may be smaller
2876 2901 * than what it can be. This should be OK.
2877 2902 */
2878 2903 if (tcps->tcps_use_smss_as_mss_opt) {
2879 2904 u1 = tcp->tcp_mss;
2880 2905 /*
2881 2906 * Note that wptr points just past the MSS
2882 2907 * option value.
2883 2908 */
2884 2909 U16_TO_BE16(u1, wptr - 2);
2885 2910 }
2886 2911
2887 2912 /*
2888 2913 * tcp_snd_ts_ok can only be set in TCPS_SYN_RCVD
2889 2914 * when the peer also uses timestamps option. And
2890 2915 * the TCP header template must have already been
2891 2916 * updated to include the timestamps option.
2892 2917 */
2893 2918 if (tcp->tcp_snd_sack_ok) {
2894 2919 if (tcp->tcp_snd_ts_ok) {
2895 2920 uint8_t *tmp_wptr;
2896 2921
2897 2922 /*
2898 2923 * Use the NOP in the header just
2899 2924 * before timestamps opton.
2900 2925 */
2901 2926 tmp_wptr = (uint8_t *)tcpha +
2902 2927 TCP_MIN_HEADER_LENGTH;
2903 2928 ASSERT(tmp_wptr[0] == TCPOPT_NOP &&
2904 2929 tmp_wptr[1] == TCPOPT_NOP);
2905 2930 tmp_wptr[0] = TCPOPT_SACK_PERMITTED;
2906 2931 tmp_wptr[1] = TCPOPT_SACK_OK_LEN;
2907 2932 } else {
2908 2933 add_sack = B_TRUE;
2909 2934 }
2910 2935 }
2911 2936
2912 2937
2913 2938 /*
2914 2939 * If the other side is ECN capable, reply
2915 2940 * that we are also ECN capable.
2916 2941 */
2917 2942 if (tcp->tcp_ecn_ok)
2918 2943 *flags |= TH_ECE;
2919 2944 break;
2920 2945
2921 2946 default:
2922 2947 /*
2923 2948 * The above ASSERT() makes sure that this
2924 2949 * must be FIN-WAIT-1 state. Our SYN has
2925 2950 * not been ack'ed so retransmit it.
2926 2951 */
2927 2952 *flags |= TH_SYN;
2928 2953 break;
2929 2954 }
2930 2955
2931 2956 if (add_sack) {
2932 2957 wptr[0] = TCPOPT_NOP;
2933 2958 wptr[1] = TCPOPT_NOP;
2934 2959 wptr[2] = TCPOPT_SACK_PERMITTED;
2935 2960 wptr[3] = TCPOPT_SACK_OK_LEN;
2936 2961 wptr += TCPOPT_REAL_SACK_OK_LEN;
2937 2962 tcpha->tha_offset_and_reserved += (1 << 4);
2938 2963 }
2939 2964
2940 2965 if (tcp->tcp_snd_ws_ok) {
2941 2966 wptr[0] = TCPOPT_NOP;
2942 2967 wptr[1] = TCPOPT_WSCALE;
2943 2968 wptr[2] = TCPOPT_WS_LEN;
2944 2969 wptr[3] = (uchar_t)tcp->tcp_rcv_ws;
2945 2970 wptr += TCPOPT_REAL_WS_LEN;
2946 2971 tcpha->tha_offset_and_reserved += (1 << 4);
2947 2972 }
2948 2973
2949 2974 mp->b_wptr = wptr;
2950 2975 u1 = (int)(mp->b_wptr - mp->b_rptr);
2951 2976 /*
2952 2977 * Get IP set to checksum on our behalf
2953 2978 * Include the adjustment for a source route if any.
2954 2979 */
2955 2980 u1 += connp->conn_sum;
2956 2981 u1 = (u1 >> 16) + (u1 & 0xFFFF);
2957 2982 tcpha->tha_sum = htons(u1);
2958 2983 TCPS_BUMP_MIB(tcps, tcpOutControl);
2959 2984 }
2960 2985
2961 2986 /*
2962 2987 * Helper function for tcp_xmit_mp() in handling connection tear down
2963 2988 * flag setting and state changes.
2964 2989 */
2965 2990 static void
2966 2991 tcp_xmit_mp_aux_fss(tcp_t *tcp, ip_xmit_attr_t *ixa, uint_t *flags)
2967 2992 {
2968 2993 if (!tcp->tcp_fin_acked) {
2969 2994 *flags |= TH_FIN;
2970 2995 TCPS_BUMP_MIB(tcp->tcp_tcps, tcpOutControl);
2971 2996 }
2972 2997 if (!tcp->tcp_fin_sent) {
2973 2998 tcp->tcp_fin_sent = B_TRUE;
2974 2999 switch (tcp->tcp_state) {
2975 3000 case TCPS_SYN_RCVD:
2976 3001 tcp->tcp_state = TCPS_FIN_WAIT_1;
2977 3002 DTRACE_TCP6(state__change, void, NULL,
2978 3003 ip_xmit_attr_t *, ixa, void, NULL,
2979 3004 tcp_t *, tcp, void, NULL,
2980 3005 int32_t, TCPS_SYN_RCVD);
2981 3006 break;
2982 3007 case TCPS_ESTABLISHED:
2983 3008 tcp->tcp_state = TCPS_FIN_WAIT_1;
2984 3009 DTRACE_TCP6(state__change, void, NULL,
2985 3010 ip_xmit_attr_t *, ixa, void, NULL,
2986 3011 tcp_t *, tcp, void, NULL,
2987 3012 int32_t, TCPS_ESTABLISHED);
2988 3013 break;
2989 3014 case TCPS_CLOSE_WAIT:
2990 3015 tcp->tcp_state = TCPS_LAST_ACK;
2991 3016 DTRACE_TCP6(state__change, void, NULL,
2992 3017 ip_xmit_attr_t *, ixa, void, NULL,
2993 3018 tcp_t *, tcp, void, NULL,
2994 3019 int32_t, TCPS_CLOSE_WAIT);
2995 3020 break;
2996 3021 }
2997 3022 if (tcp->tcp_suna == tcp->tcp_snxt)
2998 3023 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2999 3024 tcp->tcp_snxt = tcp->tcp_fss + 1;
3000 3025 }
3001 3026 }
3002 3027
3003 3028 /*
3004 3029 * tcp_xmit_mp is called to return a pointer to an mblk chain complete with
3005 3030 * ip and tcp header ready to pass down to IP. If the mp passed in is
3006 3031 * non-NULL, then up to max_to_send bytes of data will be dup'ed off that
3007 3032 * mblk. (If sendall is not set the dup'ing will stop at an mblk boundary
3008 3033 * otherwise it will dup partial mblks.)
3009 3034 * Otherwise, an appropriate ACK packet will be generated. This
3010 3035 * routine is not usually called to send new data for the first time. It
3011 3036 * is mostly called out of the timer for retransmits, and to generate ACKs.
3012 3037 *
3013 3038 * If offset is not NULL, the returned mblk chain's first mblk's b_rptr will
3014 3039 * be adjusted by *offset. And after dupb(), the offset and the ending mblk
3015 3040 * of the original mblk chain will be returned in *offset and *end_mp.
3016 3041 */
3017 3042 mblk_t *
3018 3043 tcp_xmit_mp(tcp_t *tcp, mblk_t *mp, int32_t max_to_send, int32_t *offset,
3019 3044 mblk_t **end_mp, uint32_t seq, boolean_t sendall, uint32_t *seg_len,
3020 3045 boolean_t rexmit)
3021 3046 {
3022 3047 int data_length;
3023 3048 int32_t off = 0;
3024 3049 uint_t flags;
3025 3050 mblk_t *mp1;
3026 3051 mblk_t *mp2;
3027 3052 uchar_t *rptr;
3028 3053 tcpha_t *tcpha;
3029 3054 int32_t num_sack_blk = 0;
3030 3055 int32_t sack_opt_len = 0;
3031 3056 tcp_stack_t *tcps = tcp->tcp_tcps;
3032 3057 conn_t *connp = tcp->tcp_connp;
3033 3058 ip_xmit_attr_t *ixa = connp->conn_ixa;
3034 3059
3035 3060 /* Allocate for our maximum TCP header + link-level */
3036 3061 mp1 = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
3037 3062 BPRI_MED);
3038 3063 if (mp1 == NULL)
3039 3064 return (NULL);
3040 3065 data_length = 0;
3041 3066
3042 3067 /*
3043 3068 * Note that tcp_mss has been adjusted to take into account the
3044 3069 * timestamp option if applicable. Because SACK options do not
3045 3070 * appear in every TCP segments and they are of variable lengths,
3046 3071 * they cannot be included in tcp_mss. Thus we need to calculate
3047 3072 * the actual segment length when we need to send a segment which
3048 3073 * includes SACK options.
3049 3074 */
3050 3075 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
3051 3076 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
3052 3077 tcp->tcp_num_sack_blk);
3053 3078 sack_opt_len = num_sack_blk * sizeof (sack_blk_t) +
3054 3079 TCPOPT_NOP_LEN * 2 + TCPOPT_HEADER_LEN;
3055 3080 if (max_to_send + sack_opt_len > tcp->tcp_mss)
3056 3081 max_to_send -= sack_opt_len;
3057 3082 }
3058 3083
3059 3084 if (offset != NULL) {
3060 3085 off = *offset;
3061 3086 /* We use offset as an indicator that end_mp is not NULL. */
3062 3087 *end_mp = NULL;
3063 3088 }
3064 3089 for (mp2 = mp1; mp && data_length != max_to_send; mp = mp->b_cont) {
3065 3090 /* This could be faster with cooperation from downstream */
3066 3091 if (mp2 != mp1 && !sendall &&
3067 3092 data_length + (int)(mp->b_wptr - mp->b_rptr) >
3068 3093 max_to_send)
3069 3094 /*
3070 3095 * Don't send the next mblk since the whole mblk
3071 3096 * does not fit.
3072 3097 */
3073 3098 break;
3074 3099 mp2->b_cont = dupb(mp);
3075 3100 mp2 = mp2->b_cont;
3076 3101 if (!mp2) {
3077 3102 freemsg(mp1);
3078 3103 return (NULL);
3079 3104 }
3080 3105 mp2->b_rptr += off;
3081 3106 ASSERT((uintptr_t)(mp2->b_wptr - mp2->b_rptr) <=
3082 3107 (uintptr_t)INT_MAX);
3083 3108
3084 3109 data_length += (int)(mp2->b_wptr - mp2->b_rptr);
3085 3110 if (data_length > max_to_send) {
3086 3111 mp2->b_wptr -= data_length - max_to_send;
3087 3112 data_length = max_to_send;
3088 3113 off = mp2->b_wptr - mp->b_rptr;
3089 3114 break;
3090 3115 } else {
3091 3116 off = 0;
3092 3117 }
3093 3118 }
3094 3119 if (offset != NULL) {
3095 3120 *offset = off;
3096 3121 *end_mp = mp;
3097 3122 }
3098 3123 if (seg_len != NULL) {
3099 3124 *seg_len = data_length;
3100 3125 }
3101 3126
3102 3127 /* Update the latest receive window size in TCP header. */
3103 3128 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
3104 3129
3105 3130 rptr = mp1->b_rptr + tcps->tcps_wroff_xtra;
3106 3131 mp1->b_rptr = rptr;
3107 3132 mp1->b_wptr = rptr + connp->conn_ht_iphc_len + sack_opt_len;
3108 3133 bcopy(connp->conn_ht_iphc, rptr, connp->conn_ht_iphc_len);
3109 3134 tcpha = (tcpha_t *)&rptr[ixa->ixa_ip_hdr_length];
3110 3135 tcpha->tha_seq = htonl(seq);
3111 3136
3112 3137 /*
3113 3138 * Use tcp_unsent to determine if the PUSH bit should be used assumes
3114 3139 * that this function was called from tcp_wput_data. Thus, when called
3115 3140 * to retransmit data the setting of the PUSH bit may appear some
3116 3141 * what random in that it might get set when it should not. This
3117 3142 * should not pose any performance issues.
3118 3143 */
3119 3144 if (data_length != 0 && (tcp->tcp_unsent == 0 ||
3120 3145 tcp->tcp_unsent == data_length)) {
3121 3146 flags = TH_ACK | TH_PUSH;
3122 3147 } else {
3123 3148 flags = TH_ACK;
3124 3149 }
3125 3150
3126 3151 if (tcp->tcp_ecn_ok) {
3127 3152 if (tcp->tcp_ecn_echo_on)
3128 3153 flags |= TH_ECE;
3129 3154
3130 3155 /*
3131 3156 * Only set ECT bit and ECN_CWR if a segment contains new data.
3132 3157 * There is no TCP flow control for non-data segments, and
3133 3158 * only data segment is transmitted reliably.
3134 3159 */
3135 3160 if (data_length > 0 && !rexmit) {
3136 3161 TCP_SET_ECT(tcp, rptr);
3137 3162 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3138 3163 flags |= TH_CWR;
3139 3164 tcp->tcp_ecn_cwr_sent = B_TRUE;
3140 3165 }
3141 3166 }
3142 3167 }
3143 3168
3144 3169 /* Check if there is any special processing needs to be done. */
3145 3170 if (tcp->tcp_valid_bits) {
3146 3171 uint32_t u1;
3147 3172
3148 3173 /* We don't allow having SYN and FIN in the same segment... */
3149 3174 if ((tcp->tcp_valid_bits & TCP_ISS_VALID) &&
3150 3175 seq == tcp->tcp_iss) {
3151 3176 /* Need to do connection set up processing. */
3152 3177 tcp_xmit_mp_aux_iss(tcp, connp, tcpha, mp1, &flags);
3153 3178 } else if ((tcp->tcp_valid_bits & TCP_FSS_VALID) &&
3154 3179 (seq + data_length) == tcp->tcp_fss) {
3155 3180 /* Need to do connection tear down processing. */
3156 3181 tcp_xmit_mp_aux_fss(tcp, ixa, &flags);
3157 3182 }
3158 3183
3159 3184 /*
3160 3185 * Need to do urgent pointer processing.
3161 3186 *
3162 3187 * Note the trick here. u1 is unsigned. When tcp_urg
3163 3188 * is smaller than seq, u1 will become a very huge value.
3164 3189 * So the comparison will fail. Also note that tcp_urp
3165 3190 * should be positive, see RFC 793 page 17.
3166 3191 */
3167 3192 u1 = tcp->tcp_urg - seq + TCP_OLD_URP_INTERPRETATION;
3168 3193 if ((tcp->tcp_valid_bits & TCP_URG_VALID) && u1 != 0 &&
3169 3194 u1 < (uint32_t)(64 * 1024)) {
3170 3195 flags |= TH_URG;
3171 3196 TCPS_BUMP_MIB(tcps, tcpOutUrg);
3172 3197 tcpha->tha_urp = htons(u1);
3173 3198 }
3174 3199 }
3175 3200 tcpha->tha_flags = (uchar_t)flags;
3176 3201 tcp->tcp_rack = tcp->tcp_rnxt;
3177 3202 tcp->tcp_rack_cnt = 0;
3178 3203
3179 3204 /* Fill in the current value of timestamps option. */
3180 3205 if (tcp->tcp_snd_ts_ok) {
3181 3206 if (tcp->tcp_state != TCPS_SYN_SENT) {
3182 3207 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
3183 3208
3184 3209 U32_TO_BE32(llbolt,
3185 3210 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
3186 3211 U32_TO_BE32(tcp->tcp_ts_recent,
3187 3212 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
3188 3213 }
3189 3214 }
3190 3215
3191 3216 /* Fill in the SACK blocks. */
3192 3217 if (num_sack_blk > 0) {
3193 3218 uchar_t *wptr = (uchar_t *)tcpha + connp->conn_ht_ulp_len;
3194 3219 sack_blk_t *tmp;
3195 3220 int32_t i;
3196 3221
3197 3222 wptr[0] = TCPOPT_NOP;
3198 3223 wptr[1] = TCPOPT_NOP;
3199 3224 wptr[2] = TCPOPT_SACK;
3200 3225 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3201 3226 sizeof (sack_blk_t);
3202 3227 wptr += TCPOPT_REAL_SACK_LEN;
3203 3228
3204 3229 tmp = tcp->tcp_sack_list;
3205 3230 for (i = 0; i < num_sack_blk; i++) {
3206 3231 U32_TO_BE32(tmp[i].begin, wptr);
3207 3232 wptr += sizeof (tcp_seq);
3208 3233 U32_TO_BE32(tmp[i].end, wptr);
3209 3234 wptr += sizeof (tcp_seq);
3210 3235 }
3211 3236 tcpha->tha_offset_and_reserved += ((num_sack_blk * 2 + 1) << 4);
3212 3237 }
3213 3238 ASSERT((uintptr_t)(mp1->b_wptr - rptr) <= (uintptr_t)INT_MAX);
3214 3239 data_length += (int)(mp1->b_wptr - rptr);
3215 3240
3216 3241 ixa->ixa_pktlen = data_length;
3217 3242
3218 3243 if (ixa->ixa_flags & IXAF_IS_IPV4) {
3219 3244 ((ipha_t *)rptr)->ipha_length = htons(data_length);
3220 3245 } else {
3221 3246 ip6_t *ip6 = (ip6_t *)rptr;
3222 3247
3223 3248 ip6->ip6_plen = htons(data_length - IPV6_HDR_LEN);
3224 3249 }
3225 3250
3226 3251 /*
3227 3252 * Prime pump for IP
3228 3253 * Include the adjustment for a source route if any.
3229 3254 */
3230 3255 data_length -= ixa->ixa_ip_hdr_length;
3231 3256 data_length += connp->conn_sum;
3232 3257 data_length = (data_length >> 16) + (data_length & 0xFFFF);
3233 3258 tcpha->tha_sum = htons(data_length);
3234 3259 if (tcp->tcp_ip_forward_progress) {
3235 3260 tcp->tcp_ip_forward_progress = B_FALSE;
3236 3261 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
3237 3262 } else {
3238 3263 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
3239 3264 }
3240 3265 return (mp1);
3241 3266 }
3242 3267
3243 3268 /*
3244 3269 * If this routine returns B_TRUE, TCP can generate a RST in response
3245 3270 * to a segment. If it returns B_FALSE, TCP should not respond.
3246 3271 */
3247 3272 static boolean_t
3248 3273 tcp_send_rst_chk(tcp_stack_t *tcps)
3249 3274 {
3250 3275 int64_t now;
3251 3276
3252 3277 /*
3253 3278 * TCP needs to protect itself from generating too many RSTs.
3254 3279 * This can be a DoS attack by sending us random segments
3255 3280 * soliciting RSTs.
3256 3281 *
3257 3282 * What we do here is to have a limit of tcp_rst_sent_rate RSTs
3258 3283 * in each 1 second interval. In this way, TCP still generate
3259 3284 * RSTs in normal cases but when under attack, the impact is
3260 3285 * limited.
3261 3286 */
3262 3287 if (tcps->tcps_rst_sent_rate_enabled != 0) {
3263 3288 now = ddi_get_lbolt64();
3264 3289 if (TICK_TO_MSEC(now - tcps->tcps_last_rst_intrvl) >
3265 3290 1*SECONDS) {
3266 3291 tcps->tcps_last_rst_intrvl = now;
3267 3292 tcps->tcps_rst_cnt = 1;
3268 3293 } else if (++tcps->tcps_rst_cnt > tcps->tcps_rst_sent_rate) {
3269 3294 return (B_FALSE);
3270 3295 }
3271 3296 }
3272 3297 return (B_TRUE);
3273 3298 }
3274 3299
3275 3300 /*
3276 3301 * This function handles all retransmissions if SACK is enabled for this
3277 3302 * connection. First it calculates how many segments can be retransmitted
3278 3303 * based on tcp_pipe. Then it goes thru the notsack list to find eligible
3279 3304 * segments. A segment is eligible if sack_cnt for that segment is greater
3280 3305 * than or equal tcp_dupack_fast_retransmit. After it has retransmitted
3281 3306 * all eligible segments, it checks to see if TCP can send some new segments
3282 3307 * (fast recovery). If it can, set the appropriate flag for tcp_input_data().
3283 3308 *
3284 3309 * Parameters:
3285 3310 * tcp_t *tcp: the tcp structure of the connection.
3286 3311 * uint_t *flags: in return, appropriate value will be set for
3287 3312 * tcp_input_data().
3288 3313 */
3289 3314 void
3290 3315 tcp_sack_rexmit(tcp_t *tcp, uint_t *flags)
3291 3316 {
3292 3317 notsack_blk_t *notsack_blk;
3293 3318 int32_t usable_swnd;
3294 3319 int32_t mss;
3295 3320 uint32_t seg_len;
3296 3321 mblk_t *xmit_mp;
3297 3322 tcp_stack_t *tcps = tcp->tcp_tcps;
3298 3323
3299 3324 ASSERT(tcp->tcp_notsack_list != NULL);
3300 3325 ASSERT(tcp->tcp_rexmit == B_FALSE);
3301 3326
3302 3327 /* Defensive coding in case there is a bug... */
3303 3328 if (tcp->tcp_notsack_list == NULL) {
3304 3329 return;
3305 3330 }
3306 3331 notsack_blk = tcp->tcp_notsack_list;
3307 3332 mss = tcp->tcp_mss;
3308 3333
3309 3334 /*
3310 3335 * Limit the num of outstanding data in the network to be
3311 3336 * tcp_cwnd_ssthresh, which is half of the original congestion wnd.
3312 3337 */
3313 3338 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3314 3339
3315 3340 /* At least retransmit 1 MSS of data. */
3316 3341 if (usable_swnd <= 0) {
3317 3342 usable_swnd = mss;
3318 3343 }
3319 3344
3320 3345 /* Make sure no new RTT samples will be taken. */
3321 3346 tcp->tcp_csuna = tcp->tcp_snxt;
3322 3347
3323 3348 notsack_blk = tcp->tcp_notsack_list;
3324 3349 while (usable_swnd > 0) {
3325 3350 mblk_t *snxt_mp, *tmp_mp;
3326 3351 tcp_seq begin = tcp->tcp_sack_snxt;
3327 3352 tcp_seq end;
3328 3353 int32_t off;
3329 3354
3330 3355 for (; notsack_blk != NULL; notsack_blk = notsack_blk->next) {
3331 3356 if (SEQ_GT(notsack_blk->end, begin) &&
3332 3357 (notsack_blk->sack_cnt >=
3333 3358 tcps->tcps_dupack_fast_retransmit)) {
3334 3359 end = notsack_blk->end;
3335 3360 if (SEQ_LT(begin, notsack_blk->begin)) {
3336 3361 begin = notsack_blk->begin;
3337 3362 }
3338 3363 break;
3339 3364 }
3340 3365 }
3341 3366 /*
3342 3367 * All holes are filled. Manipulate tcp_cwnd to send more
3343 3368 * if we can. Note that after the SACK recovery, tcp_cwnd is
3344 3369 * set to tcp_cwnd_ssthresh.
3345 3370 */
3346 3371 if (notsack_blk == NULL) {
3347 3372 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3348 3373 if (usable_swnd <= 0 || tcp->tcp_unsent == 0) {
3349 3374 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna;
3350 3375 ASSERT(tcp->tcp_cwnd > 0);
3351 3376 return;
3352 3377 } else {
3353 3378 usable_swnd = usable_swnd / mss;
3354 3379 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna +
3355 3380 MAX(usable_swnd * mss, mss);
3356 3381 *flags |= TH_XMIT_NEEDED;
3357 3382 return;
3358 3383 }
3359 3384 }
3360 3385
3361 3386 /*
3362 3387 * Note that we may send more than usable_swnd allows here
3363 3388 * because of round off, but no more than 1 MSS of data.
3364 3389 */
3365 3390 seg_len = end - begin;
3366 3391 if (seg_len > mss)
3367 3392 seg_len = mss;
3368 3393 snxt_mp = tcp_get_seg_mp(tcp, begin, &off);
3369 3394 ASSERT(snxt_mp != NULL);
3370 3395 /* This should not happen. Defensive coding again... */
3371 3396 if (snxt_mp == NULL) {
3372 3397 return;
3373 3398 }
3374 3399
3375 3400 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, seg_len, &off,
3376 3401 &tmp_mp, begin, B_TRUE, &seg_len, B_TRUE);
3377 3402 if (xmit_mp == NULL)
3378 3403 return;
|
↓ open down ↓ |
948 lines elided |
↑ open up ↑ |
3379 3404
3380 3405 usable_swnd -= seg_len;
3381 3406 tcp->tcp_pipe += seg_len;
3382 3407 tcp->tcp_sack_snxt = begin + seg_len;
3383 3408
3384 3409 tcp_send_data(tcp, xmit_mp);
3385 3410
3386 3411 /*
3387 3412 * Update the send timestamp to avoid false retransmission.
3388 3413 */
3414 +#ifdef KERNEL_32
3389 3415 snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3416 +#else
3417 + snxt_mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
3418 +#endif
3390 3419
3391 3420 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3392 3421 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, seg_len);
3393 3422 TCPS_BUMP_MIB(tcps, tcpOutSackRetransSegs);
3423 + tcp->tcp_cs.tcp_out_retrans_segs++;
3424 + tcp->tcp_cs.tcp_out_retrans_bytes += seg_len;
3394 3425 /*
3395 3426 * Update tcp_rexmit_max to extend this SACK recovery phase.
3396 3427 * This happens when new data sent during fast recovery is
3397 3428 * also lost. If TCP retransmits those new data, it needs
3398 3429 * to extend SACK recover phase to avoid starting another
3399 3430 * fast retransmit/recovery unnecessarily.
3400 3431 */
3401 3432 if (SEQ_GT(tcp->tcp_sack_snxt, tcp->tcp_rexmit_max)) {
3402 3433 tcp->tcp_rexmit_max = tcp->tcp_sack_snxt;
3403 3434 }
3404 3435 }
3405 3436 }
3406 3437
3407 3438 /*
3408 3439 * tcp_ss_rexmit() is called to do slow start retransmission after a timeout
3409 3440 * or ICMP errors.
3410 3441 */
3411 3442 void
3412 3443 tcp_ss_rexmit(tcp_t *tcp)
3413 3444 {
3414 3445 uint32_t snxt;
3415 3446 uint32_t smax;
3416 3447 int32_t win;
3417 3448 int32_t mss;
3418 3449 int32_t off;
3419 3450 mblk_t *snxt_mp;
3420 3451 tcp_stack_t *tcps = tcp->tcp_tcps;
3421 3452
3422 3453 /*
3423 3454 * Note that tcp_rexmit can be set even though TCP has retransmitted
3424 3455 * all unack'ed segments.
3425 3456 */
3426 3457 if (SEQ_LT(tcp->tcp_rexmit_nxt, tcp->tcp_rexmit_max)) {
3427 3458 smax = tcp->tcp_rexmit_max;
3428 3459 snxt = tcp->tcp_rexmit_nxt;
3429 3460 if (SEQ_LT(snxt, tcp->tcp_suna)) {
3430 3461 snxt = tcp->tcp_suna;
3431 3462 }
3432 3463 win = MIN(tcp->tcp_cwnd, tcp->tcp_swnd);
3433 3464 win -= snxt - tcp->tcp_suna;
3434 3465 mss = tcp->tcp_mss;
3435 3466 snxt_mp = tcp_get_seg_mp(tcp, snxt, &off);
3436 3467
3437 3468 while (SEQ_LT(snxt, smax) && (win > 0) && (snxt_mp != NULL)) {
3438 3469 mblk_t *xmit_mp;
3439 3470 mblk_t *old_snxt_mp = snxt_mp;
3440 3471 uint32_t cnt = mss;
3441 3472
3442 3473 if (win < cnt) {
3443 3474 cnt = win;
3444 3475 }
3445 3476 if (SEQ_GT(snxt + cnt, smax)) {
3446 3477 cnt = smax - snxt;
3447 3478 }
3448 3479 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, cnt, &off,
3449 3480 &snxt_mp, snxt, B_TRUE, &cnt, B_TRUE);
3450 3481 if (xmit_mp == NULL)
|
↓ open down ↓ |
47 lines elided |
↑ open up ↑ |
3451 3482 return;
3452 3483
3453 3484 tcp_send_data(tcp, xmit_mp);
3454 3485
3455 3486 snxt += cnt;
3456 3487 win -= cnt;
3457 3488 /*
3458 3489 * Update the send timestamp to avoid false
3459 3490 * retransmission.
3460 3491 */
3492 +#ifdef KERNEL_32
3461 3493 old_snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3494 +#else
3495 + old_snxt_mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
3496 +#endif
3462 3497 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3463 3498 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, cnt);
3499 + tcp->tcp_cs.tcp_out_retrans_segs++;
3500 + tcp->tcp_cs.tcp_out_retrans_bytes += cnt;
3464 3501
3465 3502 tcp->tcp_rexmit_nxt = snxt;
3466 3503 }
3467 3504 /*
3468 3505 * If we have transmitted all we have at the time
3469 3506 * we started the retranmission, we can leave
3470 3507 * the rest of the job to tcp_wput_data(). But we
3471 3508 * need to check the send window first. If the
3472 3509 * win is not 0, go on with tcp_wput_data().
3473 3510 */
3474 3511 if (SEQ_LT(snxt, smax) || win == 0) {
3475 3512 return;
3476 3513 }
3477 3514 }
3478 3515 /* Only call tcp_wput_data() if there is data to be sent. */
3479 3516 if (tcp->tcp_unsent) {
3480 3517 tcp_wput_data(tcp, NULL, B_FALSE);
3481 3518 }
3482 3519 }
3483 3520
3484 3521 /*
3485 3522 * Do slow start retransmission after ICMP errors of PMTU changes.
3486 3523 */
3487 3524 void
3488 3525 tcp_rexmit_after_error(tcp_t *tcp)
3489 3526 {
3490 3527 /*
3491 3528 * All sent data has been acknowledged or no data left to send, just
3492 3529 * to return.
3493 3530 */
3494 3531 if (!SEQ_LT(tcp->tcp_suna, tcp->tcp_snxt) ||
3495 3532 (tcp->tcp_xmit_head == NULL))
3496 3533 return;
3497 3534
3498 3535 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && (tcp->tcp_unsent == 0))
3499 3536 tcp->tcp_rexmit_max = tcp->tcp_fss;
3500 3537 else
3501 3538 tcp->tcp_rexmit_max = tcp->tcp_snxt;
3502 3539
3503 3540 tcp->tcp_rexmit_nxt = tcp->tcp_suna;
3504 3541 tcp->tcp_rexmit = B_TRUE;
3505 3542 tcp->tcp_dupack_cnt = 0;
3506 3543 tcp_ss_rexmit(tcp);
3507 3544 }
3508 3545
3509 3546 /*
3510 3547 * tcp_get_seg_mp() is called to get the pointer to a segment in the
3511 3548 * send queue which starts at the given sequence number. If the given
3512 3549 * sequence number is equal to last valid sequence number (tcp_snxt), the
3513 3550 * returned mblk is the last valid mblk, and off is set to the length of
3514 3551 * that mblk.
3515 3552 *
3516 3553 * send queue which starts at the given seq. no.
3517 3554 *
3518 3555 * Parameters:
3519 3556 * tcp_t *tcp: the tcp instance pointer.
3520 3557 * uint32_t seq: the starting seq. no of the requested segment.
3521 3558 * int32_t *off: after the execution, *off will be the offset to
3522 3559 * the returned mblk which points to the requested seq no.
3523 3560 * It is the caller's responsibility to send in a non-null off.
3524 3561 *
3525 3562 * Return:
3526 3563 * A mblk_t pointer pointing to the requested segment in send queue.
3527 3564 */
3528 3565 static mblk_t *
3529 3566 tcp_get_seg_mp(tcp_t *tcp, uint32_t seq, int32_t *off)
3530 3567 {
3531 3568 int32_t cnt;
3532 3569 mblk_t *mp;
3533 3570
3534 3571 /* Defensive coding. Make sure we don't send incorrect data. */
3535 3572 if (SEQ_LT(seq, tcp->tcp_suna) || SEQ_GT(seq, tcp->tcp_snxt))
3536 3573 return (NULL);
3537 3574
3538 3575 cnt = seq - tcp->tcp_suna;
3539 3576 mp = tcp->tcp_xmit_head;
3540 3577 while (cnt > 0 && mp != NULL) {
3541 3578 cnt -= mp->b_wptr - mp->b_rptr;
3542 3579 if (cnt <= 0) {
3543 3580 cnt += mp->b_wptr - mp->b_rptr;
3544 3581 break;
3545 3582 }
3546 3583 mp = mp->b_cont;
3547 3584 }
3548 3585 ASSERT(mp != NULL);
3549 3586 *off = cnt;
3550 3587 return (mp);
3551 3588 }
3552 3589
3553 3590 /*
3554 3591 * This routine adjusts next-to-send sequence number variables, in the
3555 3592 * case where the reciever has shrunk it's window.
3556 3593 */
3557 3594 void
3558 3595 tcp_update_xmit_tail(tcp_t *tcp, uint32_t snxt)
3559 3596 {
3560 3597 mblk_t *xmit_tail;
3561 3598 int32_t offset;
3562 3599
3563 3600 tcp->tcp_snxt = snxt;
3564 3601
3565 3602 /* Get the mblk, and the offset in it, as per the shrunk window */
3566 3603 xmit_tail = tcp_get_seg_mp(tcp, snxt, &offset);
3567 3604 ASSERT(xmit_tail != NULL);
3568 3605 tcp->tcp_xmit_tail = xmit_tail;
3569 3606 tcp->tcp_xmit_tail_unsent = xmit_tail->b_wptr -
3570 3607 xmit_tail->b_rptr - offset;
3571 3608 }
3572 3609
3573 3610 /*
3574 3611 * This handles the case when the receiver has shrunk its win. Per RFC 1122
3575 3612 * if the receiver shrinks the window, i.e. moves the right window to the
3576 3613 * left, the we should not send new data, but should retransmit normally the
3577 3614 * old unacked data between suna and suna + swnd. We might has sent data
3578 3615 * that is now outside the new window, pretend that we didn't send it.
3579 3616 */
3580 3617 static void
3581 3618 tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_count)
3582 3619 {
3583 3620 uint32_t snxt = tcp->tcp_snxt;
3584 3621
3585 3622 ASSERT(shrunk_count > 0);
3586 3623
3587 3624 if (!tcp->tcp_is_wnd_shrnk) {
3588 3625 tcp->tcp_snxt_shrunk = snxt;
3589 3626 tcp->tcp_is_wnd_shrnk = B_TRUE;
3590 3627 } else if (SEQ_GT(snxt, tcp->tcp_snxt_shrunk)) {
3591 3628 tcp->tcp_snxt_shrunk = snxt;
3592 3629 }
3593 3630
3594 3631 /* Pretend we didn't send the data outside the window */
3595 3632 snxt -= shrunk_count;
3596 3633
3597 3634 /* Reset all the values per the now shrunk window */
3598 3635 tcp_update_xmit_tail(tcp, snxt);
3599 3636 tcp->tcp_unsent += shrunk_count;
3600 3637
3601 3638 /*
3602 3639 * If the SACK option is set, delete the entire list of
3603 3640 * notsack'ed blocks.
3604 3641 */
3605 3642 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list, tcp);
3606 3643
3607 3644 if (tcp->tcp_suna == tcp->tcp_snxt && tcp->tcp_swnd == 0)
3608 3645 /*
3609 3646 * Make sure the timer is running so that we will probe a zero
3610 3647 * window.
|
↓ open down ↓ |
137 lines elided |
↑ open up ↑ |
3611 3648 */
3612 3649 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
3613 3650 }
3614 3651
3615 3652 /*
3616 3653 * tcp_fill_header is called by tcp_send() to fill the outgoing TCP header
3617 3654 * with the template header, as well as other options such as time-stamp,
3618 3655 * ECN and/or SACK.
3619 3656 */
3620 3657 static void
3621 -tcp_fill_header(tcp_t *tcp, uchar_t *rptr, clock_t now, int num_sack_blk)
3658 +tcp_fill_header(tcp_t *tcp, uchar_t *rptr, int num_sack_blk)
3622 3659 {
3623 3660 tcpha_t *tcp_tmpl, *tcpha;
3624 3661 uint32_t *dst, *src;
3625 3662 int hdrlen;
3626 3663 conn_t *connp = tcp->tcp_connp;
3627 3664
3628 3665 ASSERT(OK_32PTR(rptr));
3629 3666
3630 3667 /* Template header */
3631 3668 tcp_tmpl = tcp->tcp_tcpha;
3632 3669
|
↓ open down ↓ |
1 lines elided |
↑ open up ↑ |
3633 3670 /* Header of outgoing packet */
3634 3671 tcpha = (tcpha_t *)(rptr + connp->conn_ixa->ixa_ip_hdr_length);
3635 3672
3636 3673 /* dst and src are opaque 32-bit fields, used for copying */
3637 3674 dst = (uint32_t *)rptr;
3638 3675 src = (uint32_t *)connp->conn_ht_iphc;
3639 3676 hdrlen = connp->conn_ht_iphc_len;
3640 3677
3641 3678 /* Fill time-stamp option if needed */
3642 3679 if (tcp->tcp_snd_ts_ok) {
3643 - U32_TO_BE32((uint32_t)now,
3680 + U32_TO_BE32(LBOLT_FASTPATH,
3644 3681 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 4);
3645 3682 U32_TO_BE32(tcp->tcp_ts_recent,
3646 3683 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 8);
3647 3684 } else {
3648 3685 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
3649 3686 }
3650 3687
3651 3688 /*
3652 3689 * Copy the template header; is this really more efficient than
3653 3690 * calling bcopy()? For simple IPv4/TCP, it may be the case,
3654 3691 * but perhaps not for other scenarios.
3655 3692 */
3656 3693 dst[0] = src[0];
3657 3694 dst[1] = src[1];
3658 3695 dst[2] = src[2];
3659 3696 dst[3] = src[3];
3660 3697 dst[4] = src[4];
3661 3698 dst[5] = src[5];
3662 3699 dst[6] = src[6];
3663 3700 dst[7] = src[7];
3664 3701 dst[8] = src[8];
3665 3702 dst[9] = src[9];
3666 3703 if (hdrlen -= 40) {
3667 3704 hdrlen >>= 2;
3668 3705 dst += 10;
3669 3706 src += 10;
3670 3707 do {
3671 3708 *dst++ = *src++;
3672 3709 } while (--hdrlen);
3673 3710 }
3674 3711
3675 3712 /*
3676 3713 * Set the ECN info in the TCP header if it is not a zero
3677 3714 * window probe. Zero window probe is only sent in
3678 3715 * tcp_wput_data() and tcp_timer().
3679 3716 */
3680 3717 if (tcp->tcp_ecn_ok && !tcp->tcp_zero_win_probe) {
3681 3718 TCP_SET_ECT(tcp, rptr);
3682 3719
3683 3720 if (tcp->tcp_ecn_echo_on)
3684 3721 tcpha->tha_flags |= TH_ECE;
3685 3722 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3686 3723 tcpha->tha_flags |= TH_CWR;
3687 3724 tcp->tcp_ecn_cwr_sent = B_TRUE;
3688 3725 }
3689 3726 }
3690 3727
3691 3728 /* Fill in SACK options */
3692 3729 if (num_sack_blk > 0) {
3693 3730 uchar_t *wptr = rptr + connp->conn_ht_iphc_len;
3694 3731 sack_blk_t *tmp;
3695 3732 int32_t i;
3696 3733
3697 3734 wptr[0] = TCPOPT_NOP;
3698 3735 wptr[1] = TCPOPT_NOP;
3699 3736 wptr[2] = TCPOPT_SACK;
3700 3737 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3701 3738 sizeof (sack_blk_t);
3702 3739 wptr += TCPOPT_REAL_SACK_LEN;
3703 3740
3704 3741 tmp = tcp->tcp_sack_list;
3705 3742 for (i = 0; i < num_sack_blk; i++) {
3706 3743 U32_TO_BE32(tmp[i].begin, wptr);
3707 3744 wptr += sizeof (tcp_seq);
3708 3745 U32_TO_BE32(tmp[i].end, wptr);
3709 3746 wptr += sizeof (tcp_seq);
3710 3747 }
3711 3748 tcpha->tha_offset_and_reserved +=
3712 3749 ((num_sack_blk * 2 + 1) << 4);
3713 3750 }
3714 3751 }
|
↓ open down ↓ |
61 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX