Print this page
12976 system panics with error in IP module
Reviewed by: Andy Fiddaman <andy@omniosce.org>
Reviewed by: Paul Winder <p.winder@me.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/inet/tcp/tcp_output.c
+++ new/usr/src/uts/common/inet/tcp/tcp_output.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
25 - * Copyright 2019 Joyent, Inc.
25 + * Copyright 2020 Joyent, Inc.
26 26 */
27 27
28 28 /* This file contains all TCP output processing functions. */
29 29
30 30 #include <sys/types.h>
31 31 #include <sys/stream.h>
32 32 #include <sys/strsun.h>
33 33 #include <sys/strsubr.h>
34 34 #include <sys/stropts.h>
35 35 #include <sys/strlog.h>
36 36 #define _SUN_TPI_VERSION 2
37 37 #include <sys/tihdr.h>
38 38 #include <sys/suntpi.h>
39 39 #include <sys/xti_inet.h>
40 40 #include <sys/timod.h>
41 41 #include <sys/pattr.h>
42 42 #include <sys/squeue_impl.h>
43 43 #include <sys/squeue.h>
44 44 #include <sys/sockio.h>
45 45 #include <sys/tsol/tnet.h>
46 46
47 47 #include <inet/common.h>
48 48 #include <inet/ip.h>
49 49 #include <inet/tcp.h>
50 50 #include <inet/tcp_impl.h>
51 51 #include <inet/snmpcom.h>
52 52 #include <inet/proto_set.h>
53 53 #include <inet/ipsec_impl.h>
54 54 #include <inet/ip_ndp.h>
55 55
56 56 static mblk_t *tcp_get_seg_mp(tcp_t *, uint32_t, int32_t *);
57 57 static void tcp_wput_cmdblk(queue_t *, mblk_t *);
58 58 static void tcp_wput_flush(tcp_t *, mblk_t *);
59 59 static void tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp);
60 60 static int tcp_xmit_end(tcp_t *);
61 61 static int tcp_send(tcp_t *, const int, const int, const int,
62 62 const int, int *, uint32_t *, int *, mblk_t **, mblk_t *);
63 63 static void tcp_xmit_early_reset(char *, mblk_t *, uint32_t, uint32_t,
64 64 int, ip_recv_attr_t *, ip_stack_t *, conn_t *);
65 65 static boolean_t tcp_send_rst_chk(tcp_stack_t *);
66 66 static void tcp_process_shrunk_swnd(tcp_t *, uint32_t);
67 67 static void tcp_fill_header(tcp_t *, uchar_t *, int);
68 68
69 69 /*
70 70 * Functions called directly via squeue having a prototype of edesc_t.
71 71 */
72 72 static void tcp_wput_nondata(void *, mblk_t *, void *, ip_recv_attr_t *);
73 73 static void tcp_wput_ioctl(void *, mblk_t *, void *, ip_recv_attr_t *);
74 74 static void tcp_wput_proto(void *, mblk_t *, void *, ip_recv_attr_t *);
75 75
76 76 /*
77 77 * This controls how tiny a write must be before we try to copy it
78 78 * into the mblk on the tail of the transmit queue. Not much
79 79 * speedup is observed for values larger than sixteen. Zero will
80 80 * disable the optimisation.
81 81 */
82 82 static int tcp_tx_pull_len = 16;
83 83
84 84 static void
85 85 cc_after_idle(tcp_t *tcp)
86 86 {
87 87 uint32_t old_cwnd = tcp->tcp_cwnd;
88 88
89 89 if (CC_ALGO(tcp)->after_idle != NULL)
90 90 CC_ALGO(tcp)->after_idle(&tcp->tcp_ccv);
91 91
92 92 DTRACE_PROBE3(cwnd__cc__after__idle, tcp_t *, tcp, uint32_t, old_cwnd,
93 93 uint32_t, tcp->tcp_cwnd);
94 94 }
95 95
96 96 int
97 97 tcp_wput(queue_t *q, mblk_t *mp)
98 98 {
99 99 conn_t *connp = Q_TO_CONN(q);
100 100 tcp_t *tcp;
101 101 void (*output_proc)();
102 102 t_scalar_t type;
103 103 uchar_t *rptr;
104 104 struct iocblk *iocp;
105 105 size_t size;
106 106
107 107 ASSERT(connp->conn_ref >= 2);
108 108
109 109 switch (DB_TYPE(mp)) {
110 110 case M_DATA:
111 111 tcp = connp->conn_tcp;
112 112 ASSERT(tcp != NULL);
113 113
114 114 size = msgdsize(mp);
115 115
116 116 mutex_enter(&tcp->tcp_non_sq_lock);
117 117 tcp->tcp_squeue_bytes += size;
118 118 if (TCP_UNSENT_BYTES(tcp) > connp->conn_sndbuf) {
119 119 tcp_setqfull(tcp);
120 120 }
121 121 mutex_exit(&tcp->tcp_non_sq_lock);
122 122
123 123 CONN_INC_REF(connp);
124 124 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, tcp_output, connp,
125 125 NULL, tcp_squeue_flag, SQTAG_TCP_OUTPUT);
126 126 return (0);
127 127
128 128 case M_CMD:
129 129 tcp_wput_cmdblk(q, mp);
130 130 return (0);
131 131
132 132 case M_PROTO:
133 133 case M_PCPROTO:
134 134 /*
135 135 * if it is a snmp message, don't get behind the squeue
136 136 */
137 137 tcp = connp->conn_tcp;
138 138 rptr = mp->b_rptr;
139 139 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
140 140 type = ((union T_primitives *)rptr)->type;
141 141 } else {
142 142 if (connp->conn_debug) {
143 143 (void) strlog(TCP_MOD_ID, 0, 1,
144 144 SL_ERROR|SL_TRACE,
145 145 "tcp_wput_proto, dropping one...");
146 146 }
147 147 freemsg(mp);
148 148 return (0);
149 149 }
150 150 if (type == T_SVR4_OPTMGMT_REQ) {
151 151 /*
152 152 * All Solaris components should pass a db_credp
153 153 * for this TPI message, hence we ASSERT.
154 154 * But in case there is some other M_PROTO that looks
155 155 * like a TPI message sent by some other kernel
156 156 * component, we check and return an error.
157 157 */
158 158 cred_t *cr = msg_getcred(mp, NULL);
159 159
160 160 ASSERT(cr != NULL);
161 161 if (cr == NULL) {
162 162 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
163 163 return (0);
164 164 }
165 165 if (snmpcom_req(q, mp, tcp_snmp_set, ip_snmp_get,
166 166 cr)) {
167 167 /*
168 168 * This was a SNMP request
169 169 */
170 170 return (0);
171 171 } else {
172 172 output_proc = tcp_wput_proto;
173 173 }
174 174 } else {
175 175 output_proc = tcp_wput_proto;
176 176 }
177 177 break;
178 178 case M_IOCTL:
179 179 /*
180 180 * Most ioctls can be processed right away without going via
181 181 * squeues - process them right here. Those that do require
182 182 * squeue (currently _SIOCSOCKFALLBACK)
183 183 * are processed by tcp_wput_ioctl().
184 184 */
185 185 iocp = (struct iocblk *)mp->b_rptr;
186 186 tcp = connp->conn_tcp;
187 187
188 188 switch (iocp->ioc_cmd) {
189 189 case TCP_IOC_ABORT_CONN:
190 190 tcp_ioctl_abort_conn(q, mp);
191 191 return (0);
192 192 case TI_GETPEERNAME:
193 193 case TI_GETMYNAME:
194 194 mi_copyin(q, mp, NULL,
195 195 SIZEOF_STRUCT(strbuf, iocp->ioc_flag));
196 196 return (0);
197 197
198 198 default:
199 199 output_proc = tcp_wput_ioctl;
200 200 break;
201 201 }
202 202 break;
203 203 default:
204 204 output_proc = tcp_wput_nondata;
205 205 break;
206 206 }
207 207
208 208 CONN_INC_REF(connp);
209 209 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, output_proc, connp,
210 210 NULL, tcp_squeue_flag, SQTAG_TCP_WPUT_OTHER);
211 211 return (0);
212 212 }
213 213
214 214 /*
215 215 * The TCP normal data output path.
216 216 * NOTE: the logic of the fast path is duplicated from this function.
217 217 */
218 218 void
219 219 tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent)
220 220 {
221 221 int len;
222 222 mblk_t *local_time;
223 223 mblk_t *mp1;
224 224 uint32_t snxt;
225 225 int tail_unsent;
226 226 int tcpstate;
227 227 int usable = 0;
228 228 mblk_t *xmit_tail;
229 229 int32_t mss;
230 230 int32_t num_sack_blk = 0;
231 231 int32_t total_hdr_len;
232 232 int32_t tcp_hdr_len;
233 233 int rc;
234 234 conn_t *connp = tcp->tcp_connp;
235 235 clock_t now = LBOLT_FASTPATH;
236 236
237 237 tcpstate = tcp->tcp_state;
238 238 if (mp == NULL) {
239 239 /*
240 240 * tcp_wput_data() with NULL mp should only be called when
241 241 * there is unsent data.
242 242 */
243 243 ASSERT(tcp->tcp_unsent > 0);
244 244 /* Really tacky... but we need this for detached closes. */
245 245 len = tcp->tcp_unsent;
246 246 goto data_null;
247 247 }
248 248
249 249 ASSERT(mp->b_datap->db_type == M_DATA);
250 250 /*
251 251 * Don't allow data after T_ORDREL_REQ or T_DISCON_REQ,
252 252 * or before a connection attempt has begun.
253 253 */
254 254 if (tcpstate < TCPS_SYN_SENT || tcpstate > TCPS_CLOSE_WAIT ||
255 255 (tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
256 256 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
257 257 #ifdef DEBUG
258 258 cmn_err(CE_WARN,
259 259 "tcp_wput_data: data after ordrel, %s",
260 260 tcp_display(tcp, NULL,
261 261 DISP_ADDR_AND_PORT));
262 262 #else
263 263 if (connp->conn_debug) {
264 264 (void) strlog(TCP_MOD_ID, 0, 1,
265 265 SL_TRACE|SL_ERROR,
266 266 "tcp_wput_data: data after ordrel, %s\n",
267 267 tcp_display(tcp, NULL,
268 268 DISP_ADDR_AND_PORT));
269 269 }
270 270 #endif /* DEBUG */
271 271 }
272 272 if (tcp->tcp_snd_zcopy_aware &&
273 273 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
274 274 tcp_zcopy_notify(tcp);
275 275 freemsg(mp);
276 276 mutex_enter(&tcp->tcp_non_sq_lock);
277 277 if (tcp->tcp_flow_stopped &&
278 278 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
279 279 tcp_clrqfull(tcp);
280 280 }
281 281 mutex_exit(&tcp->tcp_non_sq_lock);
282 282 return;
283 283 }
284 284
285 285 /* Strip empties */
286 286 for (;;) {
287 287 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <=
288 288 (uintptr_t)INT_MAX);
289 289 len = (int)(mp->b_wptr - mp->b_rptr);
290 290 if (len > 0)
291 291 break;
292 292 mp1 = mp;
293 293 mp = mp->b_cont;
294 294 freeb(mp1);
295 295 if (mp == NULL) {
296 296 return;
297 297 }
298 298 }
299 299
300 300 /* If we are the first on the list ... */
301 301 if (tcp->tcp_xmit_head == NULL) {
302 302 tcp->tcp_xmit_head = mp;
303 303 tcp->tcp_xmit_tail = mp;
304 304 tcp->tcp_xmit_tail_unsent = len;
305 305 } else {
306 306 /* If tiny tx and room in txq tail, pullup to save mblks. */
307 307 struct datab *dp;
308 308
309 309 mp1 = tcp->tcp_xmit_last;
310 310 if (len < tcp_tx_pull_len &&
311 311 (dp = mp1->b_datap)->db_ref == 1 &&
312 312 dp->db_lim - mp1->b_wptr >= len) {
313 313 ASSERT(len > 0);
314 314 ASSERT(!mp1->b_cont);
315 315 if (len == 1) {
316 316 *mp1->b_wptr++ = *mp->b_rptr;
317 317 } else {
318 318 bcopy(mp->b_rptr, mp1->b_wptr, len);
319 319 mp1->b_wptr += len;
320 320 }
321 321 if (mp1 == tcp->tcp_xmit_tail)
322 322 tcp->tcp_xmit_tail_unsent += len;
323 323 mp1->b_cont = mp->b_cont;
324 324 if (tcp->tcp_snd_zcopy_aware &&
325 325 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
326 326 mp1->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
327 327 freeb(mp);
328 328 mp = mp1;
329 329 } else {
330 330 tcp->tcp_xmit_last->b_cont = mp;
331 331 }
332 332 len += tcp->tcp_unsent;
333 333 }
334 334
335 335 /* Tack on however many more positive length mblks we have */
336 336 if ((mp1 = mp->b_cont) != NULL) {
337 337 do {
338 338 int tlen;
339 339 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <=
340 340 (uintptr_t)INT_MAX);
341 341 tlen = (int)(mp1->b_wptr - mp1->b_rptr);
342 342 if (tlen <= 0) {
343 343 mp->b_cont = mp1->b_cont;
344 344 freeb(mp1);
345 345 } else {
346 346 len += tlen;
347 347 mp = mp1;
348 348 }
349 349 } while ((mp1 = mp->b_cont) != NULL);
350 350 }
351 351 tcp->tcp_xmit_last = mp;
352 352 tcp->tcp_unsent = len;
353 353
354 354 if (urgent)
355 355 usable = 1;
356 356
357 357 data_null:
358 358 snxt = tcp->tcp_snxt;
359 359 xmit_tail = tcp->tcp_xmit_tail;
360 360 tail_unsent = tcp->tcp_xmit_tail_unsent;
361 361
362 362 /*
363 363 * Note that tcp_mss has been adjusted to take into account the
364 364 * timestamp option if applicable. Because SACK options do not
365 365 * appear in every TCP segments and they are of variable lengths,
366 366 * they cannot be included in tcp_mss. Thus we need to calculate
367 367 * the actual segment length when we need to send a segment which
368 368 * includes SACK options.
369 369 */
370 370 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
371 371 int32_t opt_len;
372 372
373 373 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
374 374 tcp->tcp_num_sack_blk);
375 375 opt_len = num_sack_blk * sizeof (sack_blk_t) + TCPOPT_NOP_LEN *
376 376 2 + TCPOPT_HEADER_LEN;
377 377 mss = tcp->tcp_mss - opt_len;
378 378 total_hdr_len = connp->conn_ht_iphc_len + opt_len;
379 379 tcp_hdr_len = connp->conn_ht_ulp_len + opt_len;
380 380 } else {
381 381 mss = tcp->tcp_mss;
382 382 total_hdr_len = connp->conn_ht_iphc_len;
383 383 tcp_hdr_len = connp->conn_ht_ulp_len;
384 384 }
385 385
386 386 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
387 387 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
388 388 cc_after_idle(tcp);
389 389 }
390 390 if (tcpstate == TCPS_SYN_RCVD) {
391 391 /*
392 392 * The three-way connection establishment handshake is not
393 393 * complete yet. We want to queue the data for transmission
394 394 * after entering ESTABLISHED state (RFC793). A jump to
395 395 * "done" label effectively leaves data on the queue.
396 396 */
397 397 goto done;
398 398 } else {
399 399 int usable_r;
400 400
401 401 /*
402 402 * In the special case when cwnd is zero, which can only
403 403 * happen if the connection is ECN capable, return now.
404 404 * New segments is sent using tcp_timer(). The timer
405 405 * is set in tcp_input_data().
406 406 */
407 407 if (tcp->tcp_cwnd == 0) {
408 408 /*
409 409 * Note that tcp_cwnd is 0 before 3-way handshake is
410 410 * finished.
411 411 */
412 412 ASSERT(tcp->tcp_ecn_ok ||
413 413 tcp->tcp_state < TCPS_ESTABLISHED);
414 414 return;
415 415 }
416 416
417 417 /* NOTE: trouble if xmitting while SYN not acked? */
418 418 usable_r = snxt - tcp->tcp_suna;
419 419 usable_r = tcp->tcp_swnd - usable_r;
420 420
421 421 /*
422 422 * Check if the receiver has shrunk the window. If
423 423 * tcp_wput_data() with NULL mp is called, tcp_fin_sent
424 424 * cannot be set as there is unsent data, so FIN cannot
425 425 * be sent out. Otherwise, we need to take into account
426 426 * of FIN as it consumes an "invisible" sequence number.
427 427 */
428 428 ASSERT(tcp->tcp_fin_sent == 0);
429 429 if (usable_r < 0) {
430 430 /*
431 431 * The receiver has shrunk the window and we have sent
432 432 * -usable_r date beyond the window, re-adjust.
433 433 *
434 434 * If TCP window scaling is enabled, there can be
435 435 * round down error as the advertised receive window
436 436 * is actually right shifted n bits. This means that
437 437 * the lower n bits info is wiped out. It will look
438 438 * like the window is shrunk. Do a check here to
439 439 * see if the shrunk amount is actually within the
440 440 * error in window calculation. If it is, just
441 441 * return. Note that this check is inside the
442 442 * shrunk window check. This makes sure that even
443 443 * though tcp_process_shrunk_swnd() is not called,
444 444 * we will stop further processing.
445 445 */
446 446 if ((-usable_r >> tcp->tcp_snd_ws) > 0) {
447 447 tcp_process_shrunk_swnd(tcp, -usable_r);
448 448 }
449 449 return;
450 450 }
451 451
452 452 /* usable = MIN(swnd, cwnd) - unacked_bytes */
453 453 if (tcp->tcp_swnd > tcp->tcp_cwnd)
454 454 usable_r -= tcp->tcp_swnd - tcp->tcp_cwnd;
455 455
456 456 /* usable = MIN(usable, unsent) */
457 457 if (usable_r > len)
458 458 usable_r = len;
459 459
460 460 /* usable = MAX(usable, {1 for urgent, 0 for data}) */
461 461 if (usable_r > 0) {
462 462 usable = usable_r;
463 463 } else {
464 464 /* Bypass all other unnecessary processing. */
465 465 goto done;
466 466 }
467 467 }
468 468
469 469 local_time = (mblk_t *)(intptr_t)gethrtime();
470 470
471 471 /*
472 472 * "Our" Nagle Algorithm. This is not the same as in the old
473 473 * BSD. This is more in line with the true intent of Nagle.
474 474 *
475 475 * The conditions are:
476 476 * 1. The amount of unsent data (or amount of data which can be
477 477 * sent, whichever is smaller) is less than Nagle limit.
478 478 * 2. The last sent size is also less than Nagle limit.
479 479 * 3. There is unack'ed data.
480 480 * 4. Urgent pointer is not set. Send urgent data ignoring the
481 481 * Nagle algorithm. This reduces the probability that urgent
482 482 * bytes get "merged" together.
483 483 * 5. The app has not closed the connection. This eliminates the
484 484 * wait time of the receiving side waiting for the last piece of
485 485 * (small) data.
486 486 *
487 487 * If all are satisified, exit without sending anything. Note
488 488 * that Nagle limit can be smaller than 1 MSS. Nagle limit is
489 489 * the smaller of 1 MSS and global tcp_naglim_def (default to be
490 490 * 4095).
491 491 */
492 492 if (usable < (int)tcp->tcp_naglim &&
493 493 tcp->tcp_naglim > tcp->tcp_last_sent_len &&
494 494 snxt != tcp->tcp_suna &&
495 495 !(tcp->tcp_valid_bits & TCP_URG_VALID) &&
496 496 !(tcp->tcp_valid_bits & TCP_FSS_VALID)) {
497 497 goto done;
498 498 }
499 499
500 500 /*
501 501 * If tcp_zero_win_probe is not set and the tcp->tcp_cork option
502 502 * is set, then we have to force TCP not to send partial segment
503 503 * (smaller than MSS bytes). We are calculating the usable now
504 504 * based on full mss and will save the rest of remaining data for
505 505 * later. When tcp_zero_win_probe is set, TCP needs to send out
506 506 * something to do zero window probe.
507 507 */
508 508 if (tcp->tcp_cork && !tcp->tcp_zero_win_probe) {
509 509 if (usable < mss)
510 510 goto done;
511 511 usable = (usable / mss) * mss;
512 512 }
513 513
514 514 /* Update the latest receive window size in TCP header. */
515 515 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
516 516
517 517 /* Send the packet. */
518 518 rc = tcp_send(tcp, mss, total_hdr_len, tcp_hdr_len,
519 519 num_sack_blk, &usable, &snxt, &tail_unsent, &xmit_tail,
520 520 local_time);
521 521
522 522 /* Pretend that all we were trying to send really got sent */
523 523 if (rc < 0 && tail_unsent < 0) {
524 524 do {
525 525 xmit_tail = xmit_tail->b_cont;
526 526 xmit_tail->b_prev = local_time;
527 527 ASSERT((uintptr_t)(xmit_tail->b_wptr -
528 528 xmit_tail->b_rptr) <= (uintptr_t)INT_MAX);
529 529 tail_unsent += (int)(xmit_tail->b_wptr -
530 530 xmit_tail->b_rptr);
531 531 } while (tail_unsent < 0);
532 532 }
533 533 done:;
534 534 tcp->tcp_xmit_tail = xmit_tail;
535 535 tcp->tcp_xmit_tail_unsent = tail_unsent;
536 536 len = tcp->tcp_snxt - snxt;
537 537 if (len) {
538 538 /*
539 539 * If new data was sent, need to update the notsack
540 540 * list, which is, afterall, data blocks that have
541 541 * not been sack'ed by the receiver. New data is
542 542 * not sack'ed.
543 543 */
544 544 if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) {
545 545 /* len is a negative value. */
546 546 tcp->tcp_pipe -= len;
547 547 tcp_notsack_update(&(tcp->tcp_notsack_list),
548 548 tcp->tcp_snxt, snxt,
549 549 &(tcp->tcp_num_notsack_blk),
550 550 &(tcp->tcp_cnt_notsack_list));
551 551 }
552 552 tcp->tcp_snxt = snxt + tcp->tcp_fin_sent;
553 553 tcp->tcp_rack = tcp->tcp_rnxt;
554 554 tcp->tcp_rack_cnt = 0;
555 555 if ((snxt + len) == tcp->tcp_suna) {
556 556 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
557 557 }
558 558 } else if (snxt == tcp->tcp_suna && tcp->tcp_swnd == 0) {
559 559 /*
560 560 * Didn't send anything. Make sure the timer is running
561 561 * so that we will probe a zero window.
562 562 */
563 563 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
564 564 }
565 565 /* Note that len is the amount we just sent but with a negative sign */
566 566 tcp->tcp_unsent += len;
567 567 mutex_enter(&tcp->tcp_non_sq_lock);
568 568 if (tcp->tcp_flow_stopped) {
569 569 if (TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
570 570 tcp_clrqfull(tcp);
571 571 }
572 572 } else if (TCP_UNSENT_BYTES(tcp) >= connp->conn_sndbuf) {
573 573 if (!(tcp->tcp_detached))
574 574 tcp_setqfull(tcp);
575 575 }
576 576 mutex_exit(&tcp->tcp_non_sq_lock);
577 577 }
578 578
579 579 /*
580 580 * Initial STREAMS write side put() procedure for sockets. It tries to
581 581 * handle the T_CAPABILITY_REQ which sockfs sends down while setting
582 582 * up the socket without using the squeue. Non T_CAPABILITY_REQ messages
583 583 * are handled by tcp_wput() as usual.
584 584 *
585 585 * All further messages will also be handled by tcp_wput() because we cannot
586 586 * be sure that the above short cut is safe later.
587 587 */
588 588 int
589 589 tcp_wput_sock(queue_t *wq, mblk_t *mp)
590 590 {
591 591 conn_t *connp = Q_TO_CONN(wq);
592 592 tcp_t *tcp = connp->conn_tcp;
593 593 struct T_capability_req *car = (struct T_capability_req *)mp->b_rptr;
594 594
595 595 ASSERT(wq->q_qinfo == &tcp_sock_winit);
596 596 wq->q_qinfo = &tcp_winit;
597 597
598 598 ASSERT(IPCL_IS_TCP(connp));
599 599 ASSERT(TCP_IS_SOCKET(tcp));
600 600
601 601 if (DB_TYPE(mp) == M_PCPROTO &&
602 602 MBLKL(mp) == sizeof (struct T_capability_req) &&
603 603 car->PRIM_type == T_CAPABILITY_REQ) {
604 604 tcp_capability_req(tcp, mp);
605 605 return (0);
606 606 }
607 607
608 608 tcp_wput(wq, mp);
609 609 return (0);
610 610 }
611 611
612 612 /* ARGSUSED */
613 613 int
614 614 tcp_wput_fallback(queue_t *wq, mblk_t *mp)
615 615 {
616 616 #ifdef DEBUG
617 617 cmn_err(CE_CONT, "tcp_wput_fallback: Message during fallback \n");
618 618 #endif
619 619 freemsg(mp);
620 620 return (0);
621 621 }
622 622
623 623 /*
624 624 * Call by tcp_wput() to handle misc non M_DATA messages.
625 625 */
626 626 /* ARGSUSED */
627 627 static void
628 628 tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
629 629 {
630 630 conn_t *connp = (conn_t *)arg;
631 631 tcp_t *tcp = connp->conn_tcp;
632 632
633 633 ASSERT(DB_TYPE(mp) != M_IOCTL);
634 634 /*
635 635 * TCP is D_MP and qprocsoff() is done towards the end of the tcp_close.
636 636 * Once the close starts, streamhead and sockfs will not let any data
637 637 * packets come down (close ensures that there are no threads using the
638 638 * queue and no new threads will come down) but since qprocsoff()
639 639 * hasn't happened yet, a M_FLUSH or some non data message might
640 640 * get reflected back (in response to our own FLUSHRW) and get
641 641 * processed after tcp_close() is done. The conn would still be valid
642 642 * because a ref would have added but we need to check the state
643 643 * before actually processing the packet.
644 644 */
645 645 if (TCP_IS_DETACHED(tcp) || (tcp->tcp_state == TCPS_CLOSED)) {
646 646 freemsg(mp);
647 647 return;
648 648 }
649 649
650 650 switch (DB_TYPE(mp)) {
651 651 case M_IOCDATA:
652 652 tcp_wput_iocdata(tcp, mp);
653 653 break;
654 654 case M_FLUSH:
655 655 tcp_wput_flush(tcp, mp);
656 656 break;
657 657 default:
658 658 ip_wput_nondata(connp->conn_wq, mp);
659 659 break;
660 660 }
661 661 }
662 662
663 663 /* tcp_wput_flush is called by tcp_wput_nondata to handle M_FLUSH messages. */
664 664 static void
665 665 tcp_wput_flush(tcp_t *tcp, mblk_t *mp)
666 666 {
667 667 uchar_t fval = *mp->b_rptr;
668 668 mblk_t *tail;
669 669 conn_t *connp = tcp->tcp_connp;
670 670 queue_t *q = connp->conn_wq;
671 671
672 672 /* TODO: How should flush interact with urgent data? */
673 673 if ((fval & FLUSHW) && tcp->tcp_xmit_head != NULL &&
674 674 !(tcp->tcp_valid_bits & TCP_URG_VALID)) {
675 675 /*
676 676 * Flush only data that has not yet been put on the wire. If
677 677 * we flush data that we have already transmitted, life, as we
678 678 * know it, may come to an end.
679 679 */
680 680 tail = tcp->tcp_xmit_tail;
681 681 tail->b_wptr -= tcp->tcp_xmit_tail_unsent;
682 682 tcp->tcp_xmit_tail_unsent = 0;
683 683 tcp->tcp_unsent = 0;
684 684 if (tail->b_wptr != tail->b_rptr)
685 685 tail = tail->b_cont;
686 686 if (tail) {
687 687 mblk_t **excess = &tcp->tcp_xmit_head;
688 688 for (;;) {
689 689 mblk_t *mp1 = *excess;
690 690 if (mp1 == tail)
691 691 break;
692 692 tcp->tcp_xmit_tail = mp1;
693 693 tcp->tcp_xmit_last = mp1;
694 694 excess = &mp1->b_cont;
695 695 }
696 696 *excess = NULL;
697 697 tcp_close_mpp(&tail);
698 698 if (tcp->tcp_snd_zcopy_aware)
699 699 tcp_zcopy_notify(tcp);
700 700 }
701 701 /*
702 702 * We have no unsent data, so unsent must be less than
703 703 * conn_sndlowat, so re-enable flow.
704 704 */
705 705 mutex_enter(&tcp->tcp_non_sq_lock);
706 706 if (tcp->tcp_flow_stopped) {
707 707 tcp_clrqfull(tcp);
708 708 }
709 709 mutex_exit(&tcp->tcp_non_sq_lock);
710 710 }
711 711 /*
712 712 * TODO: you can't just flush these, you have to increase rwnd for one
713 713 * thing. For another, how should urgent data interact?
714 714 */
715 715 if (fval & FLUSHR) {
716 716 *mp->b_rptr = fval & ~FLUSHW;
717 717 /* XXX */
718 718 qreply(q, mp);
719 719 return;
720 720 }
721 721 freemsg(mp);
722 722 }
723 723
724 724 /*
725 725 * tcp_wput_iocdata is called by tcp_wput_nondata to handle all M_IOCDATA
726 726 * messages.
727 727 */
728 728 static void
729 729 tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp)
730 730 {
731 731 mblk_t *mp1;
732 732 struct iocblk *iocp = (struct iocblk *)mp->b_rptr;
733 733 STRUCT_HANDLE(strbuf, sb);
734 734 uint_t addrlen;
735 735 conn_t *connp = tcp->tcp_connp;
736 736 queue_t *q = connp->conn_wq;
737 737
738 738 /* Make sure it is one of ours. */
739 739 switch (iocp->ioc_cmd) {
740 740 case TI_GETMYNAME:
741 741 case TI_GETPEERNAME:
742 742 break;
743 743 default:
744 744 /*
745 745 * If the conn is closing, then error the ioctl here. Otherwise
746 746 * use the CONN_IOCTLREF_* macros to hold off tcp_close until
747 747 * we're done here.
748 748 */
749 749 mutex_enter(&connp->conn_lock);
750 750 if (connp->conn_state_flags & CONN_CLOSING) {
751 751 mutex_exit(&connp->conn_lock);
752 752 iocp->ioc_error = EINVAL;
753 753 mp->b_datap->db_type = M_IOCNAK;
754 754 iocp->ioc_count = 0;
755 755 qreply(q, mp);
756 756 return;
757 757 }
758 758
759 759 CONN_INC_IOCTLREF_LOCKED(connp);
760 760 ip_wput_nondata(q, mp);
761 761 CONN_DEC_IOCTLREF(connp);
762 762 return;
763 763 }
764 764 switch (mi_copy_state(q, mp, &mp1)) {
765 765 case -1:
766 766 return;
767 767 case MI_COPY_CASE(MI_COPY_IN, 1):
768 768 break;
769 769 case MI_COPY_CASE(MI_COPY_OUT, 1):
770 770 /* Copy out the strbuf. */
771 771 mi_copyout(q, mp);
772 772 return;
773 773 case MI_COPY_CASE(MI_COPY_OUT, 2):
774 774 /* All done. */
775 775 mi_copy_done(q, mp, 0);
776 776 return;
777 777 default:
778 778 mi_copy_done(q, mp, EPROTO);
779 779 return;
780 780 }
781 781 /* Check alignment of the strbuf */
782 782 if (!OK_32PTR(mp1->b_rptr)) {
783 783 mi_copy_done(q, mp, EINVAL);
784 784 return;
785 785 }
786 786
787 787 STRUCT_SET_HANDLE(sb, iocp->ioc_flag, (void *)mp1->b_rptr);
788 788
789 789 if (connp->conn_family == AF_INET)
790 790 addrlen = sizeof (sin_t);
791 791 else
792 792 addrlen = sizeof (sin6_t);
793 793
794 794 if (STRUCT_FGET(sb, maxlen) < addrlen) {
795 795 mi_copy_done(q, mp, EINVAL);
796 796 return;
797 797 }
798 798
799 799 switch (iocp->ioc_cmd) {
800 800 case TI_GETMYNAME:
801 801 break;
802 802 case TI_GETPEERNAME:
803 803 if (tcp->tcp_state < TCPS_SYN_RCVD) {
804 804 mi_copy_done(q, mp, ENOTCONN);
805 805 return;
806 806 }
807 807 break;
808 808 }
809 809 mp1 = mi_copyout_alloc(q, mp, STRUCT_FGETP(sb, buf), addrlen, B_TRUE);
810 810 if (!mp1)
811 811 return;
812 812
813 813 STRUCT_FSET(sb, len, addrlen);
814 814 switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) {
815 815 case TI_GETMYNAME:
816 816 (void) conn_getsockname(connp, (struct sockaddr *)mp1->b_wptr,
817 817 &addrlen);
818 818 break;
819 819 case TI_GETPEERNAME:
820 820 (void) conn_getpeername(connp, (struct sockaddr *)mp1->b_wptr,
821 821 &addrlen);
822 822 break;
823 823 }
824 824 mp1->b_wptr += addrlen;
825 825 /* Copy out the address */
826 826 mi_copyout(q, mp);
827 827 }
828 828
829 829 /*
830 830 * tcp_wput_ioctl is called by tcp_wput_nondata() to handle all M_IOCTL
831 831 * messages.
832 832 */
833 833 /* ARGSUSED */
834 834 static void
835 835 tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
836 836 {
837 837 conn_t *connp = (conn_t *)arg;
838 838 tcp_t *tcp = connp->conn_tcp;
839 839 queue_t *q = connp->conn_wq;
840 840 struct iocblk *iocp;
841 841
842 842 ASSERT(DB_TYPE(mp) == M_IOCTL);
843 843 /*
844 844 * Try and ASSERT the minimum possible references on the
845 845 * conn early enough. Since we are executing on write side,
846 846 * the connection is obviously not detached and that means
847 847 * there is a ref each for TCP and IP. Since we are behind
848 848 * the squeue, the minimum references needed are 3. If the
849 849 * conn is in classifier hash list, there should be an
850 850 * extra ref for that (we check both the possibilities).
851 851 */
852 852 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
853 853 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
854 854
855 855 iocp = (struct iocblk *)mp->b_rptr;
856 856 switch (iocp->ioc_cmd) {
857 857 case _SIOCSOCKFALLBACK:
858 858 /*
859 859 * Either sockmod is about to be popped and the socket
860 860 * would now be treated as a plain stream, or a module
861 861 * is about to be pushed so we could no longer use read-
862 862 * side synchronous streams for fused loopback tcp.
863 863 * Drain any queued data and disable direct sockfs
864 864 * interface from now on.
865 865 */
866 866 if (!tcp->tcp_issocket) {
867 867 DB_TYPE(mp) = M_IOCNAK;
868 868 iocp->ioc_error = EINVAL;
869 869 } else {
870 870 tcp_use_pure_tpi(tcp);
871 871 DB_TYPE(mp) = M_IOCACK;
872 872 iocp->ioc_error = 0;
873 873 }
874 874 iocp->ioc_count = 0;
875 875 iocp->ioc_rval = 0;
876 876 qreply(q, mp);
877 877 return;
878 878 }
879 879
880 880 /*
881 881 * If the conn is closing, then error the ioctl here. Otherwise bump the
882 882 * conn_ioctlref to hold off tcp_close until we're done here.
883 883 */
884 884 mutex_enter(&(connp)->conn_lock);
885 885 if ((connp)->conn_state_flags & CONN_CLOSING) {
886 886 mutex_exit(&(connp)->conn_lock);
887 887 iocp->ioc_error = EINVAL;
888 888 mp->b_datap->db_type = M_IOCNAK;
889 889 iocp->ioc_count = 0;
890 890 qreply(q, mp);
891 891 return;
892 892 }
893 893
894 894 CONN_INC_IOCTLREF_LOCKED(connp);
895 895 ip_wput_nondata(q, mp);
896 896 CONN_DEC_IOCTLREF(connp);
897 897 }
898 898
899 899 /*
900 900 * This routine is called by tcp_wput() to handle all TPI requests.
901 901 */
902 902 /* ARGSUSED */
903 903 static void
904 904 tcp_wput_proto(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
905 905 {
906 906 conn_t *connp = (conn_t *)arg;
907 907 tcp_t *tcp = connp->conn_tcp;
908 908 union T_primitives *tprim = (union T_primitives *)mp->b_rptr;
909 909 uchar_t *rptr;
910 910 t_scalar_t type;
911 911 cred_t *cr;
912 912
913 913 /*
914 914 * Try and ASSERT the minimum possible references on the
915 915 * conn early enough. Since we are executing on write side,
916 916 * the connection is obviously not detached and that means
917 917 * there is a ref each for TCP and IP. Since we are behind
918 918 * the squeue, the minimum references needed are 3. If the
919 919 * conn is in classifier hash list, there should be an
920 920 * extra ref for that (we check both the possibilities).
921 921 */
922 922 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
923 923 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
924 924
925 925 rptr = mp->b_rptr;
926 926 ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX);
927 927 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
928 928 type = ((union T_primitives *)rptr)->type;
929 929 if (type == T_EXDATA_REQ) {
930 930 tcp_output_urgent(connp, mp, arg2, NULL);
931 931 } else if (type != T_DATA_REQ) {
932 932 goto non_urgent_data;
933 933 } else {
934 934 /* TODO: options, flags, ... from user */
935 935 /* Set length to zero for reclamation below */
936 936 tcp_wput_data(tcp, mp->b_cont, B_TRUE);
937 937 freeb(mp);
938 938 }
939 939 return;
940 940 } else {
941 941 if (connp->conn_debug) {
942 942 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
943 943 "tcp_wput_proto, dropping one...");
944 944 }
945 945 freemsg(mp);
946 946 return;
947 947 }
948 948
949 949 non_urgent_data:
950 950
951 951 switch ((int)tprim->type) {
952 952 case O_T_BIND_REQ: /* bind request */
953 953 case T_BIND_REQ: /* new semantics bind request */
954 954 tcp_tpi_bind(tcp, mp);
955 955 break;
956 956 case T_UNBIND_REQ: /* unbind request */
957 957 tcp_tpi_unbind(tcp, mp);
958 958 break;
959 959 case O_T_CONN_RES: /* old connection response XXX */
960 960 case T_CONN_RES: /* connection response */
961 961 tcp_tli_accept(tcp, mp);
962 962 break;
963 963 case T_CONN_REQ: /* connection request */
964 964 tcp_tpi_connect(tcp, mp);
965 965 break;
966 966 case T_DISCON_REQ: /* disconnect request */
967 967 tcp_disconnect(tcp, mp);
968 968 break;
969 969 case T_CAPABILITY_REQ:
970 970 tcp_capability_req(tcp, mp); /* capability request */
971 971 break;
972 972 case T_INFO_REQ: /* information request */
973 973 tcp_info_req(tcp, mp);
974 974 break;
975 975 case T_SVR4_OPTMGMT_REQ: /* manage options req */
976 976 case T_OPTMGMT_REQ:
977 977 /*
978 978 * Note: no support for snmpcom_req() through new
979 979 * T_OPTMGMT_REQ. See comments in ip.c
980 980 */
981 981
982 982 /*
983 983 * All Solaris components should pass a db_credp
984 984 * for this TPI message, hence we ASSERT.
985 985 * But in case there is some other M_PROTO that looks
986 986 * like a TPI message sent by some other kernel
987 987 * component, we check and return an error.
988 988 */
989 989 cr = msg_getcred(mp, NULL);
990 990 ASSERT(cr != NULL);
991 991 if (cr == NULL) {
992 992 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
993 993 return;
994 994 }
995 995 /*
996 996 * If EINPROGRESS is returned, the request has been queued
997 997 * for subsequent processing by ip_restart_optmgmt(), which
998 998 * will do the CONN_DEC_REF().
999 999 */
1000 1000 if ((int)tprim->type == T_SVR4_OPTMGMT_REQ) {
1001 1001 svr4_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
1002 1002 } else {
1003 1003 tpi_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
1004 1004 }
1005 1005 break;
1006 1006
1007 1007 case T_UNITDATA_REQ: /* unitdata request */
1008 1008 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
1009 1009 break;
1010 1010 case T_ORDREL_REQ: /* orderly release req */
1011 1011 freemsg(mp);
1012 1012
1013 1013 if (tcp->tcp_fused)
1014 1014 tcp_unfuse(tcp);
1015 1015
1016 1016 if (tcp_xmit_end(tcp) != 0) {
1017 1017 /*
1018 1018 * We were crossing FINs and got a reset from
1019 1019 * the other side. Just ignore it.
1020 1020 */
1021 1021 if (connp->conn_debug) {
1022 1022 (void) strlog(TCP_MOD_ID, 0, 1,
1023 1023 SL_ERROR|SL_TRACE,
1024 1024 "tcp_wput_proto, T_ORDREL_REQ out of "
1025 1025 "state %s",
1026 1026 tcp_display(tcp, NULL,
1027 1027 DISP_ADDR_AND_PORT));
1028 1028 }
1029 1029 }
1030 1030 break;
1031 1031 case T_ADDR_REQ:
1032 1032 tcp_addr_req(tcp, mp);
1033 1033 break;
1034 1034 default:
1035 1035 if (connp->conn_debug) {
1036 1036 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
1037 1037 "tcp_wput_proto, bogus TPI msg, type %d",
1038 1038 tprim->type);
1039 1039 }
1040 1040 /*
1041 1041 * We used to M_ERROR. Sending TNOTSUPPORT gives the user
1042 1042 * to recover.
1043 1043 */
1044 1044 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
1045 1045 break;
1046 1046 }
1047 1047 }
1048 1048
1049 1049 /*
1050 1050 * Handle special out-of-band ioctl requests (see PSARC/2008/265).
1051 1051 */
1052 1052 static void
1053 1053 tcp_wput_cmdblk(queue_t *q, mblk_t *mp)
1054 1054 {
1055 1055 void *data;
1056 1056 mblk_t *datamp = mp->b_cont;
1057 1057 conn_t *connp = Q_TO_CONN(q);
1058 1058 tcp_t *tcp = connp->conn_tcp;
1059 1059 cmdblk_t *cmdp = (cmdblk_t *)mp->b_rptr;
1060 1060
1061 1061 if (datamp == NULL || MBLKL(datamp) < cmdp->cb_len) {
1062 1062 cmdp->cb_error = EPROTO;
1063 1063 qreply(q, mp);
1064 1064 return;
1065 1065 }
1066 1066
1067 1067 data = datamp->b_rptr;
1068 1068
1069 1069 switch (cmdp->cb_cmd) {
1070 1070 case TI_GETPEERNAME:
1071 1071 if (tcp->tcp_state < TCPS_SYN_RCVD)
1072 1072 cmdp->cb_error = ENOTCONN;
1073 1073 else
1074 1074 cmdp->cb_error = conn_getpeername(connp, data,
1075 1075 &cmdp->cb_len);
1076 1076 break;
1077 1077 case TI_GETMYNAME:
1078 1078 cmdp->cb_error = conn_getsockname(connp, data, &cmdp->cb_len);
1079 1079 break;
1080 1080 default:
1081 1081 cmdp->cb_error = EINVAL;
1082 1082 break;
1083 1083 }
1084 1084
1085 1085 qreply(q, mp);
1086 1086 }
1087 1087
1088 1088 /*
1089 1089 * The TCP fast path write put procedure.
1090 1090 * NOTE: the logic of the fast path is duplicated from tcp_wput_data()
1091 1091 */
1092 1092 /* ARGSUSED */
1093 1093 void
1094 1094 tcp_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1095 1095 {
1096 1096 int len;
1097 1097 int hdrlen;
1098 1098 int plen;
1099 1099 mblk_t *mp1;
1100 1100 uchar_t *rptr;
1101 1101 uint32_t snxt;
1102 1102 tcpha_t *tcpha;
1103 1103 struct datab *db;
1104 1104 uint32_t suna;
1105 1105 uint32_t mss;
1106 1106 ipaddr_t *dst;
1107 1107 ipaddr_t *src;
1108 1108 uint32_t sum;
1109 1109 int usable;
1110 1110 conn_t *connp = (conn_t *)arg;
1111 1111 tcp_t *tcp = connp->conn_tcp;
1112 1112 uint32_t msize;
1113 1113 tcp_stack_t *tcps = tcp->tcp_tcps;
1114 1114 ip_xmit_attr_t *ixa;
1115 1115 clock_t now;
1116 1116
1117 1117 /*
1118 1118 * Try and ASSERT the minimum possible references on the
1119 1119 * conn early enough. Since we are executing on write side,
1120 1120 * the connection is obviously not detached and that means
1121 1121 * there is a ref each for TCP and IP. Since we are behind
1122 1122 * the squeue, the minimum references needed are 3. If the
1123 1123 * conn is in classifier hash list, there should be an
1124 1124 * extra ref for that (we check both the possibilities).
1125 1125 */
1126 1126 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
1127 1127 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
1128 1128
1129 1129 ASSERT(DB_TYPE(mp) == M_DATA);
1130 1130 msize = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1131 1131
1132 1132 mutex_enter(&tcp->tcp_non_sq_lock);
1133 1133 tcp->tcp_squeue_bytes -= msize;
1134 1134 mutex_exit(&tcp->tcp_non_sq_lock);
1135 1135
1136 1136 /* Bypass tcp protocol for fused tcp loopback */
1137 1137 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1138 1138 return;
1139 1139
1140 1140 mss = tcp->tcp_mss;
1141 1141 /*
1142 1142 * If ZEROCOPY has turned off, try not to send any zero-copy message
1143 1143 * down. Do backoff, now.
1144 1144 */
1145 1145 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on)
1146 1146 mp = tcp_zcopy_backoff(tcp, mp, B_FALSE);
1147 1147
1148 1148
1149 1149 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX);
1150 1150 len = (int)(mp->b_wptr - mp->b_rptr);
1151 1151
1152 1152 /*
1153 1153 * Criteria for fast path:
1154 1154 *
1155 1155 * 1. no unsent data
1156 1156 * 2. single mblk in request
1157 1157 * 3. connection established
1158 1158 * 4. data in mblk
1159 1159 * 5. len <= mss
1160 1160 * 6. no tcp_valid bits
1161 1161 */
1162 1162 if ((tcp->tcp_unsent != 0) ||
1163 1163 (tcp->tcp_cork) ||
1164 1164 (mp->b_cont != NULL) ||
1165 1165 (tcp->tcp_state != TCPS_ESTABLISHED) ||
1166 1166 (len == 0) ||
1167 1167 (len > mss) ||
1168 1168 (tcp->tcp_valid_bits != 0)) {
1169 1169 tcp_wput_data(tcp, mp, B_FALSE);
1170 1170 return;
1171 1171 }
1172 1172
1173 1173 ASSERT(tcp->tcp_xmit_tail_unsent == 0);
1174 1174 ASSERT(tcp->tcp_fin_sent == 0);
1175 1175
1176 1176 /* queue new packet onto retransmission queue */
1177 1177 if (tcp->tcp_xmit_head == NULL) {
1178 1178 tcp->tcp_xmit_head = mp;
1179 1179 } else {
1180 1180 tcp->tcp_xmit_last->b_cont = mp;
1181 1181 }
1182 1182 tcp->tcp_xmit_last = mp;
1183 1183 tcp->tcp_xmit_tail = mp;
1184 1184
1185 1185 /* find out how much we can send */
1186 1186 /* BEGIN CSTYLED */
1187 1187 /*
1188 1188 * un-acked usable
1189 1189 * |--------------|-----------------|
1190 1190 * tcp_suna tcp_snxt tcp_suna+tcp_swnd
1191 1191 */
1192 1192 /* END CSTYLED */
1193 1193
1194 1194 /* start sending from tcp_snxt */
1195 1195 snxt = tcp->tcp_snxt;
1196 1196
1197 1197 /*
1198 1198 * Check to see if this connection has been idle for some time and no
1199 1199 * ACK is expected. If so, then the congestion window size is no longer
1200 1200 * meaningfully tied to current network conditions.
1201 1201 *
1202 1202 * We reinitialize tcp_cwnd, and slow start again to get back the
1203 1203 * connection's "self-clock" as described in Van Jacobson's 1988 paper
1204 1204 * "Congestion avoidance and control".
1205 1205 */
1206 1206 now = LBOLT_FASTPATH;
1207 1207 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
1208 1208 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
1209 1209 cc_after_idle(tcp);
1210 1210 }
1211 1211
1212 1212 usable = tcp->tcp_swnd; /* tcp window size */
1213 1213 if (usable > tcp->tcp_cwnd)
1214 1214 usable = tcp->tcp_cwnd; /* congestion window smaller */
1215 1215 usable -= snxt; /* subtract stuff already sent */
1216 1216 suna = tcp->tcp_suna;
1217 1217 usable += suna;
1218 1218 /* usable can be < 0 if the congestion window is smaller */
1219 1219 if (len > usable) {
1220 1220 /* Can't send complete M_DATA in one shot */
1221 1221 goto slow;
1222 1222 }
1223 1223
1224 1224 mutex_enter(&tcp->tcp_non_sq_lock);
1225 1225 if (tcp->tcp_flow_stopped &&
1226 1226 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
1227 1227 tcp_clrqfull(tcp);
1228 1228 }
1229 1229 mutex_exit(&tcp->tcp_non_sq_lock);
1230 1230
1231 1231 /*
1232 1232 * determine if anything to send (Nagle).
1233 1233 *
1234 1234 * 1. len < tcp_mss (i.e. small)
1235 1235 * 2. unacknowledged data present
1236 1236 * 3. len < nagle limit
1237 1237 * 4. last packet sent < nagle limit (previous packet sent)
1238 1238 */
1239 1239 if ((len < mss) && (snxt != suna) &&
1240 1240 (len < (int)tcp->tcp_naglim) &&
1241 1241 (tcp->tcp_last_sent_len < tcp->tcp_naglim)) {
1242 1242 /*
1243 1243 * This was the first unsent packet and normally
1244 1244 * mss < xmit_hiwater so there is no need to worry
1245 1245 * about flow control. The next packet will go
1246 1246 * through the flow control check in tcp_wput_data().
1247 1247 */
1248 1248 /* leftover work from above */
1249 1249 tcp->tcp_unsent = len;
1250 1250 tcp->tcp_xmit_tail_unsent = len;
1251 1251
1252 1252 return;
1253 1253 }
1254 1254
1255 1255 /*
1256 1256 * len <= tcp->tcp_mss && len == unsent so no sender silly window. Can
1257 1257 * send now.
1258 1258 */
1259 1259
1260 1260 if (snxt == suna) {
1261 1261 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1262 1262 }
1263 1263
1264 1264 /* we have always sent something */
1265 1265 tcp->tcp_rack_cnt = 0;
1266 1266
1267 1267 tcp->tcp_snxt = snxt + len;
1268 1268 tcp->tcp_rack = tcp->tcp_rnxt;
1269 1269
1270 1270 if ((mp1 = dupb(mp)) == 0)
1271 1271 goto no_memory;
1272 1272 mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
1273 1273 mp->b_next = (mblk_t *)(uintptr_t)snxt;
1274 1274
1275 1275 /* adjust tcp header information */
1276 1276 tcpha = tcp->tcp_tcpha;
1277 1277 tcpha->tha_flags = (TH_ACK|TH_PUSH);
1278 1278
1279 1279 sum = len + connp->conn_ht_ulp_len + connp->conn_sum;
1280 1280 sum = (sum >> 16) + (sum & 0xFFFF);
1281 1281 tcpha->tha_sum = htons(sum);
1282 1282
1283 1283 tcpha->tha_seq = htonl(snxt);
1284 1284
1285 1285 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1286 1286 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1287 1287 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1288 1288 tcp->tcp_cs.tcp_out_data_segs++;
1289 1289 tcp->tcp_cs.tcp_out_data_bytes += len;
1290 1290
1291 1291 /* Update the latest receive window size in TCP header. */
1292 1292 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
1293 1293
1294 1294 tcp->tcp_last_sent_len = (ushort_t)len;
1295 1295
1296 1296 plen = len + connp->conn_ht_iphc_len;
1297 1297
1298 1298 ixa = connp->conn_ixa;
1299 1299 ixa->ixa_pktlen = plen;
1300 1300
1301 1301 if (ixa->ixa_flags & IXAF_IS_IPV4) {
1302 1302 tcp->tcp_ipha->ipha_length = htons(plen);
1303 1303 } else {
1304 1304 tcp->tcp_ip6h->ip6_plen = htons(plen - IPV6_HDR_LEN);
1305 1305 }
1306 1306
1307 1307 /* see if we need to allocate a mblk for the headers */
1308 1308 hdrlen = connp->conn_ht_iphc_len;
1309 1309 rptr = mp1->b_rptr - hdrlen;
1310 1310 db = mp1->b_datap;
1311 1311 if ((db->db_ref != 2) || rptr < db->db_base ||
1312 1312 (!OK_32PTR(rptr))) {
1313 1313 /* NOTE: we assume allocb returns an OK_32PTR */
1314 1314 mp = allocb(hdrlen + tcps->tcps_wroff_xtra, BPRI_MED);
1315 1315 if (!mp) {
1316 1316 freemsg(mp1);
1317 1317 goto no_memory;
1318 1318 }
1319 1319 mp->b_cont = mp1;
1320 1320 mp1 = mp;
1321 1321 /* Leave room for Link Level header */
1322 1322 rptr = &mp1->b_rptr[tcps->tcps_wroff_xtra];
1323 1323 mp1->b_wptr = &rptr[hdrlen];
1324 1324 }
1325 1325 mp1->b_rptr = rptr;
1326 1326
1327 1327 /* Fill in the timestamp option. */
1328 1328 if (tcp->tcp_snd_ts_ok) {
1329 1329 U32_TO_BE32(now,
1330 1330 (char *)tcpha + TCP_MIN_HEADER_LENGTH + 4);
1331 1331 U32_TO_BE32(tcp->tcp_ts_recent,
1332 1332 (char *)tcpha + TCP_MIN_HEADER_LENGTH + 8);
1333 1333 } else {
1334 1334 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
1335 1335 }
1336 1336
1337 1337 /* copy header into outgoing packet */
1338 1338 dst = (ipaddr_t *)rptr;
1339 1339 src = (ipaddr_t *)connp->conn_ht_iphc;
1340 1340 dst[0] = src[0];
1341 1341 dst[1] = src[1];
1342 1342 dst[2] = src[2];
1343 1343 dst[3] = src[3];
1344 1344 dst[4] = src[4];
1345 1345 dst[5] = src[5];
1346 1346 dst[6] = src[6];
1347 1347 dst[7] = src[7];
1348 1348 dst[8] = src[8];
1349 1349 dst[9] = src[9];
1350 1350 if (hdrlen -= 40) {
1351 1351 hdrlen >>= 2;
1352 1352 dst += 10;
1353 1353 src += 10;
1354 1354 do {
1355 1355 *dst++ = *src++;
1356 1356 } while (--hdrlen);
1357 1357 }
1358 1358
1359 1359 /*
1360 1360 * Set the ECN info in the TCP header. Note that this
1361 1361 * is not the template header.
1362 1362 */
1363 1363 if (tcp->tcp_ecn_ok) {
1364 1364 TCP_SET_ECT(tcp, rptr);
1365 1365
1366 1366 tcpha = (tcpha_t *)(rptr + ixa->ixa_ip_hdr_length);
1367 1367 if (tcp->tcp_ecn_echo_on)
1368 1368 tcpha->tha_flags |= TH_ECE;
1369 1369 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
1370 1370 tcpha->tha_flags |= TH_CWR;
1371 1371 tcp->tcp_ecn_cwr_sent = B_TRUE;
1372 1372 }
1373 1373 }
1374 1374
1375 1375 if (tcp->tcp_ip_forward_progress) {
1376 1376 tcp->tcp_ip_forward_progress = B_FALSE;
1377 1377 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
1378 1378 } else {
1379 1379 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
1380 1380 }
1381 1381 tcp_send_data(tcp, mp1);
1382 1382 return;
1383 1383
1384 1384 /*
1385 1385 * If we ran out of memory, we pretend to have sent the packet
1386 1386 * and that it was lost on the wire.
1387 1387 */
1388 1388 no_memory:
1389 1389 return;
1390 1390
1391 1391 slow:
1392 1392 /* leftover work from above */
1393 1393 tcp->tcp_unsent = len;
1394 1394 tcp->tcp_xmit_tail_unsent = len;
1395 1395 tcp_wput_data(tcp, NULL, B_FALSE);
1396 1396 }
1397 1397
1398 1398 /* ARGSUSED2 */
1399 1399 void
1400 1400 tcp_output_urgent(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1401 1401 {
1402 1402 int len;
1403 1403 uint32_t msize;
1404 1404 conn_t *connp = (conn_t *)arg;
1405 1405 tcp_t *tcp = connp->conn_tcp;
1406 1406
1407 1407 msize = msgdsize(mp);
1408 1408
1409 1409 len = msize - 1;
1410 1410 if (len < 0) {
1411 1411 freemsg(mp);
1412 1412 return;
1413 1413 }
1414 1414
1415 1415 /*
1416 1416 * Try to force urgent data out on the wire. Even if we have unsent
1417 1417 * data this will at least send the urgent flag.
1418 1418 * XXX does not handle more flag correctly.
1419 1419 */
1420 1420 len += tcp->tcp_unsent;
1421 1421 len += tcp->tcp_snxt;
1422 1422 tcp->tcp_urg = len;
1423 1423 tcp->tcp_valid_bits |= TCP_URG_VALID;
1424 1424
1425 1425 /* Bypass tcp protocol for fused tcp loopback */
1426 1426 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1427 1427 return;
1428 1428
1429 1429 /* Strip off the T_EXDATA_REQ if the data is from TPI */
1430 1430 if (DB_TYPE(mp) != M_DATA) {
1431 1431 mblk_t *mp1 = mp;
1432 1432 ASSERT(!IPCL_IS_NONSTR(connp));
1433 1433 mp = mp->b_cont;
1434 1434 freeb(mp1);
1435 1435 }
1436 1436 tcp_wput_data(tcp, mp, B_TRUE);
1437 1437 }
1438 1438
1439 1439 /*
1440 1440 * Called by streams close routine via squeues when our client blows off its
1441 1441 * descriptor, we take this to mean: "close the stream state NOW, close the tcp
1442 1442 * connection politely" When SO_LINGER is set (with a non-zero linger time and
1443 1443 * it is not a nonblocking socket) then this routine sleeps until the FIN is
1444 1444 * acked.
1445 1445 *
1446 1446 * NOTE: tcp_close potentially returns error when lingering.
1447 1447 * However, the stream head currently does not pass these errors
1448 1448 * to the application. 4.4BSD only returns EINTR and EWOULDBLOCK
1449 1449 * errors to the application (from tsleep()) and not errors
1450 1450 * like ECONNRESET caused by receiving a reset packet.
1451 1451 */
1452 1452
1453 1453 /* ARGSUSED */
1454 1454 void
1455 1455 tcp_close_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1456 1456 {
1457 1457 char *msg;
1458 1458 conn_t *connp = (conn_t *)arg;
1459 1459 tcp_t *tcp = connp->conn_tcp;
1460 1460 clock_t delta = 0;
1461 1461 tcp_stack_t *tcps = tcp->tcp_tcps;
1462 1462
1463 1463 /*
1464 1464 * When a non-STREAMS socket is being closed, it does not always
1465 1465 * stick around waiting for tcp_close_output to run and can therefore
1466 1466 * have dropped a reference already. So adjust the asserts accordingly.
1467 1467 */
1468 1468 ASSERT((connp->conn_fanout != NULL &&
1469 1469 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 3 : 4)) ||
1470 1470 (connp->conn_fanout == NULL &&
1471 1471 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3)));
1472 1472
1473 1473 mutex_enter(&tcp->tcp_eager_lock);
1474 1474 if (tcp->tcp_conn_req_cnt_q0 != 0 || tcp->tcp_conn_req_cnt_q != 0) {
1475 1475 /*
1476 1476 * Cleanup for listener. For non-STREAM sockets sockfs will
1477 1477 * close all the eagers on 'q', so in that case only deal
1478 1478 * with 'q0'.
1479 1479 */
1480 1480 tcp_eager_cleanup(tcp, IPCL_IS_NONSTR(connp) ? 1 : 0);
1481 1481 tcp->tcp_wait_for_eagers = 1;
1482 1482 }
1483 1483 mutex_exit(&tcp->tcp_eager_lock);
1484 1484
1485 1485 tcp->tcp_lso = B_FALSE;
1486 1486
1487 1487 msg = NULL;
1488 1488 switch (tcp->tcp_state) {
1489 1489 case TCPS_CLOSED:
1490 1490 case TCPS_IDLE:
1491 1491 break;
1492 1492 case TCPS_BOUND:
1493 1493 if (tcp->tcp_listener != NULL) {
1494 1494 ASSERT(IPCL_IS_NONSTR(connp));
1495 1495 /*
1496 1496 * Unlink from the listener and drop the reference
1497 1497 * put on it by the eager. tcp_closei_local will not
1498 1498 * do it because tcp_tconnind_started is TRUE.
1499 1499 */
1500 1500 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1501 1501 tcp_eager_unlink(tcp);
1502 1502 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1503 1503 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1504 1504 }
1505 1505 break;
1506 1506 case TCPS_LISTEN:
1507 1507 break;
1508 1508 case TCPS_SYN_SENT:
1509 1509 msg = "tcp_close, during connect";
1510 1510 break;
1511 1511 case TCPS_SYN_RCVD:
1512 1512 /*
1513 1513 * Close during the connect 3-way handshake
1514 1514 * but here there may or may not be pending data
1515 1515 * already on queue. Process almost same as in
1516 1516 * the ESTABLISHED state.
1517 1517 */
1518 1518 /* FALLTHRU */
1519 1519 default:
1520 1520 if (tcp->tcp_fused)
1521 1521 tcp_unfuse(tcp);
1522 1522
1523 1523 /*
1524 1524 * If SO_LINGER has set a zero linger time, abort the
1525 1525 * connection with a reset.
1526 1526 */
1527 1527 if (connp->conn_linger && connp->conn_lingertime == 0) {
1528 1528 msg = "tcp_close, zero lingertime";
1529 1529 break;
1530 1530 }
1531 1531
1532 1532 /*
1533 1533 * Abort connection if there is unread data queued.
1534 1534 */
1535 1535 if (tcp->tcp_rcv_list || tcp->tcp_reass_head) {
1536 1536 msg = "tcp_close, unread data";
1537 1537 break;
1538 1538 }
1539 1539
1540 1540 /*
1541 1541 * Abort connection if it is being closed without first
1542 1542 * being accepted. This can happen if a listening non-STREAM
1543 1543 * socket wants to get rid of the socket, for example, if the
1544 1544 * listener is closing.
1545 1545 */
1546 1546 if (tcp->tcp_listener != NULL) {
1547 1547 ASSERT(IPCL_IS_NONSTR(connp));
1548 1548 msg = "tcp_close, close before accept";
1549 1549
1550 1550 /*
1551 1551 * Unlink from the listener and drop the reference
1552 1552 * put on it by the eager. tcp_closei_local will not
1553 1553 * do it because tcp_tconnind_started is TRUE.
1554 1554 */
1555 1555 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1556 1556 tcp_eager_unlink(tcp);
1557 1557 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1558 1558 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1559 1559 break;
1560 1560 }
1561 1561
1562 1562 /*
1563 1563 * Transmit the FIN before detaching the tcp_t.
1564 1564 * After tcp_detach returns this queue/perimeter
1565 1565 * no longer owns the tcp_t thus others can modify it.
1566 1566 */
1567 1567 (void) tcp_xmit_end(tcp);
1568 1568
1569 1569 /*
1570 1570 * If lingering on close then wait until the fin is acked,
1571 1571 * the SO_LINGER time passes, or a reset is sent/received.
1572 1572 */
1573 1573 if (connp->conn_linger && connp->conn_lingertime > 0 &&
1574 1574 !(tcp->tcp_fin_acked) &&
1575 1575 tcp->tcp_state >= TCPS_ESTABLISHED) {
1576 1576 if (tcp->tcp_closeflags & (FNDELAY|FNONBLOCK)) {
1577 1577 tcp->tcp_client_errno = EWOULDBLOCK;
1578 1578 } else if (tcp->tcp_client_errno == 0) {
1579 1579
1580 1580 ASSERT(tcp->tcp_linger_tid == 0);
1581 1581
1582 1582 /* conn_lingertime is in sec. */
1583 1583 tcp->tcp_linger_tid = TCP_TIMER(tcp,
1584 1584 tcp_close_linger_timeout,
1585 1585 connp->conn_lingertime * MILLISEC);
1586 1586
1587 1587 /* tcp_close_linger_timeout will finish close */
1588 1588 if (tcp->tcp_linger_tid == 0)
1589 1589 tcp->tcp_client_errno = ENOSR;
1590 1590 else
1591 1591 return;
1592 1592 }
1593 1593
1594 1594 /*
1595 1595 * Check if we need to detach or just close
1596 1596 * the instance.
1597 1597 */
1598 1598 if (tcp->tcp_state <= TCPS_LISTEN)
1599 1599 break;
1600 1600 }
1601 1601
1602 1602 /*
1603 1603 * Make sure that no other thread will access the conn_rq of
1604 1604 * this instance (through lookups etc.) as conn_rq will go
1605 1605 * away shortly.
1606 1606 */
1607 1607 tcp_acceptor_hash_remove(tcp);
1608 1608
1609 1609 mutex_enter(&tcp->tcp_non_sq_lock);
1610 1610 if (tcp->tcp_flow_stopped) {
1611 1611 tcp_clrqfull(tcp);
1612 1612 }
1613 1613 mutex_exit(&tcp->tcp_non_sq_lock);
1614 1614
1615 1615 if (tcp->tcp_timer_tid != 0) {
1616 1616 delta = TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid);
1617 1617 tcp->tcp_timer_tid = 0;
1618 1618 }
1619 1619 /*
1620 1620 * Need to cancel those timers which will not be used when
1621 1621 * TCP is detached. This has to be done before the conn_wq
1622 1622 * is set to NULL.
1623 1623 */
1624 1624 tcp_timers_stop(tcp);
1625 1625
1626 1626 tcp->tcp_detached = B_TRUE;
1627 1627 if (tcp->tcp_state == TCPS_TIME_WAIT) {
1628 1628 tcp_time_wait_append(tcp);
1629 1629 TCP_DBGSTAT(tcps, tcp_detach_time_wait);
1630 1630 ASSERT(connp->conn_ref >=
1631 1631 (IPCL_IS_NONSTR(connp) ? 2 : 3));
1632 1632 goto finish;
1633 1633 }
1634 1634
1635 1635 /*
1636 1636 * If delta is zero the timer event wasn't executed and was
1637 1637 * successfully canceled. In this case we need to restart it
1638 1638 * with the minimal delta possible.
1639 1639 */
1640 1640 if (delta >= 0)
1641 1641 tcp->tcp_timer_tid = TCP_TIMER(tcp, tcp_timer,
1642 1642 delta ? delta : 1);
1643 1643
1644 1644 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3));
1645 1645 goto finish;
1646 1646 }
1647 1647
1648 1648 /* Detach did not complete. Still need to remove q from stream. */
1649 1649 if (msg) {
1650 1650 if (tcp->tcp_state == TCPS_ESTABLISHED ||
1651 1651 tcp->tcp_state == TCPS_CLOSE_WAIT)
1652 1652 TCPS_BUMP_MIB(tcps, tcpEstabResets);
1653 1653 if (tcp->tcp_state == TCPS_SYN_SENT ||
1654 1654 tcp->tcp_state == TCPS_SYN_RCVD)
1655 1655 TCPS_BUMP_MIB(tcps, tcpAttemptFails);
1656 1656 tcp_xmit_ctl(msg, tcp, tcp->tcp_snxt, 0, TH_RST);
1657 1657 }
1658 1658
1659 1659 tcp_closei_local(tcp);
1660 1660 CONN_DEC_REF(connp);
1661 1661 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 1 : 2));
1662 1662
1663 1663 finish:
1664 1664 /*
1665 1665 * Don't change the queues in the case of a listener that has
1666 1666 * eagers in its q or q0. It could surprise the eagers.
1667 1667 * Instead wait for the eagers outside the squeue.
1668 1668 *
1669 1669 * For non-STREAMS sockets tcp_wait_for_eagers implies that
|
↓ open down ↓ |
1634 lines elided |
↑ open up ↑ |
1670 1670 * we should delay the su_closed upcall until all eagers have
1671 1671 * dropped their references.
1672 1672 */
1673 1673 if (!tcp->tcp_wait_for_eagers) {
1674 1674 tcp->tcp_detached = B_TRUE;
1675 1675 connp->conn_rq = NULL;
1676 1676 connp->conn_wq = NULL;
1677 1677
1678 1678 /* non-STREAM socket, release the upper handle */
1679 1679 if (IPCL_IS_NONSTR(connp)) {
1680 - ASSERT(connp->conn_upper_handle != NULL);
1681 - (*connp->conn_upcalls->su_closed)
1682 - (connp->conn_upper_handle);
1680 + sock_upcalls_t *upcalls = connp->conn_upcalls;
1681 + sock_upper_handle_t handle = connp->conn_upper_handle;
1682 +
1683 + ASSERT(upcalls != NULL);
1684 + ASSERT(upcalls->su_closed != NULL);
1685 + ASSERT(handle != NULL);
1686 + /*
1687 + * Set these to NULL first because closed() will free
1688 + * upper structures. Acquire conn_lock because an
1689 + * external caller like conn_get_socket_info() will
1690 + * upcall if these are non-NULL.
1691 + */
1692 + mutex_enter(&connp->conn_lock);
1683 1693 connp->conn_upper_handle = NULL;
1684 1694 connp->conn_upcalls = NULL;
1695 + mutex_exit(&connp->conn_lock);
1696 + upcalls->su_closed(handle);
1685 1697 }
1686 1698 }
1687 1699
1688 1700 /* Signal tcp_close() to finish closing. */
1689 1701 mutex_enter(&tcp->tcp_closelock);
1690 1702 tcp->tcp_closed = 1;
1691 1703 cv_signal(&tcp->tcp_closecv);
1692 1704 mutex_exit(&tcp->tcp_closelock);
1693 1705 }
1694 1706
1695 1707 /* ARGSUSED */
1696 1708 void
1697 1709 tcp_shutdown_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1698 1710 {
1699 1711 conn_t *connp = (conn_t *)arg;
1700 1712 tcp_t *tcp = connp->conn_tcp;
1701 1713
1702 1714 freemsg(mp);
1703 1715
1704 1716 if (tcp->tcp_fused)
1705 1717 tcp_unfuse(tcp);
1706 1718
1707 1719 if (tcp_xmit_end(tcp) != 0) {
1708 1720 /*
1709 1721 * We were crossing FINs and got a reset from
1710 1722 * the other side. Just ignore it.
1711 1723 */
1712 1724 if (connp->conn_debug) {
1713 1725 (void) strlog(TCP_MOD_ID, 0, 1,
1714 1726 SL_ERROR|SL_TRACE,
1715 1727 "tcp_shutdown_output() out of state %s",
1716 1728 tcp_display(tcp, NULL, DISP_ADDR_AND_PORT));
1717 1729 }
1718 1730 }
1719 1731 }
1720 1732
1721 1733 #pragma inline(tcp_send_data)
1722 1734
1723 1735 void
1724 1736 tcp_send_data(tcp_t *tcp, mblk_t *mp)
1725 1737 {
1726 1738 conn_t *connp = tcp->tcp_connp;
1727 1739
1728 1740 /*
1729 1741 * Check here to avoid sending zero-copy message down to IP when
1730 1742 * ZEROCOPY capability has turned off. We only need to deal with
1731 1743 * the race condition between sockfs and the notification here.
1732 1744 * Since we have tried to backoff the tcp_xmit_head when turning
1733 1745 * zero-copy off and new messages in tcp_output(), we simply drop
1734 1746 * the dup'ed packet here and let tcp retransmit, if tcp_xmit_zc_clean
1735 1747 * is not true.
1736 1748 */
1737 1749 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on &&
1738 1750 !tcp->tcp_xmit_zc_clean) {
1739 1751 ip_drop_output("TCP ZC was disabled but not clean", mp, NULL);
1740 1752 freemsg(mp);
1741 1753 return;
1742 1754 }
1743 1755
1744 1756 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, connp->conn_ixa,
1745 1757 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, tcp,
1746 1758 __dtrace_tcp_tcph_t *,
1747 1759 &mp->b_rptr[connp->conn_ixa->ixa_ip_hdr_length]);
1748 1760
1749 1761 ASSERT(connp->conn_ixa->ixa_notify_cookie == connp->conn_tcp);
1750 1762 (void) conn_ip_output(mp, connp->conn_ixa);
1751 1763 }
1752 1764
1753 1765 /* ARGSUSED2 */
1754 1766 void
1755 1767 tcp_send_synack(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1756 1768 {
1757 1769 conn_t *econnp = (conn_t *)arg;
1758 1770 tcp_t *tcp = econnp->conn_tcp;
1759 1771 ip_xmit_attr_t *ixa = econnp->conn_ixa;
1760 1772
1761 1773 /* Guard against a RST having blown it away while on the squeue */
1762 1774 if (tcp->tcp_state == TCPS_CLOSED) {
1763 1775 freemsg(mp);
1764 1776 return;
1765 1777 }
1766 1778
1767 1779 /*
1768 1780 * In the off-chance that the eager received and responded to
1769 1781 * some other packet while the SYN|ACK was queued, we recalculate
1770 1782 * the ixa_pktlen. It would be better to fix the SYN/accept
1771 1783 * multithreading scheme to avoid this complexity.
1772 1784 */
1773 1785 ixa->ixa_pktlen = msgdsize(mp);
1774 1786 (void) conn_ip_output(mp, ixa);
1775 1787 }
1776 1788
1777 1789 /*
1778 1790 * tcp_send() is called by tcp_wput_data() and returns one of the following:
1779 1791 *
1780 1792 * -1 = failed allocation.
1781 1793 * 0 = We've either successfully sent data, or our usable send window is too
1782 1794 * small and we'd rather wait until later before sending again.
1783 1795 */
1784 1796 static int
1785 1797 tcp_send(tcp_t *tcp, const int mss, const int total_hdr_len,
1786 1798 const int tcp_hdr_len, const int num_sack_blk, int *usable,
1787 1799 uint32_t *snxt, int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time)
1788 1800 {
1789 1801 int num_lso_seg = 1;
1790 1802 uint_t lso_usable = 0;
1791 1803 boolean_t do_lso_send = B_FALSE;
1792 1804 tcp_stack_t *tcps = tcp->tcp_tcps;
1793 1805 conn_t *connp = tcp->tcp_connp;
1794 1806 ip_xmit_attr_t *ixa = connp->conn_ixa;
1795 1807
1796 1808 /*
1797 1809 * Check LSO possibility. The value of tcp->tcp_lso indicates whether
1798 1810 * the underlying connection is LSO capable. Will check whether having
1799 1811 * enough available data to initiate LSO transmission in the for(){}
1800 1812 * loops.
1801 1813 */
1802 1814 if (tcp->tcp_lso && (tcp->tcp_valid_bits & ~TCP_FSS_VALID) == 0)
1803 1815 do_lso_send = B_TRUE;
1804 1816
1805 1817 for (;;) {
1806 1818 struct datab *db;
1807 1819 tcpha_t *tcpha;
1808 1820 uint32_t sum;
1809 1821 mblk_t *mp, *mp1;
1810 1822 uchar_t *rptr;
1811 1823 int len;
1812 1824
1813 1825 /*
1814 1826 * Calculate the maximum payload length we can send at one
1815 1827 * time.
1816 1828 */
1817 1829 if (do_lso_send) {
1818 1830 /*
1819 1831 * Determine whether or not it's possible to do LSO,
1820 1832 * and if so, how much data we can send.
1821 1833 */
1822 1834 if ((*usable - 1) / mss >= 1) {
1823 1835 lso_usable = MIN(tcp->tcp_lso_max, *usable);
1824 1836 num_lso_seg = lso_usable / mss;
1825 1837 if (lso_usable % mss) {
1826 1838 num_lso_seg++;
1827 1839 tcp->tcp_last_sent_len = (ushort_t)
1828 1840 (lso_usable % mss);
1829 1841 } else {
1830 1842 tcp->tcp_last_sent_len = (ushort_t)mss;
1831 1843 }
1832 1844 } else {
1833 1845 do_lso_send = B_FALSE;
1834 1846 num_lso_seg = 1;
1835 1847 lso_usable = mss;
1836 1848 }
1837 1849 }
1838 1850
1839 1851 ASSERT(num_lso_seg <= IP_MAXPACKET / mss + 1);
1840 1852
1841 1853 len = mss;
1842 1854 if (len > *usable) {
1843 1855 ASSERT(do_lso_send == B_FALSE);
1844 1856
1845 1857 len = *usable;
1846 1858 if (len <= 0) {
1847 1859 /* Terminate the loop */
1848 1860 break; /* success; too small */
1849 1861 }
1850 1862 /*
1851 1863 * Sender silly-window avoidance.
1852 1864 * Ignore this if we are going to send a
1853 1865 * zero window probe out.
1854 1866 *
1855 1867 * TODO: force data into microscopic window?
1856 1868 * ==> (!pushed || (unsent > usable))
1857 1869 */
1858 1870 if (len < (tcp->tcp_max_swnd >> 1) &&
1859 1871 (tcp->tcp_unsent - (*snxt - tcp->tcp_snxt)) > len &&
1860 1872 !((tcp->tcp_valid_bits & TCP_URG_VALID) &&
1861 1873 len == 1) && (! tcp->tcp_zero_win_probe)) {
1862 1874 /*
1863 1875 * If the retransmit timer is not running
1864 1876 * we start it so that we will retransmit
1865 1877 * in the case when the receiver has
1866 1878 * decremented the window.
1867 1879 */
1868 1880 if (*snxt == tcp->tcp_snxt &&
1869 1881 *snxt == tcp->tcp_suna) {
1870 1882 /*
1871 1883 * We are not supposed to send
1872 1884 * anything. So let's wait a little
1873 1885 * bit longer before breaking SWS
1874 1886 * avoidance.
1875 1887 *
1876 1888 * What should the value be?
1877 1889 * Suggestion: MAX(init rexmit time,
1878 1890 * tcp->tcp_rto)
1879 1891 */
1880 1892 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1881 1893 }
1882 1894 break; /* success; too small */
1883 1895 }
1884 1896 }
1885 1897
1886 1898 tcpha = tcp->tcp_tcpha;
1887 1899
1888 1900 /*
1889 1901 * The reason to adjust len here is that we need to set flags
1890 1902 * and calculate checksum.
1891 1903 */
1892 1904 if (do_lso_send)
1893 1905 len = lso_usable;
1894 1906
1895 1907 *usable -= len; /* Approximate - can be adjusted later */
1896 1908 if (*usable > 0)
1897 1909 tcpha->tha_flags = TH_ACK;
1898 1910 else
1899 1911 tcpha->tha_flags = (TH_ACK | TH_PUSH);
1900 1912
1901 1913 /*
1902 1914 * Prime pump for IP's checksumming on our behalf.
1903 1915 * Include the adjustment for a source route if any.
1904 1916 * In case of LSO, the partial pseudo-header checksum should
1905 1917 * exclusive TCP length, so zero tha_sum before IP calculate
1906 1918 * pseudo-header checksum for partial checksum offload.
1907 1919 */
1908 1920 if (do_lso_send) {
1909 1921 sum = 0;
1910 1922 } else {
1911 1923 sum = len + tcp_hdr_len + connp->conn_sum;
1912 1924 sum = (sum >> 16) + (sum & 0xFFFF);
1913 1925 }
1914 1926 tcpha->tha_sum = htons(sum);
1915 1927 tcpha->tha_seq = htonl(*snxt);
1916 1928
1917 1929 /*
1918 1930 * Branch off to tcp_xmit_mp() if any of the VALID bits is
1919 1931 * set. For the case when TCP_FSS_VALID is the only valid
1920 1932 * bit (normal active close), branch off only when we think
1921 1933 * that the FIN flag needs to be set. Note for this case,
1922 1934 * that (snxt + len) may not reflect the actual seg_len,
1923 1935 * as len may be further reduced in tcp_xmit_mp(). If len
1924 1936 * gets modified, we will end up here again.
1925 1937 */
1926 1938 if (tcp->tcp_valid_bits != 0 &&
1927 1939 (tcp->tcp_valid_bits != TCP_FSS_VALID ||
1928 1940 ((*snxt + len) == tcp->tcp_fss))) {
1929 1941 uchar_t *prev_rptr;
1930 1942 uint32_t prev_snxt = tcp->tcp_snxt;
1931 1943
1932 1944 if (*tail_unsent == 0) {
1933 1945 ASSERT((*xmit_tail)->b_cont != NULL);
1934 1946 *xmit_tail = (*xmit_tail)->b_cont;
1935 1947 prev_rptr = (*xmit_tail)->b_rptr;
1936 1948 *tail_unsent = (int)((*xmit_tail)->b_wptr -
1937 1949 (*xmit_tail)->b_rptr);
1938 1950 } else {
1939 1951 prev_rptr = (*xmit_tail)->b_rptr;
1940 1952 (*xmit_tail)->b_rptr = (*xmit_tail)->b_wptr -
1941 1953 *tail_unsent;
1942 1954 }
1943 1955 mp = tcp_xmit_mp(tcp, *xmit_tail, len, NULL, NULL,
1944 1956 *snxt, B_FALSE, (uint32_t *)&len, B_FALSE);
1945 1957 /* Restore tcp_snxt so we get amount sent right. */
1946 1958 tcp->tcp_snxt = prev_snxt;
1947 1959 if (prev_rptr == (*xmit_tail)->b_rptr) {
1948 1960 /*
1949 1961 * If the previous timestamp is still in use,
1950 1962 * don't stomp on it.
1951 1963 */
1952 1964 if ((*xmit_tail)->b_next == NULL) {
1953 1965 (*xmit_tail)->b_prev = local_time;
1954 1966 (*xmit_tail)->b_next =
1955 1967 (mblk_t *)(uintptr_t)(*snxt);
1956 1968 }
1957 1969 } else
1958 1970 (*xmit_tail)->b_rptr = prev_rptr;
1959 1971
1960 1972 if (mp == NULL) {
1961 1973 return (-1);
1962 1974 }
1963 1975 mp1 = mp->b_cont;
1964 1976
1965 1977 if (len <= mss) /* LSO is unusable (!do_lso_send) */
1966 1978 tcp->tcp_last_sent_len = (ushort_t)len;
1967 1979 while (mp1->b_cont) {
1968 1980 *xmit_tail = (*xmit_tail)->b_cont;
1969 1981 (*xmit_tail)->b_prev = local_time;
1970 1982 (*xmit_tail)->b_next =
1971 1983 (mblk_t *)(uintptr_t)(*snxt);
1972 1984 mp1 = mp1->b_cont;
1973 1985 }
1974 1986 *snxt += len;
1975 1987 *tail_unsent = (*xmit_tail)->b_wptr - mp1->b_wptr;
1976 1988 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1977 1989 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1978 1990 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1979 1991 tcp->tcp_cs.tcp_out_data_segs++;
1980 1992 tcp->tcp_cs.tcp_out_data_bytes += len;
1981 1993 tcp_send_data(tcp, mp);
1982 1994 continue;
1983 1995 }
1984 1996
1985 1997 *snxt += len; /* Adjust later if we don't send all of len */
1986 1998 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1987 1999 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1988 2000 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1989 2001 tcp->tcp_cs.tcp_out_data_segs++;
1990 2002 tcp->tcp_cs.tcp_out_data_bytes += len;
1991 2003
1992 2004 if (*tail_unsent) {
1993 2005 /* Are the bytes above us in flight? */
1994 2006 rptr = (*xmit_tail)->b_wptr - *tail_unsent;
1995 2007 if (rptr != (*xmit_tail)->b_rptr) {
1996 2008 *tail_unsent -= len;
1997 2009 if (len <= mss) /* LSO is unusable */
1998 2010 tcp->tcp_last_sent_len = (ushort_t)len;
1999 2011 len += total_hdr_len;
2000 2012 ixa->ixa_pktlen = len;
2001 2013
2002 2014 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2003 2015 tcp->tcp_ipha->ipha_length = htons(len);
2004 2016 } else {
2005 2017 tcp->tcp_ip6h->ip6_plen =
2006 2018 htons(len - IPV6_HDR_LEN);
2007 2019 }
2008 2020
2009 2021 mp = dupb(*xmit_tail);
2010 2022 if (mp == NULL) {
2011 2023 return (-1); /* out_of_mem */
2012 2024 }
2013 2025 mp->b_rptr = rptr;
2014 2026 /*
2015 2027 * If the old timestamp is no longer in use,
2016 2028 * sample a new timestamp now.
2017 2029 */
2018 2030 if ((*xmit_tail)->b_next == NULL) {
2019 2031 (*xmit_tail)->b_prev = local_time;
2020 2032 (*xmit_tail)->b_next =
2021 2033 (mblk_t *)(uintptr_t)(*snxt-len);
2022 2034 }
2023 2035 goto must_alloc;
2024 2036 }
2025 2037 } else {
2026 2038 *xmit_tail = (*xmit_tail)->b_cont;
2027 2039 ASSERT((uintptr_t)((*xmit_tail)->b_wptr -
2028 2040 (*xmit_tail)->b_rptr) <= (uintptr_t)INT_MAX);
2029 2041 *tail_unsent = (int)((*xmit_tail)->b_wptr -
2030 2042 (*xmit_tail)->b_rptr);
2031 2043 }
2032 2044
2033 2045 (*xmit_tail)->b_prev = local_time;
2034 2046 (*xmit_tail)->b_next = (mblk_t *)(uintptr_t)(*snxt - len);
2035 2047
2036 2048 *tail_unsent -= len;
2037 2049 if (len <= mss) /* LSO is unusable (!do_lso_send) */
2038 2050 tcp->tcp_last_sent_len = (ushort_t)len;
2039 2051
2040 2052 len += total_hdr_len;
2041 2053 ixa->ixa_pktlen = len;
2042 2054
2043 2055 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2044 2056 tcp->tcp_ipha->ipha_length = htons(len);
2045 2057 } else {
2046 2058 tcp->tcp_ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2047 2059 }
2048 2060
2049 2061 mp = dupb(*xmit_tail);
2050 2062 if (mp == NULL) {
2051 2063 return (-1); /* out_of_mem */
2052 2064 }
2053 2065
2054 2066 len = total_hdr_len;
2055 2067 /*
2056 2068 * There are four reasons to allocate a new hdr mblk:
2057 2069 * 1) The bytes above us are in use by another packet
2058 2070 * 2) We don't have good alignment
2059 2071 * 3) The mblk is being shared
2060 2072 * 4) We don't have enough room for a header
2061 2073 */
2062 2074 rptr = mp->b_rptr - len;
2063 2075 if (!OK_32PTR(rptr) ||
2064 2076 ((db = mp->b_datap), db->db_ref != 2) ||
2065 2077 rptr < db->db_base) {
2066 2078 /* NOTE: we assume allocb returns an OK_32PTR */
2067 2079
2068 2080 must_alloc:;
2069 2081 mp1 = allocb(connp->conn_ht_iphc_allocated +
2070 2082 tcps->tcps_wroff_xtra, BPRI_MED);
2071 2083 if (mp1 == NULL) {
2072 2084 freemsg(mp);
2073 2085 return (-1); /* out_of_mem */
2074 2086 }
2075 2087 mp1->b_cont = mp;
2076 2088 mp = mp1;
2077 2089 /* Leave room for Link Level header */
2078 2090 len = total_hdr_len;
2079 2091 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2080 2092 mp->b_wptr = &rptr[len];
2081 2093 }
2082 2094
2083 2095 /*
2084 2096 * Fill in the header using the template header, and add
2085 2097 * options such as time-stamp, ECN and/or SACK, as needed.
2086 2098 */
2087 2099 tcp_fill_header(tcp, rptr, num_sack_blk);
2088 2100
2089 2101 mp->b_rptr = rptr;
2090 2102
2091 2103 if (*tail_unsent) {
2092 2104 int spill = *tail_unsent;
2093 2105
2094 2106 mp1 = mp->b_cont;
2095 2107 if (mp1 == NULL)
2096 2108 mp1 = mp;
2097 2109
2098 2110 /*
2099 2111 * If we're a little short, tack on more mblks until
2100 2112 * there is no more spillover.
2101 2113 */
2102 2114 while (spill < 0) {
2103 2115 mblk_t *nmp;
2104 2116 int nmpsz;
2105 2117
2106 2118 nmp = (*xmit_tail)->b_cont;
2107 2119 nmpsz = MBLKL(nmp);
2108 2120
2109 2121 /*
2110 2122 * Excess data in mblk; can we split it?
2111 2123 * If LSO is enabled for the connection,
2112 2124 * keep on splitting as this is a transient
2113 2125 * send path.
2114 2126 */
2115 2127 if (!do_lso_send && (spill + nmpsz > 0)) {
2116 2128 /*
2117 2129 * Don't split if stream head was
2118 2130 * told to break up larger writes
2119 2131 * into smaller ones.
2120 2132 */
2121 2133 if (tcp->tcp_maxpsz_multiplier > 0)
2122 2134 break;
2123 2135
2124 2136 /*
2125 2137 * Next mblk is less than SMSS/2
2126 2138 * rounded up to nearest 64-byte;
2127 2139 * let it get sent as part of the
2128 2140 * next segment.
2129 2141 */
2130 2142 if (tcp->tcp_localnet &&
2131 2143 !tcp->tcp_cork &&
2132 2144 (nmpsz < roundup((mss >> 1), 64)))
2133 2145 break;
2134 2146 }
2135 2147
2136 2148 *xmit_tail = nmp;
2137 2149 ASSERT((uintptr_t)nmpsz <= (uintptr_t)INT_MAX);
2138 2150 /* Stash for rtt use later */
2139 2151 (*xmit_tail)->b_prev = local_time;
2140 2152 (*xmit_tail)->b_next =
2141 2153 (mblk_t *)(uintptr_t)(*snxt - len);
2142 2154 mp1->b_cont = dupb(*xmit_tail);
2143 2155 mp1 = mp1->b_cont;
2144 2156
2145 2157 spill += nmpsz;
2146 2158 if (mp1 == NULL) {
2147 2159 *tail_unsent = spill;
2148 2160 freemsg(mp);
2149 2161 return (-1); /* out_of_mem */
2150 2162 }
2151 2163 }
2152 2164
2153 2165 /* Trim back any surplus on the last mblk */
2154 2166 if (spill >= 0) {
2155 2167 mp1->b_wptr -= spill;
2156 2168 *tail_unsent = spill;
2157 2169 } else {
2158 2170 /*
2159 2171 * We did not send everything we could in
2160 2172 * order to remain within the b_cont limit.
2161 2173 */
2162 2174 *usable -= spill;
2163 2175 *snxt += spill;
2164 2176 tcp->tcp_last_sent_len += spill;
2165 2177 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, spill);
2166 2178 tcp->tcp_cs.tcp_out_data_bytes += spill;
2167 2179 /*
2168 2180 * Adjust the checksum
2169 2181 */
2170 2182 tcpha = (tcpha_t *)(rptr +
2171 2183 ixa->ixa_ip_hdr_length);
2172 2184 sum += spill;
2173 2185 sum = (sum >> 16) + (sum & 0xFFFF);
2174 2186 tcpha->tha_sum = htons(sum);
2175 2187 if (connp->conn_ipversion == IPV4_VERSION) {
2176 2188 sum = ntohs(
2177 2189 ((ipha_t *)rptr)->ipha_length) +
2178 2190 spill;
2179 2191 ((ipha_t *)rptr)->ipha_length =
2180 2192 htons(sum);
2181 2193 } else {
2182 2194 sum = ntohs(
2183 2195 ((ip6_t *)rptr)->ip6_plen) +
2184 2196 spill;
2185 2197 ((ip6_t *)rptr)->ip6_plen =
2186 2198 htons(sum);
2187 2199 }
2188 2200 ixa->ixa_pktlen += spill;
2189 2201 *tail_unsent = 0;
2190 2202 }
2191 2203 }
2192 2204 if (tcp->tcp_ip_forward_progress) {
2193 2205 tcp->tcp_ip_forward_progress = B_FALSE;
2194 2206 ixa->ixa_flags |= IXAF_REACH_CONF;
2195 2207 } else {
2196 2208 ixa->ixa_flags &= ~IXAF_REACH_CONF;
2197 2209 }
2198 2210
2199 2211 if (do_lso_send) {
2200 2212 /* Append LSO information to the mp. */
2201 2213 lso_info_set(mp, mss, HW_LSO);
2202 2214 ixa->ixa_fragsize = IP_MAXPACKET;
2203 2215 ixa->ixa_extra_ident = num_lso_seg - 1;
2204 2216
2205 2217 DTRACE_PROBE2(tcp_send_lso, int, num_lso_seg,
2206 2218 boolean_t, B_TRUE);
2207 2219
2208 2220 tcp_send_data(tcp, mp);
2209 2221
2210 2222 /*
2211 2223 * Restore values of ixa_fragsize and ixa_extra_ident.
2212 2224 */
2213 2225 ixa->ixa_fragsize = ixa->ixa_pmtu;
2214 2226 ixa->ixa_extra_ident = 0;
2215 2227 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2216 2228 TCP_STAT(tcps, tcp_lso_times);
2217 2229 TCP_STAT_UPDATE(tcps, tcp_lso_pkt_out, num_lso_seg);
2218 2230 } else {
2219 2231 /*
2220 2232 * Make sure to clean up LSO information. Wherever a
2221 2233 * new mp uses the prepended header room after dupb(),
2222 2234 * lso_info_cleanup() should be called.
2223 2235 */
2224 2236 lso_info_cleanup(mp);
2225 2237 tcp_send_data(tcp, mp);
2226 2238 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2227 2239 }
2228 2240 }
2229 2241
2230 2242 return (0);
2231 2243 }
2232 2244
2233 2245 /*
2234 2246 * Initiate closedown sequence on an active connection. (May be called as
2235 2247 * writer.) Return value zero for OK return, non-zero for error return.
2236 2248 */
2237 2249 static int
2238 2250 tcp_xmit_end(tcp_t *tcp)
2239 2251 {
2240 2252 mblk_t *mp;
2241 2253 tcp_stack_t *tcps = tcp->tcp_tcps;
2242 2254 iulp_t uinfo;
2243 2255 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip;
2244 2256 conn_t *connp = tcp->tcp_connp;
2245 2257
2246 2258 if (tcp->tcp_state < TCPS_SYN_RCVD ||
2247 2259 tcp->tcp_state > TCPS_CLOSE_WAIT) {
2248 2260 /*
2249 2261 * Invalid state, only states TCPS_SYN_RCVD,
2250 2262 * TCPS_ESTABLISHED and TCPS_CLOSE_WAIT are valid
2251 2263 */
2252 2264 return (-1);
2253 2265 }
2254 2266
2255 2267 tcp->tcp_fss = tcp->tcp_snxt + tcp->tcp_unsent;
2256 2268 tcp->tcp_valid_bits |= TCP_FSS_VALID;
2257 2269 /*
2258 2270 * If there is nothing more unsent, send the FIN now.
2259 2271 * Otherwise, it will go out with the last segment.
2260 2272 */
2261 2273 if (tcp->tcp_unsent == 0) {
2262 2274 mp = tcp_xmit_mp(tcp, NULL, 0, NULL, NULL,
2263 2275 tcp->tcp_fss, B_FALSE, NULL, B_FALSE);
2264 2276
2265 2277 if (mp) {
2266 2278 tcp_send_data(tcp, mp);
2267 2279 } else {
2268 2280 /*
2269 2281 * Couldn't allocate msg. Pretend we got it out.
2270 2282 * Wait for rexmit timeout.
2271 2283 */
2272 2284 tcp->tcp_snxt = tcp->tcp_fss + 1;
2273 2285 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2274 2286 }
2275 2287
2276 2288 /*
2277 2289 * If needed, update tcp_rexmit_snxt as tcp_snxt is
2278 2290 * changed.
2279 2291 */
2280 2292 if (tcp->tcp_rexmit && tcp->tcp_rexmit_nxt == tcp->tcp_fss) {
2281 2293 tcp->tcp_rexmit_nxt = tcp->tcp_snxt;
2282 2294 }
2283 2295 } else {
2284 2296 /*
2285 2297 * If tcp->tcp_cork is set, then the data will not get sent,
2286 2298 * so we have to check that and unset it first.
2287 2299 */
2288 2300 if (tcp->tcp_cork)
2289 2301 tcp->tcp_cork = B_FALSE;
2290 2302 tcp_wput_data(tcp, NULL, B_FALSE);
2291 2303 }
2292 2304
2293 2305 /*
2294 2306 * If TCP does not get enough samples of RTT or tcp_rtt_updates
2295 2307 * is 0, don't update the cache.
2296 2308 */
2297 2309 if (tcps->tcps_rtt_updates == 0 ||
2298 2310 tcp->tcp_rtt_update < tcps->tcps_rtt_updates)
2299 2311 return (0);
2300 2312
2301 2313 /*
2302 2314 * We do not have a good algorithm to update ssthresh at this time.
2303 2315 * So don't do any update.
2304 2316 */
2305 2317 bzero(&uinfo, sizeof (uinfo));
2306 2318 uinfo.iulp_rtt = NSEC2MSEC(tcp->tcp_rtt_sa);
2307 2319 uinfo.iulp_rtt_sd = NSEC2MSEC(tcp->tcp_rtt_sd);
2308 2320
2309 2321 /*
2310 2322 * Note that uinfo is kept for conn_faddr in the DCE. Could update even
2311 2323 * if source routed but we don't.
2312 2324 */
2313 2325 if (connp->conn_ipversion == IPV4_VERSION) {
2314 2326 if (connp->conn_faddr_v4 != tcp->tcp_ipha->ipha_dst) {
2315 2327 return (0);
2316 2328 }
2317 2329 (void) dce_update_uinfo_v4(connp->conn_faddr_v4, &uinfo, ipst);
2318 2330 } else {
2319 2331 uint_t ifindex;
2320 2332
2321 2333 if (!(IN6_ARE_ADDR_EQUAL(&connp->conn_faddr_v6,
2322 2334 &tcp->tcp_ip6h->ip6_dst))) {
2323 2335 return (0);
2324 2336 }
2325 2337 ifindex = 0;
2326 2338 if (IN6_IS_ADDR_LINKSCOPE(&connp->conn_faddr_v6)) {
2327 2339 ip_xmit_attr_t *ixa = connp->conn_ixa;
2328 2340
2329 2341 /*
2330 2342 * If we are going to create a DCE we'd better have
2331 2343 * an ifindex
2332 2344 */
2333 2345 if (ixa->ixa_nce != NULL) {
2334 2346 ifindex = ixa->ixa_nce->nce_common->ncec_ill->
2335 2347 ill_phyint->phyint_ifindex;
2336 2348 } else {
2337 2349 return (0);
2338 2350 }
2339 2351 }
2340 2352
2341 2353 (void) dce_update_uinfo(&connp->conn_faddr_v6, ifindex, &uinfo,
2342 2354 ipst);
2343 2355 }
2344 2356 return (0);
2345 2357 }
2346 2358
2347 2359 /*
2348 2360 * Send out a control packet on the tcp connection specified. This routine
2349 2361 * is typically called where we need a simple ACK or RST generated.
2350 2362 */
2351 2363 void
2352 2364 tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq, uint32_t ack, int ctl)
2353 2365 {
2354 2366 uchar_t *rptr;
2355 2367 tcpha_t *tcpha;
2356 2368 ipha_t *ipha = NULL;
2357 2369 ip6_t *ip6h = NULL;
2358 2370 uint32_t sum;
2359 2371 int total_hdr_len;
2360 2372 int ip_hdr_len;
2361 2373 mblk_t *mp;
2362 2374 tcp_stack_t *tcps = tcp->tcp_tcps;
2363 2375 conn_t *connp = tcp->tcp_connp;
2364 2376 ip_xmit_attr_t *ixa = connp->conn_ixa;
2365 2377
2366 2378 /*
2367 2379 * Save sum for use in source route later.
2368 2380 */
2369 2381 sum = connp->conn_ht_ulp_len + connp->conn_sum;
2370 2382 total_hdr_len = connp->conn_ht_iphc_len;
2371 2383 ip_hdr_len = ixa->ixa_ip_hdr_length;
2372 2384
2373 2385 /* If a text string is passed in with the request, pass it to strlog. */
2374 2386 if (str != NULL && connp->conn_debug) {
2375 2387 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2376 2388 "tcp_xmit_ctl: '%s', seq 0x%x, ack 0x%x, ctl 0x%x",
2377 2389 str, seq, ack, ctl);
2378 2390 }
2379 2391 mp = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
2380 2392 BPRI_MED);
2381 2393 if (mp == NULL) {
2382 2394 return;
2383 2395 }
2384 2396 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2385 2397 mp->b_rptr = rptr;
2386 2398 mp->b_wptr = &rptr[total_hdr_len];
2387 2399 bcopy(connp->conn_ht_iphc, rptr, total_hdr_len);
2388 2400
2389 2401 ixa->ixa_pktlen = total_hdr_len;
2390 2402
2391 2403 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2392 2404 ipha = (ipha_t *)rptr;
2393 2405 ipha->ipha_length = htons(total_hdr_len);
2394 2406 } else {
2395 2407 ip6h = (ip6_t *)rptr;
2396 2408 ip6h->ip6_plen = htons(total_hdr_len - IPV6_HDR_LEN);
2397 2409 }
2398 2410 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2399 2411 tcpha->tha_flags = (uint8_t)ctl;
2400 2412 if (ctl & TH_RST) {
2401 2413 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2402 2414 TCPS_BUMP_MIB(tcps, tcpOutControl);
2403 2415 /*
2404 2416 * Don't send TSopt w/ TH_RST packets per RFC 1323.
2405 2417 */
2406 2418 if (tcp->tcp_snd_ts_ok &&
2407 2419 tcp->tcp_state > TCPS_SYN_SENT) {
2408 2420 mp->b_wptr = &rptr[total_hdr_len - TCPOPT_REAL_TS_LEN];
2409 2421 *(mp->b_wptr) = TCPOPT_EOL;
2410 2422
2411 2423 ixa->ixa_pktlen = total_hdr_len - TCPOPT_REAL_TS_LEN;
2412 2424
2413 2425 if (connp->conn_ipversion == IPV4_VERSION) {
2414 2426 ipha->ipha_length = htons(total_hdr_len -
2415 2427 TCPOPT_REAL_TS_LEN);
2416 2428 } else {
2417 2429 ip6h->ip6_plen = htons(total_hdr_len -
2418 2430 IPV6_HDR_LEN - TCPOPT_REAL_TS_LEN);
2419 2431 }
2420 2432 tcpha->tha_offset_and_reserved -= (3 << 4);
2421 2433 sum -= TCPOPT_REAL_TS_LEN;
2422 2434 }
2423 2435 }
2424 2436 if (ctl & TH_ACK) {
2425 2437 if (tcp->tcp_snd_ts_ok) {
2426 2438 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2427 2439
2428 2440 U32_TO_BE32(llbolt,
2429 2441 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
2430 2442 U32_TO_BE32(tcp->tcp_ts_recent,
2431 2443 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
2432 2444 }
2433 2445
2434 2446 /* Update the latest receive window size in TCP header. */
2435 2447 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
2436 2448 /* Track what we sent to the peer */
2437 2449 tcp->tcp_tcpha->tha_win = tcpha->tha_win;
2438 2450 tcp->tcp_rack = ack;
2439 2451 tcp->tcp_rack_cnt = 0;
2440 2452 TCPS_BUMP_MIB(tcps, tcpOutAck);
2441 2453 }
2442 2454 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2443 2455 tcpha->tha_seq = htonl(seq);
2444 2456 tcpha->tha_ack = htonl(ack);
2445 2457 /*
2446 2458 * Include the adjustment for a source route if any.
2447 2459 */
2448 2460 sum = (sum >> 16) + (sum & 0xFFFF);
2449 2461 tcpha->tha_sum = htons(sum);
2450 2462 tcp_send_data(tcp, mp);
2451 2463 }
2452 2464
2453 2465 /*
2454 2466 * Generate a reset based on an inbound packet, connp is set by caller
2455 2467 * when RST is in response to an unexpected inbound packet for which
2456 2468 * there is active tcp state in the system.
2457 2469 *
2458 2470 * IPSEC NOTE : Try to send the reply with the same protection as it came
2459 2471 * in. We have the ip_recv_attr_t which is reversed to form the ip_xmit_attr_t.
2460 2472 * That way the packet will go out at the same level of protection as it
2461 2473 * came in with.
2462 2474 */
2463 2475 static void
2464 2476 tcp_xmit_early_reset(char *str, mblk_t *mp, uint32_t seq, uint32_t ack, int ctl,
2465 2477 ip_recv_attr_t *ira, ip_stack_t *ipst, conn_t *connp)
2466 2478 {
2467 2479 ipha_t *ipha = NULL;
2468 2480 ip6_t *ip6h = NULL;
2469 2481 ushort_t len;
2470 2482 tcpha_t *tcpha;
2471 2483 int i;
2472 2484 ipaddr_t v4addr;
2473 2485 in6_addr_t v6addr;
2474 2486 netstack_t *ns = ipst->ips_netstack;
2475 2487 tcp_stack_t *tcps = ns->netstack_tcp;
2476 2488 ip_xmit_attr_t ixas, *ixa;
2477 2489 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2478 2490 boolean_t need_refrele = B_FALSE; /* ixa_refrele(ixa) */
2479 2491 ushort_t port;
2480 2492
2481 2493 if (!tcp_send_rst_chk(tcps)) {
2482 2494 TCP_STAT(tcps, tcp_rst_unsent);
2483 2495 freemsg(mp);
2484 2496 return;
2485 2497 }
2486 2498
2487 2499 /*
2488 2500 * If connp != NULL we use conn_ixa to keep IP_NEXTHOP and other
2489 2501 * options from the listener. In that case the caller must ensure that
2490 2502 * we are running on the listener = connp squeue.
2491 2503 *
2492 2504 * We get a safe copy of conn_ixa so we don't need to restore anything
2493 2505 * we or ip_output_simple might change in the ixa.
2494 2506 */
2495 2507 if (connp != NULL) {
2496 2508 ASSERT(connp->conn_on_sqp);
2497 2509
2498 2510 ixa = conn_get_ixa_exclusive(connp);
2499 2511 if (ixa == NULL) {
2500 2512 TCP_STAT(tcps, tcp_rst_unsent);
2501 2513 freemsg(mp);
2502 2514 return;
2503 2515 }
2504 2516 need_refrele = B_TRUE;
2505 2517 } else {
2506 2518 bzero(&ixas, sizeof (ixas));
2507 2519 ixa = &ixas;
2508 2520 /*
2509 2521 * IXAF_VERIFY_SOURCE is overkill since we know the
2510 2522 * packet was for us.
2511 2523 */
2512 2524 ixa->ixa_flags |= IXAF_SET_ULP_CKSUM | IXAF_VERIFY_SOURCE;
2513 2525 ixa->ixa_protocol = IPPROTO_TCP;
2514 2526 ixa->ixa_zoneid = ira->ira_zoneid;
2515 2527 ixa->ixa_ifindex = 0;
2516 2528 ixa->ixa_ipst = ipst;
2517 2529 ixa->ixa_cred = kcred;
2518 2530 ixa->ixa_cpid = NOPID;
2519 2531 }
2520 2532
2521 2533 if (str && tcps->tcps_dbg) {
2522 2534 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2523 2535 "tcp_xmit_early_reset: '%s', seq 0x%x, ack 0x%x, "
2524 2536 "flags 0x%x",
2525 2537 str, seq, ack, ctl);
2526 2538 }
2527 2539 if (mp->b_datap->db_ref != 1) {
2528 2540 mblk_t *mp1 = copyb(mp);
2529 2541 freemsg(mp);
2530 2542 mp = mp1;
2531 2543 if (mp == NULL)
2532 2544 goto done;
2533 2545 } else if (mp->b_cont) {
2534 2546 freemsg(mp->b_cont);
2535 2547 mp->b_cont = NULL;
2536 2548 DB_CKSUMFLAGS(mp) = 0;
2537 2549 }
2538 2550 /*
2539 2551 * We skip reversing source route here.
2540 2552 * (for now we replace all IP options with EOL)
2541 2553 */
2542 2554 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2543 2555 ipha = (ipha_t *)mp->b_rptr;
2544 2556 for (i = IP_SIMPLE_HDR_LENGTH; i < (int)ip_hdr_len; i++)
2545 2557 mp->b_rptr[i] = IPOPT_EOL;
2546 2558 /*
2547 2559 * Make sure that src address isn't flagrantly invalid.
2548 2560 * Not all broadcast address checking for the src address
2549 2561 * is possible, since we don't know the netmask of the src
2550 2562 * addr. No check for destination address is done, since
2551 2563 * IP will not pass up a packet with a broadcast dest
2552 2564 * address to TCP. Similar checks are done below for IPv6.
2553 2565 */
2554 2566 if (ipha->ipha_src == 0 || ipha->ipha_src == INADDR_BROADCAST ||
2555 2567 CLASSD(ipha->ipha_src)) {
2556 2568 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsInDiscards);
2557 2569 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2558 2570 freemsg(mp);
2559 2571 goto done;
2560 2572 }
2561 2573 } else {
2562 2574 ip6h = (ip6_t *)mp->b_rptr;
2563 2575
2564 2576 if (IN6_IS_ADDR_UNSPECIFIED(&ip6h->ip6_src) ||
2565 2577 IN6_IS_ADDR_MULTICAST(&ip6h->ip6_src)) {
2566 2578 BUMP_MIB(&ipst->ips_ip6_mib, ipIfStatsInDiscards);
2567 2579 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2568 2580 freemsg(mp);
2569 2581 goto done;
2570 2582 }
2571 2583
2572 2584 /* Remove any extension headers assuming partial overlay */
2573 2585 if (ip_hdr_len > IPV6_HDR_LEN) {
2574 2586 uint8_t *to;
2575 2587
2576 2588 to = mp->b_rptr + ip_hdr_len - IPV6_HDR_LEN;
2577 2589 ovbcopy(ip6h, to, IPV6_HDR_LEN);
2578 2590 mp->b_rptr += ip_hdr_len - IPV6_HDR_LEN;
2579 2591 ip_hdr_len = IPV6_HDR_LEN;
2580 2592 ip6h = (ip6_t *)mp->b_rptr;
2581 2593 ip6h->ip6_nxt = IPPROTO_TCP;
2582 2594 }
2583 2595 }
2584 2596 tcpha = (tcpha_t *)&mp->b_rptr[ip_hdr_len];
2585 2597 if (tcpha->tha_flags & TH_RST) {
2586 2598 freemsg(mp);
2587 2599 goto done;
2588 2600 }
2589 2601 tcpha->tha_offset_and_reserved = (5 << 4);
2590 2602 len = ip_hdr_len + sizeof (tcpha_t);
2591 2603 mp->b_wptr = &mp->b_rptr[len];
2592 2604 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2593 2605 ipha->ipha_length = htons(len);
2594 2606 /* Swap addresses */
2595 2607 v4addr = ipha->ipha_src;
2596 2608 ipha->ipha_src = ipha->ipha_dst;
2597 2609 ipha->ipha_dst = v4addr;
2598 2610 ipha->ipha_ident = 0;
2599 2611 ipha->ipha_ttl = (uchar_t)tcps->tcps_ipv4_ttl;
2600 2612 ixa->ixa_flags |= IXAF_IS_IPV4;
2601 2613 ixa->ixa_ip_hdr_length = ip_hdr_len;
2602 2614 } else {
2603 2615 ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2604 2616 /* Swap addresses */
2605 2617 v6addr = ip6h->ip6_src;
2606 2618 ip6h->ip6_src = ip6h->ip6_dst;
2607 2619 ip6h->ip6_dst = v6addr;
2608 2620 ip6h->ip6_hops = (uchar_t)tcps->tcps_ipv6_hoplimit;
2609 2621 ixa->ixa_flags &= ~IXAF_IS_IPV4;
2610 2622
2611 2623 if (IN6_IS_ADDR_LINKSCOPE(&ip6h->ip6_dst)) {
2612 2624 ixa->ixa_flags |= IXAF_SCOPEID_SET;
2613 2625 ixa->ixa_scopeid = ira->ira_ruifindex;
2614 2626 }
2615 2627 ixa->ixa_ip_hdr_length = IPV6_HDR_LEN;
2616 2628 }
2617 2629 ixa->ixa_pktlen = len;
2618 2630
2619 2631 /* Swap the ports */
2620 2632 port = tcpha->tha_fport;
2621 2633 tcpha->tha_fport = tcpha->tha_lport;
2622 2634 tcpha->tha_lport = port;
2623 2635
2624 2636 tcpha->tha_ack = htonl(ack);
2625 2637 tcpha->tha_seq = htonl(seq);
2626 2638 tcpha->tha_win = 0;
2627 2639 tcpha->tha_sum = htons(sizeof (tcpha_t));
2628 2640 tcpha->tha_flags = (uint8_t)ctl;
2629 2641 if (ctl & TH_RST) {
2630 2642 if (ctl & TH_ACK) {
2631 2643 /*
2632 2644 * Probe connection rejection here.
2633 2645 * tcp_xmit_listeners_reset() drops non-SYN segments
2634 2646 * that do not specify TH_ACK in their flags without
2635 2647 * calling this function. As a consequence, if this
2636 2648 * function is called with a TH_RST|TH_ACK ctl argument,
2637 2649 * it is being called in response to a SYN segment
2638 2650 * and thus the tcp:::accept-refused probe point
2639 2651 * is valid here.
2640 2652 */
2641 2653 DTRACE_TCP5(accept__refused, mblk_t *, NULL,
2642 2654 void, NULL, void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2643 2655 tcph_t *, tcpha);
2644 2656 }
2645 2657 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2646 2658 TCPS_BUMP_MIB(tcps, tcpOutControl);
2647 2659 }
2648 2660
2649 2661 /* Discard any old label */
2650 2662 if (ixa->ixa_free_flags & IXA_FREE_TSL) {
2651 2663 ASSERT(ixa->ixa_tsl != NULL);
2652 2664 label_rele(ixa->ixa_tsl);
2653 2665 ixa->ixa_free_flags &= ~IXA_FREE_TSL;
2654 2666 }
2655 2667 ixa->ixa_tsl = ira->ira_tsl; /* Behave as a multi-level responder */
2656 2668
2657 2669 if (ira->ira_flags & IRAF_IPSEC_SECURE) {
2658 2670 /*
2659 2671 * Apply IPsec based on how IPsec was applied to
2660 2672 * the packet that caused the RST.
2661 2673 */
2662 2674 if (!ipsec_in_to_out(ira, ixa, mp, ipha, ip6h)) {
2663 2675 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsOutDiscards);
2664 2676 /* Note: mp already consumed and ip_drop_packet done */
2665 2677 goto done;
2666 2678 }
2667 2679 } else {
2668 2680 /*
2669 2681 * This is in clear. The RST message we are building
2670 2682 * here should go out in clear, independent of our policy.
2671 2683 */
2672 2684 ixa->ixa_flags |= IXAF_NO_IPSEC;
2673 2685 }
2674 2686
2675 2687 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, ixa,
2676 2688 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2677 2689 __dtrace_tcp_tcph_t *, tcpha);
2678 2690
2679 2691 /*
2680 2692 * NOTE: one might consider tracing a TCP packet here, but
2681 2693 * this function has no active TCP state and no tcp structure
2682 2694 * that has a trace buffer. If we traced here, we would have
2683 2695 * to keep a local trace buffer in tcp_record_trace().
2684 2696 */
2685 2697
2686 2698 (void) ip_output_simple(mp, ixa);
2687 2699 done:
2688 2700 ixa_cleanup(ixa);
2689 2701 if (need_refrele) {
2690 2702 ASSERT(ixa != &ixas);
2691 2703 ixa_refrele(ixa);
2692 2704 }
2693 2705 }
2694 2706
2695 2707 /*
2696 2708 * Generate a "no listener here" RST in response to an "unknown" segment.
2697 2709 * connp is set by caller when RST is in response to an unexpected
2698 2710 * inbound packet for which there is active tcp state in the system.
2699 2711 * Note that we are reusing the incoming mp to construct the outgoing RST.
2700 2712 */
2701 2713 void
2702 2714 tcp_xmit_listeners_reset(mblk_t *mp, ip_recv_attr_t *ira, ip_stack_t *ipst,
2703 2715 conn_t *connp)
2704 2716 {
2705 2717 uchar_t *rptr;
2706 2718 uint32_t seg_len;
2707 2719 tcpha_t *tcpha;
2708 2720 uint32_t seg_seq;
2709 2721 uint32_t seg_ack;
2710 2722 uint_t flags;
2711 2723 ipha_t *ipha;
2712 2724 ip6_t *ip6h;
2713 2725 boolean_t policy_present;
2714 2726 netstack_t *ns = ipst->ips_netstack;
2715 2727 tcp_stack_t *tcps = ns->netstack_tcp;
2716 2728 ipsec_stack_t *ipss = tcps->tcps_netstack->netstack_ipsec;
2717 2729 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2718 2730
2719 2731 TCP_STAT(tcps, tcp_no_listener);
2720 2732
2721 2733 /*
2722 2734 * DTrace this "unknown" segment as a tcp:::receive, as we did
2723 2735 * just receive something that was TCP.
2724 2736 */
2725 2737 DTRACE_TCP5(receive, mblk_t *, NULL, ip_xmit_attr_t *, NULL,
2726 2738 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2727 2739 __dtrace_tcp_tcph_t *, &mp->b_rptr[ip_hdr_len]);
2728 2740
2729 2741 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2730 2742 policy_present = ipss->ipsec_inbound_v4_policy_present;
2731 2743 ipha = (ipha_t *)mp->b_rptr;
2732 2744 ip6h = NULL;
2733 2745 } else {
2734 2746 policy_present = ipss->ipsec_inbound_v6_policy_present;
2735 2747 ipha = NULL;
2736 2748 ip6h = (ip6_t *)mp->b_rptr;
2737 2749 }
2738 2750
2739 2751 if (policy_present) {
2740 2752 /*
2741 2753 * The conn_t parameter is NULL because we already know
2742 2754 * nobody's home.
2743 2755 */
2744 2756 mp = ipsec_check_global_policy(mp, (conn_t *)NULL, ipha, ip6h,
2745 2757 ira, ns);
2746 2758 if (mp == NULL)
2747 2759 return;
2748 2760 }
2749 2761 if (is_system_labeled() && !tsol_can_reply_error(mp, ira)) {
2750 2762 DTRACE_PROBE2(
2751 2763 tx__ip__log__error__nolistener__tcp,
2752 2764 char *, "Could not reply with RST to mp(1)",
2753 2765 mblk_t *, mp);
2754 2766 ip2dbg(("tcp_xmit_listeners_reset: not permitted to reply\n"));
2755 2767 freemsg(mp);
2756 2768 return;
2757 2769 }
2758 2770
2759 2771 rptr = mp->b_rptr;
2760 2772
2761 2773 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2762 2774 seg_seq = ntohl(tcpha->tha_seq);
2763 2775 seg_ack = ntohl(tcpha->tha_ack);
2764 2776 flags = tcpha->tha_flags;
2765 2777
2766 2778 seg_len = msgdsize(mp) - (TCP_HDR_LENGTH(tcpha) + ip_hdr_len);
2767 2779 if (flags & TH_RST) {
2768 2780 freemsg(mp);
2769 2781 } else if (flags & TH_ACK) {
2770 2782 tcp_xmit_early_reset("no tcp, reset", mp, seg_ack, 0, TH_RST,
2771 2783 ira, ipst, connp);
2772 2784 } else {
2773 2785 if (flags & TH_SYN) {
2774 2786 seg_len++;
2775 2787 } else {
2776 2788 /*
2777 2789 * Here we violate the RFC. Note that a normal
2778 2790 * TCP will never send a segment without the ACK
2779 2791 * flag, except for RST or SYN segment. This
2780 2792 * segment is neither. Just drop it on the
2781 2793 * floor.
2782 2794 */
2783 2795 freemsg(mp);
2784 2796 TCP_STAT(tcps, tcp_rst_unsent);
2785 2797 return;
2786 2798 }
2787 2799
2788 2800 tcp_xmit_early_reset("no tcp, reset/ack", mp, 0,
2789 2801 seg_seq + seg_len, TH_RST | TH_ACK, ira, ipst, connp);
2790 2802 }
2791 2803 }
2792 2804
2793 2805 /*
2794 2806 * Helper function for tcp_xmit_mp() in handling connection set up flag
2795 2807 * options setting.
2796 2808 */
2797 2809 static void
2798 2810 tcp_xmit_mp_aux_iss(tcp_t *tcp, conn_t *connp, tcpha_t *tcpha, mblk_t *mp,
2799 2811 uint_t *flags)
2800 2812 {
2801 2813 uint32_t u1;
2802 2814 uint8_t *wptr = mp->b_wptr;
2803 2815 tcp_stack_t *tcps = tcp->tcp_tcps;
2804 2816 boolean_t add_sack = B_FALSE;
2805 2817
2806 2818 /*
2807 2819 * If TCP_ISS_VALID and the seq number is tcp_iss,
2808 2820 * TCP can only be in SYN-SENT, SYN-RCVD or
2809 2821 * FIN-WAIT-1 state. It can be FIN-WAIT-1 if
2810 2822 * our SYN is not ack'ed but the app closes this
2811 2823 * TCP connection.
2812 2824 */
2813 2825 ASSERT(tcp->tcp_state == TCPS_SYN_SENT ||
2814 2826 tcp->tcp_state == TCPS_SYN_RCVD ||
2815 2827 tcp->tcp_state == TCPS_FIN_WAIT_1);
2816 2828
2817 2829 /*
2818 2830 * Tack on the MSS option. It is always needed
2819 2831 * for both active and passive open.
2820 2832 *
2821 2833 * MSS option value should be interface MTU - MIN
2822 2834 * TCP/IP header according to RFC 793 as it means
2823 2835 * the maximum segment size TCP can receive. But
2824 2836 * to get around some broken middle boxes/end hosts
2825 2837 * out there, we allow the option value to be the
2826 2838 * same as the MSS option size on the peer side.
2827 2839 * In this way, the other side will not send
2828 2840 * anything larger than they can receive.
2829 2841 *
2830 2842 * Note that for SYN_SENT state, the ndd param
2831 2843 * tcp_use_smss_as_mss_opt has no effect as we
2832 2844 * don't know the peer's MSS option value. So
2833 2845 * the only case we need to take care of is in
2834 2846 * SYN_RCVD state, which is done later.
2835 2847 */
2836 2848 wptr[0] = TCPOPT_MAXSEG;
2837 2849 wptr[1] = TCPOPT_MAXSEG_LEN;
2838 2850 wptr += 2;
2839 2851 u1 = tcp->tcp_initial_pmtu - (connp->conn_ipversion == IPV4_VERSION ?
2840 2852 IP_SIMPLE_HDR_LENGTH : IPV6_HDR_LEN) - TCP_MIN_HEADER_LENGTH;
2841 2853 U16_TO_BE16(u1, wptr);
2842 2854 wptr += 2;
2843 2855
2844 2856 /* Update the offset to cover the additional word */
2845 2857 tcpha->tha_offset_and_reserved += (1 << 4);
2846 2858
2847 2859 switch (tcp->tcp_state) {
2848 2860 case TCPS_SYN_SENT:
2849 2861 *flags = TH_SYN;
2850 2862
2851 2863 if (tcp->tcp_snd_sack_ok)
2852 2864 add_sack = B_TRUE;
2853 2865
2854 2866 if (tcp->tcp_snd_ts_ok) {
2855 2867 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2856 2868
2857 2869 if (add_sack) {
2858 2870 wptr[0] = TCPOPT_SACK_PERMITTED;
2859 2871 wptr[1] = TCPOPT_SACK_OK_LEN;
2860 2872 add_sack = B_FALSE;
2861 2873 } else {
2862 2874 wptr[0] = TCPOPT_NOP;
2863 2875 wptr[1] = TCPOPT_NOP;
2864 2876 }
2865 2877 wptr[2] = TCPOPT_TSTAMP;
2866 2878 wptr[3] = TCPOPT_TSTAMP_LEN;
2867 2879 wptr += 4;
2868 2880 U32_TO_BE32(llbolt, wptr);
2869 2881 wptr += 4;
2870 2882 ASSERT(tcp->tcp_ts_recent == 0);
2871 2883 U32_TO_BE32(0L, wptr);
2872 2884 wptr += 4;
2873 2885 tcpha->tha_offset_and_reserved += (3 << 4);
2874 2886 }
2875 2887
2876 2888 /*
2877 2889 * Set up all the bits to tell other side
2878 2890 * we are ECN capable.
2879 2891 */
2880 2892 if (tcp->tcp_ecn_ok)
2881 2893 *flags |= (TH_ECE | TH_CWR);
2882 2894
2883 2895 break;
2884 2896
2885 2897 case TCPS_SYN_RCVD:
2886 2898 *flags |= TH_SYN;
2887 2899
2888 2900 /*
2889 2901 * Reset the MSS option value to be SMSS
2890 2902 * We should probably add back the bytes
2891 2903 * for timestamp option and IPsec. We
2892 2904 * don't do that as this is a workaround
2893 2905 * for broken middle boxes/end hosts, it
2894 2906 * is better for us to be more cautious.
2895 2907 * They may not take these things into
2896 2908 * account in their SMSS calculation. Thus
2897 2909 * the peer's calculated SMSS may be smaller
2898 2910 * than what it can be. This should be OK.
2899 2911 */
2900 2912 if (tcps->tcps_use_smss_as_mss_opt) {
2901 2913 u1 = tcp->tcp_mss;
2902 2914 /*
2903 2915 * Note that wptr points just past the MSS
2904 2916 * option value.
2905 2917 */
2906 2918 U16_TO_BE16(u1, wptr - 2);
2907 2919 }
2908 2920
2909 2921 /*
2910 2922 * tcp_snd_ts_ok can only be set in TCPS_SYN_RCVD
2911 2923 * when the peer also uses timestamps option. And
2912 2924 * the TCP header template must have already been
2913 2925 * updated to include the timestamps option.
2914 2926 */
2915 2927 if (tcp->tcp_snd_sack_ok) {
2916 2928 if (tcp->tcp_snd_ts_ok) {
2917 2929 uint8_t *tmp_wptr;
2918 2930
2919 2931 /*
2920 2932 * Use the NOP in the header just
2921 2933 * before timestamps opton.
2922 2934 */
2923 2935 tmp_wptr = (uint8_t *)tcpha +
2924 2936 TCP_MIN_HEADER_LENGTH;
2925 2937 ASSERT(tmp_wptr[0] == TCPOPT_NOP &&
2926 2938 tmp_wptr[1] == TCPOPT_NOP);
2927 2939 tmp_wptr[0] = TCPOPT_SACK_PERMITTED;
2928 2940 tmp_wptr[1] = TCPOPT_SACK_OK_LEN;
2929 2941 } else {
2930 2942 add_sack = B_TRUE;
2931 2943 }
2932 2944 }
2933 2945
2934 2946
2935 2947 /*
2936 2948 * If the other side is ECN capable, reply
2937 2949 * that we are also ECN capable.
2938 2950 */
2939 2951 if (tcp->tcp_ecn_ok)
2940 2952 *flags |= TH_ECE;
2941 2953 break;
2942 2954
2943 2955 default:
2944 2956 /*
2945 2957 * The above ASSERT() makes sure that this
2946 2958 * must be FIN-WAIT-1 state. Our SYN has
2947 2959 * not been ack'ed so retransmit it.
2948 2960 */
2949 2961 *flags |= TH_SYN;
2950 2962 break;
2951 2963 }
2952 2964
2953 2965 if (add_sack) {
2954 2966 wptr[0] = TCPOPT_NOP;
2955 2967 wptr[1] = TCPOPT_NOP;
2956 2968 wptr[2] = TCPOPT_SACK_PERMITTED;
2957 2969 wptr[3] = TCPOPT_SACK_OK_LEN;
2958 2970 wptr += TCPOPT_REAL_SACK_OK_LEN;
2959 2971 tcpha->tha_offset_and_reserved += (1 << 4);
2960 2972 }
2961 2973
2962 2974 if (tcp->tcp_snd_ws_ok) {
2963 2975 wptr[0] = TCPOPT_NOP;
2964 2976 wptr[1] = TCPOPT_WSCALE;
2965 2977 wptr[2] = TCPOPT_WS_LEN;
2966 2978 wptr[3] = (uchar_t)tcp->tcp_rcv_ws;
2967 2979 wptr += TCPOPT_REAL_WS_LEN;
2968 2980 tcpha->tha_offset_and_reserved += (1 << 4);
2969 2981 }
2970 2982
2971 2983 mp->b_wptr = wptr;
2972 2984 u1 = (int)(mp->b_wptr - mp->b_rptr);
2973 2985 /*
2974 2986 * Get IP set to checksum on our behalf
2975 2987 * Include the adjustment for a source route if any.
2976 2988 */
2977 2989 u1 += connp->conn_sum;
2978 2990 u1 = (u1 >> 16) + (u1 & 0xFFFF);
2979 2991 tcpha->tha_sum = htons(u1);
2980 2992 TCPS_BUMP_MIB(tcps, tcpOutControl);
2981 2993 }
2982 2994
2983 2995 /*
2984 2996 * Helper function for tcp_xmit_mp() in handling connection tear down
2985 2997 * flag setting and state changes.
2986 2998 */
2987 2999 static void
2988 3000 tcp_xmit_mp_aux_fss(tcp_t *tcp, ip_xmit_attr_t *ixa, uint_t *flags)
2989 3001 {
2990 3002 if (!tcp->tcp_fin_acked) {
2991 3003 *flags |= TH_FIN;
2992 3004 TCPS_BUMP_MIB(tcp->tcp_tcps, tcpOutControl);
2993 3005 }
2994 3006 if (!tcp->tcp_fin_sent) {
2995 3007 tcp->tcp_fin_sent = B_TRUE;
2996 3008 switch (tcp->tcp_state) {
2997 3009 case TCPS_SYN_RCVD:
2998 3010 tcp->tcp_state = TCPS_FIN_WAIT_1;
2999 3011 DTRACE_TCP6(state__change, void, NULL,
3000 3012 ip_xmit_attr_t *, ixa, void, NULL,
3001 3013 tcp_t *, tcp, void, NULL,
3002 3014 int32_t, TCPS_SYN_RCVD);
3003 3015 break;
3004 3016 case TCPS_ESTABLISHED:
3005 3017 tcp->tcp_state = TCPS_FIN_WAIT_1;
3006 3018 DTRACE_TCP6(state__change, void, NULL,
3007 3019 ip_xmit_attr_t *, ixa, void, NULL,
3008 3020 tcp_t *, tcp, void, NULL,
3009 3021 int32_t, TCPS_ESTABLISHED);
3010 3022 break;
3011 3023 case TCPS_CLOSE_WAIT:
3012 3024 tcp->tcp_state = TCPS_LAST_ACK;
3013 3025 DTRACE_TCP6(state__change, void, NULL,
3014 3026 ip_xmit_attr_t *, ixa, void, NULL,
3015 3027 tcp_t *, tcp, void, NULL,
3016 3028 int32_t, TCPS_CLOSE_WAIT);
3017 3029 break;
3018 3030 }
3019 3031 if (tcp->tcp_suna == tcp->tcp_snxt)
3020 3032 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
3021 3033 tcp->tcp_snxt = tcp->tcp_fss + 1;
3022 3034 }
3023 3035 }
3024 3036
3025 3037 /*
3026 3038 * tcp_xmit_mp is called to return a pointer to an mblk chain complete with
3027 3039 * ip and tcp header ready to pass down to IP. If the mp passed in is
3028 3040 * non-NULL, then up to max_to_send bytes of data will be dup'ed off that
3029 3041 * mblk. (If sendall is not set the dup'ing will stop at an mblk boundary
3030 3042 * otherwise it will dup partial mblks.)
3031 3043 * Otherwise, an appropriate ACK packet will be generated. This
3032 3044 * routine is not usually called to send new data for the first time. It
3033 3045 * is mostly called out of the timer for retransmits, and to generate ACKs.
3034 3046 *
3035 3047 * If offset is not NULL, the returned mblk chain's first mblk's b_rptr will
3036 3048 * be adjusted by *offset. And after dupb(), the offset and the ending mblk
3037 3049 * of the original mblk chain will be returned in *offset and *end_mp.
3038 3050 */
3039 3051 mblk_t *
3040 3052 tcp_xmit_mp(tcp_t *tcp, mblk_t *mp, int32_t max_to_send, int32_t *offset,
3041 3053 mblk_t **end_mp, uint32_t seq, boolean_t sendall, uint32_t *seg_len,
3042 3054 boolean_t rexmit)
3043 3055 {
3044 3056 int data_length;
3045 3057 int32_t off = 0;
3046 3058 uint_t flags;
3047 3059 mblk_t *mp1;
3048 3060 mblk_t *mp2;
3049 3061 uchar_t *rptr;
3050 3062 tcpha_t *tcpha;
3051 3063 int32_t num_sack_blk = 0;
3052 3064 int32_t sack_opt_len = 0;
3053 3065 tcp_stack_t *tcps = tcp->tcp_tcps;
3054 3066 conn_t *connp = tcp->tcp_connp;
3055 3067 ip_xmit_attr_t *ixa = connp->conn_ixa;
3056 3068
3057 3069 /* Allocate for our maximum TCP header + link-level */
3058 3070 mp1 = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
3059 3071 BPRI_MED);
3060 3072 if (mp1 == NULL)
3061 3073 return (NULL);
3062 3074 data_length = 0;
3063 3075
3064 3076 /*
3065 3077 * Note that tcp_mss has been adjusted to take into account the
3066 3078 * timestamp option if applicable. Because SACK options do not
3067 3079 * appear in every TCP segments and they are of variable lengths,
3068 3080 * they cannot be included in tcp_mss. Thus we need to calculate
3069 3081 * the actual segment length when we need to send a segment which
3070 3082 * includes SACK options.
3071 3083 */
3072 3084 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
3073 3085 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
3074 3086 tcp->tcp_num_sack_blk);
3075 3087 sack_opt_len = num_sack_blk * sizeof (sack_blk_t) +
3076 3088 TCPOPT_NOP_LEN * 2 + TCPOPT_HEADER_LEN;
3077 3089 if (max_to_send + sack_opt_len > tcp->tcp_mss)
3078 3090 max_to_send -= sack_opt_len;
3079 3091 }
3080 3092
3081 3093 if (offset != NULL) {
3082 3094 off = *offset;
3083 3095 /* We use offset as an indicator that end_mp is not NULL. */
3084 3096 *end_mp = NULL;
3085 3097 }
3086 3098 for (mp2 = mp1; mp && data_length != max_to_send; mp = mp->b_cont) {
3087 3099 /* This could be faster with cooperation from downstream */
3088 3100 if (mp2 != mp1 && !sendall &&
3089 3101 data_length + (int)(mp->b_wptr - mp->b_rptr) >
3090 3102 max_to_send)
3091 3103 /*
3092 3104 * Don't send the next mblk since the whole mblk
3093 3105 * does not fit.
3094 3106 */
3095 3107 break;
3096 3108 mp2->b_cont = dupb(mp);
3097 3109 mp2 = mp2->b_cont;
3098 3110 if (!mp2) {
3099 3111 freemsg(mp1);
3100 3112 return (NULL);
3101 3113 }
3102 3114 mp2->b_rptr += off;
3103 3115 ASSERT((uintptr_t)(mp2->b_wptr - mp2->b_rptr) <=
3104 3116 (uintptr_t)INT_MAX);
3105 3117
3106 3118 data_length += (int)(mp2->b_wptr - mp2->b_rptr);
3107 3119 if (data_length > max_to_send) {
3108 3120 mp2->b_wptr -= data_length - max_to_send;
3109 3121 data_length = max_to_send;
3110 3122 off = mp2->b_wptr - mp->b_rptr;
3111 3123 break;
3112 3124 } else {
3113 3125 off = 0;
3114 3126 }
3115 3127 }
3116 3128 if (offset != NULL) {
3117 3129 *offset = off;
3118 3130 *end_mp = mp;
3119 3131 }
3120 3132 if (seg_len != NULL) {
3121 3133 *seg_len = data_length;
3122 3134 }
3123 3135
3124 3136 /* Update the latest receive window size in TCP header. */
3125 3137 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
3126 3138
3127 3139 rptr = mp1->b_rptr + tcps->tcps_wroff_xtra;
3128 3140 mp1->b_rptr = rptr;
3129 3141 mp1->b_wptr = rptr + connp->conn_ht_iphc_len + sack_opt_len;
3130 3142 bcopy(connp->conn_ht_iphc, rptr, connp->conn_ht_iphc_len);
3131 3143 tcpha = (tcpha_t *)&rptr[ixa->ixa_ip_hdr_length];
3132 3144 tcpha->tha_seq = htonl(seq);
3133 3145
3134 3146 /*
3135 3147 * Use tcp_unsent to determine if the PUSH bit should be used assumes
3136 3148 * that this function was called from tcp_wput_data. Thus, when called
3137 3149 * to retransmit data the setting of the PUSH bit may appear some
3138 3150 * what random in that it might get set when it should not. This
3139 3151 * should not pose any performance issues.
3140 3152 */
3141 3153 if (data_length != 0 && (tcp->tcp_unsent == 0 ||
3142 3154 tcp->tcp_unsent == data_length)) {
3143 3155 flags = TH_ACK | TH_PUSH;
3144 3156 } else {
3145 3157 flags = TH_ACK;
3146 3158 }
3147 3159
3148 3160 if (tcp->tcp_ecn_ok) {
3149 3161 if (tcp->tcp_ecn_echo_on)
3150 3162 flags |= TH_ECE;
3151 3163
3152 3164 /*
3153 3165 * Only set ECT bit and ECN_CWR if a segment contains new data.
3154 3166 * There is no TCP flow control for non-data segments, and
3155 3167 * only data segment is transmitted reliably.
3156 3168 */
3157 3169 if (data_length > 0 && !rexmit) {
3158 3170 TCP_SET_ECT(tcp, rptr);
3159 3171 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3160 3172 flags |= TH_CWR;
3161 3173 tcp->tcp_ecn_cwr_sent = B_TRUE;
3162 3174 }
3163 3175 }
3164 3176 }
3165 3177
3166 3178 /* Check if there is any special processing needs to be done. */
3167 3179 if (tcp->tcp_valid_bits) {
3168 3180 uint32_t u1;
3169 3181
3170 3182 /* We don't allow having SYN and FIN in the same segment... */
3171 3183 if ((tcp->tcp_valid_bits & TCP_ISS_VALID) &&
3172 3184 seq == tcp->tcp_iss) {
3173 3185 /* Need to do connection set up processing. */
3174 3186 tcp_xmit_mp_aux_iss(tcp, connp, tcpha, mp1, &flags);
3175 3187 } else if ((tcp->tcp_valid_bits & TCP_FSS_VALID) &&
3176 3188 (seq + data_length) == tcp->tcp_fss) {
3177 3189 /* Need to do connection tear down processing. */
3178 3190 tcp_xmit_mp_aux_fss(tcp, ixa, &flags);
3179 3191 }
3180 3192
3181 3193 /*
3182 3194 * Need to do urgent pointer processing.
3183 3195 *
3184 3196 * Note the trick here. u1 is unsigned. When tcp_urg
3185 3197 * is smaller than seq, u1 will become a very huge value.
3186 3198 * So the comparison will fail. Also note that tcp_urp
3187 3199 * should be positive, see RFC 793 page 17.
3188 3200 */
3189 3201 u1 = tcp->tcp_urg - seq + TCP_OLD_URP_INTERPRETATION;
3190 3202 if ((tcp->tcp_valid_bits & TCP_URG_VALID) && u1 != 0 &&
3191 3203 u1 < (uint32_t)(64 * 1024)) {
3192 3204 flags |= TH_URG;
3193 3205 TCPS_BUMP_MIB(tcps, tcpOutUrg);
3194 3206 tcpha->tha_urp = htons(u1);
3195 3207 }
3196 3208 }
3197 3209 tcpha->tha_flags = (uchar_t)flags;
3198 3210 tcp->tcp_rack = tcp->tcp_rnxt;
3199 3211 tcp->tcp_rack_cnt = 0;
3200 3212
3201 3213 /* Fill in the current value of timestamps option. */
3202 3214 if (tcp->tcp_snd_ts_ok) {
3203 3215 if (tcp->tcp_state != TCPS_SYN_SENT) {
3204 3216 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
3205 3217
3206 3218 U32_TO_BE32(llbolt,
3207 3219 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
3208 3220 U32_TO_BE32(tcp->tcp_ts_recent,
3209 3221 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
3210 3222 }
3211 3223 }
3212 3224
3213 3225 /* Fill in the SACK blocks. */
3214 3226 if (num_sack_blk > 0) {
3215 3227 uchar_t *wptr = (uchar_t *)tcpha + connp->conn_ht_ulp_len;
3216 3228 sack_blk_t *tmp;
3217 3229 int32_t i;
3218 3230
3219 3231 wptr[0] = TCPOPT_NOP;
3220 3232 wptr[1] = TCPOPT_NOP;
3221 3233 wptr[2] = TCPOPT_SACK;
3222 3234 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3223 3235 sizeof (sack_blk_t);
3224 3236 wptr += TCPOPT_REAL_SACK_LEN;
3225 3237
3226 3238 tmp = tcp->tcp_sack_list;
3227 3239 for (i = 0; i < num_sack_blk; i++) {
3228 3240 U32_TO_BE32(tmp[i].begin, wptr);
3229 3241 wptr += sizeof (tcp_seq);
3230 3242 U32_TO_BE32(tmp[i].end, wptr);
3231 3243 wptr += sizeof (tcp_seq);
3232 3244 }
3233 3245 tcpha->tha_offset_and_reserved += ((num_sack_blk * 2 + 1) << 4);
3234 3246 }
3235 3247 ASSERT((uintptr_t)(mp1->b_wptr - rptr) <= (uintptr_t)INT_MAX);
3236 3248 data_length += (int)(mp1->b_wptr - rptr);
3237 3249
3238 3250 ixa->ixa_pktlen = data_length;
3239 3251
3240 3252 if (ixa->ixa_flags & IXAF_IS_IPV4) {
3241 3253 ((ipha_t *)rptr)->ipha_length = htons(data_length);
3242 3254 } else {
3243 3255 ip6_t *ip6 = (ip6_t *)rptr;
3244 3256
3245 3257 ip6->ip6_plen = htons(data_length - IPV6_HDR_LEN);
3246 3258 }
3247 3259
3248 3260 /*
3249 3261 * Prime pump for IP
3250 3262 * Include the adjustment for a source route if any.
3251 3263 */
3252 3264 data_length -= ixa->ixa_ip_hdr_length;
3253 3265 data_length += connp->conn_sum;
3254 3266 data_length = (data_length >> 16) + (data_length & 0xFFFF);
3255 3267 tcpha->tha_sum = htons(data_length);
3256 3268 if (tcp->tcp_ip_forward_progress) {
3257 3269 tcp->tcp_ip_forward_progress = B_FALSE;
3258 3270 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
3259 3271 } else {
3260 3272 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
3261 3273 }
3262 3274 return (mp1);
3263 3275 }
3264 3276
3265 3277 /*
3266 3278 * If this routine returns B_TRUE, TCP can generate a RST in response
3267 3279 * to a segment. If it returns B_FALSE, TCP should not respond.
3268 3280 */
3269 3281 static boolean_t
3270 3282 tcp_send_rst_chk(tcp_stack_t *tcps)
3271 3283 {
3272 3284 int64_t now;
3273 3285
3274 3286 /*
3275 3287 * TCP needs to protect itself from generating too many RSTs.
3276 3288 * This can be a DoS attack by sending us random segments
3277 3289 * soliciting RSTs.
3278 3290 *
3279 3291 * What we do here is to have a limit of tcp_rst_sent_rate RSTs
3280 3292 * in each 1 second interval. In this way, TCP still generate
3281 3293 * RSTs in normal cases but when under attack, the impact is
3282 3294 * limited.
3283 3295 */
3284 3296 if (tcps->tcps_rst_sent_rate_enabled != 0) {
3285 3297 now = ddi_get_lbolt64();
3286 3298 if (TICK_TO_MSEC(now - tcps->tcps_last_rst_intrvl) >
3287 3299 1*SECONDS) {
3288 3300 tcps->tcps_last_rst_intrvl = now;
3289 3301 tcps->tcps_rst_cnt = 1;
3290 3302 } else if (++tcps->tcps_rst_cnt > tcps->tcps_rst_sent_rate) {
3291 3303 return (B_FALSE);
3292 3304 }
3293 3305 }
3294 3306 return (B_TRUE);
3295 3307 }
3296 3308
3297 3309 /*
3298 3310 * This function handles all retransmissions if SACK is enabled for this
3299 3311 * connection. First it calculates how many segments can be retransmitted
3300 3312 * based on tcp_pipe. Then it goes thru the notsack list to find eligible
3301 3313 * segments. A segment is eligible if sack_cnt for that segment is greater
3302 3314 * than or equal tcp_dupack_fast_retransmit. After it has retransmitted
3303 3315 * all eligible segments, it checks to see if TCP can send some new segments
3304 3316 * (fast recovery). If it can, set the appropriate flag for tcp_input_data().
3305 3317 *
3306 3318 * Parameters:
3307 3319 * tcp_t *tcp: the tcp structure of the connection.
3308 3320 * uint_t *flags: in return, appropriate value will be set for
3309 3321 * tcp_input_data().
3310 3322 */
3311 3323 void
3312 3324 tcp_sack_rexmit(tcp_t *tcp, uint_t *flags)
3313 3325 {
3314 3326 notsack_blk_t *notsack_blk;
3315 3327 int32_t usable_swnd;
3316 3328 int32_t mss;
3317 3329 uint32_t seg_len;
3318 3330 mblk_t *xmit_mp;
3319 3331 tcp_stack_t *tcps = tcp->tcp_tcps;
3320 3332
3321 3333 ASSERT(tcp->tcp_notsack_list != NULL);
3322 3334 ASSERT(tcp->tcp_rexmit == B_FALSE);
3323 3335
3324 3336 /* Defensive coding in case there is a bug... */
3325 3337 if (tcp->tcp_notsack_list == NULL) {
3326 3338 return;
3327 3339 }
3328 3340 notsack_blk = tcp->tcp_notsack_list;
3329 3341 mss = tcp->tcp_mss;
3330 3342
3331 3343 /*
3332 3344 * Limit the num of outstanding data in the network to be
3333 3345 * tcp_cwnd_ssthresh, which is half of the original congestion wnd.
3334 3346 */
3335 3347 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3336 3348
3337 3349 /* At least retransmit 1 MSS of data. */
3338 3350 if (usable_swnd <= 0) {
3339 3351 usable_swnd = mss;
3340 3352 }
3341 3353
3342 3354 /* Make sure no new RTT samples will be taken. */
3343 3355 tcp->tcp_csuna = tcp->tcp_snxt;
3344 3356
3345 3357 notsack_blk = tcp->tcp_notsack_list;
3346 3358 while (usable_swnd > 0) {
3347 3359 mblk_t *snxt_mp, *tmp_mp;
3348 3360 tcp_seq begin = tcp->tcp_sack_snxt;
3349 3361 tcp_seq end;
3350 3362 int32_t off;
3351 3363
3352 3364 for (; notsack_blk != NULL; notsack_blk = notsack_blk->next) {
3353 3365 if (SEQ_GT(notsack_blk->end, begin) &&
3354 3366 (notsack_blk->sack_cnt >=
3355 3367 tcps->tcps_dupack_fast_retransmit)) {
3356 3368 end = notsack_blk->end;
3357 3369 if (SEQ_LT(begin, notsack_blk->begin)) {
3358 3370 begin = notsack_blk->begin;
3359 3371 }
3360 3372 break;
3361 3373 }
3362 3374 }
3363 3375 /*
3364 3376 * All holes are filled. Manipulate tcp_cwnd to send more
3365 3377 * if we can. Note that after the SACK recovery, tcp_cwnd is
3366 3378 * set to tcp_cwnd_ssthresh.
3367 3379 */
3368 3380 if (notsack_blk == NULL) {
3369 3381 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3370 3382 if (usable_swnd <= 0 || tcp->tcp_unsent == 0) {
3371 3383 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna;
3372 3384 ASSERT(tcp->tcp_cwnd > 0);
3373 3385 return;
3374 3386 } else {
3375 3387 usable_swnd = usable_swnd / mss;
3376 3388 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna +
3377 3389 MAX(usable_swnd * mss, mss);
3378 3390 *flags |= TH_XMIT_NEEDED;
3379 3391 return;
3380 3392 }
3381 3393 }
3382 3394
3383 3395 /*
3384 3396 * Note that we may send more than usable_swnd allows here
3385 3397 * because of round off, but no more than 1 MSS of data.
3386 3398 */
3387 3399 seg_len = end - begin;
3388 3400 if (seg_len > mss)
3389 3401 seg_len = mss;
3390 3402 snxt_mp = tcp_get_seg_mp(tcp, begin, &off);
3391 3403 ASSERT(snxt_mp != NULL);
3392 3404 /* This should not happen. Defensive coding again... */
3393 3405 if (snxt_mp == NULL) {
3394 3406 return;
3395 3407 }
3396 3408
3397 3409 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, seg_len, &off,
3398 3410 &tmp_mp, begin, B_TRUE, &seg_len, B_TRUE);
3399 3411 if (xmit_mp == NULL)
3400 3412 return;
3401 3413
3402 3414 usable_swnd -= seg_len;
3403 3415 tcp->tcp_pipe += seg_len;
3404 3416 tcp->tcp_sack_snxt = begin + seg_len;
3405 3417
3406 3418 tcp_send_data(tcp, xmit_mp);
3407 3419
3408 3420 /*
3409 3421 * Update the send timestamp to avoid false retransmission.
3410 3422 */
3411 3423 snxt_mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
3412 3424
3413 3425 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3414 3426 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, seg_len);
3415 3427 TCPS_BUMP_MIB(tcps, tcpOutSackRetransSegs);
3416 3428 tcp->tcp_cs.tcp_out_retrans_segs++;
3417 3429 tcp->tcp_cs.tcp_out_retrans_bytes += seg_len;
3418 3430 /*
3419 3431 * Update tcp_rexmit_max to extend this SACK recovery phase.
3420 3432 * This happens when new data sent during fast recovery is
3421 3433 * also lost. If TCP retransmits those new data, it needs
3422 3434 * to extend SACK recover phase to avoid starting another
3423 3435 * fast retransmit/recovery unnecessarily.
3424 3436 */
3425 3437 if (SEQ_GT(tcp->tcp_sack_snxt, tcp->tcp_rexmit_max)) {
3426 3438 tcp->tcp_rexmit_max = tcp->tcp_sack_snxt;
3427 3439 }
3428 3440 }
3429 3441 }
3430 3442
3431 3443 /*
3432 3444 * tcp_ss_rexmit() is called to do slow start retransmission after a timeout
3433 3445 * or ICMP errors.
3434 3446 */
3435 3447 void
3436 3448 tcp_ss_rexmit(tcp_t *tcp)
3437 3449 {
3438 3450 uint32_t snxt;
3439 3451 uint32_t smax;
3440 3452 int32_t win;
3441 3453 int32_t mss;
3442 3454 int32_t off;
3443 3455 mblk_t *snxt_mp;
3444 3456 tcp_stack_t *tcps = tcp->tcp_tcps;
3445 3457
3446 3458 /*
3447 3459 * Note that tcp_rexmit can be set even though TCP has retransmitted
3448 3460 * all unack'ed segments.
3449 3461 */
3450 3462 if (SEQ_LT(tcp->tcp_rexmit_nxt, tcp->tcp_rexmit_max)) {
3451 3463 smax = tcp->tcp_rexmit_max;
3452 3464 snxt = tcp->tcp_rexmit_nxt;
3453 3465 if (SEQ_LT(snxt, tcp->tcp_suna)) {
3454 3466 snxt = tcp->tcp_suna;
3455 3467 }
3456 3468 win = MIN(tcp->tcp_cwnd, tcp->tcp_swnd);
3457 3469 win -= snxt - tcp->tcp_suna;
3458 3470 mss = tcp->tcp_mss;
3459 3471 snxt_mp = tcp_get_seg_mp(tcp, snxt, &off);
3460 3472
3461 3473 while (SEQ_LT(snxt, smax) && (win > 0) && (snxt_mp != NULL)) {
3462 3474 mblk_t *xmit_mp;
3463 3475 mblk_t *old_snxt_mp = snxt_mp;
3464 3476 uint32_t cnt = mss;
3465 3477
3466 3478 if (win < cnt) {
3467 3479 cnt = win;
3468 3480 }
3469 3481 if (SEQ_GT(snxt + cnt, smax)) {
3470 3482 cnt = smax - snxt;
3471 3483 }
3472 3484 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, cnt, &off,
3473 3485 &snxt_mp, snxt, B_TRUE, &cnt, B_TRUE);
3474 3486 if (xmit_mp == NULL)
3475 3487 return;
3476 3488
3477 3489 tcp_send_data(tcp, xmit_mp);
3478 3490
3479 3491 snxt += cnt;
3480 3492 win -= cnt;
3481 3493 /*
3482 3494 * Update the send timestamp to avoid false
3483 3495 * retransmission.
3484 3496 */
3485 3497 old_snxt_mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
3486 3498 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3487 3499 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, cnt);
3488 3500 tcp->tcp_cs.tcp_out_retrans_segs++;
3489 3501 tcp->tcp_cs.tcp_out_retrans_bytes += cnt;
3490 3502
3491 3503 tcp->tcp_rexmit_nxt = snxt;
3492 3504 }
3493 3505 /*
3494 3506 * If we have transmitted all we have at the time
3495 3507 * we started the retranmission, we can leave
3496 3508 * the rest of the job to tcp_wput_data(). But we
3497 3509 * need to check the send window first. If the
3498 3510 * win is not 0, go on with tcp_wput_data().
3499 3511 */
3500 3512 if (SEQ_LT(snxt, smax) || win == 0) {
3501 3513 return;
3502 3514 }
3503 3515 }
3504 3516 /* Only call tcp_wput_data() if there is data to be sent. */
3505 3517 if (tcp->tcp_unsent) {
3506 3518 tcp_wput_data(tcp, NULL, B_FALSE);
3507 3519 }
3508 3520 }
3509 3521
3510 3522 /*
3511 3523 * Do slow start retransmission after ICMP errors of PMTU changes.
3512 3524 */
3513 3525 void
3514 3526 tcp_rexmit_after_error(tcp_t *tcp)
3515 3527 {
3516 3528 /*
3517 3529 * All sent data has been acknowledged or no data left to send, just
3518 3530 * to return.
3519 3531 */
3520 3532 if (!SEQ_LT(tcp->tcp_suna, tcp->tcp_snxt) ||
3521 3533 (tcp->tcp_xmit_head == NULL))
3522 3534 return;
3523 3535
3524 3536 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && (tcp->tcp_unsent == 0))
3525 3537 tcp->tcp_rexmit_max = tcp->tcp_fss;
3526 3538 else
3527 3539 tcp->tcp_rexmit_max = tcp->tcp_snxt;
3528 3540
3529 3541 tcp->tcp_rexmit_nxt = tcp->tcp_suna;
3530 3542 tcp->tcp_rexmit = B_TRUE;
3531 3543 tcp->tcp_dupack_cnt = 0;
3532 3544 tcp_ss_rexmit(tcp);
3533 3545 }
3534 3546
3535 3547 /*
3536 3548 * tcp_get_seg_mp() is called to get the pointer to a segment in the
3537 3549 * send queue which starts at the given sequence number. If the given
3538 3550 * sequence number is equal to last valid sequence number (tcp_snxt), the
3539 3551 * returned mblk is the last valid mblk, and off is set to the length of
3540 3552 * that mblk.
3541 3553 *
3542 3554 * send queue which starts at the given seq. no.
3543 3555 *
3544 3556 * Parameters:
3545 3557 * tcp_t *tcp: the tcp instance pointer.
3546 3558 * uint32_t seq: the starting seq. no of the requested segment.
3547 3559 * int32_t *off: after the execution, *off will be the offset to
3548 3560 * the returned mblk which points to the requested seq no.
3549 3561 * It is the caller's responsibility to send in a non-null off.
3550 3562 *
3551 3563 * Return:
3552 3564 * A mblk_t pointer pointing to the requested segment in send queue.
3553 3565 */
3554 3566 static mblk_t *
3555 3567 tcp_get_seg_mp(tcp_t *tcp, uint32_t seq, int32_t *off)
3556 3568 {
3557 3569 int32_t cnt;
3558 3570 mblk_t *mp;
3559 3571
3560 3572 /* Defensive coding. Make sure we don't send incorrect data. */
3561 3573 if (SEQ_LT(seq, tcp->tcp_suna) || SEQ_GT(seq, tcp->tcp_snxt))
3562 3574 return (NULL);
3563 3575
3564 3576 cnt = seq - tcp->tcp_suna;
3565 3577 mp = tcp->tcp_xmit_head;
3566 3578 while (cnt > 0 && mp != NULL) {
3567 3579 cnt -= mp->b_wptr - mp->b_rptr;
3568 3580 if (cnt <= 0) {
3569 3581 cnt += mp->b_wptr - mp->b_rptr;
3570 3582 break;
3571 3583 }
3572 3584 mp = mp->b_cont;
3573 3585 }
3574 3586 ASSERT(mp != NULL);
3575 3587 *off = cnt;
3576 3588 return (mp);
3577 3589 }
3578 3590
3579 3591 /*
3580 3592 * This routine adjusts next-to-send sequence number variables, in the
3581 3593 * case where the reciever has shrunk it's window.
3582 3594 */
3583 3595 void
3584 3596 tcp_update_xmit_tail(tcp_t *tcp, uint32_t snxt)
3585 3597 {
3586 3598 mblk_t *xmit_tail;
3587 3599 int32_t offset;
3588 3600
3589 3601 tcp->tcp_snxt = snxt;
3590 3602
3591 3603 /* Get the mblk, and the offset in it, as per the shrunk window */
3592 3604 xmit_tail = tcp_get_seg_mp(tcp, snxt, &offset);
3593 3605 ASSERT(xmit_tail != NULL);
3594 3606 tcp->tcp_xmit_tail = xmit_tail;
3595 3607 tcp->tcp_xmit_tail_unsent = xmit_tail->b_wptr -
3596 3608 xmit_tail->b_rptr - offset;
3597 3609 }
3598 3610
3599 3611 /*
3600 3612 * This handles the case when the receiver has shrunk its win. Per RFC 1122
3601 3613 * if the receiver shrinks the window, i.e. moves the right window to the
3602 3614 * left, the we should not send new data, but should retransmit normally the
3603 3615 * old unacked data between suna and suna + swnd. We might has sent data
3604 3616 * that is now outside the new window, pretend that we didn't send it.
3605 3617 */
3606 3618 static void
3607 3619 tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_count)
3608 3620 {
3609 3621 uint32_t snxt = tcp->tcp_snxt;
3610 3622
3611 3623 ASSERT(shrunk_count > 0);
3612 3624
3613 3625 if (!tcp->tcp_is_wnd_shrnk) {
3614 3626 tcp->tcp_snxt_shrunk = snxt;
3615 3627 tcp->tcp_is_wnd_shrnk = B_TRUE;
3616 3628 } else if (SEQ_GT(snxt, tcp->tcp_snxt_shrunk)) {
3617 3629 tcp->tcp_snxt_shrunk = snxt;
3618 3630 }
3619 3631
3620 3632 /* Pretend we didn't send the data outside the window */
3621 3633 snxt -= shrunk_count;
3622 3634
3623 3635 /* Reset all the values per the now shrunk window */
3624 3636 tcp_update_xmit_tail(tcp, snxt);
3625 3637 tcp->tcp_unsent += shrunk_count;
3626 3638
3627 3639 /*
3628 3640 * If the SACK option is set, delete the entire list of
3629 3641 * notsack'ed blocks.
3630 3642 */
3631 3643 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list, tcp);
3632 3644
3633 3645 if (tcp->tcp_suna == tcp->tcp_snxt && tcp->tcp_swnd == 0)
3634 3646 /*
3635 3647 * Make sure the timer is running so that we will probe a zero
3636 3648 * window.
3637 3649 */
3638 3650 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
3639 3651 }
3640 3652
3641 3653 /*
3642 3654 * tcp_fill_header is called by tcp_send() to fill the outgoing TCP header
3643 3655 * with the template header, as well as other options such as time-stamp,
3644 3656 * ECN and/or SACK.
3645 3657 */
3646 3658 static void
3647 3659 tcp_fill_header(tcp_t *tcp, uchar_t *rptr, int num_sack_blk)
3648 3660 {
3649 3661 tcpha_t *tcp_tmpl, *tcpha;
3650 3662 uint32_t *dst, *src;
3651 3663 int hdrlen;
3652 3664 conn_t *connp = tcp->tcp_connp;
3653 3665
3654 3666 ASSERT(OK_32PTR(rptr));
3655 3667
3656 3668 /* Template header */
3657 3669 tcp_tmpl = tcp->tcp_tcpha;
3658 3670
3659 3671 /* Header of outgoing packet */
3660 3672 tcpha = (tcpha_t *)(rptr + connp->conn_ixa->ixa_ip_hdr_length);
3661 3673
3662 3674 /* dst and src are opaque 32-bit fields, used for copying */
3663 3675 dst = (uint32_t *)rptr;
3664 3676 src = (uint32_t *)connp->conn_ht_iphc;
3665 3677 hdrlen = connp->conn_ht_iphc_len;
3666 3678
3667 3679 /* Fill time-stamp option if needed */
3668 3680 if (tcp->tcp_snd_ts_ok) {
3669 3681 U32_TO_BE32(LBOLT_FASTPATH,
3670 3682 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 4);
3671 3683 U32_TO_BE32(tcp->tcp_ts_recent,
3672 3684 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 8);
3673 3685 } else {
3674 3686 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
3675 3687 }
3676 3688
3677 3689 /*
3678 3690 * Copy the template header; is this really more efficient than
3679 3691 * calling bcopy()? For simple IPv4/TCP, it may be the case,
3680 3692 * but perhaps not for other scenarios.
3681 3693 */
3682 3694 dst[0] = src[0];
3683 3695 dst[1] = src[1];
3684 3696 dst[2] = src[2];
3685 3697 dst[3] = src[3];
3686 3698 dst[4] = src[4];
3687 3699 dst[5] = src[5];
3688 3700 dst[6] = src[6];
3689 3701 dst[7] = src[7];
3690 3702 dst[8] = src[8];
3691 3703 dst[9] = src[9];
3692 3704 if (hdrlen -= 40) {
3693 3705 hdrlen >>= 2;
3694 3706 dst += 10;
3695 3707 src += 10;
3696 3708 do {
3697 3709 *dst++ = *src++;
3698 3710 } while (--hdrlen);
3699 3711 }
3700 3712
3701 3713 /*
3702 3714 * Set the ECN info in the TCP header if it is not a zero
3703 3715 * window probe. Zero window probe is only sent in
3704 3716 * tcp_wput_data() and tcp_timer().
3705 3717 */
3706 3718 if (tcp->tcp_ecn_ok && !tcp->tcp_zero_win_probe) {
3707 3719 TCP_SET_ECT(tcp, rptr);
3708 3720
3709 3721 if (tcp->tcp_ecn_echo_on)
3710 3722 tcpha->tha_flags |= TH_ECE;
3711 3723 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3712 3724 tcpha->tha_flags |= TH_CWR;
3713 3725 tcp->tcp_ecn_cwr_sent = B_TRUE;
3714 3726 }
3715 3727 }
3716 3728
3717 3729 /* Fill in SACK options */
3718 3730 if (num_sack_blk > 0) {
3719 3731 uchar_t *wptr = rptr + connp->conn_ht_iphc_len;
3720 3732 sack_blk_t *tmp;
3721 3733 int32_t i;
3722 3734
3723 3735 wptr[0] = TCPOPT_NOP;
3724 3736 wptr[1] = TCPOPT_NOP;
3725 3737 wptr[2] = TCPOPT_SACK;
3726 3738 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3727 3739 sizeof (sack_blk_t);
3728 3740 wptr += TCPOPT_REAL_SACK_LEN;
3729 3741
3730 3742 tmp = tcp->tcp_sack_list;
3731 3743 for (i = 0; i < num_sack_blk; i++) {
3732 3744 U32_TO_BE32(tmp[i].begin, wptr);
3733 3745 wptr += sizeof (tcp_seq);
3734 3746 U32_TO_BE32(tmp[i].end, wptr);
3735 3747 wptr += sizeof (tcp_seq);
3736 3748 }
3737 3749 tcpha->tha_offset_and_reserved +=
3738 3750 ((num_sack_blk * 2 + 1) << 4);
3739 3751 }
3740 3752 }
|
↓ open down ↓ |
2046 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX