1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
25 */
26
27 /* This file contains all TCP output processing functions. */
28
29 #include <sys/types.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <sys/strsubr.h>
33 #include <sys/stropts.h>
34 #include <sys/strlog.h>
35 #define _SUN_TPI_VERSION 2
36 #include <sys/tihdr.h>
37 #include <sys/suntpi.h>
38 #include <sys/xti_inet.h>
39 #include <sys/timod.h>
40 #include <sys/pattr.h>
41 #include <sys/squeue_impl.h>
42 #include <sys/squeue.h>
43 #include <sys/sockio.h>
44 #include <sys/tsol/tnet.h>
45
46 #include <inet/common.h>
47 #include <inet/ip.h>
48 #include <inet/tcp.h>
49 #include <inet/tcp_impl.h>
50 #include <inet/snmpcom.h>
51 #include <inet/proto_set.h>
52 #include <inet/ipsec_impl.h>
53 #include <inet/ip_ndp.h>
54
55 static mblk_t *tcp_get_seg_mp(tcp_t *, uint32_t, int32_t *);
56 static void tcp_wput_cmdblk(queue_t *, mblk_t *);
57 static void tcp_wput_flush(tcp_t *, mblk_t *);
58 static void tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp);
59 static int tcp_xmit_end(tcp_t *);
60 static int tcp_send(tcp_t *, const int, const int, const int,
61 const int, int *, uint_t *, int *, mblk_t **, mblk_t *);
62 static void tcp_xmit_early_reset(char *, mblk_t *, uint32_t, uint32_t,
63 int, ip_recv_attr_t *, ip_stack_t *, conn_t *);
64 static boolean_t tcp_send_rst_chk(tcp_stack_t *);
65 static void tcp_process_shrunk_swnd(tcp_t *, uint32_t);
66 static void tcp_fill_header(tcp_t *, uchar_t *, int);
67
68 /*
69 * Functions called directly via squeue having a prototype of edesc_t.
70 */
71 static void tcp_wput_nondata(void *, mblk_t *, void *, ip_recv_attr_t *);
72 static void tcp_wput_ioctl(void *, mblk_t *, void *, ip_recv_attr_t *);
73 static void tcp_wput_proto(void *, mblk_t *, void *, ip_recv_attr_t *);
74
75 /*
76 * This controls how tiny a write must be before we try to copy it
77 * into the mblk on the tail of the transmit queue. Not much
78 * speedup is observed for values larger than sixteen. Zero will
79 * disable the optimisation.
80 */
81 static int tcp_tx_pull_len = 16;
82
83 static void
84 cc_after_idle(tcp_t *tcp)
85 {
86 uint32_t old_cwnd = tcp->tcp_cwnd;
87
88 if (CC_ALGO(tcp)->after_idle != NULL)
89 CC_ALGO(tcp)->after_idle(&tcp->tcp_ccv);
90
91 DTRACE_PROBE3(cwnd__cc__after__idle, tcp_t *, tcp, uint32_t, old_cwnd,
92 uint32_t, tcp->tcp_cwnd);
93 }
94
95 void
96 tcp_wput(queue_t *q, mblk_t *mp)
97 {
98 conn_t *connp = Q_TO_CONN(q);
99 tcp_t *tcp;
100 void (*output_proc)();
101 t_scalar_t type;
102 uchar_t *rptr;
103 struct iocblk *iocp;
104 size_t size;
105
106 ASSERT(connp->conn_ref >= 2);
107
108 switch (DB_TYPE(mp)) {
109 case M_DATA:
110 tcp = connp->conn_tcp;
111 ASSERT(tcp != NULL);
112
113 size = msgdsize(mp);
114
115 mutex_enter(&tcp->tcp_non_sq_lock);
116 tcp->tcp_squeue_bytes += size;
117 if (TCP_UNSENT_BYTES(tcp) > connp->conn_sndbuf) {
118 tcp_setqfull(tcp);
119 }
120 mutex_exit(&tcp->tcp_non_sq_lock);
121
122 CONN_INC_REF(connp);
123 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, tcp_output, connp,
124 NULL, tcp_squeue_flag, SQTAG_TCP_OUTPUT);
125 return;
126
127 case M_CMD:
128 tcp_wput_cmdblk(q, mp);
129 return;
130
131 case M_PROTO:
132 case M_PCPROTO:
133 /*
134 * if it is a snmp message, don't get behind the squeue
135 */
136 tcp = connp->conn_tcp;
137 rptr = mp->b_rptr;
138 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
139 type = ((union T_primitives *)rptr)->type;
140 } else {
141 if (connp->conn_debug) {
142 (void) strlog(TCP_MOD_ID, 0, 1,
143 SL_ERROR|SL_TRACE,
144 "tcp_wput_proto, dropping one...");
145 }
146 freemsg(mp);
147 return;
148 }
149 if (type == T_SVR4_OPTMGMT_REQ) {
150 /*
151 * All Solaris components should pass a db_credp
152 * for this TPI message, hence we ASSERT.
153 * But in case there is some other M_PROTO that looks
154 * like a TPI message sent by some other kernel
155 * component, we check and return an error.
156 */
157 cred_t *cr = msg_getcred(mp, NULL);
158
159 ASSERT(cr != NULL);
160 if (cr == NULL) {
161 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
162 return;
163 }
164 if (snmpcom_req(q, mp, tcp_snmp_set, ip_snmp_get,
165 cr)) {
166 /*
167 * This was a SNMP request
168 */
169 return;
170 } else {
171 output_proc = tcp_wput_proto;
172 }
173 } else {
174 output_proc = tcp_wput_proto;
175 }
176 break;
177 case M_IOCTL:
178 /*
179 * Most ioctls can be processed right away without going via
180 * squeues - process them right here. Those that do require
181 * squeue (currently _SIOCSOCKFALLBACK)
182 * are processed by tcp_wput_ioctl().
183 */
184 iocp = (struct iocblk *)mp->b_rptr;
185 tcp = connp->conn_tcp;
186
187 switch (iocp->ioc_cmd) {
188 case TCP_IOC_ABORT_CONN:
189 tcp_ioctl_abort_conn(q, mp);
190 return;
191 case TI_GETPEERNAME:
192 case TI_GETMYNAME:
193 mi_copyin(q, mp, NULL,
194 SIZEOF_STRUCT(strbuf, iocp->ioc_flag));
195 return;
196
197 default:
198 output_proc = tcp_wput_ioctl;
199 break;
200 }
201 break;
202 default:
203 output_proc = tcp_wput_nondata;
204 break;
205 }
206
207 CONN_INC_REF(connp);
208 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, output_proc, connp,
209 NULL, tcp_squeue_flag, SQTAG_TCP_WPUT_OTHER);
210 }
211
212 /*
213 * The TCP normal data output path.
214 * NOTE: the logic of the fast path is duplicated from this function.
215 */
216 void
217 tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent)
218 {
219 int len;
220 mblk_t *local_time;
221 mblk_t *mp1;
222 uint32_t snxt;
223 int tail_unsent;
224 int tcpstate;
225 int usable = 0;
226 mblk_t *xmit_tail;
227 int32_t mss;
228 int32_t num_sack_blk = 0;
229 int32_t total_hdr_len;
230 int32_t tcp_hdr_len;
231 int rc;
232 conn_t *connp = tcp->tcp_connp;
233 clock_t now = LBOLT_FASTPATH;
234
235 tcpstate = tcp->tcp_state;
236 if (mp == NULL) {
237 /*
238 * tcp_wput_data() with NULL mp should only be called when
239 * there is unsent data.
240 */
241 ASSERT(tcp->tcp_unsent > 0);
242 /* Really tacky... but we need this for detached closes. */
243 len = tcp->tcp_unsent;
244 goto data_null;
245 }
246
247 ASSERT(mp->b_datap->db_type == M_DATA);
248 /*
249 * Don't allow data after T_ORDREL_REQ or T_DISCON_REQ,
250 * or before a connection attempt has begun.
251 */
252 if (tcpstate < TCPS_SYN_SENT || tcpstate > TCPS_CLOSE_WAIT ||
253 (tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
254 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
255 #ifdef DEBUG
256 cmn_err(CE_WARN,
257 "tcp_wput_data: data after ordrel, %s",
258 tcp_display(tcp, NULL,
259 DISP_ADDR_AND_PORT));
260 #else
261 if (connp->conn_debug) {
262 (void) strlog(TCP_MOD_ID, 0, 1,
263 SL_TRACE|SL_ERROR,
264 "tcp_wput_data: data after ordrel, %s\n",
265 tcp_display(tcp, NULL,
266 DISP_ADDR_AND_PORT));
267 }
268 #endif /* DEBUG */
269 }
270 if (tcp->tcp_snd_zcopy_aware &&
271 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
272 tcp_zcopy_notify(tcp);
273 freemsg(mp);
274 mutex_enter(&tcp->tcp_non_sq_lock);
275 if (tcp->tcp_flow_stopped &&
276 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
277 tcp_clrqfull(tcp);
278 }
279 mutex_exit(&tcp->tcp_non_sq_lock);
280 return;
281 }
282
283 /* Strip empties */
284 for (;;) {
285 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <=
286 (uintptr_t)INT_MAX);
287 len = (int)(mp->b_wptr - mp->b_rptr);
288 if (len > 0)
289 break;
290 mp1 = mp;
291 mp = mp->b_cont;
292 freeb(mp1);
293 if (mp == NULL) {
294 return;
295 }
296 }
297
298 /* If we are the first on the list ... */
299 if (tcp->tcp_xmit_head == NULL) {
300 tcp->tcp_xmit_head = mp;
301 tcp->tcp_xmit_tail = mp;
302 tcp->tcp_xmit_tail_unsent = len;
303 } else {
304 /* If tiny tx and room in txq tail, pullup to save mblks. */
305 struct datab *dp;
306
307 mp1 = tcp->tcp_xmit_last;
308 if (len < tcp_tx_pull_len &&
309 (dp = mp1->b_datap)->db_ref == 1 &&
310 dp->db_lim - mp1->b_wptr >= len) {
311 ASSERT(len > 0);
312 ASSERT(!mp1->b_cont);
313 if (len == 1) {
314 *mp1->b_wptr++ = *mp->b_rptr;
315 } else {
316 bcopy(mp->b_rptr, mp1->b_wptr, len);
317 mp1->b_wptr += len;
318 }
319 if (mp1 == tcp->tcp_xmit_tail)
320 tcp->tcp_xmit_tail_unsent += len;
321 mp1->b_cont = mp->b_cont;
322 if (tcp->tcp_snd_zcopy_aware &&
323 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
324 mp1->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
325 freeb(mp);
326 mp = mp1;
327 } else {
328 tcp->tcp_xmit_last->b_cont = mp;
329 }
330 len += tcp->tcp_unsent;
331 }
332
333 /* Tack on however many more positive length mblks we have */
334 if ((mp1 = mp->b_cont) != NULL) {
335 do {
336 int tlen;
337 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <=
338 (uintptr_t)INT_MAX);
339 tlen = (int)(mp1->b_wptr - mp1->b_rptr);
340 if (tlen <= 0) {
341 mp->b_cont = mp1->b_cont;
342 freeb(mp1);
343 } else {
344 len += tlen;
345 mp = mp1;
346 }
347 } while ((mp1 = mp->b_cont) != NULL);
348 }
349 tcp->tcp_xmit_last = mp;
350 tcp->tcp_unsent = len;
351
352 if (urgent)
353 usable = 1;
354
355 data_null:
356 snxt = tcp->tcp_snxt;
357 xmit_tail = tcp->tcp_xmit_tail;
358 tail_unsent = tcp->tcp_xmit_tail_unsent;
359
360 /*
361 * Note that tcp_mss has been adjusted to take into account the
362 * timestamp option if applicable. Because SACK options do not
363 * appear in every TCP segments and they are of variable lengths,
364 * they cannot be included in tcp_mss. Thus we need to calculate
365 * the actual segment length when we need to send a segment which
366 * includes SACK options.
367 */
368 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
369 int32_t opt_len;
370
371 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
372 tcp->tcp_num_sack_blk);
373 opt_len = num_sack_blk * sizeof (sack_blk_t) + TCPOPT_NOP_LEN *
374 2 + TCPOPT_HEADER_LEN;
375 mss = tcp->tcp_mss - opt_len;
376 total_hdr_len = connp->conn_ht_iphc_len + opt_len;
377 tcp_hdr_len = connp->conn_ht_ulp_len + opt_len;
378 } else {
379 mss = tcp->tcp_mss;
380 total_hdr_len = connp->conn_ht_iphc_len;
381 tcp_hdr_len = connp->conn_ht_ulp_len;
382 }
383
384 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
385 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
386 cc_after_idle(tcp);
387 }
388 if (tcpstate == TCPS_SYN_RCVD) {
389 /*
390 * The three-way connection establishment handshake is not
391 * complete yet. We want to queue the data for transmission
392 * after entering ESTABLISHED state (RFC793). A jump to
393 * "done" label effectively leaves data on the queue.
394 */
395 goto done;
396 } else {
397 int usable_r;
398
399 /*
400 * In the special case when cwnd is zero, which can only
401 * happen if the connection is ECN capable, return now.
402 * New segments is sent using tcp_timer(). The timer
403 * is set in tcp_input_data().
404 */
405 if (tcp->tcp_cwnd == 0) {
406 /*
407 * Note that tcp_cwnd is 0 before 3-way handshake is
408 * finished.
409 */
410 ASSERT(tcp->tcp_ecn_ok ||
411 tcp->tcp_state < TCPS_ESTABLISHED);
412 return;
413 }
414
415 /* NOTE: trouble if xmitting while SYN not acked? */
416 usable_r = snxt - tcp->tcp_suna;
417 usable_r = tcp->tcp_swnd - usable_r;
418
419 /*
420 * Check if the receiver has shrunk the window. If
421 * tcp_wput_data() with NULL mp is called, tcp_fin_sent
422 * cannot be set as there is unsent data, so FIN cannot
423 * be sent out. Otherwise, we need to take into account
424 * of FIN as it consumes an "invisible" sequence number.
425 */
426 ASSERT(tcp->tcp_fin_sent == 0);
427 if (usable_r < 0) {
428 /*
429 * The receiver has shrunk the window and we have sent
430 * -usable_r date beyond the window, re-adjust.
431 *
432 * If TCP window scaling is enabled, there can be
433 * round down error as the advertised receive window
434 * is actually right shifted n bits. This means that
435 * the lower n bits info is wiped out. It will look
436 * like the window is shrunk. Do a check here to
437 * see if the shrunk amount is actually within the
438 * error in window calculation. If it is, just
439 * return. Note that this check is inside the
440 * shrunk window check. This makes sure that even
441 * though tcp_process_shrunk_swnd() is not called,
442 * we will stop further processing.
443 */
444 if ((-usable_r >> tcp->tcp_snd_ws) > 0) {
445 tcp_process_shrunk_swnd(tcp, -usable_r);
446 }
447 return;
448 }
449
450 /* usable = MIN(swnd, cwnd) - unacked_bytes */
451 if (tcp->tcp_swnd > tcp->tcp_cwnd)
452 usable_r -= tcp->tcp_swnd - tcp->tcp_cwnd;
453
454 /* usable = MIN(usable, unsent) */
455 if (usable_r > len)
456 usable_r = len;
457
458 /* usable = MAX(usable, {1 for urgent, 0 for data}) */
459 if (usable_r > 0) {
460 usable = usable_r;
461 } else {
462 /* Bypass all other unnecessary processing. */
463 goto done;
464 }
465 }
466
467 #ifdef KERNEL_32
468 local_time = (mblk_t *)now;
469 #else
470 local_time = (mblk_t *)(intptr_t)gethrtime();
471 #endif
472
473 /*
474 * "Our" Nagle Algorithm. This is not the same as in the old
475 * BSD. This is more in line with the true intent of Nagle.
476 *
477 * The conditions are:
478 * 1. The amount of unsent data (or amount of data which can be
479 * sent, whichever is smaller) is less than Nagle limit.
480 * 2. The last sent size is also less than Nagle limit.
481 * 3. There is unack'ed data.
482 * 4. Urgent pointer is not set. Send urgent data ignoring the
483 * Nagle algorithm. This reduces the probability that urgent
484 * bytes get "merged" together.
485 * 5. The app has not closed the connection. This eliminates the
486 * wait time of the receiving side waiting for the last piece of
487 * (small) data.
488 *
489 * If all are satisified, exit without sending anything. Note
490 * that Nagle limit can be smaller than 1 MSS. Nagle limit is
491 * the smaller of 1 MSS and global tcp_naglim_def (default to be
492 * 4095).
493 */
494 if (usable < (int)tcp->tcp_naglim &&
495 tcp->tcp_naglim > tcp->tcp_last_sent_len &&
496 snxt != tcp->tcp_suna &&
497 !(tcp->tcp_valid_bits & TCP_URG_VALID) &&
498 !(tcp->tcp_valid_bits & TCP_FSS_VALID)) {
499 goto done;
500 }
501
502 /*
503 * If tcp_zero_win_probe is not set and the tcp->tcp_cork option
504 * is set, then we have to force TCP not to send partial segment
505 * (smaller than MSS bytes). We are calculating the usable now
506 * based on full mss and will save the rest of remaining data for
507 * later. When tcp_zero_win_probe is set, TCP needs to send out
508 * something to do zero window probe.
509 */
510 if (tcp->tcp_cork && !tcp->tcp_zero_win_probe) {
511 if (usable < mss)
512 goto done;
513 usable = (usable / mss) * mss;
514 }
515
516 /* Update the latest receive window size in TCP header. */
517 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
518
519 /* Send the packet. */
520 rc = tcp_send(tcp, mss, total_hdr_len, tcp_hdr_len,
521 num_sack_blk, &usable, &snxt, &tail_unsent, &xmit_tail,
522 local_time);
523
524 /* Pretend that all we were trying to send really got sent */
525 if (rc < 0 && tail_unsent < 0) {
526 do {
527 xmit_tail = xmit_tail->b_cont;
528 xmit_tail->b_prev = local_time;
529 ASSERT((uintptr_t)(xmit_tail->b_wptr -
530 xmit_tail->b_rptr) <= (uintptr_t)INT_MAX);
531 tail_unsent += (int)(xmit_tail->b_wptr -
532 xmit_tail->b_rptr);
533 } while (tail_unsent < 0);
534 }
535 done:;
536 tcp->tcp_xmit_tail = xmit_tail;
537 tcp->tcp_xmit_tail_unsent = tail_unsent;
538 len = tcp->tcp_snxt - snxt;
539 if (len) {
540 /*
541 * If new data was sent, need to update the notsack
542 * list, which is, afterall, data blocks that have
543 * not been sack'ed by the receiver. New data is
544 * not sack'ed.
545 */
546 if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) {
547 /* len is a negative value. */
548 tcp->tcp_pipe -= len;
549 tcp_notsack_update(&(tcp->tcp_notsack_list),
550 tcp->tcp_snxt, snxt,
551 &(tcp->tcp_num_notsack_blk),
552 &(tcp->tcp_cnt_notsack_list));
553 }
554 tcp->tcp_snxt = snxt + tcp->tcp_fin_sent;
555 tcp->tcp_rack = tcp->tcp_rnxt;
556 tcp->tcp_rack_cnt = 0;
557 if ((snxt + len) == tcp->tcp_suna) {
558 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
559 }
560 } else if (snxt == tcp->tcp_suna && tcp->tcp_swnd == 0) {
561 /*
562 * Didn't send anything. Make sure the timer is running
563 * so that we will probe a zero window.
564 */
565 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
566 }
567 /* Note that len is the amount we just sent but with a negative sign */
568 tcp->tcp_unsent += len;
569 mutex_enter(&tcp->tcp_non_sq_lock);
570 if (tcp->tcp_flow_stopped) {
571 if (TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
572 tcp_clrqfull(tcp);
573 }
574 } else if (TCP_UNSENT_BYTES(tcp) >= connp->conn_sndbuf) {
575 if (!(tcp->tcp_detached))
576 tcp_setqfull(tcp);
577 }
578 mutex_exit(&tcp->tcp_non_sq_lock);
579 }
580
581 /*
582 * Initial STREAMS write side put() procedure for sockets. It tries to
583 * handle the T_CAPABILITY_REQ which sockfs sends down while setting
584 * up the socket without using the squeue. Non T_CAPABILITY_REQ messages
585 * are handled by tcp_wput() as usual.
586 *
587 * All further messages will also be handled by tcp_wput() because we cannot
588 * be sure that the above short cut is safe later.
589 */
590 void
591 tcp_wput_sock(queue_t *wq, mblk_t *mp)
592 {
593 conn_t *connp = Q_TO_CONN(wq);
594 tcp_t *tcp = connp->conn_tcp;
595 struct T_capability_req *car = (struct T_capability_req *)mp->b_rptr;
596
597 ASSERT(wq->q_qinfo == &tcp_sock_winit);
598 wq->q_qinfo = &tcp_winit;
599
600 ASSERT(IPCL_IS_TCP(connp));
601 ASSERT(TCP_IS_SOCKET(tcp));
602
603 if (DB_TYPE(mp) == M_PCPROTO &&
604 MBLKL(mp) == sizeof (struct T_capability_req) &&
605 car->PRIM_type == T_CAPABILITY_REQ) {
606 tcp_capability_req(tcp, mp);
607 return;
608 }
609
610 tcp_wput(wq, mp);
611 }
612
613 /* ARGSUSED */
614 void
615 tcp_wput_fallback(queue_t *wq, mblk_t *mp)
616 {
617 #ifdef DEBUG
618 cmn_err(CE_CONT, "tcp_wput_fallback: Message during fallback \n");
619 #endif
620 freemsg(mp);
621 }
622
623 /*
624 * Call by tcp_wput() to handle misc non M_DATA messages.
625 */
626 /* ARGSUSED */
627 static void
628 tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
629 {
630 conn_t *connp = (conn_t *)arg;
631 tcp_t *tcp = connp->conn_tcp;
632
633 ASSERT(DB_TYPE(mp) != M_IOCTL);
634 /*
635 * TCP is D_MP and qprocsoff() is done towards the end of the tcp_close.
636 * Once the close starts, streamhead and sockfs will not let any data
637 * packets come down (close ensures that there are no threads using the
638 * queue and no new threads will come down) but since qprocsoff()
639 * hasn't happened yet, a M_FLUSH or some non data message might
640 * get reflected back (in response to our own FLUSHRW) and get
641 * processed after tcp_close() is done. The conn would still be valid
642 * because a ref would have added but we need to check the state
643 * before actually processing the packet.
644 */
645 if (TCP_IS_DETACHED(tcp) || (tcp->tcp_state == TCPS_CLOSED)) {
646 freemsg(mp);
647 return;
648 }
649
650 switch (DB_TYPE(mp)) {
651 case M_IOCDATA:
652 tcp_wput_iocdata(tcp, mp);
653 break;
654 case M_FLUSH:
655 tcp_wput_flush(tcp, mp);
656 break;
657 default:
658 ip_wput_nondata(connp->conn_wq, mp);
659 break;
660 }
661 }
662
663 /* tcp_wput_flush is called by tcp_wput_nondata to handle M_FLUSH messages. */
664 static void
665 tcp_wput_flush(tcp_t *tcp, mblk_t *mp)
666 {
667 uchar_t fval = *mp->b_rptr;
668 mblk_t *tail;
669 conn_t *connp = tcp->tcp_connp;
670 queue_t *q = connp->conn_wq;
671
672 /* TODO: How should flush interact with urgent data? */
673 if ((fval & FLUSHW) && tcp->tcp_xmit_head != NULL &&
674 !(tcp->tcp_valid_bits & TCP_URG_VALID)) {
675 /*
676 * Flush only data that has not yet been put on the wire. If
677 * we flush data that we have already transmitted, life, as we
678 * know it, may come to an end.
679 */
680 tail = tcp->tcp_xmit_tail;
681 tail->b_wptr -= tcp->tcp_xmit_tail_unsent;
682 tcp->tcp_xmit_tail_unsent = 0;
683 tcp->tcp_unsent = 0;
684 if (tail->b_wptr != tail->b_rptr)
685 tail = tail->b_cont;
686 if (tail) {
687 mblk_t **excess = &tcp->tcp_xmit_head;
688 for (;;) {
689 mblk_t *mp1 = *excess;
690 if (mp1 == tail)
691 break;
692 tcp->tcp_xmit_tail = mp1;
693 tcp->tcp_xmit_last = mp1;
694 excess = &mp1->b_cont;
695 }
696 *excess = NULL;
697 tcp_close_mpp(&tail);
698 if (tcp->tcp_snd_zcopy_aware)
699 tcp_zcopy_notify(tcp);
700 }
701 /*
702 * We have no unsent data, so unsent must be less than
703 * conn_sndlowat, so re-enable flow.
704 */
705 mutex_enter(&tcp->tcp_non_sq_lock);
706 if (tcp->tcp_flow_stopped) {
707 tcp_clrqfull(tcp);
708 }
709 mutex_exit(&tcp->tcp_non_sq_lock);
710 }
711 /*
712 * TODO: you can't just flush these, you have to increase rwnd for one
713 * thing. For another, how should urgent data interact?
714 */
715 if (fval & FLUSHR) {
716 *mp->b_rptr = fval & ~FLUSHW;
717 /* XXX */
718 qreply(q, mp);
719 return;
720 }
721 freemsg(mp);
722 }
723
724 /*
725 * tcp_wput_iocdata is called by tcp_wput_nondata to handle all M_IOCDATA
726 * messages.
727 */
728 static void
729 tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp)
730 {
731 mblk_t *mp1;
732 struct iocblk *iocp = (struct iocblk *)mp->b_rptr;
733 STRUCT_HANDLE(strbuf, sb);
734 uint_t addrlen;
735 conn_t *connp = tcp->tcp_connp;
736 queue_t *q = connp->conn_wq;
737
738 /* Make sure it is one of ours. */
739 switch (iocp->ioc_cmd) {
740 case TI_GETMYNAME:
741 case TI_GETPEERNAME:
742 break;
743 default:
744 /*
745 * If the conn is closing, then error the ioctl here. Otherwise
746 * use the CONN_IOCTLREF_* macros to hold off tcp_close until
747 * we're done here.
748 */
749 mutex_enter(&connp->conn_lock);
750 if (connp->conn_state_flags & CONN_CLOSING) {
751 mutex_exit(&connp->conn_lock);
752 iocp->ioc_error = EINVAL;
753 mp->b_datap->db_type = M_IOCNAK;
754 iocp->ioc_count = 0;
755 qreply(q, mp);
756 return;
757 }
758
759 CONN_INC_IOCTLREF_LOCKED(connp);
760 ip_wput_nondata(q, mp);
761 CONN_DEC_IOCTLREF(connp);
762 return;
763 }
764 switch (mi_copy_state(q, mp, &mp1)) {
765 case -1:
766 return;
767 case MI_COPY_CASE(MI_COPY_IN, 1):
768 break;
769 case MI_COPY_CASE(MI_COPY_OUT, 1):
770 /* Copy out the strbuf. */
771 mi_copyout(q, mp);
772 return;
773 case MI_COPY_CASE(MI_COPY_OUT, 2):
774 /* All done. */
775 mi_copy_done(q, mp, 0);
776 return;
777 default:
778 mi_copy_done(q, mp, EPROTO);
779 return;
780 }
781 /* Check alignment of the strbuf */
782 if (!OK_32PTR(mp1->b_rptr)) {
783 mi_copy_done(q, mp, EINVAL);
784 return;
785 }
786
787 STRUCT_SET_HANDLE(sb, iocp->ioc_flag, (void *)mp1->b_rptr);
788
789 if (connp->conn_family == AF_INET)
790 addrlen = sizeof (sin_t);
791 else
792 addrlen = sizeof (sin6_t);
793
794 if (STRUCT_FGET(sb, maxlen) < addrlen) {
795 mi_copy_done(q, mp, EINVAL);
796 return;
797 }
798
799 switch (iocp->ioc_cmd) {
800 case TI_GETMYNAME:
801 break;
802 case TI_GETPEERNAME:
803 if (tcp->tcp_state < TCPS_SYN_RCVD) {
804 mi_copy_done(q, mp, ENOTCONN);
805 return;
806 }
807 break;
808 }
809 mp1 = mi_copyout_alloc(q, mp, STRUCT_FGETP(sb, buf), addrlen, B_TRUE);
810 if (!mp1)
811 return;
812
813 STRUCT_FSET(sb, len, addrlen);
814 switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) {
815 case TI_GETMYNAME:
816 (void) conn_getsockname(connp, (struct sockaddr *)mp1->b_wptr,
817 &addrlen);
818 break;
819 case TI_GETPEERNAME:
820 (void) conn_getpeername(connp, (struct sockaddr *)mp1->b_wptr,
821 &addrlen);
822 break;
823 }
824 mp1->b_wptr += addrlen;
825 /* Copy out the address */
826 mi_copyout(q, mp);
827 }
828
829 /*
830 * tcp_wput_ioctl is called by tcp_wput_nondata() to handle all M_IOCTL
831 * messages.
832 */
833 /* ARGSUSED */
834 static void
835 tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
836 {
837 conn_t *connp = (conn_t *)arg;
838 tcp_t *tcp = connp->conn_tcp;
839 queue_t *q = connp->conn_wq;
840 struct iocblk *iocp;
841
842 ASSERT(DB_TYPE(mp) == M_IOCTL);
843 /*
844 * Try and ASSERT the minimum possible references on the
845 * conn early enough. Since we are executing on write side,
846 * the connection is obviously not detached and that means
847 * there is a ref each for TCP and IP. Since we are behind
848 * the squeue, the minimum references needed are 3. If the
849 * conn is in classifier hash list, there should be an
850 * extra ref for that (we check both the possibilities).
851 */
852 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
853 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
854
855 iocp = (struct iocblk *)mp->b_rptr;
856 switch (iocp->ioc_cmd) {
857 case _SIOCSOCKFALLBACK:
858 /*
859 * Either sockmod is about to be popped and the socket
860 * would now be treated as a plain stream, or a module
861 * is about to be pushed so we could no longer use read-
862 * side synchronous streams for fused loopback tcp.
863 * Drain any queued data and disable direct sockfs
864 * interface from now on.
865 */
866 if (!tcp->tcp_issocket) {
867 DB_TYPE(mp) = M_IOCNAK;
868 iocp->ioc_error = EINVAL;
869 } else {
870 tcp_use_pure_tpi(tcp);
871 DB_TYPE(mp) = M_IOCACK;
872 iocp->ioc_error = 0;
873 }
874 iocp->ioc_count = 0;
875 iocp->ioc_rval = 0;
876 qreply(q, mp);
877 return;
878 }
879
880 /*
881 * If the conn is closing, then error the ioctl here. Otherwise bump the
882 * conn_ioctlref to hold off tcp_close until we're done here.
883 */
884 mutex_enter(&(connp)->conn_lock);
885 if ((connp)->conn_state_flags & CONN_CLOSING) {
886 mutex_exit(&(connp)->conn_lock);
887 iocp->ioc_error = EINVAL;
888 mp->b_datap->db_type = M_IOCNAK;
889 iocp->ioc_count = 0;
890 qreply(q, mp);
891 return;
892 }
893
894 CONN_INC_IOCTLREF_LOCKED(connp);
895 ip_wput_nondata(q, mp);
896 CONN_DEC_IOCTLREF(connp);
897 }
898
899 /*
900 * This routine is called by tcp_wput() to handle all TPI requests.
901 */
902 /* ARGSUSED */
903 static void
904 tcp_wput_proto(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
905 {
906 conn_t *connp = (conn_t *)arg;
907 tcp_t *tcp = connp->conn_tcp;
908 union T_primitives *tprim = (union T_primitives *)mp->b_rptr;
909 uchar_t *rptr;
910 t_scalar_t type;
911 cred_t *cr;
912
913 /*
914 * Try and ASSERT the minimum possible references on the
915 * conn early enough. Since we are executing on write side,
916 * the connection is obviously not detached and that means
917 * there is a ref each for TCP and IP. Since we are behind
918 * the squeue, the minimum references needed are 3. If the
919 * conn is in classifier hash list, there should be an
920 * extra ref for that (we check both the possibilities).
921 */
922 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
923 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
924
925 rptr = mp->b_rptr;
926 ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX);
927 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
928 type = ((union T_primitives *)rptr)->type;
929 if (type == T_EXDATA_REQ) {
930 tcp_output_urgent(connp, mp, arg2, NULL);
931 } else if (type != T_DATA_REQ) {
932 goto non_urgent_data;
933 } else {
934 /* TODO: options, flags, ... from user */
935 /* Set length to zero for reclamation below */
936 tcp_wput_data(tcp, mp->b_cont, B_TRUE);
937 freeb(mp);
938 }
939 return;
940 } else {
941 if (connp->conn_debug) {
942 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
943 "tcp_wput_proto, dropping one...");
944 }
945 freemsg(mp);
946 return;
947 }
948
949 non_urgent_data:
950
951 switch ((int)tprim->type) {
952 case O_T_BIND_REQ: /* bind request */
953 case T_BIND_REQ: /* new semantics bind request */
954 tcp_tpi_bind(tcp, mp);
955 break;
956 case T_UNBIND_REQ: /* unbind request */
957 tcp_tpi_unbind(tcp, mp);
958 break;
959 case O_T_CONN_RES: /* old connection response XXX */
960 case T_CONN_RES: /* connection response */
961 tcp_tli_accept(tcp, mp);
962 break;
963 case T_CONN_REQ: /* connection request */
964 tcp_tpi_connect(tcp, mp);
965 break;
966 case T_DISCON_REQ: /* disconnect request */
967 tcp_disconnect(tcp, mp);
968 break;
969 case T_CAPABILITY_REQ:
970 tcp_capability_req(tcp, mp); /* capability request */
971 break;
972 case T_INFO_REQ: /* information request */
973 tcp_info_req(tcp, mp);
974 break;
975 case T_SVR4_OPTMGMT_REQ: /* manage options req */
976 case T_OPTMGMT_REQ:
977 /*
978 * Note: no support for snmpcom_req() through new
979 * T_OPTMGMT_REQ. See comments in ip.c
980 */
981
982 /*
983 * All Solaris components should pass a db_credp
984 * for this TPI message, hence we ASSERT.
985 * But in case there is some other M_PROTO that looks
986 * like a TPI message sent by some other kernel
987 * component, we check and return an error.
988 */
989 cr = msg_getcred(mp, NULL);
990 ASSERT(cr != NULL);
991 if (cr == NULL) {
992 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
993 return;
994 }
995 /*
996 * If EINPROGRESS is returned, the request has been queued
997 * for subsequent processing by ip_restart_optmgmt(), which
998 * will do the CONN_DEC_REF().
999 */
1000 if ((int)tprim->type == T_SVR4_OPTMGMT_REQ) {
1001 svr4_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
1002 } else {
1003 tpi_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
1004 }
1005 break;
1006
1007 case T_UNITDATA_REQ: /* unitdata request */
1008 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
1009 break;
1010 case T_ORDREL_REQ: /* orderly release req */
1011 freemsg(mp);
1012
1013 if (tcp->tcp_fused)
1014 tcp_unfuse(tcp);
1015
1016 if (tcp_xmit_end(tcp) != 0) {
1017 /*
1018 * We were crossing FINs and got a reset from
1019 * the other side. Just ignore it.
1020 */
1021 if (connp->conn_debug) {
1022 (void) strlog(TCP_MOD_ID, 0, 1,
1023 SL_ERROR|SL_TRACE,
1024 "tcp_wput_proto, T_ORDREL_REQ out of "
1025 "state %s",
1026 tcp_display(tcp, NULL,
1027 DISP_ADDR_AND_PORT));
1028 }
1029 }
1030 break;
1031 case T_ADDR_REQ:
1032 tcp_addr_req(tcp, mp);
1033 break;
1034 default:
1035 if (connp->conn_debug) {
1036 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
1037 "tcp_wput_proto, bogus TPI msg, type %d",
1038 tprim->type);
1039 }
1040 /*
1041 * We used to M_ERROR. Sending TNOTSUPPORT gives the user
1042 * to recover.
1043 */
1044 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
1045 break;
1046 }
1047 }
1048
1049 /*
1050 * Handle special out-of-band ioctl requests (see PSARC/2008/265).
1051 */
1052 static void
1053 tcp_wput_cmdblk(queue_t *q, mblk_t *mp)
1054 {
1055 void *data;
1056 mblk_t *datamp = mp->b_cont;
1057 conn_t *connp = Q_TO_CONN(q);
1058 tcp_t *tcp = connp->conn_tcp;
1059 cmdblk_t *cmdp = (cmdblk_t *)mp->b_rptr;
1060
1061 if (datamp == NULL || MBLKL(datamp) < cmdp->cb_len) {
1062 cmdp->cb_error = EPROTO;
1063 qreply(q, mp);
1064 return;
1065 }
1066
1067 data = datamp->b_rptr;
1068
1069 switch (cmdp->cb_cmd) {
1070 case TI_GETPEERNAME:
1071 if (tcp->tcp_state < TCPS_SYN_RCVD)
1072 cmdp->cb_error = ENOTCONN;
1073 else
1074 cmdp->cb_error = conn_getpeername(connp, data,
1075 &cmdp->cb_len);
1076 break;
1077 case TI_GETMYNAME:
1078 cmdp->cb_error = conn_getsockname(connp, data, &cmdp->cb_len);
1079 break;
1080 default:
1081 cmdp->cb_error = EINVAL;
1082 break;
1083 }
1084
1085 qreply(q, mp);
1086 }
1087
1088 /*
1089 * The TCP fast path write put procedure.
1090 * NOTE: the logic of the fast path is duplicated from tcp_wput_data()
1091 */
1092 /* ARGSUSED */
1093 void
1094 tcp_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1095 {
1096 int len;
1097 int hdrlen;
1098 int plen;
1099 mblk_t *mp1;
1100 uchar_t *rptr;
1101 uint32_t snxt;
1102 tcpha_t *tcpha;
1103 struct datab *db;
1104 uint32_t suna;
1105 uint32_t mss;
1106 ipaddr_t *dst;
1107 ipaddr_t *src;
1108 uint32_t sum;
1109 int usable;
1110 conn_t *connp = (conn_t *)arg;
1111 tcp_t *tcp = connp->conn_tcp;
1112 uint32_t msize;
1113 tcp_stack_t *tcps = tcp->tcp_tcps;
1114 ip_xmit_attr_t *ixa;
1115 clock_t now;
1116
1117 /*
1118 * Try and ASSERT the minimum possible references on the
1119 * conn early enough. Since we are executing on write side,
1120 * the connection is obviously not detached and that means
1121 * there is a ref each for TCP and IP. Since we are behind
1122 * the squeue, the minimum references needed are 3. If the
1123 * conn is in classifier hash list, there should be an
1124 * extra ref for that (we check both the possibilities).
1125 */
1126 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
1127 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
1128
1129 ASSERT(DB_TYPE(mp) == M_DATA);
1130 msize = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1131
1132 mutex_enter(&tcp->tcp_non_sq_lock);
1133 tcp->tcp_squeue_bytes -= msize;
1134 mutex_exit(&tcp->tcp_non_sq_lock);
1135
1136 /* Bypass tcp protocol for fused tcp loopback */
1137 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1138 return;
1139
1140 mss = tcp->tcp_mss;
1141 /*
1142 * If ZEROCOPY has turned off, try not to send any zero-copy message
1143 * down. Do backoff, now.
1144 */
1145 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on)
1146 mp = tcp_zcopy_backoff(tcp, mp, B_FALSE);
1147
1148
1149 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX);
1150 len = (int)(mp->b_wptr - mp->b_rptr);
1151
1152 /*
1153 * Criteria for fast path:
1154 *
1155 * 1. no unsent data
1156 * 2. single mblk in request
1157 * 3. connection established
1158 * 4. data in mblk
1159 * 5. len <= mss
1160 * 6. no tcp_valid bits
1161 */
1162 if ((tcp->tcp_unsent != 0) ||
1163 (tcp->tcp_cork) ||
1164 (mp->b_cont != NULL) ||
1165 (tcp->tcp_state != TCPS_ESTABLISHED) ||
1166 (len == 0) ||
1167 (len > mss) ||
1168 (tcp->tcp_valid_bits != 0)) {
1169 tcp_wput_data(tcp, mp, B_FALSE);
1170 return;
1171 }
1172
1173 ASSERT(tcp->tcp_xmit_tail_unsent == 0);
1174 ASSERT(tcp->tcp_fin_sent == 0);
1175
1176 /* queue new packet onto retransmission queue */
1177 if (tcp->tcp_xmit_head == NULL) {
1178 tcp->tcp_xmit_head = mp;
1179 } else {
1180 tcp->tcp_xmit_last->b_cont = mp;
1181 }
1182 tcp->tcp_xmit_last = mp;
1183 tcp->tcp_xmit_tail = mp;
1184
1185 /* find out how much we can send */
1186 /* BEGIN CSTYLED */
1187 /*
1188 * un-acked usable
1189 * |--------------|-----------------|
1190 * tcp_suna tcp_snxt tcp_suna+tcp_swnd
1191 */
1192 /* END CSTYLED */
1193
1194 /* start sending from tcp_snxt */
1195 snxt = tcp->tcp_snxt;
1196
1197 /*
1198 * Check to see if this connection has been idled for some
1199 * time and no ACK is expected. If it is, we need to slow
1200 * start again to get back the connection's "self-clock" as
1201 * described in VJ's paper.
1202 *
1203 * Reinitialize tcp_cwnd after idle.
1204 */
1205 now = LBOLT_FASTPATH;
1206 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
1207 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
1208 cc_after_idle(tcp);
1209 }
1210
1211 usable = tcp->tcp_swnd; /* tcp window size */
1212 if (usable > tcp->tcp_cwnd)
1213 usable = tcp->tcp_cwnd; /* congestion window smaller */
1214 usable -= snxt; /* subtract stuff already sent */
1215 suna = tcp->tcp_suna;
1216 usable += suna;
1217 /* usable can be < 0 if the congestion window is smaller */
1218 if (len > usable) {
1219 /* Can't send complete M_DATA in one shot */
1220 goto slow;
1221 }
1222
1223 mutex_enter(&tcp->tcp_non_sq_lock);
1224 if (tcp->tcp_flow_stopped &&
1225 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
1226 tcp_clrqfull(tcp);
1227 }
1228 mutex_exit(&tcp->tcp_non_sq_lock);
1229
1230 /*
1231 * determine if anything to send (Nagle).
1232 *
1233 * 1. len < tcp_mss (i.e. small)
1234 * 2. unacknowledged data present
1235 * 3. len < nagle limit
1236 * 4. last packet sent < nagle limit (previous packet sent)
1237 */
1238 if ((len < mss) && (snxt != suna) &&
1239 (len < (int)tcp->tcp_naglim) &&
1240 (tcp->tcp_last_sent_len < tcp->tcp_naglim)) {
1241 /*
1242 * This was the first unsent packet and normally
1243 * mss < xmit_hiwater so there is no need to worry
1244 * about flow control. The next packet will go
1245 * through the flow control check in tcp_wput_data().
1246 */
1247 /* leftover work from above */
1248 tcp->tcp_unsent = len;
1249 tcp->tcp_xmit_tail_unsent = len;
1250
1251 return;
1252 }
1253
1254 /*
1255 * len <= tcp->tcp_mss && len == unsent so no sender silly window. Can
1256 * send now.
1257 */
1258
1259 if (snxt == suna) {
1260 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1261 }
1262
1263 /* we have always sent something */
1264 tcp->tcp_rack_cnt = 0;
1265
1266 tcp->tcp_snxt = snxt + len;
1267 tcp->tcp_rack = tcp->tcp_rnxt;
1268
1269 if ((mp1 = dupb(mp)) == 0)
1270 goto no_memory;
1271 #ifdef KERNEL_32
1272 mp->b_prev = (mblk_t *)(uintptr_t)now;
1273 #else
1274 mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
1275 #endif
1276 mp->b_next = (mblk_t *)(uintptr_t)snxt;
1277
1278 /* adjust tcp header information */
1279 tcpha = tcp->tcp_tcpha;
1280 tcpha->tha_flags = (TH_ACK|TH_PUSH);
1281
1282 sum = len + connp->conn_ht_ulp_len + connp->conn_sum;
1283 sum = (sum >> 16) + (sum & 0xFFFF);
1284 tcpha->tha_sum = htons(sum);
1285
1286 tcpha->tha_seq = htonl(snxt);
1287
1288 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1289 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1290 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1291 tcp->tcp_cs.tcp_out_data_segs++;
1292 tcp->tcp_cs.tcp_out_data_bytes += len;
1293
1294 /* Update the latest receive window size in TCP header. */
1295 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
1296
1297 tcp->tcp_last_sent_len = (ushort_t)len;
1298
1299 plen = len + connp->conn_ht_iphc_len;
1300
1301 ixa = connp->conn_ixa;
1302 ixa->ixa_pktlen = plen;
1303
1304 if (ixa->ixa_flags & IXAF_IS_IPV4) {
1305 tcp->tcp_ipha->ipha_length = htons(plen);
1306 } else {
1307 tcp->tcp_ip6h->ip6_plen = htons(plen - IPV6_HDR_LEN);
1308 }
1309
1310 /* see if we need to allocate a mblk for the headers */
1311 hdrlen = connp->conn_ht_iphc_len;
1312 rptr = mp1->b_rptr - hdrlen;
1313 db = mp1->b_datap;
1314 if ((db->db_ref != 2) || rptr < db->db_base ||
1315 (!OK_32PTR(rptr))) {
1316 /* NOTE: we assume allocb returns an OK_32PTR */
1317 mp = allocb(hdrlen + tcps->tcps_wroff_xtra, BPRI_MED);
1318 if (!mp) {
1319 freemsg(mp1);
1320 goto no_memory;
1321 }
1322 mp->b_cont = mp1;
1323 mp1 = mp;
1324 /* Leave room for Link Level header */
1325 rptr = &mp1->b_rptr[tcps->tcps_wroff_xtra];
1326 mp1->b_wptr = &rptr[hdrlen];
1327 }
1328 mp1->b_rptr = rptr;
1329
1330 /* Fill in the timestamp option. */
1331 if (tcp->tcp_snd_ts_ok) {
1332 U32_TO_BE32(now,
1333 (char *)tcpha + TCP_MIN_HEADER_LENGTH + 4);
1334 U32_TO_BE32(tcp->tcp_ts_recent,
1335 (char *)tcpha + TCP_MIN_HEADER_LENGTH + 8);
1336 } else {
1337 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
1338 }
1339
1340 /* copy header into outgoing packet */
1341 dst = (ipaddr_t *)rptr;
1342 src = (ipaddr_t *)connp->conn_ht_iphc;
1343 dst[0] = src[0];
1344 dst[1] = src[1];
1345 dst[2] = src[2];
1346 dst[3] = src[3];
1347 dst[4] = src[4];
1348 dst[5] = src[5];
1349 dst[6] = src[6];
1350 dst[7] = src[7];
1351 dst[8] = src[8];
1352 dst[9] = src[9];
1353 if (hdrlen -= 40) {
1354 hdrlen >>= 2;
1355 dst += 10;
1356 src += 10;
1357 do {
1358 *dst++ = *src++;
1359 } while (--hdrlen);
1360 }
1361
1362 /*
1363 * Set the ECN info in the TCP header. Note that this
1364 * is not the template header.
1365 */
1366 if (tcp->tcp_ecn_ok) {
1367 TCP_SET_ECT(tcp, rptr);
1368
1369 tcpha = (tcpha_t *)(rptr + ixa->ixa_ip_hdr_length);
1370 if (tcp->tcp_ecn_echo_on)
1371 tcpha->tha_flags |= TH_ECE;
1372 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
1373 tcpha->tha_flags |= TH_CWR;
1374 tcp->tcp_ecn_cwr_sent = B_TRUE;
1375 }
1376 }
1377
1378 if (tcp->tcp_ip_forward_progress) {
1379 tcp->tcp_ip_forward_progress = B_FALSE;
1380 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
1381 } else {
1382 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
1383 }
1384 tcp_send_data(tcp, mp1);
1385 return;
1386
1387 /*
1388 * If we ran out of memory, we pretend to have sent the packet
1389 * and that it was lost on the wire.
1390 */
1391 no_memory:
1392 return;
1393
1394 slow:
1395 /* leftover work from above */
1396 tcp->tcp_unsent = len;
1397 tcp->tcp_xmit_tail_unsent = len;
1398 tcp_wput_data(tcp, NULL, B_FALSE);
1399 }
1400
1401 /* ARGSUSED2 */
1402 void
1403 tcp_output_urgent(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1404 {
1405 int len;
1406 uint32_t msize;
1407 conn_t *connp = (conn_t *)arg;
1408 tcp_t *tcp = connp->conn_tcp;
1409
1410 msize = msgdsize(mp);
1411
1412 len = msize - 1;
1413 if (len < 0) {
1414 freemsg(mp);
1415 return;
1416 }
1417
1418 /*
1419 * Try to force urgent data out on the wire. Even if we have unsent
1420 * data this will at least send the urgent flag.
1421 * XXX does not handle more flag correctly.
1422 */
1423 len += tcp->tcp_unsent;
1424 len += tcp->tcp_snxt;
1425 tcp->tcp_urg = len;
1426 tcp->tcp_valid_bits |= TCP_URG_VALID;
1427
1428 /* Bypass tcp protocol for fused tcp loopback */
1429 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1430 return;
1431
1432 /* Strip off the T_EXDATA_REQ if the data is from TPI */
1433 if (DB_TYPE(mp) != M_DATA) {
1434 mblk_t *mp1 = mp;
1435 ASSERT(!IPCL_IS_NONSTR(connp));
1436 mp = mp->b_cont;
1437 freeb(mp1);
1438 }
1439 tcp_wput_data(tcp, mp, B_TRUE);
1440 }
1441
1442 /*
1443 * Called by streams close routine via squeues when our client blows off its
1444 * descriptor, we take this to mean: "close the stream state NOW, close the tcp
1445 * connection politely" When SO_LINGER is set (with a non-zero linger time and
1446 * it is not a nonblocking socket) then this routine sleeps until the FIN is
1447 * acked.
1448 *
1449 * NOTE: tcp_close potentially returns error when lingering.
1450 * However, the stream head currently does not pass these errors
1451 * to the application. 4.4BSD only returns EINTR and EWOULDBLOCK
1452 * errors to the application (from tsleep()) and not errors
1453 * like ECONNRESET caused by receiving a reset packet.
1454 */
1455
1456 /* ARGSUSED */
1457 void
1458 tcp_close_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1459 {
1460 char *msg;
1461 conn_t *connp = (conn_t *)arg;
1462 tcp_t *tcp = connp->conn_tcp;
1463 clock_t delta = 0;
1464 tcp_stack_t *tcps = tcp->tcp_tcps;
1465
1466 /*
1467 * When a non-STREAMS socket is being closed, it does not always
1468 * stick around waiting for tcp_close_output to run and can therefore
1469 * have dropped a reference already. So adjust the asserts accordingly.
1470 */
1471 ASSERT((connp->conn_fanout != NULL &&
1472 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 3 : 4)) ||
1473 (connp->conn_fanout == NULL &&
1474 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3)));
1475
1476 mutex_enter(&tcp->tcp_eager_lock);
1477 if (tcp->tcp_conn_req_cnt_q0 != 0 || tcp->tcp_conn_req_cnt_q != 0) {
1478 /*
1479 * Cleanup for listener. For non-STREAM sockets sockfs will
1480 * close all the eagers on 'q', so in that case only deal
1481 * with 'q0'.
1482 */
1483 tcp_eager_cleanup(tcp, IPCL_IS_NONSTR(connp) ? 1 : 0);
1484 tcp->tcp_wait_for_eagers = 1;
1485 }
1486 mutex_exit(&tcp->tcp_eager_lock);
1487
1488 tcp->tcp_lso = B_FALSE;
1489
1490 msg = NULL;
1491 switch (tcp->tcp_state) {
1492 case TCPS_CLOSED:
1493 case TCPS_IDLE:
1494 break;
1495 case TCPS_BOUND:
1496 if (tcp->tcp_listener != NULL) {
1497 ASSERT(IPCL_IS_NONSTR(connp));
1498 /*
1499 * Unlink from the listener and drop the reference
1500 * put on it by the eager. tcp_closei_local will not
1501 * do it because tcp_tconnind_started is TRUE.
1502 */
1503 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1504 tcp_eager_unlink(tcp);
1505 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1506 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1507 }
1508 break;
1509 case TCPS_LISTEN:
1510 break;
1511 case TCPS_SYN_SENT:
1512 msg = "tcp_close, during connect";
1513 break;
1514 case TCPS_SYN_RCVD:
1515 /*
1516 * Close during the connect 3-way handshake
1517 * but here there may or may not be pending data
1518 * already on queue. Process almost same as in
1519 * the ESTABLISHED state.
1520 */
1521 /* FALLTHRU */
1522 default:
1523 if (tcp->tcp_fused)
1524 tcp_unfuse(tcp);
1525
1526 /*
1527 * If SO_LINGER has set a zero linger time, abort the
1528 * connection with a reset.
1529 */
1530 if (connp->conn_linger && connp->conn_lingertime == 0) {
1531 msg = "tcp_close, zero lingertime";
1532 break;
1533 }
1534
1535 /*
1536 * Abort connection if there is unread data queued.
1537 */
1538 if (tcp->tcp_rcv_list || tcp->tcp_reass_head) {
1539 msg = "tcp_close, unread data";
1540 break;
1541 }
1542
1543 /*
1544 * Abort connection if it is being closed without first
1545 * being accepted. This can happen if a listening non-STREAM
1546 * socket wants to get rid of the socket, for example, if the
1547 * listener is closing.
1548 */
1549 if (tcp->tcp_listener != NULL) {
1550 ASSERT(IPCL_IS_NONSTR(connp));
1551 msg = "tcp_close, close before accept";
1552
1553 /*
1554 * Unlink from the listener and drop the reference
1555 * put on it by the eager. tcp_closei_local will not
1556 * do it because tcp_tconnind_started is TRUE.
1557 */
1558 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1559 tcp_eager_unlink(tcp);
1560 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1561 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1562 break;
1563 }
1564
1565 /*
1566 * Transmit the FIN before detaching the tcp_t.
1567 * After tcp_detach returns this queue/perimeter
1568 * no longer owns the tcp_t thus others can modify it.
1569 */
1570 (void) tcp_xmit_end(tcp);
1571
1572 /*
1573 * If lingering on close then wait until the fin is acked,
1574 * the SO_LINGER time passes, or a reset is sent/received.
1575 */
1576 if (connp->conn_linger && connp->conn_lingertime > 0 &&
1577 !(tcp->tcp_fin_acked) &&
1578 tcp->tcp_state >= TCPS_ESTABLISHED) {
1579 if (tcp->tcp_closeflags & (FNDELAY|FNONBLOCK)) {
1580 tcp->tcp_client_errno = EWOULDBLOCK;
1581 } else if (tcp->tcp_client_errno == 0) {
1582
1583 ASSERT(tcp->tcp_linger_tid == 0);
1584
1585 /* conn_lingertime is in sec. */
1586 tcp->tcp_linger_tid = TCP_TIMER(tcp,
1587 tcp_close_linger_timeout,
1588 connp->conn_lingertime * MILLISEC);
1589
1590 /* tcp_close_linger_timeout will finish close */
1591 if (tcp->tcp_linger_tid == 0)
1592 tcp->tcp_client_errno = ENOSR;
1593 else
1594 return;
1595 }
1596
1597 /*
1598 * Check if we need to detach or just close
1599 * the instance.
1600 */
1601 if (tcp->tcp_state <= TCPS_LISTEN)
1602 break;
1603 }
1604
1605 /*
1606 * Make sure that no other thread will access the conn_rq of
1607 * this instance (through lookups etc.) as conn_rq will go
1608 * away shortly.
1609 */
1610 tcp_acceptor_hash_remove(tcp);
1611
1612 mutex_enter(&tcp->tcp_non_sq_lock);
1613 if (tcp->tcp_flow_stopped) {
1614 tcp_clrqfull(tcp);
1615 }
1616 mutex_exit(&tcp->tcp_non_sq_lock);
1617
1618 if (tcp->tcp_timer_tid != 0) {
1619 delta = TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid);
1620 tcp->tcp_timer_tid = 0;
1621 }
1622 /*
1623 * Need to cancel those timers which will not be used when
1624 * TCP is detached. This has to be done before the conn_wq
1625 * is set to NULL.
1626 */
1627 tcp_timers_stop(tcp);
1628
1629 tcp->tcp_detached = B_TRUE;
1630 if (tcp->tcp_state == TCPS_TIME_WAIT) {
1631 tcp_time_wait_append(tcp);
1632 TCP_DBGSTAT(tcps, tcp_detach_time_wait);
1633 ASSERT(connp->conn_ref >=
1634 (IPCL_IS_NONSTR(connp) ? 2 : 3));
1635 goto finish;
1636 }
1637
1638 /*
1639 * If delta is zero the timer event wasn't executed and was
1640 * successfully canceled. In this case we need to restart it
1641 * with the minimal delta possible.
1642 */
1643 if (delta >= 0)
1644 tcp->tcp_timer_tid = TCP_TIMER(tcp, tcp_timer,
1645 delta ? delta : 1);
1646
1647 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3));
1648 goto finish;
1649 }
1650
1651 /* Detach did not complete. Still need to remove q from stream. */
1652 if (msg) {
1653 if (tcp->tcp_state == TCPS_ESTABLISHED ||
1654 tcp->tcp_state == TCPS_CLOSE_WAIT)
1655 TCPS_BUMP_MIB(tcps, tcpEstabResets);
1656 if (tcp->tcp_state == TCPS_SYN_SENT ||
1657 tcp->tcp_state == TCPS_SYN_RCVD)
1658 TCPS_BUMP_MIB(tcps, tcpAttemptFails);
1659 tcp_xmit_ctl(msg, tcp, tcp->tcp_snxt, 0, TH_RST);
1660 }
1661
1662 tcp_closei_local(tcp);
1663 CONN_DEC_REF(connp);
1664 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 1 : 2));
1665
1666 finish:
1667 /*
1668 * Don't change the queues in the case of a listener that has
1669 * eagers in its q or q0. It could surprise the eagers.
1670 * Instead wait for the eagers outside the squeue.
1671 *
1672 * For non-STREAMS sockets tcp_wait_for_eagers implies that
1673 * we should delay the su_closed upcall until all eagers have
1674 * dropped their references.
1675 */
1676 if (!tcp->tcp_wait_for_eagers) {
1677 tcp->tcp_detached = B_TRUE;
1678 connp->conn_rq = NULL;
1679 connp->conn_wq = NULL;
1680
1681 /* non-STREAM socket, release the upper handle */
1682 if (IPCL_IS_NONSTR(connp)) {
1683 ASSERT(connp->conn_upper_handle != NULL);
1684 (*connp->conn_upcalls->su_closed)
1685 (connp->conn_upper_handle);
1686 connp->conn_upper_handle = NULL;
1687 connp->conn_upcalls = NULL;
1688 }
1689 }
1690
1691 /* Signal tcp_close() to finish closing. */
1692 mutex_enter(&tcp->tcp_closelock);
1693 tcp->tcp_closed = 1;
1694 cv_signal(&tcp->tcp_closecv);
1695 mutex_exit(&tcp->tcp_closelock);
1696 }
1697
1698 /* ARGSUSED */
1699 void
1700 tcp_shutdown_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1701 {
1702 conn_t *connp = (conn_t *)arg;
1703 tcp_t *tcp = connp->conn_tcp;
1704
1705 freemsg(mp);
1706
1707 if (tcp->tcp_fused)
1708 tcp_unfuse(tcp);
1709
1710 if (tcp_xmit_end(tcp) != 0) {
1711 /*
1712 * We were crossing FINs and got a reset from
1713 * the other side. Just ignore it.
1714 */
1715 if (connp->conn_debug) {
1716 (void) strlog(TCP_MOD_ID, 0, 1,
1717 SL_ERROR|SL_TRACE,
1718 "tcp_shutdown_output() out of state %s",
1719 tcp_display(tcp, NULL, DISP_ADDR_AND_PORT));
1720 }
1721 }
1722 }
1723
1724 #pragma inline(tcp_send_data)
1725
1726 void
1727 tcp_send_data(tcp_t *tcp, mblk_t *mp)
1728 {
1729 conn_t *connp = tcp->tcp_connp;
1730
1731 /*
1732 * Check here to avoid sending zero-copy message down to IP when
1733 * ZEROCOPY capability has turned off. We only need to deal with
1734 * the race condition between sockfs and the notification here.
1735 * Since we have tried to backoff the tcp_xmit_head when turning
1736 * zero-copy off and new messages in tcp_output(), we simply drop
1737 * the dup'ed packet here and let tcp retransmit, if tcp_xmit_zc_clean
1738 * is not true.
1739 */
1740 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on &&
1741 !tcp->tcp_xmit_zc_clean) {
1742 ip_drop_output("TCP ZC was disabled but not clean", mp, NULL);
1743 freemsg(mp);
1744 return;
1745 }
1746
1747 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, connp->conn_ixa,
1748 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, tcp,
1749 __dtrace_tcp_tcph_t *,
1750 &mp->b_rptr[connp->conn_ixa->ixa_ip_hdr_length]);
1751
1752 ASSERT(connp->conn_ixa->ixa_notify_cookie == connp->conn_tcp);
1753 (void) conn_ip_output(mp, connp->conn_ixa);
1754 }
1755
1756 /* ARGSUSED2 */
1757 void
1758 tcp_send_synack(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1759 {
1760 conn_t *econnp = (conn_t *)arg;
1761 tcp_t *tcp = econnp->conn_tcp;
1762 ip_xmit_attr_t *ixa = econnp->conn_ixa;
1763
1764 /* Guard against a RST having blown it away while on the squeue */
1765 if (tcp->tcp_state == TCPS_CLOSED) {
1766 freemsg(mp);
1767 return;
1768 }
1769
1770 /*
1771 * In the off-chance that the eager received and responded to
1772 * some other packet while the SYN|ACK was queued, we recalculate
1773 * the ixa_pktlen. It would be better to fix the SYN/accept
1774 * multithreading scheme to avoid this complexity.
1775 */
1776 ixa->ixa_pktlen = msgdsize(mp);
1777 (void) conn_ip_output(mp, ixa);
1778 }
1779
1780 /*
1781 * tcp_send() is called by tcp_wput_data() and returns one of the following:
1782 *
1783 * -1 = failed allocation.
1784 * 0 = We've either successfully sent data, or our usable send window is too
1785 * small and we'd rather wait until later before sending again.
1786 */
1787 static int
1788 tcp_send(tcp_t *tcp, const int mss, const int total_hdr_len,
1789 const int tcp_hdr_len, const int num_sack_blk, int *usable,
1790 uint_t *snxt, int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time)
1791 {
1792 int num_lso_seg = 1;
1793 uint_t lso_usable;
1794 boolean_t do_lso_send = B_FALSE;
1795 tcp_stack_t *tcps = tcp->tcp_tcps;
1796 conn_t *connp = tcp->tcp_connp;
1797 ip_xmit_attr_t *ixa = connp->conn_ixa;
1798
1799 /*
1800 * Check LSO possibility. The value of tcp->tcp_lso indicates whether
1801 * the underlying connection is LSO capable. Will check whether having
1802 * enough available data to initiate LSO transmission in the for(){}
1803 * loops.
1804 */
1805 if (tcp->tcp_lso && (tcp->tcp_valid_bits & ~TCP_FSS_VALID) == 0)
1806 do_lso_send = B_TRUE;
1807
1808 for (;;) {
1809 struct datab *db;
1810 tcpha_t *tcpha;
1811 uint32_t sum;
1812 mblk_t *mp, *mp1;
1813 uchar_t *rptr;
1814 int len;
1815
1816 /*
1817 * Calculate the maximum payload length we can send at one
1818 * time.
1819 */
1820 if (do_lso_send) {
1821 /*
1822 * Determine whether or not it's possible to do LSO,
1823 * and if so, how much data we can send.
1824 */
1825 if ((*usable - 1) / mss >= 1) {
1826 lso_usable = MIN(tcp->tcp_lso_max, *usable);
1827 num_lso_seg = lso_usable / mss;
1828 if (lso_usable % mss) {
1829 num_lso_seg++;
1830 tcp->tcp_last_sent_len = (ushort_t)
1831 (lso_usable % mss);
1832 } else {
1833 tcp->tcp_last_sent_len = (ushort_t)mss;
1834 }
1835 } else {
1836 do_lso_send = B_FALSE;
1837 num_lso_seg = 1;
1838 lso_usable = mss;
1839 }
1840 }
1841
1842 ASSERT(num_lso_seg <= IP_MAXPACKET / mss + 1);
1843
1844 len = mss;
1845 if (len > *usable) {
1846 ASSERT(do_lso_send == B_FALSE);
1847
1848 len = *usable;
1849 if (len <= 0) {
1850 /* Terminate the loop */
1851 break; /* success; too small */
1852 }
1853 /*
1854 * Sender silly-window avoidance.
1855 * Ignore this if we are going to send a
1856 * zero window probe out.
1857 *
1858 * TODO: force data into microscopic window?
1859 * ==> (!pushed || (unsent > usable))
1860 */
1861 if (len < (tcp->tcp_max_swnd >> 1) &&
1862 (tcp->tcp_unsent - (*snxt - tcp->tcp_snxt)) > len &&
1863 !((tcp->tcp_valid_bits & TCP_URG_VALID) &&
1864 len == 1) && (! tcp->tcp_zero_win_probe)) {
1865 /*
1866 * If the retransmit timer is not running
1867 * we start it so that we will retransmit
1868 * in the case when the receiver has
1869 * decremented the window.
1870 */
1871 if (*snxt == tcp->tcp_snxt &&
1872 *snxt == tcp->tcp_suna) {
1873 /*
1874 * We are not supposed to send
1875 * anything. So let's wait a little
1876 * bit longer before breaking SWS
1877 * avoidance.
1878 *
1879 * What should the value be?
1880 * Suggestion: MAX(init rexmit time,
1881 * tcp->tcp_rto)
1882 */
1883 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1884 }
1885 break; /* success; too small */
1886 }
1887 }
1888
1889 tcpha = tcp->tcp_tcpha;
1890
1891 /*
1892 * The reason to adjust len here is that we need to set flags
1893 * and calculate checksum.
1894 */
1895 if (do_lso_send)
1896 len = lso_usable;
1897
1898 *usable -= len; /* Approximate - can be adjusted later */
1899 if (*usable > 0)
1900 tcpha->tha_flags = TH_ACK;
1901 else
1902 tcpha->tha_flags = (TH_ACK | TH_PUSH);
1903
1904 /*
1905 * Prime pump for IP's checksumming on our behalf.
1906 * Include the adjustment for a source route if any.
1907 * In case of LSO, the partial pseudo-header checksum should
1908 * exclusive TCP length, so zero tha_sum before IP calculate
1909 * pseudo-header checksum for partial checksum offload.
1910 */
1911 if (do_lso_send) {
1912 sum = 0;
1913 } else {
1914 sum = len + tcp_hdr_len + connp->conn_sum;
1915 sum = (sum >> 16) + (sum & 0xFFFF);
1916 }
1917 tcpha->tha_sum = htons(sum);
1918 tcpha->tha_seq = htonl(*snxt);
1919
1920 /*
1921 * Branch off to tcp_xmit_mp() if any of the VALID bits is
1922 * set. For the case when TCP_FSS_VALID is the only valid
1923 * bit (normal active close), branch off only when we think
1924 * that the FIN flag needs to be set. Note for this case,
1925 * that (snxt + len) may not reflect the actual seg_len,
1926 * as len may be further reduced in tcp_xmit_mp(). If len
1927 * gets modified, we will end up here again.
1928 */
1929 if (tcp->tcp_valid_bits != 0 &&
1930 (tcp->tcp_valid_bits != TCP_FSS_VALID ||
1931 ((*snxt + len) == tcp->tcp_fss))) {
1932 uchar_t *prev_rptr;
1933 uint32_t prev_snxt = tcp->tcp_snxt;
1934
1935 if (*tail_unsent == 0) {
1936 ASSERT((*xmit_tail)->b_cont != NULL);
1937 *xmit_tail = (*xmit_tail)->b_cont;
1938 prev_rptr = (*xmit_tail)->b_rptr;
1939 *tail_unsent = (int)((*xmit_tail)->b_wptr -
1940 (*xmit_tail)->b_rptr);
1941 } else {
1942 prev_rptr = (*xmit_tail)->b_rptr;
1943 (*xmit_tail)->b_rptr = (*xmit_tail)->b_wptr -
1944 *tail_unsent;
1945 }
1946 mp = tcp_xmit_mp(tcp, *xmit_tail, len, NULL, NULL,
1947 *snxt, B_FALSE, (uint32_t *)&len, B_FALSE);
1948 /* Restore tcp_snxt so we get amount sent right. */
1949 tcp->tcp_snxt = prev_snxt;
1950 if (prev_rptr == (*xmit_tail)->b_rptr) {
1951 /*
1952 * If the previous timestamp is still in use,
1953 * don't stomp on it.
1954 */
1955 if ((*xmit_tail)->b_next == NULL) {
1956 (*xmit_tail)->b_prev = local_time;
1957 (*xmit_tail)->b_next =
1958 (mblk_t *)(uintptr_t)(*snxt);
1959 }
1960 } else
1961 (*xmit_tail)->b_rptr = prev_rptr;
1962
1963 if (mp == NULL) {
1964 return (-1);
1965 }
1966 mp1 = mp->b_cont;
1967
1968 if (len <= mss) /* LSO is unusable (!do_lso_send) */
1969 tcp->tcp_last_sent_len = (ushort_t)len;
1970 while (mp1->b_cont) {
1971 *xmit_tail = (*xmit_tail)->b_cont;
1972 (*xmit_tail)->b_prev = local_time;
1973 (*xmit_tail)->b_next =
1974 (mblk_t *)(uintptr_t)(*snxt);
1975 mp1 = mp1->b_cont;
1976 }
1977 *snxt += len;
1978 *tail_unsent = (*xmit_tail)->b_wptr - mp1->b_wptr;
1979 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1980 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1981 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1982 tcp->tcp_cs.tcp_out_data_segs++;
1983 tcp->tcp_cs.tcp_out_data_bytes += len;
1984 tcp_send_data(tcp, mp);
1985 continue;
1986 }
1987
1988 *snxt += len; /* Adjust later if we don't send all of len */
1989 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
1990 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1991 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1992 tcp->tcp_cs.tcp_out_data_segs++;
1993 tcp->tcp_cs.tcp_out_data_bytes += len;
1994
1995 if (*tail_unsent) {
1996 /* Are the bytes above us in flight? */
1997 rptr = (*xmit_tail)->b_wptr - *tail_unsent;
1998 if (rptr != (*xmit_tail)->b_rptr) {
1999 *tail_unsent -= len;
2000 if (len <= mss) /* LSO is unusable */
2001 tcp->tcp_last_sent_len = (ushort_t)len;
2002 len += total_hdr_len;
2003 ixa->ixa_pktlen = len;
2004
2005 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2006 tcp->tcp_ipha->ipha_length = htons(len);
2007 } else {
2008 tcp->tcp_ip6h->ip6_plen =
2009 htons(len - IPV6_HDR_LEN);
2010 }
2011
2012 mp = dupb(*xmit_tail);
2013 if (mp == NULL) {
2014 return (-1); /* out_of_mem */
2015 }
2016 mp->b_rptr = rptr;
2017 /*
2018 * If the old timestamp is no longer in use,
2019 * sample a new timestamp now.
2020 */
2021 if ((*xmit_tail)->b_next == NULL) {
2022 (*xmit_tail)->b_prev = local_time;
2023 (*xmit_tail)->b_next =
2024 (mblk_t *)(uintptr_t)(*snxt-len);
2025 }
2026 goto must_alloc;
2027 }
2028 } else {
2029 *xmit_tail = (*xmit_tail)->b_cont;
2030 ASSERT((uintptr_t)((*xmit_tail)->b_wptr -
2031 (*xmit_tail)->b_rptr) <= (uintptr_t)INT_MAX);
2032 *tail_unsent = (int)((*xmit_tail)->b_wptr -
2033 (*xmit_tail)->b_rptr);
2034 }
2035
2036 (*xmit_tail)->b_prev = local_time;
2037 (*xmit_tail)->b_next = (mblk_t *)(uintptr_t)(*snxt - len);
2038
2039 *tail_unsent -= len;
2040 if (len <= mss) /* LSO is unusable (!do_lso_send) */
2041 tcp->tcp_last_sent_len = (ushort_t)len;
2042
2043 len += total_hdr_len;
2044 ixa->ixa_pktlen = len;
2045
2046 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2047 tcp->tcp_ipha->ipha_length = htons(len);
2048 } else {
2049 tcp->tcp_ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2050 }
2051
2052 mp = dupb(*xmit_tail);
2053 if (mp == NULL) {
2054 return (-1); /* out_of_mem */
2055 }
2056
2057 len = total_hdr_len;
2058 /*
2059 * There are four reasons to allocate a new hdr mblk:
2060 * 1) The bytes above us are in use by another packet
2061 * 2) We don't have good alignment
2062 * 3) The mblk is being shared
2063 * 4) We don't have enough room for a header
2064 */
2065 rptr = mp->b_rptr - len;
2066 if (!OK_32PTR(rptr) ||
2067 ((db = mp->b_datap), db->db_ref != 2) ||
2068 rptr < db->db_base) {
2069 /* NOTE: we assume allocb returns an OK_32PTR */
2070
2071 must_alloc:;
2072 mp1 = allocb(connp->conn_ht_iphc_allocated +
2073 tcps->tcps_wroff_xtra, BPRI_MED);
2074 if (mp1 == NULL) {
2075 freemsg(mp);
2076 return (-1); /* out_of_mem */
2077 }
2078 mp1->b_cont = mp;
2079 mp = mp1;
2080 /* Leave room for Link Level header */
2081 len = total_hdr_len;
2082 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2083 mp->b_wptr = &rptr[len];
2084 }
2085
2086 /*
2087 * Fill in the header using the template header, and add
2088 * options such as time-stamp, ECN and/or SACK, as needed.
2089 */
2090 tcp_fill_header(tcp, rptr, num_sack_blk);
2091
2092 mp->b_rptr = rptr;
2093
2094 if (*tail_unsent) {
2095 int spill = *tail_unsent;
2096
2097 mp1 = mp->b_cont;
2098 if (mp1 == NULL)
2099 mp1 = mp;
2100
2101 /*
2102 * If we're a little short, tack on more mblks until
2103 * there is no more spillover.
2104 */
2105 while (spill < 0) {
2106 mblk_t *nmp;
2107 int nmpsz;
2108
2109 nmp = (*xmit_tail)->b_cont;
2110 nmpsz = MBLKL(nmp);
2111
2112 /*
2113 * Excess data in mblk; can we split it?
2114 * If LSO is enabled for the connection,
2115 * keep on splitting as this is a transient
2116 * send path.
2117 */
2118 if (!do_lso_send && (spill + nmpsz > 0)) {
2119 /*
2120 * Don't split if stream head was
2121 * told to break up larger writes
2122 * into smaller ones.
2123 */
2124 if (tcp->tcp_maxpsz_multiplier > 0)
2125 break;
2126
2127 /*
2128 * Next mblk is less than SMSS/2
2129 * rounded up to nearest 64-byte;
2130 * let it get sent as part of the
2131 * next segment.
2132 */
2133 if (tcp->tcp_localnet &&
2134 !tcp->tcp_cork &&
2135 (nmpsz < roundup((mss >> 1), 64)))
2136 break;
2137 }
2138
2139 *xmit_tail = nmp;
2140 ASSERT((uintptr_t)nmpsz <= (uintptr_t)INT_MAX);
2141 /* Stash for rtt use later */
2142 (*xmit_tail)->b_prev = local_time;
2143 (*xmit_tail)->b_next =
2144 (mblk_t *)(uintptr_t)(*snxt - len);
2145 mp1->b_cont = dupb(*xmit_tail);
2146 mp1 = mp1->b_cont;
2147
2148 spill += nmpsz;
2149 if (mp1 == NULL) {
2150 *tail_unsent = spill;
2151 freemsg(mp);
2152 return (-1); /* out_of_mem */
2153 }
2154 }
2155
2156 /* Trim back any surplus on the last mblk */
2157 if (spill >= 0) {
2158 mp1->b_wptr -= spill;
2159 *tail_unsent = spill;
2160 } else {
2161 /*
2162 * We did not send everything we could in
2163 * order to remain within the b_cont limit.
2164 */
2165 *usable -= spill;
2166 *snxt += spill;
2167 tcp->tcp_last_sent_len += spill;
2168 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, spill);
2169 tcp->tcp_cs.tcp_out_data_bytes += spill;
2170 /*
2171 * Adjust the checksum
2172 */
2173 tcpha = (tcpha_t *)(rptr +
2174 ixa->ixa_ip_hdr_length);
2175 sum += spill;
2176 sum = (sum >> 16) + (sum & 0xFFFF);
2177 tcpha->tha_sum = htons(sum);
2178 if (connp->conn_ipversion == IPV4_VERSION) {
2179 sum = ntohs(
2180 ((ipha_t *)rptr)->ipha_length) +
2181 spill;
2182 ((ipha_t *)rptr)->ipha_length =
2183 htons(sum);
2184 } else {
2185 sum = ntohs(
2186 ((ip6_t *)rptr)->ip6_plen) +
2187 spill;
2188 ((ip6_t *)rptr)->ip6_plen =
2189 htons(sum);
2190 }
2191 ixa->ixa_pktlen += spill;
2192 *tail_unsent = 0;
2193 }
2194 }
2195 if (tcp->tcp_ip_forward_progress) {
2196 tcp->tcp_ip_forward_progress = B_FALSE;
2197 ixa->ixa_flags |= IXAF_REACH_CONF;
2198 } else {
2199 ixa->ixa_flags &= ~IXAF_REACH_CONF;
2200 }
2201
2202 if (do_lso_send) {
2203 /* Append LSO information to the mp. */
2204 lso_info_set(mp, mss, HW_LSO);
2205 ixa->ixa_fragsize = IP_MAXPACKET;
2206 ixa->ixa_extra_ident = num_lso_seg - 1;
2207
2208 DTRACE_PROBE2(tcp_send_lso, int, num_lso_seg,
2209 boolean_t, B_TRUE);
2210
2211 tcp_send_data(tcp, mp);
2212
2213 /*
2214 * Restore values of ixa_fragsize and ixa_extra_ident.
2215 */
2216 ixa->ixa_fragsize = ixa->ixa_pmtu;
2217 ixa->ixa_extra_ident = 0;
2218 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2219 TCP_STAT(tcps, tcp_lso_times);
2220 TCP_STAT_UPDATE(tcps, tcp_lso_pkt_out, num_lso_seg);
2221 } else {
2222 /*
2223 * Make sure to clean up LSO information. Wherever a
2224 * new mp uses the prepended header room after dupb(),
2225 * lso_info_cleanup() should be called.
2226 */
2227 lso_info_cleanup(mp);
2228 tcp_send_data(tcp, mp);
2229 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2230 }
2231 }
2232
2233 return (0);
2234 }
2235
2236 /*
2237 * Initiate closedown sequence on an active connection. (May be called as
2238 * writer.) Return value zero for OK return, non-zero for error return.
2239 */
2240 static int
2241 tcp_xmit_end(tcp_t *tcp)
2242 {
2243 mblk_t *mp;
2244 tcp_stack_t *tcps = tcp->tcp_tcps;
2245 iulp_t uinfo;
2246 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip;
2247 conn_t *connp = tcp->tcp_connp;
2248
2249 if (tcp->tcp_state < TCPS_SYN_RCVD ||
2250 tcp->tcp_state > TCPS_CLOSE_WAIT) {
2251 /*
2252 * Invalid state, only states TCPS_SYN_RCVD,
2253 * TCPS_ESTABLISHED and TCPS_CLOSE_WAIT are valid
2254 */
2255 return (-1);
2256 }
2257
2258 tcp->tcp_fss = tcp->tcp_snxt + tcp->tcp_unsent;
2259 tcp->tcp_valid_bits |= TCP_FSS_VALID;
2260 /*
2261 * If there is nothing more unsent, send the FIN now.
2262 * Otherwise, it will go out with the last segment.
2263 */
2264 if (tcp->tcp_unsent == 0) {
2265 mp = tcp_xmit_mp(tcp, NULL, 0, NULL, NULL,
2266 tcp->tcp_fss, B_FALSE, NULL, B_FALSE);
2267
2268 if (mp) {
2269 tcp_send_data(tcp, mp);
2270 } else {
2271 /*
2272 * Couldn't allocate msg. Pretend we got it out.
2273 * Wait for rexmit timeout.
2274 */
2275 tcp->tcp_snxt = tcp->tcp_fss + 1;
2276 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2277 }
2278
2279 /*
2280 * If needed, update tcp_rexmit_snxt as tcp_snxt is
2281 * changed.
2282 */
2283 if (tcp->tcp_rexmit && tcp->tcp_rexmit_nxt == tcp->tcp_fss) {
2284 tcp->tcp_rexmit_nxt = tcp->tcp_snxt;
2285 }
2286 } else {
2287 /*
2288 * If tcp->tcp_cork is set, then the data will not get sent,
2289 * so we have to check that and unset it first.
2290 */
2291 if (tcp->tcp_cork)
2292 tcp->tcp_cork = B_FALSE;
2293 tcp_wput_data(tcp, NULL, B_FALSE);
2294 }
2295
2296 /*
2297 * If TCP does not get enough samples of RTT or tcp_rtt_updates
2298 * is 0, don't update the cache.
2299 */
2300 if (tcps->tcps_rtt_updates == 0 ||
2301 tcp->tcp_rtt_update < tcps->tcps_rtt_updates)
2302 return (0);
2303
2304 /*
2305 * We do not have a good algorithm to update ssthresh at this time.
2306 * So don't do any update.
2307 */
2308 bzero(&uinfo, sizeof (uinfo));
2309 uinfo.iulp_rtt = NSEC2MSEC(tcp->tcp_rtt_sa);
2310 uinfo.iulp_rtt_sd = NSEC2MSEC(tcp->tcp_rtt_sd);
2311
2312 /*
2313 * Note that uinfo is kept for conn_faddr in the DCE. Could update even
2314 * if source routed but we don't.
2315 */
2316 if (connp->conn_ipversion == IPV4_VERSION) {
2317 if (connp->conn_faddr_v4 != tcp->tcp_ipha->ipha_dst) {
2318 return (0);
2319 }
2320 (void) dce_update_uinfo_v4(connp->conn_faddr_v4, &uinfo, ipst);
2321 } else {
2322 uint_t ifindex;
2323
2324 if (!(IN6_ARE_ADDR_EQUAL(&connp->conn_faddr_v6,
2325 &tcp->tcp_ip6h->ip6_dst))) {
2326 return (0);
2327 }
2328 ifindex = 0;
2329 if (IN6_IS_ADDR_LINKSCOPE(&connp->conn_faddr_v6)) {
2330 ip_xmit_attr_t *ixa = connp->conn_ixa;
2331
2332 /*
2333 * If we are going to create a DCE we'd better have
2334 * an ifindex
2335 */
2336 if (ixa->ixa_nce != NULL) {
2337 ifindex = ixa->ixa_nce->nce_common->ncec_ill->
2338 ill_phyint->phyint_ifindex;
2339 } else {
2340 return (0);
2341 }
2342 }
2343
2344 (void) dce_update_uinfo(&connp->conn_faddr_v6, ifindex, &uinfo,
2345 ipst);
2346 }
2347 return (0);
2348 }
2349
2350 /*
2351 * Send out a control packet on the tcp connection specified. This routine
2352 * is typically called where we need a simple ACK or RST generated.
2353 */
2354 void
2355 tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq, uint32_t ack, int ctl)
2356 {
2357 uchar_t *rptr;
2358 tcpha_t *tcpha;
2359 ipha_t *ipha = NULL;
2360 ip6_t *ip6h = NULL;
2361 uint32_t sum;
2362 int total_hdr_len;
2363 int ip_hdr_len;
2364 mblk_t *mp;
2365 tcp_stack_t *tcps = tcp->tcp_tcps;
2366 conn_t *connp = tcp->tcp_connp;
2367 ip_xmit_attr_t *ixa = connp->conn_ixa;
2368
2369 /*
2370 * Save sum for use in source route later.
2371 */
2372 sum = connp->conn_ht_ulp_len + connp->conn_sum;
2373 total_hdr_len = connp->conn_ht_iphc_len;
2374 ip_hdr_len = ixa->ixa_ip_hdr_length;
2375
2376 /* If a text string is passed in with the request, pass it to strlog. */
2377 if (str != NULL && connp->conn_debug) {
2378 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2379 "tcp_xmit_ctl: '%s', seq 0x%x, ack 0x%x, ctl 0x%x",
2380 str, seq, ack, ctl);
2381 }
2382 mp = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
2383 BPRI_MED);
2384 if (mp == NULL) {
2385 return;
2386 }
2387 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2388 mp->b_rptr = rptr;
2389 mp->b_wptr = &rptr[total_hdr_len];
2390 bcopy(connp->conn_ht_iphc, rptr, total_hdr_len);
2391
2392 ixa->ixa_pktlen = total_hdr_len;
2393
2394 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2395 ipha = (ipha_t *)rptr;
2396 ipha->ipha_length = htons(total_hdr_len);
2397 } else {
2398 ip6h = (ip6_t *)rptr;
2399 ip6h->ip6_plen = htons(total_hdr_len - IPV6_HDR_LEN);
2400 }
2401 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2402 tcpha->tha_flags = (uint8_t)ctl;
2403 if (ctl & TH_RST) {
2404 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2405 TCPS_BUMP_MIB(tcps, tcpOutControl);
2406 /*
2407 * Don't send TSopt w/ TH_RST packets per RFC 1323.
2408 */
2409 if (tcp->tcp_snd_ts_ok &&
2410 tcp->tcp_state > TCPS_SYN_SENT) {
2411 mp->b_wptr = &rptr[total_hdr_len - TCPOPT_REAL_TS_LEN];
2412 *(mp->b_wptr) = TCPOPT_EOL;
2413
2414 ixa->ixa_pktlen = total_hdr_len - TCPOPT_REAL_TS_LEN;
2415
2416 if (connp->conn_ipversion == IPV4_VERSION) {
2417 ipha->ipha_length = htons(total_hdr_len -
2418 TCPOPT_REAL_TS_LEN);
2419 } else {
2420 ip6h->ip6_plen = htons(total_hdr_len -
2421 IPV6_HDR_LEN - TCPOPT_REAL_TS_LEN);
2422 }
2423 tcpha->tha_offset_and_reserved -= (3 << 4);
2424 sum -= TCPOPT_REAL_TS_LEN;
2425 }
2426 }
2427 if (ctl & TH_ACK) {
2428 if (tcp->tcp_snd_ts_ok) {
2429 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2430
2431 U32_TO_BE32(llbolt,
2432 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
2433 U32_TO_BE32(tcp->tcp_ts_recent,
2434 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
2435 }
2436
2437 /* Update the latest receive window size in TCP header. */
2438 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
2439 /* Track what we sent to the peer */
2440 tcp->tcp_tcpha->tha_win = tcpha->tha_win;
2441 tcp->tcp_rack = ack;
2442 tcp->tcp_rack_cnt = 0;
2443 TCPS_BUMP_MIB(tcps, tcpOutAck);
2444 }
2445 TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
2446 tcpha->tha_seq = htonl(seq);
2447 tcpha->tha_ack = htonl(ack);
2448 /*
2449 * Include the adjustment for a source route if any.
2450 */
2451 sum = (sum >> 16) + (sum & 0xFFFF);
2452 tcpha->tha_sum = htons(sum);
2453 tcp_send_data(tcp, mp);
2454 }
2455
2456 /*
2457 * Generate a reset based on an inbound packet, connp is set by caller
2458 * when RST is in response to an unexpected inbound packet for which
2459 * there is active tcp state in the system.
2460 *
2461 * IPSEC NOTE : Try to send the reply with the same protection as it came
2462 * in. We have the ip_recv_attr_t which is reversed to form the ip_xmit_attr_t.
2463 * That way the packet will go out at the same level of protection as it
2464 * came in with.
2465 */
2466 static void
2467 tcp_xmit_early_reset(char *str, mblk_t *mp, uint32_t seq, uint32_t ack, int ctl,
2468 ip_recv_attr_t *ira, ip_stack_t *ipst, conn_t *connp)
2469 {
2470 ipha_t *ipha = NULL;
2471 ip6_t *ip6h = NULL;
2472 ushort_t len;
2473 tcpha_t *tcpha;
2474 int i;
2475 ipaddr_t v4addr;
2476 in6_addr_t v6addr;
2477 netstack_t *ns = ipst->ips_netstack;
2478 tcp_stack_t *tcps = ns->netstack_tcp;
2479 ip_xmit_attr_t ixas, *ixa;
2480 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2481 boolean_t need_refrele = B_FALSE; /* ixa_refrele(ixa) */
2482 ushort_t port;
2483
2484 if (!tcp_send_rst_chk(tcps)) {
2485 TCP_STAT(tcps, tcp_rst_unsent);
2486 freemsg(mp);
2487 return;
2488 }
2489
2490 /*
2491 * If connp != NULL we use conn_ixa to keep IP_NEXTHOP and other
2492 * options from the listener. In that case the caller must ensure that
2493 * we are running on the listener = connp squeue.
2494 *
2495 * We get a safe copy of conn_ixa so we don't need to restore anything
2496 * we or ip_output_simple might change in the ixa.
2497 */
2498 if (connp != NULL) {
2499 ASSERT(connp->conn_on_sqp);
2500
2501 ixa = conn_get_ixa_exclusive(connp);
2502 if (ixa == NULL) {
2503 TCP_STAT(tcps, tcp_rst_unsent);
2504 freemsg(mp);
2505 return;
2506 }
2507 need_refrele = B_TRUE;
2508 } else {
2509 bzero(&ixas, sizeof (ixas));
2510 ixa = &ixas;
2511 /*
2512 * IXAF_VERIFY_SOURCE is overkill since we know the
2513 * packet was for us.
2514 */
2515 ixa->ixa_flags |= IXAF_SET_ULP_CKSUM | IXAF_VERIFY_SOURCE;
2516 ixa->ixa_protocol = IPPROTO_TCP;
2517 ixa->ixa_zoneid = ira->ira_zoneid;
2518 ixa->ixa_ifindex = 0;
2519 ixa->ixa_ipst = ipst;
2520 ixa->ixa_cred = kcred;
2521 ixa->ixa_cpid = NOPID;
2522 }
2523
2524 if (str && tcps->tcps_dbg) {
2525 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2526 "tcp_xmit_early_reset: '%s', seq 0x%x, ack 0x%x, "
2527 "flags 0x%x",
2528 str, seq, ack, ctl);
2529 }
2530 if (mp->b_datap->db_ref != 1) {
2531 mblk_t *mp1 = copyb(mp);
2532 freemsg(mp);
2533 mp = mp1;
2534 if (mp == NULL)
2535 goto done;
2536 } else if (mp->b_cont) {
2537 freemsg(mp->b_cont);
2538 mp->b_cont = NULL;
2539 DB_CKSUMFLAGS(mp) = 0;
2540 }
2541 /*
2542 * We skip reversing source route here.
2543 * (for now we replace all IP options with EOL)
2544 */
2545 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2546 ipha = (ipha_t *)mp->b_rptr;
2547 for (i = IP_SIMPLE_HDR_LENGTH; i < (int)ip_hdr_len; i++)
2548 mp->b_rptr[i] = IPOPT_EOL;
2549 /*
2550 * Make sure that src address isn't flagrantly invalid.
2551 * Not all broadcast address checking for the src address
2552 * is possible, since we don't know the netmask of the src
2553 * addr. No check for destination address is done, since
2554 * IP will not pass up a packet with a broadcast dest
2555 * address to TCP. Similar checks are done below for IPv6.
2556 */
2557 if (ipha->ipha_src == 0 || ipha->ipha_src == INADDR_BROADCAST ||
2558 CLASSD(ipha->ipha_src)) {
2559 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsInDiscards);
2560 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2561 freemsg(mp);
2562 goto done;
2563 }
2564 } else {
2565 ip6h = (ip6_t *)mp->b_rptr;
2566
2567 if (IN6_IS_ADDR_UNSPECIFIED(&ip6h->ip6_src) ||
2568 IN6_IS_ADDR_MULTICAST(&ip6h->ip6_src)) {
2569 BUMP_MIB(&ipst->ips_ip6_mib, ipIfStatsInDiscards);
2570 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2571 freemsg(mp);
2572 goto done;
2573 }
2574
2575 /* Remove any extension headers assuming partial overlay */
2576 if (ip_hdr_len > IPV6_HDR_LEN) {
2577 uint8_t *to;
2578
2579 to = mp->b_rptr + ip_hdr_len - IPV6_HDR_LEN;
2580 ovbcopy(ip6h, to, IPV6_HDR_LEN);
2581 mp->b_rptr += ip_hdr_len - IPV6_HDR_LEN;
2582 ip_hdr_len = IPV6_HDR_LEN;
2583 ip6h = (ip6_t *)mp->b_rptr;
2584 ip6h->ip6_nxt = IPPROTO_TCP;
2585 }
2586 }
2587 tcpha = (tcpha_t *)&mp->b_rptr[ip_hdr_len];
2588 if (tcpha->tha_flags & TH_RST) {
2589 freemsg(mp);
2590 goto done;
2591 }
2592 tcpha->tha_offset_and_reserved = (5 << 4);
2593 len = ip_hdr_len + sizeof (tcpha_t);
2594 mp->b_wptr = &mp->b_rptr[len];
2595 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2596 ipha->ipha_length = htons(len);
2597 /* Swap addresses */
2598 v4addr = ipha->ipha_src;
2599 ipha->ipha_src = ipha->ipha_dst;
2600 ipha->ipha_dst = v4addr;
2601 ipha->ipha_ident = 0;
2602 ipha->ipha_ttl = (uchar_t)tcps->tcps_ipv4_ttl;
2603 ixa->ixa_flags |= IXAF_IS_IPV4;
2604 ixa->ixa_ip_hdr_length = ip_hdr_len;
2605 } else {
2606 ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2607 /* Swap addresses */
2608 v6addr = ip6h->ip6_src;
2609 ip6h->ip6_src = ip6h->ip6_dst;
2610 ip6h->ip6_dst = v6addr;
2611 ip6h->ip6_hops = (uchar_t)tcps->tcps_ipv6_hoplimit;
2612 ixa->ixa_flags &= ~IXAF_IS_IPV4;
2613
2614 if (IN6_IS_ADDR_LINKSCOPE(&ip6h->ip6_dst)) {
2615 ixa->ixa_flags |= IXAF_SCOPEID_SET;
2616 ixa->ixa_scopeid = ira->ira_ruifindex;
2617 }
2618 ixa->ixa_ip_hdr_length = IPV6_HDR_LEN;
2619 }
2620 ixa->ixa_pktlen = len;
2621
2622 /* Swap the ports */
2623 port = tcpha->tha_fport;
2624 tcpha->tha_fport = tcpha->tha_lport;
2625 tcpha->tha_lport = port;
2626
2627 tcpha->tha_ack = htonl(ack);
2628 tcpha->tha_seq = htonl(seq);
2629 tcpha->tha_win = 0;
2630 tcpha->tha_sum = htons(sizeof (tcpha_t));
2631 tcpha->tha_flags = (uint8_t)ctl;
2632 if (ctl & TH_RST) {
2633 if (ctl & TH_ACK) {
2634 /*
2635 * Probe connection rejection here.
2636 * tcp_xmit_listeners_reset() drops non-SYN segments
2637 * that do not specify TH_ACK in their flags without
2638 * calling this function. As a consequence, if this
2639 * function is called with a TH_RST|TH_ACK ctl argument,
2640 * it is being called in response to a SYN segment
2641 * and thus the tcp:::accept-refused probe point
2642 * is valid here.
2643 */
2644 DTRACE_TCP5(accept__refused, mblk_t *, NULL,
2645 void, NULL, void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2646 tcph_t *, tcpha);
2647 }
2648 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2649 TCPS_BUMP_MIB(tcps, tcpOutControl);
2650 }
2651
2652 /* Discard any old label */
2653 if (ixa->ixa_free_flags & IXA_FREE_TSL) {
2654 ASSERT(ixa->ixa_tsl != NULL);
2655 label_rele(ixa->ixa_tsl);
2656 ixa->ixa_free_flags &= ~IXA_FREE_TSL;
2657 }
2658 ixa->ixa_tsl = ira->ira_tsl; /* Behave as a multi-level responder */
2659
2660 if (ira->ira_flags & IRAF_IPSEC_SECURE) {
2661 /*
2662 * Apply IPsec based on how IPsec was applied to
2663 * the packet that caused the RST.
2664 */
2665 if (!ipsec_in_to_out(ira, ixa, mp, ipha, ip6h)) {
2666 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsOutDiscards);
2667 /* Note: mp already consumed and ip_drop_packet done */
2668 goto done;
2669 }
2670 } else {
2671 /*
2672 * This is in clear. The RST message we are building
2673 * here should go out in clear, independent of our policy.
2674 */
2675 ixa->ixa_flags |= IXAF_NO_IPSEC;
2676 }
2677
2678 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, ixa,
2679 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2680 __dtrace_tcp_tcph_t *, tcpha);
2681
2682 /*
2683 * NOTE: one might consider tracing a TCP packet here, but
2684 * this function has no active TCP state and no tcp structure
2685 * that has a trace buffer. If we traced here, we would have
2686 * to keep a local trace buffer in tcp_record_trace().
2687 */
2688
2689 (void) ip_output_simple(mp, ixa);
2690 done:
2691 ixa_cleanup(ixa);
2692 if (need_refrele) {
2693 ASSERT(ixa != &ixas);
2694 ixa_refrele(ixa);
2695 }
2696 }
2697
2698 /*
2699 * Generate a "no listener here" RST in response to an "unknown" segment.
2700 * connp is set by caller when RST is in response to an unexpected
2701 * inbound packet for which there is active tcp state in the system.
2702 * Note that we are reusing the incoming mp to construct the outgoing RST.
2703 */
2704 void
2705 tcp_xmit_listeners_reset(mblk_t *mp, ip_recv_attr_t *ira, ip_stack_t *ipst,
2706 conn_t *connp)
2707 {
2708 uchar_t *rptr;
2709 uint32_t seg_len;
2710 tcpha_t *tcpha;
2711 uint32_t seg_seq;
2712 uint32_t seg_ack;
2713 uint_t flags;
2714 ipha_t *ipha;
2715 ip6_t *ip6h;
2716 boolean_t policy_present;
2717 netstack_t *ns = ipst->ips_netstack;
2718 tcp_stack_t *tcps = ns->netstack_tcp;
2719 ipsec_stack_t *ipss = tcps->tcps_netstack->netstack_ipsec;
2720 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2721
2722 TCP_STAT(tcps, tcp_no_listener);
2723
2724 /*
2725 * DTrace this "unknown" segment as a tcp:::receive, as we did
2726 * just receive something that was TCP.
2727 */
2728 DTRACE_TCP5(receive, mblk_t *, NULL, ip_xmit_attr_t *, NULL,
2729 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2730 __dtrace_tcp_tcph_t *, &mp->b_rptr[ip_hdr_len]);
2731
2732 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2733 policy_present = ipss->ipsec_inbound_v4_policy_present;
2734 ipha = (ipha_t *)mp->b_rptr;
2735 ip6h = NULL;
2736 } else {
2737 policy_present = ipss->ipsec_inbound_v6_policy_present;
2738 ipha = NULL;
2739 ip6h = (ip6_t *)mp->b_rptr;
2740 }
2741
2742 if (policy_present) {
2743 /*
2744 * The conn_t parameter is NULL because we already know
2745 * nobody's home.
2746 */
2747 mp = ipsec_check_global_policy(mp, (conn_t *)NULL, ipha, ip6h,
2748 ira, ns);
2749 if (mp == NULL)
2750 return;
2751 }
2752 if (is_system_labeled() && !tsol_can_reply_error(mp, ira)) {
2753 DTRACE_PROBE2(
2754 tx__ip__log__error__nolistener__tcp,
2755 char *, "Could not reply with RST to mp(1)",
2756 mblk_t *, mp);
2757 ip2dbg(("tcp_xmit_listeners_reset: not permitted to reply\n"));
2758 freemsg(mp);
2759 return;
2760 }
2761
2762 rptr = mp->b_rptr;
2763
2764 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2765 seg_seq = ntohl(tcpha->tha_seq);
2766 seg_ack = ntohl(tcpha->tha_ack);
2767 flags = tcpha->tha_flags;
2768
2769 seg_len = msgdsize(mp) - (TCP_HDR_LENGTH(tcpha) + ip_hdr_len);
2770 if (flags & TH_RST) {
2771 freemsg(mp);
2772 } else if (flags & TH_ACK) {
2773 tcp_xmit_early_reset("no tcp, reset", mp, seg_ack, 0, TH_RST,
2774 ira, ipst, connp);
2775 } else {
2776 if (flags & TH_SYN) {
2777 seg_len++;
2778 } else {
2779 /*
2780 * Here we violate the RFC. Note that a normal
2781 * TCP will never send a segment without the ACK
2782 * flag, except for RST or SYN segment. This
2783 * segment is neither. Just drop it on the
2784 * floor.
2785 */
2786 freemsg(mp);
2787 TCP_STAT(tcps, tcp_rst_unsent);
2788 return;
2789 }
2790
2791 tcp_xmit_early_reset("no tcp, reset/ack", mp, 0,
2792 seg_seq + seg_len, TH_RST | TH_ACK, ira, ipst, connp);
2793 }
2794 }
2795
2796 /*
2797 * Helper function for tcp_xmit_mp() in handling connection set up flag
2798 * options setting.
2799 */
2800 static void
2801 tcp_xmit_mp_aux_iss(tcp_t *tcp, conn_t *connp, tcpha_t *tcpha, mblk_t *mp,
2802 uint_t *flags)
2803 {
2804 uint32_t u1;
2805 uint8_t *wptr = mp->b_wptr;
2806 tcp_stack_t *tcps = tcp->tcp_tcps;
2807 boolean_t add_sack = B_FALSE;
2808
2809 /*
2810 * If TCP_ISS_VALID and the seq number is tcp_iss,
2811 * TCP can only be in SYN-SENT, SYN-RCVD or
2812 * FIN-WAIT-1 state. It can be FIN-WAIT-1 if
2813 * our SYN is not ack'ed but the app closes this
2814 * TCP connection.
2815 */
2816 ASSERT(tcp->tcp_state == TCPS_SYN_SENT ||
2817 tcp->tcp_state == TCPS_SYN_RCVD ||
2818 tcp->tcp_state == TCPS_FIN_WAIT_1);
2819
2820 /*
2821 * Tack on the MSS option. It is always needed
2822 * for both active and passive open.
2823 *
2824 * MSS option value should be interface MTU - MIN
2825 * TCP/IP header according to RFC 793 as it means
2826 * the maximum segment size TCP can receive. But
2827 * to get around some broken middle boxes/end hosts
2828 * out there, we allow the option value to be the
2829 * same as the MSS option size on the peer side.
2830 * In this way, the other side will not send
2831 * anything larger than they can receive.
2832 *
2833 * Note that for SYN_SENT state, the ndd param
2834 * tcp_use_smss_as_mss_opt has no effect as we
2835 * don't know the peer's MSS option value. So
2836 * the only case we need to take care of is in
2837 * SYN_RCVD state, which is done later.
2838 */
2839 wptr[0] = TCPOPT_MAXSEG;
2840 wptr[1] = TCPOPT_MAXSEG_LEN;
2841 wptr += 2;
2842 u1 = tcp->tcp_initial_pmtu - (connp->conn_ipversion == IPV4_VERSION ?
2843 IP_SIMPLE_HDR_LENGTH : IPV6_HDR_LEN) - TCP_MIN_HEADER_LENGTH;
2844 U16_TO_BE16(u1, wptr);
2845 wptr += 2;
2846
2847 /* Update the offset to cover the additional word */
2848 tcpha->tha_offset_and_reserved += (1 << 4);
2849
2850 switch (tcp->tcp_state) {
2851 case TCPS_SYN_SENT:
2852 *flags = TH_SYN;
2853
2854 if (tcp->tcp_snd_sack_ok)
2855 add_sack = B_TRUE;
2856
2857 if (tcp->tcp_snd_ts_ok) {
2858 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2859
2860 if (add_sack) {
2861 wptr[0] = TCPOPT_SACK_PERMITTED;
2862 wptr[1] = TCPOPT_SACK_OK_LEN;
2863 add_sack = B_FALSE;
2864 } else {
2865 wptr[0] = TCPOPT_NOP;
2866 wptr[1] = TCPOPT_NOP;
2867 }
2868 wptr[2] = TCPOPT_TSTAMP;
2869 wptr[3] = TCPOPT_TSTAMP_LEN;
2870 wptr += 4;
2871 U32_TO_BE32(llbolt, wptr);
2872 wptr += 4;
2873 ASSERT(tcp->tcp_ts_recent == 0);
2874 U32_TO_BE32(0L, wptr);
2875 wptr += 4;
2876 tcpha->tha_offset_and_reserved += (3 << 4);
2877 }
2878
2879 /*
2880 * Set up all the bits to tell other side
2881 * we are ECN capable.
2882 */
2883 if (tcp->tcp_ecn_ok)
2884 *flags |= (TH_ECE | TH_CWR);
2885
2886 break;
2887
2888 case TCPS_SYN_RCVD:
2889 *flags |= TH_SYN;
2890
2891 /*
2892 * Reset the MSS option value to be SMSS
2893 * We should probably add back the bytes
2894 * for timestamp option and IPsec. We
2895 * don't do that as this is a workaround
2896 * for broken middle boxes/end hosts, it
2897 * is better for us to be more cautious.
2898 * They may not take these things into
2899 * account in their SMSS calculation. Thus
2900 * the peer's calculated SMSS may be smaller
2901 * than what it can be. This should be OK.
2902 */
2903 if (tcps->tcps_use_smss_as_mss_opt) {
2904 u1 = tcp->tcp_mss;
2905 /*
2906 * Note that wptr points just past the MSS
2907 * option value.
2908 */
2909 U16_TO_BE16(u1, wptr - 2);
2910 }
2911
2912 /*
2913 * tcp_snd_ts_ok can only be set in TCPS_SYN_RCVD
2914 * when the peer also uses timestamps option. And
2915 * the TCP header template must have already been
2916 * updated to include the timestamps option.
2917 */
2918 if (tcp->tcp_snd_sack_ok) {
2919 if (tcp->tcp_snd_ts_ok) {
2920 uint8_t *tmp_wptr;
2921
2922 /*
2923 * Use the NOP in the header just
2924 * before timestamps opton.
2925 */
2926 tmp_wptr = (uint8_t *)tcpha +
2927 TCP_MIN_HEADER_LENGTH;
2928 ASSERT(tmp_wptr[0] == TCPOPT_NOP &&
2929 tmp_wptr[1] == TCPOPT_NOP);
2930 tmp_wptr[0] = TCPOPT_SACK_PERMITTED;
2931 tmp_wptr[1] = TCPOPT_SACK_OK_LEN;
2932 } else {
2933 add_sack = B_TRUE;
2934 }
2935 }
2936
2937
2938 /*
2939 * If the other side is ECN capable, reply
2940 * that we are also ECN capable.
2941 */
2942 if (tcp->tcp_ecn_ok)
2943 *flags |= TH_ECE;
2944 break;
2945
2946 default:
2947 /*
2948 * The above ASSERT() makes sure that this
2949 * must be FIN-WAIT-1 state. Our SYN has
2950 * not been ack'ed so retransmit it.
2951 */
2952 *flags |= TH_SYN;
2953 break;
2954 }
2955
2956 if (add_sack) {
2957 wptr[0] = TCPOPT_NOP;
2958 wptr[1] = TCPOPT_NOP;
2959 wptr[2] = TCPOPT_SACK_PERMITTED;
2960 wptr[3] = TCPOPT_SACK_OK_LEN;
2961 wptr += TCPOPT_REAL_SACK_OK_LEN;
2962 tcpha->tha_offset_and_reserved += (1 << 4);
2963 }
2964
2965 if (tcp->tcp_snd_ws_ok) {
2966 wptr[0] = TCPOPT_NOP;
2967 wptr[1] = TCPOPT_WSCALE;
2968 wptr[2] = TCPOPT_WS_LEN;
2969 wptr[3] = (uchar_t)tcp->tcp_rcv_ws;
2970 wptr += TCPOPT_REAL_WS_LEN;
2971 tcpha->tha_offset_and_reserved += (1 << 4);
2972 }
2973
2974 mp->b_wptr = wptr;
2975 u1 = (int)(mp->b_wptr - mp->b_rptr);
2976 /*
2977 * Get IP set to checksum on our behalf
2978 * Include the adjustment for a source route if any.
2979 */
2980 u1 += connp->conn_sum;
2981 u1 = (u1 >> 16) + (u1 & 0xFFFF);
2982 tcpha->tha_sum = htons(u1);
2983 TCPS_BUMP_MIB(tcps, tcpOutControl);
2984 }
2985
2986 /*
2987 * Helper function for tcp_xmit_mp() in handling connection tear down
2988 * flag setting and state changes.
2989 */
2990 static void
2991 tcp_xmit_mp_aux_fss(tcp_t *tcp, ip_xmit_attr_t *ixa, uint_t *flags)
2992 {
2993 if (!tcp->tcp_fin_acked) {
2994 *flags |= TH_FIN;
2995 TCPS_BUMP_MIB(tcp->tcp_tcps, tcpOutControl);
2996 }
2997 if (!tcp->tcp_fin_sent) {
2998 tcp->tcp_fin_sent = B_TRUE;
2999 switch (tcp->tcp_state) {
3000 case TCPS_SYN_RCVD:
3001 tcp->tcp_state = TCPS_FIN_WAIT_1;
3002 DTRACE_TCP6(state__change, void, NULL,
3003 ip_xmit_attr_t *, ixa, void, NULL,
3004 tcp_t *, tcp, void, NULL,
3005 int32_t, TCPS_SYN_RCVD);
3006 break;
3007 case TCPS_ESTABLISHED:
3008 tcp->tcp_state = TCPS_FIN_WAIT_1;
3009 DTRACE_TCP6(state__change, void, NULL,
3010 ip_xmit_attr_t *, ixa, void, NULL,
3011 tcp_t *, tcp, void, NULL,
3012 int32_t, TCPS_ESTABLISHED);
3013 break;
3014 case TCPS_CLOSE_WAIT:
3015 tcp->tcp_state = TCPS_LAST_ACK;
3016 DTRACE_TCP6(state__change, void, NULL,
3017 ip_xmit_attr_t *, ixa, void, NULL,
3018 tcp_t *, tcp, void, NULL,
3019 int32_t, TCPS_CLOSE_WAIT);
3020 break;
3021 }
3022 if (tcp->tcp_suna == tcp->tcp_snxt)
3023 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
3024 tcp->tcp_snxt = tcp->tcp_fss + 1;
3025 }
3026 }
3027
3028 /*
3029 * tcp_xmit_mp is called to return a pointer to an mblk chain complete with
3030 * ip and tcp header ready to pass down to IP. If the mp passed in is
3031 * non-NULL, then up to max_to_send bytes of data will be dup'ed off that
3032 * mblk. (If sendall is not set the dup'ing will stop at an mblk boundary
3033 * otherwise it will dup partial mblks.)
3034 * Otherwise, an appropriate ACK packet will be generated. This
3035 * routine is not usually called to send new data for the first time. It
3036 * is mostly called out of the timer for retransmits, and to generate ACKs.
3037 *
3038 * If offset is not NULL, the returned mblk chain's first mblk's b_rptr will
3039 * be adjusted by *offset. And after dupb(), the offset and the ending mblk
3040 * of the original mblk chain will be returned in *offset and *end_mp.
3041 */
3042 mblk_t *
3043 tcp_xmit_mp(tcp_t *tcp, mblk_t *mp, int32_t max_to_send, int32_t *offset,
3044 mblk_t **end_mp, uint32_t seq, boolean_t sendall, uint32_t *seg_len,
3045 boolean_t rexmit)
3046 {
3047 int data_length;
3048 int32_t off = 0;
3049 uint_t flags;
3050 mblk_t *mp1;
3051 mblk_t *mp2;
3052 uchar_t *rptr;
3053 tcpha_t *tcpha;
3054 int32_t num_sack_blk = 0;
3055 int32_t sack_opt_len = 0;
3056 tcp_stack_t *tcps = tcp->tcp_tcps;
3057 conn_t *connp = tcp->tcp_connp;
3058 ip_xmit_attr_t *ixa = connp->conn_ixa;
3059
3060 /* Allocate for our maximum TCP header + link-level */
3061 mp1 = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
3062 BPRI_MED);
3063 if (mp1 == NULL)
3064 return (NULL);
3065 data_length = 0;
3066
3067 /*
3068 * Note that tcp_mss has been adjusted to take into account the
3069 * timestamp option if applicable. Because SACK options do not
3070 * appear in every TCP segments and they are of variable lengths,
3071 * they cannot be included in tcp_mss. Thus we need to calculate
3072 * the actual segment length when we need to send a segment which
3073 * includes SACK options.
3074 */
3075 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
3076 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
3077 tcp->tcp_num_sack_blk);
3078 sack_opt_len = num_sack_blk * sizeof (sack_blk_t) +
3079 TCPOPT_NOP_LEN * 2 + TCPOPT_HEADER_LEN;
3080 if (max_to_send + sack_opt_len > tcp->tcp_mss)
3081 max_to_send -= sack_opt_len;
3082 }
3083
3084 if (offset != NULL) {
3085 off = *offset;
3086 /* We use offset as an indicator that end_mp is not NULL. */
3087 *end_mp = NULL;
3088 }
3089 for (mp2 = mp1; mp && data_length != max_to_send; mp = mp->b_cont) {
3090 /* This could be faster with cooperation from downstream */
3091 if (mp2 != mp1 && !sendall &&
3092 data_length + (int)(mp->b_wptr - mp->b_rptr) >
3093 max_to_send)
3094 /*
3095 * Don't send the next mblk since the whole mblk
3096 * does not fit.
3097 */
3098 break;
3099 mp2->b_cont = dupb(mp);
3100 mp2 = mp2->b_cont;
3101 if (!mp2) {
3102 freemsg(mp1);
3103 return (NULL);
3104 }
3105 mp2->b_rptr += off;
3106 ASSERT((uintptr_t)(mp2->b_wptr - mp2->b_rptr) <=
3107 (uintptr_t)INT_MAX);
3108
3109 data_length += (int)(mp2->b_wptr - mp2->b_rptr);
3110 if (data_length > max_to_send) {
3111 mp2->b_wptr -= data_length - max_to_send;
3112 data_length = max_to_send;
3113 off = mp2->b_wptr - mp->b_rptr;
3114 break;
3115 } else {
3116 off = 0;
3117 }
3118 }
3119 if (offset != NULL) {
3120 *offset = off;
3121 *end_mp = mp;
3122 }
3123 if (seg_len != NULL) {
3124 *seg_len = data_length;
3125 }
3126
3127 /* Update the latest receive window size in TCP header. */
3128 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
3129
3130 rptr = mp1->b_rptr + tcps->tcps_wroff_xtra;
3131 mp1->b_rptr = rptr;
3132 mp1->b_wptr = rptr + connp->conn_ht_iphc_len + sack_opt_len;
3133 bcopy(connp->conn_ht_iphc, rptr, connp->conn_ht_iphc_len);
3134 tcpha = (tcpha_t *)&rptr[ixa->ixa_ip_hdr_length];
3135 tcpha->tha_seq = htonl(seq);
3136
3137 /*
3138 * Use tcp_unsent to determine if the PUSH bit should be used assumes
3139 * that this function was called from tcp_wput_data. Thus, when called
3140 * to retransmit data the setting of the PUSH bit may appear some
3141 * what random in that it might get set when it should not. This
3142 * should not pose any performance issues.
3143 */
3144 if (data_length != 0 && (tcp->tcp_unsent == 0 ||
3145 tcp->tcp_unsent == data_length)) {
3146 flags = TH_ACK | TH_PUSH;
3147 } else {
3148 flags = TH_ACK;
3149 }
3150
3151 if (tcp->tcp_ecn_ok) {
3152 if (tcp->tcp_ecn_echo_on)
3153 flags |= TH_ECE;
3154
3155 /*
3156 * Only set ECT bit and ECN_CWR if a segment contains new data.
3157 * There is no TCP flow control for non-data segments, and
3158 * only data segment is transmitted reliably.
3159 */
3160 if (data_length > 0 && !rexmit) {
3161 TCP_SET_ECT(tcp, rptr);
3162 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3163 flags |= TH_CWR;
3164 tcp->tcp_ecn_cwr_sent = B_TRUE;
3165 }
3166 }
3167 }
3168
3169 /* Check if there is any special processing needs to be done. */
3170 if (tcp->tcp_valid_bits) {
3171 uint32_t u1;
3172
3173 /* We don't allow having SYN and FIN in the same segment... */
3174 if ((tcp->tcp_valid_bits & TCP_ISS_VALID) &&
3175 seq == tcp->tcp_iss) {
3176 /* Need to do connection set up processing. */
3177 tcp_xmit_mp_aux_iss(tcp, connp, tcpha, mp1, &flags);
3178 } else if ((tcp->tcp_valid_bits & TCP_FSS_VALID) &&
3179 (seq + data_length) == tcp->tcp_fss) {
3180 /* Need to do connection tear down processing. */
3181 tcp_xmit_mp_aux_fss(tcp, ixa, &flags);
3182 }
3183
3184 /*
3185 * Need to do urgent pointer processing.
3186 *
3187 * Note the trick here. u1 is unsigned. When tcp_urg
3188 * is smaller than seq, u1 will become a very huge value.
3189 * So the comparison will fail. Also note that tcp_urp
3190 * should be positive, see RFC 793 page 17.
3191 */
3192 u1 = tcp->tcp_urg - seq + TCP_OLD_URP_INTERPRETATION;
3193 if ((tcp->tcp_valid_bits & TCP_URG_VALID) && u1 != 0 &&
3194 u1 < (uint32_t)(64 * 1024)) {
3195 flags |= TH_URG;
3196 TCPS_BUMP_MIB(tcps, tcpOutUrg);
3197 tcpha->tha_urp = htons(u1);
3198 }
3199 }
3200 tcpha->tha_flags = (uchar_t)flags;
3201 tcp->tcp_rack = tcp->tcp_rnxt;
3202 tcp->tcp_rack_cnt = 0;
3203
3204 /* Fill in the current value of timestamps option. */
3205 if (tcp->tcp_snd_ts_ok) {
3206 if (tcp->tcp_state != TCPS_SYN_SENT) {
3207 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
3208
3209 U32_TO_BE32(llbolt,
3210 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
3211 U32_TO_BE32(tcp->tcp_ts_recent,
3212 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
3213 }
3214 }
3215
3216 /* Fill in the SACK blocks. */
3217 if (num_sack_blk > 0) {
3218 uchar_t *wptr = (uchar_t *)tcpha + connp->conn_ht_ulp_len;
3219 sack_blk_t *tmp;
3220 int32_t i;
3221
3222 wptr[0] = TCPOPT_NOP;
3223 wptr[1] = TCPOPT_NOP;
3224 wptr[2] = TCPOPT_SACK;
3225 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3226 sizeof (sack_blk_t);
3227 wptr += TCPOPT_REAL_SACK_LEN;
3228
3229 tmp = tcp->tcp_sack_list;
3230 for (i = 0; i < num_sack_blk; i++) {
3231 U32_TO_BE32(tmp[i].begin, wptr);
3232 wptr += sizeof (tcp_seq);
3233 U32_TO_BE32(tmp[i].end, wptr);
3234 wptr += sizeof (tcp_seq);
3235 }
3236 tcpha->tha_offset_and_reserved += ((num_sack_blk * 2 + 1) << 4);
3237 }
3238 ASSERT((uintptr_t)(mp1->b_wptr - rptr) <= (uintptr_t)INT_MAX);
3239 data_length += (int)(mp1->b_wptr - rptr);
3240
3241 ixa->ixa_pktlen = data_length;
3242
3243 if (ixa->ixa_flags & IXAF_IS_IPV4) {
3244 ((ipha_t *)rptr)->ipha_length = htons(data_length);
3245 } else {
3246 ip6_t *ip6 = (ip6_t *)rptr;
3247
3248 ip6->ip6_plen = htons(data_length - IPV6_HDR_LEN);
3249 }
3250
3251 /*
3252 * Prime pump for IP
3253 * Include the adjustment for a source route if any.
3254 */
3255 data_length -= ixa->ixa_ip_hdr_length;
3256 data_length += connp->conn_sum;
3257 data_length = (data_length >> 16) + (data_length & 0xFFFF);
3258 tcpha->tha_sum = htons(data_length);
3259 if (tcp->tcp_ip_forward_progress) {
3260 tcp->tcp_ip_forward_progress = B_FALSE;
3261 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
3262 } else {
3263 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
3264 }
3265 return (mp1);
3266 }
3267
3268 /*
3269 * If this routine returns B_TRUE, TCP can generate a RST in response
3270 * to a segment. If it returns B_FALSE, TCP should not respond.
3271 */
3272 static boolean_t
3273 tcp_send_rst_chk(tcp_stack_t *tcps)
3274 {
3275 int64_t now;
3276
3277 /*
3278 * TCP needs to protect itself from generating too many RSTs.
3279 * This can be a DoS attack by sending us random segments
3280 * soliciting RSTs.
3281 *
3282 * What we do here is to have a limit of tcp_rst_sent_rate RSTs
3283 * in each 1 second interval. In this way, TCP still generate
3284 * RSTs in normal cases but when under attack, the impact is
3285 * limited.
3286 */
3287 if (tcps->tcps_rst_sent_rate_enabled != 0) {
3288 now = ddi_get_lbolt64();
3289 if (TICK_TO_MSEC(now - tcps->tcps_last_rst_intrvl) >
3290 1*SECONDS) {
3291 tcps->tcps_last_rst_intrvl = now;
3292 tcps->tcps_rst_cnt = 1;
3293 } else if (++tcps->tcps_rst_cnt > tcps->tcps_rst_sent_rate) {
3294 return (B_FALSE);
3295 }
3296 }
3297 return (B_TRUE);
3298 }
3299
3300 /*
3301 * This function handles all retransmissions if SACK is enabled for this
3302 * connection. First it calculates how many segments can be retransmitted
3303 * based on tcp_pipe. Then it goes thru the notsack list to find eligible
3304 * segments. A segment is eligible if sack_cnt for that segment is greater
3305 * than or equal tcp_dupack_fast_retransmit. After it has retransmitted
3306 * all eligible segments, it checks to see if TCP can send some new segments
3307 * (fast recovery). If it can, set the appropriate flag for tcp_input_data().
3308 *
3309 * Parameters:
3310 * tcp_t *tcp: the tcp structure of the connection.
3311 * uint_t *flags: in return, appropriate value will be set for
3312 * tcp_input_data().
3313 */
3314 void
3315 tcp_sack_rexmit(tcp_t *tcp, uint_t *flags)
3316 {
3317 notsack_blk_t *notsack_blk;
3318 int32_t usable_swnd;
3319 int32_t mss;
3320 uint32_t seg_len;
3321 mblk_t *xmit_mp;
3322 tcp_stack_t *tcps = tcp->tcp_tcps;
3323
3324 ASSERT(tcp->tcp_notsack_list != NULL);
3325 ASSERT(tcp->tcp_rexmit == B_FALSE);
3326
3327 /* Defensive coding in case there is a bug... */
3328 if (tcp->tcp_notsack_list == NULL) {
3329 return;
3330 }
3331 notsack_blk = tcp->tcp_notsack_list;
3332 mss = tcp->tcp_mss;
3333
3334 /*
3335 * Limit the num of outstanding data in the network to be
3336 * tcp_cwnd_ssthresh, which is half of the original congestion wnd.
3337 */
3338 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3339
3340 /* At least retransmit 1 MSS of data. */
3341 if (usable_swnd <= 0) {
3342 usable_swnd = mss;
3343 }
3344
3345 /* Make sure no new RTT samples will be taken. */
3346 tcp->tcp_csuna = tcp->tcp_snxt;
3347
3348 notsack_blk = tcp->tcp_notsack_list;
3349 while (usable_swnd > 0) {
3350 mblk_t *snxt_mp, *tmp_mp;
3351 tcp_seq begin = tcp->tcp_sack_snxt;
3352 tcp_seq end;
3353 int32_t off;
3354
3355 for (; notsack_blk != NULL; notsack_blk = notsack_blk->next) {
3356 if (SEQ_GT(notsack_blk->end, begin) &&
3357 (notsack_blk->sack_cnt >=
3358 tcps->tcps_dupack_fast_retransmit)) {
3359 end = notsack_blk->end;
3360 if (SEQ_LT(begin, notsack_blk->begin)) {
3361 begin = notsack_blk->begin;
3362 }
3363 break;
3364 }
3365 }
3366 /*
3367 * All holes are filled. Manipulate tcp_cwnd to send more
3368 * if we can. Note that after the SACK recovery, tcp_cwnd is
3369 * set to tcp_cwnd_ssthresh.
3370 */
3371 if (notsack_blk == NULL) {
3372 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3373 if (usable_swnd <= 0 || tcp->tcp_unsent == 0) {
3374 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna;
3375 ASSERT(tcp->tcp_cwnd > 0);
3376 return;
3377 } else {
3378 usable_swnd = usable_swnd / mss;
3379 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna +
3380 MAX(usable_swnd * mss, mss);
3381 *flags |= TH_XMIT_NEEDED;
3382 return;
3383 }
3384 }
3385
3386 /*
3387 * Note that we may send more than usable_swnd allows here
3388 * because of round off, but no more than 1 MSS of data.
3389 */
3390 seg_len = end - begin;
3391 if (seg_len > mss)
3392 seg_len = mss;
3393 snxt_mp = tcp_get_seg_mp(tcp, begin, &off);
3394 ASSERT(snxt_mp != NULL);
3395 /* This should not happen. Defensive coding again... */
3396 if (snxt_mp == NULL) {
3397 return;
3398 }
3399
3400 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, seg_len, &off,
3401 &tmp_mp, begin, B_TRUE, &seg_len, B_TRUE);
3402 if (xmit_mp == NULL)
3403 return;
3404
3405 usable_swnd -= seg_len;
3406 tcp->tcp_pipe += seg_len;
3407 tcp->tcp_sack_snxt = begin + seg_len;
3408
3409 tcp_send_data(tcp, xmit_mp);
3410
3411 /*
3412 * Update the send timestamp to avoid false retransmission.
3413 */
3414 #ifdef KERNEL_32
3415 snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3416 #else
3417 snxt_mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
3418 #endif
3419
3420 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3421 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, seg_len);
3422 TCPS_BUMP_MIB(tcps, tcpOutSackRetransSegs);
3423 tcp->tcp_cs.tcp_out_retrans_segs++;
3424 tcp->tcp_cs.tcp_out_retrans_bytes += seg_len;
3425 /*
3426 * Update tcp_rexmit_max to extend this SACK recovery phase.
3427 * This happens when new data sent during fast recovery is
3428 * also lost. If TCP retransmits those new data, it needs
3429 * to extend SACK recover phase to avoid starting another
3430 * fast retransmit/recovery unnecessarily.
3431 */
3432 if (SEQ_GT(tcp->tcp_sack_snxt, tcp->tcp_rexmit_max)) {
3433 tcp->tcp_rexmit_max = tcp->tcp_sack_snxt;
3434 }
3435 }
3436 }
3437
3438 /*
3439 * tcp_ss_rexmit() is called to do slow start retransmission after a timeout
3440 * or ICMP errors.
3441 */
3442 void
3443 tcp_ss_rexmit(tcp_t *tcp)
3444 {
3445 uint32_t snxt;
3446 uint32_t smax;
3447 int32_t win;
3448 int32_t mss;
3449 int32_t off;
3450 mblk_t *snxt_mp;
3451 tcp_stack_t *tcps = tcp->tcp_tcps;
3452
3453 /*
3454 * Note that tcp_rexmit can be set even though TCP has retransmitted
3455 * all unack'ed segments.
3456 */
3457 if (SEQ_LT(tcp->tcp_rexmit_nxt, tcp->tcp_rexmit_max)) {
3458 smax = tcp->tcp_rexmit_max;
3459 snxt = tcp->tcp_rexmit_nxt;
3460 if (SEQ_LT(snxt, tcp->tcp_suna)) {
3461 snxt = tcp->tcp_suna;
3462 }
3463 win = MIN(tcp->tcp_cwnd, tcp->tcp_swnd);
3464 win -= snxt - tcp->tcp_suna;
3465 mss = tcp->tcp_mss;
3466 snxt_mp = tcp_get_seg_mp(tcp, snxt, &off);
3467
3468 while (SEQ_LT(snxt, smax) && (win > 0) && (snxt_mp != NULL)) {
3469 mblk_t *xmit_mp;
3470 mblk_t *old_snxt_mp = snxt_mp;
3471 uint32_t cnt = mss;
3472
3473 if (win < cnt) {
3474 cnt = win;
3475 }
3476 if (SEQ_GT(snxt + cnt, smax)) {
3477 cnt = smax - snxt;
3478 }
3479 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, cnt, &off,
3480 &snxt_mp, snxt, B_TRUE, &cnt, B_TRUE);
3481 if (xmit_mp == NULL)
3482 return;
3483
3484 tcp_send_data(tcp, xmit_mp);
3485
3486 snxt += cnt;
3487 win -= cnt;
3488 /*
3489 * Update the send timestamp to avoid false
3490 * retransmission.
3491 */
3492 #ifdef KERNEL_32
3493 old_snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3494 #else
3495 old_snxt_mp->b_prev = (mblk_t *)(intptr_t)gethrtime();
3496 #endif
3497 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3498 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, cnt);
3499 tcp->tcp_cs.tcp_out_retrans_segs++;
3500 tcp->tcp_cs.tcp_out_retrans_bytes += cnt;
3501
3502 tcp->tcp_rexmit_nxt = snxt;
3503 }
3504 /*
3505 * If we have transmitted all we have at the time
3506 * we started the retranmission, we can leave
3507 * the rest of the job to tcp_wput_data(). But we
3508 * need to check the send window first. If the
3509 * win is not 0, go on with tcp_wput_data().
3510 */
3511 if (SEQ_LT(snxt, smax) || win == 0) {
3512 return;
3513 }
3514 }
3515 /* Only call tcp_wput_data() if there is data to be sent. */
3516 if (tcp->tcp_unsent) {
3517 tcp_wput_data(tcp, NULL, B_FALSE);
3518 }
3519 }
3520
3521 /*
3522 * Do slow start retransmission after ICMP errors of PMTU changes.
3523 */
3524 void
3525 tcp_rexmit_after_error(tcp_t *tcp)
3526 {
3527 /*
3528 * All sent data has been acknowledged or no data left to send, just
3529 * to return.
3530 */
3531 if (!SEQ_LT(tcp->tcp_suna, tcp->tcp_snxt) ||
3532 (tcp->tcp_xmit_head == NULL))
3533 return;
3534
3535 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && (tcp->tcp_unsent == 0))
3536 tcp->tcp_rexmit_max = tcp->tcp_fss;
3537 else
3538 tcp->tcp_rexmit_max = tcp->tcp_snxt;
3539
3540 tcp->tcp_rexmit_nxt = tcp->tcp_suna;
3541 tcp->tcp_rexmit = B_TRUE;
3542 tcp->tcp_dupack_cnt = 0;
3543 tcp_ss_rexmit(tcp);
3544 }
3545
3546 /*
3547 * tcp_get_seg_mp() is called to get the pointer to a segment in the
3548 * send queue which starts at the given sequence number. If the given
3549 * sequence number is equal to last valid sequence number (tcp_snxt), the
3550 * returned mblk is the last valid mblk, and off is set to the length of
3551 * that mblk.
3552 *
3553 * send queue which starts at the given seq. no.
3554 *
3555 * Parameters:
3556 * tcp_t *tcp: the tcp instance pointer.
3557 * uint32_t seq: the starting seq. no of the requested segment.
3558 * int32_t *off: after the execution, *off will be the offset to
3559 * the returned mblk which points to the requested seq no.
3560 * It is the caller's responsibility to send in a non-null off.
3561 *
3562 * Return:
3563 * A mblk_t pointer pointing to the requested segment in send queue.
3564 */
3565 static mblk_t *
3566 tcp_get_seg_mp(tcp_t *tcp, uint32_t seq, int32_t *off)
3567 {
3568 int32_t cnt;
3569 mblk_t *mp;
3570
3571 /* Defensive coding. Make sure we don't send incorrect data. */
3572 if (SEQ_LT(seq, tcp->tcp_suna) || SEQ_GT(seq, tcp->tcp_snxt))
3573 return (NULL);
3574
3575 cnt = seq - tcp->tcp_suna;
3576 mp = tcp->tcp_xmit_head;
3577 while (cnt > 0 && mp != NULL) {
3578 cnt -= mp->b_wptr - mp->b_rptr;
3579 if (cnt <= 0) {
3580 cnt += mp->b_wptr - mp->b_rptr;
3581 break;
3582 }
3583 mp = mp->b_cont;
3584 }
3585 ASSERT(mp != NULL);
3586 *off = cnt;
3587 return (mp);
3588 }
3589
3590 /*
3591 * This routine adjusts next-to-send sequence number variables, in the
3592 * case where the reciever has shrunk it's window.
3593 */
3594 void
3595 tcp_update_xmit_tail(tcp_t *tcp, uint32_t snxt)
3596 {
3597 mblk_t *xmit_tail;
3598 int32_t offset;
3599
3600 tcp->tcp_snxt = snxt;
3601
3602 /* Get the mblk, and the offset in it, as per the shrunk window */
3603 xmit_tail = tcp_get_seg_mp(tcp, snxt, &offset);
3604 ASSERT(xmit_tail != NULL);
3605 tcp->tcp_xmit_tail = xmit_tail;
3606 tcp->tcp_xmit_tail_unsent = xmit_tail->b_wptr -
3607 xmit_tail->b_rptr - offset;
3608 }
3609
3610 /*
3611 * This handles the case when the receiver has shrunk its win. Per RFC 1122
3612 * if the receiver shrinks the window, i.e. moves the right window to the
3613 * left, the we should not send new data, but should retransmit normally the
3614 * old unacked data between suna and suna + swnd. We might has sent data
3615 * that is now outside the new window, pretend that we didn't send it.
3616 */
3617 static void
3618 tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_count)
3619 {
3620 uint32_t snxt = tcp->tcp_snxt;
3621
3622 ASSERT(shrunk_count > 0);
3623
3624 if (!tcp->tcp_is_wnd_shrnk) {
3625 tcp->tcp_snxt_shrunk = snxt;
3626 tcp->tcp_is_wnd_shrnk = B_TRUE;
3627 } else if (SEQ_GT(snxt, tcp->tcp_snxt_shrunk)) {
3628 tcp->tcp_snxt_shrunk = snxt;
3629 }
3630
3631 /* Pretend we didn't send the data outside the window */
3632 snxt -= shrunk_count;
3633
3634 /* Reset all the values per the now shrunk window */
3635 tcp_update_xmit_tail(tcp, snxt);
3636 tcp->tcp_unsent += shrunk_count;
3637
3638 /*
3639 * If the SACK option is set, delete the entire list of
3640 * notsack'ed blocks.
3641 */
3642 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list, tcp);
3643
3644 if (tcp->tcp_suna == tcp->tcp_snxt && tcp->tcp_swnd == 0)
3645 /*
3646 * Make sure the timer is running so that we will probe a zero
3647 * window.
3648 */
3649 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
3650 }
3651
3652 /*
3653 * tcp_fill_header is called by tcp_send() to fill the outgoing TCP header
3654 * with the template header, as well as other options such as time-stamp,
3655 * ECN and/or SACK.
3656 */
3657 static void
3658 tcp_fill_header(tcp_t *tcp, uchar_t *rptr, int num_sack_blk)
3659 {
3660 tcpha_t *tcp_tmpl, *tcpha;
3661 uint32_t *dst, *src;
3662 int hdrlen;
3663 conn_t *connp = tcp->tcp_connp;
3664
3665 ASSERT(OK_32PTR(rptr));
3666
3667 /* Template header */
3668 tcp_tmpl = tcp->tcp_tcpha;
3669
3670 /* Header of outgoing packet */
3671 tcpha = (tcpha_t *)(rptr + connp->conn_ixa->ixa_ip_hdr_length);
3672
3673 /* dst and src are opaque 32-bit fields, used for copying */
3674 dst = (uint32_t *)rptr;
3675 src = (uint32_t *)connp->conn_ht_iphc;
3676 hdrlen = connp->conn_ht_iphc_len;
3677
3678 /* Fill time-stamp option if needed */
3679 if (tcp->tcp_snd_ts_ok) {
3680 U32_TO_BE32(LBOLT_FASTPATH,
3681 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 4);
3682 U32_TO_BE32(tcp->tcp_ts_recent,
3683 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 8);
3684 } else {
3685 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
3686 }
3687
3688 /*
3689 * Copy the template header; is this really more efficient than
3690 * calling bcopy()? For simple IPv4/TCP, it may be the case,
3691 * but perhaps not for other scenarios.
3692 */
3693 dst[0] = src[0];
3694 dst[1] = src[1];
3695 dst[2] = src[2];
3696 dst[3] = src[3];
3697 dst[4] = src[4];
3698 dst[5] = src[5];
3699 dst[6] = src[6];
3700 dst[7] = src[7];
3701 dst[8] = src[8];
3702 dst[9] = src[9];
3703 if (hdrlen -= 40) {
3704 hdrlen >>= 2;
3705 dst += 10;
3706 src += 10;
3707 do {
3708 *dst++ = *src++;
3709 } while (--hdrlen);
3710 }
3711
3712 /*
3713 * Set the ECN info in the TCP header if it is not a zero
3714 * window probe. Zero window probe is only sent in
3715 * tcp_wput_data() and tcp_timer().
3716 */
3717 if (tcp->tcp_ecn_ok && !tcp->tcp_zero_win_probe) {
3718 TCP_SET_ECT(tcp, rptr);
3719
3720 if (tcp->tcp_ecn_echo_on)
3721 tcpha->tha_flags |= TH_ECE;
3722 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3723 tcpha->tha_flags |= TH_CWR;
3724 tcp->tcp_ecn_cwr_sent = B_TRUE;
3725 }
3726 }
3727
3728 /* Fill in SACK options */
3729 if (num_sack_blk > 0) {
3730 uchar_t *wptr = rptr + connp->conn_ht_iphc_len;
3731 sack_blk_t *tmp;
3732 int32_t i;
3733
3734 wptr[0] = TCPOPT_NOP;
3735 wptr[1] = TCPOPT_NOP;
3736 wptr[2] = TCPOPT_SACK;
3737 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3738 sizeof (sack_blk_t);
3739 wptr += TCPOPT_REAL_SACK_LEN;
3740
3741 tmp = tcp->tcp_sack_list;
3742 for (i = 0; i < num_sack_blk; i++) {
3743 U32_TO_BE32(tmp[i].begin, wptr);
3744 wptr += sizeof (tcp_seq);
3745 U32_TO_BE32(tmp[i].end, wptr);
3746 wptr += sizeof (tcp_seq);
3747 }
3748 tcpha->tha_offset_and_reserved +=
3749 ((num_sack_blk * 2 + 1) << 4);
3750 }
3751 }