1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2014 by Delphix. All rights reserved.
25 */
26
27 /* This file contains all TCP output processing functions. */
28
29 #include <sys/types.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <sys/strsubr.h>
33 #include <sys/stropts.h>
34 #include <sys/strlog.h>
35 #define _SUN_TPI_VERSION 2
36 #include <sys/tihdr.h>
37 #include <sys/suntpi.h>
38 #include <sys/xti_inet.h>
39 #include <sys/timod.h>
40 #include <sys/pattr.h>
41 #include <sys/squeue_impl.h>
42 #include <sys/squeue.h>
43 #include <sys/sockio.h>
44 #include <sys/tsol/tnet.h>
45
46 #include <inet/common.h>
47 #include <inet/ip.h>
48 #include <inet/tcp.h>
49 #include <inet/tcp_impl.h>
50 #include <inet/snmpcom.h>
51 #include <inet/proto_set.h>
52 #include <inet/ipsec_impl.h>
53 #include <inet/ip_ndp.h>
54
55 static mblk_t *tcp_get_seg_mp(tcp_t *, uint32_t, int32_t *);
56 static void tcp_wput_cmdblk(queue_t *, mblk_t *);
57 static void tcp_wput_flush(tcp_t *, mblk_t *);
58 static void tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp);
59 static int tcp_xmit_end(tcp_t *);
60 static int tcp_send(tcp_t *, const int, const int, const int,
61 const int, int *, uint_t *, int *, mblk_t **, mblk_t *);
62 static void tcp_xmit_early_reset(char *, mblk_t *, uint32_t, uint32_t,
63 int, ip_recv_attr_t *, ip_stack_t *, conn_t *);
64 static boolean_t tcp_send_rst_chk(tcp_stack_t *);
65 static void tcp_process_shrunk_swnd(tcp_t *, uint32_t);
66 static void tcp_fill_header(tcp_t *, uchar_t *, clock_t, int);
67
68 /*
69 * Functions called directly via squeue having a prototype of edesc_t.
70 */
71 static void tcp_wput_nondata(void *, mblk_t *, void *, ip_recv_attr_t *);
72 static void tcp_wput_ioctl(void *, mblk_t *, void *, ip_recv_attr_t *);
73 static void tcp_wput_proto(void *, mblk_t *, void *, ip_recv_attr_t *);
74
75 /*
76 * This controls how tiny a write must be before we try to copy it
77 * into the mblk on the tail of the transmit queue. Not much
78 * speedup is observed for values larger than sixteen. Zero will
79 * disable the optimisation.
80 */
81 static int tcp_tx_pull_len = 16;
82
83 void
84 tcp_wput(queue_t *q, mblk_t *mp)
85 {
86 conn_t *connp = Q_TO_CONN(q);
87 tcp_t *tcp;
88 void (*output_proc)();
89 t_scalar_t type;
90 uchar_t *rptr;
91 struct iocblk *iocp;
92 size_t size;
93
94 ASSERT(connp->conn_ref >= 2);
95
96 switch (DB_TYPE(mp)) {
97 case M_DATA:
98 tcp = connp->conn_tcp;
99 ASSERT(tcp != NULL);
100
101 size = msgdsize(mp);
102
103 mutex_enter(&tcp->tcp_non_sq_lock);
104 tcp->tcp_squeue_bytes += size;
105 if (TCP_UNSENT_BYTES(tcp) > connp->conn_sndbuf) {
106 tcp_setqfull(tcp);
107 }
108 mutex_exit(&tcp->tcp_non_sq_lock);
109
110 CONN_INC_REF(connp);
111 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, tcp_output, connp,
112 NULL, tcp_squeue_flag, SQTAG_TCP_OUTPUT);
113 return;
114
115 case M_CMD:
116 tcp_wput_cmdblk(q, mp);
117 return;
118
119 case M_PROTO:
120 case M_PCPROTO:
121 /*
122 * if it is a snmp message, don't get behind the squeue
123 */
124 tcp = connp->conn_tcp;
125 rptr = mp->b_rptr;
126 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
127 type = ((union T_primitives *)rptr)->type;
128 } else {
129 if (connp->conn_debug) {
130 (void) strlog(TCP_MOD_ID, 0, 1,
131 SL_ERROR|SL_TRACE,
132 "tcp_wput_proto, dropping one...");
133 }
134 freemsg(mp);
135 return;
136 }
137 if (type == T_SVR4_OPTMGMT_REQ) {
138 /*
139 * All Solaris components should pass a db_credp
140 * for this TPI message, hence we ASSERT.
141 * But in case there is some other M_PROTO that looks
142 * like a TPI message sent by some other kernel
143 * component, we check and return an error.
144 */
145 cred_t *cr = msg_getcred(mp, NULL);
146
147 ASSERT(cr != NULL);
148 if (cr == NULL) {
149 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
150 return;
151 }
152 if (snmpcom_req(q, mp, tcp_snmp_set, ip_snmp_get,
153 cr)) {
154 /*
155 * This was a SNMP request
156 */
157 return;
158 } else {
159 output_proc = tcp_wput_proto;
160 }
161 } else {
162 output_proc = tcp_wput_proto;
163 }
164 break;
165 case M_IOCTL:
166 /*
167 * Most ioctls can be processed right away without going via
168 * squeues - process them right here. Those that do require
169 * squeue (currently _SIOCSOCKFALLBACK)
170 * are processed by tcp_wput_ioctl().
171 */
172 iocp = (struct iocblk *)mp->b_rptr;
173 tcp = connp->conn_tcp;
174
175 switch (iocp->ioc_cmd) {
176 case TCP_IOC_ABORT_CONN:
177 tcp_ioctl_abort_conn(q, mp);
178 return;
179 case TI_GETPEERNAME:
180 case TI_GETMYNAME:
181 mi_copyin(q, mp, NULL,
182 SIZEOF_STRUCT(strbuf, iocp->ioc_flag));
183 return;
184
185 default:
186 output_proc = tcp_wput_ioctl;
187 break;
188 }
189 break;
190 default:
191 output_proc = tcp_wput_nondata;
192 break;
193 }
194
195 CONN_INC_REF(connp);
196 SQUEUE_ENTER_ONE(connp->conn_sqp, mp, output_proc, connp,
197 NULL, tcp_squeue_flag, SQTAG_TCP_WPUT_OTHER);
198 }
199
200 /*
201 * The TCP normal data output path.
202 * NOTE: the logic of the fast path is duplicated from this function.
203 */
204 void
205 tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent)
206 {
207 int len;
208 mblk_t *local_time;
209 mblk_t *mp1;
210 uint32_t snxt;
211 int tail_unsent;
212 int tcpstate;
213 int usable = 0;
214 mblk_t *xmit_tail;
215 int32_t mss;
216 int32_t num_sack_blk = 0;
217 int32_t total_hdr_len;
218 int32_t tcp_hdr_len;
219 int rc;
220 tcp_stack_t *tcps = tcp->tcp_tcps;
221 conn_t *connp = tcp->tcp_connp;
222 clock_t now = LBOLT_FASTPATH;
223
224 tcpstate = tcp->tcp_state;
225 if (mp == NULL) {
226 /*
227 * tcp_wput_data() with NULL mp should only be called when
228 * there is unsent data.
229 */
230 ASSERT(tcp->tcp_unsent > 0);
231 /* Really tacky... but we need this for detached closes. */
232 len = tcp->tcp_unsent;
233 goto data_null;
234 }
235
236 ASSERT(mp->b_datap->db_type == M_DATA);
237 /*
238 * Don't allow data after T_ORDREL_REQ or T_DISCON_REQ,
239 * or before a connection attempt has begun.
240 */
241 if (tcpstate < TCPS_SYN_SENT || tcpstate > TCPS_CLOSE_WAIT ||
242 (tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
243 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
244 #ifdef DEBUG
245 cmn_err(CE_WARN,
246 "tcp_wput_data: data after ordrel, %s",
247 tcp_display(tcp, NULL,
248 DISP_ADDR_AND_PORT));
249 #else
250 if (connp->conn_debug) {
251 (void) strlog(TCP_MOD_ID, 0, 1,
252 SL_TRACE|SL_ERROR,
253 "tcp_wput_data: data after ordrel, %s\n",
254 tcp_display(tcp, NULL,
255 DISP_ADDR_AND_PORT));
256 }
257 #endif /* DEBUG */
258 }
259 if (tcp->tcp_snd_zcopy_aware &&
260 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
261 tcp_zcopy_notify(tcp);
262 freemsg(mp);
263 mutex_enter(&tcp->tcp_non_sq_lock);
264 if (tcp->tcp_flow_stopped &&
265 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
266 tcp_clrqfull(tcp);
267 }
268 mutex_exit(&tcp->tcp_non_sq_lock);
269 return;
270 }
271
272 /* Strip empties */
273 for (;;) {
274 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <=
275 (uintptr_t)INT_MAX);
276 len = (int)(mp->b_wptr - mp->b_rptr);
277 if (len > 0)
278 break;
279 mp1 = mp;
280 mp = mp->b_cont;
281 freeb(mp1);
282 if (mp == NULL) {
283 return;
284 }
285 }
286
287 /* If we are the first on the list ... */
288 if (tcp->tcp_xmit_head == NULL) {
289 tcp->tcp_xmit_head = mp;
290 tcp->tcp_xmit_tail = mp;
291 tcp->tcp_xmit_tail_unsent = len;
292 } else {
293 /* If tiny tx and room in txq tail, pullup to save mblks. */
294 struct datab *dp;
295
296 mp1 = tcp->tcp_xmit_last;
297 if (len < tcp_tx_pull_len &&
298 (dp = mp1->b_datap)->db_ref == 1 &&
299 dp->db_lim - mp1->b_wptr >= len) {
300 ASSERT(len > 0);
301 ASSERT(!mp1->b_cont);
302 if (len == 1) {
303 *mp1->b_wptr++ = *mp->b_rptr;
304 } else {
305 bcopy(mp->b_rptr, mp1->b_wptr, len);
306 mp1->b_wptr += len;
307 }
308 if (mp1 == tcp->tcp_xmit_tail)
309 tcp->tcp_xmit_tail_unsent += len;
310 mp1->b_cont = mp->b_cont;
311 if (tcp->tcp_snd_zcopy_aware &&
312 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
313 mp1->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
314 freeb(mp);
315 mp = mp1;
316 } else {
317 tcp->tcp_xmit_last->b_cont = mp;
318 }
319 len += tcp->tcp_unsent;
320 }
321
322 /* Tack on however many more positive length mblks we have */
323 if ((mp1 = mp->b_cont) != NULL) {
324 do {
325 int tlen;
326 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <=
327 (uintptr_t)INT_MAX);
328 tlen = (int)(mp1->b_wptr - mp1->b_rptr);
329 if (tlen <= 0) {
330 mp->b_cont = mp1->b_cont;
331 freeb(mp1);
332 } else {
333 len += tlen;
334 mp = mp1;
335 }
336 } while ((mp1 = mp->b_cont) != NULL);
337 }
338 tcp->tcp_xmit_last = mp;
339 tcp->tcp_unsent = len;
340
341 if (urgent)
342 usable = 1;
343
344 data_null:
345 snxt = tcp->tcp_snxt;
346 xmit_tail = tcp->tcp_xmit_tail;
347 tail_unsent = tcp->tcp_xmit_tail_unsent;
348
349 /*
350 * Note that tcp_mss has been adjusted to take into account the
351 * timestamp option if applicable. Because SACK options do not
352 * appear in every TCP segments and they are of variable lengths,
353 * they cannot be included in tcp_mss. Thus we need to calculate
354 * the actual segment length when we need to send a segment which
355 * includes SACK options.
356 */
357 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
358 int32_t opt_len;
359
360 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
361 tcp->tcp_num_sack_blk);
362 opt_len = num_sack_blk * sizeof (sack_blk_t) + TCPOPT_NOP_LEN *
363 2 + TCPOPT_HEADER_LEN;
364 mss = tcp->tcp_mss - opt_len;
365 total_hdr_len = connp->conn_ht_iphc_len + opt_len;
366 tcp_hdr_len = connp->conn_ht_ulp_len + opt_len;
367 } else {
368 mss = tcp->tcp_mss;
369 total_hdr_len = connp->conn_ht_iphc_len;
370 tcp_hdr_len = connp->conn_ht_ulp_len;
371 }
372
373 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
374 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
375 TCP_SET_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle);
376 }
377 if (tcpstate == TCPS_SYN_RCVD) {
378 /*
379 * The three-way connection establishment handshake is not
380 * complete yet. We want to queue the data for transmission
381 * after entering ESTABLISHED state (RFC793). A jump to
382 * "done" label effectively leaves data on the queue.
383 */
384 goto done;
385 } else {
386 int usable_r;
387
388 /*
389 * In the special case when cwnd is zero, which can only
390 * happen if the connection is ECN capable, return now.
391 * New segments is sent using tcp_timer(). The timer
392 * is set in tcp_input_data().
393 */
394 if (tcp->tcp_cwnd == 0) {
395 /*
396 * Note that tcp_cwnd is 0 before 3-way handshake is
397 * finished.
398 */
399 ASSERT(tcp->tcp_ecn_ok ||
400 tcp->tcp_state < TCPS_ESTABLISHED);
401 return;
402 }
403
404 /* NOTE: trouble if xmitting while SYN not acked? */
405 usable_r = snxt - tcp->tcp_suna;
406 usable_r = tcp->tcp_swnd - usable_r;
407
408 /*
409 * Check if the receiver has shrunk the window. If
410 * tcp_wput_data() with NULL mp is called, tcp_fin_sent
411 * cannot be set as there is unsent data, so FIN cannot
412 * be sent out. Otherwise, we need to take into account
413 * of FIN as it consumes an "invisible" sequence number.
414 */
415 ASSERT(tcp->tcp_fin_sent == 0);
416 if (usable_r < 0) {
417 /*
418 * The receiver has shrunk the window and we have sent
419 * -usable_r date beyond the window, re-adjust.
420 *
421 * If TCP window scaling is enabled, there can be
422 * round down error as the advertised receive window
423 * is actually right shifted n bits. This means that
424 * the lower n bits info is wiped out. It will look
425 * like the window is shrunk. Do a check here to
426 * see if the shrunk amount is actually within the
427 * error in window calculation. If it is, just
428 * return. Note that this check is inside the
429 * shrunk window check. This makes sure that even
430 * though tcp_process_shrunk_swnd() is not called,
431 * we will stop further processing.
432 */
433 if ((-usable_r >> tcp->tcp_snd_ws) > 0) {
434 tcp_process_shrunk_swnd(tcp, -usable_r);
435 }
436 return;
437 }
438
439 /* usable = MIN(swnd, cwnd) - unacked_bytes */
440 if (tcp->tcp_swnd > tcp->tcp_cwnd)
441 usable_r -= tcp->tcp_swnd - tcp->tcp_cwnd;
442
443 /* usable = MIN(usable, unsent) */
444 if (usable_r > len)
445 usable_r = len;
446
447 /* usable = MAX(usable, {1 for urgent, 0 for data}) */
448 if (usable_r > 0) {
449 usable = usable_r;
450 } else {
451 /* Bypass all other unnecessary processing. */
452 goto done;
453 }
454 }
455
456 local_time = (mblk_t *)now;
457
458 /*
459 * "Our" Nagle Algorithm. This is not the same as in the old
460 * BSD. This is more in line with the true intent of Nagle.
461 *
462 * The conditions are:
463 * 1. The amount of unsent data (or amount of data which can be
464 * sent, whichever is smaller) is less than Nagle limit.
465 * 2. The last sent size is also less than Nagle limit.
466 * 3. There is unack'ed data.
467 * 4. Urgent pointer is not set. Send urgent data ignoring the
468 * Nagle algorithm. This reduces the probability that urgent
469 * bytes get "merged" together.
470 * 5. The app has not closed the connection. This eliminates the
471 * wait time of the receiving side waiting for the last piece of
472 * (small) data.
473 *
474 * If all are satisified, exit without sending anything. Note
475 * that Nagle limit can be smaller than 1 MSS. Nagle limit is
476 * the smaller of 1 MSS and global tcp_naglim_def (default to be
477 * 4095).
478 */
479 if (usable < (int)tcp->tcp_naglim &&
480 tcp->tcp_naglim > tcp->tcp_last_sent_len &&
481 snxt != tcp->tcp_suna &&
482 !(tcp->tcp_valid_bits & TCP_URG_VALID) &&
483 !(tcp->tcp_valid_bits & TCP_FSS_VALID)) {
484 goto done;
485 }
486
487 /*
488 * If tcp_zero_win_probe is not set and the tcp->tcp_cork option
489 * is set, then we have to force TCP not to send partial segment
490 * (smaller than MSS bytes). We are calculating the usable now
491 * based on full mss and will save the rest of remaining data for
492 * later. When tcp_zero_win_probe is set, TCP needs to send out
493 * something to do zero window probe.
494 */
495 if (tcp->tcp_cork && !tcp->tcp_zero_win_probe) {
496 if (usable < mss)
497 goto done;
498 usable = (usable / mss) * mss;
499 }
500
501 /* Update the latest receive window size in TCP header. */
502 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
503
504 /* Send the packet. */
505 rc = tcp_send(tcp, mss, total_hdr_len, tcp_hdr_len,
506 num_sack_blk, &usable, &snxt, &tail_unsent, &xmit_tail,
507 local_time);
508
509 /* Pretend that all we were trying to send really got sent */
510 if (rc < 0 && tail_unsent < 0) {
511 do {
512 xmit_tail = xmit_tail->b_cont;
513 xmit_tail->b_prev = local_time;
514 ASSERT((uintptr_t)(xmit_tail->b_wptr -
515 xmit_tail->b_rptr) <= (uintptr_t)INT_MAX);
516 tail_unsent += (int)(xmit_tail->b_wptr -
517 xmit_tail->b_rptr);
518 } while (tail_unsent < 0);
519 }
520 done:;
521 tcp->tcp_xmit_tail = xmit_tail;
522 tcp->tcp_xmit_tail_unsent = tail_unsent;
523 len = tcp->tcp_snxt - snxt;
524 if (len) {
525 /*
526 * If new data was sent, need to update the notsack
527 * list, which is, afterall, data blocks that have
528 * not been sack'ed by the receiver. New data is
529 * not sack'ed.
530 */
531 if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) {
532 /* len is a negative value. */
533 tcp->tcp_pipe -= len;
534 tcp_notsack_update(&(tcp->tcp_notsack_list),
535 tcp->tcp_snxt, snxt,
536 &(tcp->tcp_num_notsack_blk),
537 &(tcp->tcp_cnt_notsack_list));
538 }
539 tcp->tcp_snxt = snxt + tcp->tcp_fin_sent;
540 tcp->tcp_rack = tcp->tcp_rnxt;
541 tcp->tcp_rack_cnt = 0;
542 if ((snxt + len) == tcp->tcp_suna) {
543 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
544 }
545 } else if (snxt == tcp->tcp_suna && tcp->tcp_swnd == 0) {
546 /*
547 * Didn't send anything. Make sure the timer is running
548 * so that we will probe a zero window.
549 */
550 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
551 }
552 /* Note that len is the amount we just sent but with a negative sign */
553 tcp->tcp_unsent += len;
554 mutex_enter(&tcp->tcp_non_sq_lock);
555 if (tcp->tcp_flow_stopped) {
556 if (TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
557 tcp_clrqfull(tcp);
558 }
559 } else if (TCP_UNSENT_BYTES(tcp) >= connp->conn_sndbuf) {
560 if (!(tcp->tcp_detached))
561 tcp_setqfull(tcp);
562 }
563 mutex_exit(&tcp->tcp_non_sq_lock);
564 }
565
566 /*
567 * Initial STREAMS write side put() procedure for sockets. It tries to
568 * handle the T_CAPABILITY_REQ which sockfs sends down while setting
569 * up the socket without using the squeue. Non T_CAPABILITY_REQ messages
570 * are handled by tcp_wput() as usual.
571 *
572 * All further messages will also be handled by tcp_wput() because we cannot
573 * be sure that the above short cut is safe later.
574 */
575 void
576 tcp_wput_sock(queue_t *wq, mblk_t *mp)
577 {
578 conn_t *connp = Q_TO_CONN(wq);
579 tcp_t *tcp = connp->conn_tcp;
580 struct T_capability_req *car = (struct T_capability_req *)mp->b_rptr;
581
582 ASSERT(wq->q_qinfo == &tcp_sock_winit);
583 wq->q_qinfo = &tcp_winit;
584
585 ASSERT(IPCL_IS_TCP(connp));
586 ASSERT(TCP_IS_SOCKET(tcp));
587
588 if (DB_TYPE(mp) == M_PCPROTO &&
589 MBLKL(mp) == sizeof (struct T_capability_req) &&
590 car->PRIM_type == T_CAPABILITY_REQ) {
591 tcp_capability_req(tcp, mp);
592 return;
593 }
594
595 tcp_wput(wq, mp);
596 }
597
598 /* ARGSUSED */
599 void
600 tcp_wput_fallback(queue_t *wq, mblk_t *mp)
601 {
602 #ifdef DEBUG
603 cmn_err(CE_CONT, "tcp_wput_fallback: Message during fallback \n");
604 #endif
605 freemsg(mp);
606 }
607
608 /*
609 * Call by tcp_wput() to handle misc non M_DATA messages.
610 */
611 /* ARGSUSED */
612 static void
613 tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
614 {
615 conn_t *connp = (conn_t *)arg;
616 tcp_t *tcp = connp->conn_tcp;
617
618 ASSERT(DB_TYPE(mp) != M_IOCTL);
619 /*
620 * TCP is D_MP and qprocsoff() is done towards the end of the tcp_close.
621 * Once the close starts, streamhead and sockfs will not let any data
622 * packets come down (close ensures that there are no threads using the
623 * queue and no new threads will come down) but since qprocsoff()
624 * hasn't happened yet, a M_FLUSH or some non data message might
625 * get reflected back (in response to our own FLUSHRW) and get
626 * processed after tcp_close() is done. The conn would still be valid
627 * because a ref would have added but we need to check the state
628 * before actually processing the packet.
629 */
630 if (TCP_IS_DETACHED(tcp) || (tcp->tcp_state == TCPS_CLOSED)) {
631 freemsg(mp);
632 return;
633 }
634
635 switch (DB_TYPE(mp)) {
636 case M_IOCDATA:
637 tcp_wput_iocdata(tcp, mp);
638 break;
639 case M_FLUSH:
640 tcp_wput_flush(tcp, mp);
641 break;
642 default:
643 ip_wput_nondata(connp->conn_wq, mp);
644 break;
645 }
646 }
647
648 /* tcp_wput_flush is called by tcp_wput_nondata to handle M_FLUSH messages. */
649 static void
650 tcp_wput_flush(tcp_t *tcp, mblk_t *mp)
651 {
652 uchar_t fval = *mp->b_rptr;
653 mblk_t *tail;
654 conn_t *connp = tcp->tcp_connp;
655 queue_t *q = connp->conn_wq;
656
657 /* TODO: How should flush interact with urgent data? */
658 if ((fval & FLUSHW) && tcp->tcp_xmit_head != NULL &&
659 !(tcp->tcp_valid_bits & TCP_URG_VALID)) {
660 /*
661 * Flush only data that has not yet been put on the wire. If
662 * we flush data that we have already transmitted, life, as we
663 * know it, may come to an end.
664 */
665 tail = tcp->tcp_xmit_tail;
666 tail->b_wptr -= tcp->tcp_xmit_tail_unsent;
667 tcp->tcp_xmit_tail_unsent = 0;
668 tcp->tcp_unsent = 0;
669 if (tail->b_wptr != tail->b_rptr)
670 tail = tail->b_cont;
671 if (tail) {
672 mblk_t **excess = &tcp->tcp_xmit_head;
673 for (;;) {
674 mblk_t *mp1 = *excess;
675 if (mp1 == tail)
676 break;
677 tcp->tcp_xmit_tail = mp1;
678 tcp->tcp_xmit_last = mp1;
679 excess = &mp1->b_cont;
680 }
681 *excess = NULL;
682 tcp_close_mpp(&tail);
683 if (tcp->tcp_snd_zcopy_aware)
684 tcp_zcopy_notify(tcp);
685 }
686 /*
687 * We have no unsent data, so unsent must be less than
688 * conn_sndlowat, so re-enable flow.
689 */
690 mutex_enter(&tcp->tcp_non_sq_lock);
691 if (tcp->tcp_flow_stopped) {
692 tcp_clrqfull(tcp);
693 }
694 mutex_exit(&tcp->tcp_non_sq_lock);
695 }
696 /*
697 * TODO: you can't just flush these, you have to increase rwnd for one
698 * thing. For another, how should urgent data interact?
699 */
700 if (fval & FLUSHR) {
701 *mp->b_rptr = fval & ~FLUSHW;
702 /* XXX */
703 qreply(q, mp);
704 return;
705 }
706 freemsg(mp);
707 }
708
709 /*
710 * tcp_wput_iocdata is called by tcp_wput_nondata to handle all M_IOCDATA
711 * messages.
712 */
713 static void
714 tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp)
715 {
716 mblk_t *mp1;
717 struct iocblk *iocp = (struct iocblk *)mp->b_rptr;
718 STRUCT_HANDLE(strbuf, sb);
719 uint_t addrlen;
720 conn_t *connp = tcp->tcp_connp;
721 queue_t *q = connp->conn_wq;
722
723 /* Make sure it is one of ours. */
724 switch (iocp->ioc_cmd) {
725 case TI_GETMYNAME:
726 case TI_GETPEERNAME:
727 break;
728 default:
729 /*
730 * If the conn is closing, then error the ioctl here. Otherwise
731 * use the CONN_IOCTLREF_* macros to hold off tcp_close until
732 * we're done here.
733 */
734 mutex_enter(&connp->conn_lock);
735 if (connp->conn_state_flags & CONN_CLOSING) {
736 mutex_exit(&connp->conn_lock);
737 iocp->ioc_error = EINVAL;
738 mp->b_datap->db_type = M_IOCNAK;
739 iocp->ioc_count = 0;
740 qreply(q, mp);
741 return;
742 }
743
744 CONN_INC_IOCTLREF_LOCKED(connp);
745 ip_wput_nondata(q, mp);
746 CONN_DEC_IOCTLREF(connp);
747 return;
748 }
749 switch (mi_copy_state(q, mp, &mp1)) {
750 case -1:
751 return;
752 case MI_COPY_CASE(MI_COPY_IN, 1):
753 break;
754 case MI_COPY_CASE(MI_COPY_OUT, 1):
755 /* Copy out the strbuf. */
756 mi_copyout(q, mp);
757 return;
758 case MI_COPY_CASE(MI_COPY_OUT, 2):
759 /* All done. */
760 mi_copy_done(q, mp, 0);
761 return;
762 default:
763 mi_copy_done(q, mp, EPROTO);
764 return;
765 }
766 /* Check alignment of the strbuf */
767 if (!OK_32PTR(mp1->b_rptr)) {
768 mi_copy_done(q, mp, EINVAL);
769 return;
770 }
771
772 STRUCT_SET_HANDLE(sb, iocp->ioc_flag, (void *)mp1->b_rptr);
773
774 if (connp->conn_family == AF_INET)
775 addrlen = sizeof (sin_t);
776 else
777 addrlen = sizeof (sin6_t);
778
779 if (STRUCT_FGET(sb, maxlen) < addrlen) {
780 mi_copy_done(q, mp, EINVAL);
781 return;
782 }
783
784 switch (iocp->ioc_cmd) {
785 case TI_GETMYNAME:
786 break;
787 case TI_GETPEERNAME:
788 if (tcp->tcp_state < TCPS_SYN_RCVD) {
789 mi_copy_done(q, mp, ENOTCONN);
790 return;
791 }
792 break;
793 }
794 mp1 = mi_copyout_alloc(q, mp, STRUCT_FGETP(sb, buf), addrlen, B_TRUE);
795 if (!mp1)
796 return;
797
798 STRUCT_FSET(sb, len, addrlen);
799 switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) {
800 case TI_GETMYNAME:
801 (void) conn_getsockname(connp, (struct sockaddr *)mp1->b_wptr,
802 &addrlen);
803 break;
804 case TI_GETPEERNAME:
805 (void) conn_getpeername(connp, (struct sockaddr *)mp1->b_wptr,
806 &addrlen);
807 break;
808 }
809 mp1->b_wptr += addrlen;
810 /* Copy out the address */
811 mi_copyout(q, mp);
812 }
813
814 /*
815 * tcp_wput_ioctl is called by tcp_wput_nondata() to handle all M_IOCTL
816 * messages.
817 */
818 /* ARGSUSED */
819 static void
820 tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
821 {
822 conn_t *connp = (conn_t *)arg;
823 tcp_t *tcp = connp->conn_tcp;
824 queue_t *q = connp->conn_wq;
825 struct iocblk *iocp;
826
827 ASSERT(DB_TYPE(mp) == M_IOCTL);
828 /*
829 * Try and ASSERT the minimum possible references on the
830 * conn early enough. Since we are executing on write side,
831 * the connection is obviously not detached and that means
832 * there is a ref each for TCP and IP. Since we are behind
833 * the squeue, the minimum references needed are 3. If the
834 * conn is in classifier hash list, there should be an
835 * extra ref for that (we check both the possibilities).
836 */
837 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
838 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
839
840 iocp = (struct iocblk *)mp->b_rptr;
841 switch (iocp->ioc_cmd) {
842 case _SIOCSOCKFALLBACK:
843 /*
844 * Either sockmod is about to be popped and the socket
845 * would now be treated as a plain stream, or a module
846 * is about to be pushed so we could no longer use read-
847 * side synchronous streams for fused loopback tcp.
848 * Drain any queued data and disable direct sockfs
849 * interface from now on.
850 */
851 if (!tcp->tcp_issocket) {
852 DB_TYPE(mp) = M_IOCNAK;
853 iocp->ioc_error = EINVAL;
854 } else {
855 tcp_use_pure_tpi(tcp);
856 DB_TYPE(mp) = M_IOCACK;
857 iocp->ioc_error = 0;
858 }
859 iocp->ioc_count = 0;
860 iocp->ioc_rval = 0;
861 qreply(q, mp);
862 return;
863 }
864
865 /*
866 * If the conn is closing, then error the ioctl here. Otherwise bump the
867 * conn_ioctlref to hold off tcp_close until we're done here.
868 */
869 mutex_enter(&(connp)->conn_lock);
870 if ((connp)->conn_state_flags & CONN_CLOSING) {
871 mutex_exit(&(connp)->conn_lock);
872 iocp->ioc_error = EINVAL;
873 mp->b_datap->db_type = M_IOCNAK;
874 iocp->ioc_count = 0;
875 qreply(q, mp);
876 return;
877 }
878
879 CONN_INC_IOCTLREF_LOCKED(connp);
880 ip_wput_nondata(q, mp);
881 CONN_DEC_IOCTLREF(connp);
882 }
883
884 /*
885 * This routine is called by tcp_wput() to handle all TPI requests.
886 */
887 /* ARGSUSED */
888 static void
889 tcp_wput_proto(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
890 {
891 conn_t *connp = (conn_t *)arg;
892 tcp_t *tcp = connp->conn_tcp;
893 union T_primitives *tprim = (union T_primitives *)mp->b_rptr;
894 uchar_t *rptr;
895 t_scalar_t type;
896 cred_t *cr;
897
898 /*
899 * Try and ASSERT the minimum possible references on the
900 * conn early enough. Since we are executing on write side,
901 * the connection is obviously not detached and that means
902 * there is a ref each for TCP and IP. Since we are behind
903 * the squeue, the minimum references needed are 3. If the
904 * conn is in classifier hash list, there should be an
905 * extra ref for that (we check both the possibilities).
906 */
907 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
908 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
909
910 rptr = mp->b_rptr;
911 ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX);
912 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
913 type = ((union T_primitives *)rptr)->type;
914 if (type == T_EXDATA_REQ) {
915 tcp_output_urgent(connp, mp, arg2, NULL);
916 } else if (type != T_DATA_REQ) {
917 goto non_urgent_data;
918 } else {
919 /* TODO: options, flags, ... from user */
920 /* Set length to zero for reclamation below */
921 tcp_wput_data(tcp, mp->b_cont, B_TRUE);
922 freeb(mp);
923 }
924 return;
925 } else {
926 if (connp->conn_debug) {
927 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
928 "tcp_wput_proto, dropping one...");
929 }
930 freemsg(mp);
931 return;
932 }
933
934 non_urgent_data:
935
936 switch ((int)tprim->type) {
937 case O_T_BIND_REQ: /* bind request */
938 case T_BIND_REQ: /* new semantics bind request */
939 tcp_tpi_bind(tcp, mp);
940 break;
941 case T_UNBIND_REQ: /* unbind request */
942 tcp_tpi_unbind(tcp, mp);
943 break;
944 case O_T_CONN_RES: /* old connection response XXX */
945 case T_CONN_RES: /* connection response */
946 tcp_tli_accept(tcp, mp);
947 break;
948 case T_CONN_REQ: /* connection request */
949 tcp_tpi_connect(tcp, mp);
950 break;
951 case T_DISCON_REQ: /* disconnect request */
952 tcp_disconnect(tcp, mp);
953 break;
954 case T_CAPABILITY_REQ:
955 tcp_capability_req(tcp, mp); /* capability request */
956 break;
957 case T_INFO_REQ: /* information request */
958 tcp_info_req(tcp, mp);
959 break;
960 case T_SVR4_OPTMGMT_REQ: /* manage options req */
961 case T_OPTMGMT_REQ:
962 /*
963 * Note: no support for snmpcom_req() through new
964 * T_OPTMGMT_REQ. See comments in ip.c
965 */
966
967 /*
968 * All Solaris components should pass a db_credp
969 * for this TPI message, hence we ASSERT.
970 * But in case there is some other M_PROTO that looks
971 * like a TPI message sent by some other kernel
972 * component, we check and return an error.
973 */
974 cr = msg_getcred(mp, NULL);
975 ASSERT(cr != NULL);
976 if (cr == NULL) {
977 tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
978 return;
979 }
980 /*
981 * If EINPROGRESS is returned, the request has been queued
982 * for subsequent processing by ip_restart_optmgmt(), which
983 * will do the CONN_DEC_REF().
984 */
985 if ((int)tprim->type == T_SVR4_OPTMGMT_REQ) {
986 svr4_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
987 } else {
988 tpi_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
989 }
990 break;
991
992 case T_UNITDATA_REQ: /* unitdata request */
993 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
994 break;
995 case T_ORDREL_REQ: /* orderly release req */
996 freemsg(mp);
997
998 if (tcp->tcp_fused)
999 tcp_unfuse(tcp);
1000
1001 if (tcp_xmit_end(tcp) != 0) {
1002 /*
1003 * We were crossing FINs and got a reset from
1004 * the other side. Just ignore it.
1005 */
1006 if (connp->conn_debug) {
1007 (void) strlog(TCP_MOD_ID, 0, 1,
1008 SL_ERROR|SL_TRACE,
1009 "tcp_wput_proto, T_ORDREL_REQ out of "
1010 "state %s",
1011 tcp_display(tcp, NULL,
1012 DISP_ADDR_AND_PORT));
1013 }
1014 }
1015 break;
1016 case T_ADDR_REQ:
1017 tcp_addr_req(tcp, mp);
1018 break;
1019 default:
1020 if (connp->conn_debug) {
1021 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
1022 "tcp_wput_proto, bogus TPI msg, type %d",
1023 tprim->type);
1024 }
1025 /*
1026 * We used to M_ERROR. Sending TNOTSUPPORT gives the user
1027 * to recover.
1028 */
1029 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
1030 break;
1031 }
1032 }
1033
1034 /*
1035 * Handle special out-of-band ioctl requests (see PSARC/2008/265).
1036 */
1037 static void
1038 tcp_wput_cmdblk(queue_t *q, mblk_t *mp)
1039 {
1040 void *data;
1041 mblk_t *datamp = mp->b_cont;
1042 conn_t *connp = Q_TO_CONN(q);
1043 tcp_t *tcp = connp->conn_tcp;
1044 cmdblk_t *cmdp = (cmdblk_t *)mp->b_rptr;
1045
1046 if (datamp == NULL || MBLKL(datamp) < cmdp->cb_len) {
1047 cmdp->cb_error = EPROTO;
1048 qreply(q, mp);
1049 return;
1050 }
1051
1052 data = datamp->b_rptr;
1053
1054 switch (cmdp->cb_cmd) {
1055 case TI_GETPEERNAME:
1056 if (tcp->tcp_state < TCPS_SYN_RCVD)
1057 cmdp->cb_error = ENOTCONN;
1058 else
1059 cmdp->cb_error = conn_getpeername(connp, data,
1060 &cmdp->cb_len);
1061 break;
1062 case TI_GETMYNAME:
1063 cmdp->cb_error = conn_getsockname(connp, data, &cmdp->cb_len);
1064 break;
1065 default:
1066 cmdp->cb_error = EINVAL;
1067 break;
1068 }
1069
1070 qreply(q, mp);
1071 }
1072
1073 /*
1074 * The TCP fast path write put procedure.
1075 * NOTE: the logic of the fast path is duplicated from tcp_wput_data()
1076 */
1077 /* ARGSUSED */
1078 void
1079 tcp_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1080 {
1081 int len;
1082 int hdrlen;
1083 int plen;
1084 mblk_t *mp1;
1085 uchar_t *rptr;
1086 uint32_t snxt;
1087 tcpha_t *tcpha;
1088 struct datab *db;
1089 uint32_t suna;
1090 uint32_t mss;
1091 ipaddr_t *dst;
1092 ipaddr_t *src;
1093 uint32_t sum;
1094 int usable;
1095 conn_t *connp = (conn_t *)arg;
1096 tcp_t *tcp = connp->conn_tcp;
1097 uint32_t msize;
1098 tcp_stack_t *tcps = tcp->tcp_tcps;
1099 ip_xmit_attr_t *ixa;
1100 clock_t now;
1101
1102 /*
1103 * Try and ASSERT the minimum possible references on the
1104 * conn early enough. Since we are executing on write side,
1105 * the connection is obviously not detached and that means
1106 * there is a ref each for TCP and IP. Since we are behind
1107 * the squeue, the minimum references needed are 3. If the
1108 * conn is in classifier hash list, there should be an
1109 * extra ref for that (we check both the possibilities).
1110 */
1111 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
1112 (connp->conn_fanout == NULL && connp->conn_ref >= 3));
1113
1114 ASSERT(DB_TYPE(mp) == M_DATA);
1115 msize = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1116
1117 mutex_enter(&tcp->tcp_non_sq_lock);
1118 tcp->tcp_squeue_bytes -= msize;
1119 mutex_exit(&tcp->tcp_non_sq_lock);
1120
1121 /* Bypass tcp protocol for fused tcp loopback */
1122 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1123 return;
1124
1125 mss = tcp->tcp_mss;
1126 /*
1127 * If ZEROCOPY has turned off, try not to send any zero-copy message
1128 * down. Do backoff, now.
1129 */
1130 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on)
1131 mp = tcp_zcopy_backoff(tcp, mp, B_FALSE);
1132
1133
1134 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX);
1135 len = (int)(mp->b_wptr - mp->b_rptr);
1136
1137 /*
1138 * Criteria for fast path:
1139 *
1140 * 1. no unsent data
1141 * 2. single mblk in request
1142 * 3. connection established
1143 * 4. data in mblk
1144 * 5. len <= mss
1145 * 6. no tcp_valid bits
1146 */
1147 if ((tcp->tcp_unsent != 0) ||
1148 (tcp->tcp_cork) ||
1149 (mp->b_cont != NULL) ||
1150 (tcp->tcp_state != TCPS_ESTABLISHED) ||
1151 (len == 0) ||
1152 (len > mss) ||
1153 (tcp->tcp_valid_bits != 0)) {
1154 tcp_wput_data(tcp, mp, B_FALSE);
1155 return;
1156 }
1157
1158 ASSERT(tcp->tcp_xmit_tail_unsent == 0);
1159 ASSERT(tcp->tcp_fin_sent == 0);
1160
1161 /* queue new packet onto retransmission queue */
1162 if (tcp->tcp_xmit_head == NULL) {
1163 tcp->tcp_xmit_head = mp;
1164 } else {
1165 tcp->tcp_xmit_last->b_cont = mp;
1166 }
1167 tcp->tcp_xmit_last = mp;
1168 tcp->tcp_xmit_tail = mp;
1169
1170 /* find out how much we can send */
1171 /* BEGIN CSTYLED */
1172 /*
1173 * un-acked usable
1174 * |--------------|-----------------|
1175 * tcp_suna tcp_snxt tcp_suna+tcp_swnd
1176 */
1177 /* END CSTYLED */
1178
1179 /* start sending from tcp_snxt */
1180 snxt = tcp->tcp_snxt;
1181
1182 /*
1183 * Check to see if this connection has been idled for some
1184 * time and no ACK is expected. If it is, we need to slow
1185 * start again to get back the connection's "self-clock" as
1186 * described in VJ's paper.
1187 *
1188 * Reinitialize tcp_cwnd after idle.
1189 */
1190 now = LBOLT_FASTPATH;
1191 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
1192 (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
1193 TCP_SET_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle);
1194 }
1195
1196 usable = tcp->tcp_swnd; /* tcp window size */
1197 if (usable > tcp->tcp_cwnd)
1198 usable = tcp->tcp_cwnd; /* congestion window smaller */
1199 usable -= snxt; /* subtract stuff already sent */
1200 suna = tcp->tcp_suna;
1201 usable += suna;
1202 /* usable can be < 0 if the congestion window is smaller */
1203 if (len > usable) {
1204 /* Can't send complete M_DATA in one shot */
1205 goto slow;
1206 }
1207
1208 mutex_enter(&tcp->tcp_non_sq_lock);
1209 if (tcp->tcp_flow_stopped &&
1210 TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
1211 tcp_clrqfull(tcp);
1212 }
1213 mutex_exit(&tcp->tcp_non_sq_lock);
1214
1215 /*
1216 * determine if anything to send (Nagle).
1217 *
1218 * 1. len < tcp_mss (i.e. small)
1219 * 2. unacknowledged data present
1220 * 3. len < nagle limit
1221 * 4. last packet sent < nagle limit (previous packet sent)
1222 */
1223 if ((len < mss) && (snxt != suna) &&
1224 (len < (int)tcp->tcp_naglim) &&
1225 (tcp->tcp_last_sent_len < tcp->tcp_naglim)) {
1226 /*
1227 * This was the first unsent packet and normally
1228 * mss < xmit_hiwater so there is no need to worry
1229 * about flow control. The next packet will go
1230 * through the flow control check in tcp_wput_data().
1231 */
1232 /* leftover work from above */
1233 tcp->tcp_unsent = len;
1234 tcp->tcp_xmit_tail_unsent = len;
1235
1236 return;
1237 }
1238
1239 /*
1240 * len <= tcp->tcp_mss && len == unsent so no sender silly window. Can
1241 * send now.
1242 */
1243
1244 if (snxt == suna) {
1245 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1246 }
1247
1248 /* we have always sent something */
1249 tcp->tcp_rack_cnt = 0;
1250
1251 tcp->tcp_snxt = snxt + len;
1252 tcp->tcp_rack = tcp->tcp_rnxt;
1253
1254 if ((mp1 = dupb(mp)) == 0)
1255 goto no_memory;
1256 mp->b_prev = (mblk_t *)(uintptr_t)now;
1257 mp->b_next = (mblk_t *)(uintptr_t)snxt;
1258
1259 /* adjust tcp header information */
1260 tcpha = tcp->tcp_tcpha;
1261 tcpha->tha_flags = (TH_ACK|TH_PUSH);
1262
1263 sum = len + connp->conn_ht_ulp_len + connp->conn_sum;
1264 sum = (sum >> 16) + (sum & 0xFFFF);
1265 tcpha->tha_sum = htons(sum);
1266
1267 tcpha->tha_seq = htonl(snxt);
1268
1269 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1270 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1271 BUMP_LOCAL(tcp->tcp_obsegs);
1272
1273 /* Update the latest receive window size in TCP header. */
1274 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
1275
1276 tcp->tcp_last_sent_len = (ushort_t)len;
1277
1278 plen = len + connp->conn_ht_iphc_len;
1279
1280 ixa = connp->conn_ixa;
1281 ixa->ixa_pktlen = plen;
1282
1283 if (ixa->ixa_flags & IXAF_IS_IPV4) {
1284 tcp->tcp_ipha->ipha_length = htons(plen);
1285 } else {
1286 tcp->tcp_ip6h->ip6_plen = htons(plen - IPV6_HDR_LEN);
1287 }
1288
1289 /* see if we need to allocate a mblk for the headers */
1290 hdrlen = connp->conn_ht_iphc_len;
1291 rptr = mp1->b_rptr - hdrlen;
1292 db = mp1->b_datap;
1293 if ((db->db_ref != 2) || rptr < db->db_base ||
1294 (!OK_32PTR(rptr))) {
1295 /* NOTE: we assume allocb returns an OK_32PTR */
1296 mp = allocb(hdrlen + tcps->tcps_wroff_xtra, BPRI_MED);
1297 if (!mp) {
1298 freemsg(mp1);
1299 goto no_memory;
1300 }
1301 mp->b_cont = mp1;
1302 mp1 = mp;
1303 /* Leave room for Link Level header */
1304 rptr = &mp1->b_rptr[tcps->tcps_wroff_xtra];
1305 mp1->b_wptr = &rptr[hdrlen];
1306 }
1307 mp1->b_rptr = rptr;
1308
1309 /* Fill in the timestamp option. */
1310 if (tcp->tcp_snd_ts_ok) {
1311 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
1312
1313 U32_TO_BE32(llbolt,
1314 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
1315 U32_TO_BE32(tcp->tcp_ts_recent,
1316 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
1317 } else {
1318 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
1319 }
1320
1321 /* copy header into outgoing packet */
1322 dst = (ipaddr_t *)rptr;
1323 src = (ipaddr_t *)connp->conn_ht_iphc;
1324 dst[0] = src[0];
1325 dst[1] = src[1];
1326 dst[2] = src[2];
1327 dst[3] = src[3];
1328 dst[4] = src[4];
1329 dst[5] = src[5];
1330 dst[6] = src[6];
1331 dst[7] = src[7];
1332 dst[8] = src[8];
1333 dst[9] = src[9];
1334 if (hdrlen -= 40) {
1335 hdrlen >>= 2;
1336 dst += 10;
1337 src += 10;
1338 do {
1339 *dst++ = *src++;
1340 } while (--hdrlen);
1341 }
1342
1343 /*
1344 * Set the ECN info in the TCP header. Note that this
1345 * is not the template header.
1346 */
1347 if (tcp->tcp_ecn_ok) {
1348 TCP_SET_ECT(tcp, rptr);
1349
1350 tcpha = (tcpha_t *)(rptr + ixa->ixa_ip_hdr_length);
1351 if (tcp->tcp_ecn_echo_on)
1352 tcpha->tha_flags |= TH_ECE;
1353 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
1354 tcpha->tha_flags |= TH_CWR;
1355 tcp->tcp_ecn_cwr_sent = B_TRUE;
1356 }
1357 }
1358
1359 if (tcp->tcp_ip_forward_progress) {
1360 tcp->tcp_ip_forward_progress = B_FALSE;
1361 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
1362 } else {
1363 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
1364 }
1365 tcp_send_data(tcp, mp1);
1366 return;
1367
1368 /*
1369 * If we ran out of memory, we pretend to have sent the packet
1370 * and that it was lost on the wire.
1371 */
1372 no_memory:
1373 return;
1374
1375 slow:
1376 /* leftover work from above */
1377 tcp->tcp_unsent = len;
1378 tcp->tcp_xmit_tail_unsent = len;
1379 tcp_wput_data(tcp, NULL, B_FALSE);
1380 }
1381
1382 /* ARGSUSED2 */
1383 void
1384 tcp_output_urgent(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1385 {
1386 int len;
1387 uint32_t msize;
1388 conn_t *connp = (conn_t *)arg;
1389 tcp_t *tcp = connp->conn_tcp;
1390
1391 msize = msgdsize(mp);
1392
1393 len = msize - 1;
1394 if (len < 0) {
1395 freemsg(mp);
1396 return;
1397 }
1398
1399 /*
1400 * Try to force urgent data out on the wire. Even if we have unsent
1401 * data this will at least send the urgent flag.
1402 * XXX does not handle more flag correctly.
1403 */
1404 len += tcp->tcp_unsent;
1405 len += tcp->tcp_snxt;
1406 tcp->tcp_urg = len;
1407 tcp->tcp_valid_bits |= TCP_URG_VALID;
1408
1409 /* Bypass tcp protocol for fused tcp loopback */
1410 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1411 return;
1412
1413 /* Strip off the T_EXDATA_REQ if the data is from TPI */
1414 if (DB_TYPE(mp) != M_DATA) {
1415 mblk_t *mp1 = mp;
1416 ASSERT(!IPCL_IS_NONSTR(connp));
1417 mp = mp->b_cont;
1418 freeb(mp1);
1419 }
1420 tcp_wput_data(tcp, mp, B_TRUE);
1421 }
1422
1423 /*
1424 * Called by streams close routine via squeues when our client blows off its
1425 * descriptor, we take this to mean: "close the stream state NOW, close the tcp
1426 * connection politely" When SO_LINGER is set (with a non-zero linger time and
1427 * it is not a nonblocking socket) then this routine sleeps until the FIN is
1428 * acked.
1429 *
1430 * NOTE: tcp_close potentially returns error when lingering.
1431 * However, the stream head currently does not pass these errors
1432 * to the application. 4.4BSD only returns EINTR and EWOULDBLOCK
1433 * errors to the application (from tsleep()) and not errors
1434 * like ECONNRESET caused by receiving a reset packet.
1435 */
1436
1437 /* ARGSUSED */
1438 void
1439 tcp_close_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1440 {
1441 char *msg;
1442 conn_t *connp = (conn_t *)arg;
1443 tcp_t *tcp = connp->conn_tcp;
1444 clock_t delta = 0;
1445 tcp_stack_t *tcps = tcp->tcp_tcps;
1446
1447 /*
1448 * When a non-STREAMS socket is being closed, it does not always
1449 * stick around waiting for tcp_close_output to run and can therefore
1450 * have dropped a reference already. So adjust the asserts accordingly.
1451 */
1452 ASSERT((connp->conn_fanout != NULL &&
1453 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 3 : 4)) ||
1454 (connp->conn_fanout == NULL &&
1455 connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3)));
1456
1457 mutex_enter(&tcp->tcp_eager_lock);
1458 if (tcp->tcp_conn_req_cnt_q0 != 0 || tcp->tcp_conn_req_cnt_q != 0) {
1459 /*
1460 * Cleanup for listener. For non-STREAM sockets sockfs will
1461 * close all the eagers on 'q', so in that case only deal
1462 * with 'q0'.
1463 */
1464 tcp_eager_cleanup(tcp, IPCL_IS_NONSTR(connp) ? 1 : 0);
1465 tcp->tcp_wait_for_eagers = 1;
1466 }
1467 mutex_exit(&tcp->tcp_eager_lock);
1468
1469 tcp->tcp_lso = B_FALSE;
1470
1471 msg = NULL;
1472 switch (tcp->tcp_state) {
1473 case TCPS_CLOSED:
1474 case TCPS_IDLE:
1475 break;
1476 case TCPS_BOUND:
1477 if (tcp->tcp_listener != NULL) {
1478 ASSERT(IPCL_IS_NONSTR(connp));
1479 /*
1480 * Unlink from the listener and drop the reference
1481 * put on it by the eager. tcp_closei_local will not
1482 * do it because tcp_tconnind_started is TRUE.
1483 */
1484 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1485 tcp_eager_unlink(tcp);
1486 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1487 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1488 }
1489 break;
1490 case TCPS_LISTEN:
1491 break;
1492 case TCPS_SYN_SENT:
1493 msg = "tcp_close, during connect";
1494 break;
1495 case TCPS_SYN_RCVD:
1496 /*
1497 * Close during the connect 3-way handshake
1498 * but here there may or may not be pending data
1499 * already on queue. Process almost same as in
1500 * the ESTABLISHED state.
1501 */
1502 /* FALLTHRU */
1503 default:
1504 if (tcp->tcp_fused)
1505 tcp_unfuse(tcp);
1506
1507 /*
1508 * If SO_LINGER has set a zero linger time, abort the
1509 * connection with a reset.
1510 */
1511 if (connp->conn_linger && connp->conn_lingertime == 0) {
1512 msg = "tcp_close, zero lingertime";
1513 break;
1514 }
1515
1516 /*
1517 * Abort connection if there is unread data queued.
1518 */
1519 if (tcp->tcp_rcv_list || tcp->tcp_reass_head) {
1520 msg = "tcp_close, unread data";
1521 break;
1522 }
1523
1524 /*
1525 * Abort connection if it is being closed without first
1526 * being accepted. This can happen if a listening non-STREAM
1527 * socket wants to get rid of the socket, for example, if the
1528 * listener is closing.
1529 */
1530 if (tcp->tcp_listener != NULL) {
1531 ASSERT(IPCL_IS_NONSTR(connp));
1532 msg = "tcp_close, close before accept";
1533
1534 /*
1535 * Unlink from the listener and drop the reference
1536 * put on it by the eager. tcp_closei_local will not
1537 * do it because tcp_tconnind_started is TRUE.
1538 */
1539 mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1540 tcp_eager_unlink(tcp);
1541 mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1542 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1543 break;
1544 }
1545
1546 /*
1547 * Transmit the FIN before detaching the tcp_t.
1548 * After tcp_detach returns this queue/perimeter
1549 * no longer owns the tcp_t thus others can modify it.
1550 */
1551 (void) tcp_xmit_end(tcp);
1552
1553 /*
1554 * If lingering on close then wait until the fin is acked,
1555 * the SO_LINGER time passes, or a reset is sent/received.
1556 */
1557 if (connp->conn_linger && connp->conn_lingertime > 0 &&
1558 !(tcp->tcp_fin_acked) &&
1559 tcp->tcp_state >= TCPS_ESTABLISHED) {
1560 if (tcp->tcp_closeflags & (FNDELAY|FNONBLOCK)) {
1561 tcp->tcp_client_errno = EWOULDBLOCK;
1562 } else if (tcp->tcp_client_errno == 0) {
1563
1564 ASSERT(tcp->tcp_linger_tid == 0);
1565
1566 /* conn_lingertime is in sec. */
1567 tcp->tcp_linger_tid = TCP_TIMER(tcp,
1568 tcp_close_linger_timeout,
1569 connp->conn_lingertime * MILLISEC);
1570
1571 /* tcp_close_linger_timeout will finish close */
1572 if (tcp->tcp_linger_tid == 0)
1573 tcp->tcp_client_errno = ENOSR;
1574 else
1575 return;
1576 }
1577
1578 /*
1579 * Check if we need to detach or just close
1580 * the instance.
1581 */
1582 if (tcp->tcp_state <= TCPS_LISTEN)
1583 break;
1584 }
1585
1586 /*
1587 * Make sure that no other thread will access the conn_rq of
1588 * this instance (through lookups etc.) as conn_rq will go
1589 * away shortly.
1590 */
1591 tcp_acceptor_hash_remove(tcp);
1592
1593 mutex_enter(&tcp->tcp_non_sq_lock);
1594 if (tcp->tcp_flow_stopped) {
1595 tcp_clrqfull(tcp);
1596 }
1597 mutex_exit(&tcp->tcp_non_sq_lock);
1598
1599 if (tcp->tcp_timer_tid != 0) {
1600 delta = TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid);
1601 tcp->tcp_timer_tid = 0;
1602 }
1603 /*
1604 * Need to cancel those timers which will not be used when
1605 * TCP is detached. This has to be done before the conn_wq
1606 * is set to NULL.
1607 */
1608 tcp_timers_stop(tcp);
1609
1610 tcp->tcp_detached = B_TRUE;
1611 if (tcp->tcp_state == TCPS_TIME_WAIT) {
1612 tcp_time_wait_append(tcp);
1613 TCP_DBGSTAT(tcps, tcp_detach_time_wait);
1614 ASSERT(connp->conn_ref >=
1615 (IPCL_IS_NONSTR(connp) ? 2 : 3));
1616 goto finish;
1617 }
1618
1619 /*
1620 * If delta is zero the timer event wasn't executed and was
1621 * successfully canceled. In this case we need to restart it
1622 * with the minimal delta possible.
1623 */
1624 if (delta >= 0)
1625 tcp->tcp_timer_tid = TCP_TIMER(tcp, tcp_timer,
1626 delta ? delta : 1);
1627
1628 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3));
1629 goto finish;
1630 }
1631
1632 /* Detach did not complete. Still need to remove q from stream. */
1633 if (msg) {
1634 if (tcp->tcp_state == TCPS_ESTABLISHED ||
1635 tcp->tcp_state == TCPS_CLOSE_WAIT)
1636 TCPS_BUMP_MIB(tcps, tcpEstabResets);
1637 if (tcp->tcp_state == TCPS_SYN_SENT ||
1638 tcp->tcp_state == TCPS_SYN_RCVD)
1639 TCPS_BUMP_MIB(tcps, tcpAttemptFails);
1640 tcp_xmit_ctl(msg, tcp, tcp->tcp_snxt, 0, TH_RST);
1641 }
1642
1643 tcp_closei_local(tcp);
1644 CONN_DEC_REF(connp);
1645 ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 1 : 2));
1646
1647 finish:
1648 /*
1649 * Don't change the queues in the case of a listener that has
1650 * eagers in its q or q0. It could surprise the eagers.
1651 * Instead wait for the eagers outside the squeue.
1652 *
1653 * For non-STREAMS sockets tcp_wait_for_eagers implies that
1654 * we should delay the su_closed upcall until all eagers have
1655 * dropped their references.
1656 */
1657 if (!tcp->tcp_wait_for_eagers) {
1658 tcp->tcp_detached = B_TRUE;
1659 connp->conn_rq = NULL;
1660 connp->conn_wq = NULL;
1661
1662 /* non-STREAM socket, release the upper handle */
1663 if (IPCL_IS_NONSTR(connp)) {
1664 ASSERT(connp->conn_upper_handle != NULL);
1665 (*connp->conn_upcalls->su_closed)
1666 (connp->conn_upper_handle);
1667 connp->conn_upper_handle = NULL;
1668 connp->conn_upcalls = NULL;
1669 }
1670 }
1671
1672 /* Signal tcp_close() to finish closing. */
1673 mutex_enter(&tcp->tcp_closelock);
1674 tcp->tcp_closed = 1;
1675 cv_signal(&tcp->tcp_closecv);
1676 mutex_exit(&tcp->tcp_closelock);
1677 }
1678
1679 /* ARGSUSED */
1680 void
1681 tcp_shutdown_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1682 {
1683 conn_t *connp = (conn_t *)arg;
1684 tcp_t *tcp = connp->conn_tcp;
1685
1686 freemsg(mp);
1687
1688 if (tcp->tcp_fused)
1689 tcp_unfuse(tcp);
1690
1691 if (tcp_xmit_end(tcp) != 0) {
1692 /*
1693 * We were crossing FINs and got a reset from
1694 * the other side. Just ignore it.
1695 */
1696 if (connp->conn_debug) {
1697 (void) strlog(TCP_MOD_ID, 0, 1,
1698 SL_ERROR|SL_TRACE,
1699 "tcp_shutdown_output() out of state %s",
1700 tcp_display(tcp, NULL, DISP_ADDR_AND_PORT));
1701 }
1702 }
1703 }
1704
1705 #pragma inline(tcp_send_data)
1706
1707 void
1708 tcp_send_data(tcp_t *tcp, mblk_t *mp)
1709 {
1710 conn_t *connp = tcp->tcp_connp;
1711
1712 /*
1713 * Check here to avoid sending zero-copy message down to IP when
1714 * ZEROCOPY capability has turned off. We only need to deal with
1715 * the race condition between sockfs and the notification here.
1716 * Since we have tried to backoff the tcp_xmit_head when turning
1717 * zero-copy off and new messages in tcp_output(), we simply drop
1718 * the dup'ed packet here and let tcp retransmit, if tcp_xmit_zc_clean
1719 * is not true.
1720 */
1721 if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on &&
1722 !tcp->tcp_xmit_zc_clean) {
1723 ip_drop_output("TCP ZC was disabled but not clean", mp, NULL);
1724 freemsg(mp);
1725 return;
1726 }
1727
1728 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, connp->conn_ixa,
1729 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, tcp,
1730 __dtrace_tcp_tcph_t *,
1731 &mp->b_rptr[connp->conn_ixa->ixa_ip_hdr_length]);
1732
1733 ASSERT(connp->conn_ixa->ixa_notify_cookie == connp->conn_tcp);
1734 (void) conn_ip_output(mp, connp->conn_ixa);
1735 }
1736
1737 /* ARGSUSED2 */
1738 void
1739 tcp_send_synack(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1740 {
1741 conn_t *econnp = (conn_t *)arg;
1742 tcp_t *tcp = econnp->conn_tcp;
1743 ip_xmit_attr_t *ixa = econnp->conn_ixa;
1744
1745 /* Guard against a RST having blown it away while on the squeue */
1746 if (tcp->tcp_state == TCPS_CLOSED) {
1747 freemsg(mp);
1748 return;
1749 }
1750
1751 /*
1752 * In the off-chance that the eager received and responded to
1753 * some other packet while the SYN|ACK was queued, we recalculate
1754 * the ixa_pktlen. It would be better to fix the SYN/accept
1755 * multithreading scheme to avoid this complexity.
1756 */
1757 ixa->ixa_pktlen = msgdsize(mp);
1758 (void) conn_ip_output(mp, ixa);
1759 }
1760
1761 /*
1762 * tcp_send() is called by tcp_wput_data() and returns one of the following:
1763 *
1764 * -1 = failed allocation.
1765 * 0 = We've either successfully sent data, or our usable send window is too
1766 * small and we'd rather wait until later before sending again.
1767 */
1768 static int
1769 tcp_send(tcp_t *tcp, const int mss, const int total_hdr_len,
1770 const int tcp_hdr_len, const int num_sack_blk, int *usable,
1771 uint_t *snxt, int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time)
1772 {
1773 int num_lso_seg = 1;
1774 uint_t lso_usable;
1775 boolean_t do_lso_send = B_FALSE;
1776 tcp_stack_t *tcps = tcp->tcp_tcps;
1777 conn_t *connp = tcp->tcp_connp;
1778 ip_xmit_attr_t *ixa = connp->conn_ixa;
1779
1780 /*
1781 * Check LSO possibility. The value of tcp->tcp_lso indicates whether
1782 * the underlying connection is LSO capable. Will check whether having
1783 * enough available data to initiate LSO transmission in the for(){}
1784 * loops.
1785 */
1786 if (tcp->tcp_lso && (tcp->tcp_valid_bits & ~TCP_FSS_VALID) == 0)
1787 do_lso_send = B_TRUE;
1788
1789 for (;;) {
1790 struct datab *db;
1791 tcpha_t *tcpha;
1792 uint32_t sum;
1793 mblk_t *mp, *mp1;
1794 uchar_t *rptr;
1795 int len;
1796
1797 /*
1798 * Calculate the maximum payload length we can send at one
1799 * time.
1800 */
1801 if (do_lso_send) {
1802 /*
1803 * Determine whether or not it's possible to do LSO,
1804 * and if so, how much data we can send.
1805 */
1806 if ((*usable - 1) / mss >= 1) {
1807 lso_usable = MIN(tcp->tcp_lso_max, *usable);
1808 num_lso_seg = lso_usable / mss;
1809 if (lso_usable % mss) {
1810 num_lso_seg++;
1811 tcp->tcp_last_sent_len = (ushort_t)
1812 (lso_usable % mss);
1813 } else {
1814 tcp->tcp_last_sent_len = (ushort_t)mss;
1815 }
1816 } else {
1817 do_lso_send = B_FALSE;
1818 num_lso_seg = 1;
1819 lso_usable = mss;
1820 }
1821 }
1822
1823 ASSERT(num_lso_seg <= IP_MAXPACKET / mss + 1);
1824
1825 len = mss;
1826 if (len > *usable) {
1827 ASSERT(do_lso_send == B_FALSE);
1828
1829 len = *usable;
1830 if (len <= 0) {
1831 /* Terminate the loop */
1832 break; /* success; too small */
1833 }
1834 /*
1835 * Sender silly-window avoidance.
1836 * Ignore this if we are going to send a
1837 * zero window probe out.
1838 *
1839 * TODO: force data into microscopic window?
1840 * ==> (!pushed || (unsent > usable))
1841 */
1842 if (len < (tcp->tcp_max_swnd >> 1) &&
1843 (tcp->tcp_unsent - (*snxt - tcp->tcp_snxt)) > len &&
1844 !((tcp->tcp_valid_bits & TCP_URG_VALID) &&
1845 len == 1) && (! tcp->tcp_zero_win_probe)) {
1846 /*
1847 * If the retransmit timer is not running
1848 * we start it so that we will retransmit
1849 * in the case when the receiver has
1850 * decremented the window.
1851 */
1852 if (*snxt == tcp->tcp_snxt &&
1853 *snxt == tcp->tcp_suna) {
1854 /*
1855 * We are not supposed to send
1856 * anything. So let's wait a little
1857 * bit longer before breaking SWS
1858 * avoidance.
1859 *
1860 * What should the value be?
1861 * Suggestion: MAX(init rexmit time,
1862 * tcp->tcp_rto)
1863 */
1864 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1865 }
1866 break; /* success; too small */
1867 }
1868 }
1869
1870 tcpha = tcp->tcp_tcpha;
1871
1872 /*
1873 * The reason to adjust len here is that we need to set flags
1874 * and calculate checksum.
1875 */
1876 if (do_lso_send)
1877 len = lso_usable;
1878
1879 *usable -= len; /* Approximate - can be adjusted later */
1880 if (*usable > 0)
1881 tcpha->tha_flags = TH_ACK;
1882 else
1883 tcpha->tha_flags = (TH_ACK | TH_PUSH);
1884
1885 /*
1886 * Prime pump for IP's checksumming on our behalf.
1887 * Include the adjustment for a source route if any.
1888 * In case of LSO, the partial pseudo-header checksum should
1889 * exclusive TCP length, so zero tha_sum before IP calculate
1890 * pseudo-header checksum for partial checksum offload.
1891 */
1892 if (do_lso_send) {
1893 sum = 0;
1894 } else {
1895 sum = len + tcp_hdr_len + connp->conn_sum;
1896 sum = (sum >> 16) + (sum & 0xFFFF);
1897 }
1898 tcpha->tha_sum = htons(sum);
1899 tcpha->tha_seq = htonl(*snxt);
1900
1901 /*
1902 * Branch off to tcp_xmit_mp() if any of the VALID bits is
1903 * set. For the case when TCP_FSS_VALID is the only valid
1904 * bit (normal active close), branch off only when we think
1905 * that the FIN flag needs to be set. Note for this case,
1906 * that (snxt + len) may not reflect the actual seg_len,
1907 * as len may be further reduced in tcp_xmit_mp(). If len
1908 * gets modified, we will end up here again.
1909 */
1910 if (tcp->tcp_valid_bits != 0 &&
1911 (tcp->tcp_valid_bits != TCP_FSS_VALID ||
1912 ((*snxt + len) == tcp->tcp_fss))) {
1913 uchar_t *prev_rptr;
1914 uint32_t prev_snxt = tcp->tcp_snxt;
1915
1916 if (*tail_unsent == 0) {
1917 ASSERT((*xmit_tail)->b_cont != NULL);
1918 *xmit_tail = (*xmit_tail)->b_cont;
1919 prev_rptr = (*xmit_tail)->b_rptr;
1920 *tail_unsent = (int)((*xmit_tail)->b_wptr -
1921 (*xmit_tail)->b_rptr);
1922 } else {
1923 prev_rptr = (*xmit_tail)->b_rptr;
1924 (*xmit_tail)->b_rptr = (*xmit_tail)->b_wptr -
1925 *tail_unsent;
1926 }
1927 mp = tcp_xmit_mp(tcp, *xmit_tail, len, NULL, NULL,
1928 *snxt, B_FALSE, (uint32_t *)&len, B_FALSE);
1929 /* Restore tcp_snxt so we get amount sent right. */
1930 tcp->tcp_snxt = prev_snxt;
1931 if (prev_rptr == (*xmit_tail)->b_rptr) {
1932 /*
1933 * If the previous timestamp is still in use,
1934 * don't stomp on it.
1935 */
1936 if ((*xmit_tail)->b_next == NULL) {
1937 (*xmit_tail)->b_prev = local_time;
1938 (*xmit_tail)->b_next =
1939 (mblk_t *)(uintptr_t)(*snxt);
1940 }
1941 } else
1942 (*xmit_tail)->b_rptr = prev_rptr;
1943
1944 if (mp == NULL) {
1945 return (-1);
1946 }
1947 mp1 = mp->b_cont;
1948
1949 if (len <= mss) /* LSO is unusable (!do_lso_send) */
1950 tcp->tcp_last_sent_len = (ushort_t)len;
1951 while (mp1->b_cont) {
1952 *xmit_tail = (*xmit_tail)->b_cont;
1953 (*xmit_tail)->b_prev = local_time;
1954 (*xmit_tail)->b_next =
1955 (mblk_t *)(uintptr_t)(*snxt);
1956 mp1 = mp1->b_cont;
1957 }
1958 *snxt += len;
1959 *tail_unsent = (*xmit_tail)->b_wptr - mp1->b_wptr;
1960 BUMP_LOCAL(tcp->tcp_obsegs);
1961 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1962 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1963 tcp_send_data(tcp, mp);
1964 continue;
1965 }
1966
1967 *snxt += len; /* Adjust later if we don't send all of len */
1968 TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1969 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1970
1971 if (*tail_unsent) {
1972 /* Are the bytes above us in flight? */
1973 rptr = (*xmit_tail)->b_wptr - *tail_unsent;
1974 if (rptr != (*xmit_tail)->b_rptr) {
1975 *tail_unsent -= len;
1976 if (len <= mss) /* LSO is unusable */
1977 tcp->tcp_last_sent_len = (ushort_t)len;
1978 len += total_hdr_len;
1979 ixa->ixa_pktlen = len;
1980
1981 if (ixa->ixa_flags & IXAF_IS_IPV4) {
1982 tcp->tcp_ipha->ipha_length = htons(len);
1983 } else {
1984 tcp->tcp_ip6h->ip6_plen =
1985 htons(len - IPV6_HDR_LEN);
1986 }
1987
1988 mp = dupb(*xmit_tail);
1989 if (mp == NULL) {
1990 return (-1); /* out_of_mem */
1991 }
1992 mp->b_rptr = rptr;
1993 /*
1994 * If the old timestamp is no longer in use,
1995 * sample a new timestamp now.
1996 */
1997 if ((*xmit_tail)->b_next == NULL) {
1998 (*xmit_tail)->b_prev = local_time;
1999 (*xmit_tail)->b_next =
2000 (mblk_t *)(uintptr_t)(*snxt-len);
2001 }
2002 goto must_alloc;
2003 }
2004 } else {
2005 *xmit_tail = (*xmit_tail)->b_cont;
2006 ASSERT((uintptr_t)((*xmit_tail)->b_wptr -
2007 (*xmit_tail)->b_rptr) <= (uintptr_t)INT_MAX);
2008 *tail_unsent = (int)((*xmit_tail)->b_wptr -
2009 (*xmit_tail)->b_rptr);
2010 }
2011
2012 (*xmit_tail)->b_prev = local_time;
2013 (*xmit_tail)->b_next = (mblk_t *)(uintptr_t)(*snxt - len);
2014
2015 *tail_unsent -= len;
2016 if (len <= mss) /* LSO is unusable (!do_lso_send) */
2017 tcp->tcp_last_sent_len = (ushort_t)len;
2018
2019 len += total_hdr_len;
2020 ixa->ixa_pktlen = len;
2021
2022 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2023 tcp->tcp_ipha->ipha_length = htons(len);
2024 } else {
2025 tcp->tcp_ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2026 }
2027
2028 mp = dupb(*xmit_tail);
2029 if (mp == NULL) {
2030 return (-1); /* out_of_mem */
2031 }
2032
2033 len = total_hdr_len;
2034 /*
2035 * There are four reasons to allocate a new hdr mblk:
2036 * 1) The bytes above us are in use by another packet
2037 * 2) We don't have good alignment
2038 * 3) The mblk is being shared
2039 * 4) We don't have enough room for a header
2040 */
2041 rptr = mp->b_rptr - len;
2042 if (!OK_32PTR(rptr) ||
2043 ((db = mp->b_datap), db->db_ref != 2) ||
2044 rptr < db->db_base) {
2045 /* NOTE: we assume allocb returns an OK_32PTR */
2046
2047 must_alloc:;
2048 mp1 = allocb(connp->conn_ht_iphc_allocated +
2049 tcps->tcps_wroff_xtra, BPRI_MED);
2050 if (mp1 == NULL) {
2051 freemsg(mp);
2052 return (-1); /* out_of_mem */
2053 }
2054 mp1->b_cont = mp;
2055 mp = mp1;
2056 /* Leave room for Link Level header */
2057 len = total_hdr_len;
2058 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2059 mp->b_wptr = &rptr[len];
2060 }
2061
2062 /*
2063 * Fill in the header using the template header, and add
2064 * options such as time-stamp, ECN and/or SACK, as needed.
2065 */
2066 tcp_fill_header(tcp, rptr, (clock_t)local_time, num_sack_blk);
2067
2068 mp->b_rptr = rptr;
2069
2070 if (*tail_unsent) {
2071 int spill = *tail_unsent;
2072
2073 mp1 = mp->b_cont;
2074 if (mp1 == NULL)
2075 mp1 = mp;
2076
2077 /*
2078 * If we're a little short, tack on more mblks until
2079 * there is no more spillover.
2080 */
2081 while (spill < 0) {
2082 mblk_t *nmp;
2083 int nmpsz;
2084
2085 nmp = (*xmit_tail)->b_cont;
2086 nmpsz = MBLKL(nmp);
2087
2088 /*
2089 * Excess data in mblk; can we split it?
2090 * If LSO is enabled for the connection,
2091 * keep on splitting as this is a transient
2092 * send path.
2093 */
2094 if (!do_lso_send && (spill + nmpsz > 0)) {
2095 /*
2096 * Don't split if stream head was
2097 * told to break up larger writes
2098 * into smaller ones.
2099 */
2100 if (tcp->tcp_maxpsz_multiplier > 0)
2101 break;
2102
2103 /*
2104 * Next mblk is less than SMSS/2
2105 * rounded up to nearest 64-byte;
2106 * let it get sent as part of the
2107 * next segment.
2108 */
2109 if (tcp->tcp_localnet &&
2110 !tcp->tcp_cork &&
2111 (nmpsz < roundup((mss >> 1), 64)))
2112 break;
2113 }
2114
2115 *xmit_tail = nmp;
2116 ASSERT((uintptr_t)nmpsz <= (uintptr_t)INT_MAX);
2117 /* Stash for rtt use later */
2118 (*xmit_tail)->b_prev = local_time;
2119 (*xmit_tail)->b_next =
2120 (mblk_t *)(uintptr_t)(*snxt - len);
2121 mp1->b_cont = dupb(*xmit_tail);
2122 mp1 = mp1->b_cont;
2123
2124 spill += nmpsz;
2125 if (mp1 == NULL) {
2126 *tail_unsent = spill;
2127 freemsg(mp);
2128 return (-1); /* out_of_mem */
2129 }
2130 }
2131
2132 /* Trim back any surplus on the last mblk */
2133 if (spill >= 0) {
2134 mp1->b_wptr -= spill;
2135 *tail_unsent = spill;
2136 } else {
2137 /*
2138 * We did not send everything we could in
2139 * order to remain within the b_cont limit.
2140 */
2141 *usable -= spill;
2142 *snxt += spill;
2143 tcp->tcp_last_sent_len += spill;
2144 TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, spill);
2145 /*
2146 * Adjust the checksum
2147 */
2148 tcpha = (tcpha_t *)(rptr +
2149 ixa->ixa_ip_hdr_length);
2150 sum += spill;
2151 sum = (sum >> 16) + (sum & 0xFFFF);
2152 tcpha->tha_sum = htons(sum);
2153 if (connp->conn_ipversion == IPV4_VERSION) {
2154 sum = ntohs(
2155 ((ipha_t *)rptr)->ipha_length) +
2156 spill;
2157 ((ipha_t *)rptr)->ipha_length =
2158 htons(sum);
2159 } else {
2160 sum = ntohs(
2161 ((ip6_t *)rptr)->ip6_plen) +
2162 spill;
2163 ((ip6_t *)rptr)->ip6_plen =
2164 htons(sum);
2165 }
2166 ixa->ixa_pktlen += spill;
2167 *tail_unsent = 0;
2168 }
2169 }
2170 if (tcp->tcp_ip_forward_progress) {
2171 tcp->tcp_ip_forward_progress = B_FALSE;
2172 ixa->ixa_flags |= IXAF_REACH_CONF;
2173 } else {
2174 ixa->ixa_flags &= ~IXAF_REACH_CONF;
2175 }
2176
2177 if (do_lso_send) {
2178 /* Append LSO information to the mp. */
2179 lso_info_set(mp, mss, HW_LSO);
2180 ixa->ixa_fragsize = IP_MAXPACKET;
2181 ixa->ixa_extra_ident = num_lso_seg - 1;
2182
2183 DTRACE_PROBE2(tcp_send_lso, int, num_lso_seg,
2184 boolean_t, B_TRUE);
2185
2186 tcp_send_data(tcp, mp);
2187
2188 /*
2189 * Restore values of ixa_fragsize and ixa_extra_ident.
2190 */
2191 ixa->ixa_fragsize = ixa->ixa_pmtu;
2192 ixa->ixa_extra_ident = 0;
2193 tcp->tcp_obsegs += num_lso_seg;
2194 TCP_STAT(tcps, tcp_lso_times);
2195 TCP_STAT_UPDATE(tcps, tcp_lso_pkt_out, num_lso_seg);
2196 } else {
2197 /*
2198 * Make sure to clean up LSO information. Wherever a
2199 * new mp uses the prepended header room after dupb(),
2200 * lso_info_cleanup() should be called.
2201 */
2202 lso_info_cleanup(mp);
2203 tcp_send_data(tcp, mp);
2204 BUMP_LOCAL(tcp->tcp_obsegs);
2205 }
2206 }
2207
2208 return (0);
2209 }
2210
2211 /*
2212 * Initiate closedown sequence on an active connection. (May be called as
2213 * writer.) Return value zero for OK return, non-zero for error return.
2214 */
2215 static int
2216 tcp_xmit_end(tcp_t *tcp)
2217 {
2218 mblk_t *mp;
2219 tcp_stack_t *tcps = tcp->tcp_tcps;
2220 iulp_t uinfo;
2221 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip;
2222 conn_t *connp = tcp->tcp_connp;
2223
2224 if (tcp->tcp_state < TCPS_SYN_RCVD ||
2225 tcp->tcp_state > TCPS_CLOSE_WAIT) {
2226 /*
2227 * Invalid state, only states TCPS_SYN_RCVD,
2228 * TCPS_ESTABLISHED and TCPS_CLOSE_WAIT are valid
2229 */
2230 return (-1);
2231 }
2232
2233 tcp->tcp_fss = tcp->tcp_snxt + tcp->tcp_unsent;
2234 tcp->tcp_valid_bits |= TCP_FSS_VALID;
2235 /*
2236 * If there is nothing more unsent, send the FIN now.
2237 * Otherwise, it will go out with the last segment.
2238 */
2239 if (tcp->tcp_unsent == 0) {
2240 mp = tcp_xmit_mp(tcp, NULL, 0, NULL, NULL,
2241 tcp->tcp_fss, B_FALSE, NULL, B_FALSE);
2242
2243 if (mp) {
2244 tcp_send_data(tcp, mp);
2245 } else {
2246 /*
2247 * Couldn't allocate msg. Pretend we got it out.
2248 * Wait for rexmit timeout.
2249 */
2250 tcp->tcp_snxt = tcp->tcp_fss + 1;
2251 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2252 }
2253
2254 /*
2255 * If needed, update tcp_rexmit_snxt as tcp_snxt is
2256 * changed.
2257 */
2258 if (tcp->tcp_rexmit && tcp->tcp_rexmit_nxt == tcp->tcp_fss) {
2259 tcp->tcp_rexmit_nxt = tcp->tcp_snxt;
2260 }
2261 } else {
2262 /*
2263 * If tcp->tcp_cork is set, then the data will not get sent,
2264 * so we have to check that and unset it first.
2265 */
2266 if (tcp->tcp_cork)
2267 tcp->tcp_cork = B_FALSE;
2268 tcp_wput_data(tcp, NULL, B_FALSE);
2269 }
2270
2271 /*
2272 * If TCP does not get enough samples of RTT or tcp_rtt_updates
2273 * is 0, don't update the cache.
2274 */
2275 if (tcps->tcps_rtt_updates == 0 ||
2276 tcp->tcp_rtt_update < tcps->tcps_rtt_updates)
2277 return (0);
2278
2279 /*
2280 * We do not have a good algorithm to update ssthresh at this time.
2281 * So don't do any update.
2282 */
2283 bzero(&uinfo, sizeof (uinfo));
2284 uinfo.iulp_rtt = tcp->tcp_rtt_sa;
2285 uinfo.iulp_rtt_sd = tcp->tcp_rtt_sd;
2286
2287 /*
2288 * Note that uinfo is kept for conn_faddr in the DCE. Could update even
2289 * if source routed but we don't.
2290 */
2291 if (connp->conn_ipversion == IPV4_VERSION) {
2292 if (connp->conn_faddr_v4 != tcp->tcp_ipha->ipha_dst) {
2293 return (0);
2294 }
2295 (void) dce_update_uinfo_v4(connp->conn_faddr_v4, &uinfo, ipst);
2296 } else {
2297 uint_t ifindex;
2298
2299 if (!(IN6_ARE_ADDR_EQUAL(&connp->conn_faddr_v6,
2300 &tcp->tcp_ip6h->ip6_dst))) {
2301 return (0);
2302 }
2303 ifindex = 0;
2304 if (IN6_IS_ADDR_LINKSCOPE(&connp->conn_faddr_v6)) {
2305 ip_xmit_attr_t *ixa = connp->conn_ixa;
2306
2307 /*
2308 * If we are going to create a DCE we'd better have
2309 * an ifindex
2310 */
2311 if (ixa->ixa_nce != NULL) {
2312 ifindex = ixa->ixa_nce->nce_common->ncec_ill->
2313 ill_phyint->phyint_ifindex;
2314 } else {
2315 return (0);
2316 }
2317 }
2318
2319 (void) dce_update_uinfo(&connp->conn_faddr_v6, ifindex, &uinfo,
2320 ipst);
2321 }
2322 return (0);
2323 }
2324
2325 /*
2326 * Send out a control packet on the tcp connection specified. This routine
2327 * is typically called where we need a simple ACK or RST generated.
2328 */
2329 void
2330 tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq, uint32_t ack, int ctl)
2331 {
2332 uchar_t *rptr;
2333 tcpha_t *tcpha;
2334 ipha_t *ipha = NULL;
2335 ip6_t *ip6h = NULL;
2336 uint32_t sum;
2337 int total_hdr_len;
2338 int ip_hdr_len;
2339 mblk_t *mp;
2340 tcp_stack_t *tcps = tcp->tcp_tcps;
2341 conn_t *connp = tcp->tcp_connp;
2342 ip_xmit_attr_t *ixa = connp->conn_ixa;
2343
2344 /*
2345 * Save sum for use in source route later.
2346 */
2347 sum = connp->conn_ht_ulp_len + connp->conn_sum;
2348 total_hdr_len = connp->conn_ht_iphc_len;
2349 ip_hdr_len = ixa->ixa_ip_hdr_length;
2350
2351 /* If a text string is passed in with the request, pass it to strlog. */
2352 if (str != NULL && connp->conn_debug) {
2353 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2354 "tcp_xmit_ctl: '%s', seq 0x%x, ack 0x%x, ctl 0x%x",
2355 str, seq, ack, ctl);
2356 }
2357 mp = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
2358 BPRI_MED);
2359 if (mp == NULL) {
2360 return;
2361 }
2362 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2363 mp->b_rptr = rptr;
2364 mp->b_wptr = &rptr[total_hdr_len];
2365 bcopy(connp->conn_ht_iphc, rptr, total_hdr_len);
2366
2367 ixa->ixa_pktlen = total_hdr_len;
2368
2369 if (ixa->ixa_flags & IXAF_IS_IPV4) {
2370 ipha = (ipha_t *)rptr;
2371 ipha->ipha_length = htons(total_hdr_len);
2372 } else {
2373 ip6h = (ip6_t *)rptr;
2374 ip6h->ip6_plen = htons(total_hdr_len - IPV6_HDR_LEN);
2375 }
2376 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2377 tcpha->tha_flags = (uint8_t)ctl;
2378 if (ctl & TH_RST) {
2379 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2380 TCPS_BUMP_MIB(tcps, tcpOutControl);
2381 /*
2382 * Don't send TSopt w/ TH_RST packets per RFC 1323.
2383 */
2384 if (tcp->tcp_snd_ts_ok &&
2385 tcp->tcp_state > TCPS_SYN_SENT) {
2386 mp->b_wptr = &rptr[total_hdr_len - TCPOPT_REAL_TS_LEN];
2387 *(mp->b_wptr) = TCPOPT_EOL;
2388
2389 ixa->ixa_pktlen = total_hdr_len - TCPOPT_REAL_TS_LEN;
2390
2391 if (connp->conn_ipversion == IPV4_VERSION) {
2392 ipha->ipha_length = htons(total_hdr_len -
2393 TCPOPT_REAL_TS_LEN);
2394 } else {
2395 ip6h->ip6_plen = htons(total_hdr_len -
2396 IPV6_HDR_LEN - TCPOPT_REAL_TS_LEN);
2397 }
2398 tcpha->tha_offset_and_reserved -= (3 << 4);
2399 sum -= TCPOPT_REAL_TS_LEN;
2400 }
2401 }
2402 if (ctl & TH_ACK) {
2403 if (tcp->tcp_snd_ts_ok) {
2404 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2405
2406 U32_TO_BE32(llbolt,
2407 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
2408 U32_TO_BE32(tcp->tcp_ts_recent,
2409 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
2410 }
2411
2412 /* Update the latest receive window size in TCP header. */
2413 tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
2414 /* Track what we sent to the peer */
2415 tcp->tcp_tcpha->tha_win = tcpha->tha_win;
2416 tcp->tcp_rack = ack;
2417 tcp->tcp_rack_cnt = 0;
2418 TCPS_BUMP_MIB(tcps, tcpOutAck);
2419 }
2420 BUMP_LOCAL(tcp->tcp_obsegs);
2421 tcpha->tha_seq = htonl(seq);
2422 tcpha->tha_ack = htonl(ack);
2423 /*
2424 * Include the adjustment for a source route if any.
2425 */
2426 sum = (sum >> 16) + (sum & 0xFFFF);
2427 tcpha->tha_sum = htons(sum);
2428 tcp_send_data(tcp, mp);
2429 }
2430
2431 /*
2432 * Generate a reset based on an inbound packet, connp is set by caller
2433 * when RST is in response to an unexpected inbound packet for which
2434 * there is active tcp state in the system.
2435 *
2436 * IPSEC NOTE : Try to send the reply with the same protection as it came
2437 * in. We have the ip_recv_attr_t which is reversed to form the ip_xmit_attr_t.
2438 * That way the packet will go out at the same level of protection as it
2439 * came in with.
2440 */
2441 static void
2442 tcp_xmit_early_reset(char *str, mblk_t *mp, uint32_t seq, uint32_t ack, int ctl,
2443 ip_recv_attr_t *ira, ip_stack_t *ipst, conn_t *connp)
2444 {
2445 ipha_t *ipha = NULL;
2446 ip6_t *ip6h = NULL;
2447 ushort_t len;
2448 tcpha_t *tcpha;
2449 int i;
2450 ipaddr_t v4addr;
2451 in6_addr_t v6addr;
2452 netstack_t *ns = ipst->ips_netstack;
2453 tcp_stack_t *tcps = ns->netstack_tcp;
2454 ip_xmit_attr_t ixas, *ixa;
2455 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2456 boolean_t need_refrele = B_FALSE; /* ixa_refrele(ixa) */
2457 ushort_t port;
2458
2459 if (!tcp_send_rst_chk(tcps)) {
2460 TCP_STAT(tcps, tcp_rst_unsent);
2461 freemsg(mp);
2462 return;
2463 }
2464
2465 /*
2466 * If connp != NULL we use conn_ixa to keep IP_NEXTHOP and other
2467 * options from the listener. In that case the caller must ensure that
2468 * we are running on the listener = connp squeue.
2469 *
2470 * We get a safe copy of conn_ixa so we don't need to restore anything
2471 * we or ip_output_simple might change in the ixa.
2472 */
2473 if (connp != NULL) {
2474 ASSERT(connp->conn_on_sqp);
2475
2476 ixa = conn_get_ixa_exclusive(connp);
2477 if (ixa == NULL) {
2478 TCP_STAT(tcps, tcp_rst_unsent);
2479 freemsg(mp);
2480 return;
2481 }
2482 need_refrele = B_TRUE;
2483 } else {
2484 bzero(&ixas, sizeof (ixas));
2485 ixa = &ixas;
2486 /*
2487 * IXAF_VERIFY_SOURCE is overkill since we know the
2488 * packet was for us.
2489 */
2490 ixa->ixa_flags |= IXAF_SET_ULP_CKSUM | IXAF_VERIFY_SOURCE;
2491 ixa->ixa_protocol = IPPROTO_TCP;
2492 ixa->ixa_zoneid = ira->ira_zoneid;
2493 ixa->ixa_ifindex = 0;
2494 ixa->ixa_ipst = ipst;
2495 ixa->ixa_cred = kcred;
2496 ixa->ixa_cpid = NOPID;
2497 }
2498
2499 if (str && tcps->tcps_dbg) {
2500 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2501 "tcp_xmit_early_reset: '%s', seq 0x%x, ack 0x%x, "
2502 "flags 0x%x",
2503 str, seq, ack, ctl);
2504 }
2505 if (mp->b_datap->db_ref != 1) {
2506 mblk_t *mp1 = copyb(mp);
2507 freemsg(mp);
2508 mp = mp1;
2509 if (mp == NULL)
2510 goto done;
2511 } else if (mp->b_cont) {
2512 freemsg(mp->b_cont);
2513 mp->b_cont = NULL;
2514 DB_CKSUMFLAGS(mp) = 0;
2515 }
2516 /*
2517 * We skip reversing source route here.
2518 * (for now we replace all IP options with EOL)
2519 */
2520 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2521 ipha = (ipha_t *)mp->b_rptr;
2522 for (i = IP_SIMPLE_HDR_LENGTH; i < (int)ip_hdr_len; i++)
2523 mp->b_rptr[i] = IPOPT_EOL;
2524 /*
2525 * Make sure that src address isn't flagrantly invalid.
2526 * Not all broadcast address checking for the src address
2527 * is possible, since we don't know the netmask of the src
2528 * addr. No check for destination address is done, since
2529 * IP will not pass up a packet with a broadcast dest
2530 * address to TCP. Similar checks are done below for IPv6.
2531 */
2532 if (ipha->ipha_src == 0 || ipha->ipha_src == INADDR_BROADCAST ||
2533 CLASSD(ipha->ipha_src)) {
2534 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsInDiscards);
2535 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2536 freemsg(mp);
2537 goto done;
2538 }
2539 } else {
2540 ip6h = (ip6_t *)mp->b_rptr;
2541
2542 if (IN6_IS_ADDR_UNSPECIFIED(&ip6h->ip6_src) ||
2543 IN6_IS_ADDR_MULTICAST(&ip6h->ip6_src)) {
2544 BUMP_MIB(&ipst->ips_ip6_mib, ipIfStatsInDiscards);
2545 ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2546 freemsg(mp);
2547 goto done;
2548 }
2549
2550 /* Remove any extension headers assuming partial overlay */
2551 if (ip_hdr_len > IPV6_HDR_LEN) {
2552 uint8_t *to;
2553
2554 to = mp->b_rptr + ip_hdr_len - IPV6_HDR_LEN;
2555 ovbcopy(ip6h, to, IPV6_HDR_LEN);
2556 mp->b_rptr += ip_hdr_len - IPV6_HDR_LEN;
2557 ip_hdr_len = IPV6_HDR_LEN;
2558 ip6h = (ip6_t *)mp->b_rptr;
2559 ip6h->ip6_nxt = IPPROTO_TCP;
2560 }
2561 }
2562 tcpha = (tcpha_t *)&mp->b_rptr[ip_hdr_len];
2563 if (tcpha->tha_flags & TH_RST) {
2564 freemsg(mp);
2565 goto done;
2566 }
2567 tcpha->tha_offset_and_reserved = (5 << 4);
2568 len = ip_hdr_len + sizeof (tcpha_t);
2569 mp->b_wptr = &mp->b_rptr[len];
2570 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2571 ipha->ipha_length = htons(len);
2572 /* Swap addresses */
2573 v4addr = ipha->ipha_src;
2574 ipha->ipha_src = ipha->ipha_dst;
2575 ipha->ipha_dst = v4addr;
2576 ipha->ipha_ident = 0;
2577 ipha->ipha_ttl = (uchar_t)tcps->tcps_ipv4_ttl;
2578 ixa->ixa_flags |= IXAF_IS_IPV4;
2579 ixa->ixa_ip_hdr_length = ip_hdr_len;
2580 } else {
2581 ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2582 /* Swap addresses */
2583 v6addr = ip6h->ip6_src;
2584 ip6h->ip6_src = ip6h->ip6_dst;
2585 ip6h->ip6_dst = v6addr;
2586 ip6h->ip6_hops = (uchar_t)tcps->tcps_ipv6_hoplimit;
2587 ixa->ixa_flags &= ~IXAF_IS_IPV4;
2588
2589 if (IN6_IS_ADDR_LINKSCOPE(&ip6h->ip6_dst)) {
2590 ixa->ixa_flags |= IXAF_SCOPEID_SET;
2591 ixa->ixa_scopeid = ira->ira_ruifindex;
2592 }
2593 ixa->ixa_ip_hdr_length = IPV6_HDR_LEN;
2594 }
2595 ixa->ixa_pktlen = len;
2596
2597 /* Swap the ports */
2598 port = tcpha->tha_fport;
2599 tcpha->tha_fport = tcpha->tha_lport;
2600 tcpha->tha_lport = port;
2601
2602 tcpha->tha_ack = htonl(ack);
2603 tcpha->tha_seq = htonl(seq);
2604 tcpha->tha_win = 0;
2605 tcpha->tha_sum = htons(sizeof (tcpha_t));
2606 tcpha->tha_flags = (uint8_t)ctl;
2607 if (ctl & TH_RST) {
2608 if (ctl & TH_ACK) {
2609 /*
2610 * Probe connection rejection here.
2611 * tcp_xmit_listeners_reset() drops non-SYN segments
2612 * that do not specify TH_ACK in their flags without
2613 * calling this function. As a consequence, if this
2614 * function is called with a TH_RST|TH_ACK ctl argument,
2615 * it is being called in response to a SYN segment
2616 * and thus the tcp:::accept-refused probe point
2617 * is valid here.
2618 */
2619 DTRACE_TCP5(accept__refused, mblk_t *, NULL,
2620 void, NULL, void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2621 tcph_t *, tcpha);
2622 }
2623 TCPS_BUMP_MIB(tcps, tcpOutRsts);
2624 TCPS_BUMP_MIB(tcps, tcpOutControl);
2625 }
2626
2627 /* Discard any old label */
2628 if (ixa->ixa_free_flags & IXA_FREE_TSL) {
2629 ASSERT(ixa->ixa_tsl != NULL);
2630 label_rele(ixa->ixa_tsl);
2631 ixa->ixa_free_flags &= ~IXA_FREE_TSL;
2632 }
2633 ixa->ixa_tsl = ira->ira_tsl; /* Behave as a multi-level responder */
2634
2635 if (ira->ira_flags & IRAF_IPSEC_SECURE) {
2636 /*
2637 * Apply IPsec based on how IPsec was applied to
2638 * the packet that caused the RST.
2639 */
2640 if (!ipsec_in_to_out(ira, ixa, mp, ipha, ip6h)) {
2641 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsOutDiscards);
2642 /* Note: mp already consumed and ip_drop_packet done */
2643 goto done;
2644 }
2645 } else {
2646 /*
2647 * This is in clear. The RST message we are building
2648 * here should go out in clear, independent of our policy.
2649 */
2650 ixa->ixa_flags |= IXAF_NO_IPSEC;
2651 }
2652
2653 DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, ixa,
2654 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2655 __dtrace_tcp_tcph_t *, tcpha);
2656
2657 /*
2658 * NOTE: one might consider tracing a TCP packet here, but
2659 * this function has no active TCP state and no tcp structure
2660 * that has a trace buffer. If we traced here, we would have
2661 * to keep a local trace buffer in tcp_record_trace().
2662 */
2663
2664 (void) ip_output_simple(mp, ixa);
2665 done:
2666 ixa_cleanup(ixa);
2667 if (need_refrele) {
2668 ASSERT(ixa != &ixas);
2669 ixa_refrele(ixa);
2670 }
2671 }
2672
2673 /*
2674 * Generate a "no listener here" RST in response to an "unknown" segment.
2675 * connp is set by caller when RST is in response to an unexpected
2676 * inbound packet for which there is active tcp state in the system.
2677 * Note that we are reusing the incoming mp to construct the outgoing RST.
2678 */
2679 void
2680 tcp_xmit_listeners_reset(mblk_t *mp, ip_recv_attr_t *ira, ip_stack_t *ipst,
2681 conn_t *connp)
2682 {
2683 uchar_t *rptr;
2684 uint32_t seg_len;
2685 tcpha_t *tcpha;
2686 uint32_t seg_seq;
2687 uint32_t seg_ack;
2688 uint_t flags;
2689 ipha_t *ipha;
2690 ip6_t *ip6h;
2691 boolean_t policy_present;
2692 netstack_t *ns = ipst->ips_netstack;
2693 tcp_stack_t *tcps = ns->netstack_tcp;
2694 ipsec_stack_t *ipss = tcps->tcps_netstack->netstack_ipsec;
2695 uint_t ip_hdr_len = ira->ira_ip_hdr_length;
2696
2697 TCP_STAT(tcps, tcp_no_listener);
2698
2699 /*
2700 * DTrace this "unknown" segment as a tcp:::receive, as we did
2701 * just receive something that was TCP.
2702 */
2703 DTRACE_TCP5(receive, mblk_t *, NULL, ip_xmit_attr_t *, NULL,
2704 __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2705 __dtrace_tcp_tcph_t *, &mp->b_rptr[ip_hdr_len]);
2706
2707 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2708 policy_present = ipss->ipsec_inbound_v4_policy_present;
2709 ipha = (ipha_t *)mp->b_rptr;
2710 ip6h = NULL;
2711 } else {
2712 policy_present = ipss->ipsec_inbound_v6_policy_present;
2713 ipha = NULL;
2714 ip6h = (ip6_t *)mp->b_rptr;
2715 }
2716
2717 if (policy_present) {
2718 /*
2719 * The conn_t parameter is NULL because we already know
2720 * nobody's home.
2721 */
2722 mp = ipsec_check_global_policy(mp, (conn_t *)NULL, ipha, ip6h,
2723 ira, ns);
2724 if (mp == NULL)
2725 return;
2726 }
2727 if (is_system_labeled() && !tsol_can_reply_error(mp, ira)) {
2728 DTRACE_PROBE2(
2729 tx__ip__log__error__nolistener__tcp,
2730 char *, "Could not reply with RST to mp(1)",
2731 mblk_t *, mp);
2732 ip2dbg(("tcp_xmit_listeners_reset: not permitted to reply\n"));
2733 freemsg(mp);
2734 return;
2735 }
2736
2737 rptr = mp->b_rptr;
2738
2739 tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2740 seg_seq = ntohl(tcpha->tha_seq);
2741 seg_ack = ntohl(tcpha->tha_ack);
2742 flags = tcpha->tha_flags;
2743
2744 seg_len = msgdsize(mp) - (TCP_HDR_LENGTH(tcpha) + ip_hdr_len);
2745 if (flags & TH_RST) {
2746 freemsg(mp);
2747 } else if (flags & TH_ACK) {
2748 tcp_xmit_early_reset("no tcp, reset", mp, seg_ack, 0, TH_RST,
2749 ira, ipst, connp);
2750 } else {
2751 if (flags & TH_SYN) {
2752 seg_len++;
2753 } else {
2754 /*
2755 * Here we violate the RFC. Note that a normal
2756 * TCP will never send a segment without the ACK
2757 * flag, except for RST or SYN segment. This
2758 * segment is neither. Just drop it on the
2759 * floor.
2760 */
2761 freemsg(mp);
2762 TCP_STAT(tcps, tcp_rst_unsent);
2763 return;
2764 }
2765
2766 tcp_xmit_early_reset("no tcp, reset/ack", mp, 0,
2767 seg_seq + seg_len, TH_RST | TH_ACK, ira, ipst, connp);
2768 }
2769 }
2770
2771 /*
2772 * Helper function for tcp_xmit_mp() in handling connection set up flag
2773 * options setting.
2774 */
2775 static void
2776 tcp_xmit_mp_aux_iss(tcp_t *tcp, conn_t *connp, tcpha_t *tcpha, mblk_t *mp,
2777 uint_t *flags)
2778 {
2779 uint32_t u1;
2780 uint8_t *wptr = mp->b_wptr;
2781 tcp_stack_t *tcps = tcp->tcp_tcps;
2782 boolean_t add_sack = B_FALSE;
2783
2784 /*
2785 * If TCP_ISS_VALID and the seq number is tcp_iss,
2786 * TCP can only be in SYN-SENT, SYN-RCVD or
2787 * FIN-WAIT-1 state. It can be FIN-WAIT-1 if
2788 * our SYN is not ack'ed but the app closes this
2789 * TCP connection.
2790 */
2791 ASSERT(tcp->tcp_state == TCPS_SYN_SENT ||
2792 tcp->tcp_state == TCPS_SYN_RCVD ||
2793 tcp->tcp_state == TCPS_FIN_WAIT_1);
2794
2795 /*
2796 * Tack on the MSS option. It is always needed
2797 * for both active and passive open.
2798 *
2799 * MSS option value should be interface MTU - MIN
2800 * TCP/IP header according to RFC 793 as it means
2801 * the maximum segment size TCP can receive. But
2802 * to get around some broken middle boxes/end hosts
2803 * out there, we allow the option value to be the
2804 * same as the MSS option size on the peer side.
2805 * In this way, the other side will not send
2806 * anything larger than they can receive.
2807 *
2808 * Note that for SYN_SENT state, the ndd param
2809 * tcp_use_smss_as_mss_opt has no effect as we
2810 * don't know the peer's MSS option value. So
2811 * the only case we need to take care of is in
2812 * SYN_RCVD state, which is done later.
2813 */
2814 wptr[0] = TCPOPT_MAXSEG;
2815 wptr[1] = TCPOPT_MAXSEG_LEN;
2816 wptr += 2;
2817 u1 = tcp->tcp_initial_pmtu - (connp->conn_ipversion == IPV4_VERSION ?
2818 IP_SIMPLE_HDR_LENGTH : IPV6_HDR_LEN) - TCP_MIN_HEADER_LENGTH;
2819 U16_TO_BE16(u1, wptr);
2820 wptr += 2;
2821
2822 /* Update the offset to cover the additional word */
2823 tcpha->tha_offset_and_reserved += (1 << 4);
2824
2825 switch (tcp->tcp_state) {
2826 case TCPS_SYN_SENT:
2827 *flags = TH_SYN;
2828
2829 if (tcp->tcp_snd_sack_ok)
2830 add_sack = B_TRUE;
2831
2832 if (tcp->tcp_snd_ts_ok) {
2833 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2834
2835 if (add_sack) {
2836 wptr[0] = TCPOPT_SACK_PERMITTED;
2837 wptr[1] = TCPOPT_SACK_OK_LEN;
2838 add_sack = B_FALSE;
2839 } else {
2840 wptr[0] = TCPOPT_NOP;
2841 wptr[1] = TCPOPT_NOP;
2842 }
2843 wptr[2] = TCPOPT_TSTAMP;
2844 wptr[3] = TCPOPT_TSTAMP_LEN;
2845 wptr += 4;
2846 U32_TO_BE32(llbolt, wptr);
2847 wptr += 4;
2848 ASSERT(tcp->tcp_ts_recent == 0);
2849 U32_TO_BE32(0L, wptr);
2850 wptr += 4;
2851 tcpha->tha_offset_and_reserved += (3 << 4);
2852 }
2853
2854 /*
2855 * Set up all the bits to tell other side
2856 * we are ECN capable.
2857 */
2858 if (tcp->tcp_ecn_ok)
2859 *flags |= (TH_ECE | TH_CWR);
2860
2861 break;
2862
2863 case TCPS_SYN_RCVD:
2864 *flags |= TH_SYN;
2865
2866 /*
2867 * Reset the MSS option value to be SMSS
2868 * We should probably add back the bytes
2869 * for timestamp option and IPsec. We
2870 * don't do that as this is a workaround
2871 * for broken middle boxes/end hosts, it
2872 * is better for us to be more cautious.
2873 * They may not take these things into
2874 * account in their SMSS calculation. Thus
2875 * the peer's calculated SMSS may be smaller
2876 * than what it can be. This should be OK.
2877 */
2878 if (tcps->tcps_use_smss_as_mss_opt) {
2879 u1 = tcp->tcp_mss;
2880 /*
2881 * Note that wptr points just past the MSS
2882 * option value.
2883 */
2884 U16_TO_BE16(u1, wptr - 2);
2885 }
2886
2887 /*
2888 * tcp_snd_ts_ok can only be set in TCPS_SYN_RCVD
2889 * when the peer also uses timestamps option. And
2890 * the TCP header template must have already been
2891 * updated to include the timestamps option.
2892 */
2893 if (tcp->tcp_snd_sack_ok) {
2894 if (tcp->tcp_snd_ts_ok) {
2895 uint8_t *tmp_wptr;
2896
2897 /*
2898 * Use the NOP in the header just
2899 * before timestamps opton.
2900 */
2901 tmp_wptr = (uint8_t *)tcpha +
2902 TCP_MIN_HEADER_LENGTH;
2903 ASSERT(tmp_wptr[0] == TCPOPT_NOP &&
2904 tmp_wptr[1] == TCPOPT_NOP);
2905 tmp_wptr[0] = TCPOPT_SACK_PERMITTED;
2906 tmp_wptr[1] = TCPOPT_SACK_OK_LEN;
2907 } else {
2908 add_sack = B_TRUE;
2909 }
2910 }
2911
2912
2913 /*
2914 * If the other side is ECN capable, reply
2915 * that we are also ECN capable.
2916 */
2917 if (tcp->tcp_ecn_ok)
2918 *flags |= TH_ECE;
2919 break;
2920
2921 default:
2922 /*
2923 * The above ASSERT() makes sure that this
2924 * must be FIN-WAIT-1 state. Our SYN has
2925 * not been ack'ed so retransmit it.
2926 */
2927 *flags |= TH_SYN;
2928 break;
2929 }
2930
2931 if (add_sack) {
2932 wptr[0] = TCPOPT_NOP;
2933 wptr[1] = TCPOPT_NOP;
2934 wptr[2] = TCPOPT_SACK_PERMITTED;
2935 wptr[3] = TCPOPT_SACK_OK_LEN;
2936 wptr += TCPOPT_REAL_SACK_OK_LEN;
2937 tcpha->tha_offset_and_reserved += (1 << 4);
2938 }
2939
2940 if (tcp->tcp_snd_ws_ok) {
2941 wptr[0] = TCPOPT_NOP;
2942 wptr[1] = TCPOPT_WSCALE;
2943 wptr[2] = TCPOPT_WS_LEN;
2944 wptr[3] = (uchar_t)tcp->tcp_rcv_ws;
2945 wptr += TCPOPT_REAL_WS_LEN;
2946 tcpha->tha_offset_and_reserved += (1 << 4);
2947 }
2948
2949 mp->b_wptr = wptr;
2950 u1 = (int)(mp->b_wptr - mp->b_rptr);
2951 /*
2952 * Get IP set to checksum on our behalf
2953 * Include the adjustment for a source route if any.
2954 */
2955 u1 += connp->conn_sum;
2956 u1 = (u1 >> 16) + (u1 & 0xFFFF);
2957 tcpha->tha_sum = htons(u1);
2958 TCPS_BUMP_MIB(tcps, tcpOutControl);
2959 }
2960
2961 /*
2962 * Helper function for tcp_xmit_mp() in handling connection tear down
2963 * flag setting and state changes.
2964 */
2965 static void
2966 tcp_xmit_mp_aux_fss(tcp_t *tcp, ip_xmit_attr_t *ixa, uint_t *flags)
2967 {
2968 if (!tcp->tcp_fin_acked) {
2969 *flags |= TH_FIN;
2970 TCPS_BUMP_MIB(tcp->tcp_tcps, tcpOutControl);
2971 }
2972 if (!tcp->tcp_fin_sent) {
2973 tcp->tcp_fin_sent = B_TRUE;
2974 switch (tcp->tcp_state) {
2975 case TCPS_SYN_RCVD:
2976 tcp->tcp_state = TCPS_FIN_WAIT_1;
2977 DTRACE_TCP6(state__change, void, NULL,
2978 ip_xmit_attr_t *, ixa, void, NULL,
2979 tcp_t *, tcp, void, NULL,
2980 int32_t, TCPS_SYN_RCVD);
2981 break;
2982 case TCPS_ESTABLISHED:
2983 tcp->tcp_state = TCPS_FIN_WAIT_1;
2984 DTRACE_TCP6(state__change, void, NULL,
2985 ip_xmit_attr_t *, ixa, void, NULL,
2986 tcp_t *, tcp, void, NULL,
2987 int32_t, TCPS_ESTABLISHED);
2988 break;
2989 case TCPS_CLOSE_WAIT:
2990 tcp->tcp_state = TCPS_LAST_ACK;
2991 DTRACE_TCP6(state__change, void, NULL,
2992 ip_xmit_attr_t *, ixa, void, NULL,
2993 tcp_t *, tcp, void, NULL,
2994 int32_t, TCPS_CLOSE_WAIT);
2995 break;
2996 }
2997 if (tcp->tcp_suna == tcp->tcp_snxt)
2998 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2999 tcp->tcp_snxt = tcp->tcp_fss + 1;
3000 }
3001 }
3002
3003 /*
3004 * tcp_xmit_mp is called to return a pointer to an mblk chain complete with
3005 * ip and tcp header ready to pass down to IP. If the mp passed in is
3006 * non-NULL, then up to max_to_send bytes of data will be dup'ed off that
3007 * mblk. (If sendall is not set the dup'ing will stop at an mblk boundary
3008 * otherwise it will dup partial mblks.)
3009 * Otherwise, an appropriate ACK packet will be generated. This
3010 * routine is not usually called to send new data for the first time. It
3011 * is mostly called out of the timer for retransmits, and to generate ACKs.
3012 *
3013 * If offset is not NULL, the returned mblk chain's first mblk's b_rptr will
3014 * be adjusted by *offset. And after dupb(), the offset and the ending mblk
3015 * of the original mblk chain will be returned in *offset and *end_mp.
3016 */
3017 mblk_t *
3018 tcp_xmit_mp(tcp_t *tcp, mblk_t *mp, int32_t max_to_send, int32_t *offset,
3019 mblk_t **end_mp, uint32_t seq, boolean_t sendall, uint32_t *seg_len,
3020 boolean_t rexmit)
3021 {
3022 int data_length;
3023 int32_t off = 0;
3024 uint_t flags;
3025 mblk_t *mp1;
3026 mblk_t *mp2;
3027 uchar_t *rptr;
3028 tcpha_t *tcpha;
3029 int32_t num_sack_blk = 0;
3030 int32_t sack_opt_len = 0;
3031 tcp_stack_t *tcps = tcp->tcp_tcps;
3032 conn_t *connp = tcp->tcp_connp;
3033 ip_xmit_attr_t *ixa = connp->conn_ixa;
3034
3035 /* Allocate for our maximum TCP header + link-level */
3036 mp1 = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
3037 BPRI_MED);
3038 if (mp1 == NULL)
3039 return (NULL);
3040 data_length = 0;
3041
3042 /*
3043 * Note that tcp_mss has been adjusted to take into account the
3044 * timestamp option if applicable. Because SACK options do not
3045 * appear in every TCP segments and they are of variable lengths,
3046 * they cannot be included in tcp_mss. Thus we need to calculate
3047 * the actual segment length when we need to send a segment which
3048 * includes SACK options.
3049 */
3050 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
3051 num_sack_blk = MIN(tcp->tcp_max_sack_blk,
3052 tcp->tcp_num_sack_blk);
3053 sack_opt_len = num_sack_blk * sizeof (sack_blk_t) +
3054 TCPOPT_NOP_LEN * 2 + TCPOPT_HEADER_LEN;
3055 if (max_to_send + sack_opt_len > tcp->tcp_mss)
3056 max_to_send -= sack_opt_len;
3057 }
3058
3059 if (offset != NULL) {
3060 off = *offset;
3061 /* We use offset as an indicator that end_mp is not NULL. */
3062 *end_mp = NULL;
3063 }
3064 for (mp2 = mp1; mp && data_length != max_to_send; mp = mp->b_cont) {
3065 /* This could be faster with cooperation from downstream */
3066 if (mp2 != mp1 && !sendall &&
3067 data_length + (int)(mp->b_wptr - mp->b_rptr) >
3068 max_to_send)
3069 /*
3070 * Don't send the next mblk since the whole mblk
3071 * does not fit.
3072 */
3073 break;
3074 mp2->b_cont = dupb(mp);
3075 mp2 = mp2->b_cont;
3076 if (!mp2) {
3077 freemsg(mp1);
3078 return (NULL);
3079 }
3080 mp2->b_rptr += off;
3081 ASSERT((uintptr_t)(mp2->b_wptr - mp2->b_rptr) <=
3082 (uintptr_t)INT_MAX);
3083
3084 data_length += (int)(mp2->b_wptr - mp2->b_rptr);
3085 if (data_length > max_to_send) {
3086 mp2->b_wptr -= data_length - max_to_send;
3087 data_length = max_to_send;
3088 off = mp2->b_wptr - mp->b_rptr;
3089 break;
3090 } else {
3091 off = 0;
3092 }
3093 }
3094 if (offset != NULL) {
3095 *offset = off;
3096 *end_mp = mp;
3097 }
3098 if (seg_len != NULL) {
3099 *seg_len = data_length;
3100 }
3101
3102 /* Update the latest receive window size in TCP header. */
3103 tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
3104
3105 rptr = mp1->b_rptr + tcps->tcps_wroff_xtra;
3106 mp1->b_rptr = rptr;
3107 mp1->b_wptr = rptr + connp->conn_ht_iphc_len + sack_opt_len;
3108 bcopy(connp->conn_ht_iphc, rptr, connp->conn_ht_iphc_len);
3109 tcpha = (tcpha_t *)&rptr[ixa->ixa_ip_hdr_length];
3110 tcpha->tha_seq = htonl(seq);
3111
3112 /*
3113 * Use tcp_unsent to determine if the PUSH bit should be used assumes
3114 * that this function was called from tcp_wput_data. Thus, when called
3115 * to retransmit data the setting of the PUSH bit may appear some
3116 * what random in that it might get set when it should not. This
3117 * should not pose any performance issues.
3118 */
3119 if (data_length != 0 && (tcp->tcp_unsent == 0 ||
3120 tcp->tcp_unsent == data_length)) {
3121 flags = TH_ACK | TH_PUSH;
3122 } else {
3123 flags = TH_ACK;
3124 }
3125
3126 if (tcp->tcp_ecn_ok) {
3127 if (tcp->tcp_ecn_echo_on)
3128 flags |= TH_ECE;
3129
3130 /*
3131 * Only set ECT bit and ECN_CWR if a segment contains new data.
3132 * There is no TCP flow control for non-data segments, and
3133 * only data segment is transmitted reliably.
3134 */
3135 if (data_length > 0 && !rexmit) {
3136 TCP_SET_ECT(tcp, rptr);
3137 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3138 flags |= TH_CWR;
3139 tcp->tcp_ecn_cwr_sent = B_TRUE;
3140 }
3141 }
3142 }
3143
3144 /* Check if there is any special processing needs to be done. */
3145 if (tcp->tcp_valid_bits) {
3146 uint32_t u1;
3147
3148 /* We don't allow having SYN and FIN in the same segment... */
3149 if ((tcp->tcp_valid_bits & TCP_ISS_VALID) &&
3150 seq == tcp->tcp_iss) {
3151 /* Need to do connection set up processing. */
3152 tcp_xmit_mp_aux_iss(tcp, connp, tcpha, mp1, &flags);
3153 } else if ((tcp->tcp_valid_bits & TCP_FSS_VALID) &&
3154 (seq + data_length) == tcp->tcp_fss) {
3155 /* Need to do connection tear down processing. */
3156 tcp_xmit_mp_aux_fss(tcp, ixa, &flags);
3157 }
3158
3159 /*
3160 * Need to do urgent pointer processing.
3161 *
3162 * Note the trick here. u1 is unsigned. When tcp_urg
3163 * is smaller than seq, u1 will become a very huge value.
3164 * So the comparison will fail. Also note that tcp_urp
3165 * should be positive, see RFC 793 page 17.
3166 */
3167 u1 = tcp->tcp_urg - seq + TCP_OLD_URP_INTERPRETATION;
3168 if ((tcp->tcp_valid_bits & TCP_URG_VALID) && u1 != 0 &&
3169 u1 < (uint32_t)(64 * 1024)) {
3170 flags |= TH_URG;
3171 TCPS_BUMP_MIB(tcps, tcpOutUrg);
3172 tcpha->tha_urp = htons(u1);
3173 }
3174 }
3175 tcpha->tha_flags = (uchar_t)flags;
3176 tcp->tcp_rack = tcp->tcp_rnxt;
3177 tcp->tcp_rack_cnt = 0;
3178
3179 /* Fill in the current value of timestamps option. */
3180 if (tcp->tcp_snd_ts_ok) {
3181 if (tcp->tcp_state != TCPS_SYN_SENT) {
3182 uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
3183
3184 U32_TO_BE32(llbolt,
3185 (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
3186 U32_TO_BE32(tcp->tcp_ts_recent,
3187 (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
3188 }
3189 }
3190
3191 /* Fill in the SACK blocks. */
3192 if (num_sack_blk > 0) {
3193 uchar_t *wptr = (uchar_t *)tcpha + connp->conn_ht_ulp_len;
3194 sack_blk_t *tmp;
3195 int32_t i;
3196
3197 wptr[0] = TCPOPT_NOP;
3198 wptr[1] = TCPOPT_NOP;
3199 wptr[2] = TCPOPT_SACK;
3200 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3201 sizeof (sack_blk_t);
3202 wptr += TCPOPT_REAL_SACK_LEN;
3203
3204 tmp = tcp->tcp_sack_list;
3205 for (i = 0; i < num_sack_blk; i++) {
3206 U32_TO_BE32(tmp[i].begin, wptr);
3207 wptr += sizeof (tcp_seq);
3208 U32_TO_BE32(tmp[i].end, wptr);
3209 wptr += sizeof (tcp_seq);
3210 }
3211 tcpha->tha_offset_and_reserved += ((num_sack_blk * 2 + 1) << 4);
3212 }
3213 ASSERT((uintptr_t)(mp1->b_wptr - rptr) <= (uintptr_t)INT_MAX);
3214 data_length += (int)(mp1->b_wptr - rptr);
3215
3216 ixa->ixa_pktlen = data_length;
3217
3218 if (ixa->ixa_flags & IXAF_IS_IPV4) {
3219 ((ipha_t *)rptr)->ipha_length = htons(data_length);
3220 } else {
3221 ip6_t *ip6 = (ip6_t *)rptr;
3222
3223 ip6->ip6_plen = htons(data_length - IPV6_HDR_LEN);
3224 }
3225
3226 /*
3227 * Prime pump for IP
3228 * Include the adjustment for a source route if any.
3229 */
3230 data_length -= ixa->ixa_ip_hdr_length;
3231 data_length += connp->conn_sum;
3232 data_length = (data_length >> 16) + (data_length & 0xFFFF);
3233 tcpha->tha_sum = htons(data_length);
3234 if (tcp->tcp_ip_forward_progress) {
3235 tcp->tcp_ip_forward_progress = B_FALSE;
3236 connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
3237 } else {
3238 connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
3239 }
3240 return (mp1);
3241 }
3242
3243 /*
3244 * If this routine returns B_TRUE, TCP can generate a RST in response
3245 * to a segment. If it returns B_FALSE, TCP should not respond.
3246 */
3247 static boolean_t
3248 tcp_send_rst_chk(tcp_stack_t *tcps)
3249 {
3250 int64_t now;
3251
3252 /*
3253 * TCP needs to protect itself from generating too many RSTs.
3254 * This can be a DoS attack by sending us random segments
3255 * soliciting RSTs.
3256 *
3257 * What we do here is to have a limit of tcp_rst_sent_rate RSTs
3258 * in each 1 second interval. In this way, TCP still generate
3259 * RSTs in normal cases but when under attack, the impact is
3260 * limited.
3261 */
3262 if (tcps->tcps_rst_sent_rate_enabled != 0) {
3263 now = ddi_get_lbolt64();
3264 if (TICK_TO_MSEC(now - tcps->tcps_last_rst_intrvl) >
3265 1*SECONDS) {
3266 tcps->tcps_last_rst_intrvl = now;
3267 tcps->tcps_rst_cnt = 1;
3268 } else if (++tcps->tcps_rst_cnt > tcps->tcps_rst_sent_rate) {
3269 return (B_FALSE);
3270 }
3271 }
3272 return (B_TRUE);
3273 }
3274
3275 /*
3276 * This function handles all retransmissions if SACK is enabled for this
3277 * connection. First it calculates how many segments can be retransmitted
3278 * based on tcp_pipe. Then it goes thru the notsack list to find eligible
3279 * segments. A segment is eligible if sack_cnt for that segment is greater
3280 * than or equal tcp_dupack_fast_retransmit. After it has retransmitted
3281 * all eligible segments, it checks to see if TCP can send some new segments
3282 * (fast recovery). If it can, set the appropriate flag for tcp_input_data().
3283 *
3284 * Parameters:
3285 * tcp_t *tcp: the tcp structure of the connection.
3286 * uint_t *flags: in return, appropriate value will be set for
3287 * tcp_input_data().
3288 */
3289 void
3290 tcp_sack_rexmit(tcp_t *tcp, uint_t *flags)
3291 {
3292 notsack_blk_t *notsack_blk;
3293 int32_t usable_swnd;
3294 int32_t mss;
3295 uint32_t seg_len;
3296 mblk_t *xmit_mp;
3297 tcp_stack_t *tcps = tcp->tcp_tcps;
3298
3299 ASSERT(tcp->tcp_notsack_list != NULL);
3300 ASSERT(tcp->tcp_rexmit == B_FALSE);
3301
3302 /* Defensive coding in case there is a bug... */
3303 if (tcp->tcp_notsack_list == NULL) {
3304 return;
3305 }
3306 notsack_blk = tcp->tcp_notsack_list;
3307 mss = tcp->tcp_mss;
3308
3309 /*
3310 * Limit the num of outstanding data in the network to be
3311 * tcp_cwnd_ssthresh, which is half of the original congestion wnd.
3312 */
3313 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3314
3315 /* At least retransmit 1 MSS of data. */
3316 if (usable_swnd <= 0) {
3317 usable_swnd = mss;
3318 }
3319
3320 /* Make sure no new RTT samples will be taken. */
3321 tcp->tcp_csuna = tcp->tcp_snxt;
3322
3323 notsack_blk = tcp->tcp_notsack_list;
3324 while (usable_swnd > 0) {
3325 mblk_t *snxt_mp, *tmp_mp;
3326 tcp_seq begin = tcp->tcp_sack_snxt;
3327 tcp_seq end;
3328 int32_t off;
3329
3330 for (; notsack_blk != NULL; notsack_blk = notsack_blk->next) {
3331 if (SEQ_GT(notsack_blk->end, begin) &&
3332 (notsack_blk->sack_cnt >=
3333 tcps->tcps_dupack_fast_retransmit)) {
3334 end = notsack_blk->end;
3335 if (SEQ_LT(begin, notsack_blk->begin)) {
3336 begin = notsack_blk->begin;
3337 }
3338 break;
3339 }
3340 }
3341 /*
3342 * All holes are filled. Manipulate tcp_cwnd to send more
3343 * if we can. Note that after the SACK recovery, tcp_cwnd is
3344 * set to tcp_cwnd_ssthresh.
3345 */
3346 if (notsack_blk == NULL) {
3347 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3348 if (usable_swnd <= 0 || tcp->tcp_unsent == 0) {
3349 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna;
3350 ASSERT(tcp->tcp_cwnd > 0);
3351 return;
3352 } else {
3353 usable_swnd = usable_swnd / mss;
3354 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna +
3355 MAX(usable_swnd * mss, mss);
3356 *flags |= TH_XMIT_NEEDED;
3357 return;
3358 }
3359 }
3360
3361 /*
3362 * Note that we may send more than usable_swnd allows here
3363 * because of round off, but no more than 1 MSS of data.
3364 */
3365 seg_len = end - begin;
3366 if (seg_len > mss)
3367 seg_len = mss;
3368 snxt_mp = tcp_get_seg_mp(tcp, begin, &off);
3369 ASSERT(snxt_mp != NULL);
3370 /* This should not happen. Defensive coding again... */
3371 if (snxt_mp == NULL) {
3372 return;
3373 }
3374
3375 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, seg_len, &off,
3376 &tmp_mp, begin, B_TRUE, &seg_len, B_TRUE);
3377 if (xmit_mp == NULL)
3378 return;
3379
3380 usable_swnd -= seg_len;
3381 tcp->tcp_pipe += seg_len;
3382 tcp->tcp_sack_snxt = begin + seg_len;
3383
3384 tcp_send_data(tcp, xmit_mp);
3385
3386 /*
3387 * Update the send timestamp to avoid false retransmission.
3388 */
3389 snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3390
3391 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3392 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, seg_len);
3393 TCPS_BUMP_MIB(tcps, tcpOutSackRetransSegs);
3394 /*
3395 * Update tcp_rexmit_max to extend this SACK recovery phase.
3396 * This happens when new data sent during fast recovery is
3397 * also lost. If TCP retransmits those new data, it needs
3398 * to extend SACK recover phase to avoid starting another
3399 * fast retransmit/recovery unnecessarily.
3400 */
3401 if (SEQ_GT(tcp->tcp_sack_snxt, tcp->tcp_rexmit_max)) {
3402 tcp->tcp_rexmit_max = tcp->tcp_sack_snxt;
3403 }
3404 }
3405 }
3406
3407 /*
3408 * tcp_ss_rexmit() is called to do slow start retransmission after a timeout
3409 * or ICMP errors.
3410 */
3411 void
3412 tcp_ss_rexmit(tcp_t *tcp)
3413 {
3414 uint32_t snxt;
3415 uint32_t smax;
3416 int32_t win;
3417 int32_t mss;
3418 int32_t off;
3419 mblk_t *snxt_mp;
3420 tcp_stack_t *tcps = tcp->tcp_tcps;
3421
3422 /*
3423 * Note that tcp_rexmit can be set even though TCP has retransmitted
3424 * all unack'ed segments.
3425 */
3426 if (SEQ_LT(tcp->tcp_rexmit_nxt, tcp->tcp_rexmit_max)) {
3427 smax = tcp->tcp_rexmit_max;
3428 snxt = tcp->tcp_rexmit_nxt;
3429 if (SEQ_LT(snxt, tcp->tcp_suna)) {
3430 snxt = tcp->tcp_suna;
3431 }
3432 win = MIN(tcp->tcp_cwnd, tcp->tcp_swnd);
3433 win -= snxt - tcp->tcp_suna;
3434 mss = tcp->tcp_mss;
3435 snxt_mp = tcp_get_seg_mp(tcp, snxt, &off);
3436
3437 while (SEQ_LT(snxt, smax) && (win > 0) && (snxt_mp != NULL)) {
3438 mblk_t *xmit_mp;
3439 mblk_t *old_snxt_mp = snxt_mp;
3440 uint32_t cnt = mss;
3441
3442 if (win < cnt) {
3443 cnt = win;
3444 }
3445 if (SEQ_GT(snxt + cnt, smax)) {
3446 cnt = smax - snxt;
3447 }
3448 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, cnt, &off,
3449 &snxt_mp, snxt, B_TRUE, &cnt, B_TRUE);
3450 if (xmit_mp == NULL)
3451 return;
3452
3453 tcp_send_data(tcp, xmit_mp);
3454
3455 snxt += cnt;
3456 win -= cnt;
3457 /*
3458 * Update the send timestamp to avoid false
3459 * retransmission.
3460 */
3461 old_snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3462 TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3463 TCPS_UPDATE_MIB(tcps, tcpRetransBytes, cnt);
3464
3465 tcp->tcp_rexmit_nxt = snxt;
3466 }
3467 /*
3468 * If we have transmitted all we have at the time
3469 * we started the retranmission, we can leave
3470 * the rest of the job to tcp_wput_data(). But we
3471 * need to check the send window first. If the
3472 * win is not 0, go on with tcp_wput_data().
3473 */
3474 if (SEQ_LT(snxt, smax) || win == 0) {
3475 return;
3476 }
3477 }
3478 /* Only call tcp_wput_data() if there is data to be sent. */
3479 if (tcp->tcp_unsent) {
3480 tcp_wput_data(tcp, NULL, B_FALSE);
3481 }
3482 }
3483
3484 /*
3485 * Do slow start retransmission after ICMP errors of PMTU changes.
3486 */
3487 void
3488 tcp_rexmit_after_error(tcp_t *tcp)
3489 {
3490 /*
3491 * All sent data has been acknowledged or no data left to send, just
3492 * to return.
3493 */
3494 if (!SEQ_LT(tcp->tcp_suna, tcp->tcp_snxt) ||
3495 (tcp->tcp_xmit_head == NULL))
3496 return;
3497
3498 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && (tcp->tcp_unsent == 0))
3499 tcp->tcp_rexmit_max = tcp->tcp_fss;
3500 else
3501 tcp->tcp_rexmit_max = tcp->tcp_snxt;
3502
3503 tcp->tcp_rexmit_nxt = tcp->tcp_suna;
3504 tcp->tcp_rexmit = B_TRUE;
3505 tcp->tcp_dupack_cnt = 0;
3506 tcp_ss_rexmit(tcp);
3507 }
3508
3509 /*
3510 * tcp_get_seg_mp() is called to get the pointer to a segment in the
3511 * send queue which starts at the given sequence number. If the given
3512 * sequence number is equal to last valid sequence number (tcp_snxt), the
3513 * returned mblk is the last valid mblk, and off is set to the length of
3514 * that mblk.
3515 *
3516 * send queue which starts at the given seq. no.
3517 *
3518 * Parameters:
3519 * tcp_t *tcp: the tcp instance pointer.
3520 * uint32_t seq: the starting seq. no of the requested segment.
3521 * int32_t *off: after the execution, *off will be the offset to
3522 * the returned mblk which points to the requested seq no.
3523 * It is the caller's responsibility to send in a non-null off.
3524 *
3525 * Return:
3526 * A mblk_t pointer pointing to the requested segment in send queue.
3527 */
3528 static mblk_t *
3529 tcp_get_seg_mp(tcp_t *tcp, uint32_t seq, int32_t *off)
3530 {
3531 int32_t cnt;
3532 mblk_t *mp;
3533
3534 /* Defensive coding. Make sure we don't send incorrect data. */
3535 if (SEQ_LT(seq, tcp->tcp_suna) || SEQ_GT(seq, tcp->tcp_snxt))
3536 return (NULL);
3537
3538 cnt = seq - tcp->tcp_suna;
3539 mp = tcp->tcp_xmit_head;
3540 while (cnt > 0 && mp != NULL) {
3541 cnt -= mp->b_wptr - mp->b_rptr;
3542 if (cnt <= 0) {
3543 cnt += mp->b_wptr - mp->b_rptr;
3544 break;
3545 }
3546 mp = mp->b_cont;
3547 }
3548 ASSERT(mp != NULL);
3549 *off = cnt;
3550 return (mp);
3551 }
3552
3553 /*
3554 * This routine adjusts next-to-send sequence number variables, in the
3555 * case where the reciever has shrunk it's window.
3556 */
3557 void
3558 tcp_update_xmit_tail(tcp_t *tcp, uint32_t snxt)
3559 {
3560 mblk_t *xmit_tail;
3561 int32_t offset;
3562
3563 tcp->tcp_snxt = snxt;
3564
3565 /* Get the mblk, and the offset in it, as per the shrunk window */
3566 xmit_tail = tcp_get_seg_mp(tcp, snxt, &offset);
3567 ASSERT(xmit_tail != NULL);
3568 tcp->tcp_xmit_tail = xmit_tail;
3569 tcp->tcp_xmit_tail_unsent = xmit_tail->b_wptr -
3570 xmit_tail->b_rptr - offset;
3571 }
3572
3573 /*
3574 * This handles the case when the receiver has shrunk its win. Per RFC 1122
3575 * if the receiver shrinks the window, i.e. moves the right window to the
3576 * left, the we should not send new data, but should retransmit normally the
3577 * old unacked data between suna and suna + swnd. We might has sent data
3578 * that is now outside the new window, pretend that we didn't send it.
3579 */
3580 static void
3581 tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_count)
3582 {
3583 uint32_t snxt = tcp->tcp_snxt;
3584
3585 ASSERT(shrunk_count > 0);
3586
3587 if (!tcp->tcp_is_wnd_shrnk) {
3588 tcp->tcp_snxt_shrunk = snxt;
3589 tcp->tcp_is_wnd_shrnk = B_TRUE;
3590 } else if (SEQ_GT(snxt, tcp->tcp_snxt_shrunk)) {
3591 tcp->tcp_snxt_shrunk = snxt;
3592 }
3593
3594 /* Pretend we didn't send the data outside the window */
3595 snxt -= shrunk_count;
3596
3597 /* Reset all the values per the now shrunk window */
3598 tcp_update_xmit_tail(tcp, snxt);
3599 tcp->tcp_unsent += shrunk_count;
3600
3601 /*
3602 * If the SACK option is set, delete the entire list of
3603 * notsack'ed blocks.
3604 */
3605 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list, tcp);
3606
3607 if (tcp->tcp_suna == tcp->tcp_snxt && tcp->tcp_swnd == 0)
3608 /*
3609 * Make sure the timer is running so that we will probe a zero
3610 * window.
3611 */
3612 TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
3613 }
3614
3615 /*
3616 * tcp_fill_header is called by tcp_send() to fill the outgoing TCP header
3617 * with the template header, as well as other options such as time-stamp,
3618 * ECN and/or SACK.
3619 */
3620 static void
3621 tcp_fill_header(tcp_t *tcp, uchar_t *rptr, clock_t now, int num_sack_blk)
3622 {
3623 tcpha_t *tcp_tmpl, *tcpha;
3624 uint32_t *dst, *src;
3625 int hdrlen;
3626 conn_t *connp = tcp->tcp_connp;
3627
3628 ASSERT(OK_32PTR(rptr));
3629
3630 /* Template header */
3631 tcp_tmpl = tcp->tcp_tcpha;
3632
3633 /* Header of outgoing packet */
3634 tcpha = (tcpha_t *)(rptr + connp->conn_ixa->ixa_ip_hdr_length);
3635
3636 /* dst and src are opaque 32-bit fields, used for copying */
3637 dst = (uint32_t *)rptr;
3638 src = (uint32_t *)connp->conn_ht_iphc;
3639 hdrlen = connp->conn_ht_iphc_len;
3640
3641 /* Fill time-stamp option if needed */
3642 if (tcp->tcp_snd_ts_ok) {
3643 U32_TO_BE32((uint32_t)now,
3644 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 4);
3645 U32_TO_BE32(tcp->tcp_ts_recent,
3646 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 8);
3647 } else {
3648 ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
3649 }
3650
3651 /*
3652 * Copy the template header; is this really more efficient than
3653 * calling bcopy()? For simple IPv4/TCP, it may be the case,
3654 * but perhaps not for other scenarios.
3655 */
3656 dst[0] = src[0];
3657 dst[1] = src[1];
3658 dst[2] = src[2];
3659 dst[3] = src[3];
3660 dst[4] = src[4];
3661 dst[5] = src[5];
3662 dst[6] = src[6];
3663 dst[7] = src[7];
3664 dst[8] = src[8];
3665 dst[9] = src[9];
3666 if (hdrlen -= 40) {
3667 hdrlen >>= 2;
3668 dst += 10;
3669 src += 10;
3670 do {
3671 *dst++ = *src++;
3672 } while (--hdrlen);
3673 }
3674
3675 /*
3676 * Set the ECN info in the TCP header if it is not a zero
3677 * window probe. Zero window probe is only sent in
3678 * tcp_wput_data() and tcp_timer().
3679 */
3680 if (tcp->tcp_ecn_ok && !tcp->tcp_zero_win_probe) {
3681 TCP_SET_ECT(tcp, rptr);
3682
3683 if (tcp->tcp_ecn_echo_on)
3684 tcpha->tha_flags |= TH_ECE;
3685 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3686 tcpha->tha_flags |= TH_CWR;
3687 tcp->tcp_ecn_cwr_sent = B_TRUE;
3688 }
3689 }
3690
3691 /* Fill in SACK options */
3692 if (num_sack_blk > 0) {
3693 uchar_t *wptr = rptr + connp->conn_ht_iphc_len;
3694 sack_blk_t *tmp;
3695 int32_t i;
3696
3697 wptr[0] = TCPOPT_NOP;
3698 wptr[1] = TCPOPT_NOP;
3699 wptr[2] = TCPOPT_SACK;
3700 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3701 sizeof (sack_blk_t);
3702 wptr += TCPOPT_REAL_SACK_LEN;
3703
3704 tmp = tcp->tcp_sack_list;
3705 for (i = 0; i < num_sack_blk; i++) {
3706 U32_TO_BE32(tmp[i].begin, wptr);
3707 wptr += sizeof (tcp_seq);
3708 U32_TO_BE32(tmp[i].end, wptr);
3709 wptr += sizeof (tcp_seq);
3710 }
3711 tcpha->tha_offset_and_reserved +=
3712 ((num_sack_blk * 2 + 1) << 4);
3713 }
3714 }