7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /* Copyright (c) 1990 Mentat Inc. */
26
27 #include <sys/types.h>
28 #include <sys/stream.h>
29 #include <sys/strsun.h>
30 #include <sys/zone.h>
31 #include <sys/ddi.h>
32 #include <sys/sunddi.h>
33 #include <sys/cmn_err.h>
34 #include <sys/debug.h>
35 #include <sys/atomic.h>
36
37 #include <sys/systm.h>
38 #include <sys/param.h>
39 #include <sys/kmem.h>
40 #include <sys/sdt.h>
41 #include <sys/socket.h>
42 #include <sys/mac.h>
43 #include <net/if.h>
44 #include <net/if_arp.h>
45 #include <net/route.h>
46 #include <sys/sockio.h>
80 #include <net/pfkeyv2.h>
81 #include <inet/sadb.h>
82 #include <inet/ipsec_impl.h>
83 #include <inet/ipdrop.h>
84 #include <inet/ip_netinfo.h>
85 #include <sys/squeue_impl.h>
86 #include <sys/squeue.h>
87
88 #include <inet/ipclassifier.h>
89 #include <inet/sctp_ip.h>
90 #include <inet/sctp/sctp_impl.h>
91 #include <inet/udp_impl.h>
92 #include <sys/sunddi.h>
93
94 #include <sys/tsol/label.h>
95 #include <sys/tsol/tnet.h>
96
97 /*
98 * Release a reference on ip_xmit_attr.
99 * The reference is acquired by conn_get_ixa()
100 */
101 #define IXA_REFRELE(ixa) \
102 { \
103 if (atomic_dec_32_nv(&(ixa)->ixa_refcnt) == 0) \
104 ixa_inactive(ixa); \
105 }
106
107 #define IXA_REFHOLD(ixa) \
108 { \
109 ASSERT((ixa)->ixa_refcnt != 0); \
110 atomic_inc_32(&(ixa)->ixa_refcnt); \
111 }
112
113 /*
114 * When we need to handle a transmit side asynchronous operation, then we need
115 * to save sufficient information so that we can call the fragment and postfrag
116 * functions. That information is captured in an mblk containing this structure.
117 *
118 * Since this is currently only used for IPsec, we include information for
119 * the kernel crypto framework.
120 */
121 typedef struct ixamblk_s {
122 boolean_t ixm_inbound; /* B_FALSE */
123 iaflags_t ixm_flags; /* ixa_flags */
124 netstackid_t ixm_stackid; /* Verify it didn't go away */
125 uint_t ixm_ifindex; /* Used to find the nce */
126 in6_addr_t ixm_nceaddr_v6; /* Used to find nce */
127 #define ixm_nceaddr_v4 V4_PART_OF_V6(ixm_nceaddr_v6)
128 uint32_t ixm_fragsize;
129 uint_t ixm_pktlen;
729 if (mp->b_wptr == NULL || mp->b_wptr == (uchar_t *)-1)
730 return (B_FALSE);
731
732 #ifdef DEBUG
733 iramblk_t *irm;
734
735 if (DB_TYPE(mp) != M_BREAK)
736 return (B_FALSE);
737
738 irm = (iramblk_t *)mp->b_rptr;
739 ASSERT(irm->irm_inbound);
740 return (B_TRUE);
741 #else
742 return (DB_TYPE(mp) == M_BREAK);
743 #endif
744 }
745
746 static ip_xmit_attr_t *
747 conn_get_ixa_impl(conn_t *connp, boolean_t replace, int kmflag)
748 {
749 ip_xmit_attr_t *ixa;
750 ip_xmit_attr_t *oldixa;
751
752 mutex_enter(&connp->conn_lock);
753 ixa = connp->conn_ixa;
754
755 /* At least one references for the conn_t */
756 ASSERT(ixa->ixa_refcnt >= 1);
757 if (atomic_inc_32_nv(&ixa->ixa_refcnt) == 2) {
758 /* No other thread using conn_ixa */
759 mutex_exit(&connp->conn_lock);
760 return (ixa);
761 }
762 ixa = kmem_alloc(sizeof (*ixa), kmflag);
763 if (ixa == NULL) {
764 mutex_exit(&connp->conn_lock);
765 ixa_refrele(connp->conn_ixa);
766 return (NULL);
767 }
768 ixa_safe_copy(connp->conn_ixa, ixa);
769
770 /* Make sure we drop conn_lock before any refrele */
771 if (replace) {
772 ixa->ixa_refcnt++; /* No atomic needed - not visible */
773 oldixa = connp->conn_ixa;
774 connp->conn_ixa = ixa;
775 mutex_exit(&connp->conn_lock);
776 IXA_REFRELE(oldixa); /* Undo refcnt from conn_t */
777 } else {
778 oldixa = connp->conn_ixa;
779 mutex_exit(&connp->conn_lock);
780 }
781 IXA_REFRELE(oldixa); /* Undo above atomic_add_32_nv */
782
783 return (ixa);
784 }
785
786 /*
787 * Return an ip_xmit_attr_t to use with a conn_t that ensures that only
788 * the caller can access the ip_xmit_attr_t.
789 *
790 * If nobody else is using conn_ixa we return it.
791 * Otherwise we make a "safe" copy of conn_ixa
792 * and return it. The "safe" copy has the pointers set to NULL
793 * (since the pointers might be changed by another thread using
794 * conn_ixa). The caller needs to check for NULL pointers to see
795 * if ip_set_destination needs to be called to re-establish the pointers.
796 *
797 * If 'replace' is set then we replace conn_ixa with the new ip_xmit_attr_t.
798 * That is used when we connect() the ULP.
830
831 oldixa = connp->conn_ixa;
832 IXA_REFHOLD(ixa);
833 ixa->ixa_conn_id = oldixa->ixa_conn_id;
834 connp->conn_ixa = ixa;
835 return (oldixa);
836 }
837
838 /*
839 * Return a ip_xmit_attr_t to use with a conn_t that is based on but
840 * separate from conn_ixa.
841 *
842 * This "safe" copy has the pointers set to NULL
843 * (since the pointers might be changed by another thread using
844 * conn_ixa). The caller needs to check for NULL pointers to see
845 * if ip_set_destination needs to be called to re-establish the pointers.
846 */
847 ip_xmit_attr_t *
848 conn_get_ixa_exclusive(conn_t *connp)
849 {
850 ip_xmit_attr_t *ixa;
851
852 mutex_enter(&connp->conn_lock);
853 ixa = connp->conn_ixa;
854
855 /* At least one references for the conn_t */
856 ASSERT(ixa->ixa_refcnt >= 1);
857
858 /* Make sure conn_ixa doesn't disappear while we copy it */
859 atomic_inc_32(&ixa->ixa_refcnt);
860
861 ixa = kmem_alloc(sizeof (*ixa), KM_NOSLEEP);
862 if (ixa == NULL) {
863 mutex_exit(&connp->conn_lock);
864 ixa_refrele(connp->conn_ixa);
865 return (NULL);
866 }
867 ixa_safe_copy(connp->conn_ixa, ixa);
868 mutex_exit(&connp->conn_lock);
869 IXA_REFRELE(connp->conn_ixa);
870 return (ixa);
871 }
872
873 void
874 ixa_safe_copy(ip_xmit_attr_t *src, ip_xmit_attr_t *ixa)
875 {
876 bcopy(src, ixa, sizeof (*ixa));
877 ixa->ixa_refcnt = 1;
878 /*
879 * Clear any pointers that have references and might be changed
880 * by ip_set_destination or the ULP
881 */
882 ixa->ixa_ire = NULL;
883 ixa->ixa_nce = NULL;
884 ixa->ixa_dce = NULL;
885 ixa->ixa_ire_generation = IRE_GENERATION_VERIFY;
886 ixa->ixa_dce_generation = DCE_GENERATION_VERIFY;
887 #ifdef DEBUG
888 ixa->ixa_curthread = NULL;
889 #endif
1340 * due to memory reclaim. In the former case we wait for
1341 * memory since we must remove the refcnts on the ill.
1342 */
1343 if (tryhard) {
1344 ixa = conn_get_ixa_tryhard(connp, B_TRUE);
1345 ASSERT(ixa != NULL);
1346 } else {
1347 ixa = conn_get_ixa(connp, B_TRUE);
1348 if (ixa == NULL) {
1349 /*
1350 * Somebody else was using it and kmem_alloc
1351 * failed! Next memory reclaim will try to
1352 * clean up.
1353 */
1354 DTRACE_PROBE1(conn__ixa__cleanup__bail,
1355 conn_t *, connp);
1356 return;
1357 }
1358 }
1359 ixa_cleanup_stale(ixa);
1360 ixa_refrele(ixa);
1361 }
1362 }
1363
1364 /*
1365 * ixa needs to be an exclusive copy so that no one changes the cookie
1366 * or the ixa_nce.
1367 */
1368 boolean_t
1369 ixa_check_drain_insert(conn_t *connp, ip_xmit_attr_t *ixa)
1370 {
1371 uintptr_t cookie = ixa->ixa_cookie;
1372 ill_dld_direct_t *idd;
1373 idl_tx_list_t *idl_txl;
1374 ill_t *ill = ixa->ixa_nce->nce_ill;
1375 boolean_t inserted = B_FALSE;
1376
1377 idd = &(ill)->ill_dld_capab->idc_direct;
1378 idl_txl = &ixa->ixa_ipst->ips_idl_tx_list[IDLHASHINDEX(cookie)];
1379 mutex_enter(&idl_txl->txl_lock);
1380
|
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /* Copyright (c) 1990 Mentat Inc. */
26
27 /*
28 * Copyright 2019 Joyent, Inc.
29 */
30
31 #include <sys/types.h>
32 #include <sys/stream.h>
33 #include <sys/strsun.h>
34 #include <sys/zone.h>
35 #include <sys/ddi.h>
36 #include <sys/sunddi.h>
37 #include <sys/cmn_err.h>
38 #include <sys/debug.h>
39 #include <sys/atomic.h>
40
41 #include <sys/systm.h>
42 #include <sys/param.h>
43 #include <sys/kmem.h>
44 #include <sys/sdt.h>
45 #include <sys/socket.h>
46 #include <sys/mac.h>
47 #include <net/if.h>
48 #include <net/if_arp.h>
49 #include <net/route.h>
50 #include <sys/sockio.h>
84 #include <net/pfkeyv2.h>
85 #include <inet/sadb.h>
86 #include <inet/ipsec_impl.h>
87 #include <inet/ipdrop.h>
88 #include <inet/ip_netinfo.h>
89 #include <sys/squeue_impl.h>
90 #include <sys/squeue.h>
91
92 #include <inet/ipclassifier.h>
93 #include <inet/sctp_ip.h>
94 #include <inet/sctp/sctp_impl.h>
95 #include <inet/udp_impl.h>
96 #include <sys/sunddi.h>
97
98 #include <sys/tsol/label.h>
99 #include <sys/tsol/tnet.h>
100
101 /*
102 * Release a reference on ip_xmit_attr.
103 * The reference is acquired by conn_get_ixa()
104 *
105 * This macro has a lowercase function-call version for callers outside
106 * this file.
107 */
108 #define IXA_REFRELE(ixa) \
109 { \
110 if (atomic_dec_32_nv(&(ixa)->ixa_refcnt) == 0) \
111 ixa_inactive(ixa); \
112 }
113
114 #define IXA_REFHOLD(ixa) \
115 { \
116 ASSERT3U((ixa)->ixa_refcnt, !=, 0); \
117 atomic_inc_32(&(ixa)->ixa_refcnt); \
118 }
119
120 /*
121 * When we need to handle a transmit side asynchronous operation, then we need
122 * to save sufficient information so that we can call the fragment and postfrag
123 * functions. That information is captured in an mblk containing this structure.
124 *
125 * Since this is currently only used for IPsec, we include information for
126 * the kernel crypto framework.
127 */
128 typedef struct ixamblk_s {
129 boolean_t ixm_inbound; /* B_FALSE */
130 iaflags_t ixm_flags; /* ixa_flags */
131 netstackid_t ixm_stackid; /* Verify it didn't go away */
132 uint_t ixm_ifindex; /* Used to find the nce */
133 in6_addr_t ixm_nceaddr_v6; /* Used to find nce */
134 #define ixm_nceaddr_v4 V4_PART_OF_V6(ixm_nceaddr_v6)
135 uint32_t ixm_fragsize;
136 uint_t ixm_pktlen;
736 if (mp->b_wptr == NULL || mp->b_wptr == (uchar_t *)-1)
737 return (B_FALSE);
738
739 #ifdef DEBUG
740 iramblk_t *irm;
741
742 if (DB_TYPE(mp) != M_BREAK)
743 return (B_FALSE);
744
745 irm = (iramblk_t *)mp->b_rptr;
746 ASSERT(irm->irm_inbound);
747 return (B_TRUE);
748 #else
749 return (DB_TYPE(mp) == M_BREAK);
750 #endif
751 }
752
753 static ip_xmit_attr_t *
754 conn_get_ixa_impl(conn_t *connp, boolean_t replace, int kmflag)
755 {
756 ip_xmit_attr_t *oldixa; /* Already attached to conn_t */
757 ip_xmit_attr_t *ixa; /* New one, which we return. */
758
759 /*
760 * NOTE: If the marked-below common case isn't, move the
761 * kmem_alloc() up here and put a free in what was marked as the
762 * (not really) common case instead.
763 */
764
765 mutex_enter(&connp->conn_lock);
766 oldixa = connp->conn_ixa;
767
768 /* At least one reference for the conn_t */
769 ASSERT3U(oldixa->ixa_refcnt, >=, 1);
770 if (atomic_inc_32_nv(&oldixa->ixa_refcnt) == 2) {
771 /* No other thread using conn_ixa (common case) */
772 mutex_exit(&connp->conn_lock);
773 return (oldixa);
774 }
775 /* Do allocation inside-the-conn_lock because it's less common. */
776 ixa = kmem_alloc(sizeof (*ixa), kmflag);
777 if (ixa == NULL) {
778 mutex_exit(&connp->conn_lock);
779 IXA_REFRELE(oldixa);
780 return (NULL);
781 }
782 ixa_safe_copy(oldixa, ixa);
783
784 /* Make sure we drop conn_lock before any refrele */
785 if (replace) {
786 ixa->ixa_refcnt++; /* No atomic needed - not visible */
787 connp->conn_ixa = ixa;
788 mutex_exit(&connp->conn_lock);
789 IXA_REFRELE(oldixa); /* Undo refcnt from conn_t */
790 } else {
791 mutex_exit(&connp->conn_lock);
792 }
793 IXA_REFRELE(oldixa); /* Undo above atomic_add_32_nv */
794
795 return (ixa);
796 }
797
798 /*
799 * Return an ip_xmit_attr_t to use with a conn_t that ensures that only
800 * the caller can access the ip_xmit_attr_t.
801 *
802 * If nobody else is using conn_ixa we return it.
803 * Otherwise we make a "safe" copy of conn_ixa
804 * and return it. The "safe" copy has the pointers set to NULL
805 * (since the pointers might be changed by another thread using
806 * conn_ixa). The caller needs to check for NULL pointers to see
807 * if ip_set_destination needs to be called to re-establish the pointers.
808 *
809 * If 'replace' is set then we replace conn_ixa with the new ip_xmit_attr_t.
810 * That is used when we connect() the ULP.
842
843 oldixa = connp->conn_ixa;
844 IXA_REFHOLD(ixa);
845 ixa->ixa_conn_id = oldixa->ixa_conn_id;
846 connp->conn_ixa = ixa;
847 return (oldixa);
848 }
849
850 /*
851 * Return a ip_xmit_attr_t to use with a conn_t that is based on but
852 * separate from conn_ixa.
853 *
854 * This "safe" copy has the pointers set to NULL
855 * (since the pointers might be changed by another thread using
856 * conn_ixa). The caller needs to check for NULL pointers to see
857 * if ip_set_destination needs to be called to re-establish the pointers.
858 */
859 ip_xmit_attr_t *
860 conn_get_ixa_exclusive(conn_t *connp)
861 {
862 ip_xmit_attr_t *oldixa;
863 ip_xmit_attr_t *ixa;
864
865 ixa = kmem_alloc(sizeof (*ixa), KM_NOSLEEP | KM_NORMALPRI);
866 if (ixa == NULL)
867 return (NULL);
868
869 mutex_enter(&connp->conn_lock);
870
871 oldixa = connp->conn_ixa;
872 IXA_REFHOLD(oldixa);
873
874 ixa_safe_copy(oldixa, ixa);
875 mutex_exit(&connp->conn_lock);
876 IXA_REFRELE(oldixa);
877 return (ixa);
878 }
879
880 void
881 ixa_safe_copy(ip_xmit_attr_t *src, ip_xmit_attr_t *ixa)
882 {
883 bcopy(src, ixa, sizeof (*ixa));
884 ixa->ixa_refcnt = 1;
885 /*
886 * Clear any pointers that have references and might be changed
887 * by ip_set_destination or the ULP
888 */
889 ixa->ixa_ire = NULL;
890 ixa->ixa_nce = NULL;
891 ixa->ixa_dce = NULL;
892 ixa->ixa_ire_generation = IRE_GENERATION_VERIFY;
893 ixa->ixa_dce_generation = DCE_GENERATION_VERIFY;
894 #ifdef DEBUG
895 ixa->ixa_curthread = NULL;
896 #endif
1347 * due to memory reclaim. In the former case we wait for
1348 * memory since we must remove the refcnts on the ill.
1349 */
1350 if (tryhard) {
1351 ixa = conn_get_ixa_tryhard(connp, B_TRUE);
1352 ASSERT(ixa != NULL);
1353 } else {
1354 ixa = conn_get_ixa(connp, B_TRUE);
1355 if (ixa == NULL) {
1356 /*
1357 * Somebody else was using it and kmem_alloc
1358 * failed! Next memory reclaim will try to
1359 * clean up.
1360 */
1361 DTRACE_PROBE1(conn__ixa__cleanup__bail,
1362 conn_t *, connp);
1363 return;
1364 }
1365 }
1366 ixa_cleanup_stale(ixa);
1367 IXA_REFRELE(ixa);
1368 }
1369 }
1370
1371 /*
1372 * ixa needs to be an exclusive copy so that no one changes the cookie
1373 * or the ixa_nce.
1374 */
1375 boolean_t
1376 ixa_check_drain_insert(conn_t *connp, ip_xmit_attr_t *ixa)
1377 {
1378 uintptr_t cookie = ixa->ixa_cookie;
1379 ill_dld_direct_t *idd;
1380 idl_tx_list_t *idl_txl;
1381 ill_t *ill = ixa->ixa_nce->nce_ill;
1382 boolean_t inserted = B_FALSE;
1383
1384 idd = &(ill)->ill_dld_capab->idc_direct;
1385 idl_txl = &ixa->ixa_ipst->ips_idl_tx_list[IDLHASHINDEX(cookie)];
1386 mutex_enter(&idl_txl->txl_lock);
1387
|