--- old/usr/src/uts/common/inet/vxlnat/vxlnat_nat.c	Thu Oct 11 14:05:19 2018
+++ new/usr/src/uts/common/inet/vxlnat/vxlnat_nat.c	Thu Oct 11 14:05:19 2018
@@ -349,6 +349,102 @@
 }
 
 /*
+ * See if the inbound VXLAN packet hits a 1-1/fixed mapping, and process if it
+ * does.  B_TRUE means the packet was handled, and we shouldn't continue
+ * processing (even if "was handled" means droppage).
+ */
+static boolean_t
+vxlnat_one_vxlan_fixed(vxlnat_vnet_t *vnet, mblk_t *mp, ipha_t *ipha,
+    ip6_t *ip6h)
+{
+	vxlnat_fixed_t *fixed, fsearch;
+	mblk_t *newmp;
+	ire_t *outbound_ire;
+	/* Use C99's initializers for fun & profit. */
+	ip_recv_attr_t iras = { IRAF_IS_IPV4 | IRAF_VERIFIED_SRC };
+
+	if (ipha != NULL) {
+		IN6_INADDR_TO_V4MAPPED((struct in_addr *)(&ipha->ipha_src),
+		    &fsearch.vxnf_addr);
+	} else {
+		/* vxlnat_cache_remote() did reality checks... */
+		ASSERT(ipha == NULL && ip6h != NULL);
+		fsearch.vxnf_addr = ip6h->ip6_src;
+	}
+
+	rw_enter(&vnet->vxnv_fixed_lock, RW_READER);
+	fixed = avl_find(&vnet->vxnv_fixed_ips, &fsearch, NULL);
+	if (fixed != NULL)
+		VXNF_REFHOLD(fixed);
+	rw_exit(&vnet->vxnv_fixed_lock);
+	if (fixed == NULL)
+		return (B_FALSE);	/* Try another method of processing. */
+
+	newmp = NULL;
+	/*
+	 * XXX KEBE ASKS --> Do an MTU check NOW?!  That way, we have
+	 * pre-natted data.  One gotcha, external dests may have
+	 * different PathMTUs so see below about EMSGSIZE...
+	 *
+	 * For now, let the post-NAT crunch through
+	 * ire_recv_forward_v4() take care of all of that.
+	 */
+
+	if (ipha != NULL)
+		newmp = vxlnat_fixed_fixv4(mp, fixed, B_FALSE);
+	else {
+		freemsg(mp); /* XXX handle ip6h */
+		return (B_TRUE);
+	}
+
+	if (newmp == NULL)
+		return (B_TRUE);	/* mp eaten by vxlnat_fixed_fixv4() */
+
+
+	ASSERT3P(ipha, ==, newmp->b_rptr);
+	/* XXX KEBE ASKS, IRR_ALLOCATE okay?!? */
+	/* XXX KEBE SAYS XMIT HINT! */
+	outbound_ire = ire_route_recursive_dstonly_v4(ipha->ipha_dst,
+	    IRR_ALLOCATE, 0, vxlnat_netstack->netstack_ip);
+	VERIFY3P(outbound_ire, !=, NULL);
+	if (outbound_ire->ire_type == IRE_NOROUTE) {
+		/* Bail! */
+		DTRACE_PROBE2(vxlnat__in__drop__fixedire, ipaddr_t,
+		    ipha->ipha_dst, mblk_t *, mp);
+		VXNF_REFRELE(fixed);
+		freemsg(mp);
+		return (B_TRUE);
+	}
+
+	iras.ira_ip_hdr_length = IPH_HDR_LENGTH(ipha);
+	if (iras.ira_ip_hdr_length > sizeof (ipha_t))
+		iras.ira_flags |= IRAF_IPV4_OPTIONS;
+	iras.ira_xmit_hint = 0; /* XXX KEBE SAYS FIX ME! */
+	iras.ira_zoneid = outbound_ire->ire_zoneid;
+	iras.ira_pktlen = ntohs(ipha->ipha_length);
+	iras.ira_protocol = ipha->ipha_protocol;
+	/* XXX KEBE ASKS rifindex & ruifindex ?!? */
+	/*
+	 * NOTE: AT LEAST ira_ill needs ILLF_ROUTER set, as
+	 * well as the ill for the external NIC (where
+	 * off-link destinations live).  For fixed, ira_ill
+	 * should be the ill of the external source.
+	 */
+	iras.ira_rill = vxlnat_underlay_ire->ire_ill;
+	iras.ira_ill = fixed->vxnf_ire->ire_ill;
+	/* XXX KEBE ASKS cred & cpid ? */
+	iras.ira_verified_src = ipha->ipha_src;
+	/* XXX KEBE SAYS don't sweat IPsec stuff. */
+	/* XXX KEBE SAYS ALSO don't sweat l2src & mhip */
+
+	/* Okay, we're good! Let's pretend we're forwarding. */
+	ire_recv_forward_v4(outbound_ire, mp, ipha, &iras);
+	ire_refrele(outbound_ire);
+
+	return (B_TRUE);
+}
+
+/*
  * Process exactly one VXLAN packet.
  */
 static void
@@ -358,7 +454,6 @@
 	vxlnat_vnet_t *vnet;
 	ipha_t *ipha;
 	ip6_t *ip6h;
-	vxlnat_fixed_t *fixed, fsearch;
 
 	if (MBLKL(mp) < sizeof (*vxh)) {
 		/* XXX KEBE ASKS -- should we be more forgiving? */
@@ -406,103 +501,48 @@
 		freeb(oldmp);
 	}
 	mp = vxlnat_cache_remote(mp, underlay_src, vnet);
-	if (mp == NULL) {
-		VXNV_REFRELE(vnet);
-		return;
-	}
+	if (mp == NULL)
+		goto bail_no_free;
 
-	/* 2.) Search 1-1s, process if hit. */
+	/* Let's cache the IP header here... */
 	ipha = (ipha_t *)mp->b_rptr;
-	if (IPH_HDR_VERSION(ipha) == IPV4_VERSION) {
+	switch (IPH_HDR_VERSION(ipha)) {
+	case IPV4_VERSION:
 		ip6h = NULL;
-		IN6_INADDR_TO_V4MAPPED((struct in_addr *)(&ipha->ipha_src),
-		    &fsearch.vxnf_addr);
-	} else {
-		/* vxlnat_cache_remote() did reality checks... */
-		ASSERT(IPH_HDR_VERSION(ipha) == IPV6_VERSION);
+		break;
+	case IPV6_VERSION:
 		ip6h = (ip6_t *)ipha;
 		ipha = NULL;
-		fsearch.vxnf_addr = ip6h->ip6_src;
+		break;
+	default:
+		DTRACE_PROBE2(vxlnat__in__drop__ipvers, int,
+		    IPH_HDR_VERSION(ipha), mblk_t *, mp);
+		goto bail_and_free;
 	}
-	rw_enter(&vnet->vxnv_fixed_lock, RW_READER);
-	fixed = avl_find(&vnet->vxnv_fixed_ips, &fsearch, NULL);
-	if (fixed != NULL)
-		VXNF_REFHOLD(fixed);
-	rw_exit(&vnet->vxnv_fixed_lock);
-	if (fixed != NULL) {
-		mblk_t *newmp = NULL;
 
-		/*
-		 * XXX KEBE ASKS --> Do MTU check NOW?!  That way, we have
-		 * pre-natted data.  One gotcha, external dests may have
-		 * different PathMTUs so see below about EMSGSIZE...
-		 */
+	/* 2.) Search 1-1s, process if hit. */
+	if (vxlnat_one_vxlan_fixed(vnet, mp, ipha, ip6h))
+		goto bail_no_free;	/* Success means mp was consumed. */
 
-		/* XXX KEBE SAYS -- FILL ME IN... but for now: */
-		if (ipha != NULL)
-			newmp = vxlnat_fixed_fixv4(mp, fixed, B_FALSE);
-		else
-			freemsg(mp); /* XXX handle ip6h */
+#ifdef notyet
+	/* 3.) Search flows, process if hit. */
+	if (vxlnat_one_vxlan_flow(vnet, mp, ipha, ip6h))
+		goto bail_no_free;	/* Success means mp was consumed. */
 
-		if (newmp != NULL) {
-			ire_t *outbound_ire;
-			/* Use C99's initializers for fun & profit. */
-			ip_recv_attr_t iras =
-			    { IRAF_IS_IPV4 | IRAF_VERIFIED_SRC };
+	/* 4.) Search rules, create new flow (or not) if hit. */
+	if (vxlnat_one_vxlan_rule(vnet, mp, ipha, ip6h))
+		goto bail_no_free;	/* Success means mp was consumed. */
+#endif
 
-			ASSERT3P(ipha, !=, NULL);
-			ASSERT3P(ipha, ==, newmp->b_rptr);
-			/* XXX KEBE ASKS, IRR_ALLOCATE okay?!? */
-			outbound_ire = ire_route_recursive_dstonly_v4(
-			    ipha->ipha_dst, IRR_ALLOCATE,
-			    0 /* XXX KEBE SAYS XMIT HINT! */,
-			    vxlnat_netstack->netstack_ip);
-			VERIFY3P(outbound_ire, !=, NULL);
-			if (outbound_ire->ire_type == IRE_NOROUTE) {
-				/* Bail! */
-				VXNF_REFRELE(fixed);
-				VXNV_REFRELE(vnet);
-				return;
-			}
+	/* 5.) Nothing, drop the packet. */
 
-			iras.ira_ip_hdr_length = IPH_HDR_LENGTH(ipha);
-			if (iras.ira_ip_hdr_length > sizeof (ipha_t))
-				iras.ira_flags |= IRAF_IPV4_OPTIONS;
-			iras.ira_xmit_hint = 0; /* XXX KEBE SAYS FIX ME! */
-			iras.ira_zoneid = outbound_ire->ire_zoneid;
-			iras.ira_pktlen = ntohs(ipha->ipha_length);
-			iras.ira_protocol = ipha->ipha_protocol;
-			/* XXX KEBE ASKS rifindex & ruifindex ?!? */
-			/*
-			 * NOTE: AT LEAST ira_ill needs ILLF_ROUTER set, as
-			 * well as the ill for the external NIC (where
-			 * off-link destinations live).  For fixed, ira_ill
-			 * should be the ill of the external source.
-			 */
-			iras.ira_rill = vxlnat_underlay_ire->ire_ill;
-			iras.ira_ill = fixed->vxnf_ire->ire_ill;
-			/* XXX KEBE ASKS cred & cpid ? */
-			iras.ira_verified_src = ipha->ipha_src;
-			/* XXX KEBE SAYS don't sweat IPsec stuff. */
-			/* XXX KEBE SAYS ALSO don't sweat l2src & mhip */
+	DTRACE_PROBE2(vxlnat__in___drop__nohits, vxlnat_vnet_t *, vnet,
+	    mblk_t *, mp);
 
-			/* Okay, we're good! Let's pretend we're forwarding. */
-			ire_recv_forward_v4(outbound_ire, mp, ipha, &iras);
-			ire_refrele(outbound_ire);
-		}
-
-		/* All done... */
-		VXNF_REFRELE(fixed);
-		VXNV_REFRELE(vnet);
-		return;
-	}
-
-	/* XXX KEBE SAYS BUILD STEPS 3-4. */
-
-	/* 5.) Nothing, drop the packet. */
-	/* XXX KEBE ASKS DIAGNOSTIC? */
-	VXNV_REFRELE(vnet);
+bail_and_free:
 	freemsg(mp);
+bail_no_free:
+	VXNV_REFRELE(vnet);
 }
 /*
  * ONLY return B_FALSE if we get a packet-clogging event.
@@ -638,6 +678,7 @@
 /*
  * Take a 1-1/fixed IPv4 packet and convert it for transmission out the
  * appropriate end. "to_private" is what it says on the tin.
+ * ALWAYS consumes "mp", regardless of return value.
  */
 static mblk_t *
 vxlnat_fixed_fixv4(mblk_t *mp, vxlnat_fixed_t *fixed, boolean_t to_private)
--- old/usr/src/uts/common/inet/vxlnat/vxlnat_rules.c	Thu Oct 11 14:05:19 2018
+++ new/usr/src/uts/common/inet/vxlnat/vxlnat_rules.c	Thu Oct 11 14:05:19 2018
@@ -96,6 +96,37 @@
 }
 
 /*
+ * Comparison function for NAT flow.
+ */
+static int
+vxlnat_flow_cmp_v4(const void *first, const void *second)
+{
+	vxlnat_flow_t *first_flow = (vxlnat_flow_t *)first;
+	vxlnat_flow_t *second_flow = (vxlnat_flow_t *)second;
+	uint64_t firstaddrs, secondaddrs, firstportproto, secondportproto;
+
+	firstaddrs = first_flow->vxnfl_src._S6_un._S6_u32[3] |
+	    (((uint64_t)first_flow->vxnfl_dst._S6_un._S6_u32[3]) << 32ULL);
+	secondaddrs = second_flow->vxnfl_src._S6_un._S6_u32[3] |
+	    (((uint64_t)second_flow->vxnfl_dst._S6_un._S6_u32[3]) << 32ULL);
+	firstportproto = first_flow->vxnfl_ports |
+	    (((uint64_t)first_flow->vxnfl_protocol) << 32ULL);
+	secondportproto = second_flow->vxnfl_ports |
+	    (((uint64_t)second_flow->vxnfl_protocol) << 32ULL);
+
+	if (firstaddrs > secondaddrs)
+		return (1);
+	else if (firstaddrs < secondaddrs)
+		return (-1);
+	else if (firstportproto > secondportproto)
+		return (1);
+	else if (firstportproto < secondportproto)
+		return (-1);
+
+	return (0);
+}
+
+/*
  * Find-and-reference-hold a vnet.  If none present, create one.
  * "vnetid" MUST be in wire-order and its one byte cleared.
  */
@@ -121,9 +152,12 @@
 		    sizeof (vxlnat_fixed_t), 0);
 		/* Initialize NAT rules.  (NAT mutex is zeroed-out.) */
 		list_create(&vnet->vxnv_rules, sizeof (vxlnat_rule_t), 0);
-#ifdef notyet
-		/* XXX KEBE SAYS INITIALIZE NAT flows... */
-#endif /* notyet */
+
+		/* Initialize NAT flows... */
+		rw_init(&vnet->vxnv_flowv4_lock, NULL, RW_DRIVER, NULL);
+		avl_create(&vnet->vxnv_flows_v4, vxlnat_flow_cmp_v4,
+		    sizeof (vxlnat_flow_t), 0);
+
 		/*
 		 * Initialize remote VXLAN destination cache.
 		 * (remotes mutex is zeroed-out.)
@@ -434,8 +468,8 @@
 		avl_insert(&vnet->vxnv_fixed_ips, fixed, where);
 		rc = 0;
 		/*
-		 * CHEESY USE OF POINTERS WARNING: I'm going to use
-		 * ire_dep_children for this IRE_LOCAL as a backpointer to
+		 * ODD USE OF POINTERS WARNING: I'm going to use
+		 * ire_dep_sib_next for this IRE_LOCAL as a backpointer to
 		 * this 'fixed'.  This'll allow rapid packet processing.
 		 * Inspection seems to indicate that IRE_LOCAL ires NEVER use
 		 * the ire_dep* pointers, so we'll use one (and independent of
@@ -443,6 +477,8 @@
 		 * fix it here and add a new pointer in ip.h for ire_t.
 		 */
 		ire->ire_dep_sib_next = (ire_t *)fixed;
+		VXNF_REFHOLD(fixed);	/* ire holds us too... */
+		fixed->vxnf_ire = ire;
 		/* and then rewire the ire receive and send functions. */
 		if (ire->ire_ipversion == IPV4_VERSION) {
 			ire->ire_recvfn = vxlnat_fixed_ire_recv_v4;
@@ -452,8 +488,7 @@
 			ire->ire_recvfn = vxlnat_fixed_ire_recv_v6;
 			ire->ire_sendfn = vxlnat_fixed_ire_send_v6;
 		}
-		VXNF_REFHOLD(fixed);	/* ire holds us too... */
-		fixed->vxnf_ire = ire;
+#if 1	/* Cheesy hack */
 		/*
 		 * XXX KEBE SAYS CHEESY HACK:
 		 */
@@ -464,6 +499,7 @@
 			/* Just so we're clear... */
 			fixed->vxnf_clear_router = B_FALSE;
 		}
+#endif	/* Cheesy hack */
 	}
 	rw_exit(&vnet->vxnv_fixed_lock);
 
--- old/usr/src/uts/common/inet/vxlnat_impl.h	Thu Oct 11 14:05:20 2018
+++ new/usr/src/uts/common/inet/vxlnat_impl.h	Thu Oct 11 14:05:20 2018
@@ -69,14 +69,44 @@
 extern void vxlnat_rule_free(vxlnat_rule_t *);
 
 /*
+ * NAT FLOWS.  These are per-vnet, and keyed/searched by:
+ * <inner-IP-source,IP-dest,inner-source-port,dest-port,protocol>.
+ * They will be tied-to/part-of
+ */
+typedef struct vxlnat_flow_s {
+	avl_node_t vxnfl_treenode;
+	/*
+	 * I'm guessing that dst varies more than src.  Also
+	 * the plan is for the comparitor function to bcmp() both
+	 * of these as one call for IPv6 (if we ever get to that..).
+	 */
+	in6_addr_t vxnfl_dst;
+	in6_addr_t vxnfl_src;	/* INNER source address. */
+	uint32_t vxnfl_ports;
+	uint8_t vxnfl_protocol;
+	uint8_t vxnfl_isv4 : 1,	/* Will save us 12 bytes of compares... */
+		vxlfl_reserved1 : 7;
+	conn_t *vxnfl_connp;	/* Question - embed instead? */
+	vxlnat_rule_t *vxnfl_rule; /* Refhold to rule that generated me. */
+} vxlnat_flow_t;
+/* Exploit endianisms, maintain network order... */
+#ifdef _BIG_ENDIAN
+#define	VXNFL_SPORT(ports) (uint16_t)((ports) >> 16) /* Unsigned all around. */
+#define	VXNFL_DPORT(ports) ((ports) & 0xFFFF)
+#else
+#define	VXNFL_SPORT(ports) ((ports) & 0xFFFF)
+#define	VXNFL_DPORT(ports) (uint16_t)((ports) >> 16) /* Unsigned all around. */
+#endif
+
+/*
  * 1-1 IP mapping.
  */
 typedef struct vxlnat_fixed_s {
 	avl_node_t vxnf_treenode;
-	in6_addr_t vxnf_addr;	/* XXX KEBE ASKS - must it match to a rule? */
+	in6_addr_t vxnf_addr;	/* For now it needn't match to a rule. */
 	in6_addr_t vxnf_pubaddr; /* External IP. */
 	struct vxlnat_vnet_s *vxnf_vnet;
-	ire_t *vxnf_ire;	/* Should be a local IRE from the ftable. */
+	ire_t *vxnf_ire;	/* Should be an IRE_LOCAL from the ftable. */
 	struct vxlnat_remote_s *vxnf_remote;
 	uint8_t vxnf_myether[ETHERADDRL];
 	uint16_t vxnf_vlanid;	/* Stored in network order for quick xmit. */
@@ -100,7 +130,7 @@
  */
 typedef struct vxlnat_remote_s {
 	avl_node_t vxnrem_treenode;
-	in6_addr_t vxnrem_addr;	/* Same prefix as one in rule. */
+	in6_addr_t vxnrem_addr;	/* Same prefix as one in rule, or fixed addr. */
 	in6_addr_t vxnrem_uaddr; /* Underlay VXLAN destination. */
 	struct vxlnat_vnet_s *vxnrem_vnet;	/* Reference-held. */
 	uint32_t vxnrem_refcount;
@@ -133,21 +163,26 @@
 	avl_node_t vxnv_treenode;
 	/*
 	 * 1-1 IP mappings. (1st lookup for an in-to-out packet.)
-	 * Will map to SOMETHING in IP.
-	 * XXX KEBE ASKS - conn_t or something else TBD?!
+	 * Will map to an IRE_LOCAL in IP.
 	 */
 	krwlock_t vxnv_fixed_lock;
 	avl_tree_t vxnv_fixed_ips;
+
 	/*
 	 * NAT flows. (2nd lookup for an in-to-out packet.)
 	 * These are also conn_ts with outer-packet fields for out-to-in
 	 * matches against a conn_t.
+	 *
+	 * NOTE: We're going to keep a separate tree for inner IPv6 NAT, if
+	 * we ever need it.
 	 */
-	krwlock_t vxnv_flow_lock;
-	avl_tree_t vxnv_flows;
+	krwlock_t vxnv_flowv4_lock;
+	avl_tree_t vxnv_flows_v4;
+
 	/* NAT rules. (3rd lookup for an in-to-out packet.) */
 	kmutex_t vxnv_rule_lock;
 	list_t vxnv_rules;
+
 	/*
 	 * Internal-network remote-nodes. (only lookup for out-to-in packet.)
 	 * Entries here are also refheld by 1-1s or NAT flows.
