--- /dev/null	Tue Apr  3 17:58:12 2018
+++ new/usr/src/uts/common/io/overlay/overlay_router.c	Tue Apr  3 17:58:12 2018
@@ -0,0 +1,457 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2018, Joyent, Inc.
+ */
+
+#include <sys/types.h>
+#include <sys/ethernet.h>
+#include <sys/mac_client.h>
+#include <sys/vlan.h>
+#include <sys/ddi.h>
+#include <sys/strsun.h>
+#include <inet/ip.h>
+#include <inet/ip6.h>
+#include <netinet/in.h>
+
+#include <sys/overlay_impl.h>
+
+extern kmem_cache_t *overlay_entry_cache;
+extern int overlay_ent_size;
+
+int
+overlay_fabric_avl(const void *a, const void *b)
+{
+	const overlay_fabric_t *l;
+	const overlay_fabric_t *r;
+	int i;
+
+	l = &((const overlay_fabric_entry_t *)a)->ofe_fabric;
+	r = &((const overlay_fabric_entry_t *)b)->ofe_fabric;
+
+	if (l->ofb_dcid < r->ofb_dcid)
+		return (-1);
+	if (l->ofb_dcid > r->ofb_dcid)
+		return (1);
+	if (l->ofb_vid < r->ofb_vid)
+		return (-1);
+	if (l->ofb_vid > r->ofb_vid)
+		return (1);
+	if (l->ofb_vlan < r->ofb_vlan)
+		return (-1);
+	if (l->ofb_vlan > r->ofb_vlan)
+		return (1);
+	for (i = 0; i < 4; i++) {
+		if (l->ofb_addr.s6_addr32[i] < r->ofb_addr.s6_addr32[i])
+			return (-1);
+		if (l->ofb_addr.s6_addr32[i] > r->ofb_addr.s6_addr32[i])
+			return (1);
+	}
+	return (0);
+}
+
+overlay_fabric_entry_t *
+overlay_fabric_entry_new(uint64_t vid, uint32_t dcid, uint16_t vlan,
+    const struct in6_addr *addr, uint8_t prefixlen, const uint8_t *mac)
+{
+	overlay_fabric_entry_t *ofe = NULL;
+
+	ofe = kmem_zalloc(sizeof (*ofe), KM_SLEEP);
+
+	ofe->ofe_fabric.ofb_vid = vid;
+	ofe->ofe_fabric.ofb_dcid = dcid;
+	ofe->ofe_fabric.ofb_vlan = vlan;
+	ofe->ofe_fabric.ofb_prefixlen = prefixlen;
+	bcopy(addr, &ofe->ofe_fabric.ofb_addr, sizeof (*addr));
+	bcopy(mac, ofe->ofe_fabric.ofb_mac, ETHERADDRL);
+	return (ofe);
+}
+
+void
+overlay_fabric_entry_free(overlay_fabric_entry_t *ofe)
+{
+	if (ofe == NULL)
+		return;
+
+	VERIFY3U(ofe->ofe_refcnt, ==, 0);
+	kmem_free(ofe, sizeof (*ofe));
+}
+
+/*
+ * Using the source and destination IP address, locate the target fabric, or
+ * create larval entries and queue for missing information.
+ */
+static overlay_fabric_entry_t *
+overlay_route_find(overlay_dev_t *odd, const mac_header_info_t *mhi,
+    const struct in6_addr *dst)
+{
+	overlay_target_t *ott = odd->odd_target;
+	overlay_router_t *otr;
+	overlay_fabric_entry_t *ofb_src, *ofb_dst;
+	overlay_target_entry_t *src_entry;
+	overlay_route_table_t *orte;
+	uint16_t vlan = VLAN_ID(mhi->mhi_tci);
+	boolean_t dst_found = B_FALSE;
+
+	/* First attempt to find the overlay_fabric_t for the source */
+	otr = odd->odd_router;
+
+	mutex_enter(&ott->ott_lock);
+	if ((src_entry = refhash_lookup(ott->ott_u.ott_dyn.ott_dhash,
+	    mhi->mhi_saddr)) != NULL)
+		OVERLAY_TARG_ENTRY_REFHOLD(src_entry);
+	mutex_exit(&ott->ott_lock);
+
+	if (src_entry == NULL)
+		return (NULL);
+
+	mutex_enter(&src_entry->ote_lock);
+	if (src_entry->ote_fab == NULL) {
+		mutex_exit(&src_entry->ote_lock);
+		OVERLAY_TARG_ENTRY_REFRELE(src_entry);
+		return (NULL);
+	}
+	if ((ofb_src = src_entry->ote_fab) != NULL)
+		OVERLAY_FAB_ENTRY_REFHOLD(ofb_src);
+	mutex_exit(&src_entry->ote_lock);
+
+	OVERLAY_TARG_ENTRY_REFRELE(src_entry);
+	src_entry = NULL;
+
+	if (ofb_src == NULL || ofb_src->ofe_route_table == NULL) {
+		mutex_exit(&otr->otr_lock);
+		return (NULL);
+	}
+
+	/* Go through the fabric route table and try to find a destination */
+	orte = ofb_src->ofe_route_table;
+	for (size_t i = 0; orte->ort_dest[i] != NULL; i++) {
+		ofb_dst = orte->ort_dest[i];
+		if (IN6_ARE_PREFIXEDADDR_EQUAL(dst,
+		    &ofb_dst->ofe_fabric.ofb_addr,
+		    ofb_dst->ofe_fabric.ofb_prefixlen)) {
+			dst_found = B_TRUE;
+			break;
+		}
+	}
+
+	if (dst_found)
+		OVERLAY_FAB_ENTRY_REFHOLD(ofb_dst);
+
+	mutex_exit(&otr->otr_lock);
+
+	return (dst_found ? ofb_dst : NULL);
+}
+
+/*
+ * Adjust packet fields as necessary for delivery of an encapsulated packet
+ * to a remote target (e.g. cross-DC).
+ */
+static int
+overlay_route(overlay_dev_t *odd, mblk_t *mp, const overlay_fabric_entry_t *ofe,
+    const overlay_target_entry_t *ote)
+{
+	if (MBLKL(mp) >= sizeof (struct ether_vlan_header)) {
+		/* The easy way */
+		struct ether_vlan_header *evh;
+
+		evh = (struct ether_vlan_header *)mp->b_rptr;
+
+		/*
+		 * Today, we require all encapsulated frames to be vlan tagged.
+		 * In the future, we might relax this and insert the vlan tag if
+		 * the destination is tagged and the source is not.
+		 */
+		if (evh->ether_tpid != ETHERTYPE_VLAN)
+			return (OVERLAY_TARGET_DROP);
+
+		evh->ether_tci = evh->ether_tci & ~(VLAN_ID_MASK);
+		evh->ether_tci |= ofe->ofe_fabric.ofb_vlan;
+		bcopy(ote->ote_addr, &evh->ether_dhost, ETHERADDRL);
+		bcopy(ofe->ofe_fabric.ofb_mac, &evh->ether_shost, ETHERADDRL);
+		return (OVERLAY_TARGET_OK);
+	}
+
+	/* The painful, long, tedious way */
+	unsigned char *p = mp->b_rptr;
+	unsigned char *end = mp->b_wptr;
+	unsigned char *vlanp = NULL;
+	uint16_t sap = 0;
+	uint16_t vlan = 0;
+	size_t i;
+
+	/*
+	 * XXX: This method seems so absolutely awful, I wonder if there
+	 * shouldn't be a counter and/or dtrace probe to allow us to find
+	 * out what upstack causes this path to be taken.
+	 */
+	for (i = 0; i < ETHERADDRL; i++) {
+		if (p == end) {
+			mp = mp->b_cont;
+			if (mp == NULL)
+				return (OVERLAY_TARGET_DROP);
+			p = mp->b_rptr;
+			end = mp->b_wptr;
+		}
+		*p++ = ote->ote_addr[i];
+	}
+
+	for (i = 0; i < ETHERADDRL; i++) {
+		if (p == end) {
+			mp = mp->b_cont;
+			if (mp == NULL)
+				return (OVERLAY_TARGET_DROP);
+			p = mp->b_rptr;
+			end = mp->b_wptr;
+		}
+		*p++ = ofe->ofe_fabric.ofb_mac[i];
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (p == end) {
+			mp = mp->b_cont;
+			if (mp == NULL)
+				return (OVERLAY_TARGET_DROP);
+			p = mp->b_rptr;
+			end = mp->b_wptr;
+		}
+		sap |= *p++ << (8 - i*8);
+	}
+
+	if (sap != ETHERTYPE_VLAN)
+		return (OVERLAY_TARGET_DROP);
+
+	if (p == end) {
+		mp = mp->b_cont;
+		if (mp == NULL)
+			return (OVERLAY_TARGET_DROP);
+		p = mp->b_rptr;
+		end = mp->b_wptr;
+	}
+	vlanp = p;
+
+	for (i = 0; i < 2; i++) {
+		if (p == end) {
+			mp = mp->b_cont;
+			if (mp == NULL)
+				return (OVERLAY_TARGET_DROP);
+			p = mp->b_rptr;
+			end = mp->b_wptr;
+		}
+		vlan |= *p++ << (8 - i*8);
+	}
+
+	vlan &= ~(VLAN_ID_MASK);
+	vlan |= ofe->ofe_fabric.ofb_vlan;
+
+	for (p = vlanp, i = 0; i < 2; i++) {
+		if (p == end) {
+			mp = mp->b_cont;
+			if (mp == NULL)
+				return (OVERLAY_TARGET_DROP);
+			p = mp->b_rptr;
+			end = mp->b_wptr;
+		}
+		*p++ = vlan >> (8 - i*8);
+	}
+
+	return (OVERLAY_TARGET_OK);
+}
+
+int
+overlay_route_lookup(overlay_dev_t *odd, mblk_t *mp,
+    const mac_header_info_t *mhi, struct sockaddr *sock, socklen_t *slenp,
+	uint64_t *vidp)
+{
+	overlay_target_t *ott = odd->odd_target;
+	overlay_router_t *otr = odd->odd_router;
+	overlay_fabric_entry_t *dst_fab;
+	overlay_target_entry_t *entry;
+	overlay_target_entry_t search = { 0 };
+	struct sockaddr_in6 *v6;
+	struct in6_addr src, dst;
+	int ret;
+
+	ASSERT(ott->ott_mode == OVERLAY_TARGET_DYNAMIC);
+
+	if (odd->odd_router == NULL)
+		return (OVERLAY_TARGET_DROP);
+
+	if ((ret = overlay_mblk_vl3ip(mp, &src, &dst)) != 0)
+		return (OVERLAY_TARGET_DROP);
+
+	if ((dst_fab = overlay_route_find(odd, mhi, &dst)) == NULL)
+		return (OVERLAY_TARGET_DROP);
+
+	v6 = (struct sockaddr_in6 *)sock;
+	bzero(v6, sizeof (struct sockaddr_in6));
+	v6->sin6_family = AF_INET6;
+
+	bcopy(&dst, &search.ote_ip, sizeof (dst));
+	search.ote_fab = dst_fab;
+
+	mutex_enter(&ott->ott_lock);
+	if ((entry = refhash_lookup(ott->ott_u.ott_dyn.ott_l3dhash,
+	    &search)) == NULL) {
+		if ((entry = kmem_cache_alloc(overlay_entry_cache,
+		    KM_NOSLEEP | KM_NORMALPRI)) == NULL) {
+			mutex_exit(&ott->ott_lock);
+			return (OVERLAY_TARGET_DROP);
+		}
+
+		bcopy(&dst, &entry->ote_ip, sizeof (dst));
+		entry->ote_chead = entry->ote_ctail = mp;
+		entry->ote_mbsize = msgsize(mp);
+		entry->ote_flags |= OVERLAY_ENTRY_F_VL3_PENDING;
+		entry->ote_ott = ott;
+		entry->ote_odd = odd;
+		OVERLAY_TARG_ENTRY_REFHOLD(entry);
+		refhash_insert(ott->ott_u.ott_dyn.ott_l3dhash, entry);
+		mutex_exit(&ott->ott_lock);
+		overlay_target_queue(entry);
+		OVERLAY_FAB_ENTRY_REFRELE(dst_fab);
+		return (OVERLAY_TARGET_ASYNC);
+	}
+	OVERLAY_TARG_ENTRY_REFHOLD(entry);
+	mutex_exit(&ott->ott_lock);
+
+	mutex_enter(&entry->ote_lock);
+	/* There's no point in sending packets to a remote fabric's router IP */
+	if (entry->ote_flags & (OVERLAY_ENTRY_F_DROP|OVERLAY_ENTRY_F_ROUTER)) {
+		ret = OVERLAY_TARGET_DROP;
+	} else if (entry->ote_flags & OVERLAY_ENTRY_F_VALID) {
+		*vidp = dst_fab->ofe_fabric.ofb_vid;
+
+		bcopy(&entry->ote_dest.otp_ip, &v6->sin6_addr,
+		    sizeof (struct in6_addr));
+		v6->sin6_port = htons(entry->ote_dest.otp_port);
+		*slenp = sizeof (struct sockaddr_in6);
+
+		ret = overlay_route(odd, mp, dst_fab, entry);
+	} else {
+		size_t mlen = msgsize(mp);
+
+		if (mlen + entry->ote_mbsize > overlay_ent_size) {
+			ret = OVERLAY_TARGET_DROP;
+		} else {
+			if (entry->ote_ctail != NULL) {
+				ASSERT(entry->ote_ctail->b_next == NULL);
+				entry->ote_ctail->b_next = mp;
+				entry->ote_ctail = mp;
+			} else {
+				entry->ote_chead = mp;
+				entry->ote_ctail = mp;
+			}
+			entry->ote_mbsize += mlen;
+			if ((entry->ote_flags & OVERLAY_ENTRY_F_PENDING) == 0) {
+				entry->ote_flags |= OVERLAY_ENTRY_F_PENDING;
+				overlay_target_queue(entry);
+			}
+			ret = OVERLAY_TARGET_ASYNC;
+		}
+	}
+	mutex_exit(&entry->ote_lock);
+
+	OVERLAY_TARG_ENTRY_REFRELE(entry);
+	OVERLAY_FAB_ENTRY_REFRELE(dst_fab);
+
+	return (ret);
+}
+
+/*
+ * Obtain the source and/or destination VL3 IPs of a packet.  If this is
+ * IPV4 packet, the addresses are returned as IPV6 mapped addresses.
+ *
+ * XXX: The name seems a bit ugly, anyone have better suggestions?
+ */
+int
+overlay_mblk_vl3ip(mblk_t *mp, struct in6_addr *src, struct in6_addr *dst)
+{
+	struct ether_header *ehp;
+	ipha_t *iphp;
+	ip6_t *ip6hp;
+	size_t skip_len;
+	uint16_t sap;
+
+	/*
+	 * This is only used for outbound packets, so we use a similar
+	 * approach as inet_pkt_hash().
+	 */
+	ASSERT(IS_P2ALIGNED(mp->b_rptr, sizeof (uint16_t)));
+	ASSERT(MBLKL(mp) >= sizeof (struct ether_header));
+
+	ehp = (struct ether_header *)mp->b_rptr;
+	sap = ntohs(ehp->ether_type);
+	if (sap == ETHERTYPE_VLAN) {
+		struct ether_vlan_header *evhp;
+		mblk_t *newmp = NULL;
+
+		skip_len = sizeof (struct ether_vlan_header);
+		if (MBLKL(mp) < skip_len) {
+			/* the vlan tag is the payload, pull up first */
+			newmp = msgpullup(mp, -1);
+			if ((newmp == NULL) || (MBLKL(newmp) < skip_len)) {
+				freemsg(newmp);
+				return (ENOMEM);
+			}
+			evhp = (struct ether_vlan_header *)newmp->b_rptr;
+		} else {
+			evhp = (struct ether_vlan_header *)mp->b_rptr;
+		}
+		sap = ntohs(evhp->ether_type);
+		freemsg(newmp);
+	} else {
+		skip_len = sizeof (struct ether_header);
+	}
+
+	/* if the ethernet header is in its own mblk, skip it */
+	if (MBLKL(mp) <= skip_len) {
+		skip_len -= MBLKL(mp);
+		mp = mp->b_cont;
+		if (mp == NULL)
+			return (EINVAL);
+	}
+
+	switch (sap) {
+	case ETHERTYPE_IP:
+		iphp = (ipha_t *)(mp->b_rptr + skip_len);
+		if (((unsigned char *)iphp + sizeof (ipha_t) > mp->b_wptr) ||
+		    !OK_32PTR((char *)iphp))
+			return (EINVAL);	/* XXX: better error code? */
+
+		IN6_IPADDR_TO_V4MAPPED(iphp->ipha_src, src);
+		IN6_IPADDR_TO_V4MAPPED(iphp->ipha_dst, dst);
+		return (0);
+	case ETHERTYPE_IPV6:
+		ip6hp = (ip6_t *)(mp->b_rptr + skip_len);
+		if (((unsigned char *)ip6hp + IPV6_HDR_LEN > mp->b_wptr) ||
+		    !OK_32PTR((char *)ip6hp))
+			return (EINVAL);
+
+		bcopy(&ip6hp->ip6_src, src, sizeof (*src));
+		bcopy(&ip6hp->ip6_dst, dst, sizeof (*dst));
+		return (0);
+	default:
+		return (EINVAL);
+	}
+
+	return (0);
+}
+
+/* ARGSUSED */
+int
+overlay_router_ioctl(dev_t dev, int cmt, intptr_t arg, int mode, cred_t *credp,
+    int *rvalp)
+{
+	/* TODO */
+	return (ENOTTY);
+}
