--- old/usr/src/lib/libipadm/common/ipadm_addr.c	Mon Aug  5 17:16:19 2019
+++ new/usr/src/lib/libipadm/common/ipadm_addr.c	Mon Aug  5 17:16:19 2019
@@ -18,9 +18,11 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright 2016 Nexenta Systems, Inc.
  * Copyright (c) 2016-2017, Chris Fraire <cfraire@me.com>.
  */
 
@@ -297,10 +299,12 @@
 	}
 	assert(af != AF_UNSPEC);
 	if (nvpair_value_nvlist(nvp, &nvladdr) != 0 ||
-	    nvlist_lookup_string(nvladdr, IPADM_NVP_IPADDRHNAME, &sname) != 0 ||
-	    ipadm_set_addr(ipaddr, sname, af) != IPADM_SUCCESS) {
+	    nvlist_lookup_string(nvladdr, IPADM_NVP_IPADDRHNAME, &sname) != 0)
 		goto fail;
-	}
+
+	if (ipadm_set_addr(ipaddr, sname, af) != IPADM_SUCCESS)
+		goto fail;
+
 	nvlist_free(onvl);
 	return (IPADM_SUCCESS);
 fail:
@@ -383,7 +387,7 @@
  * Gets all the addresses from active configuration and populates the
  * address information in `addrinfo'.
  */
-static ipadm_status_t
+ipadm_status_t
 i_ipadm_active_addr_info(ipadm_handle_t iph, const char *ifname,
     ipadm_addr_info_t **addrinfo, uint32_t ipadm_flags, int64_t lifc_flags)
 {
@@ -1713,7 +1717,6 @@
 	if ((pflags & IPADM_OPT_PERSIST) &&
 	    !(ipaddr.ipadm_flags & IPMGMT_PERSIST))
 		return (IPADM_TEMPORARY_OBJ);
-
 	/*
 	 * Currently, setting an address property on an address object of type
 	 * IPADM_ADDR_IPV6_ADDRCONF is not supported. Supporting it involves
@@ -2409,11 +2412,9 @@
 	/* Check if the interface name is valid. */
 	if (!ifparse_ifspec(ifname, &ifsp))
 		return (IPADM_INVALID_ARG);
-
 	/* Check if the given addrobj name is valid. */
 	if (aname != NULL && !i_ipadm_is_user_aobjname_valid(aname))
 		return (IPADM_INVALID_ARG);
-
 	if ((newaddr = calloc(1, sizeof (struct ipadm_addrobj_s))) == NULL)
 		return (IPADM_NO_MEMORY);
 
@@ -2595,10 +2596,6 @@
     const char *aobjname, nvlist_t **onvl)
 {
 	ipmgmt_getaddr_arg_t	garg;
-	ipmgmt_get_rval_t	*rvalp;
-	int			err;
-	size_t			nvlsize;
-	char			*nvlbuf;
 
 	/* Populate the door_call argument structure */
 	bzero(&garg, sizeof (garg));
@@ -2609,16 +2606,7 @@
 	if (ifname != NULL)
 		(void) strlcpy(garg.ia_ifname, ifname, sizeof (garg.ia_ifname));
 
-	rvalp = malloc(sizeof (ipmgmt_get_rval_t));
-	err = ipadm_door_call(iph, &garg, sizeof (garg), (void **)&rvalp,
-	    sizeof (*rvalp), B_TRUE);
-	if (err == 0) {
-		nvlsize = rvalp->ir_nvlsize;
-		nvlbuf = (char *)rvalp + sizeof (ipmgmt_get_rval_t);
-		err = nvlist_unpack(nvlbuf, nvlsize, onvl, NV_ENCODE_NATIVE);
-	}
-	free(rvalp);
-	return (ipadm_errno2status(err));
+	return (i_ipadm_call_ipmgmtd(iph, (void *) &garg, sizeof (garg), onvl));
 }
 
 /*
@@ -2654,6 +2642,8 @@
 	struct lifreq		lifr;
 	uint64_t		ifflags;
 	boolean_t		is_boot = (iph->iph_flags & IPH_IPMGMTD);
+	boolean_t		is_ipmp;
+	char			gifname[LIFGRNAMSIZ];
 
 	/* check for solaris.network.interface.config authorization */
 	if (!ipadm_check_auth())
@@ -2663,7 +2653,6 @@
 	status = i_ipadm_validate_create_addr(iph, addr, flags);
 	if (status != IPADM_SUCCESS)
 		return (status);
-
 	/*
 	 * For Legacy case, check if an addrobj already exists for the
 	 * given logical interface name. If one does not exist,
@@ -2798,6 +2787,17 @@
 			return (IPADM_SUCCESS);
 	}
 
+	/*
+	 * If interface is an IPMP group member, move it out of the group before
+	 * performing any operations on it.
+	 */
+	if ((is_ipmp = i_ipadm_is_under_ipmp(iph, addr->ipadm_ifname))) {
+		(void) i_ipadm_get_groupname_active(iph, addr->ipadm_ifname,
+		    gifname, sizeof (gifname));
+		(void) i_ipadm_set_groupname_active(iph, addr->ipadm_ifname,
+		    "");
+	}
+
 	/* Create the address. */
 	type = addr->ipadm_atype;
 	switch (type) {
@@ -2815,6 +2815,12 @@
 		break;
 	}
 
+	/* Move the underlying IPMP interface back to the group */
+	if (is_ipmp) {
+		(void) i_ipadm_set_groupname_active(iph, addr->ipadm_ifname,
+		    gifname);
+	}
+
 	/*
 	 * If address was not created successfully, unplumb the interface
 	 * if it was plumbed implicitly in this function and remove the
@@ -2887,25 +2893,10 @@
 	 * Create a new logical interface if needed; otherwise, just
 	 * use the 0th logical interface.
 	 */
-retry:
 	if (!(iph->iph_flags & IPH_LEGACY)) {
 		status = i_ipadm_do_addif(iph, ipaddr);
 		if (status != IPADM_SUCCESS)
 			return (status);
-		/*
-		 * We don't have to set the lifnum for IPH_INIT case, because
-		 * there is no placeholder created for the address object in
-		 * this case. For IPH_LEGACY, we don't do this because the
-		 * lifnum is given by the caller and it will be set in the
-		 * end while we call the i_ipadm_addr_persist().
-		 */
-		if (!(iph->iph_flags & IPH_INIT)) {
-			status = i_ipadm_setlifnum_addrobj(iph, ipaddr);
-			if (status == IPADM_ADDROBJ_EXISTS)
-				goto retry;
-			if (status != IPADM_SUCCESS)
-				return (status);
-		}
 	}
 	i_ipadm_addrobj2lifname(ipaddr, lifr.lifr_name,
 	    sizeof (lifr.lifr_name));
@@ -2929,9 +2920,19 @@
 	}
 
 	if (flags & IPADM_OPT_UP) {
-		status = i_ipadm_set_flags(iph, lifr.lifr_name, af, IFF_UP, 0);
+		uint32_t	iff_flags = IFF_UP;
 
 		/*
+		 * Set the NOFAILOVER flag only on underlying IPMP interface
+		 * and not the IPMP group interface itself.
+		 */
+		if (i_ipadm_is_under_ipmp(iph, lifr.lifr_name) &&
+		    !i_ipadm_is_ipmp(iph, lifr.lifr_name))
+			iff_flags |= IFF_NOFAILOVER;
+		status = i_ipadm_set_flags(iph, lifr.lifr_name,
+		    af, iff_flags, 0);
+
+		/*
 		 * IPADM_DAD_FOUND is a soft-error for create-addr.
 		 * No need to tear down the address.
 		 */
@@ -2964,6 +2965,7 @@
 ret:
 	if (status != IPADM_SUCCESS && !legacy)
 		(void) i_ipadm_delete_addr(iph, ipaddr);
+
 	return (status);
 }
 
@@ -3028,6 +3030,21 @@
 	 * kernel.
 	 */
 	if (ipaddr.ipadm_flags & IPMGMT_ACTIVE) {
+		boolean_t	is_ipmp;
+		char		gifname[LIFGRNAMSIZ];
+
+		/*
+		 * If interface is an IPMP group member, move it out of the
+		 * group before performing any operations on it.
+		 */
+		if ((is_ipmp = i_ipadm_is_under_ipmp(iph,
+		    ipaddr.ipadm_ifname))) {
+			(void) i_ipadm_get_groupname_active(iph,
+			    ipaddr.ipadm_ifname, gifname, sizeof (gifname));
+			(void) i_ipadm_set_groupname_active(iph,
+			    ipaddr.ipadm_ifname, "");
+		}
+
 		switch (ipaddr.ipadm_atype) {
 		case IPADM_ADDR_STATIC:
 			status = i_ipadm_delete_addr(iph, &ipaddr);
@@ -3047,6 +3064,12 @@
 			break;
 		}
 
+		/* Move the underlying IPMP interface back to the group */
+		if (is_ipmp) {
+			(void) i_ipadm_set_groupname_active(iph,
+			    ipaddr.ipadm_ifname, gifname);
+		}
+
 		/*
 		 * If the address was previously deleted from the active
 		 * config, we will get a IPADM_ENXIO from kernel.
@@ -3568,9 +3591,11 @@
 
 	if (!(ipaddr->ipadm_flags & IPMGMT_ACTIVE))
 		return (IPADM_OP_DISABLE_OBJ);
+
 	if ((ipadm_flags & IPADM_OPT_PERSIST) &&
 	    !(ipaddr->ipadm_flags & IPMGMT_PERSIST))
 		return (IPADM_TEMPORARY_OBJ);
+
 	if (ipaddr->ipadm_atype == IPADM_ADDR_IPV6_ADDRCONF ||
 	    (ipaddr->ipadm_atype == IPADM_ADDR_DHCP &&
 	    (ipadm_flags & IPADM_OPT_PERSIST)))
@@ -3577,6 +3602,7 @@
 		return (IPADM_NOTSUP);
 
 	i_ipadm_addrobj2lifname(ipaddr, lifname, sizeof (lifname));
+
 	return (i_ipadm_get_flags(iph, lifname, ipaddr->ipadm_af, ifflags));
 }
 
@@ -3803,8 +3829,13 @@
 
 	ifname = ipaddr->ipadm_ifname;
 
-	if (i_ipadm_is_ipmp(iph, ifname) || i_ipadm_is_under_ipmp(iph, ifname))
-		return (IPADM_NOTSUP);
+	/*
+	 * Do not go further when we are under ipmp.
+	 * The interface is plumbed up and we are going to add
+	 * NOFAILOVER address to make in.mpathd happy.
+	 */
+	if (i_ipadm_is_under_ipmp(iph, ifname))
+		return (IPADM_SUCCESS);
 
 	af = ipaddr->ipadm_af;
 	af_exists = ipadm_if_enabled(iph, ifname, af);
@@ -3828,16 +3859,10 @@
 	status = i_ipadm_if_pexists(iph, ifname, af, &p_exists);
 	if (status != IPADM_SUCCESS)
 		return (status);
+
 	if (!a_exists && p_exists)
 		return (IPADM_OP_DISABLE_OBJ);
-	if ((flags & IPADM_OPT_PERSIST) && a_exists && !p_exists) {
-		/*
-		 * If address has to be created persistently,
-		 * and the interface does not exist in the persistent
-		 * store but in active config, fail.
-		 */
-		return (IPADM_TEMPORARY_OBJ);
-	}
+
 	if (af_exists) {
 		status = i_ipadm_get_flags(iph, ifname, af, &ifflags);
 		if (status != IPADM_SUCCESS)
