--- old/usr/src/cmd/zpool/zpool_main.c	Mon Aug  5 17:15:43 2019
+++ new/usr/src/cmd/zpool/zpool_main.c	Mon Aug  5 17:15:43 2019
@@ -21,8 +21,9 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
+ * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  * Copyright 2016 Nexenta Systems, Inc.
@@ -58,6 +59,10 @@
 
 #include "statcommon.h"
 
+#ifndef	MAX
+#define	MAX(x, y)	((x) > (y) ? (x) : (y))
+#endif	/* MAX */
+
 static int zpool_do_create(int, char **);
 static int zpool_do_destroy(int, char **);
 
@@ -82,6 +87,7 @@
 static int zpool_do_split(int, char **);
 
 static int zpool_do_scrub(int, char **);
+static int zpool_do_trim(int, char **);
 
 static int zpool_do_import(int, char **);
 static int zpool_do_export(int, char **);
@@ -93,6 +99,17 @@
 static int zpool_do_get(int, char **);
 static int zpool_do_set(int, char **);
 
+static int zpool_do_vdev_get(int, char **);
+static int zpool_do_vdev_set(int, char **);
+
+static int zpool_do_cos_alloc(int, char **);
+static int zpool_do_cos_free(int, char **);
+static int zpool_do_cos_list(int, char **);
+static int zpool_do_cos_get(int, char **);
+static int zpool_do_cos_set(int, char **);
+
+static boolean_t nexenta_meta_enable();
+
 /*
  * These libumem hooks provide a reasonable set of defaults for the allocator's
  * debugging facilities.
@@ -130,6 +147,7 @@
 	HELP_REPLACE,
 	HELP_REMOVE,
 	HELP_SCRUB,
+	HELP_TRIM,
 	HELP_STATUS,
 	HELP_UPGRADE,
 	HELP_GET,
@@ -136,7 +154,14 @@
 	HELP_SET,
 	HELP_SPLIT,
 	HELP_REGUID,
-	HELP_REOPEN
+	HELP_REOPEN,
+	HELP_VDEV_GET,
+	HELP_VDEV_SET,
+	HELP_COS_ALLOC,
+	HELP_COS_FREE,
+	HELP_COS_LIST,
+	HELP_COS_GET,
+	HELP_COS_SET
 } zpool_help_t;
 
 
@@ -180,6 +205,8 @@
 	{ NULL },
 	{ "scrub",	zpool_do_scrub,		HELP_SCRUB		},
 	{ NULL },
+	{ "trim",	zpool_do_trim,		HELP_TRIM		},
+	{ NULL },
 	{ "import",	zpool_do_import,	HELP_IMPORT		},
 	{ "export",	zpool_do_export,	HELP_EXPORT		},
 	{ "upgrade",	zpool_do_upgrade,	HELP_UPGRADE		},
@@ -188,6 +215,13 @@
 	{ "history",	zpool_do_history,	HELP_HISTORY		},
 	{ "get",	zpool_do_get,		HELP_GET		},
 	{ "set",	zpool_do_set,		HELP_SET		},
+	{ "vdev-get",	zpool_do_vdev_get,	HELP_VDEV_GET		},
+	{ "vdev-set",   zpool_do_vdev_set,	HELP_VDEV_SET		},
+	{ "cos-alloc",  zpool_do_cos_alloc,	HELP_COS_ALLOC		},
+	{ "cos-free",   zpool_do_cos_free,	HELP_COS_FREE		},
+	{ "cos-list",   zpool_do_cos_list,	HELP_COS_LIST		},
+	{ "cos-get",    zpool_do_cos_get,	HELP_COS_GET		},
+	{ "cos-set",    zpool_do_cos_set,	HELP_COS_SET		}
 };
 
 #define	NCOMMAND	(sizeof (command_table) / sizeof (command_table[0]))
@@ -222,11 +256,11 @@
 	case HELP_HISTORY:
 		return (gettext("\thistory [-il] [<pool>] ...\n"));
 	case HELP_IMPORT:
-		return (gettext("\timport [-d dir] [-D]\n"
+		return (gettext("\timport [-d dir] [-D] [-t num]\n"
 		    "\timport [-d dir | -c cachefile] [-F [-n]] <pool | id>\n"
 		    "\timport [-o mntopts] [-o property=value] ... \n"
 		    "\t    [-d dir | -c cachefile] [-D] [-f] [-m] [-N] "
-		    "[-R root] [-F [-n]] -a\n"
+		    "[-R root] [-F [-n]] [-t num] -a\n"
 		    "\timport [-o mntopts] [-o property=value] ... \n"
 		    "\t    [-d dir | -c cachefile] [-D] [-f] [-m] [-N] "
 		    "[-R root] [-F [-n]]\n"
@@ -247,11 +281,13 @@
 		return (gettext("\treplace [-f] <pool> <device> "
 		    "[new-device]\n"));
 	case HELP_REMOVE:
-		return (gettext("\tremove [-nps] <pool> <device> ...\n"));
+		return (gettext("\tremove <pool> <device> ...\n"));
 	case HELP_REOPEN:
 		return (gettext("\treopen <pool>\n"));
 	case HELP_SCRUB:
-		return (gettext("\tscrub [-s | -p] <pool> ...\n"));
+		return (gettext("\tscrub [-m|-M|-p|-s] <pool> ...\n"));
+	case HELP_TRIM:
+		return (gettext("\ttrim [-s|-r <rate>] <pool> ...\n"));
 	case HELP_STATUS:
 		return (gettext("\tstatus [-vx] [-T d|u] [pool] ... [interval "
 		    "[count]]\n"));
@@ -270,13 +306,44 @@
 		    "[<device> ...]\n"));
 	case HELP_REGUID:
 		return (gettext("\treguid <pool>\n"));
+	case HELP_VDEV_GET:
+		return (gettext("\tvdev-get <property | all> <pool> "
+		    "<vdev name | GUID>\n"));
+	case HELP_VDEV_SET:
+		return (gettext("\tvdev-set <property=value> <pool> "
+		    "<vdev name | GUID>\n"));
+	case HELP_COS_ALLOC:
+		return (gettext("\tcos-alloc <pool> <cos name | GUID>\n"));
+	case HELP_COS_FREE:
+		return (gettext("\tcos-free [-f] <pool> <cos name | GUID>\n"));
+	case HELP_COS_LIST:
+		return (gettext("\tcos-list <pool>\n"));
+	case HELP_COS_GET:
+		return (gettext("\tcos-get <property | all> <pool>"
+		    "<cos name | GUID>\n"));
+	case HELP_COS_SET:
+		return (gettext("\tcos-set <property=value> <pool>"
+		    "<cos name | GUID>\n"));
 	}
 
 	abort();
 	/* NOTREACHED */
 }
+/*
+ * Check if additional ZFS meta features are enabled.
+ */
+static boolean_t
+nexenta_meta_enable()
+{
+	if (getenv("nexenta_meta_enable") == NULL) {
+		(void) fprintf(stderr, gettext("feature not enabled\n"));
+		(void) fprintf(stderr,
+		    gettext("set nexenta_meta_enable to access\n"));
+		return (B_FALSE);
+	}
+	return (B_TRUE);
+}
 
-
 /*
  * Callback routine that will print out a pool property value.
  */
@@ -411,7 +478,7 @@
 add_prop_list(const char *propname, char *propval, nvlist_t **props,
     boolean_t poolprop)
 {
-	zpool_prop_t prop = ZPOOL_PROP_INVAL;
+	zpool_prop_t prop = ZPROP_INVAL;
 	zfs_prop_t fprop;
 	nvlist_t *proplist;
 	const char *normnm;
@@ -429,7 +496,7 @@
 	if (poolprop) {
 		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
 
-		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
+		if ((prop = zpool_name_to_prop(propname)) == ZPROP_INVAL &&
 		    !zpool_prop_feature(propname)) {
 			(void) fprintf(stderr, gettext("property '%s' is "
 			    "not a valid pool property\n"), propname);
@@ -440,7 +507,7 @@
 		 * feature@ properties and version should not be specified
 		 * at the same time.
 		 */
-		if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
+		if ((prop == ZPROP_INVAL && zpool_prop_feature(propname) &&
 		    nvlist_exists(proplist, vname)) ||
 		    (prop == ZPOOL_PROP_VERSION &&
 		    prop_list_contains_feature(proplist))) {
@@ -480,6 +547,72 @@
 }
 
 /*
+ * Add a property pair (name, string-value) into a vdev property nvlist.
+ */
+static int
+add_vdev_prop_list(const char *propname, char *propval, nvlist_t **props)
+{
+	vdev_prop_t prop = ZPROP_INVAL;
+	nvlist_t *proplist;
+	const char *normnm;
+
+	if (*props == NULL && nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
+		(void) fprintf(stderr,
+		    gettext("internal error: out of memory\n"));
+		return (1);
+	}
+
+	proplist = *props;
+	if ((prop = vdev_name_to_prop(propname)) == ZPROP_INVAL) {
+		(void) fprintf(stderr, gettext("property '%s' is "
+		    "not a valid vdev property\n"), propname);
+		return (2);
+	}
+	normnm = vdev_prop_to_name(prop);
+
+	if (nvlist_add_string(proplist, normnm, propval) != 0) {
+		(void) fprintf(stderr, gettext("internal "
+		    "error: out of memory\n"));
+		return (1);
+	}
+
+	return (0);
+}
+
+/*
+ * Add a property pair (name, string-value) into a cos property nvlist.
+ */
+static int
+add_cos_prop_list(const char *propname, char *propval, nvlist_t **props)
+{
+	cos_prop_t prop = ZPROP_INVAL;
+	nvlist_t *proplist;
+	const char *normnm;
+
+	if (*props == NULL && nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
+		(void) fprintf(stderr,
+		    gettext("internal error: out of memory\n"));
+		return (1);
+	}
+
+	proplist = *props;
+	if ((prop = cos_name_to_prop(propname)) == ZPROP_INVAL) {
+		(void) fprintf(stderr, gettext("property '%s' is "
+		    "not a valid cos property\n"), propname);
+		return (2);
+	}
+	normnm = cos_prop_to_name(prop);
+
+	if (nvlist_add_string(proplist, normnm, propval) != 0) {
+		(void) fprintf(stderr, gettext("internal "
+		    "error: out of memory\n"));
+		return (1);
+	}
+
+	return (0);
+}
+
+/*
  * zpool add [-fn] <pool> <vdev> ...
  *
  *	-f	Force addition of devices, even if they appear in use
@@ -597,7 +730,8 @@
 /*
  * zpool remove  <pool> <vdev> ...
  *
- * Removes the given vdev from the pool.
+ * Removes the given vdev from the pool.  Currently, this supports removing
+ * spares, cache, and log devices from the pool.
  */
 int
 zpool_do_remove(int argc, char **argv)
@@ -605,38 +739,19 @@
 	char *poolname;
 	int i, ret = 0;
 	zpool_handle_t *zhp;
-	boolean_t stop = B_FALSE;
-	boolean_t noop = B_FALSE;
-	boolean_t parsable = B_FALSE;
-	char c;
 
-	/* check options */
-	while ((c = getopt(argc, argv, "nps")) != -1) {
-		switch (c) {
-		case 'n':
-			noop = B_TRUE;
-			break;
-		case 'p':
-			parsable = B_TRUE;
-			break;
-		case 's':
-			stop = B_TRUE;
-			break;
-		case '?':
-			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
-			    optopt);
-			usage(B_FALSE);
-		}
-	}
+	argc--;
+	argv++;
 
-	argc -= optind;
-	argv += optind;
-
 	/* get pool name and check number of arguments */
 	if (argc < 1) {
 		(void) fprintf(stderr, gettext("missing pool name argument\n"));
 		usage(B_FALSE);
 	}
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing device\n"));
+		usage(B_FALSE);
+	}
 
 	poolname = argv[0];
 
@@ -643,49 +758,9 @@
 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
 		return (1);
 
-	if (stop && noop) {
-		(void) fprintf(stderr, gettext("stop request ignored\n"));
-		return (0);
-	}
-
-	if (stop) {
-		if (argc > 1) {
-			(void) fprintf(stderr, gettext("too many arguments\n"));
-			usage(B_FALSE);
-		}
-		if (zpool_vdev_remove_cancel(zhp) != 0)
+	for (i = 1; i < argc; i++) {
+		if (zpool_vdev_remove(zhp, argv[i]) != 0)
 			ret = 1;
-	} else {
-		if (argc < 2) {
-			(void) fprintf(stderr, gettext("missing device\n"));
-			usage(B_FALSE);
-		}
-
-		for (i = 1; i < argc; i++) {
-			if (noop) {
-				uint64_t size;
-
-				if (zpool_vdev_indirect_size(zhp, argv[i],
-				    &size) != 0) {
-					ret = 1;
-					break;
-				}
-				if (parsable) {
-					(void) printf("%s %llu\n",
-					    argv[i], size);
-				} else {
-					char valstr[32];
-					zfs_nicenum(size, valstr,
-					    sizeof (valstr));
-					(void) printf("Memory that will be "
-					    "used after removing %s: %s\n",
-					    argv[i], valstr);
-				}
-			} else {
-				if (zpool_vdev_remove(zhp, argv[i]) != 0)
-					ret = 1;
-			}
-		}
 	}
 
 	return (ret);
@@ -1264,6 +1339,26 @@
 		return (1);
 	}
 
+	ret = zfs_check_krrp(g_zfs, argv[0]);
+	/*
+	 * ENOTSUP means that autosnaper doesn't handle this pool.
+	 */
+	if (ret != ENOTSUP) {
+		if (ret == ECHILD || ret == EBUSY || ret == EUSERS)
+			ret = EBUSY;
+
+		if (ret) {
+			(void) zpool_standard_error(g_zfs,
+			    ret, gettext("cannnot destroy pool because "
+			    "of krrp"));
+			zpool_close(zhp);
+			ret = 1;
+			return (ret);
+		}
+	} else {
+		ret = 0;
+	}
+
 	if (zpool_disable_datasets(zhp, force) != 0) {
 		(void) fprintf(stderr, gettext("could not destroy '%s': "
 		    "could not unmount datasets\n"), zpool_get_name(zhp));
@@ -1294,13 +1389,15 @@
 {
 	boolean_t force = B_FALSE;
 	boolean_t hardforce = B_FALSE;
+	boolean_t saveconfig = B_FALSE;
 	int c;
+	int n_threads = sysconf(_SC_NPROCESSORS_ONLN) * 2;
 	zpool_handle_t *zhp;
 	int ret;
 	int i;
 
 	/* check options */
-	while ((c = getopt(argc, argv, "fF")) != -1) {
+	while ((c = getopt(argc, argv, ":fFct:")) != -1) {
 		switch (c) {
 		case 'f':
 			force = B_TRUE;
@@ -1308,10 +1405,22 @@
 		case 'F':
 			hardforce = B_TRUE;
 			break;
+		case 'c':
+			saveconfig = B_TRUE;
+			break;
+		case 't':
+			n_threads = atoi(optarg);
+			break;
 		case '?':
 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
 			    optopt);
 			usage(B_FALSE);
+			break;
+		case ':':
+			(void) fprintf(stderr, gettext("missing argument "
+			    "for option '%c'\n"), optopt);
+			usage(B_FALSE);
+			break;
 		}
 	}
 
@@ -1331,7 +1440,27 @@
 			continue;
 		}
 
-		if (zpool_disable_datasets(zhp, force) != 0) {
+		ret = zfs_check_krrp(g_zfs, argv[0]);
+		/*
+		 * ENOTSUP means that autosnaper doesn't handle this pool.
+		 */
+		if (ret != ENOTSUP) {
+			if (ret == ECHILD || ret == EBUSY || ret == EUSERS)
+				ret = EBUSY;
+
+			if (ret) {
+				(void) zpool_standard_error(g_zfs,
+				    ret, gettext("cannnot export pool because "
+				    "of krrp"));
+				zpool_close(zhp);
+				ret = 1;
+				continue;
+			}
+		} else {
+			ret = 0;
+		}
+
+		if (zpool_disable_datasets_ex(zhp, force, n_threads) != 0) {
 			ret = 1;
 			zpool_close(zhp);
 			continue;
@@ -1341,9 +1470,11 @@
 		log_history = B_FALSE;
 
 		if (hardforce) {
-			if (zpool_export_force(zhp, history_str) != 0)
+			if (zpool_export_force(zhp, saveconfig, history_str)
+			    != 0)
 				ret = 1;
-		} else if (zpool_export(zhp, force, history_str) != 0) {
+		} else if (zpool_export(zhp, force, saveconfig, history_str)
+		    != 0) {
 			ret = 1;
 		}
 
@@ -1459,7 +1590,6 @@
 	uint64_t notpresent;
 	spare_cbdata_t cb;
 	const char *state;
-	char *type;
 
 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 	    &child, &children) != 0)
@@ -1468,11 +1598,6 @@
 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
 	    (uint64_t **)&vs, &c) == 0);
 
-	verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
-
-	if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
-		return;
-
 	state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
 	if (isspare) {
 		/*
@@ -1562,10 +1687,6 @@
 			(void) printf(gettext("split into new pool"));
 			break;
 
-		case VDEV_AUX_CHILDREN_OFFLINE:
-			(void) printf(gettext("all children offline"));
-			break;
-
 		default:
 			(void) printf(gettext("corrupted data"));
 			break;
@@ -1575,8 +1696,9 @@
 	(void) nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_SCAN_STATS,
 	    (uint64_t **)&ps, &c);
 
-	if (ps && ps->pss_state == DSS_SCANNING &&
-	    vs->vs_scan_processed != 0 && children == 0) {
+	if (ps != NULL && (ps->pss_state == DSS_SCANNING ||
+	    ps->pss_state == DSS_FINISHING) && vs->vs_scan_processed != 0 &&
+	    children == 0) {
 		(void) printf(gettext("  (%s)"),
 		    (ps->pss_func == POOL_SCAN_RESILVER) ?
 		    "resilvering" : "repairing");
@@ -1585,7 +1707,7 @@
 	(void) printf("\n");
 
 	for (c = 0; c < children; c++) {
-		uint64_t islog = B_FALSE, ishole = B_FALSE;
+		uint64_t islog = B_FALSE, ishole = B_FALSE, isspecial = B_FALSE;
 
 		/* Don't print logs or holes here */
 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
@@ -1592,7 +1714,9 @@
 		    &islog);
 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
 		    &ishole);
-		if (islog || ishole)
+		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_SPECIAL,
+		    &isspecial);
+		if (islog || ishole || isspecial)
 			continue;
 		vname = zpool_vdev_name(g_zfs, zhp, child[c], B_TRUE);
 		print_status_config(zhp, vname, child[c],
@@ -1653,10 +1777,6 @@
 			(void) printf(gettext("too many errors"));
 			break;
 
-		case VDEV_AUX_CHILDREN_OFFLINE:
-			(void) printf(gettext("all children offline"));
-			break;
-
 		default:
 			(void) printf(gettext("corrupted data"));
 			break;
@@ -1669,11 +1789,13 @@
 		return;
 
 	for (c = 0; c < children; c++) {
-		uint64_t is_log = B_FALSE;
+		uint64_t is_log = B_FALSE, is_special = B_FALSE;
 
 		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 		    &is_log);
-		if (is_log)
+		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_SPECIAL,
+		    &is_special);
+		if (is_log || is_special)
 			continue;
 
 		vname = zpool_vdev_name(g_zfs, NULL, child[c], B_TRUE);
@@ -1741,6 +1863,45 @@
 }
 
 /*
+ * Print special vdevs.
+ * Special vdevs are recorded as top level vdevs in the main pool child array
+ * but with "is_special" set to 1. We use either print_status_config() or
+ * print_import_config() to print the top level logs then any log
+ * children (eg mirrored slogs) are printed recursively - which
+ * works because only the top level vdev is marked "is_special"
+ */
+static void
+print_special(zpool_handle_t *zhp, nvlist_t *nv, int namewidth,
+    boolean_t verbose)
+{
+	uint_t c, children;
+	nvlist_t **child;
+
+	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
+	    &children) != 0)
+		return;
+
+	(void) printf(gettext("\tspecial\n"));
+
+	for (c = 0; c < children; c++) {
+		uint64_t is_special = B_FALSE;
+		char *name;
+
+		(void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_SPECIAL,
+		    &is_special);
+		if (!is_special)
+			continue;
+		name = zpool_vdev_name(g_zfs, zhp, child[c], B_TRUE);
+		if (verbose)
+			print_status_config(zhp, name, child[c], namewidth,
+			    2, B_FALSE);
+		else
+			print_import_config(name, child[c], namewidth, 2);
+		free(name);
+	}
+}
+
+/*
  * Display the status for the given pool.
  */
 static void
@@ -1955,6 +2116,9 @@
 		namewidth = 10;
 
 	print_import_config(name, nvroot, namewidth, 0);
+	if (num_special(nvroot) > 0)
+		print_special(NULL, nvroot, namewidth, B_FALSE);
+
 	if (num_logs(nvroot) > 0)
 		print_logs(NULL, nvroot, namewidth, B_FALSE);
 
@@ -1972,7 +2136,7 @@
  */
 static int
 do_import(nvlist_t *config, const char *newname, const char *mntopts,
-    nvlist_t *props, int flags)
+    nvlist_t *props, int flags, int n_threads)
 {
 	zpool_handle_t *zhp;
 	char *name;
@@ -2036,7 +2200,7 @@
 
 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
 	    !(flags & ZFS_IMPORT_ONLY) &&
-	    zpool_enable_datasets(zhp, mntopts, 0) != 0) {
+	    zpool_enable_datasets_ex(zhp, mntopts, 0, n_threads) != 0) {
 		zpool_close(zhp);
 		return (1);
 	}
@@ -2047,9 +2211,9 @@
 
 /*
  * zpool import [-d dir] [-D]
- *       import [-o mntopts] [-o prop=value] ... [-R root] [-D]
+ *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-t num]
  *              [-d dir | -c cachefile] [-f] -a
- *       import [-o mntopts] [-o prop=value] ... [-R root] [-D]
+ *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-t num]
  *              [-d dir | -c cachefile] [-f] [-n] [-F] <pool | id> [newpool]
  *
  *	 -c	Read pool information from a cachefile instead of searching
@@ -2066,8 +2230,8 @@
  *		is rebooted.
  *
  *       -V	Import even in the presence of faulted vdevs.  This is an
- *       	intentionally undocumented option for testing purposes, and
- *       	treats the pool configuration as complete, leaving any bad
+ *		intentionally undocumented option for testing purposes, and
+ *		treats the pool configuration as complete, leaving any bad
  *		vdevs in the FAULTED state. In other words, it does verbatim
  *		import.
  *
@@ -2079,8 +2243,10 @@
  *
  *       -N     Import the pool but don't mount datasets.
  *
+ *       -t     Use up to num threads to mount datasets in parallel.
+ *
  *       -T     Specify a starting txg to use for import. This option is
- *       	intentionally undocumented option for testing purposes.
+ *		intentionally undocumented option for testing purposes.
  *
  *       -a	Import all pools found.
  *
@@ -2117,10 +2283,11 @@
 	uint64_t pool_state, txg = -1ULL;
 	char *cachefile = NULL;
 	importargs_t idata = { 0 };
+	unsigned long n_threads = sysconf(_SC_NPROCESSORS_ONLN) * 2;
 	char *endptr;
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":aCc:d:DEfFmnNo:rR:T:VX")) != -1) {
+	while ((c = getopt(argc, argv, ":aCc:d:DEfFmnNo:rR:t:T:VX")) != -1) {
 		switch (c) {
 		case 'a':
 			do_all = B_TRUE;
@@ -2182,6 +2349,15 @@
 			    ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
 				goto error;
 			break;
+		case 't':
+			errno = 0;
+			n_threads = strtol(optarg, &endptr, 10);
+			if (errno != 0 || *endptr != '\0') {
+				(void) fprintf(stderr,
+				    gettext("invalid num value\n"));
+				usage(B_FALSE);
+			}
+			break;
 		case 'T':
 			errno = 0;
 			txg = strtoull(optarg, &endptr, 0);
@@ -2304,7 +2480,6 @@
 	idata.poolname = searchname;
 	idata.guid = searchguid;
 	idata.cachefile = cachefile;
-	idata.policy = policy;
 
 	pools = zpool_search_import(g_zfs, &idata);
 
@@ -2369,7 +2544,7 @@
 
 			if (do_all) {
 				err |= do_import(config, NULL, mntopts,
-				    props, flags);
+				    props, flags, n_threads);
 			} else {
 				show_import(config);
 			}
@@ -2418,7 +2593,7 @@
 			err = B_TRUE;
 		} else {
 			err |= do_import(found_config, argc == 1 ? NULL :
-			    argv[1], mntopts, props, flags);
+			    argv[1], mntopts, props, flags, n_threads);
 		}
 	}
 
@@ -2453,16 +2628,17 @@
 
 	for (i = 0; i < cb->cb_namewidth; i++)
 		(void) printf("-");
-	(void) printf("  -----  -----  -----  -----  -----  -----\n");
+	(void) printf("  -----  -----  -----  -----  -----  -----  -----  "
+	    "-----\n");
 }
 
 static void
 print_iostat_header(iostat_cbdata_t *cb)
 {
-	(void) printf("%*s     capacity     operations    bandwidth\n",
-	    cb->cb_namewidth, "");
-	(void) printf("%-*s  alloc   free   read  write   read  write\n",
-	    cb->cb_namewidth, "pool");
+	(void) printf("%*s     capacity     operations    bandwidth      "
+	    "latency\n", cb->cb_namewidth, "");
+	(void) printf("%-*s  alloc   free   read  write   read  write   read  "
+	    "write\n", cb->cb_namewidth, "pool");
 	print_iostat_separator(cb);
 }
 
@@ -2479,6 +2655,25 @@
 }
 
 /*
+ * Display latency statistic (extra care needed)
+ */
+static void
+print_one_latency_stat(uint64_t iotime, uint64_t ops)
+{
+	char buf[64];
+	double value = 0.0; /* latency in milliseconds */
+
+	if (ops != 0) {
+		value = (double)iotime;
+		value /= (double)ops;
+		value /= (double)(MICROSEC / MILLISEC);
+	}
+
+	(void) sprintf(buf, "%.2f", value);
+	(void) printf("  %5s", buf);
+}
+
+/*
  * Print out all the statistics for the given vdev.  This can either be the
  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
  * is a verbose output, and we don't want to display the toplevel pool stats.
@@ -2495,9 +2690,6 @@
 	double scale;
 	char *vname;
 
-	if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
-		return;
-
 	if (oldnv != NULL) {
 		verify(nvlist_lookup_uint64_array(oldnv,
 		    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
@@ -2541,6 +2733,19 @@
 	print_one_stat((uint64_t)(scale * (newvs->vs_bytes[ZIO_TYPE_WRITE] -
 	    oldvs->vs_bytes[ZIO_TYPE_WRITE])));
 
+	/*
+	 * No scale needed here since we are dividing in
+	 * print_one_latency_stat()
+	 */
+
+	print_one_latency_stat(
+	    newvs->vs_iotime[ZIO_TYPE_READ] - oldvs->vs_iotime[ZIO_TYPE_READ],
+	    newvs->vs_ops[ZIO_TYPE_READ] - oldvs->vs_ops[ZIO_TYPE_READ]);
+
+	print_one_latency_stat(
+	    newvs->vs_iotime[ZIO_TYPE_WRITE] - oldvs->vs_iotime[ZIO_TYPE_WRITE],
+	    newvs->vs_ops[ZIO_TYPE_WRITE] - oldvs->vs_ops[ZIO_TYPE_WRITE]);
+
 	(void) printf("\n");
 
 	if (!cb->cb_verbose)
@@ -2555,7 +2760,7 @@
 		return;
 
 	for (c = 0; c < children; c++) {
-		uint64_t ishole = B_FALSE, islog = B_FALSE;
+		uint64_t ishole = B_FALSE, islog = B_FALSE, isspec = B_FALSE;
 
 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
 		    &ishole);
@@ -2563,7 +2768,10 @@
 		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
 		    &islog);
 
-		if (ishole || islog)
+		(void) nvlist_lookup_uint64(newchild[c],
+		    ZPOOL_CONFIG_IS_SPECIAL, &isspec);
+
+		if (ishole || islog || isspec)
 			continue;
 
 		vname = zpool_vdev_name(g_zfs, zhp, newchild[c], B_FALSE);
@@ -2594,7 +2802,30 @@
 				free(vname);
 			}
 		}
+	}
 
+	/*
+	 * Special device section
+	 */
+
+	if (num_special(newnv) > 0) {
+		(void) printf("%-*s      -      -      -      -      -      "
+		    "-\n", cb->cb_namewidth, "special");
+
+		for (c = 0; c < children; c++) {
+			uint64_t isspec = B_FALSE;
+			(void) nvlist_lookup_uint64(newchild[c],
+			    ZPOOL_CONFIG_IS_SPECIAL, &isspec);
+
+			if (isspec) {
+				vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
+				    B_FALSE);
+				print_vdev_stats(zhp, vname, oldnv ?
+				    oldchild[c] : NULL, newchild[c],
+				    cb, depth + 2);
+				free(vname);
+			}
+		}
 	}
 
 	/*
@@ -3104,9 +3335,6 @@
 		boolean_t toplevel = (vs->vs_space != 0);
 		uint64_t cap;
 
-		if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
-			return;
-
 		if (scripted)
 			(void) printf("\t%s", name);
 		else if (strlen(name) + depth > cb->cb_namewidth)
@@ -3238,7 +3466,7 @@
 zpool_do_list(int argc, char **argv)
 {
 	int c;
-	int ret;
+	int ret = 0;
 	list_cbdata_t cb = { 0 };
 	static char default_props[] =
 	    "name,size,allocated,free,expandsize,fragmentation,capacity,"
@@ -3982,6 +4210,31 @@
 	return (err != 0);
 }
 
+typedef struct trim_cbdata {
+	boolean_t	cb_start;
+	uint64_t	cb_rate;
+} trim_cbdata_t;
+
+int
+trim_callback(zpool_handle_t *zhp, void *data)
+{
+	trim_cbdata_t *cb = data;
+	int err;
+
+	/*
+	 * Ignore faulted pools.
+	 */
+	if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
+		(void) fprintf(stderr, gettext("cannot trim '%s': pool is "
+		    "currently unavailable\n"), zpool_get_name(zhp));
+		return (1);
+	}
+
+	err = zpool_trim(zhp, cb->cb_start, cb->cb_rate);
+
+	return (err != 0);
+}
+
 /*
  * zpool scrub [-s | -p] <pool> ...
  *
@@ -3998,11 +4251,32 @@
 	cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
 
 	/* check options */
-	while ((c = getopt(argc, argv, "sp")) != -1) {
+	while ((c = getopt(argc, argv, "mMps")) != -1) {
 		switch (c) {
 		case 's':
-			cb.cb_type = POOL_SCAN_NONE;
+			if (cb.cb_type != POOL_SCAN_SCRUB) {
+				(void) fprintf(stderr,
+				    gettext("incompatible options\n"));
+				usage(B_FALSE);
+			} else
+				cb.cb_type = POOL_SCAN_NONE;
 			break;
+		case 'M':
+			if (cb.cb_type != POOL_SCAN_SCRUB) {
+				(void) fprintf(stderr,
+				    gettext("incompatible options\n"));
+				usage(B_FALSE);
+			} else
+				cb.cb_type = POOL_SCAN_MOS;
+			break;
+		case 'm':
+			if (cb.cb_type != POOL_SCAN_SCRUB) {
+				(void) fprintf(stderr,
+				    gettext("incompatible options\n"));
+				usage(B_FALSE);
+			} else
+				cb.cb_type = POOL_SCAN_META;
+			break;
 		case 'p':
 			cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
 			break;
@@ -4033,6 +4307,52 @@
 	return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
 }
 
+/*
+ * zpool trim [-s|-r <rate>] <pool> ...
+ *
+ *	-s		Stop. Stops any in-progress trim.
+ *	-r <rate>	Sets the TRIM rate.
+ */
+int
+zpool_do_trim(int argc, char **argv)
+{
+	int c;
+	trim_cbdata_t cb;
+
+	cb.cb_start = B_TRUE;
+	cb.cb_rate = 0;
+
+	/* check options */
+	while ((c = getopt(argc, argv, "sr:")) != -1) {
+		switch (c) {
+		case 's':
+			cb.cb_start = B_FALSE;
+			break;
+		case 'r':
+			if (zfs_nicestrtonum(NULL, optarg, &cb.cb_rate) == -1) {
+				(void) fprintf(stderr,
+				    gettext("invalid value for rate\n"));
+				usage(B_FALSE);
+			}
+			break;
+		case '?':
+			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
+			    optopt);
+			usage(B_FALSE);
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 1) {
+		(void) fprintf(stderr, gettext("missing pool name argument\n"));
+		usage(B_FALSE);
+	}
+
+	return (for_each_pool(argc, argv, B_TRUE, NULL, trim_callback, &cb));
+}
+
 typedef struct status_cbdata {
 	int		cb_count;
 	boolean_t	cb_allpools;
@@ -4045,15 +4365,16 @@
 /*
  * Print out detailed scrub status.
  */
-static void
+void
 print_scan_status(pool_scan_stat_t *ps)
 {
 	time_t start, end, pause;
 	uint64_t elapsed, mins_left, hours_left;
-	uint64_t pass_exam, examined, total;
-	uint_t rate;
+	uint64_t examined, total;
+	uint64_t rate, proc_rate;
 	double fraction_done;
-	char processed_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
+	char processed_buf[7], examined_buf[7], total_buf[7], rate_buf[7],
+	    issued_buf[7];
 
 	(void) printf(gettext("  scan: "));
 
@@ -4070,7 +4391,9 @@
 	zfs_nicenum(ps->pss_processed, processed_buf, sizeof (processed_buf));
 
 	assert(ps->pss_func == POOL_SCAN_SCRUB ||
-	    ps->pss_func == POOL_SCAN_RESILVER);
+	    ps->pss_func == POOL_SCAN_RESILVER ||
+	    ps->pss_func == POOL_SCAN_MOS ||
+	    ps->pss_func == POOL_SCAN_META);
 	/*
 	 * Scan is finished or canceled.
 	 */
@@ -4081,6 +4404,12 @@
 		if (ps->pss_func == POOL_SCAN_SCRUB) {
 			fmt = gettext("scrub repaired %s in %lluh%um with "
 			    "%llu errors on %s");
+		} else if (ps->pss_func == POOL_SCAN_MOS) {
+			fmt = gettext("MOS scrub repaired %s in %lluh%um with "
+			    "%llu errors on %s");
+		} else if (ps->pss_func == POOL_SCAN_META) {
+			fmt = gettext("meta scrub repaired %s in %lluh%um with "
+			    "%llu errors on %s");
 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
 			fmt = gettext("resilvered %s in %lluh%um with "
 			    "%llu errors on %s");
@@ -4096,6 +4425,12 @@
 		if (ps->pss_func == POOL_SCAN_SCRUB) {
 			(void) printf(gettext("scrub canceled on %s"),
 			    ctime(&end));
+		} else if (ps->pss_func == POOL_SCAN_MOS) {
+			(void) printf(gettext("MOS scrub canceled on %s"),
+			    ctime(&end));
+		} else if (ps->pss_func == POOL_SCAN_META) {
+			(void) printf(gettext("meta scrub canceled on %s"),
+			    ctime(&end));
 		} else if (ps->pss_func == POOL_SCAN_RESILVER) {
 			(void) printf(gettext("resilver canceled on %s"),
 			    ctime(&end));
@@ -4103,7 +4438,7 @@
 		return;
 	}
 
-	assert(ps->pss_state == DSS_SCANNING);
+	assert(ps->pss_state == DSS_SCANNING || ps->pss_state == DSS_FINISHING);
 
 	/*
 	 * Scan is in progress.
@@ -4120,36 +4455,66 @@
 			(void) printf(gettext("\tscrub started on   %s"),
 			    ctime(&start));
 		}
+	} else if (ps->pss_func == POOL_SCAN_MOS) {
+		(void) printf(gettext("MOS scrub in progress since %s"),
+		    ctime(&start));
+	} else if (ps->pss_func == POOL_SCAN_META) {
+		(void) printf(gettext("meta scrub in progress since %s"),
+		    ctime(&start));
 	} else if (ps->pss_func == POOL_SCAN_RESILVER) {
 		(void) printf(gettext("resilver in progress since %s"),
 		    ctime(&start));
 	}
 
-	examined = ps->pss_examined ? ps->pss_examined : 1;
+	examined = ps->pss_examined;
 	total = ps->pss_to_examine;
-	fraction_done = (double)examined / total;
+	fraction_done = (double)ps->pss_issued / total;
 
 	/* elapsed time for this pass */
-	elapsed = time(NULL) - ps->pss_pass_start;
-	elapsed -= ps->pss_pass_scrub_spent_paused;
-	elapsed = elapsed ? elapsed : 1;
-	pass_exam = ps->pss_pass_exam ? ps->pss_pass_exam : 1;
-	rate = pass_exam / elapsed;
-	rate = rate ? rate : 1;
-	mins_left = ((total - examined) / rate) / 60;
+	elapsed = MAX(time(NULL) - ps->pss_start_time -
+	    ps->pss_pass_scrub_spent_paused, 1);
+	if (ps->pss_func == POOL_SCAN_RESILVER) {
+		rate = MAX(((ps->pss_issued + ps->pss_processed) / 2) /
+		    elapsed, 1);
+	} else {
+		rate = MAX(ps->pss_issued / elapsed, 1);
+	}
+	proc_rate = MAX(ps->pss_processed / elapsed, 1);
+	if (ps->pss_func == POOL_SCAN_RESILVER)
+		mins_left = ((total - ps->pss_issued) / proc_rate) / 60;
+	else
+		mins_left = ((total - ps->pss_issued) / rate) / 60;
 	hours_left = mins_left / 60;
 
 	zfs_nicenum(examined, examined_buf, sizeof (examined_buf));
+	zfs_nicenum(ps->pss_issued, issued_buf, sizeof (issued_buf));
 	zfs_nicenum(total, total_buf, sizeof (total_buf));
+	zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
 
+	if (pause == 0) {
+		(void) printf(gettext("        %s scanned, %s verified "
+		    "out of %s at %s/s, %.2f%% done\n"), examined_buf,
+		    issued_buf, total_buf, rate_buf, 100 * fraction_done);
+	}
+
+	if (ps->pss_func == POOL_SCAN_RESILVER) {
+		char proc_rate_buf[7];
+		zfs_nicenum(proc_rate, proc_rate_buf,
+		    sizeof (proc_rate_buf));
+		(void) printf(gettext("        %s resilvered at %s/s"),
+		    processed_buf, proc_rate_buf);
+	} else if (ps->pss_func == POOL_SCAN_SCRUB ||
+	    ps->pss_func == POOL_SCAN_MOS ||
+		    ps->pss_func == POOL_SCAN_META) {
+		(void) printf(gettext("        %s repaired"),
+		    processed_buf);
+	}
+
 	/*
 	 * do not print estimated time if hours_left is more than 30 days
 	 * or we have a paused scrub
 	 */
 	if (pause == 0) {
-		zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
-		(void) printf(gettext("\t%s scanned out of %s at %s/s"),
-		    examined_buf, total_buf, rate_buf);
 		if (hours_left < (30 * 24)) {
 			(void) printf(gettext(", %lluh%um to go\n"),
 			    (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
@@ -4158,122 +4523,62 @@
 			    ", (scan is slow, no estimated time)\n"));
 		}
 	} else {
-		(void) printf(gettext("\t%s scanned out of %s\n"),
-		    examined_buf, total_buf);
+		(void) printf(gettext("\t%s scanned, %s verified out of %s\n"),
+		    examined_buf, issued_buf, total_buf);
 	}
-
-	if (ps->pss_func == POOL_SCAN_RESILVER) {
-		(void) printf(gettext("    %s resilvered, %.2f%% done\n"),
-		    processed_buf, 100 * fraction_done);
-	} else if (ps->pss_func == POOL_SCAN_SCRUB) {
-		(void) printf(gettext("    %s repaired, %.2f%% done\n"),
-		    processed_buf, 100 * fraction_done);
-	}
 }
 
-/*
- * Print out detailed removal status.
- */
 static void
-print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
+print_trim_status(uint64_t trim_prog, uint64_t total_size, uint64_t rate,
+    uint64_t start_time_u64, uint64_t end_time_u64)
 {
-	char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
-	time_t start, end;
-	nvlist_t *config, *nvroot;
-	nvlist_t **child;
-	uint_t children;
-	char *vdev_name;
+	time_t start_time = start_time_u64, end_time = end_time_u64;
+	char *buf;
 
-	if (prs == NULL || prs->prs_state == DSS_NONE)
-		return;
-
-	/*
-	 * Determine name of vdev.
-	 */
-	config = zpool_get_config(zhp, NULL);
-	nvroot = fnvlist_lookup_nvlist(config,
-	    ZPOOL_CONFIG_VDEV_TREE);
-	verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
-	    &child, &children) == 0);
-	assert(prs->prs_removing_vdev < children);
-	vdev_name = zpool_vdev_name(g_zfs, zhp,
-	    child[prs->prs_removing_vdev], B_TRUE);
-
-	(void) printf(gettext("remove: "));
-
-	start = prs->prs_start_time;
-	end = prs->prs_end_time;
-	zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
-
-	/*
-	 * Removal is finished or canceled.
-	 */
-	if (prs->prs_state == DSS_FINISHED) {
-		uint64_t minutes_taken = (end - start) / 60;
-
-		(void) printf(gettext("Removal of vdev %llu copied %s "
-		    "in %lluh%um, completed on %s"),
-		    (longlong_t)prs->prs_removing_vdev,
-		    copied_buf,
-		    (u_longlong_t)(minutes_taken / 60),
-		    (uint_t)(minutes_taken % 60),
-		    ctime((time_t *)&end));
-	} else if (prs->prs_state == DSS_CANCELED) {
-		(void) printf(gettext("Removal of %s canceled on %s"),
-		    vdev_name, ctime(&end));
+	assert(trim_prog <= total_size);
+	if (trim_prog != 0 && trim_prog != total_size) {
+		buf = ctime(&start_time);
+		buf[strlen(buf) - 1] = '\0';	/* strip trailing newline */
+		if (rate != 0) {
+			char rate_str[32];
+			zfs_nicenum(rate, rate_str, sizeof (rate_str));
+			(void) printf("  trim: %.02f%%\tstarted: %s\t"
+			    "(rate: %s/s)\n", (((double)trim_prog) /
+			    total_size) * 100, buf, rate_str);
+		} else {
+			(void) printf("  trim: %.02f%%\tstarted: %s\t"
+			    "(rate: max)\n", (((double)trim_prog) /
+			    total_size) * 100, buf);
+		}
 	} else {
-		uint64_t copied, total, elapsed, mins_left, hours_left;
-		double fraction_done;
-		uint_t rate;
+		if (start_time != 0) {
+			/*
+			 * Non-zero start time means we were run at some point
+			 * in the past.
+			 */
+			if (end_time != 0) {
+				/* Non-zero end time means we completed */
+				time_t diff = end_time - start_time;
+				int hrs, mins;
 
-		assert(prs->prs_state == DSS_SCANNING);
-
-		/*
-		 * Removal is in progress.
-		 */
-		(void) printf(gettext(
-		    "Evacuation of %s in progress since %s"),
-		    vdev_name, ctime(&start));
-
-		copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
-		total = prs->prs_to_copy;
-		fraction_done = (double)copied / total;
-
-		/* elapsed time for this pass */
-		elapsed = time(NULL) - prs->prs_start_time;
-		elapsed = elapsed > 0 ? elapsed : 1;
-		rate = copied / elapsed;
-		rate = rate > 0 ? rate : 1;
-		mins_left = ((total - copied) / rate) / 60;
-		hours_left = mins_left / 60;
-
-		zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
-		zfs_nicenum(total, total_buf, sizeof (total_buf));
-		zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
-
-		/*
-		 * do not print estimated time if hours_left is more than
-		 * 30 days
-		 */
-		(void) printf(gettext("    %s copied out of %s at %s/s, "
-		    "%.2f%% done"),
-		    examined_buf, total_buf, rate_buf, 100 * fraction_done);
-		if (hours_left < (30 * 24)) {
-			(void) printf(gettext(", %lluh%um to go\n"),
-			    (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
+				buf = ctime(&end_time);
+				buf[strlen(buf) - 1] = '\0';
+				hrs = diff / 3600;
+				mins = (diff % 3600) / 60;
+				(void) printf(gettext("  trim: completed on %s "
+				    "(after %dh%dm)\n"), buf, hrs, mins);
+			} else {
+				buf = ctime(&start_time);
+				buf[strlen(buf) - 1] = '\0';
+				/* Zero end time means we were interrupted */
+				(void) printf(gettext("  trim: interrupted\t"
+				    "(started %s)\n"), buf);
+			}
 		} else {
-			(void) printf(gettext(
-			    ", (copy is slow, no estimated time)\n"));
+			/* trim was never run */
+			(void) printf(gettext("  trim: none requested\n"));
 		}
 	}
-
-	if (prs->prs_mapping_memory > 0) {
-		char mem_buf[7];
-		zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
-		(void) printf(gettext("    %s memory used for "
-		    "removed device mappings\n"),
-		    mem_buf);
-	}
 }
 
 static void
@@ -4388,6 +4693,43 @@
 }
 
 /*
+ * Calculates the total space available on log devices on the pool.
+ * For whatever reason, this is not counted in the root vdev's space stats.
+ */
+static uint64_t
+zpool_slog_space(nvlist_t *nvroot)
+{
+	nvlist_t **newchild;
+	uint_t c, children;
+	uint64_t space = 0;
+
+	verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
+	    &newchild, &children) == 0);
+
+	for (c = 0; c < children; c++) {
+		uint64_t islog = B_FALSE;
+		vdev_stat_t *vs;
+		uint_t n;
+		uint_t n_subchildren = 1;
+		nvlist_t **subchild;
+
+		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
+		    &islog);
+		if (!islog)
+			continue;
+		verify(nvlist_lookup_uint64_array(newchild[c],
+		    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &n) == 0);
+
+		/* vdev can be non-leaf, so multiply by number of children */
+		(void) nvlist_lookup_nvlist_array(newchild[c],
+		    ZPOOL_CONFIG_CHILDREN, &subchild, &n_subchildren);
+		space += n_subchildren * vs->vs_space;
+	}
+
+	return (space);
+}
+
+/*
  * Display a summary of pool status.  Displays a summary such as:
  *
  *        pool: tank
@@ -4440,7 +4782,8 @@
 	else
 		(void) printf("\n");
 
-	nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
+	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
+	    &nvroot) == 0);
 	verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
 	    (uint64_t **)&vs, &c) == 0);
 	health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
@@ -4648,15 +4991,29 @@
 		nvlist_t **spares, **l2cache;
 		uint_t nspares, nl2cache;
 		pool_scan_stat_t *ps = NULL;
-		pool_removal_stat_t *prs = NULL;
+		uint64_t trim_prog, trim_rate, trim_start_time, trim_stop_time;
 
 		(void) nvlist_lookup_uint64_array(nvroot,
 		    ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c);
 		print_scan_status(ps);
 
-		(void) nvlist_lookup_uint64_array(nvroot,
-		    ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
-		print_removal_status(zhp, prs);
+		/* Grab trim stats if the pool supports it */
+		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_TRIM_PROG,
+		    &trim_prog) == 0 &&
+		    nvlist_lookup_uint64(config, ZPOOL_CONFIG_TRIM_RATE,
+		    &trim_rate) == 0 &&
+		    nvlist_lookup_uint64(config, ZPOOL_CONFIG_TRIM_START_TIME,
+		    &trim_start_time) == 0 &&
+		    nvlist_lookup_uint64(config, ZPOOL_CONFIG_TRIM_STOP_TIME,
+		    &trim_stop_time) == 0) {
+			/*
+			 * For whatever reason, root vdev_stats_t don't
+			 * include log devices.
+			 */
+			print_trim_status(trim_prog, vs->vs_space +
+			    zpool_slog_space(nvroot), trim_rate,
+			    trim_start_time, trim_stop_time);
+		}
 
 		namewidth = max_width(zhp, nvroot, 0, 0);
 		if (namewidth < 10)
@@ -4668,6 +5025,8 @@
 		print_status_config(zhp, zpool_get_name(zhp), nvroot,
 		    namewidth, 0, B_FALSE);
 
+		if (num_special(nvroot) > 0)
+			print_special(zhp, nvroot, namewidth, B_TRUE);
 		if (num_logs(nvroot) > 0)
 			print_logs(zhp, nvroot, namewidth, B_TRUE);
 		if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
@@ -4847,10 +5206,19 @@
 
 	count = 0;
 	for (i = 0; i < SPA_FEATURES; i++) {
-		const char *fname = spa_feature_table[i].fi_uname;
-		const char *fguid = spa_feature_table[i].fi_guid;
+		zfeature_info_t *finfo = &spa_feature_table[i];
+		const char *fname = finfo->fi_uname;
+		const char *fguid = finfo->fi_guid;
 		if (!nvlist_exists(enabled, fguid)) {
 			char *propname;
+
+			/*
+			 * SPA_FEATURE_WBC can be enabled only
+			 * if 'special' vdev available
+			 */
+			if (finfo->fi_feature == SPA_FEATURE_WBC)
+				continue;
+
 			verify(-1 != asprintf(&propname, "feature@%s", fname));
 			ret = zpool_set_prop(zhp, propname,
 			    ZFS_FEATURE_ENABLED);
@@ -5608,8 +5976,7 @@
 }
 
 typedef struct set_cbdata {
-	char *cb_propname;
-	char *cb_value;
+	nvlist_t *cb_nvl;
 	boolean_t cb_any_successful;
 } set_cbdata_t;
 
@@ -5619,8 +5986,7 @@
 	int error;
 	set_cbdata_t *cb = (set_cbdata_t *)data;
 
-	error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
-
+	error = zpool_set_proplist(zhp, cb->cb_nvl);
 	if (!error)
 		cb->cb_any_successful = B_TRUE;
 
@@ -5627,6 +5993,48 @@
 	return (error);
 }
 
+static void
+parse_props(char *propname, nvlist_t **nvl, zfs_type_t prop_type)
+{
+	char *propval;
+	char *delim;
+	int err;
+
+	do {
+		delim = strchr(propname, ',');
+		if (delim != NULL)
+			*delim++ = '\0';
+
+		propval = strchr(propname, '=');
+		if (propval == NULL) {
+			(void) fprintf(stderr, gettext("missing value "
+			    "in property=value argument\n"));
+			if (*nvl != NULL)
+				nvlist_free(*nvl);
+			usage(B_FALSE);
+		}
+		*propval++ = '\0';
+		switch (prop_type) {
+		case ZFS_TYPE_VDEV:
+			err = add_vdev_prop_list(propname, propval, nvl);
+			break;
+		case ZFS_TYPE_COS:
+			err = add_cos_prop_list(propname, propval, nvl);
+			break;
+		default:
+			err = add_prop_list(propname, propval, nvl, B_TRUE);
+		}
+
+		if (err) {
+			if (*nvl != NULL)
+				nvlist_free(*nvl);
+			usage(B_FALSE);
+		}
+
+		propname = delim;
+	} while (delim != NULL);
+}
+
 int
 zpool_do_set(int argc, char **argv)
 {
@@ -5655,24 +6063,466 @@
 		usage(B_FALSE);
 	}
 
-	cb.cb_propname = argv[1];
-	cb.cb_value = strchr(cb.cb_propname, '=');
-	if (cb.cb_value == NULL) {
-		(void) fprintf(stderr, gettext("missing value in "
-		    "property=value argument\n"));
+	parse_props(argv[1], &cb.cb_nvl, ZFS_TYPE_POOL);
+
+	error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
+	    set_callback, &cb);
+
+	return (error);
+}
+
+typedef struct vdev_cbdata {
+	char *vcb_vdev;
+	nvlist_t *vcb_nvl; /* values */
+	boolean_t vcb_any_successful;
+} vdev_cbdata_t;
+
+typedef struct vdev_get_cbdata {
+	zprop_get_cbdata_t vcb_zprop_get_cbdata;
+	char *vcb_vdev;
+	nvlist_t *vcb_nvl; /* values */
+	boolean_t vcb_any_successful;
+} vdev_get_cbdata_t;
+
+static int
+vdev_get_callback(zpool_handle_t *zhp, void *data)
+{
+	int err;
+	boolean_t l2cache, avail_space;
+	vdev_get_cbdata_t *vcb = (vdev_get_cbdata_t *)data;
+	zprop_get_cbdata_t *cb = &vcb->vcb_zprop_get_cbdata;
+	char value[MAXNAMELEN];
+	zprop_list_t *pl;
+
+	(void) zpool_find_vdev(zhp, vcb->vcb_vdev, &avail_space,
+	    &l2cache, NULL, NULL);
+	for (pl = cb->cb_proplist; pl != NULL; pl = pl->pl_next) {
+		if ((err = vdev_get_prop(zhp, vcb->vcb_vdev,
+		    pl->pl_prop, value, sizeof (value))) != 0)
+			return (err);
+
+		/* don't show L2ARC prop for non L2ARC dev, can't set anyway */
+		if (!l2cache && pl->pl_prop == VDEV_PROP_L2ADDDT)
+			continue;
+		vdev_print_one_property(zpool_get_name(zhp), vcb->vcb_vdev, cb,
+		    vdev_prop_to_name(pl->pl_prop), value);
+	}
+
+	return (0);
+}
+
+
+int
+zpool_do_vdev_get(int argc, char **argv)
+{
+	vdev_get_cbdata_t vcb = { 0 };
+	zprop_get_cbdata_t *cb = &vcb.vcb_zprop_get_cbdata;
+	int error;
+
+	if (argc > 1 && argv[1][0] == '-') {
+		(void) fprintf(stderr, gettext("invalid option '%c'\n"),
+		    argv[1][1]);
 		usage(B_FALSE);
 	}
 
-	*(cb.cb_value) = '\0';
-	cb.cb_value++;
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing property name\n"));
+		usage(B_FALSE);
+	}
 
-	error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
-	    set_callback, &cb);
+	if (argc < 3) {
+		(void) fprintf(stderr, gettext("missing pool name\n"));
+		usage(B_FALSE);
+	}
 
+	if (argc < 4) {
+		(void) fprintf(stderr,
+		    gettext("at least one vdev name or guid is required\n"));
+		usage(B_FALSE);
+	}
+
+	cb->cb_first = B_TRUE;
+	cb->cb_sources = ZPROP_SRC_ALL;
+	cb->cb_columns[0] = GET_COL_NAME;
+	cb->cb_columns[1] = GET_COL_SOURCE;
+	cb->cb_columns[2] = GET_COL_PROPERTY;
+	cb->cb_columns[3] = GET_COL_VALUE;
+	cb->cb_type = ZFS_TYPE_VDEV;
+
+	if (vdev_get_proplist(g_zfs, argv[1], &cb->cb_proplist) != 0)
+		usage(B_FALSE);
+
+	vcb.vcb_vdev = argv[3];
+	error = for_each_pool(1, argv + 2, B_TRUE, NULL,
+	    vdev_get_callback, &vcb);
+
+	zprop_free_list(cb->cb_proplist);
+
 	return (error);
 }
 
+int
+vdev_set_callback(zpool_handle_t *zhp, void *data)
+{
+	int error;
+	vdev_cbdata_t *cb = (vdev_cbdata_t *)data;
+
+	error = vdev_set_proplist(zhp, cb->vcb_vdev, cb->vcb_nvl);
+	if (!error)
+		cb->vcb_any_successful = B_TRUE;
+
+	return (error);
+}
+
+int
+zpool_do_vdev_set(int argc, char **argv)
+{
+	vdev_cbdata_t cb = { 0 };
+	int error;
+
+	if (argc > 1 && argv[1][0] == '-') {
+		(void) fprintf(stderr, gettext("invalid option '%c'\n"),
+		    argv[1][1]);
+		usage(B_FALSE);
+	}
+
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing property=value "
+		    "argument\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 3) {
+		(void) fprintf(stderr, gettext("missing pool name\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 4) {
+		(void) fprintf(stderr,
+		    gettext("at least one vdev name or guid is required\n"));
+		usage(B_FALSE);
+	}
+
+	parse_props(argv[1], &cb.vcb_nvl, ZFS_TYPE_VDEV);
+
+	cb.vcb_vdev = argv[3];
+	error = for_each_pool(1, argv + 2, B_TRUE, NULL,
+	    vdev_set_callback, &cb);
+
+	return (error);
+}
+
+typedef struct cos_af_cbdata {
+	char *cb_cos;
+	uint64_t cb_guid;
+	boolean_t cb_alloc;
+	boolean_t cb_any_successful;
+	boolean_t cb_force;
+	nvlist_t *cb_nvl;
+} cos_af_cbdata_t;
+
+int
+cos_alloc_callback(zpool_handle_t *zhp, void *data)
+{
+	int error;
+	cos_af_cbdata_t *cb = (cos_af_cbdata_t *)data;
+
+	if (cb->cb_alloc)
+		error = cos_alloc(zhp, cb->cb_cos, cb->cb_nvl);
+	else
+		error = cos_free(zhp, cb->cb_cos, cb->cb_guid, cb->cb_force);
+	if (!error)
+		cb->cb_any_successful = B_TRUE;
+
+	if (error == ENOTSUP) {
+		(void) fprintf(stderr, gettext("operation failed: "
+		    "CoS feature is disabled.\n"));
+	}
+
+	return (error);
+}
+
+int
+zpool_do_cos_alloc(int argc, char **argv)
+{
+	nvlist_t *nvl;
+	cos_af_cbdata_t cb = { 0 };
+	int error;
+
+	if (!nexenta_meta_enable())
+		return (-1);
+
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing pool name\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 3) {
+		(void) fprintf(stderr,
+		    gettext("at least one cos name or id is required\n"));
+		usage(B_FALSE);
+	}
+
+	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0) {
+		(void) fprintf(stderr,
+		    gettext("internal error: out of memory\n"));
+		return (1);
+	}
+
+	cb.cb_cos = argv[2];
+	cb.cb_alloc = B_TRUE;
+	cb.cb_nvl = nvl;
+	error = for_each_pool(1, argv + 1, B_TRUE, NULL,
+	    cos_alloc_callback, &cb);
+
+	nvlist_free(nvl);
+
+	return (error);
+}
+
+int
+zpool_do_cos_free(int argc, char **argv)
+{
+	cos_af_cbdata_t cb = { 0 };
+	char *endp;
+	int error;
+
+	if (!nexenta_meta_enable())
+		return (-1);
+
+	if (argc > 1 && strncmp(argv[1], "-f", sizeof ("-f")) == 0) {
+		/* -f - force option */
+		cb.cb_force = B_TRUE;
+		argc--;
+		argv++;
+	}
+
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing pool name\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 3) {
+		(void) fprintf(stderr,
+		    gettext("at least one cos name or id is required\n"));
+		usage(B_FALSE);
+	}
+
+	cb.cb_guid = strtoll(argv[2], &endp, 10);
+	if (endp <  argv[2] + strlen(argv[2])) {
+		cb.cb_guid = 0;
+	}
+
+	if (cb.cb_guid == 0)
+		cb.cb_cos = argv[2];
+	else
+		cb.cb_cos = "";
+
+	error = for_each_pool(1, argv + 1, B_TRUE, NULL,
+	    cos_alloc_callback, &cb);
+
+	return (error);
+}
+
+
 static int
+cos_list_callback(zpool_handle_t *zhp, void *data)
+{
+	int err, i;
+	nvlist_t *nvl = (nvlist_t *)data;
+	nvpair_t *nvp;
+
+	if ((err = cos_list(zhp, &nvl)) == 0) {
+		(void) printf("%-*s %s\n", MAXCOSNAMELEN, gettext("COSNAME"),
+		    gettext("COSID"));
+		for (i = 0, nvp = nvlist_next_nvpair(nvl, NULL); nvp != NULL;
+		    nvp = nvlist_next_nvpair(nvl, nvp), i++) {
+			uint64_t cosid;
+			char *cosname = nvpair_name(nvp);
+			if (nvpair_value_uint64(nvp, &cosid) == 0)
+				(void) printf("%-*s %llu\n", MAXCOSNAMELEN,
+				    cosname, cosid);
+		}
+
+		if (!i)
+			(void) printf("%s\n", gettext("<no classes found>"));
+	}
+	return (err);
+}
+
+int
+zpool_do_cos_list(int argc, char **argv)
+{
+	nvlist_t *nvl;
+	int error;
+
+	if (!nexenta_meta_enable())
+		return (-1);
+
+	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0) {
+		(void) fprintf(stderr,
+		    gettext("internal error: out of memory\n"));
+		return (1);
+	}
+
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing pool name\n"));
+		usage(B_FALSE);
+	}
+
+	error = for_each_pool(1, argv+1, B_TRUE, NULL,
+	    cos_list_callback, nvl);
+
+	nvlist_free(nvl);
+
+	return (error);
+
+}
+
+typedef struct cos_cbdata {
+	char *vcb_cos;
+	nvlist_t *vcb_nvl; /* values */
+	boolean_t vcb_any_successful;
+} cos_cbdata_t;
+
+typedef struct cos_get_cbdata {
+	zprop_get_cbdata_t vcb_zprop_get_cbdata;
+	char *vcb_cos;
+	nvlist_t *vcb_nvl; /* values */
+	boolean_t vcb_any_successful;
+} cos_get_cbdata_t;
+
+static int
+cos_get_callback(zpool_handle_t *zhp, void *data)
+{
+	int err;
+	cos_get_cbdata_t *vcb = (cos_get_cbdata_t *)data;
+	zprop_get_cbdata_t *cb = &vcb->vcb_zprop_get_cbdata;
+	char value[MAXNAMELEN];
+	zprop_list_t *pl;
+	nvlist_t *nvl = NULL;
+
+	for (pl = cb->cb_proplist; pl != NULL; pl = pl->pl_next) {
+		if ((err = cos_get_prop(zhp, vcb->vcb_cos,
+		    pl->pl_prop, value, sizeof (value), &nvl)) != 0)
+			return (err);
+
+		cos_print_one_property(zpool_get_name(zhp), vcb->vcb_cos, cb,
+		    cos_prop_to_name(pl->pl_prop), value);
+	}
+
+	return (0);
+}
+
+
+int
+zpool_do_cos_get(int argc, char **argv)
+{
+	cos_get_cbdata_t vcb = { 0 };
+	zprop_get_cbdata_t *cb = &vcb.vcb_zprop_get_cbdata;
+	int error;
+
+	if (!nexenta_meta_enable())
+		return (-1);
+
+	if (argc > 1 && argv[1][0] == '-') {
+		(void) fprintf(stderr, gettext("invalid option '%c'\n"),
+		    argv[1][1]);
+		usage(B_FALSE);
+	}
+
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing property name\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 3) {
+		(void) fprintf(stderr, gettext("missing pool name\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 4) {
+		(void) fprintf(stderr,
+		    gettext("at least one cos name or guid is required\n"));
+		usage(B_FALSE);
+	}
+
+	cb->cb_first = B_TRUE;
+	cb->cb_sources = ZPROP_SRC_ALL;
+	cb->cb_columns[0] = GET_COL_NAME;
+	cb->cb_columns[1] = GET_COL_SOURCE;
+	cb->cb_columns[2] = GET_COL_PROPERTY;
+	cb->cb_columns[3] = GET_COL_VALUE;
+	cb->cb_type = ZFS_TYPE_COS;
+
+	if (cos_get_proplist(g_zfs, argv[1],  &cb->cb_proplist) != 0)
+		usage(B_FALSE);
+
+	vcb.vcb_cos = argv[3];
+	error = for_each_pool(1, argv + 2, B_TRUE, NULL,
+	    cos_get_callback, &vcb);
+
+	zprop_free_list(cb->cb_proplist);
+
+	return (error);
+
+}
+
+int
+cos_set_callback(zpool_handle_t *zhp, void *data)
+{
+	int error;
+	cos_cbdata_t *cb = (cos_cbdata_t *)data;
+
+	error = cos_set_proplist(zhp, cb->vcb_cos, cb->vcb_nvl);
+	if (!error)
+		cb->vcb_any_successful = B_TRUE;
+
+	return (error);
+}
+
+int
+zpool_do_cos_set(int argc, char **argv)
+{
+	cos_cbdata_t cb = { 0 };
+	int error;
+
+	if (!nexenta_meta_enable())
+		return (-1);
+
+	if (argc > 1 && argv[1][0] == '-') {
+		(void) fprintf(stderr, gettext("invalid option '%c'\n"),
+		    argv[1][1]);
+		usage(B_FALSE);
+	}
+
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing property=value "
+		    "argument\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 3) {
+		(void) fprintf(stderr, gettext("missing pool name\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc < 4) {
+		(void) fprintf(stderr,
+		    gettext("at least one cos name or id is required\n"));
+		usage(B_FALSE);
+	}
+
+	parse_props(argv[1], &cb.vcb_nvl, ZFS_TYPE_COS);
+
+	cb.vcb_cos = argv[3];
+	error = for_each_pool(1, argv + 2, B_TRUE, NULL,
+	    cos_set_callback, &cb);
+
+	return (error);
+}
+
+static int
 find_command_idx(char *command, int *idx)
 {
 	int i;
@@ -5726,6 +6576,7 @@
 		usage(B_TRUE);
 
 	zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
+	verify(zpool_stage_history(g_zfs, history_str) == 0);
 
 	/*
 	 * Run the appropriate command.
