--- old/usr/src/cmd/ztest/ztest.c	Mon Aug  5 17:15:44 2019
+++ new/usr/src/cmd/ztest/ztest.c	Mon Aug  5 17:15:44 2019
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
- * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2014 Integros [integros.com]
  * Copyright 2017 Joyent, Inc.
@@ -113,6 +113,7 @@
 #include <sys/refcount.h>
 #include <sys/zfeature.h>
 #include <sys/dsl_userhold.h>
+#include <libzfs.h>
 #include <sys/abd.h>
 #include <stdio.h>
 #include <stdio_ext.h>
@@ -127,6 +128,8 @@
 #include <libnvpair.h>
 #include <libcmdutils.h>
 
+#include <sys/special.h>
+
 static int ztest_fd_data = -1;
 static int ztest_fd_rand = -1;
 
@@ -328,6 +331,8 @@
 ztest_func_t ztest_dmu_snapshot_create_destroy;
 ztest_func_t ztest_dsl_prop_get_set;
 ztest_func_t ztest_spa_prop_get_set;
+ztest_func_t ztest_vdev_prop_get_set;
+ztest_func_t ztest_cos_prop_get_set;
 ztest_func_t ztest_spa_create_destroy;
 ztest_func_t ztest_fault_inject;
 ztest_func_t ztest_ddt_repair;
@@ -342,8 +347,6 @@
 ztest_func_t ztest_split_pool;
 ztest_func_t ztest_reguid;
 ztest_func_t ztest_spa_upgrade;
-ztest_func_t ztest_device_removal;
-ztest_func_t ztest_remap_blocks;
 
 uint64_t zopt_always = 0ULL * NANOSEC;		/* all the time */
 uint64_t zopt_incessant = 1ULL * NANOSEC / 10;	/* every 1/10 second */
@@ -365,6 +368,8 @@
 	{ ztest_dmu_objset_create_destroy,	1,	&zopt_often	},
 	{ ztest_dsl_prop_get_set,		1,	&zopt_often	},
 	{ ztest_spa_prop_get_set,		1,	&zopt_sometimes	},
+	{ ztest_vdev_prop_get_set,		1,	&zopt_often	},
+	{ ztest_cos_prop_get_set,		1,	&zopt_often	},
 #if 0
 	{ ztest_dmu_prealloc,			1,	&zopt_sometimes	},
 #endif
@@ -385,8 +390,6 @@
 	    &ztest_opts.zo_vdevtime				},
 	{ ztest_vdev_aux_add_remove,		1,
 	    &ztest_opts.zo_vdevtime				},
-	{ ztest_device_removal,			1,	&zopt_sometimes	},
-	{ ztest_remap_blocks,			1,	&zopt_sometimes }
 };
 
 #define	ZTEST_FUNCS	(sizeof (ztest_info) / sizeof (ztest_info_t))
@@ -434,6 +437,12 @@
 static kmutex_t ztest_vdev_lock;
 
 /*
+ * Make sure the "set/get/test" test does not interfere with other
+ * concurrent tests on the same vdev/cos property
+ */
+static kmutex_t ztest_props_lock;
+
+/*
  * The ztest_name_lock protects the pool and dataset namespace used by
  * the individual tests. To modify the namespace, consumers must grab
  * this lock as writer. Grabbing the lock as reader will ensure that the
@@ -789,10 +798,10 @@
 
 	/*
 	 * Before we kill off ztest, make sure that the config is updated.
-	 * See comment above spa_write_cachefile().
+	 * See comment above spa_config_sync().
 	 */
 	mutex_enter(&spa_namespace_lock);
-	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE);
+	spa_config_sync(ztest_spa, B_FALSE, B_FALSE);
 	mutex_exit(&spa_namespace_lock);
 
 	zfs_dbgmsg_print(FTAG);
@@ -831,7 +840,8 @@
 }
 
 static nvlist_t *
-make_vdev_file(char *path, char *aux, char *pool, size_t size, uint64_t ashift)
+make_vdev_file(char *path, char *aux, char *pool, size_t size,
+    uint64_t ashift, boolean_t is_special)
 {
 	char pathbuf[MAXPATHLEN];
 	uint64_t vdev;
@@ -862,7 +872,7 @@
 		if (fd == -1)
 			fatal(1, "can't open %s", path);
 		if (ftruncate(fd, size) != 0)
-			fatal(1, "can't ftruncate %s", path);
+			fatal(1, "can't ftruncate %s to %lld", path, size);
 		(void) close(fd);
 	}
 
@@ -870,23 +880,26 @@
 	VERIFY(nvlist_add_string(file, ZPOOL_CONFIG_TYPE, VDEV_TYPE_FILE) == 0);
 	VERIFY(nvlist_add_string(file, ZPOOL_CONFIG_PATH, path) == 0);
 	VERIFY(nvlist_add_uint64(file, ZPOOL_CONFIG_ASHIFT, ashift) == 0);
-
+	VERIFY(nvlist_add_uint64(file, ZPOOL_CONFIG_IS_SPECIAL, is_special)
+	    == 0);
 	return (file);
 }
 
 static nvlist_t *
 make_vdev_raidz(char *path, char *aux, char *pool, size_t size,
-    uint64_t ashift, int r)
+    uint64_t ashift, int r, boolean_t is_special)
 {
 	nvlist_t *raidz, **child;
 	int c;
 
 	if (r < 2)
-		return (make_vdev_file(path, aux, pool, size, ashift));
+		return (make_vdev_file(path, aux, pool, size, ashift,
+		    is_special));
 	child = umem_alloc(r * sizeof (nvlist_t *), UMEM_NOFAIL);
 
 	for (c = 0; c < r; c++)
-		child[c] = make_vdev_file(path, aux, pool, size, ashift);
+		child[c] = make_vdev_file(path, aux, pool, size, ashift,
+		    B_FALSE);
 
 	VERIFY(nvlist_alloc(&raidz, NV_UNIQUE_NAME, 0) == 0);
 	VERIFY(nvlist_add_string(raidz, ZPOOL_CONFIG_TYPE,
@@ -906,18 +919,20 @@
 
 static nvlist_t *
 make_vdev_mirror(char *path, char *aux, char *pool, size_t size,
-    uint64_t ashift, int r, int m)
+    uint64_t ashift, int r, int m, boolean_t is_special)
 {
 	nvlist_t *mirror, **child;
 	int c;
 
 	if (m < 1)
-		return (make_vdev_raidz(path, aux, pool, size, ashift, r));
+		return (make_vdev_raidz(path, aux, pool, size, ashift, r,
+		    is_special));
 
 	child = umem_alloc(m * sizeof (nvlist_t *), UMEM_NOFAIL);
 
 	for (c = 0; c < m; c++)
-		child[c] = make_vdev_raidz(path, aux, pool, size, ashift, r);
+		child[c] = make_vdev_raidz(path, aux, pool, size, ashift,
+		    r, B_FALSE);
 
 	VERIFY(nvlist_alloc(&mirror, NV_UNIQUE_NAME, 0) == 0);
 	VERIFY(nvlist_add_string(mirror, ZPOOL_CONFIG_TYPE,
@@ -924,6 +939,8 @@
 	    VDEV_TYPE_MIRROR) == 0);
 	VERIFY(nvlist_add_nvlist_array(mirror, ZPOOL_CONFIG_CHILDREN,
 	    child, m) == 0);
+	VERIFY(nvlist_add_uint64(mirror, ZPOOL_CONFIG_IS_SPECIAL, is_special)
+	    == 0);
 
 	for (c = 0; c < m; c++)
 		nvlist_free(child[c]);
@@ -935,7 +952,7 @@
 
 static nvlist_t *
 make_vdev_root(char *path, char *aux, char *pool, size_t size, uint64_t ashift,
-    int log, int r, int m, int t)
+    int log, int r, int m, int t, boolean_t special)
 {
 	nvlist_t *root, **child;
 	int c;
@@ -946,7 +963,7 @@
 
 	for (c = 0; c < t; c++) {
 		child[c] = make_vdev_mirror(path, aux, pool, size, ashift,
-		    r, m);
+		    r, m, special);
 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 		    log) == 0);
 	}
@@ -965,6 +982,62 @@
 }
 
 /*
+ * Add special top-level vdev(s) to the vdev tree
+ */
+static void
+add_special_vdevs(nvlist_t *root, size_t size, int r, int m, int t)
+{
+	nvlist_t **child = NULL, **prev_child = NULL, **new_child = NULL;
+	int c = 0, new = 0;
+	unsigned int prev = 0;
+
+	if ((m == 0) || (t == 0))
+		return;
+
+	child = umem_alloc(t * sizeof (nvlist_t *), UMEM_NOFAIL);
+
+	/*
+	 * special flag that is added to the top-level vdevs
+	 */
+	for (c = 0; c < t; c++) {
+		child[c] = make_vdev_mirror(NULL, NULL, NULL, size, 0, r, m,
+		    B_TRUE);
+	}
+
+	/*
+	 * Extend the children's array in the root"
+	 *  - get previously added children
+	 *  - allocate new array
+	 *  - and copy the previous and new children there
+	 *  - replace the children nvlist adday with the new one
+	 */
+	VERIFY(nvlist_lookup_nvlist_array(root, ZPOOL_CONFIG_CHILDREN,
+	    &prev_child, &prev) == 0);
+
+	new = prev + t;
+
+	new_child = umem_alloc(new * sizeof (nvlist_t *),
+	    UMEM_NOFAIL);
+	for (c = 0; c < prev; c++) {
+		VERIFY(nvlist_dup(prev_child[c], &new_child[c], 0) == 0);
+	}
+	for (; c < new; c++) {
+		new_child[c] = child[c-prev];
+	}
+
+	VERIFY(nvlist_add_nvlist_array(root, ZPOOL_CONFIG_CHILDREN,
+	    new_child, new) == 0);
+
+	/* free children */
+	for (c = 0; c < new; c++) {
+		nvlist_free(new_child[c]);
+	}
+	umem_free(child, t * sizeof (nvlist_t *));
+
+	umem_free(new_child, new * sizeof (nvlist_t *));
+}
+
+/*
  * Find a random spa version. Returns back a random spa version in the
  * range [initial_version, SPA_VERSION_FEATURES].
  */
@@ -1019,7 +1092,7 @@
 	do {
 		top = ztest_random(rvd->vdev_children);
 		tvd = rvd->vdev_child[top];
-	} while (!vdev_is_concrete(tvd) || (tvd->vdev_islog && !log_ok) ||
+	} while (tvd->vdev_ishole || (tvd->vdev_islog && !log_ok) ||
 	    tvd->vdev_mg == NULL || tvd->vdev_mg->mg_class == NULL);
 
 	return (top);
@@ -1834,6 +1907,7 @@
  * ZIL get_data callbacks
  */
 
+/* ARGSUSED */
 static void
 ztest_get_done(zgd_t *zgd, int error)
 {
@@ -1846,9 +1920,6 @@
 	ztest_range_unlock(zgd->zgd_rl);
 	ztest_object_unlock(zd, object);
 
-	if (error == 0 && zgd->zgd_bp)
-		zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
-
 	umem_free(zgd, sizeof (*zgd));
 }
 
@@ -2376,7 +2447,8 @@
 	/*
 	 * Attempt to create using a bad file.
 	 */
-	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, 0, 0, 0, 1);
+	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, 0, 0, 0, 1,
+	    B_FALSE);
 	VERIFY3U(ENOENT, ==,
 	    spa_create("ztest_bad_file", nvroot, NULL, NULL));
 	nvlist_free(nvroot);
@@ -2384,7 +2456,8 @@
 	/*
 	 * Attempt to create using a bad mirror.
 	 */
-	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, 0, 0, 2, 1);
+	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, 0, 0, 2, 1,
+	    B_FALSE);
 	VERIFY3U(ENOENT, ==,
 	    spa_create("ztest_bad_mirror", nvroot, NULL, NULL));
 	nvlist_free(nvroot);
@@ -2394,7 +2467,8 @@
 	 * what's in the nvroot; we should fail with EEXIST.
 	 */
 	rw_enter(&ztest_name_lock, RW_READER);
-	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, 0, 0, 0, 1);
+	nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, 0, 0, 0, 1,
+	    B_FALSE);
 	VERIFY3U(EEXIST, ==, spa_create(zo->zo_pool, nvroot, NULL, NULL));
 	nvlist_free(nvroot);
 	VERIFY3U(0, ==, spa_open(zo->zo_pool, &spa, FTAG));
@@ -2423,7 +2497,7 @@
 	(void) spa_destroy(name);
 
 	nvroot = make_vdev_root(NULL, NULL, name, ztest_opts.zo_vdev_size, 0,
-	    0, ztest_opts.zo_raidz, ztest_opts.zo_mirrors, 1);
+	    0, ztest_opts.zo_raidz, ztest_opts.zo_mirrors, 1, B_FALSE);
 
 	/*
 	 * If we're configuring a RAIDZ device then make sure that the
@@ -2567,7 +2641,7 @@
 		nvroot = make_vdev_root(NULL, NULL, NULL,
 		    ztest_opts.zo_vdev_size, 0,
 		    ztest_random(4) == 0, ztest_opts.zo_raidz,
-		    zs->zs_mirrors, 1);
+		    zs->zs_mirrors, 1, B_FALSE);
 
 		error = spa_vdev_add(spa, nvroot);
 		nvlist_free(nvroot);
@@ -2642,7 +2716,7 @@
 		 * Add a new device.
 		 */
 		nvlist_t *nvroot = make_vdev_root(NULL, aux, NULL,
-		    (ztest_opts.zo_vdev_size * 5) / 4, 0, 0, 0, 0, 1);
+		    (ztest_opts.zo_vdev_size * 5) / 4, 0, 0, 0, 0, 1, B_FALSE);
 		error = spa_vdev_add(spa, nvroot);
 		if (error != 0)
 			fatal(0, "spa_vdev_add(%p) = %d", nvroot, error);
@@ -2788,21 +2862,9 @@
 	mutex_enter(&ztest_vdev_lock);
 	leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raidz;
 
-	spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
+	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
 
 	/*
-	 * If a vdev is in the process of being removed, its removal may
-	 * finish while we are in progress, leading to an unexpected error
-	 * value.  Don't bother trying to attach while we are in the middle
-	 * of removal.
-	 */
-	if (spa->spa_vdev_removal != NULL) {
-		spa_config_exit(spa, SCL_ALL, FTAG);
-		mutex_exit(&ztest_vdev_lock);
-		return;
-	}
-
-	/*
 	 * Decide whether to do an attach or a replace.
 	 */
 	replacing = ztest_random(2);
@@ -2853,7 +2915,7 @@
 	 * If oldvd has siblings, then half of the time, detach it.
 	 */
 	if (oldvd_has_siblings && ztest_random(2) == 0) {
-		spa_config_exit(spa, SCL_ALL, FTAG);
+		spa_config_exit(spa, SCL_VDEV, FTAG);
 		error = spa_vdev_detach(spa, oldguid, pguid, B_FALSE);
 		if (error != 0 && error != ENODEV && error != EBUSY &&
 		    error != ENOTSUP)
@@ -2880,10 +2942,6 @@
 	}
 
 	if (newvd) {
-		/*
-		 * Reopen to ensure the vdev's asize field isn't stale.
-		 */
-		vdev_reopen(newvd);
 		newsize = vdev_get_min_asize(newvd);
 	} else {
 		/*
@@ -2921,13 +2979,13 @@
 	else
 		expected_error = 0;
 
-	spa_config_exit(spa, SCL_ALL, FTAG);
+	spa_config_exit(spa, SCL_VDEV, FTAG);
 
 	/*
 	 * Build the nvlist describing newpath.
 	 */
 	root = make_vdev_root(newpath, NULL, NULL, newvd == NULL ? newsize : 0,
-	    ashift, 0, 0, 0, 1);
+	    ashift, 0, 0, 0, 1, replacing ? oldvd->vdev_isspecial : B_FALSE);
 
 	error = spa_vdev_attach(spa, oldguid, root, replacing);
 
@@ -2959,26 +3017,6 @@
 	mutex_exit(&ztest_vdev_lock);
 }
 
-/* ARGSUSED */
-void
-ztest_device_removal(ztest_ds_t *zd, uint64_t id)
-{
-	spa_t *spa = ztest_spa;
-	vdev_t *vd;
-	uint64_t guid;
-
-	mutex_enter(&ztest_vdev_lock);
-
-	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
-	vd = vdev_lookup_top(spa, ztest_random_vdev_top(spa, B_FALSE));
-	guid = vd->vdev_guid;
-	spa_config_exit(spa, SCL_VDEV, FTAG);
-
-	(void) spa_vdev_remove(spa, guid, B_FALSE);
-
-	mutex_exit(&ztest_vdev_lock);
-}
-
 /*
  * Callback function which expands the physical size of the vdev.
  */
@@ -3065,6 +3103,28 @@
 }
 
 /*
+ * Callback function which checks that the given vdev is
+ * - not a part of replacing group
+ * - not being removed
+ * - healthy
+ */
+/* ARGSUSED */
+vdev_t *
+check_valid_vdev(vdev_t *vd, void *arg)
+{
+	spa_t *spa = vd->vdev_spa;
+
+	ASSERT(spa_config_held(spa, SCL_STATE, RW_READER) == SCL_STATE);
+	ASSERT(vd->vdev_ops->vdev_op_leaf);
+
+	if (vd->vdev_parent->vdev_ops == &vdev_replacing_ops ||
+	    vd->vdev_removing || vd->vdev_state != VDEV_STATE_HEALTHY)
+		return (NULL);
+
+	return (vd);
+}
+
+/*
  * Traverse the vdev tree calling the supplied function.
  * We continue to walk the tree until we either have walked all
  * children or we receive a non-NULL return from the callback.
@@ -3107,18 +3167,6 @@
 	mutex_enter(&ztest_vdev_lock);
 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
 
-	/*
-	 * If there is a vdev removal in progress, it could complete while
-	 * we are running, in which case we would not be able to verify
-	 * that the metaslab_class space increased (because it decreases
-	 * when the device removal completes).
-	 */
-	if (spa->spa_vdev_removal != NULL) {
-		spa_config_exit(spa, SCL_STATE, FTAG);
-		mutex_exit(&ztest_vdev_lock);
-		return;
-	}
-
 	top = ztest_random_vdev_top(spa, B_TRUE);
 
 	tvd = spa->spa_root_vdev->vdev_child[top];
@@ -3210,18 +3258,16 @@
 	/*
 	 * Make sure we were able to grow the vdev.
 	 */
-	if (new_ms_count <= old_ms_count) {
-		fatal(0, "LUN expansion failed: ms_count %llu < %llu\n",
+	if (new_ms_count <= old_ms_count)
+		fatal(0, "LUN expansion failed: ms_count %llu <= %llu\n",
 		    old_ms_count, new_ms_count);
-	}
 
 	/*
 	 * Make sure we were able to grow the pool.
 	 */
-	if (new_class_space <= old_class_space) {
-		fatal(0, "LUN expansion failed: class_space %llu < %llu\n",
+	if (new_class_space <= old_class_space)
+		fatal(0, "LUN expansion failed: class_space %llu <= %llu\n",
 		    old_class_space, new_class_space);
-	}
 
 	if (ztest_opts.zo_verbose >= 5) {
 		char oldnumbuf[NN_NUMBUF_SZ], newnumbuf[NN_NUMBUF_SZ];
@@ -4694,20 +4740,6 @@
 
 /* ARGSUSED */
 void
-ztest_remap_blocks(ztest_ds_t *zd, uint64_t id)
-{
-	rw_enter(&ztest_name_lock, RW_READER);
-
-	int error = dmu_objset_remap_indirects(zd->zd_name);
-	if (error == ENOSPC)
-		error = 0;
-	ASSERT0(error);
-
-	rw_exit(&ztest_name_lock);
-}
-
-/* ARGSUSED */
-void
 ztest_spa_prop_get_set(ztest_ds_t *zd, uint64_t id)
 {
 	nvlist_t *props = NULL;
@@ -4727,6 +4759,357 @@
 	rw_exit(&ztest_name_lock);
 }
 
+/* vdev and cos property tests */
+typedef enum {
+	VDEV_PROP_UINT64,
+	VDEV_PROP_STRING,
+	COS_PROP_UINT64
+} ztest_prop_t;
+
+/* common functions */
+static vdev_t *
+ztest_get_random_vdev_leaf(spa_t *spa)
+{
+	vdev_t *lvd = NULL, *tvd = NULL;
+	uint64_t top = 0;
+
+	spa_config_enter(spa, SCL_ALL, FTAG, RW_READER);
+
+	for (;;) {
+		/* Pick a leaf of a random top-level vdev */
+		top = ztest_random_vdev_top(spa, B_TRUE);
+		tvd = spa->spa_root_vdev->vdev_child[top];
+		lvd = vdev_walk_tree(tvd, check_valid_vdev, NULL);
+		if (lvd == NULL) {
+			/*
+			 * We cannot  return NULL and no reasons to crash.
+			 * Just let other threads to finish their work and
+			 * maybe next time we will have leaf-vdev
+			 */
+			spa_config_exit(spa, SCL_ALL, FTAG);
+			(void) poll(NULL, 0, 100);
+			spa_config_enter(spa, SCL_ALL, FTAG, RW_READER);
+			continue;
+		}
+
+		ASSERT(lvd->vdev_ops->vdev_op_leaf);
+		break;
+	}
+
+	spa_config_exit(spa, SCL_ALL, FTAG);
+
+	return (lvd);
+}
+
+#define	ZTEST_COS_NAME		"ztest_cos"
+
+/*ARGSUSED*/
+static nvlist_t *
+ztest_props_set(const vdev_t *lvd, const char *name, const ztest_prop_t t,
+    const void *props, const size_t size)
+{
+	spa_t *spa = ztest_spa;
+	nvlist_t *sprops;
+	int error = 0;
+
+	VERIFY(0 == nvlist_alloc(&sprops, NV_UNIQUE_NAME, 0));
+
+	for (int p = 0; p < size; p++) {
+		uint64_t ival;
+		char sval[16];
+		const char *pname =
+		    (t == VDEV_PROP_UINT64 || t == VDEV_PROP_STRING) ?
+		    vdev_prop_to_name(((vdev_prop_t *)props)[p]) :
+		    cos_prop_to_name(((cos_prop_t *)props)[p]);
+
+		switch (t) {
+		case VDEV_PROP_UINT64:
+		case COS_PROP_UINT64:
+			/* range 0...10 is valid for all properties */
+			ival = ztest_random(10) + 1;
+			VERIFY(0 == nvlist_add_uint64(sprops, pname, ival));
+			break;
+		case VDEV_PROP_STRING:
+			/* use a well known name for cos property */
+			if (((vdev_prop_t *)props)[p] == VDEV_PROP_COS) {
+				(void) snprintf(sval, 15, "%s", ZTEST_COS_NAME);
+			} else {
+				/* any short string will do */
+				(void) snprintf(sval, 15, "prop_value%d", p);
+			}
+			VERIFY(0 == nvlist_add_string(sprops, pname, sval));
+			break;
+		default:
+			/* unknown property */
+			error = EINVAL;
+			break;
+		}
+	}
+	VERIFY3U(0, ==, error);
+
+	/* set the props */
+	switch (t) {
+	case VDEV_PROP_UINT64:
+	case VDEV_PROP_STRING:
+		error = spa_vdev_prop_set(spa, lvd->vdev_guid, sprops);
+		break;
+	case COS_PROP_UINT64:
+		error = spa_cos_prop_set(spa, name, sprops);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+	if (error == ENOSPC) {
+		ztest_record_enospc(FTAG);
+		nvlist_free(sprops);
+		return (NULL);
+	}
+	ASSERT0(error);
+	return (sprops);
+}
+
+static nvlist_t *
+ztest_props_get(const vdev_t *lvd, const char *name)
+{
+	spa_t *spa = ztest_spa;
+	nvlist_t *gprops = NULL;
+	int error = 0;
+
+	if (lvd)
+		error = spa_vdev_prop_get(spa, lvd->vdev_guid, &gprops);
+	else
+		error = spa_cos_prop_get(spa, name, &gprops);
+	if (error == ENOSPC) {
+		ztest_record_enospc(FTAG);
+		return (NULL);
+	}
+	ASSERT0(error);
+	return (gprops);
+}
+
+static void
+ztest_props_test(const ztest_prop_t t, const void *props, const size_t size,
+    nvlist_t *sprops, nvlist_t *gprops)
+{
+	for (int p = 0; p < size; p++) {
+		const char *pname =
+		    (t == VDEV_PROP_UINT64 || t == VDEV_PROP_STRING) ?
+		    vdev_prop_to_name(((vdev_prop_t *)props)[p]) :
+		    cos_prop_to_name(((cos_prop_t *)props)[p]);
+
+		switch (t) {
+		case VDEV_PROP_UINT64:
+		case COS_PROP_UINT64:
+		{
+			uint64_t sival, gival;
+			VERIFY3U(0, ==, nvlist_lookup_uint64(sprops, pname,
+			    &sival));
+			VERIFY3U(0, ==, nvlist_lookup_uint64(gprops, pname,
+			    &gival));
+			VERIFY3U(gival, ==, sival);
+		}
+		break;
+		case VDEV_PROP_STRING:
+		{
+			char *ssval, *gsval;
+			VERIFY3U(0, ==, nvlist_lookup_string(sprops, pname,
+			    &ssval));
+			VERIFY3U(0, ==, nvlist_lookup_string(gprops, pname,
+			    &gsval));
+			VERIFY3U(0, ==, strcmp(ssval, gsval));
+		}
+		break;
+		default:
+			/* unknown property */
+			VERIFY(0);
+			break;
+		}
+	}
+
+	nvlist_free(sprops);
+	nvlist_free(gprops);
+}
+
+static const cos_prop_t cprops_uint64[] = {
+	COS_PROP_READ_MINACTIVE,
+	COS_PROP_AREAD_MINACTIVE,
+	COS_PROP_WRITE_MINACTIVE,
+	COS_PROP_AWRITE_MINACTIVE,
+	COS_PROP_SCRUB_MINACTIVE,
+	COS_PROP_RESILVER_MINACTIVE,
+	COS_PROP_READ_MAXACTIVE,
+	COS_PROP_AREAD_MAXACTIVE,
+	COS_PROP_WRITE_MAXACTIVE,
+	COS_PROP_AWRITE_MAXACTIVE,
+	COS_PROP_SCRUB_MAXACTIVE,
+	COS_PROP_RESILVER_MAXACTIVE,
+	COS_PROP_PREFERRED_READ
+};
+
+/* ARGSUSED */
+void
+ztest_cos_prop_get_set(ztest_ds_t *zd, uint64_t id)
+{
+	spa_t *spa = ztest_spa;
+	nvlist_t *sprops = NULL, *gprops = NULL, *cos_list = NULL;
+	char cos_name[MAXCOSNAMELEN];
+	const char *pname = NULL;
+	char *sval = NULL;
+	uint64_t cos_id = ztest_random(~0ULL), val = 0;
+	vdev_t *lvd = NULL;
+
+	(void) snprintf(cos_name, MAXCOSNAMELEN-1, "cos_%llu", cos_id);
+
+	mutex_enter(&ztest_props_lock);
+
+	VERIFY3U(0, ==, spa_alloc_cos(spa, cos_name, cos_id));
+
+	sprops = ztest_props_set(NULL, cos_name,
+	    COS_PROP_UINT64, (void *)&cprops_uint64[0],
+	    sizeof (cprops_uint64) / sizeof (cprops_uint64[0]));
+	gprops = ztest_props_get(NULL, cos_name);
+	ztest_props_test(COS_PROP_UINT64, (void *)&cprops_uint64[0],
+	    sizeof (cprops_uint64) / sizeof (cprops_uint64[0]),
+	    sprops, gprops);
+
+	VERIFY3U(0, ==, nvlist_alloc(&cos_list, NV_UNIQUE_NAME, 0));
+	VERIFY3U(0, ==, spa_list_cos(spa, cos_list));
+	VERIFY3U(0, ==, nvlist_lookup_uint64(cos_list, cos_name, &val));
+	VERIFY3U(cos_id, ==, val);
+	nvlist_free(cos_list);
+
+	VERIFY3U(0, ==, spa_free_cos(spa, cos_name, B_FALSE));
+	VERIFY3U(0, ==, nvlist_alloc(&cos_list, NV_UNIQUE_NAME, 0));
+	VERIFY3U(0, ==, spa_list_cos(spa, cos_list));
+	VERIFY3U(ENOENT, ==, nvlist_lookup_uint64(cos_list, cos_name, &val));
+	nvlist_free(cos_list);
+
+	/*
+	 * force spa_free_cos() test
+	 * - allocate cos property, set vdev's cos, then free cos forcefuly
+	 * - verify everything succeeds
+	 * - verify no cos property on vdev
+	 * - verify no cos descriptor remains
+	 */
+	VERIFY3U(0, ==, spa_alloc_cos(spa, cos_name, cos_id));
+
+	/* Make sure vdevs will stay in place */
+	mutex_enter(&ztest_vdev_lock);
+
+	lvd = ztest_get_random_vdev_leaf(spa);
+
+	VERIFY(0 == nvlist_alloc(&sprops, NV_UNIQUE_NAME, 0));
+
+	pname = vdev_prop_to_name(VDEV_PROP_COS);
+	VERIFY3U(0, ==, nvlist_add_string(sprops, pname, cos_name));
+	VERIFY3U(0, ==, spa_vdev_prop_set(spa, lvd->vdev_guid, sprops));
+
+	VERIFY3U(0, ==, spa_free_cos(spa, cos_name, B_TRUE));
+
+	VERIFY3U(0, ==, spa_vdev_prop_get(spa, lvd->vdev_guid, &gprops));
+
+	mutex_exit(&ztest_vdev_lock);
+
+	/* verify the vdev cos prop gone */
+	VERIFY3U(ENOENT, ==, nvlist_lookup_string(gprops, cos_name, &sval));
+
+	/* verify the cos descriptor gone */
+	VERIFY3U(0, ==, nvlist_alloc(&cos_list, NV_UNIQUE_NAME, 0));
+	VERIFY3U(0, ==, spa_list_cos(spa, cos_list));
+	VERIFY3U(ENOENT, ==, nvlist_lookup_uint64(cos_list, cos_name, &val));
+
+	mutex_exit(&ztest_props_lock);
+
+	nvlist_free(cos_list);
+}
+
+/* vdev tests */
+static const vdev_prop_t vprops_uint64[] = {
+	VDEV_PROP_READ_MINACTIVE,
+	VDEV_PROP_AREAD_MINACTIVE,
+	VDEV_PROP_WRITE_MINACTIVE,
+	VDEV_PROP_AWRITE_MINACTIVE,
+	VDEV_PROP_SCRUB_MINACTIVE,
+	VDEV_PROP_RESILVER_MINACTIVE,
+	VDEV_PROP_READ_MAXACTIVE,
+	VDEV_PROP_AREAD_MAXACTIVE,
+	VDEV_PROP_WRITE_MAXACTIVE,
+	VDEV_PROP_AWRITE_MAXACTIVE,
+	VDEV_PROP_SCRUB_MAXACTIVE,
+	VDEV_PROP_RESILVER_MAXACTIVE,
+	VDEV_PROP_PREFERRED_READ
+};
+static const vdev_prop_t vprops_string[] = {
+	VDEV_PROP_COS,
+	VDEV_PROP_SPAREGROUP
+};
+
+static void
+ztest_cos_free(spa_t *spa, vdev_t *lvd, const char *name)
+{
+	nvlist_t *sprops = NULL;
+	int error = 0;
+	VERIFY(0 == nvlist_alloc(&sprops, NV_UNIQUE_NAME, 0));
+	VERIFY(0 == nvlist_add_string(sprops,
+	    vdev_prop_to_name(VDEV_PROP_COS), ""));
+	VERIFY3U(0, ==, spa_vdev_prop_set(spa, lvd->vdev_guid, sprops));
+	/*
+	 * this can be called in cleanup code paths when we do not know
+	 * if CoS was allocated
+	 */
+	error = spa_free_cos(spa, name, B_TRUE);
+	if (error)
+		VERIFY3U(error, ==, ENOENT);
+	nvlist_free(sprops);
+}
+
+/* ARGSUSED */
+void
+ztest_vdev_prop_get_set(ztest_ds_t *zd, uint64_t id)
+{
+	spa_t *spa = ztest_spa;
+	nvlist_t *sprops = NULL, *gprops = NULL;
+	vdev_t *lvd = NULL;
+	int error = 0;
+	/* Make sure vdevs will stay in place */
+	mutex_enter(&ztest_props_lock);
+
+	mutex_enter(&ztest_vdev_lock);
+
+	lvd = ztest_get_random_vdev_leaf(spa);
+
+	/* Test uint64 properties */
+	sprops = ztest_props_set(lvd, NULL, VDEV_PROP_UINT64,
+	    (void *)&vprops_uint64[0],
+	    sizeof (vprops_uint64) / sizeof (vprops_uint64[0]));
+	gprops = ztest_props_get(lvd, NULL);
+	ztest_props_test(VDEV_PROP_UINT64, (void *)&vprops_uint64[0],
+	    sizeof (vprops_uint64) / sizeof (vprops_uint64[0]), sprops, gprops);
+
+	/* Test string properties */
+	/* Allocate CoS descriptor to have vdev-set of cos succeed */
+	error = spa_alloc_cos(spa, ZTEST_COS_NAME, 0);
+	if (error)
+		VERIFY3U(error, ==, EEXIST);
+
+	sprops = ztest_props_set(lvd, NULL, VDEV_PROP_STRING,
+	    (void *)&vprops_string[0],
+	    sizeof (vprops_string) / sizeof (vprops_string[0]));
+	gprops = ztest_props_get(lvd, NULL);
+	ztest_props_test(VDEV_PROP_STRING, (void *)&vprops_string[0],
+	    sizeof (vprops_string) / sizeof (vprops_string[0]), sprops, gprops);
+
+	/* Done, free cos to avoid collisions with other tests */
+	ztest_cos_free(spa, lvd, ZTEST_COS_NAME);
+
+	mutex_exit(&ztest_vdev_lock);
+
+	mutex_exit(&ztest_props_lock);
+}
+
+/* end vdev and cos property tests */
+
 static int
 user_release_one(const char *snapname, const char *holdname)
 {
@@ -4957,9 +5340,6 @@
 			 */
 			vdev_file_t *vf = vd0->vdev_tsd;
 
-			zfs_dbgmsg("injecting fault to vdev %llu; maxfaults=%d",
-			    (long long)vd0->vdev_id, (int)maxfaults);
-
 			if (vf != NULL && ztest_random(3) == 0) {
 				(void) close(vf->vf_vnode->v_fd);
 				vf->vf_vnode->v_fd = -1;
@@ -5433,7 +5813,8 @@
 	/*
 	 * Export it.
 	 */
-	VERIFY3U(0, ==, spa_export(oldname, &config, B_FALSE, B_FALSE));
+	VERIFY3U(0, ==, spa_export(oldname, &config, B_FALSE, B_FALSE,
+	    B_FALSE));
 
 	ztest_walk_pool_directory("pools after export");
 
@@ -5752,6 +6133,7 @@
 	 * Initialize parent/child shared state.
 	 */
 	mutex_init(&ztest_vdev_lock, NULL, USYNC_THREAD, NULL);
+	mutex_init(&ztest_props_lock, NULL, USYNC_THREAD, NULL);
 	rw_init(&ztest_name_lock, NULL, USYNC_THREAD, NULL);
 
 	zs->zs_thread_start = gethrtime();
@@ -6041,13 +6423,51 @@
 	nvlist_t *props;
 
 	VERIFY(nvlist_alloc(&props, NV_UNIQUE_NAME, 0) == 0);
-	if (ztest_random(2) == 0)
-		return (props);
-	VERIFY(nvlist_add_uint64(props, "autoreplace", 1) == 0);
+	switch (ztest_random(5)) {
+	case 0:
+		break;
+	case 1:
+		VERIFY(nvlist_add_uint64(props, "autoreplace", 1) == 0);
+		break;
+	case 2:
+		VERIFY(nvlist_add_uint64(props, "enablespecial", 1) == 0);
+		VERIFY(nvlist_add_uint64(props, "small_data_to_metadev", 1) ==
+		    0);
+		break;
+	case 3:
+		VERIFY(nvlist_add_uint64(props, "enablespecial", 1) == 0);
+		VERIFY(nvlist_add_uint64(props, "autoreplace", 1) == 0);
+		break;
+	case 4:
+		VERIFY(nvlist_add_uint64(props, "enablespecial", 1) == 0);
+		VERIFY(nvlist_add_uint64(props, "meta_placement", 1) == 0);
+		VERIFY(nvlist_add_uint64(props, "zfs_meta_to_metadev", 1) == 0);
+		break;
+	}
 
 	return (props);
 }
 
+static void
+set_random_ds_props(char *dsname)
+{
+	uint64_t value = META_PLACEMENT_OFF;
+
+	switch (ztest_random(3)) {
+	case 0:
+		break;
+	case 1:
+		value = META_PLACEMENT_ON;
+		break;
+	case 2:
+		value = META_PLACEMENT_DUAL;
+		break;
+	}
+
+	VERIFY(ztest_dsl_prop_set_uint64(dsname, ZFS_PROP_ZPL_META_TO_METADEV,
+	    value, B_TRUE) == 0);
+}
+
 /*
  * Create a storage pool with the given name and initial vdev size.
  * Then test spa_freeze() functionality.
@@ -6071,7 +6491,13 @@
 	zs->zs_splits = 0;
 	zs->zs_mirrors = ztest_opts.zo_mirrors;
 	nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
-	    0, ztest_opts.zo_raidz, zs->zs_mirrors, 1);
+	    0, ztest_opts.zo_raidz, zs->zs_mirrors, 1, B_FALSE);
+	/*
+	 * Add special vdevs
+	 */
+	add_special_vdevs(nvroot, ztest_opts.zo_vdev_size, ztest_opts.zo_raidz,
+	    zs->zs_mirrors, 1);
+
 	props = make_random_props();
 	for (int i = 0; i < SPA_FEATURES; i++) {
 		char buf[1024];
@@ -6087,6 +6513,9 @@
 	zs->zs_metaslab_sz =
 	    1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
 
+	/* set props on the root dataset */
+	set_random_ds_props(ztest_opts.zo_pool);
+
 	spa_close(spa, FTAG);
 
 	kernel_fini();
@@ -6098,6 +6527,7 @@
 	ztest_run_zdb(ztest_opts.zo_pool);
 
 	rw_destroy(&ztest_name_lock);
+	mutex_destroy(&ztest_props_lock);
 	mutex_destroy(&ztest_vdev_lock);
 }
 
