--- old/usr/src/lib/libfakekernel/common/kmem.c	Mon Jan 22 17:29:09 2018
+++ new/usr/src/lib/libfakekernel/common/kmem.c	Mon Jan 22 17:29:09 2018
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 #include <sys/kmem.h>
@@ -128,6 +129,18 @@
 	kmem_cbrc_t (*fun)(void *, void *, size_t, void *))
 {
 }
+
+boolean_t
+kmem_cache_reap_active(void)
+{
+	return (B_FALSE);
+}
+
+/* ARGSUSED */
+void
+kmem_cache_reap_soon(kmem_cache_t *kc)
+{
+}
 
 /* ARGSUSED */
 void
--- old/usr/src/lib/libfakekernel/common/mapfile-vers	Mon Jan 22 17:29:09 2018
+++ new/usr/src/lib/libfakekernel/common/mapfile-vers	Mon Jan 22 17:29:09 2018
@@ -11,6 +11,7 @@
 
 #
 # Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
+# Copyright (c) 2017, Joyent, Inc.
 #
 
 #
@@ -85,7 +86,8 @@
 	kmem_cache_create;
 	kmem_cache_destroy;
 	kmem_cache_free;
-	kmem_cache_reap_now;
+	kmem_cache_reap_active;
+	kmem_cache_reap_soon;
 	kmem_cache_set_move;
 	kmem_debugging;
 	kmem_free;
@@ -173,6 +175,7 @@
 	taskq_dispatch;
 	taskq_dispatch_ent;
 	taskq_member;
+	taskq_empty;
 	taskq_wait;
 
 	thread_join;
--- old/usr/src/lib/libfakekernel/common/taskq.c	Mon Jan 22 17:29:11 2018
+++ new/usr/src/lib/libfakekernel/common/taskq.c	Mon Jan 22 17:29:11 2018
@@ -25,6 +25,7 @@
 /*
  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 #include <sys/taskq_impl.h>
@@ -208,6 +209,18 @@
 	mutex_exit(&tq->tq_lock);
 }
 
+boolean_t
+taskq_empty(taskq_t *tq)
+{
+	boolean_t rv;
+
+	mutex_enter(&tq->tq_lock);
+	rv = (tq->tq_task.tqent_next == &tq->tq_task) && (tq->tq_active == 0);
+	mutex_exit(&tq->tq_lock);
+
+	return (rv);
+}
+
 void
 taskq_wait(taskq_t *tq)
 {
--- old/usr/src/lib/libzpool/common/sys/zfs_context.h	Mon Jan 22 17:29:11 2018
+++ new/usr/src/lib/libzpool/common/sys/zfs_context.h	Mon Jan 22 17:29:11 2018
@@ -22,7 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc. All rights reserved.
  */
 
 #ifndef _SYS_ZFS_CONTEXT_H
@@ -333,7 +333,8 @@
 #define	kmem_cache_alloc(_c, _f) umem_cache_alloc(_c, _f)
 #define	kmem_cache_free(_c, _b)	umem_cache_free(_c, _b)
 #define	kmem_debugging()	0
-#define	kmem_cache_reap_now(_c)		/* nothing */
+#define	kmem_cache_reap_active()	(B_FALSE)
+#define	kmem_cache_reap_soon(_c)	/* nothing */
 #define	kmem_cache_set_move(_c, _cb)	/* nothing */
 #define	vmem_qcache_reap(_v)		/* nothing */
 #define	POINTER_INVALIDATE(_pp)		/* nothing */
@@ -391,6 +392,7 @@
 extern void	taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t,
     taskq_ent_t *);
 extern void	taskq_destroy(taskq_t *);
+extern boolean_t taskq_empty(taskq_t *);
 extern void	taskq_wait(taskq_t *);
 extern int	taskq_member(taskq_t *, void *);
 extern void	system_taskq_init(void);
--- old/usr/src/lib/libzpool/common/taskq.c	Mon Jan 22 17:29:12 2018
+++ new/usr/src/lib/libzpool/common/taskq.c	Mon Jan 22 17:29:12 2018
@@ -26,6 +26,7 @@
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
  * Copyright (c) 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 #include <sys/zfs_context.h>
@@ -178,6 +179,18 @@
 	mutex_exit(&tq->tq_lock);
 }
 
+boolean_t
+taskq_empty(taskq_t *tq)
+{
+	boolean_t rv;
+
+	mutex_enter(&tq->tq_lock);
+	rv = (tq->tq_task.tqent_next == &tq->tq_task) && (tq->tq_active == 0);
+	mutex_exit(&tq->tq_lock);
+
+	return (rv);
+}
+
 void
 taskq_wait(taskq_t *tq)
 {
--- old/usr/src/uts/common/fs/zfs/arc.c	Mon Jan 22 17:29:12 2018
+++ new/usr/src/uts/common/fs/zfs/arc.c	Mon Jan 22 17:29:12 2018
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc. All rights reserved.
  * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
  * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
  * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
@@ -301,6 +301,9 @@
 /* number of seconds before growing cache again */
 static int		arc_grow_retry = 60;
 
+/* number of milliseconds before attempting a kmem-cache-reap */
+static int		arc_kmem_cache_reap_retry_ms = 1000;
+
 /* shift of arc_c for calculating overflow limit in arc_get_data_impl */
 int		zfs_arc_overflow_shift = 8;
 
@@ -4047,21 +4050,31 @@
 #endif
 #endif
 
+	/*
+	 * If a kmem reap is already active, don't schedule more.  We must
+	 * check for this because kmem_cache_reap_soon() won't actually
+	 * block on the cache being reaped (this is to prevent callers from
+	 * becoming implicitly blocked by a system-wide kmem reap -- which,
+	 * on a system with many, many full magazines, can take minutes).
+	 */
+	if (kmem_cache_reap_active())
+		return;
+
 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
 		if (zio_buf_cache[i] != prev_cache) {
 			prev_cache = zio_buf_cache[i];
-			kmem_cache_reap_now(zio_buf_cache[i]);
+			kmem_cache_reap_soon(zio_buf_cache[i]);
 		}
 		if (zio_data_buf_cache[i] != prev_data_cache) {
 			prev_data_cache = zio_data_buf_cache[i];
-			kmem_cache_reap_now(zio_data_buf_cache[i]);
+			kmem_cache_reap_soon(zio_data_buf_cache[i]);
 		}
 	}
-	kmem_cache_reap_now(abd_chunk_cache);
-	kmem_cache_reap_now(buf_cache);
-	kmem_cache_reap_now(hdr_full_cache);
-	kmem_cache_reap_now(hdr_l2only_cache);
-	kmem_cache_reap_now(range_seg_cache);
+	kmem_cache_reap_soon(abd_chunk_cache);
+	kmem_cache_reap_soon(buf_cache);
+	kmem_cache_reap_soon(hdr_full_cache);
+	kmem_cache_reap_soon(hdr_l2only_cache);
+	kmem_cache_reap_soon(range_seg_cache);
 
 	if (zio_arena != NULL) {
 		/*
@@ -4093,6 +4106,7 @@
 arc_reclaim_thread(void *unused)
 {
 	hrtime_t		growtime = 0;
+	hrtime_t		kmem_reap_time = 0;
 	callb_cpr_t		cpr;
 
 	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
@@ -4126,7 +4140,7 @@
 
 		int64_t free_memory = arc_available_memory();
 		if (free_memory < 0) {
-
+			hrtime_t curtime = gethrtime();
 			arc_no_grow = B_TRUE;
 			arc_warm = B_TRUE;
 
@@ -4134,9 +4148,20 @@
 			 * Wait at least zfs_grow_retry (default 60) seconds
 			 * before considering growing.
 			 */
-			growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
+			growtime = curtime + SEC2NSEC(arc_grow_retry);
 
-			arc_kmem_reap_now();
+			/*
+			 * Wait at least arc_kmem_cache_reap_retry_ms
+			 * between arc_kmem_reap_now() calls. Without
+			 * this check it is possible to end up in a
+			 * situation where we spend lots of time
+			 * reaping caches, while we're near arc_c_min.
+			 */
+			if (curtime >= kmem_reap_time) {
+				arc_kmem_reap_now();
+				kmem_reap_time = gethrtime() +
+				    MSEC2NSEC(arc_kmem_cache_reap_retry_ms);
+			}
 
 			/*
 			 * If we are still low on memory, shrink the ARC
--- old/usr/src/uts/common/os/kmem.c	Mon Jan 22 17:29:14 2018
+++ new/usr/src/uts/common/os/kmem.c	Mon Jan 22 17:29:14 2018
@@ -22,6 +22,7 @@
  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 /*
@@ -3249,10 +3250,27 @@
 }
 
 /*
- * Reap (almost) everything right now.
+ * Allow our caller to determine if there are running reaps.
+ *
+ * This call is very conservative and may return B_TRUE even when
+ * reaping activity isn't active. If it returns B_FALSE, then reaping
+ * activity is definitely inactive.
  */
+boolean_t
+kmem_cache_reap_active(void)
+{
+	return (!taskq_empty(kmem_taskq));
+}
+
+/*
+ * Reap (almost) everything soon.
+ *
+ * Note: this does not wait for the reap-tasks to complete. Caller
+ * should use kmem_cache_reap_active() (above) and/or moderation to
+ * avoid scheduling too many reap-tasks.
+ */
 void
-kmem_cache_reap_now(kmem_cache_t *cp)
+kmem_cache_reap_soon(kmem_cache_t *cp)
 {
 	ASSERT(list_link_active(&cp->cache_link));
 
@@ -3260,7 +3278,6 @@
 
 	(void) taskq_dispatch(kmem_taskq,
 	    (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
-	taskq_wait(kmem_taskq);
 }
 
 /*
--- old/usr/src/uts/common/os/taskq.c	Mon Jan 22 17:29:15 2018
+++ new/usr/src/uts/common/os/taskq.c	Mon Jan 22 17:29:15 2018
@@ -26,6 +26,7 @@
 /*
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2017 by Delphix. All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 /*
@@ -1319,6 +1320,22 @@
 	mutex_exit(&tq->tq_lock);
 }
 
+/*
+ * Allow our caller to ask if there are tasks pending on the queue.
+ */
+boolean_t
+taskq_empty(taskq_t *tq)
+{
+	boolean_t rv;
+
+	ASSERT3P(tq, !=, curthread->t_taskq);
+	mutex_enter(&tq->tq_lock);
+	rv = (tq->tq_task.tqent_next == &tq->tq_task) && (tq->tq_active == 0);
+	mutex_exit(&tq->tq_lock);
+
+	return (rv);
+}
+
 /*
  * Wait for all pending tasks to complete.
  * Calling taskq_wait from a task will cause deadlock.
--- old/usr/src/uts/common/os/vmem.c	Mon Jan 22 17:29:16 2018
+++ new/usr/src/uts/common/os/vmem.c	Mon Jan 22 17:29:16 2018
@@ -25,7 +25,7 @@
 
 /*
  * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 /*
@@ -1745,7 +1745,7 @@
 	 */
 	for (i = 0; i < VMEM_NQCACHE_MAX; i++)
 		if (vmp->vm_qcache[i])
-			kmem_cache_reap_now(vmp->vm_qcache[i]);
+			kmem_cache_reap_soon(vmp->vm_qcache[i]);
 }
 
 /*
--- old/usr/src/uts/common/sys/kmem.h	Mon Jan 22 17:29:16 2018
+++ new/usr/src/uts/common/sys/kmem.h	Mon Jan 22 17:29:16 2018
@@ -23,6 +23,7 @@
  * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
@@ -128,7 +129,8 @@
 extern void *kmem_cache_alloc(kmem_cache_t *, int);
 extern void kmem_cache_free(kmem_cache_t *, void *);
 extern uint64_t kmem_cache_stat(kmem_cache_t *, char *);
-extern void kmem_cache_reap_now(kmem_cache_t *);
+extern boolean_t kmem_cache_reap_active(void);
+extern void kmem_cache_reap_soon(kmem_cache_t *);
 extern void kmem_cache_move_notify(kmem_cache_t *, void *);
 
 #endif	/* _KERNEL */
--- old/usr/src/uts/common/sys/taskq.h	Mon Jan 22 17:29:17 2018
+++ new/usr/src/uts/common/sys/taskq.h	Mon Jan 22 17:29:17 2018
@@ -23,6 +23,7 @@
  * Use is subject to license terms.
  *
  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 #ifndef	_SYS_TASKQ_H
@@ -80,6 +81,7 @@
 extern void	nulltask(void *);
 extern void	taskq_destroy(taskq_t *);
 extern void	taskq_wait(taskq_t *);
+extern boolean_t taskq_empty(taskq_t *);
 extern void	taskq_suspend(taskq_t *);
 extern int	taskq_suspended(taskq_t *);
 extern void	taskq_resume(taskq_t *);
