OS-6363 system went to dark side of moon for ~467 seconds OS-6404 ARC reclaim should throttle its calls to arc_kmem_reap_now() Reviewed by: Bryan Cantrill <bryan@joyent.com> Reviewed by: Dan McDonald <danmcd@joyent.com>

*** 20,29 ****
--- 20,30 ----
   */
  /*
   * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
   * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
   * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
+  * Copyright (c) 2017, Joyent, Inc.
   */
  
  /*
   * Kernel memory allocator, as described in the following two papers and a
   * statement about the consolidator:
*** 3247,3268 ****
          }
  
  }
  
  /*
!  * Reap (almost) everything right now.
   */
  void
! kmem_cache_reap_now(kmem_cache_t *cp)
  {
          ASSERT(list_link_active(&cp->cache_link));
  
          kmem_depot_ws_zero(cp);
  
          (void) taskq_dispatch(kmem_taskq,
              (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
-         taskq_wait(kmem_taskq);
  }
  
  /*
   * Recompute a cache's magazine size.  The trade-off is that larger magazines
   * provide a higher transfer rate with the depot, while smaller magazines
--- 3248,3285 ----
          }
  
  }
  
  /*
!  * Allow our caller to determine if there are running reaps.
!  *
!  * This call is very conservative and may return B_TRUE even when
!  * reaping activity isn't active. If it returns B_FALSE, then reaping
!  * activity is definitely inactive.
   */
+ boolean_t
+ kmem_cache_reap_active(void)
+ {
+         return (!taskq_empty(kmem_taskq));
+ }
+ 
+ /*
+  * Reap (almost) everything soon.
+  *
+  * Note: this does not wait for the reap-tasks to complete. Caller
+  * should use kmem_cache_reap_active() (above) and/or moderation to
+  * avoid scheduling too many reap-tasks.
+  */
  void
! kmem_cache_reap_soon(kmem_cache_t *cp)
  {
          ASSERT(list_link_active(&cp->cache_link));
  
          kmem_depot_ws_zero(cp);
  
          (void) taskq_dispatch(kmem_taskq,
              (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
  }
  
  /*
   * Recompute a cache's magazine size.  The trade-off is that larger magazines
   * provide a higher transfer rate with the depot, while smaller magazines