Print this page
8493 kmem_move taskq appears to be inducing significant system latency
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Dan McDonald <danmcd@joyent.com>

@@ -157,14 +157,26 @@
  *               wherever it expects to find known objects
  *            b) the client has removed the object from wherever it expects to
  *               find known objects and is about to free it, or
  *            c) the client has freed the object.
  *            In all these cases (a, b, and c) kmem frees the new object (the
- *            unused copy destination) and searches for the old object in the
- *            magazine layer. If found, the object is removed from the magazine
- *            layer and freed to the slab layer so it will no longer hold the
- *            slab hostage.
+ *            unused copy destination).  In the first case, the object is in
+ *            use and the correct action is that for LATER; in the latter two
+ *            cases, we know that the object is either freed or about to be
+ *            freed, in which case it is either already in a magazine or about
+ *            to be in one.  In these cases, we know that the object will either
+ *            be reallocated and reused, or it will end up in a full magazine
+ *            that will be reaped (thereby liberating the slab).  Because it
+ *            is prohibitively expensive to differentiate these cases, and
+ *            because the defrag code is executed when we're low on memory
+ *            (thereby biasing the system to reclaim full magazines) we treat
+ *            all DONT_KNOW cases as LATER and rely on cache reaping to
+ *            generally clean up full magazines.  While we take the same action
+ *            for these cases, we maintain their semantic distinction:  if
+ *            defragmentation is not occurring, it is useful to know if this
+ *            is due to objects in use (LATER) or objects in an unknown state
+ *            of transition (DONT_KNOW).
  *
  * 2.3 Object States
  *
  * Neither kmem nor the client can be assumed to know the object's whereabouts
  * at the time of the callback. An object belonging to a kmem cache may be in

@@ -283,14 +295,14 @@
  * be reused on a subsequent allocation without the overhead of calling the
  * constructor. While in the magazine it appears allocated from the point of
  * view of the slab layer, making it a candidate for the move callback. Most
  * objects unrecognized by the client in the move callback fall into this
  * category and are cheaply distinguished from known objects by the test
- * described earlier. Since recognition is cheap for the client, and searching
- * magazines is expensive for kmem, kmem defers searching until the client first
- * returns KMEM_CBRC_DONT_KNOW. As long as the needed effort is reasonable, kmem
- * elsewhere does what it can to avoid bothering the client unnecessarily.
+ * described earlier. Because searching magazines is prohibitively expensive
+ * for kmem, clients that do not mark freed objects (and therefore return
+ * KMEM_CBRC_DONT_KNOW for large numbers of objects) may find defragmentation
+ * efficacy reduced.
  *
  * Invalidating the designated pointer member before freeing the object marks
  * the object to be avoided in the callback, and conversely, assigning a valid
  * value to the designated pointer member after allocating the object makes the
  * object fair game for the callback:

@@ -1037,26 +1049,10 @@
 static vmem_t           *kmem_default_arena;
 static vmem_t           *kmem_firewall_va_arena;
 static vmem_t           *kmem_firewall_arena;
 
 /*
- * Define KMEM_STATS to turn on statistic gathering. By default, it is only
- * turned on when DEBUG is also defined.
- */
-#ifdef  DEBUG
-#define KMEM_STATS
-#endif  /* DEBUG */
-
-#ifdef  KMEM_STATS
-#define KMEM_STAT_ADD(stat)                     ((stat)++)
-#define KMEM_STAT_COND_ADD(cond, stat)          ((void) (!(cond) || (stat)++))
-#else
-#define KMEM_STAT_ADD(stat)                     /* nothing */
-#define KMEM_STAT_COND_ADD(cond, stat)          /* nothing */
-#endif  /* KMEM_STATS */
-
-/*
  * kmem slab consolidator thresholds (tunables)
  */
 size_t kmem_frag_minslabs = 101;        /* minimum total slabs */
 size_t kmem_frag_numer = 1;             /* free buffers (numerator) */
 size_t kmem_frag_denom = KMEM_VOID_FRACTION; /* buffers (denominator) */

@@ -1069,51 +1065,10 @@
  * Number of slabs to scan backwards from the end of the partial slab list
  * when searching for buffers to relocate.
  */
 size_t kmem_reclaim_scan_range = 12;
 
-#ifdef  KMEM_STATS
-static struct {
-        uint64_t kms_callbacks;
-        uint64_t kms_yes;
-        uint64_t kms_no;
-        uint64_t kms_later;
-        uint64_t kms_dont_need;
-        uint64_t kms_dont_know;
-        uint64_t kms_hunt_found_mag;
-        uint64_t kms_hunt_found_slab;
-        uint64_t kms_hunt_alloc_fail;
-        uint64_t kms_hunt_lucky;
-        uint64_t kms_notify;
-        uint64_t kms_notify_callbacks;
-        uint64_t kms_disbelief;
-        uint64_t kms_already_pending;
-        uint64_t kms_callback_alloc_fail;
-        uint64_t kms_callback_taskq_fail;
-        uint64_t kms_endscan_slab_dead;
-        uint64_t kms_endscan_slab_destroyed;
-        uint64_t kms_endscan_nomem;
-        uint64_t kms_endscan_refcnt_changed;
-        uint64_t kms_endscan_nomove_changed;
-        uint64_t kms_endscan_freelist;
-        uint64_t kms_avl_update;
-        uint64_t kms_avl_noupdate;
-        uint64_t kms_no_longer_reclaimable;
-        uint64_t kms_notify_no_longer_reclaimable;
-        uint64_t kms_notify_slab_dead;
-        uint64_t kms_notify_slab_destroyed;
-        uint64_t kms_alloc_fail;
-        uint64_t kms_constructor_fail;
-        uint64_t kms_dead_slabs_freed;
-        uint64_t kms_defrags;
-        uint64_t kms_scans;
-        uint64_t kms_scan_depot_ws_reaps;
-        uint64_t kms_debug_reaps;
-        uint64_t kms_debug_scans;
-} kmem_move_stats;
-#endif  /* KMEM_STATS */
-
 /* consolidator knobs */
 boolean_t kmem_move_noreap;
 boolean_t kmem_move_blocked;
 boolean_t kmem_move_fulltilt;
 boolean_t kmem_move_any_partial;

@@ -1920,19 +1875,11 @@
                 ASSERT(sp->slab_chunks > 1);
                 list_remove(&cp->cache_complete_slabs, sp);
                 cp->cache_complete_slab_count--;
                 avl_add(&cp->cache_partial_slabs, sp);
         } else {
-#ifdef  DEBUG
-                if (avl_update_gt(&cp->cache_partial_slabs, sp)) {
-                        KMEM_STAT_ADD(kmem_move_stats.kms_avl_update);
-                } else {
-                        KMEM_STAT_ADD(kmem_move_stats.kms_avl_noupdate);
-                }
-#else
                 (void) avl_update_gt(&cp->cache_partial_slabs, sp);
-#endif
         }
 
         ASSERT((cp->cache_slab_create - cp->cache_slab_destroy) ==
             (cp->cache_complete_slab_count +
             avl_numnodes(&cp->cache_partial_slabs) +

@@ -3577,11 +3524,11 @@
                 kmcp->kmc_move_yes.value.ui64           = kd->kmd_yes;
                 kmcp->kmc_move_no.value.ui64            = kd->kmd_no;
                 kmcp->kmc_move_later.value.ui64         = kd->kmd_later;
                 kmcp->kmc_move_dont_need.value.ui64     = kd->kmd_dont_need;
                 kmcp->kmc_move_dont_know.value.ui64     = kd->kmd_dont_know;
-                kmcp->kmc_move_hunt_found.value.ui64    = kd->kmd_hunt_found;
+                kmcp->kmc_move_hunt_found.value.ui64    = 0;
                 kmcp->kmc_move_slabs_freed.value.ui64   = kd->kmd_slabs_freed;
                 kmcp->kmc_defrag.value.ui64             = kd->kmd_defrags;
                 kmcp->kmc_scan.value.ui64               = kd->kmd_scans;
 
                 reclaimable = cp->cache_bufslab - (cp->cache_maxchunks - 1);

@@ -4148,11 +4095,12 @@
         list_remove(&kmem_caches, cp);
         mutex_exit(&kmem_cache_lock);
 
         if (kmem_taskq != NULL)
                 taskq_wait(kmem_taskq);
-        if (kmem_move_taskq != NULL)
+
+        if (kmem_move_taskq != NULL && cp->cache_defrag != NULL)
                 taskq_wait(kmem_move_taskq);
 
         kmem_cache_magazine_purge(cp);
 
         mutex_enter(&cp->cache_lock);

@@ -4675,99 +4623,11 @@
          */
         return ((refcnt * KMEM_VOID_FRACTION) <
             (sp->slab_chunks * cp->cache_defrag->kmd_reclaim_numer));
 }
 
-static void *
-kmem_hunt_mag(kmem_cache_t *cp, kmem_magazine_t *m, int n, void *buf,
-    void *tbuf)
-{
-        int i;          /* magazine round index */
-
-        for (i = 0; i < n; i++) {
-                if (buf == m->mag_round[i]) {
-                        if (cp->cache_flags & KMF_BUFTAG) {
-                                (void) kmem_cache_free_debug(cp, tbuf,
-                                    caller());
-                        }
-                        m->mag_round[i] = tbuf;
-                        return (buf);
-                }
-        }
-
-        return (NULL);
-}
-
 /*
- * Hunt the magazine layer for the given buffer. If found, the buffer is
- * removed from the magazine layer and returned, otherwise NULL is returned.
- * The state of the returned buffer is freed and constructed.
- */
-static void *
-kmem_hunt_mags(kmem_cache_t *cp, void *buf)
-{
-        kmem_cpu_cache_t *ccp;
-        kmem_magazine_t *m;
-        int cpu_seqid;
-        int n;          /* magazine rounds */
-        void *tbuf;     /* temporary swap buffer */
-
-        ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
-
-        /*
-         * Allocated a buffer to swap with the one we hope to pull out of a
-         * magazine when found.
-         */
-        tbuf = kmem_cache_alloc(cp, KM_NOSLEEP);
-        if (tbuf == NULL) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_hunt_alloc_fail);
-                return (NULL);
-        }
-        if (tbuf == buf) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_hunt_lucky);
-                if (cp->cache_flags & KMF_BUFTAG) {
-                        (void) kmem_cache_free_debug(cp, buf, caller());
-                }
-                return (buf);
-        }
-
-        /* Hunt the depot. */
-        mutex_enter(&cp->cache_depot_lock);
-        n = cp->cache_magtype->mt_magsize;
-        for (m = cp->cache_full.ml_list; m != NULL; m = m->mag_next) {
-                if (kmem_hunt_mag(cp, m, n, buf, tbuf) != NULL) {
-                        mutex_exit(&cp->cache_depot_lock);
-                        return (buf);
-                }
-        }
-        mutex_exit(&cp->cache_depot_lock);
-
-        /* Hunt the per-CPU magazines. */
-        for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
-                ccp = &cp->cache_cpu[cpu_seqid];
-
-                mutex_enter(&ccp->cc_lock);
-                m = ccp->cc_loaded;
-                n = ccp->cc_rounds;
-                if (kmem_hunt_mag(cp, m, n, buf, tbuf) != NULL) {
-                        mutex_exit(&ccp->cc_lock);
-                        return (buf);
-                }
-                m = ccp->cc_ploaded;
-                n = ccp->cc_prounds;
-                if (kmem_hunt_mag(cp, m, n, buf, tbuf) != NULL) {
-                        mutex_exit(&ccp->cc_lock);
-                        return (buf);
-                }
-                mutex_exit(&ccp->cc_lock);
-        }
-
-        kmem_cache_free(cp, tbuf);
-        return (NULL);
-}
-
-/*
  * May be called from the kmem_move_taskq, from kmem_cache_move_notify_task(),
  * or when the buffer is freed.
  */
 static void
 kmem_slab_move_yes(kmem_cache_t *cp, kmem_slab_t *sp, void *from_buf)

@@ -4826,11 +4686,11 @@
  *
  * YES          kmem frees the old buffer (the move was successful)
  * NO           kmem frees the new buffer, marks the slab of the old buffer
  *              non-reclaimable to avoid bothering the client again
  * LATER        kmem frees the new buffer, increments slab_later_count
- * DONT_KNOW    kmem frees the new buffer, searches mags for the old buffer
+ * DONT_KNOW    kmem frees the new buffer
  * DONT_NEED    kmem frees both the old buffer and the new buffer
  *
  * The pending callback argument now being processed contains both of the
  * buffers (old and new) passed to the move callback function, the slab of the
  * old buffer, and flags related to the move request, such as whether or not the

@@ -4860,31 +4720,25 @@
          * last checked the slab's reclaimability (when the pending move was
          * enqueued), or the client may have responded NO when asked to move
          * another buffer on the same slab.
          */
         if (!kmem_slab_is_reclaimable(cp, sp, callback->kmm_flags)) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_no_longer_reclaimable);
-                KMEM_STAT_COND_ADD((callback->kmm_flags & KMM_NOTIFY),
-                    kmem_move_stats.kms_notify_no_longer_reclaimable);
                 kmem_slab_free(cp, callback->kmm_to_buf);
                 kmem_move_end(cp, callback);
                 return;
         }
 
         /*
-         * Hunting magazines is expensive, so we'll wait to do that until the
-         * client responds KMEM_CBRC_DONT_KNOW. However, checking the slab layer
-         * is cheap, so we might as well do that here in case we can avoid
-         * bothering the client.
+         * Checking the slab layer is easy, so we might as well do that here
+         * in case we can avoid bothering the client.
          */
         mutex_enter(&cp->cache_lock);
         free_on_slab = (kmem_slab_allocated(cp, sp,
             callback->kmm_from_buf) == NULL);
         mutex_exit(&cp->cache_lock);
 
         if (free_on_slab) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_hunt_found_slab);
                 kmem_slab_free(cp, callback->kmm_to_buf);
                 kmem_move_end(cp, callback);
                 return;
         }
 

@@ -4892,27 +4746,22 @@
                 /*
                  * Make kmem_cache_alloc_debug() apply the constructor for us.
                  */
                 if (kmem_cache_alloc_debug(cp, callback->kmm_to_buf,
                     KM_NOSLEEP, 1, caller()) != 0) {
-                        KMEM_STAT_ADD(kmem_move_stats.kms_alloc_fail);
                         kmem_move_end(cp, callback);
                         return;
                 }
         } else if (cp->cache_constructor != NULL &&
             cp->cache_constructor(callback->kmm_to_buf, cp->cache_private,
             KM_NOSLEEP) != 0) {
                 atomic_inc_64(&cp->cache_alloc_fail);
-                KMEM_STAT_ADD(kmem_move_stats.kms_constructor_fail);
                 kmem_slab_free(cp, callback->kmm_to_buf);
                 kmem_move_end(cp, callback);
                 return;
         }
 
-        KMEM_STAT_ADD(kmem_move_stats.kms_callbacks);
-        KMEM_STAT_COND_ADD((callback->kmm_flags & KMM_NOTIFY),
-            kmem_move_stats.kms_notify_callbacks);
         cp->cache_defrag->kmd_callbacks++;
         cp->cache_defrag->kmd_thread = curthread;
         cp->cache_defrag->kmd_from_buf = callback->kmm_from_buf;
         cp->cache_defrag->kmd_to_buf = callback->kmm_to_buf;
         DTRACE_PROBE2(kmem__move__start, kmem_cache_t *, cp, kmem_move_t *,

@@ -4926,11 +4775,10 @@
         cp->cache_defrag->kmd_thread = NULL;
         cp->cache_defrag->kmd_from_buf = NULL;
         cp->cache_defrag->kmd_to_buf = NULL;
 
         if (response == KMEM_CBRC_YES) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_yes);
                 cp->cache_defrag->kmd_yes++;
                 kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE);
                 /* slab safe to access until kmem_move_end() */
                 if (sp->slab_refcnt == 0)
                         cp->cache_defrag->kmd_slabs_freed++;

@@ -4941,58 +4789,56 @@
                 return;
         }
 
         switch (response) {
         case KMEM_CBRC_NO:
-                KMEM_STAT_ADD(kmem_move_stats.kms_no);
                 cp->cache_defrag->kmd_no++;
                 mutex_enter(&cp->cache_lock);
                 kmem_slab_move_no(cp, sp, callback->kmm_from_buf);
                 mutex_exit(&cp->cache_lock);
                 break;
         case KMEM_CBRC_LATER:
-                KMEM_STAT_ADD(kmem_move_stats.kms_later);
                 cp->cache_defrag->kmd_later++;
                 mutex_enter(&cp->cache_lock);
                 if (!KMEM_SLAB_IS_PARTIAL(sp)) {
                         mutex_exit(&cp->cache_lock);
                         break;
                 }
 
                 if (++sp->slab_later_count >= KMEM_DISBELIEF) {
-                        KMEM_STAT_ADD(kmem_move_stats.kms_disbelief);
                         kmem_slab_move_no(cp, sp, callback->kmm_from_buf);
                 } else if (!(sp->slab_flags & KMEM_SLAB_NOMOVE)) {
                         sp->slab_stuck_offset = KMEM_SLAB_OFFSET(sp,
                             callback->kmm_from_buf);
                 }
                 mutex_exit(&cp->cache_lock);
                 break;
         case KMEM_CBRC_DONT_NEED:
-                KMEM_STAT_ADD(kmem_move_stats.kms_dont_need);
                 cp->cache_defrag->kmd_dont_need++;
                 kmem_slab_free_constructed(cp, callback->kmm_from_buf, B_FALSE);
                 if (sp->slab_refcnt == 0)
                         cp->cache_defrag->kmd_slabs_freed++;
                 mutex_enter(&cp->cache_lock);
                 kmem_slab_move_yes(cp, sp, callback->kmm_from_buf);
                 mutex_exit(&cp->cache_lock);
                 break;
         case KMEM_CBRC_DONT_KNOW:
-                KMEM_STAT_ADD(kmem_move_stats.kms_dont_know);
+                /*
+                 * If we don't know if we can move this buffer or not, we'll
+                 * just assume that we can't:  if the buffer is in fact free,
+                 * then it is sitting in one of the per-CPU magazines or in
+                 * a full magazine in the depot layer.  Either way, because
+                 * defrag is induced in the same logic that reaps a cache,
+                 * it's likely that full magazines will be returned to the
+                 * system soon (thereby accomplishing what we're trying to
+                 * accomplish here: return those magazines to their slabs).
+                 * Given this, any work that we might do now to locate a buffer
+                 * in a magazine is wasted (and expensive!) work; we bump
+                 * a counter in this case and otherwise assume that we can't
+                 * move it.
+                 */
                 cp->cache_defrag->kmd_dont_know++;
-                if (kmem_hunt_mags(cp, callback->kmm_from_buf) != NULL) {
-                        KMEM_STAT_ADD(kmem_move_stats.kms_hunt_found_mag);
-                        cp->cache_defrag->kmd_hunt_found++;
-                        kmem_slab_free_constructed(cp, callback->kmm_from_buf,
-                            B_TRUE);
-                        if (sp->slab_refcnt == 0)
-                                cp->cache_defrag->kmd_slabs_freed++;
-                        mutex_enter(&cp->cache_lock);
-                        kmem_slab_move_yes(cp, sp, callback->kmm_from_buf);
-                        mutex_exit(&cp->cache_lock);
-                }
                 break;
         default:
                 panic("'%s' (%p) unexpected move callback response %d\n",
                     cp->cache_name, (void *)cp, response);
         }

@@ -5013,14 +4859,13 @@
         ASSERT(taskq_member(kmem_taskq, curthread));
         ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
         ASSERT(sp->slab_flags & KMEM_SLAB_MOVE_PENDING);
 
         callback = kmem_cache_alloc(kmem_move_cache, KM_NOSLEEP);
-        if (callback == NULL) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_callback_alloc_fail);
+
+        if (callback == NULL)
                 return (B_FALSE);
-        }
 
         callback->kmm_from_slab = sp;
         callback->kmm_from_buf = buf;
         callback->kmm_flags = flags;
 

@@ -5041,11 +4886,10 @@
                  */
                 if (flags & KMM_DESPERATE) {
                         pending->kmm_flags |= KMM_DESPERATE;
                 }
                 mutex_exit(&cp->cache_lock);
-                KMEM_STAT_ADD(kmem_move_stats.kms_already_pending);
                 kmem_cache_free(kmem_move_cache, callback);
                 return (B_TRUE);
         }
 
         to_buf = kmem_slab_alloc_impl(cp, avl_first(&cp->cache_partial_slabs),

@@ -5055,11 +4899,10 @@
 
         mutex_exit(&cp->cache_lock);
 
         if (!taskq_dispatch(kmem_move_taskq, (task_func_t *)kmem_move_buffer,
             callback, TQ_NOSLEEP)) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_callback_taskq_fail);
                 mutex_enter(&cp->cache_lock);
                 avl_remove(&cp->cache_defrag->kmd_moves_pending, callback);
                 mutex_exit(&cp->cache_lock);
                 kmem_slab_free(cp, to_buf);
                 kmem_cache_free(kmem_move_cache, callback);

@@ -5101,11 +4944,10 @@
                         }
                         cp->cache_defrag->kmd_deadcount--;
                         cp->cache_slab_destroy++;
                         mutex_exit(&cp->cache_lock);
                         kmem_slab_destroy(cp, sp);
-                        KMEM_STAT_ADD(kmem_move_stats.kms_dead_slabs_freed);
                         mutex_enter(&cp->cache_lock);
                 }
         }
         mutex_exit(&cp->cache_lock);
         kmem_cache_free(kmem_move_cache, callback);

@@ -5246,12 +5088,10 @@
                                          * destroyed along with any other slabs
                                          * on the deadlist after the last
                                          * pending move completes.
                                          */
                                         list_insert_head(deadlist, sp);
-                                        KMEM_STAT_ADD(kmem_move_stats.
-                                            kms_endscan_slab_dead);
                                         return (-1);
                                 }
 
                                 /*
                                  * Destroy the slab now if it was completely

@@ -5262,14 +5102,10 @@
                                  */
                                 cp->cache_defrag->kmd_deadcount--;
                                 cp->cache_slab_destroy++;
                                 mutex_exit(&cp->cache_lock);
                                 kmem_slab_destroy(cp, sp);
-                                KMEM_STAT_ADD(kmem_move_stats.
-                                    kms_dead_slabs_freed);
-                                KMEM_STAT_ADD(kmem_move_stats.
-                                    kms_endscan_slab_destroyed);
                                 mutex_enter(&cp->cache_lock);
                                 /*
                                  * Since we can't pick up the scan where we left
                                  * off, abort the scan and say nothing about the
                                  * number of reclaimable slabs.

@@ -5281,12 +5117,10 @@
                                 /*
                                  * Abort the scan if there is not enough memory
                                  * for the request and say nothing about the
                                  * number of reclaimable slabs.
                                  */
-                                KMEM_STAT_COND_ADD(s < max_slabs,
-                                    kmem_move_stats.kms_endscan_nomem);
                                 return (-1);
                         }
 
                         /*
                          * The slab's position changed while the lock was

@@ -5298,20 +5132,14 @@
                                  * If this is a KMM_DEBUG move, the slab_refcnt
                                  * may have changed because we allocated a
                                  * destination buffer on the same slab. In that
                                  * case, we're not interested in counting it.
                                  */
-                                KMEM_STAT_COND_ADD(!(flags & KMM_DEBUG) &&
-                                    (s < max_slabs),
-                                    kmem_move_stats.kms_endscan_refcnt_changed);
                                 return (-1);
                         }
-                        if ((sp->slab_flags & KMEM_SLAB_NOMOVE) != nomove) {
-                                KMEM_STAT_COND_ADD(s < max_slabs,
-                                    kmem_move_stats.kms_endscan_nomove_changed);
+                        if ((sp->slab_flags & KMEM_SLAB_NOMOVE) != nomove)
                                 return (-1);
-                        }
 
                         /*
                          * Generating a move request allocates a destination
                          * buffer from the slab layer, bumping the first partial
                          * slab if it is completely allocated. If the current

@@ -5334,15 +5162,10 @@
                         }
                 }
         }
 end_scan:
 
-        KMEM_STAT_COND_ADD(!(flags & KMM_DEBUG) &&
-            (s < max_slabs) &&
-            (sp == avl_first(&cp->cache_partial_slabs)),
-            kmem_move_stats.kms_endscan_freelist);
-
         return (s);
 }
 
 typedef struct kmem_move_notify_args {
         kmem_cache_t *kmna_cache;

@@ -5398,22 +5221,17 @@
 
                         if (!avl_is_empty(
                             &cp->cache_defrag->kmd_moves_pending)) {
                                 list_insert_head(deadlist, sp);
                                 mutex_exit(&cp->cache_lock);
-                                KMEM_STAT_ADD(kmem_move_stats.
-                                    kms_notify_slab_dead);
                                 return;
                         }
 
                         cp->cache_defrag->kmd_deadcount--;
                         cp->cache_slab_destroy++;
                         mutex_exit(&cp->cache_lock);
                         kmem_slab_destroy(cp, sp);
-                        KMEM_STAT_ADD(kmem_move_stats.kms_dead_slabs_freed);
-                        KMEM_STAT_ADD(kmem_move_stats.
-                            kms_notify_slab_destroyed);
                         return;
                 }
         } else {
                 kmem_slab_move_yes(cp, sp, buf);
         }

@@ -5423,11 +5241,10 @@
 void
 kmem_cache_move_notify(kmem_cache_t *cp, void *buf)
 {
         kmem_move_notify_args_t *args;
 
-        KMEM_STAT_ADD(kmem_move_stats.kms_notify);
         args = kmem_alloc(sizeof (kmem_move_notify_args_t), KM_NOSLEEP);
         if (args != NULL) {
                 args->kmna_cache = cp;
                 args->kmna_buf = buf;
                 if (!taskq_dispatch(kmem_taskq,

@@ -5446,11 +5263,10 @@
 
         mutex_enter(&cp->cache_lock);
         n = avl_numnodes(&cp->cache_partial_slabs);
         if (n > 1) {
                 /* kmem_move_buffers() drops and reacquires cache_lock */
-                KMEM_STAT_ADD(kmem_move_stats.kms_defrags);
                 cp->cache_defrag->kmd_defrags++;
                 (void) kmem_move_buffers(cp, n, 0, KMM_DESPERATE);
         }
         mutex_exit(&cp->cache_lock);
 }

@@ -5545,11 +5361,10 @@
                  * the definition of a candidate slab if we're having trouble
                  * finding them.
                  *
                  * kmem_move_buffers() drops and reacquires cache_lock.
                  */
-                KMEM_STAT_ADD(kmem_move_stats.kms_scans);
                 kmd->kmd_scans++;
                 slabs_found = kmem_move_buffers(cp, kmem_reclaim_scan_range,
                     kmem_reclaim_max_slabs, 0);
                 if (slabs_found >= 0) {
                         kmd->kmd_slabs_sought += kmem_reclaim_max_slabs;

@@ -5586,16 +5401,13 @@
 
                         (void) random_get_bytes((uint8_t *)&debug_rand, 2);
                         if (!kmem_move_noreap &&
                             ((debug_rand % kmem_mtb_reap) == 0)) {
                                 mutex_exit(&cp->cache_lock);
-                                KMEM_STAT_ADD(kmem_move_stats.kms_debug_reaps);
                                 kmem_cache_reap(cp);
                                 return;
                         } else if ((debug_rand % kmem_mtb_move) == 0) {
-                                KMEM_STAT_ADD(kmem_move_stats.kms_scans);
-                                KMEM_STAT_ADD(kmem_move_stats.kms_debug_scans);
                                 kmd->kmd_scans++;
                                 (void) kmem_move_buffers(cp,
                                     kmem_reclaim_scan_range, 1, KMM_DEBUG);
                         }
                 }

@@ -5602,10 +5414,8 @@
 #endif  /* DEBUG */
         }
 
         mutex_exit(&cp->cache_lock);
 
-        if (reap) {
-                KMEM_STAT_ADD(kmem_move_stats.kms_scan_depot_ws_reaps);
+        if (reap)
                 kmem_depot_ws_reap(cp);
-        }
 }