1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2013, Joyent, Inc. All rights reserved.
24 * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
27 */
28
29 /*
30 * DVA-based Adjustable Replacement Cache
31 *
32 * While much of the theory of operation used here is
33 * based on the self-tuning, low overhead replacement cache
34 * presented by Megiddo and Modha at FAST 2003, there are some
35 * significant differences:
36 *
37 * 1. The Megiddo and Modha model assumes any page is evictable.
38 * Pages in its cache cannot be "locked" into memory. This makes
39 * the eviction algorithm simple: evict the last page in the list.
40 * This also make the performance characteristics easy to reason
41 * about. Our cache is not so simple. At any given moment, some
42 * subset of the blocks in the cache are un-evictable because we
43 * have handed out a reference to them. Blocks are only evictable
44 * when there are no external references active. This makes
45 * eviction far more problematic: we choose to evict the evictable
46 * blocks that are the "lowest" in the list.
47 *
48 * There are times when it is not possible to evict the requested
49 * space. In these circumstances we are unable to adjust the cache
50 * size. To prevent the cache growing unbounded at these times we
51 * implement a "cache throttle" that slows the flow of new data
52 * into the cache until we can make space available.
53 *
54 * 2. The Megiddo and Modha model assumes a fixed cache size.
55 * Pages are evicted when the cache is full and there is a cache
56 * miss. Our model has a variable sized cache. It grows with
57 * high use, but also tries to react to memory pressure from the
58 * operating system: decreasing its size when system memory is
59 * tight.
60 *
61 * 3. The Megiddo and Modha model assumes a fixed page size. All
62 * elements of the cache are therefore exactly the same size. So
63 * when adjusting the cache size following a cache miss, its simply
64 * a matter of choosing a single page to evict. In our model, we
65 * have variable sized cache blocks (rangeing from 512 bytes to
66 * 128K bytes). We therefore choose a set of blocks to evict to make
67 * space for a cache miss that approximates as closely as possible
68 * the space used by the new block.
69 *
70 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71 * by N. Megiddo & D. Modha, FAST 2003
72 */
73
74 /*
75 * The locking model:
76 *
77 * A new reference to a cache buffer can be obtained in two
78 * ways: 1) via a hash table lookup using the DVA as a key,
79 * or 2) via one of the ARC lists. The arc_read() interface
80 * uses method 1, while the internal arc algorithms for
81 * adjusting the cache use method 2. We therefore provide two
82 * types of locks: 1) the hash table lock array, and 2) the
83 * arc list locks.
84 *
85 * Buffers do not have their own mutexes, rather they rely on the
86 * hash table mutexes for the bulk of their protection (i.e. most
87 * fields in the arc_buf_hdr_t are protected by these mutexes).
88 *
89 * buf_hash_find() returns the appropriate mutex (held) when it
90 * locates the requested buffer in the hash table. It returns
91 * NULL for the mutex if the buffer was not in the table.
92 *
93 * buf_hash_remove() expects the appropriate hash mutex to be
94 * already held before it is invoked.
95 *
96 * Each arc state also has a mutex which is used to protect the
97 * buffer list associated with the state. When attempting to
98 * obtain a hash table lock while holding an arc list lock you
99 * must use: mutex_tryenter() to avoid deadlock. Also note that
100 * the active state mutex must be held before the ghost state mutex.
101 *
102 * Arc buffers may have an associated eviction callback function.
103 * This function will be invoked prior to removing the buffer (e.g.
104 * in arc_do_user_evicts()). Note however that the data associated
105 * with the buffer may be evicted prior to the callback. The callback
106 * must be made with *no locks held* (to prevent deadlock). Additionally,
107 * the users of callbacks must ensure that their private data is
108 * protected from simultaneous callbacks from arc_clear_callback()
109 * and arc_do_user_evicts().
110 *
111 * Note that the majority of the performance stats are manipulated
112 * with atomic operations.
113 *
114 * The L2ARC uses the l2ad_mtx on each vdev for the following:
115 *
116 * - L2ARC buflist creation
117 * - L2ARC buflist eviction
118 * - L2ARC write completion, which walks L2ARC buflists
119 * - ARC header destruction, as it removes from L2ARC buflists
120 * - ARC header release, as it removes from L2ARC buflists
121 */
122
123 #include <sys/spa.h>
124 #include <sys/zio.h>
125 #include <sys/zio_compress.h>
126 #include <sys/zfs_context.h>
127 #include <sys/arc.h>
128 #include <sys/refcount.h>
129 #include <sys/vdev.h>
130 #include <sys/vdev_impl.h>
131 #include <sys/dsl_pool.h>
132 #include <sys/zfs_zone.h>
133 #include <sys/multilist.h>
134 #ifdef _KERNEL
135 #include <sys/vmsystm.h>
136 #include <vm/anon.h>
137 #include <sys/fs/swapnode.h>
138 #include <sys/dnlc.h>
139 #endif
140 #include <sys/callb.h>
141 #include <sys/kstat.h>
142 #include <zfs_fletcher.h>
143
144 #ifndef _KERNEL
145 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
146 boolean_t arc_watch = B_FALSE;
147 int arc_procfd;
148 #endif
149
150 static kmutex_t arc_reclaim_lock;
151 static kcondvar_t arc_reclaim_thread_cv;
152 static boolean_t arc_reclaim_thread_exit;
153 static kcondvar_t arc_reclaim_waiters_cv;
154
155 static kmutex_t arc_user_evicts_lock;
156 static kcondvar_t arc_user_evicts_cv;
157 static boolean_t arc_user_evicts_thread_exit;
158
159 uint_t arc_reduce_dnlc_percent = 3;
160
161 /*
162 * The number of headers to evict in arc_evict_state_impl() before
163 * dropping the sublist lock and evicting from another sublist. A lower
164 * value means we're more likely to evict the "correct" header (i.e. the
165 * oldest header in the arc state), but comes with higher overhead
166 * (i.e. more invocations of arc_evict_state_impl()).
167 */
168 int zfs_arc_evict_batch_limit = 10;
169
170 /*
171 * The number of sublists used for each of the arc state lists. If this
172 * is not set to a suitable value by the user, it will be configured to
173 * the number of CPUs on the system in arc_init().
174 */
175 int zfs_arc_num_sublists_per_state = 0;
176
177 /* number of seconds before growing cache again */
178 static int arc_grow_retry = 60;
179
180 /* shift of arc_c for calculating overflow limit in arc_get_data_buf */
181 int zfs_arc_overflow_shift = 8;
182
183 /* shift of arc_c for calculating both min and max arc_p */
184 static int arc_p_min_shift = 4;
185
186 /* log2(fraction of arc to reclaim) */
187 static int arc_shrink_shift = 7;
188
189 /*
190 * log2(fraction of ARC which must be free to allow growing).
191 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
192 * when reading a new block into the ARC, we will evict an equal-sized block
193 * from the ARC.
194 *
195 * This must be less than arc_shrink_shift, so that when we shrink the ARC,
196 * we will still not allow it to grow.
197 */
198 int arc_no_grow_shift = 5;
199
200
201 /*
202 * minimum lifespan of a prefetch block in clock ticks
203 * (initialized in arc_init())
204 */
205 static int arc_min_prefetch_lifespan;
206
207 /*
208 * If this percent of memory is free, don't throttle.
209 */
210 int arc_lotsfree_percent = 10;
211
212 static int arc_dead;
213
214 /*
215 * The arc has filled available memory and has now warmed up.
216 */
217 static boolean_t arc_warm;
218
219 /*
220 * These tunables are for performance analysis.
221 */
222 uint64_t zfs_arc_max;
223 uint64_t zfs_arc_min;
224 uint64_t zfs_arc_meta_limit = 0;
225 uint64_t zfs_arc_meta_min = 0;
226 int zfs_arc_grow_retry = 0;
227 int zfs_arc_shrink_shift = 0;
228 int zfs_arc_p_min_shift = 0;
229 int zfs_disable_dup_eviction = 0;
230 int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
231
232 /*
233 * Note that buffers can be in one of 6 states:
234 * ARC_anon - anonymous (discussed below)
235 * ARC_mru - recently used, currently cached
236 * ARC_mru_ghost - recentely used, no longer in cache
237 * ARC_mfu - frequently used, currently cached
238 * ARC_mfu_ghost - frequently used, no longer in cache
239 * ARC_l2c_only - exists in L2ARC but not other states
240 * When there are no active references to the buffer, they are
241 * are linked onto a list in one of these arc states. These are
242 * the only buffers that can be evicted or deleted. Within each
243 * state there are multiple lists, one for meta-data and one for
244 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes,
245 * etc.) is tracked separately so that it can be managed more
246 * explicitly: favored over data, limited explicitly.
247 *
248 * Anonymous buffers are buffers that are not associated with
249 * a DVA. These are buffers that hold dirty block copies
250 * before they are written to stable storage. By definition,
251 * they are "ref'd" and are considered part of arc_mru
252 * that cannot be freed. Generally, they will aquire a DVA
253 * as they are written and migrate onto the arc_mru list.
254 *
255 * The ARC_l2c_only state is for buffers that are in the second
256 * level ARC but no longer in any of the ARC_m* lists. The second
257 * level ARC itself may also contain buffers that are in any of
258 * the ARC_m* states - meaning that a buffer can exist in two
259 * places. The reason for the ARC_l2c_only state is to keep the
260 * buffer header in the hash table, so that reads that hit the
261 * second level ARC benefit from these fast lookups.
262 */
263
264 typedef struct arc_state {
265 /*
266 * list of evictable buffers
267 */
268 multilist_t arcs_list[ARC_BUFC_NUMTYPES];
269 /*
270 * total amount of evictable data in this state
271 */
272 uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];
273 /*
274 * total amount of data in this state; this includes: evictable,
275 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
276 */
277 refcount_t arcs_size;
278 } arc_state_t;
279
280 /* The 6 states: */
281 static arc_state_t ARC_anon;
282 static arc_state_t ARC_mru;
283 static arc_state_t ARC_mru_ghost;
284 static arc_state_t ARC_mfu;
285 static arc_state_t ARC_mfu_ghost;
286 static arc_state_t ARC_l2c_only;
287
288 typedef struct arc_stats {
289 kstat_named_t arcstat_hits;
290 kstat_named_t arcstat_misses;
291 kstat_named_t arcstat_demand_data_hits;
292 kstat_named_t arcstat_demand_data_misses;
293 kstat_named_t arcstat_demand_metadata_hits;
294 kstat_named_t arcstat_demand_metadata_misses;
295 kstat_named_t arcstat_prefetch_data_hits;
296 kstat_named_t arcstat_prefetch_data_misses;
297 kstat_named_t arcstat_prefetch_metadata_hits;
298 kstat_named_t arcstat_prefetch_metadata_misses;
299 kstat_named_t arcstat_mru_hits;
300 kstat_named_t arcstat_mru_ghost_hits;
301 kstat_named_t arcstat_mfu_hits;
302 kstat_named_t arcstat_mfu_ghost_hits;
303 kstat_named_t arcstat_deleted;
304 /*
305 * Number of buffers that could not be evicted because the hash lock
306 * was held by another thread. The lock may not necessarily be held
307 * by something using the same buffer, since hash locks are shared
308 * by multiple buffers.
309 */
310 kstat_named_t arcstat_mutex_miss;
311 /*
312 * Number of buffers skipped because they have I/O in progress, are
313 * indrect prefetch buffers that have not lived long enough, or are
314 * not from the spa we're trying to evict from.
315 */
316 kstat_named_t arcstat_evict_skip;
317 /*
318 * Number of times arc_evict_state() was unable to evict enough
319 * buffers to reach it's target amount.
320 */
321 kstat_named_t arcstat_evict_not_enough;
322 kstat_named_t arcstat_evict_l2_cached;
323 kstat_named_t arcstat_evict_l2_eligible;
324 kstat_named_t arcstat_evict_l2_ineligible;
325 kstat_named_t arcstat_evict_l2_skip;
326 kstat_named_t arcstat_hash_elements;
327 kstat_named_t arcstat_hash_elements_max;
328 kstat_named_t arcstat_hash_collisions;
329 kstat_named_t arcstat_hash_chains;
330 kstat_named_t arcstat_hash_chain_max;
331 kstat_named_t arcstat_p;
332 kstat_named_t arcstat_c;
333 kstat_named_t arcstat_c_min;
334 kstat_named_t arcstat_c_max;
335 kstat_named_t arcstat_size;
336 /*
337 * Number of bytes consumed by internal ARC structures necessary
338 * for tracking purposes; these structures are not actually
339 * backed by ARC buffers. This includes arc_buf_hdr_t structures
340 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
341 * caches), and arc_buf_t structures (allocated via arc_buf_t
342 * cache).
343 */
344 kstat_named_t arcstat_hdr_size;
345 /*
346 * Number of bytes consumed by ARC buffers of type equal to
347 * ARC_BUFC_DATA. This is generally consumed by buffers backing
348 * on disk user data (e.g. plain file contents).
349 */
350 kstat_named_t arcstat_data_size;
351 /*
352 * Number of bytes consumed by ARC buffers of type equal to
353 * ARC_BUFC_METADATA. This is generally consumed by buffers
354 * backing on disk data that is used for internal ZFS
355 * structures (e.g. ZAP, dnode, indirect blocks, etc).
356 */
357 kstat_named_t arcstat_metadata_size;
358 /*
359 * Number of bytes consumed by various buffers and structures
360 * not actually backed with ARC buffers. This includes bonus
361 * buffers (allocated directly via zio_buf_* functions),
362 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
363 * cache), and dnode_t structures (allocated via dnode_t cache).
364 */
365 kstat_named_t arcstat_other_size;
366 /*
367 * Total number of bytes consumed by ARC buffers residing in the
368 * arc_anon state. This includes *all* buffers in the arc_anon
369 * state; e.g. data, metadata, evictable, and unevictable buffers
370 * are all included in this value.
371 */
372 kstat_named_t arcstat_anon_size;
373 /*
374 * Number of bytes consumed by ARC buffers that meet the
375 * following criteria: backing buffers of type ARC_BUFC_DATA,
376 * residing in the arc_anon state, and are eligible for eviction
377 * (e.g. have no outstanding holds on the buffer).
378 */
379 kstat_named_t arcstat_anon_evictable_data;
380 /*
381 * Number of bytes consumed by ARC buffers that meet the
382 * following criteria: backing buffers of type ARC_BUFC_METADATA,
383 * residing in the arc_anon state, and are eligible for eviction
384 * (e.g. have no outstanding holds on the buffer).
385 */
386 kstat_named_t arcstat_anon_evictable_metadata;
387 /*
388 * Total number of bytes consumed by ARC buffers residing in the
389 * arc_mru state. This includes *all* buffers in the arc_mru
390 * state; e.g. data, metadata, evictable, and unevictable buffers
391 * are all included in this value.
392 */
393 kstat_named_t arcstat_mru_size;
394 /*
395 * Number of bytes consumed by ARC buffers that meet the
396 * following criteria: backing buffers of type ARC_BUFC_DATA,
397 * residing in the arc_mru state, and are eligible for eviction
398 * (e.g. have no outstanding holds on the buffer).
399 */
400 kstat_named_t arcstat_mru_evictable_data;
401 /*
402 * Number of bytes consumed by ARC buffers that meet the
403 * following criteria: backing buffers of type ARC_BUFC_METADATA,
404 * residing in the arc_mru state, and are eligible for eviction
405 * (e.g. have no outstanding holds on the buffer).
406 */
407 kstat_named_t arcstat_mru_evictable_metadata;
408 /*
409 * Total number of bytes that *would have been* consumed by ARC
410 * buffers in the arc_mru_ghost state. The key thing to note
411 * here, is the fact that this size doesn't actually indicate
412 * RAM consumption. The ghost lists only consist of headers and
413 * don't actually have ARC buffers linked off of these headers.
414 * Thus, *if* the headers had associated ARC buffers, these
415 * buffers *would have* consumed this number of bytes.
416 */
417 kstat_named_t arcstat_mru_ghost_size;
418 /*
419 * Number of bytes that *would have been* consumed by ARC
420 * buffers that are eligible for eviction, of type
421 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
422 */
423 kstat_named_t arcstat_mru_ghost_evictable_data;
424 /*
425 * Number of bytes that *would have been* consumed by ARC
426 * buffers that are eligible for eviction, of type
427 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
428 */
429 kstat_named_t arcstat_mru_ghost_evictable_metadata;
430 /*
431 * Total number of bytes consumed by ARC buffers residing in the
432 * arc_mfu state. This includes *all* buffers in the arc_mfu
433 * state; e.g. data, metadata, evictable, and unevictable buffers
434 * are all included in this value.
435 */
436 kstat_named_t arcstat_mfu_size;
437 /*
438 * Number of bytes consumed by ARC buffers that are eligible for
439 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
440 * state.
441 */
442 kstat_named_t arcstat_mfu_evictable_data;
443 /*
444 * Number of bytes consumed by ARC buffers that are eligible for
445 * eviction, of type ARC_BUFC_METADATA, and reside in the
446 * arc_mfu state.
447 */
448 kstat_named_t arcstat_mfu_evictable_metadata;
449 /*
450 * Total number of bytes that *would have been* consumed by ARC
451 * buffers in the arc_mfu_ghost state. See the comment above
452 * arcstat_mru_ghost_size for more details.
453 */
454 kstat_named_t arcstat_mfu_ghost_size;
455 /*
456 * Number of bytes that *would have been* consumed by ARC
457 * buffers that are eligible for eviction, of type
458 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
459 */
460 kstat_named_t arcstat_mfu_ghost_evictable_data;
461 /*
462 * Number of bytes that *would have been* consumed by ARC
463 * buffers that are eligible for eviction, of type
464 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
465 */
466 kstat_named_t arcstat_mfu_ghost_evictable_metadata;
467 kstat_named_t arcstat_l2_hits;
468 kstat_named_t arcstat_l2_misses;
469 kstat_named_t arcstat_l2_feeds;
470 kstat_named_t arcstat_l2_rw_clash;
471 kstat_named_t arcstat_l2_read_bytes;
472 kstat_named_t arcstat_l2_write_bytes;
473 kstat_named_t arcstat_l2_writes_sent;
474 kstat_named_t arcstat_l2_writes_done;
475 kstat_named_t arcstat_l2_writes_error;
476 kstat_named_t arcstat_l2_writes_lock_retry;
477 kstat_named_t arcstat_l2_evict_lock_retry;
478 kstat_named_t arcstat_l2_evict_reading;
479 kstat_named_t arcstat_l2_evict_l1cached;
480 kstat_named_t arcstat_l2_free_on_write;
481 kstat_named_t arcstat_l2_cdata_free_on_write;
482 kstat_named_t arcstat_l2_abort_lowmem;
483 kstat_named_t arcstat_l2_cksum_bad;
484 kstat_named_t arcstat_l2_io_error;
485 kstat_named_t arcstat_l2_size;
486 kstat_named_t arcstat_l2_asize;
487 kstat_named_t arcstat_l2_hdr_size;
488 kstat_named_t arcstat_l2_compress_successes;
489 kstat_named_t arcstat_l2_compress_zeros;
490 kstat_named_t arcstat_l2_compress_failures;
491 kstat_named_t arcstat_memory_throttle_count;
492 kstat_named_t arcstat_duplicate_buffers;
493 kstat_named_t arcstat_duplicate_buffers_size;
494 kstat_named_t arcstat_duplicate_reads;
495 kstat_named_t arcstat_meta_used;
496 kstat_named_t arcstat_meta_limit;
497 kstat_named_t arcstat_meta_max;
498 kstat_named_t arcstat_meta_min;
499 } arc_stats_t;
500
501 static arc_stats_t arc_stats = {
502 { "hits", KSTAT_DATA_UINT64 },
503 { "misses", KSTAT_DATA_UINT64 },
504 { "demand_data_hits", KSTAT_DATA_UINT64 },
505 { "demand_data_misses", KSTAT_DATA_UINT64 },
506 { "demand_metadata_hits", KSTAT_DATA_UINT64 },
507 { "demand_metadata_misses", KSTAT_DATA_UINT64 },
508 { "prefetch_data_hits", KSTAT_DATA_UINT64 },
509 { "prefetch_data_misses", KSTAT_DATA_UINT64 },
510 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 },
511 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 },
512 { "mru_hits", KSTAT_DATA_UINT64 },
513 { "mru_ghost_hits", KSTAT_DATA_UINT64 },
514 { "mfu_hits", KSTAT_DATA_UINT64 },
515 { "mfu_ghost_hits", KSTAT_DATA_UINT64 },
516 { "deleted", KSTAT_DATA_UINT64 },
517 { "mutex_miss", KSTAT_DATA_UINT64 },
518 { "evict_skip", KSTAT_DATA_UINT64 },
519 { "evict_not_enough", KSTAT_DATA_UINT64 },
520 { "evict_l2_cached", KSTAT_DATA_UINT64 },
521 { "evict_l2_eligible", KSTAT_DATA_UINT64 },
522 { "evict_l2_ineligible", KSTAT_DATA_UINT64 },
523 { "evict_l2_skip", KSTAT_DATA_UINT64 },
524 { "hash_elements", KSTAT_DATA_UINT64 },
525 { "hash_elements_max", KSTAT_DATA_UINT64 },
526 { "hash_collisions", KSTAT_DATA_UINT64 },
527 { "hash_chains", KSTAT_DATA_UINT64 },
528 { "hash_chain_max", KSTAT_DATA_UINT64 },
529 { "p", KSTAT_DATA_UINT64 },
530 { "c", KSTAT_DATA_UINT64 },
531 { "c_min", KSTAT_DATA_UINT64 },
532 { "c_max", KSTAT_DATA_UINT64 },
533 { "size", KSTAT_DATA_UINT64 },
534 { "hdr_size", KSTAT_DATA_UINT64 },
535 { "data_size", KSTAT_DATA_UINT64 },
536 { "metadata_size", KSTAT_DATA_UINT64 },
537 { "other_size", KSTAT_DATA_UINT64 },
538 { "anon_size", KSTAT_DATA_UINT64 },
539 { "anon_evictable_data", KSTAT_DATA_UINT64 },
540 { "anon_evictable_metadata", KSTAT_DATA_UINT64 },
541 { "mru_size", KSTAT_DATA_UINT64 },
542 { "mru_evictable_data", KSTAT_DATA_UINT64 },
543 { "mru_evictable_metadata", KSTAT_DATA_UINT64 },
544 { "mru_ghost_size", KSTAT_DATA_UINT64 },
545 { "mru_ghost_evictable_data", KSTAT_DATA_UINT64 },
546 { "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
547 { "mfu_size", KSTAT_DATA_UINT64 },
548 { "mfu_evictable_data", KSTAT_DATA_UINT64 },
549 { "mfu_evictable_metadata", KSTAT_DATA_UINT64 },
550 { "mfu_ghost_size", KSTAT_DATA_UINT64 },
551 { "mfu_ghost_evictable_data", KSTAT_DATA_UINT64 },
552 { "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
553 { "l2_hits", KSTAT_DATA_UINT64 },
554 { "l2_misses", KSTAT_DATA_UINT64 },
555 { "l2_feeds", KSTAT_DATA_UINT64 },
556 { "l2_rw_clash", KSTAT_DATA_UINT64 },
557 { "l2_read_bytes", KSTAT_DATA_UINT64 },
558 { "l2_write_bytes", KSTAT_DATA_UINT64 },
559 { "l2_writes_sent", KSTAT_DATA_UINT64 },
560 { "l2_writes_done", KSTAT_DATA_UINT64 },
561 { "l2_writes_error", KSTAT_DATA_UINT64 },
562 { "l2_writes_lock_retry", KSTAT_DATA_UINT64 },
563 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 },
564 { "l2_evict_reading", KSTAT_DATA_UINT64 },
565 { "l2_evict_l1cached", KSTAT_DATA_UINT64 },
566 { "l2_free_on_write", KSTAT_DATA_UINT64 },
567 { "l2_cdata_free_on_write", KSTAT_DATA_UINT64 },
568 { "l2_abort_lowmem", KSTAT_DATA_UINT64 },
569 { "l2_cksum_bad", KSTAT_DATA_UINT64 },
570 { "l2_io_error", KSTAT_DATA_UINT64 },
571 { "l2_size", KSTAT_DATA_UINT64 },
572 { "l2_asize", KSTAT_DATA_UINT64 },
573 { "l2_hdr_size", KSTAT_DATA_UINT64 },
574 { "l2_compress_successes", KSTAT_DATA_UINT64 },
575 { "l2_compress_zeros", KSTAT_DATA_UINT64 },
576 { "l2_compress_failures", KSTAT_DATA_UINT64 },
577 { "memory_throttle_count", KSTAT_DATA_UINT64 },
578 { "duplicate_buffers", KSTAT_DATA_UINT64 },
579 { "duplicate_buffers_size", KSTAT_DATA_UINT64 },
580 { "duplicate_reads", KSTAT_DATA_UINT64 },
581 { "arc_meta_used", KSTAT_DATA_UINT64 },
582 { "arc_meta_limit", KSTAT_DATA_UINT64 },
583 { "arc_meta_max", KSTAT_DATA_UINT64 },
584 { "arc_meta_min", KSTAT_DATA_UINT64 }
585 };
586
587 #define ARCSTAT(stat) (arc_stats.stat.value.ui64)
588
589 #define ARCSTAT_INCR(stat, val) \
590 atomic_add_64(&arc_stats.stat.value.ui64, (val))
591
592 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1)
593 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1)
594
595 #define ARCSTAT_MAX(stat, val) { \
596 uint64_t m; \
597 while ((val) > (m = arc_stats.stat.value.ui64) && \
598 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \
599 continue; \
600 }
601
602 #define ARCSTAT_MAXSTAT(stat) \
603 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
604
605 /*
606 * We define a macro to allow ARC hits/misses to be easily broken down by
607 * two separate conditions, giving a total of four different subtypes for
608 * each of hits and misses (so eight statistics total).
609 */
610 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
611 if (cond1) { \
612 if (cond2) { \
613 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
614 } else { \
615 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
616 } \
617 } else { \
618 if (cond2) { \
619 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
620 } else { \
621 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
622 } \
623 }
624
625 kstat_t *arc_ksp;
626 static arc_state_t *arc_anon;
627 static arc_state_t *arc_mru;
628 static arc_state_t *arc_mru_ghost;
629 static arc_state_t *arc_mfu;
630 static arc_state_t *arc_mfu_ghost;
631 static arc_state_t *arc_l2c_only;
632
633 /*
634 * There are several ARC variables that are critical to export as kstats --
635 * but we don't want to have to grovel around in the kstat whenever we wish to
636 * manipulate them. For these variables, we therefore define them to be in
637 * terms of the statistic variable. This assures that we are not introducing
638 * the possibility of inconsistency by having shadow copies of the variables,
639 * while still allowing the code to be readable.
640 */
641 #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */
642 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */
643 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */
644 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */
645 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */
646 #define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */
647 #define arc_meta_min ARCSTAT(arcstat_meta_min) /* min size for metadata */
648 #define arc_meta_used ARCSTAT(arcstat_meta_used) /* size of metadata */
649 #define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */
650
651 #define L2ARC_IS_VALID_COMPRESS(_c_) \
652 ((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
653
654 static int arc_no_grow; /* Don't try to grow cache size */
655 static uint64_t arc_tempreserve;
656 static uint64_t arc_loaned_bytes;
657
658 typedef struct arc_callback arc_callback_t;
659
660 struct arc_callback {
661 void *acb_private;
662 arc_done_func_t *acb_done;
663 arc_buf_t *acb_buf;
664 zio_t *acb_zio_dummy;
665 arc_callback_t *acb_next;
666 };
667
668 typedef struct arc_write_callback arc_write_callback_t;
669
670 struct arc_write_callback {
671 void *awcb_private;
672 arc_done_func_t *awcb_ready;
673 arc_done_func_t *awcb_physdone;
674 arc_done_func_t *awcb_done;
675 arc_buf_t *awcb_buf;
676 };
677
678 /*
679 * ARC buffers are separated into multiple structs as a memory saving measure:
680 * - Common fields struct, always defined, and embedded within it:
681 * - L2-only fields, always allocated but undefined when not in L2ARC
682 * - L1-only fields, only allocated when in L1ARC
683 *
684 * Buffer in L1 Buffer only in L2
685 * +------------------------+ +------------------------+
686 * | arc_buf_hdr_t | | arc_buf_hdr_t |
687 * | | | |
688 * | | | |
689 * | | | |
690 * +------------------------+ +------------------------+
691 * | l2arc_buf_hdr_t | | l2arc_buf_hdr_t |
692 * | (undefined if L1-only) | | |
693 * +------------------------+ +------------------------+
694 * | l1arc_buf_hdr_t |
695 * | |
696 * | |
697 * | |
698 * | |
699 * +------------------------+
700 *
701 * Because it's possible for the L2ARC to become extremely large, we can wind
702 * up eating a lot of memory in L2ARC buffer headers, so the size of a header
703 * is minimized by only allocating the fields necessary for an L1-cached buffer
704 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
705 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
706 * words in pointers. arc_hdr_realloc() is used to switch a header between
707 * these two allocation states.
708 */
709 typedef struct l1arc_buf_hdr {
710 kmutex_t b_freeze_lock;
711 #ifdef ZFS_DEBUG
712 /*
713 * used for debugging wtih kmem_flags - by allocating and freeing
714 * b_thawed when the buffer is thawed, we get a record of the stack
715 * trace that thawed it.
716 */
717 void *b_thawed;
718 #endif
719
720 arc_buf_t *b_buf;
721 uint32_t b_datacnt;
722 /* for waiting on writes to complete */
723 kcondvar_t b_cv;
724
725 /* protected by arc state mutex */
726 arc_state_t *b_state;
727 multilist_node_t b_arc_node;
728
729 /* updated atomically */
730 clock_t b_arc_access;
731
732 /* self protecting */
733 refcount_t b_refcnt;
734
735 arc_callback_t *b_acb;
736 /* temporary buffer holder for in-flight compressed data */
737 void *b_tmp_cdata;
738 } l1arc_buf_hdr_t;
739
740 typedef struct l2arc_dev l2arc_dev_t;
741
742 typedef struct l2arc_buf_hdr {
743 /* protected by arc_buf_hdr mutex */
744 l2arc_dev_t *b_dev; /* L2ARC device */
745 uint64_t b_daddr; /* disk address, offset byte */
746 /* real alloc'd buffer size depending on b_compress applied */
747 int32_t b_asize;
748 uint8_t b_compress;
749
750 list_node_t b_l2node;
751 } l2arc_buf_hdr_t;
752
753 struct arc_buf_hdr {
754 /* protected by hash lock */
755 dva_t b_dva;
756 uint64_t b_birth;
757 /*
758 * Even though this checksum is only set/verified when a buffer is in
759 * the L1 cache, it needs to be in the set of common fields because it
760 * must be preserved from the time before a buffer is written out to
761 * L2ARC until after it is read back in.
762 */
763 zio_cksum_t *b_freeze_cksum;
764
765 arc_buf_hdr_t *b_hash_next;
766 arc_flags_t b_flags;
767
768 /* immutable */
769 int32_t b_size;
770 uint64_t b_spa;
771
772 /* L2ARC fields. Undefined when not in L2ARC. */
773 l2arc_buf_hdr_t b_l2hdr;
774 /* L1ARC fields. Undefined when in l2arc_only state */
775 l1arc_buf_hdr_t b_l1hdr;
776 };
777
778 static arc_buf_t *arc_eviction_list;
779 static arc_buf_hdr_t arc_eviction_hdr;
780
781 #define GHOST_STATE(state) \
782 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
783 (state) == arc_l2c_only)
784
785 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
786 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
787 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
788 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
789 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FLAG_FREED_IN_READ)
790 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE)
791
792 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
793 #define HDR_L2COMPRESS(hdr) ((hdr)->b_flags & ARC_FLAG_L2COMPRESS)
794 #define HDR_L2_READING(hdr) \
795 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
796 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
797 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
798 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
799 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
800
801 #define HDR_ISTYPE_METADATA(hdr) \
802 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
803 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr))
804
805 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
806 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
807
808 /*
809 * Other sizes
810 */
811
812 #define HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
813 #define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
814
815 /*
816 * Hash table routines
817 */
818
819 #define HT_LOCK_PAD 64
820
821 struct ht_lock {
822 kmutex_t ht_lock;
823 #ifdef _KERNEL
824 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
825 #endif
826 };
827
828 #define BUF_LOCKS 256
829 typedef struct buf_hash_table {
830 uint64_t ht_mask;
831 arc_buf_hdr_t **ht_table;
832 struct ht_lock ht_locks[BUF_LOCKS];
833 } buf_hash_table_t;
834
835 static buf_hash_table_t buf_hash_table;
836
837 #define BUF_HASH_INDEX(spa, dva, birth) \
838 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
839 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
840 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
841 #define HDR_LOCK(hdr) \
842 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
843
844 uint64_t zfs_crc64_table[256];
845
846 /*
847 * Level 2 ARC
848 */
849
850 #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */
851 #define L2ARC_HEADROOM 2 /* num of writes */
852 /*
853 * If we discover during ARC scan any buffers to be compressed, we boost
854 * our headroom for the next scanning cycle by this percentage multiple.
855 */
856 #define L2ARC_HEADROOM_BOOST 200
857 #define L2ARC_FEED_SECS 1 /* caching interval secs */
858 #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */
859
860 /*
861 * Used to distinguish headers that are being process by
862 * l2arc_write_buffers(), but have yet to be assigned to a l2arc disk
863 * address. This can happen when the header is added to the l2arc's list
864 * of buffers to write in the first stage of l2arc_write_buffers(), but
865 * has not yet been written out which happens in the second stage of
866 * l2arc_write_buffers().
867 */
868 #define L2ARC_ADDR_UNSET ((uint64_t)(-1))
869
870 #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent)
871 #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done)
872
873 /* L2ARC Performance Tunables */
874 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */
875 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */
876 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */
877 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
878 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */
879 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */
880 boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */
881 boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */
882 boolean_t l2arc_norw = B_TRUE; /* no reads during writes */
883
884 /*
885 * L2ARC Internals
886 */
887 struct l2arc_dev {
888 vdev_t *l2ad_vdev; /* vdev */
889 spa_t *l2ad_spa; /* spa */
890 uint64_t l2ad_hand; /* next write location */
891 uint64_t l2ad_start; /* first addr on device */
892 uint64_t l2ad_end; /* last addr on device */
893 boolean_t l2ad_first; /* first sweep through */
894 boolean_t l2ad_writing; /* currently writing */
895 kmutex_t l2ad_mtx; /* lock for buffer list */
896 list_t l2ad_buflist; /* buffer list */
897 list_node_t l2ad_node; /* device list node */
898 refcount_t l2ad_alloc; /* allocated bytes */
899 };
900
901 static list_t L2ARC_dev_list; /* device list */
902 static list_t *l2arc_dev_list; /* device list pointer */
903 static kmutex_t l2arc_dev_mtx; /* device list mutex */
904 static l2arc_dev_t *l2arc_dev_last; /* last device used */
905 static list_t L2ARC_free_on_write; /* free after write buf list */
906 static list_t *l2arc_free_on_write; /* free after write list ptr */
907 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */
908 static uint64_t l2arc_ndev; /* number of devices */
909
910 typedef struct l2arc_read_callback {
911 arc_buf_t *l2rcb_buf; /* read buffer */
912 spa_t *l2rcb_spa; /* spa */
913 blkptr_t l2rcb_bp; /* original blkptr */
914 zbookmark_phys_t l2rcb_zb; /* original bookmark */
915 int l2rcb_flags; /* original flags */
916 enum zio_compress l2rcb_compress; /* applied compress */
917 } l2arc_read_callback_t;
918
919 typedef struct l2arc_write_callback {
920 l2arc_dev_t *l2wcb_dev; /* device info */
921 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */
922 } l2arc_write_callback_t;
923
924 typedef struct l2arc_data_free {
925 /* protected by l2arc_free_on_write_mtx */
926 void *l2df_data;
927 size_t l2df_size;
928 void (*l2df_func)(void *, size_t);
929 list_node_t l2df_list_node;
930 } l2arc_data_free_t;
931
932 static kmutex_t l2arc_feed_thr_lock;
933 static kcondvar_t l2arc_feed_thr_cv;
934 static uint8_t l2arc_thread_exit;
935
936 static void arc_get_data_buf(arc_buf_t *);
937 static void arc_access(arc_buf_hdr_t *, kmutex_t *);
938 static boolean_t arc_is_overflowing();
939 static void arc_buf_watch(arc_buf_t *);
940
941 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
942 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
943
944 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
945 static void l2arc_read_done(zio_t *);
946
947 static boolean_t l2arc_compress_buf(arc_buf_hdr_t *);
948 static void l2arc_decompress_zio(zio_t *, arc_buf_hdr_t *, enum zio_compress);
949 static void l2arc_release_cdata_buf(arc_buf_hdr_t *);
950
951 static uint64_t
952 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
953 {
954 uint8_t *vdva = (uint8_t *)dva;
955 uint64_t crc = -1ULL;
956 int i;
957
958 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
959
960 for (i = 0; i < sizeof (dva_t); i++)
961 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
962
963 crc ^= (spa>>8) ^ birth;
964
965 return (crc);
966 }
967
968 #define BUF_EMPTY(buf) \
969 ((buf)->b_dva.dva_word[0] == 0 && \
970 (buf)->b_dva.dva_word[1] == 0)
971
972 #define BUF_EQUAL(spa, dva, birth, buf) \
973 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
974 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
975 ((buf)->b_birth == birth) && ((buf)->b_spa == spa)
976
977 static void
978 buf_discard_identity(arc_buf_hdr_t *hdr)
979 {
980 hdr->b_dva.dva_word[0] = 0;
981 hdr->b_dva.dva_word[1] = 0;
982 hdr->b_birth = 0;
983 }
984
985 static arc_buf_hdr_t *
986 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
987 {
988 const dva_t *dva = BP_IDENTITY(bp);
989 uint64_t birth = BP_PHYSICAL_BIRTH(bp);
990 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
991 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
992 arc_buf_hdr_t *hdr;
993
994 mutex_enter(hash_lock);
995 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
996 hdr = hdr->b_hash_next) {
997 if (BUF_EQUAL(spa, dva, birth, hdr)) {
998 *lockp = hash_lock;
999 return (hdr);
1000 }
1001 }
1002 mutex_exit(hash_lock);
1003 *lockp = NULL;
1004 return (NULL);
1005 }
1006
1007 /*
1008 * Insert an entry into the hash table. If there is already an element
1009 * equal to elem in the hash table, then the already existing element
1010 * will be returned and the new element will not be inserted.
1011 * Otherwise returns NULL.
1012 * If lockp == NULL, the caller is assumed to already hold the hash lock.
1013 */
1014 static arc_buf_hdr_t *
1015 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1016 {
1017 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1018 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1019 arc_buf_hdr_t *fhdr;
1020 uint32_t i;
1021
1022 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1023 ASSERT(hdr->b_birth != 0);
1024 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1025
1026 if (lockp != NULL) {
1027 *lockp = hash_lock;
1028 mutex_enter(hash_lock);
1029 } else {
1030 ASSERT(MUTEX_HELD(hash_lock));
1031 }
1032
1033 for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1034 fhdr = fhdr->b_hash_next, i++) {
1035 if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1036 return (fhdr);
1037 }
1038
1039 hdr->b_hash_next = buf_hash_table.ht_table[idx];
1040 buf_hash_table.ht_table[idx] = hdr;
1041 hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
1042
1043 /* collect some hash table performance data */
1044 if (i > 0) {
1045 ARCSTAT_BUMP(arcstat_hash_collisions);
1046 if (i == 1)
1047 ARCSTAT_BUMP(arcstat_hash_chains);
1048
1049 ARCSTAT_MAX(arcstat_hash_chain_max, i);
1050 }
1051
1052 ARCSTAT_BUMP(arcstat_hash_elements);
1053 ARCSTAT_MAXSTAT(arcstat_hash_elements);
1054
1055 return (NULL);
1056 }
1057
1058 static void
1059 buf_hash_remove(arc_buf_hdr_t *hdr)
1060 {
1061 arc_buf_hdr_t *fhdr, **hdrp;
1062 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1063
1064 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1065 ASSERT(HDR_IN_HASH_TABLE(hdr));
1066
1067 hdrp = &buf_hash_table.ht_table[idx];
1068 while ((fhdr = *hdrp) != hdr) {
1069 ASSERT(fhdr != NULL);
1070 hdrp = &fhdr->b_hash_next;
1071 }
1072 *hdrp = hdr->b_hash_next;
1073 hdr->b_hash_next = NULL;
1074 hdr->b_flags &= ~ARC_FLAG_IN_HASH_TABLE;
1075
1076 /* collect some hash table performance data */
1077 ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1078
1079 if (buf_hash_table.ht_table[idx] &&
1080 buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1081 ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1082 }
1083
1084 /*
1085 * Global data structures and functions for the buf kmem cache.
1086 */
1087 static kmem_cache_t *hdr_full_cache;
1088 static kmem_cache_t *hdr_l2only_cache;
1089 static kmem_cache_t *buf_cache;
1090
1091 static void
1092 buf_fini(void)
1093 {
1094 int i;
1095
1096 kmem_free(buf_hash_table.ht_table,
1097 (buf_hash_table.ht_mask + 1) * sizeof (void *));
1098 for (i = 0; i < BUF_LOCKS; i++)
1099 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1100 kmem_cache_destroy(hdr_full_cache);
1101 kmem_cache_destroy(hdr_l2only_cache);
1102 kmem_cache_destroy(buf_cache);
1103 }
1104
1105 /*
1106 * Constructor callback - called when the cache is empty
1107 * and a new buf is requested.
1108 */
1109 /* ARGSUSED */
1110 static int
1111 hdr_full_cons(void *vbuf, void *unused, int kmflag)
1112 {
1113 arc_buf_hdr_t *hdr = vbuf;
1114
1115 bzero(hdr, HDR_FULL_SIZE);
1116 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1117 refcount_create(&hdr->b_l1hdr.b_refcnt);
1118 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1119 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1120 arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1121
1122 return (0);
1123 }
1124
1125 /* ARGSUSED */
1126 static int
1127 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1128 {
1129 arc_buf_hdr_t *hdr = vbuf;
1130
1131 bzero(hdr, HDR_L2ONLY_SIZE);
1132 arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1133
1134 return (0);
1135 }
1136
1137 /* ARGSUSED */
1138 static int
1139 buf_cons(void *vbuf, void *unused, int kmflag)
1140 {
1141 arc_buf_t *buf = vbuf;
1142
1143 bzero(buf, sizeof (arc_buf_t));
1144 mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1145 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1146
1147 return (0);
1148 }
1149
1150 /*
1151 * Destructor callback - called when a cached buf is
1152 * no longer required.
1153 */
1154 /* ARGSUSED */
1155 static void
1156 hdr_full_dest(void *vbuf, void *unused)
1157 {
1158 arc_buf_hdr_t *hdr = vbuf;
1159
1160 ASSERT(BUF_EMPTY(hdr));
1161 cv_destroy(&hdr->b_l1hdr.b_cv);
1162 refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1163 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1164 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1165 arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1166 }
1167
1168 /* ARGSUSED */
1169 static void
1170 hdr_l2only_dest(void *vbuf, void *unused)
1171 {
1172 arc_buf_hdr_t *hdr = vbuf;
1173
1174 ASSERT(BUF_EMPTY(hdr));
1175 arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1176 }
1177
1178 /* ARGSUSED */
1179 static void
1180 buf_dest(void *vbuf, void *unused)
1181 {
1182 arc_buf_t *buf = vbuf;
1183
1184 mutex_destroy(&buf->b_evict_lock);
1185 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1186 }
1187
1188 /*
1189 * Reclaim callback -- invoked when memory is low.
1190 */
1191 /* ARGSUSED */
1192 static void
1193 hdr_recl(void *unused)
1194 {
1195 dprintf("hdr_recl called\n");
1196 /*
1197 * umem calls the reclaim func when we destroy the buf cache,
1198 * which is after we do arc_fini().
1199 */
1200 if (!arc_dead)
1201 cv_signal(&arc_reclaim_thread_cv);
1202 }
1203
1204 static void
1205 buf_init(void)
1206 {
1207 uint64_t *ct;
1208 uint64_t hsize = 1ULL << 12;
1209 int i, j;
1210
1211 /*
1212 * The hash table is big enough to fill all of physical memory
1213 * with an average block size of zfs_arc_average_blocksize (default 8K).
1214 * By default, the table will take up
1215 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1216 */
1217 while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE)
1218 hsize <<= 1;
1219 retry:
1220 buf_hash_table.ht_mask = hsize - 1;
1221 buf_hash_table.ht_table =
1222 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1223 if (buf_hash_table.ht_table == NULL) {
1224 ASSERT(hsize > (1ULL << 8));
1225 hsize >>= 1;
1226 goto retry;
1227 }
1228
1229 hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1230 0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1231 hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1232 HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1233 NULL, NULL, 0);
1234 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1235 0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1236
1237 for (i = 0; i < 256; i++)
1238 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1239 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1240
1241 for (i = 0; i < BUF_LOCKS; i++) {
1242 mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1243 NULL, MUTEX_DEFAULT, NULL);
1244 }
1245 }
1246
1247 /*
1248 * Transition between the two allocation states for the arc_buf_hdr struct.
1249 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
1250 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
1251 * version is used when a cache buffer is only in the L2ARC in order to reduce
1252 * memory usage.
1253 */
1254 static arc_buf_hdr_t *
1255 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
1256 {
1257 ASSERT(HDR_HAS_L2HDR(hdr));
1258
1259 arc_buf_hdr_t *nhdr;
1260 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1261
1262 ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
1263 (old == hdr_l2only_cache && new == hdr_full_cache));
1264
1265 nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
1266
1267 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
1268 buf_hash_remove(hdr);
1269
1270 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
1271
1272 if (new == hdr_full_cache) {
1273 nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1274 /*
1275 * arc_access and arc_change_state need to be aware that a
1276 * header has just come out of L2ARC, so we set its state to
1277 * l2c_only even though it's about to change.
1278 */
1279 nhdr->b_l1hdr.b_state = arc_l2c_only;
1280
1281 /* Verify previous threads set to NULL before freeing */
1282 ASSERT3P(nhdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1283 } else {
1284 ASSERT(hdr->b_l1hdr.b_buf == NULL);
1285 ASSERT0(hdr->b_l1hdr.b_datacnt);
1286
1287 /*
1288 * If we've reached here, We must have been called from
1289 * arc_evict_hdr(), as such we should have already been
1290 * removed from any ghost list we were previously on
1291 * (which protects us from racing with arc_evict_state),
1292 * thus no locking is needed during this check.
1293 */
1294 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1295
1296 /*
1297 * A buffer must not be moved into the arc_l2c_only
1298 * state if it's not finished being written out to the
1299 * l2arc device. Otherwise, the b_l1hdr.b_tmp_cdata field
1300 * might try to be accessed, even though it was removed.
1301 */
1302 VERIFY(!HDR_L2_WRITING(hdr));
1303 VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1304
1305 nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
1306 }
1307 /*
1308 * The header has been reallocated so we need to re-insert it into any
1309 * lists it was on.
1310 */
1311 (void) buf_hash_insert(nhdr, NULL);
1312
1313 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
1314
1315 mutex_enter(&dev->l2ad_mtx);
1316
1317 /*
1318 * We must place the realloc'ed header back into the list at
1319 * the same spot. Otherwise, if it's placed earlier in the list,
1320 * l2arc_write_buffers() could find it during the function's
1321 * write phase, and try to write it out to the l2arc.
1322 */
1323 list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
1324 list_remove(&dev->l2ad_buflist, hdr);
1325
1326 mutex_exit(&dev->l2ad_mtx);
1327
1328 /*
1329 * Since we're using the pointer address as the tag when
1330 * incrementing and decrementing the l2ad_alloc refcount, we
1331 * must remove the old pointer (that we're about to destroy) and
1332 * add the new pointer to the refcount. Otherwise we'd remove
1333 * the wrong pointer address when calling arc_hdr_destroy() later.
1334 */
1335
1336 (void) refcount_remove_many(&dev->l2ad_alloc,
1337 hdr->b_l2hdr.b_asize, hdr);
1338
1339 (void) refcount_add_many(&dev->l2ad_alloc,
1340 nhdr->b_l2hdr.b_asize, nhdr);
1341
1342 buf_discard_identity(hdr);
1343 hdr->b_freeze_cksum = NULL;
1344 kmem_cache_free(old, hdr);
1345
1346 return (nhdr);
1347 }
1348
1349
1350 #define ARC_MINTIME (hz>>4) /* 62 ms */
1351
1352 static void
1353 arc_cksum_verify(arc_buf_t *buf)
1354 {
1355 zio_cksum_t zc;
1356
1357 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1358 return;
1359
1360 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1361 if (buf->b_hdr->b_freeze_cksum == NULL || HDR_IO_ERROR(buf->b_hdr)) {
1362 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1363 return;
1364 }
1365 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1366 if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
1367 panic("buffer modified while frozen!");
1368 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1369 }
1370
1371 static int
1372 arc_cksum_equal(arc_buf_t *buf)
1373 {
1374 zio_cksum_t zc;
1375 int equal;
1376
1377 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1378 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1379 equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
1380 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1381
1382 return (equal);
1383 }
1384
1385 static void
1386 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
1387 {
1388 if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
1389 return;
1390
1391 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1392 if (buf->b_hdr->b_freeze_cksum != NULL) {
1393 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1394 return;
1395 }
1396 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1397 fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1398 buf->b_hdr->b_freeze_cksum);
1399 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1400 arc_buf_watch(buf);
1401 }
1402
1403 #ifndef _KERNEL
1404 typedef struct procctl {
1405 long cmd;
1406 prwatch_t prwatch;
1407 } procctl_t;
1408 #endif
1409
1410 /* ARGSUSED */
1411 static void
1412 arc_buf_unwatch(arc_buf_t *buf)
1413 {
1414 #ifndef _KERNEL
1415 if (arc_watch) {
1416 int result;
1417 procctl_t ctl;
1418 ctl.cmd = PCWATCH;
1419 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1420 ctl.prwatch.pr_size = 0;
1421 ctl.prwatch.pr_wflags = 0;
1422 result = write(arc_procfd, &ctl, sizeof (ctl));
1423 ASSERT3U(result, ==, sizeof (ctl));
1424 }
1425 #endif
1426 }
1427
1428 /* ARGSUSED */
1429 static void
1430 arc_buf_watch(arc_buf_t *buf)
1431 {
1432 #ifndef _KERNEL
1433 if (arc_watch) {
1434 int result;
1435 procctl_t ctl;
1436 ctl.cmd = PCWATCH;
1437 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1438 ctl.prwatch.pr_size = buf->b_hdr->b_size;
1439 ctl.prwatch.pr_wflags = WA_WRITE;
1440 result = write(arc_procfd, &ctl, sizeof (ctl));
1441 ASSERT3U(result, ==, sizeof (ctl));
1442 }
1443 #endif
1444 }
1445
1446 static arc_buf_contents_t
1447 arc_buf_type(arc_buf_hdr_t *hdr)
1448 {
1449 if (HDR_ISTYPE_METADATA(hdr)) {
1450 return (ARC_BUFC_METADATA);
1451 } else {
1452 return (ARC_BUFC_DATA);
1453 }
1454 }
1455
1456 static uint32_t
1457 arc_bufc_to_flags(arc_buf_contents_t type)
1458 {
1459 switch (type) {
1460 case ARC_BUFC_DATA:
1461 /* metadata field is 0 if buffer contains normal data */
1462 return (0);
1463 case ARC_BUFC_METADATA:
1464 return (ARC_FLAG_BUFC_METADATA);
1465 default:
1466 break;
1467 }
1468 panic("undefined ARC buffer type!");
1469 return ((uint32_t)-1);
1470 }
1471
1472 void
1473 arc_buf_thaw(arc_buf_t *buf)
1474 {
1475 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1476 if (buf->b_hdr->b_l1hdr.b_state != arc_anon)
1477 panic("modifying non-anon buffer!");
1478 if (HDR_IO_IN_PROGRESS(buf->b_hdr))
1479 panic("modifying buffer while i/o in progress!");
1480 arc_cksum_verify(buf);
1481 }
1482
1483 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1484 if (buf->b_hdr->b_freeze_cksum != NULL) {
1485 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1486 buf->b_hdr->b_freeze_cksum = NULL;
1487 }
1488
1489 #ifdef ZFS_DEBUG
1490 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1491 if (buf->b_hdr->b_l1hdr.b_thawed != NULL)
1492 kmem_free(buf->b_hdr->b_l1hdr.b_thawed, 1);
1493 buf->b_hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
1494 }
1495 #endif
1496
1497 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1498
1499 arc_buf_unwatch(buf);
1500 }
1501
1502 void
1503 arc_buf_freeze(arc_buf_t *buf)
1504 {
1505 kmutex_t *hash_lock;
1506
1507 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1508 return;
1509
1510 hash_lock = HDR_LOCK(buf->b_hdr);
1511 mutex_enter(hash_lock);
1512
1513 ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1514 buf->b_hdr->b_l1hdr.b_state == arc_anon);
1515 arc_cksum_compute(buf, B_FALSE);
1516 mutex_exit(hash_lock);
1517
1518 }
1519
1520 static void
1521 add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1522 {
1523 ASSERT(HDR_HAS_L1HDR(hdr));
1524 ASSERT(MUTEX_HELD(hash_lock));
1525 arc_state_t *state = hdr->b_l1hdr.b_state;
1526
1527 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
1528 (state != arc_anon)) {
1529 /* We don't use the L2-only state list. */
1530 if (state != arc_l2c_only) {
1531 arc_buf_contents_t type = arc_buf_type(hdr);
1532 uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
1533 multilist_t *list = &state->arcs_list[type];
1534 uint64_t *size = &state->arcs_lsize[type];
1535
1536 multilist_remove(list, hdr);
1537
1538 if (GHOST_STATE(state)) {
1539 ASSERT0(hdr->b_l1hdr.b_datacnt);
1540 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
1541 delta = hdr->b_size;
1542 }
1543 ASSERT(delta > 0);
1544 ASSERT3U(*size, >=, delta);
1545 atomic_add_64(size, -delta);
1546 }
1547 /* remove the prefetch flag if we get a reference */
1548 hdr->b_flags &= ~ARC_FLAG_PREFETCH;
1549 }
1550 }
1551
1552 static int
1553 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1554 {
1555 int cnt;
1556 arc_state_t *state = hdr->b_l1hdr.b_state;
1557
1558 ASSERT(HDR_HAS_L1HDR(hdr));
1559 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1560 ASSERT(!GHOST_STATE(state));
1561
1562 /*
1563 * arc_l2c_only counts as a ghost state so we don't need to explicitly
1564 * check to prevent usage of the arc_l2c_only list.
1565 */
1566 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
1567 (state != arc_anon)) {
1568 arc_buf_contents_t type = arc_buf_type(hdr);
1569 multilist_t *list = &state->arcs_list[type];
1570 uint64_t *size = &state->arcs_lsize[type];
1571
1572 multilist_insert(list, hdr);
1573
1574 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
1575 atomic_add_64(size, hdr->b_size *
1576 hdr->b_l1hdr.b_datacnt);
1577 }
1578 return (cnt);
1579 }
1580
1581 /*
1582 * Move the supplied buffer to the indicated state. The hash lock
1583 * for the buffer must be held by the caller.
1584 */
1585 static void
1586 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
1587 kmutex_t *hash_lock)
1588 {
1589 arc_state_t *old_state;
1590 int64_t refcnt;
1591 uint32_t datacnt;
1592 uint64_t from_delta, to_delta;
1593 arc_buf_contents_t buftype = arc_buf_type(hdr);
1594
1595 /*
1596 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
1597 * in arc_read() when bringing a buffer out of the L2ARC. However, the
1598 * L1 hdr doesn't always exist when we change state to arc_anon before
1599 * destroying a header, in which case reallocating to add the L1 hdr is
1600 * pointless.
1601 */
1602 if (HDR_HAS_L1HDR(hdr)) {
1603 old_state = hdr->b_l1hdr.b_state;
1604 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
1605 datacnt = hdr->b_l1hdr.b_datacnt;
1606 } else {
1607 old_state = arc_l2c_only;
1608 refcnt = 0;
1609 datacnt = 0;
1610 }
1611
1612 ASSERT(MUTEX_HELD(hash_lock));
1613 ASSERT3P(new_state, !=, old_state);
1614 ASSERT(refcnt == 0 || datacnt > 0);
1615 ASSERT(!GHOST_STATE(new_state) || datacnt == 0);
1616 ASSERT(old_state != arc_anon || datacnt <= 1);
1617
1618 from_delta = to_delta = datacnt * hdr->b_size;
1619
1620 /*
1621 * If this buffer is evictable, transfer it from the
1622 * old state list to the new state list.
1623 */
1624 if (refcnt == 0) {
1625 if (old_state != arc_anon && old_state != arc_l2c_only) {
1626 uint64_t *size = &old_state->arcs_lsize[buftype];
1627
1628 ASSERT(HDR_HAS_L1HDR(hdr));
1629 multilist_remove(&old_state->arcs_list[buftype], hdr);
1630
1631 /*
1632 * If prefetching out of the ghost cache,
1633 * we will have a non-zero datacnt.
1634 */
1635 if (GHOST_STATE(old_state) && datacnt == 0) {
1636 /* ghost elements have a ghost size */
1637 ASSERT(hdr->b_l1hdr.b_buf == NULL);
1638 from_delta = hdr->b_size;
1639 }
1640 ASSERT3U(*size, >=, from_delta);
1641 atomic_add_64(size, -from_delta);
1642 }
1643 if (new_state != arc_anon && new_state != arc_l2c_only) {
1644 uint64_t *size = &new_state->arcs_lsize[buftype];
1645
1646 /*
1647 * An L1 header always exists here, since if we're
1648 * moving to some L1-cached state (i.e. not l2c_only or
1649 * anonymous), we realloc the header to add an L1hdr
1650 * beforehand.
1651 */
1652 ASSERT(HDR_HAS_L1HDR(hdr));
1653 multilist_insert(&new_state->arcs_list[buftype], hdr);
1654
1655 /* ghost elements have a ghost size */
1656 if (GHOST_STATE(new_state)) {
1657 ASSERT0(datacnt);
1658 ASSERT(hdr->b_l1hdr.b_buf == NULL);
1659 to_delta = hdr->b_size;
1660 }
1661 atomic_add_64(size, to_delta);
1662 }
1663 }
1664
1665 ASSERT(!BUF_EMPTY(hdr));
1666 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
1667 buf_hash_remove(hdr);
1668
1669 /* adjust state sizes (ignore arc_l2c_only) */
1670
1671 if (to_delta && new_state != arc_l2c_only) {
1672 ASSERT(HDR_HAS_L1HDR(hdr));
1673 if (GHOST_STATE(new_state)) {
1674 ASSERT0(datacnt);
1675
1676 /*
1677 * We moving a header to a ghost state, we first
1678 * remove all arc buffers. Thus, we'll have a
1679 * datacnt of zero, and no arc buffer to use for
1680 * the reference. As a result, we use the arc
1681 * header pointer for the reference.
1682 */
1683 (void) refcount_add_many(&new_state->arcs_size,
1684 hdr->b_size, hdr);
1685 } else {
1686 ASSERT3U(datacnt, !=, 0);
1687
1688 /*
1689 * Each individual buffer holds a unique reference,
1690 * thus we must remove each of these references one
1691 * at a time.
1692 */
1693 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
1694 buf = buf->b_next) {
1695 (void) refcount_add_many(&new_state->arcs_size,
1696 hdr->b_size, buf);
1697 }
1698 }
1699 }
1700
1701 if (from_delta && old_state != arc_l2c_only) {
1702 ASSERT(HDR_HAS_L1HDR(hdr));
1703 if (GHOST_STATE(old_state)) {
1704 /*
1705 * When moving a header off of a ghost state,
1706 * there's the possibility for datacnt to be
1707 * non-zero. This is because we first add the
1708 * arc buffer to the header prior to changing
1709 * the header's state. Since we used the header
1710 * for the reference when putting the header on
1711 * the ghost state, we must balance that and use
1712 * the header when removing off the ghost state
1713 * (even though datacnt is non zero).
1714 */
1715
1716 IMPLY(datacnt == 0, new_state == arc_anon ||
1717 new_state == arc_l2c_only);
1718
1719 (void) refcount_remove_many(&old_state->arcs_size,
1720 hdr->b_size, hdr);
1721 } else {
1722 ASSERT3P(datacnt, !=, 0);
1723
1724 /*
1725 * Each individual buffer holds a unique reference,
1726 * thus we must remove each of these references one
1727 * at a time.
1728 */
1729 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
1730 buf = buf->b_next) {
1731 (void) refcount_remove_many(
1732 &old_state->arcs_size, hdr->b_size, buf);
1733 }
1734 }
1735 }
1736
1737 if (HDR_HAS_L1HDR(hdr))
1738 hdr->b_l1hdr.b_state = new_state;
1739
1740 /*
1741 * L2 headers should never be on the L2 state list since they don't
1742 * have L1 headers allocated.
1743 */
1744 ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
1745 multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
1746 }
1747
1748 void
1749 arc_space_consume(uint64_t space, arc_space_type_t type)
1750 {
1751 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1752
1753 switch (type) {
1754 case ARC_SPACE_DATA:
1755 ARCSTAT_INCR(arcstat_data_size, space);
1756 break;
1757 case ARC_SPACE_META:
1758 ARCSTAT_INCR(arcstat_metadata_size, space);
1759 break;
1760 case ARC_SPACE_OTHER:
1761 ARCSTAT_INCR(arcstat_other_size, space);
1762 break;
1763 case ARC_SPACE_HDRS:
1764 ARCSTAT_INCR(arcstat_hdr_size, space);
1765 break;
1766 case ARC_SPACE_L2HDRS:
1767 ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1768 break;
1769 }
1770
1771 if (type != ARC_SPACE_DATA)
1772 ARCSTAT_INCR(arcstat_meta_used, space);
1773
1774 atomic_add_64(&arc_size, space);
1775 }
1776
1777 void
1778 arc_space_return(uint64_t space, arc_space_type_t type)
1779 {
1780 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1781
1782 switch (type) {
1783 case ARC_SPACE_DATA:
1784 ARCSTAT_INCR(arcstat_data_size, -space);
1785 break;
1786 case ARC_SPACE_META:
1787 ARCSTAT_INCR(arcstat_metadata_size, -space);
1788 break;
1789 case ARC_SPACE_OTHER:
1790 ARCSTAT_INCR(arcstat_other_size, -space);
1791 break;
1792 case ARC_SPACE_HDRS:
1793 ARCSTAT_INCR(arcstat_hdr_size, -space);
1794 break;
1795 case ARC_SPACE_L2HDRS:
1796 ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1797 break;
1798 }
1799
1800 if (type != ARC_SPACE_DATA) {
1801 ASSERT(arc_meta_used >= space);
1802 if (arc_meta_max < arc_meta_used)
1803 arc_meta_max = arc_meta_used;
1804 ARCSTAT_INCR(arcstat_meta_used, -space);
1805 }
1806
1807 ASSERT(arc_size >= space);
1808 atomic_add_64(&arc_size, -space);
1809 }
1810
1811 arc_buf_t *
1812 arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
1813 {
1814 arc_buf_hdr_t *hdr;
1815 arc_buf_t *buf;
1816
1817 ASSERT3U(size, >, 0);
1818 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
1819 ASSERT(BUF_EMPTY(hdr));
1820 ASSERT3P(hdr->b_freeze_cksum, ==, NULL);
1821 hdr->b_size = size;
1822 hdr->b_spa = spa_load_guid(spa);
1823
1824 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1825 buf->b_hdr = hdr;
1826 buf->b_data = NULL;
1827 buf->b_efunc = NULL;
1828 buf->b_private = NULL;
1829 buf->b_next = NULL;
1830
1831 hdr->b_flags = arc_bufc_to_flags(type);
1832 hdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1833
1834 hdr->b_l1hdr.b_buf = buf;
1835 hdr->b_l1hdr.b_state = arc_anon;
1836 hdr->b_l1hdr.b_arc_access = 0;
1837 hdr->b_l1hdr.b_datacnt = 1;
1838 hdr->b_l1hdr.b_tmp_cdata = NULL;
1839
1840 arc_get_data_buf(buf);
1841 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
1842 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1843
1844 return (buf);
1845 }
1846
1847 static char *arc_onloan_tag = "onloan";
1848
1849 /*
1850 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1851 * flight data by arc_tempreserve_space() until they are "returned". Loaned
1852 * buffers must be returned to the arc before they can be used by the DMU or
1853 * freed.
1854 */
1855 arc_buf_t *
1856 arc_loan_buf(spa_t *spa, int size)
1857 {
1858 arc_buf_t *buf;
1859
1860 buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1861
1862 atomic_add_64(&arc_loaned_bytes, size);
1863 return (buf);
1864 }
1865
1866 /*
1867 * Return a loaned arc buffer to the arc.
1868 */
1869 void
1870 arc_return_buf(arc_buf_t *buf, void *tag)
1871 {
1872 arc_buf_hdr_t *hdr = buf->b_hdr;
1873
1874 ASSERT(buf->b_data != NULL);
1875 ASSERT(HDR_HAS_L1HDR(hdr));
1876 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1877 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1878
1879 atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1880 }
1881
1882 /* Detach an arc_buf from a dbuf (tag) */
1883 void
1884 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1885 {
1886 arc_buf_hdr_t *hdr = buf->b_hdr;
1887
1888 ASSERT(buf->b_data != NULL);
1889 ASSERT(HDR_HAS_L1HDR(hdr));
1890 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1891 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
1892 buf->b_efunc = NULL;
1893 buf->b_private = NULL;
1894
1895 atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1896 }
1897
1898 static arc_buf_t *
1899 arc_buf_clone(arc_buf_t *from)
1900 {
1901 arc_buf_t *buf;
1902 arc_buf_hdr_t *hdr = from->b_hdr;
1903 uint64_t size = hdr->b_size;
1904
1905 ASSERT(HDR_HAS_L1HDR(hdr));
1906 ASSERT(hdr->b_l1hdr.b_state != arc_anon);
1907
1908 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1909 buf->b_hdr = hdr;
1910 buf->b_data = NULL;
1911 buf->b_efunc = NULL;
1912 buf->b_private = NULL;
1913 buf->b_next = hdr->b_l1hdr.b_buf;
1914 hdr->b_l1hdr.b_buf = buf;
1915 arc_get_data_buf(buf);
1916 bcopy(from->b_data, buf->b_data, size);
1917
1918 /*
1919 * This buffer already exists in the arc so create a duplicate
1920 * copy for the caller. If the buffer is associated with user data
1921 * then track the size and number of duplicates. These stats will be
1922 * updated as duplicate buffers are created and destroyed.
1923 */
1924 if (HDR_ISTYPE_DATA(hdr)) {
1925 ARCSTAT_BUMP(arcstat_duplicate_buffers);
1926 ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1927 }
1928 hdr->b_l1hdr.b_datacnt += 1;
1929 return (buf);
1930 }
1931
1932 void
1933 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1934 {
1935 arc_buf_hdr_t *hdr;
1936 kmutex_t *hash_lock;
1937
1938 /*
1939 * Check to see if this buffer is evicted. Callers
1940 * must verify b_data != NULL to know if the add_ref
1941 * was successful.
1942 */
1943 mutex_enter(&buf->b_evict_lock);
1944 if (buf->b_data == NULL) {
1945 mutex_exit(&buf->b_evict_lock);
1946 return;
1947 }
1948 hash_lock = HDR_LOCK(buf->b_hdr);
1949 mutex_enter(hash_lock);
1950 hdr = buf->b_hdr;
1951 ASSERT(HDR_HAS_L1HDR(hdr));
1952 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1953 mutex_exit(&buf->b_evict_lock);
1954
1955 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
1956 hdr->b_l1hdr.b_state == arc_mfu);
1957
1958 add_reference(hdr, hash_lock, tag);
1959 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1960 arc_access(hdr, hash_lock);
1961 mutex_exit(hash_lock);
1962 ARCSTAT_BUMP(arcstat_hits);
1963 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
1964 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
1965 data, metadata, hits);
1966 }
1967
1968 static void
1969 arc_buf_free_on_write(void *data, size_t size,
1970 void (*free_func)(void *, size_t))
1971 {
1972 l2arc_data_free_t *df;
1973
1974 df = kmem_alloc(sizeof (*df), KM_SLEEP);
1975 df->l2df_data = data;
1976 df->l2df_size = size;
1977 df->l2df_func = free_func;
1978 mutex_enter(&l2arc_free_on_write_mtx);
1979 list_insert_head(l2arc_free_on_write, df);
1980 mutex_exit(&l2arc_free_on_write_mtx);
1981 }
1982
1983 /*
1984 * Free the arc data buffer. If it is an l2arc write in progress,
1985 * the buffer is placed on l2arc_free_on_write to be freed later.
1986 */
1987 static void
1988 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1989 {
1990 arc_buf_hdr_t *hdr = buf->b_hdr;
1991
1992 if (HDR_L2_WRITING(hdr)) {
1993 arc_buf_free_on_write(buf->b_data, hdr->b_size, free_func);
1994 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1995 } else {
1996 free_func(buf->b_data, hdr->b_size);
1997 }
1998 }
1999
2000 static void
2001 arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
2002 {
2003 ASSERT(HDR_HAS_L2HDR(hdr));
2004 ASSERT(MUTEX_HELD(&hdr->b_l2hdr.b_dev->l2ad_mtx));
2005
2006 /*
2007 * The b_tmp_cdata field is linked off of the b_l1hdr, so if
2008 * that doesn't exist, the header is in the arc_l2c_only state,
2009 * and there isn't anything to free (it's already been freed).
2010 */
2011 if (!HDR_HAS_L1HDR(hdr))
2012 return;
2013
2014 /*
2015 * The header isn't being written to the l2arc device, thus it
2016 * shouldn't have a b_tmp_cdata to free.
2017 */
2018 if (!HDR_L2_WRITING(hdr)) {
2019 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
2020 return;
2021 }
2022
2023 /*
2024 * The header does not have compression enabled. This can be due
2025 * to the buffer not being compressible, or because we're
2026 * freeing the buffer before the second phase of
2027 * l2arc_write_buffer() has started (which does the compression
2028 * step). In either case, b_tmp_cdata does not point to a
2029 * separately compressed buffer, so there's nothing to free (it
2030 * points to the same buffer as the arc_buf_t's b_data field).
2031 */
2032 if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_OFF) {
2033 hdr->b_l1hdr.b_tmp_cdata = NULL;
2034 return;
2035 }
2036
2037 /*
2038 * There's nothing to free since the buffer was all zero's and
2039 * compressed to a zero length buffer.
2040 */
2041 if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_EMPTY) {
2042 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
2043 return;
2044 }
2045
2046 ASSERT(L2ARC_IS_VALID_COMPRESS(hdr->b_l2hdr.b_compress));
2047
2048 arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata,
2049 hdr->b_size, zio_data_buf_free);
2050
2051 ARCSTAT_BUMP(arcstat_l2_cdata_free_on_write);
2052 hdr->b_l1hdr.b_tmp_cdata = NULL;
2053 }
2054
2055 /*
2056 * Free up buf->b_data and if 'remove' is set, then pull the
2057 * arc_buf_t off of the the arc_buf_hdr_t's list and free it.
2058 */
2059 static void
2060 arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
2061 {
2062 arc_buf_t **bufp;
2063
2064 /* free up data associated with the buf */
2065 if (buf->b_data != NULL) {
2066 arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
2067 uint64_t size = buf->b_hdr->b_size;
2068 arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
2069
2070 arc_cksum_verify(buf);
2071 arc_buf_unwatch(buf);
2072
2073 if (type == ARC_BUFC_METADATA) {
2074 arc_buf_data_free(buf, zio_buf_free);
2075 arc_space_return(size, ARC_SPACE_META);
2076 } else {
2077 ASSERT(type == ARC_BUFC_DATA);
2078 arc_buf_data_free(buf, zio_data_buf_free);
2079 arc_space_return(size, ARC_SPACE_DATA);
2080 }
2081
2082 /* protected by hash lock, if in the hash table */
2083 if (multilist_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
2084 uint64_t *cnt = &state->arcs_lsize[type];
2085
2086 ASSERT(refcount_is_zero(
2087 &buf->b_hdr->b_l1hdr.b_refcnt));
2088 ASSERT(state != arc_anon && state != arc_l2c_only);
2089
2090 ASSERT3U(*cnt, >=, size);
2091 atomic_add_64(cnt, -size);
2092 }
2093
2094 (void) refcount_remove_many(&state->arcs_size, size, buf);
2095 buf->b_data = NULL;
2096
2097 /*
2098 * If we're destroying a duplicate buffer make sure
2099 * that the appropriate statistics are updated.
2100 */
2101 if (buf->b_hdr->b_l1hdr.b_datacnt > 1 &&
2102 HDR_ISTYPE_DATA(buf->b_hdr)) {
2103 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
2104 ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
2105 }
2106 ASSERT(buf->b_hdr->b_l1hdr.b_datacnt > 0);
2107 buf->b_hdr->b_l1hdr.b_datacnt -= 1;
2108 }
2109
2110 /* only remove the buf if requested */
2111 if (!remove)
2112 return;
2113
2114 /* remove the buf from the hdr list */
2115 for (bufp = &buf->b_hdr->b_l1hdr.b_buf; *bufp != buf;
2116 bufp = &(*bufp)->b_next)
2117 continue;
2118 *bufp = buf->b_next;
2119 buf->b_next = NULL;
2120
2121 ASSERT(buf->b_efunc == NULL);
2122
2123 /* clean up the buf */
2124 buf->b_hdr = NULL;
2125 kmem_cache_free(buf_cache, buf);
2126 }
2127
2128 static void
2129 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
2130 {
2131 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
2132 l2arc_dev_t *dev = l2hdr->b_dev;
2133
2134 ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
2135 ASSERT(HDR_HAS_L2HDR(hdr));
2136
2137 list_remove(&dev->l2ad_buflist, hdr);
2138
2139 /*
2140 * We don't want to leak the b_tmp_cdata buffer that was
2141 * allocated in l2arc_write_buffers()
2142 */
2143 arc_buf_l2_cdata_free(hdr);
2144
2145 /*
2146 * If the l2hdr's b_daddr is equal to L2ARC_ADDR_UNSET, then
2147 * this header is being processed by l2arc_write_buffers() (i.e.
2148 * it's in the first stage of l2arc_write_buffers()).
2149 * Re-affirming that truth here, just to serve as a reminder. If
2150 * b_daddr does not equal L2ARC_ADDR_UNSET, then the header may or
2151 * may not have its HDR_L2_WRITING flag set. (the write may have
2152 * completed, in which case HDR_L2_WRITING will be false and the
2153 * b_daddr field will point to the address of the buffer on disk).
2154 */
2155 IMPLY(l2hdr->b_daddr == L2ARC_ADDR_UNSET, HDR_L2_WRITING(hdr));
2156
2157 /*
2158 * If b_daddr is equal to L2ARC_ADDR_UNSET, we're racing with
2159 * l2arc_write_buffers(). Since we've just removed this header
2160 * from the l2arc buffer list, this header will never reach the
2161 * second stage of l2arc_write_buffers(), which increments the
2162 * accounting stats for this header. Thus, we must be careful
2163 * not to decrement them for this header either.
2164 */
2165 if (l2hdr->b_daddr != L2ARC_ADDR_UNSET) {
2166 ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
2167 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
2168
2169 vdev_space_update(dev->l2ad_vdev,
2170 -l2hdr->b_asize, 0, 0);
2171
2172 (void) refcount_remove_many(&dev->l2ad_alloc,
2173 l2hdr->b_asize, hdr);
2174 }
2175
2176 hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
2177 }
2178
2179 static void
2180 arc_hdr_destroy(arc_buf_hdr_t *hdr)
2181 {
2182 if (HDR_HAS_L1HDR(hdr)) {
2183 ASSERT(hdr->b_l1hdr.b_buf == NULL ||
2184 hdr->b_l1hdr.b_datacnt > 0);
2185 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2186 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2187 }
2188 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2189 ASSERT(!HDR_IN_HASH_TABLE(hdr));
2190
2191 if (HDR_HAS_L2HDR(hdr)) {
2192 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
2193 boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
2194
2195 if (!buflist_held)
2196 mutex_enter(&dev->l2ad_mtx);
2197
2198 /*
2199 * Even though we checked this conditional above, we
2200 * need to check this again now that we have the
2201 * l2ad_mtx. This is because we could be racing with
2202 * another thread calling l2arc_evict() which might have
2203 * destroyed this header's L2 portion as we were waiting
2204 * to acquire the l2ad_mtx. If that happens, we don't
2205 * want to re-destroy the header's L2 portion.
2206 */
2207 if (HDR_HAS_L2HDR(hdr))
2208 arc_hdr_l2hdr_destroy(hdr);
2209
2210 if (!buflist_held)
2211 mutex_exit(&dev->l2ad_mtx);
2212 }
2213
2214 if (!BUF_EMPTY(hdr))
2215 buf_discard_identity(hdr);
2216
2217 if (hdr->b_freeze_cksum != NULL) {
2218 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
2219 hdr->b_freeze_cksum = NULL;
2220 }
2221
2222 if (HDR_HAS_L1HDR(hdr)) {
2223 while (hdr->b_l1hdr.b_buf) {
2224 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2225
2226 if (buf->b_efunc != NULL) {
2227 mutex_enter(&arc_user_evicts_lock);
2228 mutex_enter(&buf->b_evict_lock);
2229 ASSERT(buf->b_hdr != NULL);
2230 arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE);
2231 hdr->b_l1hdr.b_buf = buf->b_next;
2232 buf->b_hdr = &arc_eviction_hdr;
2233 buf->b_next = arc_eviction_list;
2234 arc_eviction_list = buf;
2235 mutex_exit(&buf->b_evict_lock);
2236 cv_signal(&arc_user_evicts_cv);
2237 mutex_exit(&arc_user_evicts_lock);
2238 } else {
2239 arc_buf_destroy(hdr->b_l1hdr.b_buf, TRUE);
2240 }
2241 }
2242 #ifdef ZFS_DEBUG
2243 if (hdr->b_l1hdr.b_thawed != NULL) {
2244 kmem_free(hdr->b_l1hdr.b_thawed, 1);
2245 hdr->b_l1hdr.b_thawed = NULL;
2246 }
2247 #endif
2248 }
2249
2250 ASSERT3P(hdr->b_hash_next, ==, NULL);
2251 if (HDR_HAS_L1HDR(hdr)) {
2252 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
2253 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
2254 kmem_cache_free(hdr_full_cache, hdr);
2255 } else {
2256 kmem_cache_free(hdr_l2only_cache, hdr);
2257 }
2258 }
2259
2260 void
2261 arc_buf_free(arc_buf_t *buf, void *tag)
2262 {
2263 arc_buf_hdr_t *hdr = buf->b_hdr;
2264 int hashed = hdr->b_l1hdr.b_state != arc_anon;
2265
2266 ASSERT(buf->b_efunc == NULL);
2267 ASSERT(buf->b_data != NULL);
2268
2269 if (hashed) {
2270 kmutex_t *hash_lock = HDR_LOCK(hdr);
2271
2272 mutex_enter(hash_lock);
2273 hdr = buf->b_hdr;
2274 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2275
2276 (void) remove_reference(hdr, hash_lock, tag);
2277 if (hdr->b_l1hdr.b_datacnt > 1) {
2278 arc_buf_destroy(buf, TRUE);
2279 } else {
2280 ASSERT(buf == hdr->b_l1hdr.b_buf);
2281 ASSERT(buf->b_efunc == NULL);
2282 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2283 }
2284 mutex_exit(hash_lock);
2285 } else if (HDR_IO_IN_PROGRESS(hdr)) {
2286 int destroy_hdr;
2287 /*
2288 * We are in the middle of an async write. Don't destroy
2289 * this buffer unless the write completes before we finish
2290 * decrementing the reference count.
2291 */
2292 mutex_enter(&arc_user_evicts_lock);
2293 (void) remove_reference(hdr, NULL, tag);
2294 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2295 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
2296 mutex_exit(&arc_user_evicts_lock);
2297 if (destroy_hdr)
2298 arc_hdr_destroy(hdr);
2299 } else {
2300 if (remove_reference(hdr, NULL, tag) > 0)
2301 arc_buf_destroy(buf, TRUE);
2302 else
2303 arc_hdr_destroy(hdr);
2304 }
2305 }
2306
2307 boolean_t
2308 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
2309 {
2310 arc_buf_hdr_t *hdr = buf->b_hdr;
2311 kmutex_t *hash_lock = HDR_LOCK(hdr);
2312 boolean_t no_callback = (buf->b_efunc == NULL);
2313
2314 if (hdr->b_l1hdr.b_state == arc_anon) {
2315 ASSERT(hdr->b_l1hdr.b_datacnt == 1);
2316 arc_buf_free(buf, tag);
2317 return (no_callback);
2318 }
2319
2320 mutex_enter(hash_lock);
2321 hdr = buf->b_hdr;
2322 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
2323 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2324 ASSERT(hdr->b_l1hdr.b_state != arc_anon);
2325 ASSERT(buf->b_data != NULL);
2326
2327 (void) remove_reference(hdr, hash_lock, tag);
2328 if (hdr->b_l1hdr.b_datacnt > 1) {
2329 if (no_callback)
2330 arc_buf_destroy(buf, TRUE);
2331 } else if (no_callback) {
2332 ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
2333 ASSERT(buf->b_efunc == NULL);
2334 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2335 }
2336 ASSERT(no_callback || hdr->b_l1hdr.b_datacnt > 1 ||
2337 refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2338 mutex_exit(hash_lock);
2339 return (no_callback);
2340 }
2341
2342 int32_t
2343 arc_buf_size(arc_buf_t *buf)
2344 {
2345 return (buf->b_hdr->b_size);
2346 }
2347
2348 /*
2349 * Called from the DMU to determine if the current buffer should be
2350 * evicted. In order to ensure proper locking, the eviction must be initiated
2351 * from the DMU. Return true if the buffer is associated with user data and
2352 * duplicate buffers still exist.
2353 */
2354 boolean_t
2355 arc_buf_eviction_needed(arc_buf_t *buf)
2356 {
2357 arc_buf_hdr_t *hdr;
2358 boolean_t evict_needed = B_FALSE;
2359
2360 if (zfs_disable_dup_eviction)
2361 return (B_FALSE);
2362
2363 mutex_enter(&buf->b_evict_lock);
2364 hdr = buf->b_hdr;
2365 if (hdr == NULL) {
2366 /*
2367 * We are in arc_do_user_evicts(); let that function
2368 * perform the eviction.
2369 */
2370 ASSERT(buf->b_data == NULL);
2371 mutex_exit(&buf->b_evict_lock);
2372 return (B_FALSE);
2373 } else if (buf->b_data == NULL) {
2374 /*
2375 * We have already been added to the arc eviction list;
2376 * recommend eviction.
2377 */
2378 ASSERT3P(hdr, ==, &arc_eviction_hdr);
2379 mutex_exit(&buf->b_evict_lock);
2380 return (B_TRUE);
2381 }
2382
2383 if (hdr->b_l1hdr.b_datacnt > 1 && HDR_ISTYPE_DATA(hdr))
2384 evict_needed = B_TRUE;
2385
2386 mutex_exit(&buf->b_evict_lock);
2387 return (evict_needed);
2388 }
2389
2390 /*
2391 * Evict the arc_buf_hdr that is provided as a parameter. The resultant
2392 * state of the header is dependent on it's state prior to entering this
2393 * function. The following transitions are possible:
2394 *
2395 * - arc_mru -> arc_mru_ghost
2396 * - arc_mfu -> arc_mfu_ghost
2397 * - arc_mru_ghost -> arc_l2c_only
2398 * - arc_mru_ghost -> deleted
2399 * - arc_mfu_ghost -> arc_l2c_only
2400 * - arc_mfu_ghost -> deleted
2401 */
2402 static int64_t
2403 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
2404 {
2405 arc_state_t *evicted_state, *state;
2406 int64_t bytes_evicted = 0;
2407
2408 ASSERT(MUTEX_HELD(hash_lock));
2409 ASSERT(HDR_HAS_L1HDR(hdr));
2410
2411 state = hdr->b_l1hdr.b_state;
2412 if (GHOST_STATE(state)) {
2413 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2414 ASSERT(hdr->b_l1hdr.b_buf == NULL);
2415
2416 /*
2417 * l2arc_write_buffers() relies on a header's L1 portion
2418 * (i.e. it's b_tmp_cdata field) during it's write phase.
2419 * Thus, we cannot push a header onto the arc_l2c_only
2420 * state (removing it's L1 piece) until the header is
2421 * done being written to the l2arc.
2422 */
2423 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
2424 ARCSTAT_BUMP(arcstat_evict_l2_skip);
2425 return (bytes_evicted);
2426 }
2427
2428 ARCSTAT_BUMP(arcstat_deleted);
2429 bytes_evicted += hdr->b_size;
2430
2431 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
2432
2433 if (HDR_HAS_L2HDR(hdr)) {
2434 /*
2435 * This buffer is cached on the 2nd Level ARC;
2436 * don't destroy the header.
2437 */
2438 arc_change_state(arc_l2c_only, hdr, hash_lock);
2439 /*
2440 * dropping from L1+L2 cached to L2-only,
2441 * realloc to remove the L1 header.
2442 */
2443 hdr = arc_hdr_realloc(hdr, hdr_full_cache,
2444 hdr_l2only_cache);
2445 } else {
2446 arc_change_state(arc_anon, hdr, hash_lock);
2447 arc_hdr_destroy(hdr);
2448 }
2449 return (bytes_evicted);
2450 }
2451
2452 ASSERT(state == arc_mru || state == arc_mfu);
2453 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
2454
2455 /* prefetch buffers have a minimum lifespan */
2456 if (HDR_IO_IN_PROGRESS(hdr) ||
2457 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
2458 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
2459 arc_min_prefetch_lifespan)) {
2460 ARCSTAT_BUMP(arcstat_evict_skip);
2461 return (bytes_evicted);
2462 }
2463
2464 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
2465 ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
2466 while (hdr->b_l1hdr.b_buf) {
2467 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2468 if (!mutex_tryenter(&buf->b_evict_lock)) {
2469 ARCSTAT_BUMP(arcstat_mutex_miss);
2470 break;
2471 }
2472 if (buf->b_data != NULL)
2473 bytes_evicted += hdr->b_size;
2474 if (buf->b_efunc != NULL) {
2475 mutex_enter(&arc_user_evicts_lock);
2476 arc_buf_destroy(buf, FALSE);
2477 hdr->b_l1hdr.b_buf = buf->b_next;
2478 buf->b_hdr = &arc_eviction_hdr;
2479 buf->b_next = arc_eviction_list;
2480 arc_eviction_list = buf;
2481 cv_signal(&arc_user_evicts_cv);
2482 mutex_exit(&arc_user_evicts_lock);
2483 mutex_exit(&buf->b_evict_lock);
2484 } else {
2485 mutex_exit(&buf->b_evict_lock);
2486 arc_buf_destroy(buf, TRUE);
2487 }
2488 }
2489
2490 if (HDR_HAS_L2HDR(hdr)) {
2491 ARCSTAT_INCR(arcstat_evict_l2_cached, hdr->b_size);
2492 } else {
2493 if (l2arc_write_eligible(hdr->b_spa, hdr))
2494 ARCSTAT_INCR(arcstat_evict_l2_eligible, hdr->b_size);
2495 else
2496 ARCSTAT_INCR(arcstat_evict_l2_ineligible, hdr->b_size);
2497 }
2498
2499 if (hdr->b_l1hdr.b_datacnt == 0) {
2500 arc_change_state(evicted_state, hdr, hash_lock);
2501 ASSERT(HDR_IN_HASH_TABLE(hdr));
2502 hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
2503 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
2504 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
2505 }
2506
2507 return (bytes_evicted);
2508 }
2509
2510 static uint64_t
2511 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
2512 uint64_t spa, int64_t bytes)
2513 {
2514 multilist_sublist_t *mls;
2515 uint64_t bytes_evicted = 0;
2516 arc_buf_hdr_t *hdr;
2517 kmutex_t *hash_lock;
2518 int evict_count = 0;
2519
2520 ASSERT3P(marker, !=, NULL);
2521 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2522
2523 mls = multilist_sublist_lock(ml, idx);
2524
2525 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
2526 hdr = multilist_sublist_prev(mls, marker)) {
2527 if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
2528 (evict_count >= zfs_arc_evict_batch_limit))
2529 break;
2530
2531 /*
2532 * To keep our iteration location, move the marker
2533 * forward. Since we're not holding hdr's hash lock, we
2534 * must be very careful and not remove 'hdr' from the
2535 * sublist. Otherwise, other consumers might mistake the
2536 * 'hdr' as not being on a sublist when they call the
2537 * multilist_link_active() function (they all rely on
2538 * the hash lock protecting concurrent insertions and
2539 * removals). multilist_sublist_move_forward() was
2540 * specifically implemented to ensure this is the case
2541 * (only 'marker' will be removed and re-inserted).
2542 */
2543 multilist_sublist_move_forward(mls, marker);
2544
2545 /*
2546 * The only case where the b_spa field should ever be
2547 * zero, is the marker headers inserted by
2548 * arc_evict_state(). It's possible for multiple threads
2549 * to be calling arc_evict_state() concurrently (e.g.
2550 * dsl_pool_close() and zio_inject_fault()), so we must
2551 * skip any markers we see from these other threads.
2552 */
2553 if (hdr->b_spa == 0)
2554 continue;
2555
2556 /* we're only interested in evicting buffers of a certain spa */
2557 if (spa != 0 && hdr->b_spa != spa) {
2558 ARCSTAT_BUMP(arcstat_evict_skip);
2559 continue;
2560 }
2561
2562 hash_lock = HDR_LOCK(hdr);
2563
2564 /*
2565 * We aren't calling this function from any code path
2566 * that would already be holding a hash lock, so we're
2567 * asserting on this assumption to be defensive in case
2568 * this ever changes. Without this check, it would be
2569 * possible to incorrectly increment arcstat_mutex_miss
2570 * below (e.g. if the code changed such that we called
2571 * this function with a hash lock held).
2572 */
2573 ASSERT(!MUTEX_HELD(hash_lock));
2574
2575 if (mutex_tryenter(hash_lock)) {
2576 uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
2577 mutex_exit(hash_lock);
2578
2579 bytes_evicted += evicted;
2580
2581 /*
2582 * If evicted is zero, arc_evict_hdr() must have
2583 * decided to skip this header, don't increment
2584 * evict_count in this case.
2585 */
2586 if (evicted != 0)
2587 evict_count++;
2588
2589 /*
2590 * If arc_size isn't overflowing, signal any
2591 * threads that might happen to be waiting.
2592 *
2593 * For each header evicted, we wake up a single
2594 * thread. If we used cv_broadcast, we could
2595 * wake up "too many" threads causing arc_size
2596 * to significantly overflow arc_c; since
2597 * arc_get_data_buf() doesn't check for overflow
2598 * when it's woken up (it doesn't because it's
2599 * possible for the ARC to be overflowing while
2600 * full of un-evictable buffers, and the
2601 * function should proceed in this case).
2602 *
2603 * If threads are left sleeping, due to not
2604 * using cv_broadcast, they will be woken up
2605 * just before arc_reclaim_thread() sleeps.
2606 */
2607 mutex_enter(&arc_reclaim_lock);
2608 if (!arc_is_overflowing())
2609 cv_signal(&arc_reclaim_waiters_cv);
2610 mutex_exit(&arc_reclaim_lock);
2611 } else {
2612 ARCSTAT_BUMP(arcstat_mutex_miss);
2613 }
2614 }
2615
2616 multilist_sublist_unlock(mls);
2617
2618 return (bytes_evicted);
2619 }
2620
2621 /*
2622 * Evict buffers from the given arc state, until we've removed the
2623 * specified number of bytes. Move the removed buffers to the
2624 * appropriate evict state.
2625 *
2626 * This function makes a "best effort". It skips over any buffers
2627 * it can't get a hash_lock on, and so, may not catch all candidates.
2628 * It may also return without evicting as much space as requested.
2629 *
2630 * If bytes is specified using the special value ARC_EVICT_ALL, this
2631 * will evict all available (i.e. unlocked and evictable) buffers from
2632 * the given arc state; which is used by arc_flush().
2633 */
2634 static uint64_t
2635 arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
2636 arc_buf_contents_t type)
2637 {
2638 uint64_t total_evicted = 0;
2639 multilist_t *ml = &state->arcs_list[type];
2640 int num_sublists;
2641 arc_buf_hdr_t **markers;
2642
2643 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2644
2645 num_sublists = multilist_get_num_sublists(ml);
2646
2647 /*
2648 * If we've tried to evict from each sublist, made some
2649 * progress, but still have not hit the target number of bytes
2650 * to evict, we want to keep trying. The markers allow us to
2651 * pick up where we left off for each individual sublist, rather
2652 * than starting from the tail each time.
2653 */
2654 markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
2655 for (int i = 0; i < num_sublists; i++) {
2656 markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
2657
2658 /*
2659 * A b_spa of 0 is used to indicate that this header is
2660 * a marker. This fact is used in arc_adjust_type() and
2661 * arc_evict_state_impl().
2662 */
2663 markers[i]->b_spa = 0;
2664
2665 multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2666 multilist_sublist_insert_tail(mls, markers[i]);
2667 multilist_sublist_unlock(mls);
2668 }
2669
2670 /*
2671 * While we haven't hit our target number of bytes to evict, or
2672 * we're evicting all available buffers.
2673 */
2674 while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
2675 /*
2676 * Start eviction using a randomly selected sublist,
2677 * this is to try and evenly balance eviction across all
2678 * sublists. Always starting at the same sublist
2679 * (e.g. index 0) would cause evictions to favor certain
2680 * sublists over others.
2681 */
2682 int sublist_idx = multilist_get_random_index(ml);
2683 uint64_t scan_evicted = 0;
2684
2685 for (int i = 0; i < num_sublists; i++) {
2686 uint64_t bytes_remaining;
2687 uint64_t bytes_evicted;
2688
2689 if (bytes == ARC_EVICT_ALL)
2690 bytes_remaining = ARC_EVICT_ALL;
2691 else if (total_evicted < bytes)
2692 bytes_remaining = bytes - total_evicted;
2693 else
2694 break;
2695
2696 bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
2697 markers[sublist_idx], spa, bytes_remaining);
2698
2699 scan_evicted += bytes_evicted;
2700 total_evicted += bytes_evicted;
2701
2702 /* we've reached the end, wrap to the beginning */
2703 if (++sublist_idx >= num_sublists)
2704 sublist_idx = 0;
2705 }
2706
2707 /*
2708 * If we didn't evict anything during this scan, we have
2709 * no reason to believe we'll evict more during another
2710 * scan, so break the loop.
2711 */
2712 if (scan_evicted == 0) {
2713 /* This isn't possible, let's make that obvious */
2714 ASSERT3S(bytes, !=, 0);
2715
2716 /*
2717 * When bytes is ARC_EVICT_ALL, the only way to
2718 * break the loop is when scan_evicted is zero.
2719 * In that case, we actually have evicted enough,
2720 * so we don't want to increment the kstat.
2721 */
2722 if (bytes != ARC_EVICT_ALL) {
2723 ASSERT3S(total_evicted, <, bytes);
2724 ARCSTAT_BUMP(arcstat_evict_not_enough);
2725 }
2726
2727 break;
2728 }
2729 }
2730
2731 for (int i = 0; i < num_sublists; i++) {
2732 multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2733 multilist_sublist_remove(mls, markers[i]);
2734 multilist_sublist_unlock(mls);
2735
2736 kmem_cache_free(hdr_full_cache, markers[i]);
2737 }
2738 kmem_free(markers, sizeof (*markers) * num_sublists);
2739
2740 return (total_evicted);
2741 }
2742
2743 /*
2744 * Flush all "evictable" data of the given type from the arc state
2745 * specified. This will not evict any "active" buffers (i.e. referenced).
2746 *
2747 * When 'retry' is set to FALSE, the function will make a single pass
2748 * over the state and evict any buffers that it can. Since it doesn't
2749 * continually retry the eviction, it might end up leaving some buffers
2750 * in the ARC due to lock misses.
2751 *
2752 * When 'retry' is set to TRUE, the function will continually retry the
2753 * eviction until *all* evictable buffers have been removed from the
2754 * state. As a result, if concurrent insertions into the state are
2755 * allowed (e.g. if the ARC isn't shutting down), this function might
2756 * wind up in an infinite loop, continually trying to evict buffers.
2757 */
2758 static uint64_t
2759 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
2760 boolean_t retry)
2761 {
2762 uint64_t evicted = 0;
2763
2764 while (state->arcs_lsize[type] != 0) {
2765 evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
2766
2767 if (!retry)
2768 break;
2769 }
2770
2771 return (evicted);
2772 }
2773
2774 /*
2775 * Evict the specified number of bytes from the state specified,
2776 * restricting eviction to the spa and type given. This function
2777 * prevents us from trying to evict more from a state's list than
2778 * is "evictable", and to skip evicting altogether when passed a
2779 * negative value for "bytes". In contrast, arc_evict_state() will
2780 * evict everything it can, when passed a negative value for "bytes".
2781 */
2782 static uint64_t
2783 arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
2784 arc_buf_contents_t type)
2785 {
2786 int64_t delta;
2787
2788 if (bytes > 0 && state->arcs_lsize[type] > 0) {
2789 delta = MIN(state->arcs_lsize[type], bytes);
2790 return (arc_evict_state(state, spa, delta, type));
2791 }
2792
2793 return (0);
2794 }
2795
2796 /*
2797 * Evict metadata buffers from the cache, such that arc_meta_used is
2798 * capped by the arc_meta_limit tunable.
2799 */
2800 static uint64_t
2801 arc_adjust_meta(void)
2802 {
2803 uint64_t total_evicted = 0;
2804 int64_t target;
2805
2806 /*
2807 * If we're over the meta limit, we want to evict enough
2808 * metadata to get back under the meta limit. We don't want to
2809 * evict so much that we drop the MRU below arc_p, though. If
2810 * we're over the meta limit more than we're over arc_p, we
2811 * evict some from the MRU here, and some from the MFU below.
2812 */
2813 target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2814 (int64_t)(refcount_count(&arc_anon->arcs_size) +
2815 refcount_count(&arc_mru->arcs_size) - arc_p));
2816
2817 total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2818
2819 /*
2820 * Similar to the above, we want to evict enough bytes to get us
2821 * below the meta limit, but not so much as to drop us below the
2822 * space alloted to the MFU (which is defined as arc_c - arc_p).
2823 */
2824 target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2825 (int64_t)(refcount_count(&arc_mfu->arcs_size) - (arc_c - arc_p)));
2826
2827 total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2828
2829 return (total_evicted);
2830 }
2831
2832 /*
2833 * Return the type of the oldest buffer in the given arc state
2834 *
2835 * This function will select a random sublist of type ARC_BUFC_DATA and
2836 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
2837 * is compared, and the type which contains the "older" buffer will be
2838 * returned.
2839 */
2840 static arc_buf_contents_t
2841 arc_adjust_type(arc_state_t *state)
2842 {
2843 multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
2844 multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
2845 int data_idx = multilist_get_random_index(data_ml);
2846 int meta_idx = multilist_get_random_index(meta_ml);
2847 multilist_sublist_t *data_mls;
2848 multilist_sublist_t *meta_mls;
2849 arc_buf_contents_t type;
2850 arc_buf_hdr_t *data_hdr;
2851 arc_buf_hdr_t *meta_hdr;
2852
2853 /*
2854 * We keep the sublist lock until we're finished, to prevent
2855 * the headers from being destroyed via arc_evict_state().
2856 */
2857 data_mls = multilist_sublist_lock(data_ml, data_idx);
2858 meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
2859
2860 /*
2861 * These two loops are to ensure we skip any markers that
2862 * might be at the tail of the lists due to arc_evict_state().
2863 */
2864
2865 for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
2866 data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
2867 if (data_hdr->b_spa != 0)
2868 break;
2869 }
2870
2871 for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
2872 meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
2873 if (meta_hdr->b_spa != 0)
2874 break;
2875 }
2876
2877 if (data_hdr == NULL && meta_hdr == NULL) {
2878 type = ARC_BUFC_DATA;
2879 } else if (data_hdr == NULL) {
2880 ASSERT3P(meta_hdr, !=, NULL);
2881 type = ARC_BUFC_METADATA;
2882 } else if (meta_hdr == NULL) {
2883 ASSERT3P(data_hdr, !=, NULL);
2884 type = ARC_BUFC_DATA;
2885 } else {
2886 ASSERT3P(data_hdr, !=, NULL);
2887 ASSERT3P(meta_hdr, !=, NULL);
2888
2889 /* The headers can't be on the sublist without an L1 header */
2890 ASSERT(HDR_HAS_L1HDR(data_hdr));
2891 ASSERT(HDR_HAS_L1HDR(meta_hdr));
2892
2893 if (data_hdr->b_l1hdr.b_arc_access <
2894 meta_hdr->b_l1hdr.b_arc_access) {
2895 type = ARC_BUFC_DATA;
2896 } else {
2897 type = ARC_BUFC_METADATA;
2898 }
2899 }
2900
2901 multilist_sublist_unlock(meta_mls);
2902 multilist_sublist_unlock(data_mls);
2903
2904 return (type);
2905 }
2906
2907 /*
2908 * Evict buffers from the cache, such that arc_size is capped by arc_c.
2909 */
2910 static uint64_t
2911 arc_adjust(void)
2912 {
2913 uint64_t total_evicted = 0;
2914 uint64_t bytes;
2915 int64_t target;
2916
2917 /*
2918 * If we're over arc_meta_limit, we want to correct that before
2919 * potentially evicting data buffers below.
2920 */
2921 total_evicted += arc_adjust_meta();
2922
2923 /*
2924 * Adjust MRU size
2925 *
2926 * If we're over the target cache size, we want to evict enough
2927 * from the list to get back to our target size. We don't want
2928 * to evict too much from the MRU, such that it drops below
2929 * arc_p. So, if we're over our target cache size more than
2930 * the MRU is over arc_p, we'll evict enough to get back to
2931 * arc_p here, and then evict more from the MFU below.
2932 */
2933 target = MIN((int64_t)(arc_size - arc_c),
2934 (int64_t)(refcount_count(&arc_anon->arcs_size) +
2935 refcount_count(&arc_mru->arcs_size) + arc_meta_used - arc_p));
2936
2937 /*
2938 * If we're below arc_meta_min, always prefer to evict data.
2939 * Otherwise, try to satisfy the requested number of bytes to
2940 * evict from the type which contains older buffers; in an
2941 * effort to keep newer buffers in the cache regardless of their
2942 * type. If we cannot satisfy the number of bytes from this
2943 * type, spill over into the next type.
2944 */
2945 if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2946 arc_meta_used > arc_meta_min) {
2947 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2948 total_evicted += bytes;
2949
2950 /*
2951 * If we couldn't evict our target number of bytes from
2952 * metadata, we try to get the rest from data.
2953 */
2954 target -= bytes;
2955
2956 total_evicted +=
2957 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2958 } else {
2959 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2960 total_evicted += bytes;
2961
2962 /*
2963 * If we couldn't evict our target number of bytes from
2964 * data, we try to get the rest from metadata.
2965 */
2966 target -= bytes;
2967
2968 total_evicted +=
2969 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2970 }
2971
2972 /*
2973 * Adjust MFU size
2974 *
2975 * Now that we've tried to evict enough from the MRU to get its
2976 * size back to arc_p, if we're still above the target cache
2977 * size, we evict the rest from the MFU.
2978 */
2979 target = arc_size - arc_c;
2980
2981 if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2982 arc_meta_used > arc_meta_min) {
2983 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2984 total_evicted += bytes;
2985
2986 /*
2987 * If we couldn't evict our target number of bytes from
2988 * metadata, we try to get the rest from data.
2989 */
2990 target -= bytes;
2991
2992 total_evicted +=
2993 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2994 } else {
2995 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2996 total_evicted += bytes;
2997
2998 /*
2999 * If we couldn't evict our target number of bytes from
3000 * data, we try to get the rest from data.
3001 */
3002 target -= bytes;
3003
3004 total_evicted +=
3005 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
3006 }
3007
3008 /*
3009 * Adjust ghost lists
3010 *
3011 * In addition to the above, the ARC also defines target values
3012 * for the ghost lists. The sum of the mru list and mru ghost
3013 * list should never exceed the target size of the cache, and
3014 * the sum of the mru list, mfu list, mru ghost list, and mfu
3015 * ghost list should never exceed twice the target size of the
3016 * cache. The following logic enforces these limits on the ghost
3017 * caches, and evicts from them as needed.
3018 */
3019 target = refcount_count(&arc_mru->arcs_size) +
3020 refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
3021
3022 bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
3023 total_evicted += bytes;
3024
3025 target -= bytes;
3026
3027 total_evicted +=
3028 arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
3029
3030 /*
3031 * We assume the sum of the mru list and mfu list is less than
3032 * or equal to arc_c (we enforced this above), which means we
3033 * can use the simpler of the two equations below:
3034 *
3035 * mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
3036 * mru ghost + mfu ghost <= arc_c
3037 */
3038 target = refcount_count(&arc_mru_ghost->arcs_size) +
3039 refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
3040
3041 bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
3042 total_evicted += bytes;
3043
3044 target -= bytes;
3045
3046 total_evicted +=
3047 arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
3048
3049 return (total_evicted);
3050 }
3051
3052 static void
3053 arc_do_user_evicts(void)
3054 {
3055 mutex_enter(&arc_user_evicts_lock);
3056 while (arc_eviction_list != NULL) {
3057 arc_buf_t *buf = arc_eviction_list;
3058 arc_eviction_list = buf->b_next;
3059 mutex_enter(&buf->b_evict_lock);
3060 buf->b_hdr = NULL;
3061 mutex_exit(&buf->b_evict_lock);
3062 mutex_exit(&arc_user_evicts_lock);
3063
3064 if (buf->b_efunc != NULL)
3065 VERIFY0(buf->b_efunc(buf->b_private));
3066
3067 buf->b_efunc = NULL;
3068 buf->b_private = NULL;
3069 kmem_cache_free(buf_cache, buf);
3070 mutex_enter(&arc_user_evicts_lock);
3071 }
3072 mutex_exit(&arc_user_evicts_lock);
3073 }
3074
3075 void
3076 arc_flush(spa_t *spa, boolean_t retry)
3077 {
3078 uint64_t guid = 0;
3079
3080 /*
3081 * If retry is TRUE, a spa must not be specified since we have
3082 * no good way to determine if all of a spa's buffers have been
3083 * evicted from an arc state.
3084 */
3085 ASSERT(!retry || spa == 0);
3086
3087 if (spa != NULL)
3088 guid = spa_load_guid(spa);
3089
3090 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
3091 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
3092
3093 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
3094 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
3095
3096 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
3097 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
3098
3099 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
3100 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
3101
3102 arc_do_user_evicts();
3103 ASSERT(spa || arc_eviction_list == NULL);
3104 }
3105
3106 void
3107 arc_shrink(int64_t to_free)
3108 {
3109 if (arc_c > arc_c_min) {
3110
3111 if (arc_c > arc_c_min + to_free)
3112 atomic_add_64(&arc_c, -to_free);
3113 else
3114 arc_c = arc_c_min;
3115
3116 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
3117 if (arc_c > arc_size)
3118 arc_c = MAX(arc_size, arc_c_min);
3119 if (arc_p > arc_c)
3120 arc_p = (arc_c >> 1);
3121 ASSERT(arc_c >= arc_c_min);
3122 ASSERT((int64_t)arc_p >= 0);
3123 }
3124
3125 if (arc_size > arc_c)
3126 (void) arc_adjust();
3127 }
3128
3129 typedef enum free_memory_reason_t {
3130 FMR_UNKNOWN,
3131 FMR_NEEDFREE,
3132 FMR_LOTSFREE,
3133 FMR_SWAPFS_MINFREE,
3134 FMR_PAGES_PP_MAXIMUM,
3135 FMR_HEAP_ARENA,
3136 FMR_ZIO_ARENA,
3137 } free_memory_reason_t;
3138
3139 int64_t last_free_memory;
3140 free_memory_reason_t last_free_reason;
3141
3142 /*
3143 * Additional reserve of pages for pp_reserve.
3144 */
3145 int64_t arc_pages_pp_reserve = 64;
3146
3147 /*
3148 * Additional reserve of pages for swapfs.
3149 */
3150 int64_t arc_swapfs_reserve = 64;
3151
3152 /*
3153 * Return the amount of memory that can be consumed before reclaim will be
3154 * needed. Positive if there is sufficient free memory, negative indicates
3155 * the amount of memory that needs to be freed up.
3156 */
3157 static int64_t
3158 arc_available_memory(void)
3159 {
3160 int64_t lowest = INT64_MAX;
3161 int64_t n;
3162 free_memory_reason_t r = FMR_UNKNOWN;
3163
3164 #ifdef _KERNEL
3165 if (needfree > 0) {
3166 n = PAGESIZE * (-needfree);
3167 if (n < lowest) {
3168 lowest = n;
3169 r = FMR_NEEDFREE;
3170 }
3171 }
3172
3173 /*
3174 * check that we're out of range of the pageout scanner. It starts to
3175 * schedule paging if freemem is less than lotsfree and needfree.
3176 * lotsfree is the high-water mark for pageout, and needfree is the
3177 * number of needed free pages. We add extra pages here to make sure
3178 * the scanner doesn't start up while we're freeing memory.
3179 */
3180 n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
3181 if (n < lowest) {
3182 lowest = n;
3183 r = FMR_LOTSFREE;
3184 }
3185
3186 /*
3187 * check to make sure that swapfs has enough space so that anon
3188 * reservations can still succeed. anon_resvmem() checks that the
3189 * availrmem is greater than swapfs_minfree, and the number of reserved
3190 * swap pages. We also add a bit of extra here just to prevent
3191 * circumstances from getting really dire.
3192 */
3193 n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
3194 desfree - arc_swapfs_reserve);
3195 if (n < lowest) {
3196 lowest = n;
3197 r = FMR_SWAPFS_MINFREE;
3198 }
3199
3200
3201 /*
3202 * Check that we have enough availrmem that memory locking (e.g., via
3203 * mlock(3C) or memcntl(2)) can still succeed. (pages_pp_maximum
3204 * stores the number of pages that cannot be locked; when availrmem
3205 * drops below pages_pp_maximum, page locking mechanisms such as
3206 * page_pp_lock() will fail.)
3207 */
3208 n = PAGESIZE * (availrmem - pages_pp_maximum -
3209 arc_pages_pp_reserve);
3210 if (n < lowest) {
3211 lowest = n;
3212 r = FMR_PAGES_PP_MAXIMUM;
3213 }
3214
3215 #if defined(__i386)
3216 /*
3217 * If we're on an i386 platform, it's possible that we'll exhaust the
3218 * kernel heap space before we ever run out of available physical
3219 * memory. Most checks of the size of the heap_area compare against
3220 * tune.t_minarmem, which is the minimum available real memory that we
3221 * can have in the system. However, this is generally fixed at 25 pages
3222 * which is so low that it's useless. In this comparison, we seek to
3223 * calculate the total heap-size, and reclaim if more than 3/4ths of the
3224 * heap is allocated. (Or, in the calculation, if less than 1/4th is
3225 * free)
3226 */
3227 n = vmem_size(heap_arena, VMEM_FREE) -
3228 (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2);
3229 if (n < lowest) {
3230 lowest = n;
3231 r = FMR_HEAP_ARENA;
3232 }
3233 #endif
3234
3235 /*
3236 * If zio data pages are being allocated out of a separate heap segment,
3237 * then enforce that the size of available vmem for this arena remains
3238 * above about 1/16th free.
3239 *
3240 * Note: The 1/16th arena free requirement was put in place
3241 * to aggressively evict memory from the arc in order to avoid
3242 * memory fragmentation issues.
3243 */
3244 if (zio_arena != NULL) {
3245 n = vmem_size(zio_arena, VMEM_FREE) -
3246 (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
3247 if (n < lowest) {
3248 lowest = n;
3249 r = FMR_ZIO_ARENA;
3250 }
3251 }
3252 #else
3253 /* Every 100 calls, free a small amount */
3254 if (spa_get_random(100) == 0)
3255 lowest = -1024;
3256 #endif
3257
3258 last_free_memory = lowest;
3259 last_free_reason = r;
3260
3261 return (lowest);
3262 }
3263
3264
3265 /*
3266 * Determine if the system is under memory pressure and is asking
3267 * to reclaim memory. A return value of TRUE indicates that the system
3268 * is under memory pressure and that the arc should adjust accordingly.
3269 */
3270 static boolean_t
3271 arc_reclaim_needed(void)
3272 {
3273 return (arc_available_memory() < 0);
3274 }
3275
3276 static void
3277 arc_kmem_reap_now(void)
3278 {
3279 size_t i;
3280 kmem_cache_t *prev_cache = NULL;
3281 kmem_cache_t *prev_data_cache = NULL;
3282 extern kmem_cache_t *zio_buf_cache[];
3283 extern kmem_cache_t *zio_data_buf_cache[];
3284 extern kmem_cache_t *range_seg_cache;
3285
3286 #ifdef _KERNEL
3287 if (arc_meta_used >= arc_meta_limit) {
3288 /*
3289 * We are exceeding our meta-data cache limit.
3290 * Purge some DNLC entries to release holds on meta-data.
3291 */
3292 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
3293 }
3294 #if defined(__i386)
3295 /*
3296 * Reclaim unused memory from all kmem caches.
3297 */
3298 kmem_reap();
3299 #endif
3300 #endif
3301
3302 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
3303 if (zio_buf_cache[i] != prev_cache) {
3304 prev_cache = zio_buf_cache[i];
3305 kmem_cache_reap_now(zio_buf_cache[i]);
3306 }
3307 if (zio_data_buf_cache[i] != prev_data_cache) {
3308 prev_data_cache = zio_data_buf_cache[i];
3309 kmem_cache_reap_now(zio_data_buf_cache[i]);
3310 }
3311 }
3312 kmem_cache_reap_now(buf_cache);
3313 kmem_cache_reap_now(hdr_full_cache);
3314 kmem_cache_reap_now(hdr_l2only_cache);
3315 kmem_cache_reap_now(range_seg_cache);
3316
3317 if (zio_arena != NULL) {
3318 /*
3319 * Ask the vmem arena to reclaim unused memory from its
3320 * quantum caches.
3321 */
3322 vmem_qcache_reap(zio_arena);
3323 }
3324 }
3325
3326 /*
3327 * Threads can block in arc_get_data_buf() waiting for this thread to evict
3328 * enough data and signal them to proceed. When this happens, the threads in
3329 * arc_get_data_buf() are sleeping while holding the hash lock for their
3330 * particular arc header. Thus, we must be careful to never sleep on a
3331 * hash lock in this thread. This is to prevent the following deadlock:
3332 *
3333 * - Thread A sleeps on CV in arc_get_data_buf() holding hash lock "L",
3334 * waiting for the reclaim thread to signal it.
3335 *
3336 * - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
3337 * fails, and goes to sleep forever.
3338 *
3339 * This possible deadlock is avoided by always acquiring a hash lock
3340 * using mutex_tryenter() from arc_reclaim_thread().
3341 */
3342 static void
3343 arc_reclaim_thread(void)
3344 {
3345 clock_t growtime = 0;
3346 callb_cpr_t cpr;
3347
3348 CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
3349
3350 mutex_enter(&arc_reclaim_lock);
3351 while (!arc_reclaim_thread_exit) {
3352 int64_t free_memory = arc_available_memory();
3353 uint64_t evicted = 0;
3354
3355 mutex_exit(&arc_reclaim_lock);
3356
3357 if (free_memory < 0) {
3358
3359 arc_no_grow = B_TRUE;
3360 arc_warm = B_TRUE;
3361
3362 /*
3363 * Wait at least zfs_grow_retry (default 60) seconds
3364 * before considering growing.
3365 */
3366 growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
3367
3368 arc_kmem_reap_now();
3369
3370 /*
3371 * If we are still low on memory, shrink the ARC
3372 * so that we have arc_shrink_min free space.
3373 */
3374 free_memory = arc_available_memory();
3375
3376 int64_t to_free =
3377 (arc_c >> arc_shrink_shift) - free_memory;
3378 if (to_free > 0) {
3379 #ifdef _KERNEL
3380 to_free = MAX(to_free, ptob(needfree));
3381 #endif
3382 arc_shrink(to_free);
3383 }
3384 } else if (free_memory < arc_c >> arc_no_grow_shift) {
3385 arc_no_grow = B_TRUE;
3386 } else if (ddi_get_lbolt() >= growtime) {
3387 arc_no_grow = B_FALSE;
3388 }
3389
3390 evicted = arc_adjust();
3391
3392 mutex_enter(&arc_reclaim_lock);
3393
3394 /*
3395 * If evicted is zero, we couldn't evict anything via
3396 * arc_adjust(). This could be due to hash lock
3397 * collisions, but more likely due to the majority of
3398 * arc buffers being unevictable. Therefore, even if
3399 * arc_size is above arc_c, another pass is unlikely to
3400 * be helpful and could potentially cause us to enter an
3401 * infinite loop.
3402 */
3403 if (arc_size <= arc_c || evicted == 0) {
3404 /*
3405 * We're either no longer overflowing, or we
3406 * can't evict anything more, so we should wake
3407 * up any threads before we go to sleep.
3408 */
3409 cv_broadcast(&arc_reclaim_waiters_cv);
3410
3411 /*
3412 * Block until signaled, or after one second (we
3413 * might need to perform arc_kmem_reap_now()
3414 * even if we aren't being signalled)
3415 */
3416 CALLB_CPR_SAFE_BEGIN(&cpr);
3417 (void) cv_timedwait(&arc_reclaim_thread_cv,
3418 &arc_reclaim_lock, ddi_get_lbolt() + hz);
3419 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
3420 }
3421 }
3422
3423 arc_reclaim_thread_exit = FALSE;
3424 cv_broadcast(&arc_reclaim_thread_cv);
3425 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_lock */
3426 thread_exit();
3427 }
3428
3429 static void
3430 arc_user_evicts_thread(void)
3431 {
3432 callb_cpr_t cpr;
3433
3434 CALLB_CPR_INIT(&cpr, &arc_user_evicts_lock, callb_generic_cpr, FTAG);
3435
3436 mutex_enter(&arc_user_evicts_lock);
3437 while (!arc_user_evicts_thread_exit) {
3438 mutex_exit(&arc_user_evicts_lock);
3439
3440 arc_do_user_evicts();
3441
3442 /*
3443 * This is necessary in order for the mdb ::arc dcmd to
3444 * show up to date information. Since the ::arc command
3445 * does not call the kstat's update function, without
3446 * this call, the command may show stale stats for the
3447 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
3448 * with this change, the data might be up to 1 second
3449 * out of date; but that should suffice. The arc_state_t
3450 * structures can be queried directly if more accurate
3451 * information is needed.
3452 */
3453 if (arc_ksp != NULL)
3454 arc_ksp->ks_update(arc_ksp, KSTAT_READ);
3455
3456 mutex_enter(&arc_user_evicts_lock);
3457
3458 /*
3459 * Block until signaled, or after one second (we need to
3460 * call the arc's kstat update function regularly).
3461 */
3462 CALLB_CPR_SAFE_BEGIN(&cpr);
3463 (void) cv_timedwait(&arc_user_evicts_cv,
3464 &arc_user_evicts_lock, ddi_get_lbolt() + hz);
3465 CALLB_CPR_SAFE_END(&cpr, &arc_user_evicts_lock);
3466 }
3467
3468 arc_user_evicts_thread_exit = FALSE;
3469 cv_broadcast(&arc_user_evicts_cv);
3470 CALLB_CPR_EXIT(&cpr); /* drops arc_user_evicts_lock */
3471 thread_exit();
3472 }
3473
3474 /*
3475 * Adapt arc info given the number of bytes we are trying to add and
3476 * the state that we are comming from. This function is only called
3477 * when we are adding new content to the cache.
3478 */
3479 static void
3480 arc_adapt(int bytes, arc_state_t *state)
3481 {
3482 int mult;
3483 uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
3484 int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size);
3485 int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size);
3486
3487 if (state == arc_l2c_only)
3488 return;
3489
3490 ASSERT(bytes > 0);
3491 /*
3492 * Adapt the target size of the MRU list:
3493 * - if we just hit in the MRU ghost list, then increase
3494 * the target size of the MRU list.
3495 * - if we just hit in the MFU ghost list, then increase
3496 * the target size of the MFU list by decreasing the
3497 * target size of the MRU list.
3498 */
3499 if (state == arc_mru_ghost) {
3500 mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
3501 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
3502
3503 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
3504 } else if (state == arc_mfu_ghost) {
3505 uint64_t delta;
3506
3507 mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
3508 mult = MIN(mult, 10);
3509
3510 delta = MIN(bytes * mult, arc_p);
3511 arc_p = MAX(arc_p_min, arc_p - delta);
3512 }
3513 ASSERT((int64_t)arc_p >= 0);
3514
3515 if (arc_reclaim_needed()) {
3516 cv_signal(&arc_reclaim_thread_cv);
3517 return;
3518 }
3519
3520 if (arc_no_grow)
3521 return;
3522
3523 if (arc_c >= arc_c_max)
3524 return;
3525
3526 /*
3527 * If we're within (2 * maxblocksize) bytes of the target
3528 * cache size, increment the target cache size
3529 */
3530 if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
3531 atomic_add_64(&arc_c, (int64_t)bytes);
3532 if (arc_c > arc_c_max)
3533 arc_c = arc_c_max;
3534 else if (state == arc_anon)
3535 atomic_add_64(&arc_p, (int64_t)bytes);
3536 if (arc_p > arc_c)
3537 arc_p = arc_c;
3538 }
3539 ASSERT((int64_t)arc_p >= 0);
3540 }
3541
3542 /*
3543 * Check if arc_size has grown past our upper threshold, determined by
3544 * zfs_arc_overflow_shift.
3545 */
3546 static boolean_t
3547 arc_is_overflowing(void)
3548 {
3549 /* Always allow at least one block of overflow */
3550 uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
3551 arc_c >> zfs_arc_overflow_shift);
3552
3553 return (arc_size >= arc_c + overflow);
3554 }
3555
3556 /*
3557 * The buffer, supplied as the first argument, needs a data block. If we
3558 * are hitting the hard limit for the cache size, we must sleep, waiting
3559 * for the eviction thread to catch up. If we're past the target size
3560 * but below the hard limit, we'll only signal the reclaim thread and
3561 * continue on.
3562 */
3563 static void
3564 arc_get_data_buf(arc_buf_t *buf)
3565 {
3566 arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
3567 uint64_t size = buf->b_hdr->b_size;
3568 arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
3569
3570 arc_adapt(size, state);
3571
3572 /*
3573 * If arc_size is currently overflowing, and has grown past our
3574 * upper limit, we must be adding data faster than the evict
3575 * thread can evict. Thus, to ensure we don't compound the
3576 * problem by adding more data and forcing arc_size to grow even
3577 * further past it's target size, we halt and wait for the
3578 * eviction thread to catch up.
3579 *
3580 * It's also possible that the reclaim thread is unable to evict
3581 * enough buffers to get arc_size below the overflow limit (e.g.
3582 * due to buffers being un-evictable, or hash lock collisions).
3583 * In this case, we want to proceed regardless if we're
3584 * overflowing; thus we don't use a while loop here.
3585 */
3586 if (arc_is_overflowing()) {
3587 mutex_enter(&arc_reclaim_lock);
3588
3589 /*
3590 * Now that we've acquired the lock, we may no longer be
3591 * over the overflow limit, lets check.
3592 *
3593 * We're ignoring the case of spurious wake ups. If that
3594 * were to happen, it'd let this thread consume an ARC
3595 * buffer before it should have (i.e. before we're under
3596 * the overflow limit and were signalled by the reclaim
3597 * thread). As long as that is a rare occurrence, it
3598 * shouldn't cause any harm.
3599 */
3600 if (arc_is_overflowing()) {
3601 cv_signal(&arc_reclaim_thread_cv);
3602 cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
3603 }
3604
3605 mutex_exit(&arc_reclaim_lock);
3606 }
3607
3608 if (type == ARC_BUFC_METADATA) {
3609 buf->b_data = zio_buf_alloc(size);
3610 arc_space_consume(size, ARC_SPACE_META);
3611 } else {
3612 ASSERT(type == ARC_BUFC_DATA);
3613 buf->b_data = zio_data_buf_alloc(size);
3614 arc_space_consume(size, ARC_SPACE_DATA);
3615 }
3616
3617 /*
3618 * Update the state size. Note that ghost states have a
3619 * "ghost size" and so don't need to be updated.
3620 */
3621 if (!GHOST_STATE(buf->b_hdr->b_l1hdr.b_state)) {
3622 arc_buf_hdr_t *hdr = buf->b_hdr;
3623 arc_state_t *state = hdr->b_l1hdr.b_state;
3624
3625 (void) refcount_add_many(&state->arcs_size, size, buf);
3626
3627 /*
3628 * If this is reached via arc_read, the link is
3629 * protected by the hash lock. If reached via
3630 * arc_buf_alloc, the header should not be accessed by
3631 * any other thread. And, if reached via arc_read_done,
3632 * the hash lock will protect it if it's found in the
3633 * hash table; otherwise no other thread should be
3634 * trying to [add|remove]_reference it.
3635 */
3636 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
3637 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3638 atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
3639 size);
3640 }
3641 /*
3642 * If we are growing the cache, and we are adding anonymous
3643 * data, and we have outgrown arc_p, update arc_p
3644 */
3645 if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
3646 (refcount_count(&arc_anon->arcs_size) +
3647 refcount_count(&arc_mru->arcs_size) > arc_p))
3648 arc_p = MIN(arc_c, arc_p + size);
3649 }
3650 }
3651
3652 /*
3653 * This routine is called whenever a buffer is accessed.
3654 * NOTE: the hash lock is dropped in this function.
3655 */
3656 static void
3657 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3658 {
3659 clock_t now;
3660
3661 ASSERT(MUTEX_HELD(hash_lock));
3662 ASSERT(HDR_HAS_L1HDR(hdr));
3663
3664 if (hdr->b_l1hdr.b_state == arc_anon) {
3665 /*
3666 * This buffer is not in the cache, and does not
3667 * appear in our "ghost" list. Add the new buffer
3668 * to the MRU state.
3669 */
3670
3671 ASSERT0(hdr->b_l1hdr.b_arc_access);
3672 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3673 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3674 arc_change_state(arc_mru, hdr, hash_lock);
3675
3676 } else if (hdr->b_l1hdr.b_state == arc_mru) {
3677 now = ddi_get_lbolt();
3678
3679 /*
3680 * If this buffer is here because of a prefetch, then either:
3681 * - clear the flag if this is a "referencing" read
3682 * (any subsequent access will bump this into the MFU state).
3683 * or
3684 * - move the buffer to the head of the list if this is
3685 * another prefetch (to make it less likely to be evicted).
3686 */
3687 if (HDR_PREFETCH(hdr)) {
3688 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
3689 /* link protected by hash lock */
3690 ASSERT(multilist_link_active(
3691 &hdr->b_l1hdr.b_arc_node));
3692 } else {
3693 hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3694 ARCSTAT_BUMP(arcstat_mru_hits);
3695 }
3696 hdr->b_l1hdr.b_arc_access = now;
3697 return;
3698 }
3699
3700 /*
3701 * This buffer has been "accessed" only once so far,
3702 * but it is still in the cache. Move it to the MFU
3703 * state.
3704 */
3705 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
3706 /*
3707 * More than 125ms have passed since we
3708 * instantiated this buffer. Move it to the
3709 * most frequently used state.
3710 */
3711 hdr->b_l1hdr.b_arc_access = now;
3712 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3713 arc_change_state(arc_mfu, hdr, hash_lock);
3714 }
3715 ARCSTAT_BUMP(arcstat_mru_hits);
3716 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
3717 arc_state_t *new_state;
3718 /*
3719 * This buffer has been "accessed" recently, but
3720 * was evicted from the cache. Move it to the
3721 * MFU state.
3722 */
3723
3724 if (HDR_PREFETCH(hdr)) {
3725 new_state = arc_mru;
3726 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
3727 hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3728 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3729 } else {
3730 new_state = arc_mfu;
3731 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3732 }
3733
3734 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3735 arc_change_state(new_state, hdr, hash_lock);
3736
3737 ARCSTAT_BUMP(arcstat_mru_ghost_hits);
3738 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
3739 /*
3740 * This buffer has been accessed more than once and is
3741 * still in the cache. Keep it in the MFU state.
3742 *
3743 * NOTE: an add_reference() that occurred when we did
3744 * the arc_read() will have kicked this off the list.
3745 * If it was a prefetch, we will explicitly move it to
3746 * the head of the list now.
3747 */
3748 if ((HDR_PREFETCH(hdr)) != 0) {
3749 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3750 /* link protected by hash_lock */
3751 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3752 }
3753 ARCSTAT_BUMP(arcstat_mfu_hits);
3754 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3755 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
3756 arc_state_t *new_state = arc_mfu;
3757 /*
3758 * This buffer has been accessed more than once but has
3759 * been evicted from the cache. Move it back to the
3760 * MFU state.
3761 */
3762
3763 if (HDR_PREFETCH(hdr)) {
3764 /*
3765 * This is a prefetch access...
3766 * move this block back to the MRU state.
3767 */
3768 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3769 new_state = arc_mru;
3770 }
3771
3772 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3773 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3774 arc_change_state(new_state, hdr, hash_lock);
3775
3776 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
3777 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
3778 /*
3779 * This buffer is on the 2nd Level ARC.
3780 */
3781
3782 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3783 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3784 arc_change_state(arc_mfu, hdr, hash_lock);
3785 } else {
3786 ASSERT(!"invalid arc state");
3787 }
3788 }
3789
3790 /* a generic arc_done_func_t which you can use */
3791 /* ARGSUSED */
3792 void
3793 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
3794 {
3795 if (zio == NULL || zio->io_error == 0)
3796 bcopy(buf->b_data, arg, buf->b_hdr->b_size);
3797 VERIFY(arc_buf_remove_ref(buf, arg));
3798 }
3799
3800 /* a generic arc_done_func_t */
3801 void
3802 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
3803 {
3804 arc_buf_t **bufp = arg;
3805 if (zio && zio->io_error) {
3806 VERIFY(arc_buf_remove_ref(buf, arg));
3807 *bufp = NULL;
3808 } else {
3809 *bufp = buf;
3810 ASSERT(buf->b_data);
3811 }
3812 }
3813
3814 static void
3815 arc_read_done(zio_t *zio)
3816 {
3817 arc_buf_hdr_t *hdr;
3818 arc_buf_t *buf;
3819 arc_buf_t *abuf; /* buffer we're assigning to callback */
3820 kmutex_t *hash_lock = NULL;
3821 arc_callback_t *callback_list, *acb;
3822 int freeable = FALSE;
3823
3824 buf = zio->io_private;
3825 hdr = buf->b_hdr;
3826
3827 /*
3828 * The hdr was inserted into hash-table and removed from lists
3829 * prior to starting I/O. We should find this header, since
3830 * it's in the hash table, and it should be legit since it's
3831 * not possible to evict it during the I/O. The only possible
3832 * reason for it not to be found is if we were freed during the
3833 * read.
3834 */
3835 if (HDR_IN_HASH_TABLE(hdr)) {
3836 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
3837 ASSERT3U(hdr->b_dva.dva_word[0], ==,
3838 BP_IDENTITY(zio->io_bp)->dva_word[0]);
3839 ASSERT3U(hdr->b_dva.dva_word[1], ==,
3840 BP_IDENTITY(zio->io_bp)->dva_word[1]);
3841
3842 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
3843 &hash_lock);
3844
3845 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) &&
3846 hash_lock == NULL) ||
3847 (found == hdr &&
3848 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
3849 (found == hdr && HDR_L2_READING(hdr)));
3850 }
3851
3852 hdr->b_flags &= ~ARC_FLAG_L2_EVICTED;
3853 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
3854 hdr->b_flags &= ~ARC_FLAG_L2CACHE;
3855
3856 /* byteswap if necessary */
3857 callback_list = hdr->b_l1hdr.b_acb;
3858 ASSERT(callback_list != NULL);
3859 if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
3860 dmu_object_byteswap_t bswap =
3861 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
3862 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
3863 byteswap_uint64_array :
3864 dmu_ot_byteswap[bswap].ob_func;
3865 func(buf->b_data, hdr->b_size);
3866 }
3867
3868 arc_cksum_compute(buf, B_FALSE);
3869 arc_buf_watch(buf);
3870
3871 if (hash_lock && zio->io_error == 0 &&
3872 hdr->b_l1hdr.b_state == arc_anon) {
3873 /*
3874 * Only call arc_access on anonymous buffers. This is because
3875 * if we've issued an I/O for an evicted buffer, we've already
3876 * called arc_access (to prevent any simultaneous readers from
3877 * getting confused).
3878 */
3879 arc_access(hdr, hash_lock);
3880 }
3881
3882 /* create copies of the data buffer for the callers */
3883 abuf = buf;
3884 for (acb = callback_list; acb; acb = acb->acb_next) {
3885 if (acb->acb_done) {
3886 if (abuf == NULL) {
3887 ARCSTAT_BUMP(arcstat_duplicate_reads);
3888 abuf = arc_buf_clone(buf);
3889 }
3890 acb->acb_buf = abuf;
3891 abuf = NULL;
3892 }
3893 }
3894 hdr->b_l1hdr.b_acb = NULL;
3895 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
3896 ASSERT(!HDR_BUF_AVAILABLE(hdr));
3897 if (abuf == buf) {
3898 ASSERT(buf->b_efunc == NULL);
3899 ASSERT(hdr->b_l1hdr.b_datacnt == 1);
3900 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
3901 }
3902
3903 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
3904 callback_list != NULL);
3905
3906 if (zio->io_error != 0) {
3907 hdr->b_flags |= ARC_FLAG_IO_ERROR;
3908 if (hdr->b_l1hdr.b_state != arc_anon)
3909 arc_change_state(arc_anon, hdr, hash_lock);
3910 if (HDR_IN_HASH_TABLE(hdr))
3911 buf_hash_remove(hdr);
3912 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3913 }
3914
3915 /*
3916 * Broadcast before we drop the hash_lock to avoid the possibility
3917 * that the hdr (and hence the cv) might be freed before we get to
3918 * the cv_broadcast().
3919 */
3920 cv_broadcast(&hdr->b_l1hdr.b_cv);
3921
3922 if (hash_lock != NULL) {
3923 mutex_exit(hash_lock);
3924 } else {
3925 /*
3926 * This block was freed while we waited for the read to
3927 * complete. It has been removed from the hash table and
3928 * moved to the anonymous state (so that it won't show up
3929 * in the cache).
3930 */
3931 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3932 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3933 }
3934
3935 /* execute each callback and free its structure */
3936 while ((acb = callback_list) != NULL) {
3937 if (acb->acb_done)
3938 acb->acb_done(zio, acb->acb_buf, acb->acb_private);
3939
3940 if (acb->acb_zio_dummy != NULL) {
3941 acb->acb_zio_dummy->io_error = zio->io_error;
3942 zio_nowait(acb->acb_zio_dummy);
3943 }
3944
3945 callback_list = acb->acb_next;
3946 kmem_free(acb, sizeof (arc_callback_t));
3947 }
3948
3949 if (freeable)
3950 arc_hdr_destroy(hdr);
3951 }
3952
3953 /*
3954 * "Read" the block at the specified DVA (in bp) via the
3955 * cache. If the block is found in the cache, invoke the provided
3956 * callback immediately and return. Note that the `zio' parameter
3957 * in the callback will be NULL in this case, since no IO was
3958 * required. If the block is not in the cache pass the read request
3959 * on to the spa with a substitute callback function, so that the
3960 * requested block will be added to the cache.
3961 *
3962 * If a read request arrives for a block that has a read in-progress,
3963 * either wait for the in-progress read to complete (and return the
3964 * results); or, if this is a read with a "done" func, add a record
3965 * to the read to invoke the "done" func when the read completes,
3966 * and return; or just return.
3967 *
3968 * arc_read_done() will invoke all the requested "done" functions
3969 * for readers of this block.
3970 */
3971 int
3972 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
3973 void *private, zio_priority_t priority, int zio_flags,
3974 arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
3975 {
3976 arc_buf_hdr_t *hdr = NULL;
3977 arc_buf_t *buf = NULL;
3978 kmutex_t *hash_lock = NULL;
3979 zio_t *rzio;
3980 uint64_t guid = spa_load_guid(spa);
3981
3982 ASSERT(!BP_IS_EMBEDDED(bp) ||
3983 BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
3984
3985 top:
3986 if (!BP_IS_EMBEDDED(bp)) {
3987 /*
3988 * Embedded BP's have no DVA and require no I/O to "read".
3989 * Create an anonymous arc buf to back it.
3990 */
3991 hdr = buf_hash_find(guid, bp, &hash_lock);
3992 }
3993
3994 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_datacnt > 0) {
3995
3996 *arc_flags |= ARC_FLAG_CACHED;
3997
3998 if (HDR_IO_IN_PROGRESS(hdr)) {
3999
4000 if (*arc_flags & ARC_FLAG_WAIT) {
4001 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
4002 mutex_exit(hash_lock);
4003 goto top;
4004 }
4005 ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
4006
4007 if (done) {
4008 arc_callback_t *acb = NULL;
4009
4010 acb = kmem_zalloc(sizeof (arc_callback_t),
4011 KM_SLEEP);
4012 acb->acb_done = done;
4013 acb->acb_private = private;
4014 if (pio != NULL)
4015 acb->acb_zio_dummy = zio_null(pio,
4016 spa, NULL, NULL, NULL, zio_flags);
4017
4018 ASSERT(acb->acb_done != NULL);
4019 acb->acb_next = hdr->b_l1hdr.b_acb;
4020 hdr->b_l1hdr.b_acb = acb;
4021 add_reference(hdr, hash_lock, private);
4022 mutex_exit(hash_lock);
4023 return (0);
4024 }
4025 mutex_exit(hash_lock);
4026 return (0);
4027 }
4028
4029 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4030 hdr->b_l1hdr.b_state == arc_mfu);
4031
4032 if (done) {
4033 add_reference(hdr, hash_lock, private);
4034 /*
4035 * If this block is already in use, create a new
4036 * copy of the data so that we will be guaranteed
4037 * that arc_release() will always succeed.
4038 */
4039 buf = hdr->b_l1hdr.b_buf;
4040 ASSERT(buf);
4041 ASSERT(buf->b_data);
4042 if (HDR_BUF_AVAILABLE(hdr)) {
4043 ASSERT(buf->b_efunc == NULL);
4044 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
4045 } else {
4046 buf = arc_buf_clone(buf);
4047 }
4048
4049 } else if (*arc_flags & ARC_FLAG_PREFETCH &&
4050 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4051 hdr->b_flags |= ARC_FLAG_PREFETCH;
4052 }
4053 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
4054 arc_access(hdr, hash_lock);
4055 if (*arc_flags & ARC_FLAG_L2CACHE)
4056 hdr->b_flags |= ARC_FLAG_L2CACHE;
4057 if (*arc_flags & ARC_FLAG_L2COMPRESS)
4058 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4059 mutex_exit(hash_lock);
4060 ARCSTAT_BUMP(arcstat_hits);
4061 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4062 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4063 data, metadata, hits);
4064
4065 if (done)
4066 done(NULL, buf, private);
4067 } else {
4068 uint64_t size = BP_GET_LSIZE(bp);
4069 arc_callback_t *acb;
4070 vdev_t *vd = NULL;
4071 uint64_t addr = 0;
4072 boolean_t devw = B_FALSE;
4073 enum zio_compress b_compress = ZIO_COMPRESS_OFF;
4074 int32_t b_asize = 0;
4075
4076 if (hdr == NULL) {
4077 /* this block is not in the cache */
4078 arc_buf_hdr_t *exists = NULL;
4079 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
4080 buf = arc_buf_alloc(spa, size, private, type);
4081 hdr = buf->b_hdr;
4082 if (!BP_IS_EMBEDDED(bp)) {
4083 hdr->b_dva = *BP_IDENTITY(bp);
4084 hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
4085 exists = buf_hash_insert(hdr, &hash_lock);
4086 }
4087 if (exists != NULL) {
4088 /* somebody beat us to the hash insert */
4089 mutex_exit(hash_lock);
4090 buf_discard_identity(hdr);
4091 (void) arc_buf_remove_ref(buf, private);
4092 goto top; /* restart the IO request */
4093 }
4094
4095 /* if this is a prefetch, we don't have a reference */
4096 if (*arc_flags & ARC_FLAG_PREFETCH) {
4097 (void) remove_reference(hdr, hash_lock,
4098 private);
4099 hdr->b_flags |= ARC_FLAG_PREFETCH;
4100 }
4101 if (*arc_flags & ARC_FLAG_L2CACHE)
4102 hdr->b_flags |= ARC_FLAG_L2CACHE;
4103 if (*arc_flags & ARC_FLAG_L2COMPRESS)
4104 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4105 if (BP_GET_LEVEL(bp) > 0)
4106 hdr->b_flags |= ARC_FLAG_INDIRECT;
4107 } else {
4108 /*
4109 * This block is in the ghost cache. If it was L2-only
4110 * (and thus didn't have an L1 hdr), we realloc the
4111 * header to add an L1 hdr.
4112 */
4113 if (!HDR_HAS_L1HDR(hdr)) {
4114 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
4115 hdr_full_cache);
4116 }
4117
4118 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
4119 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4120 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4121 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
4122
4123 /* if this is a prefetch, we don't have a reference */
4124 if (*arc_flags & ARC_FLAG_PREFETCH)
4125 hdr->b_flags |= ARC_FLAG_PREFETCH;
4126 else
4127 add_reference(hdr, hash_lock, private);
4128 if (*arc_flags & ARC_FLAG_L2CACHE)
4129 hdr->b_flags |= ARC_FLAG_L2CACHE;
4130 if (*arc_flags & ARC_FLAG_L2COMPRESS)
4131 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4132 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
4133 buf->b_hdr = hdr;
4134 buf->b_data = NULL;
4135 buf->b_efunc = NULL;
4136 buf->b_private = NULL;
4137 buf->b_next = NULL;
4138 hdr->b_l1hdr.b_buf = buf;
4139 ASSERT0(hdr->b_l1hdr.b_datacnt);
4140 hdr->b_l1hdr.b_datacnt = 1;
4141 arc_get_data_buf(buf);
4142 arc_access(hdr, hash_lock);
4143 }
4144
4145 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
4146
4147 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
4148 acb->acb_done = done;
4149 acb->acb_private = private;
4150
4151 ASSERT(hdr->b_l1hdr.b_acb == NULL);
4152 hdr->b_l1hdr.b_acb = acb;
4153 hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4154
4155 if (HDR_HAS_L2HDR(hdr) &&
4156 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
4157 devw = hdr->b_l2hdr.b_dev->l2ad_writing;
4158 addr = hdr->b_l2hdr.b_daddr;
4159 b_compress = hdr->b_l2hdr.b_compress;
4160 b_asize = hdr->b_l2hdr.b_asize;
4161 /*
4162 * Lock out device removal.
4163 */
4164 if (vdev_is_dead(vd) ||
4165 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
4166 vd = NULL;
4167 }
4168
4169 if (hash_lock != NULL)
4170 mutex_exit(hash_lock);
4171
4172 /*
4173 * At this point, we have a level 1 cache miss. Try again in
4174 * L2ARC if possible.
4175 */
4176 ASSERT3U(hdr->b_size, ==, size);
4177 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
4178 uint64_t, size, zbookmark_phys_t *, zb);
4179 ARCSTAT_BUMP(arcstat_misses);
4180 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4181 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4182 data, metadata, misses);
4183
4184 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
4185 /*
4186 * Read from the L2ARC if the following are true:
4187 * 1. The L2ARC vdev was previously cached.
4188 * 2. This buffer still has L2ARC metadata.
4189 * 3. This buffer isn't currently writing to the L2ARC.
4190 * 4. The L2ARC entry wasn't evicted, which may
4191 * also have invalidated the vdev.
4192 * 5. This isn't prefetch and l2arc_noprefetch is set.
4193 */
4194 if (HDR_HAS_L2HDR(hdr) &&
4195 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
4196 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
4197 l2arc_read_callback_t *cb;
4198
4199 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
4200 ARCSTAT_BUMP(arcstat_l2_hits);
4201
4202 cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
4203 KM_SLEEP);
4204 cb->l2rcb_buf = buf;
4205 cb->l2rcb_spa = spa;
4206 cb->l2rcb_bp = *bp;
4207 cb->l2rcb_zb = *zb;
4208 cb->l2rcb_flags = zio_flags;
4209 cb->l2rcb_compress = b_compress;
4210
4211 ASSERT(addr >= VDEV_LABEL_START_SIZE &&
4212 addr + size < vd->vdev_psize -
4213 VDEV_LABEL_END_SIZE);
4214
4215 /*
4216 * l2arc read. The SCL_L2ARC lock will be
4217 * released by l2arc_read_done().
4218 * Issue a null zio if the underlying buffer
4219 * was squashed to zero size by compression.
4220 */
4221 if (b_compress == ZIO_COMPRESS_EMPTY) {
4222 rzio = zio_null(pio, spa, vd,
4223 l2arc_read_done, cb,
4224 zio_flags | ZIO_FLAG_DONT_CACHE |
4225 ZIO_FLAG_CANFAIL |
4226 ZIO_FLAG_DONT_PROPAGATE |
4227 ZIO_FLAG_DONT_RETRY);
4228 } else {
4229 rzio = zio_read_phys(pio, vd, addr,
4230 b_asize, buf->b_data,
4231 ZIO_CHECKSUM_OFF,
4232 l2arc_read_done, cb, priority,
4233 zio_flags | ZIO_FLAG_DONT_CACHE |
4234 ZIO_FLAG_CANFAIL |
4235 ZIO_FLAG_DONT_PROPAGATE |
4236 ZIO_FLAG_DONT_RETRY, B_FALSE);
4237 }
4238 DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
4239 zio_t *, rzio);
4240 ARCSTAT_INCR(arcstat_l2_read_bytes, b_asize);
4241
4242 if (*arc_flags & ARC_FLAG_NOWAIT) {
4243 zio_nowait(rzio);
4244 return (0);
4245 }
4246
4247 ASSERT(*arc_flags & ARC_FLAG_WAIT);
4248 if (zio_wait(rzio) == 0)
4249 return (0);
4250
4251 /* l2arc read error; goto zio_read() */
4252 } else {
4253 DTRACE_PROBE1(l2arc__miss,
4254 arc_buf_hdr_t *, hdr);
4255 ARCSTAT_BUMP(arcstat_l2_misses);
4256 if (HDR_L2_WRITING(hdr))
4257 ARCSTAT_BUMP(arcstat_l2_rw_clash);
4258 spa_config_exit(spa, SCL_L2ARC, vd);
4259 }
4260 } else {
4261 if (vd != NULL)
4262 spa_config_exit(spa, SCL_L2ARC, vd);
4263 if (l2arc_ndev != 0) {
4264 DTRACE_PROBE1(l2arc__miss,
4265 arc_buf_hdr_t *, hdr);
4266 ARCSTAT_BUMP(arcstat_l2_misses);
4267 }
4268 }
4269
4270 rzio = zio_read(pio, spa, bp, buf->b_data, size,
4271 arc_read_done, buf, priority, zio_flags, zb);
4272
4273 /*
4274 * At this point, this read I/O has already missed in the ARC
4275 * and will be going through to the disk. The I/O throttle
4276 * should delay this I/O if this zone is using more than its I/O
4277 * priority allows.
4278 */
4279 zfs_zone_io_throttle(ZFS_ZONE_IOP_READ);
4280
4281 if (*arc_flags & ARC_FLAG_WAIT)
4282 return (zio_wait(rzio));
4283
4284 ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
4285 zio_nowait(rzio);
4286 }
4287 return (0);
4288 }
4289
4290 void
4291 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
4292 {
4293 ASSERT(buf->b_hdr != NULL);
4294 ASSERT(buf->b_hdr->b_l1hdr.b_state != arc_anon);
4295 ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt) ||
4296 func == NULL);
4297 ASSERT(buf->b_efunc == NULL);
4298 ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
4299
4300 buf->b_efunc = func;
4301 buf->b_private = private;
4302 }
4303
4304 /*
4305 * Notify the arc that a block was freed, and thus will never be used again.
4306 */
4307 void
4308 arc_freed(spa_t *spa, const blkptr_t *bp)
4309 {
4310 arc_buf_hdr_t *hdr;
4311 kmutex_t *hash_lock;
4312 uint64_t guid = spa_load_guid(spa);
4313
4314 ASSERT(!BP_IS_EMBEDDED(bp));
4315
4316 hdr = buf_hash_find(guid, bp, &hash_lock);
4317 if (hdr == NULL)
4318 return;
4319 if (HDR_BUF_AVAILABLE(hdr)) {
4320 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
4321 add_reference(hdr, hash_lock, FTAG);
4322 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
4323 mutex_exit(hash_lock);
4324
4325 arc_release(buf, FTAG);
4326 (void) arc_buf_remove_ref(buf, FTAG);
4327 } else {
4328 mutex_exit(hash_lock);
4329 }
4330
4331 }
4332
4333 /*
4334 * Clear the user eviction callback set by arc_set_callback(), first calling
4335 * it if it exists. Because the presence of a callback keeps an arc_buf cached
4336 * clearing the callback may result in the arc_buf being destroyed. However,
4337 * it will not result in the *last* arc_buf being destroyed, hence the data
4338 * will remain cached in the ARC. We make a copy of the arc buffer here so
4339 * that we can process the callback without holding any locks.
4340 *
4341 * It's possible that the callback is already in the process of being cleared
4342 * by another thread. In this case we can not clear the callback.
4343 *
4344 * Returns B_TRUE if the callback was successfully called and cleared.
4345 */
4346 boolean_t
4347 arc_clear_callback(arc_buf_t *buf)
4348 {
4349 arc_buf_hdr_t *hdr;
4350 kmutex_t *hash_lock;
4351 arc_evict_func_t *efunc = buf->b_efunc;
4352 void *private = buf->b_private;
4353
4354 mutex_enter(&buf->b_evict_lock);
4355 hdr = buf->b_hdr;
4356 if (hdr == NULL) {
4357 /*
4358 * We are in arc_do_user_evicts().
4359 */
4360 ASSERT(buf->b_data == NULL);
4361 mutex_exit(&buf->b_evict_lock);
4362 return (B_FALSE);
4363 } else if (buf->b_data == NULL) {
4364 /*
4365 * We are on the eviction list; process this buffer now
4366 * but let arc_do_user_evicts() do the reaping.
4367 */
4368 buf->b_efunc = NULL;
4369 mutex_exit(&buf->b_evict_lock);
4370 VERIFY0(efunc(private));
4371 return (B_TRUE);
4372 }
4373 hash_lock = HDR_LOCK(hdr);
4374 mutex_enter(hash_lock);
4375 hdr = buf->b_hdr;
4376 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4377
4378 ASSERT3U(refcount_count(&hdr->b_l1hdr.b_refcnt), <,
4379 hdr->b_l1hdr.b_datacnt);
4380 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4381 hdr->b_l1hdr.b_state == arc_mfu);
4382
4383 buf->b_efunc = NULL;
4384 buf->b_private = NULL;
4385
4386 if (hdr->b_l1hdr.b_datacnt > 1) {
4387 mutex_exit(&buf->b_evict_lock);
4388 arc_buf_destroy(buf, TRUE);
4389 } else {
4390 ASSERT(buf == hdr->b_l1hdr.b_buf);
4391 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
4392 mutex_exit(&buf->b_evict_lock);
4393 }
4394
4395 mutex_exit(hash_lock);
4396 VERIFY0(efunc(private));
4397 return (B_TRUE);
4398 }
4399
4400 /*
4401 * Release this buffer from the cache, making it an anonymous buffer. This
4402 * must be done after a read and prior to modifying the buffer contents.
4403 * If the buffer has more than one reference, we must make
4404 * a new hdr for the buffer.
4405 */
4406 void
4407 arc_release(arc_buf_t *buf, void *tag)
4408 {
4409 arc_buf_hdr_t *hdr = buf->b_hdr;
4410
4411 /*
4412 * It would be nice to assert that if it's DMU metadata (level >
4413 * 0 || it's the dnode file), then it must be syncing context.
4414 * But we don't know that information at this level.
4415 */
4416
4417 mutex_enter(&buf->b_evict_lock);
4418
4419 ASSERT(HDR_HAS_L1HDR(hdr));
4420
4421 /*
4422 * We don't grab the hash lock prior to this check, because if
4423 * the buffer's header is in the arc_anon state, it won't be
4424 * linked into the hash table.
4425 */
4426 if (hdr->b_l1hdr.b_state == arc_anon) {
4427 mutex_exit(&buf->b_evict_lock);
4428 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4429 ASSERT(!HDR_IN_HASH_TABLE(hdr));
4430 ASSERT(!HDR_HAS_L2HDR(hdr));
4431 ASSERT(BUF_EMPTY(hdr));
4432
4433 ASSERT3U(hdr->b_l1hdr.b_datacnt, ==, 1);
4434 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
4435 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
4436
4437 ASSERT3P(buf->b_efunc, ==, NULL);
4438 ASSERT3P(buf->b_private, ==, NULL);
4439
4440 hdr->b_l1hdr.b_arc_access = 0;
4441 arc_buf_thaw(buf);
4442
4443 return;
4444 }
4445
4446 kmutex_t *hash_lock = HDR_LOCK(hdr);
4447 mutex_enter(hash_lock);
4448
4449 /*
4450 * This assignment is only valid as long as the hash_lock is
4451 * held, we must be careful not to reference state or the
4452 * b_state field after dropping the lock.
4453 */
4454 arc_state_t *state = hdr->b_l1hdr.b_state;
4455 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4456 ASSERT3P(state, !=, arc_anon);
4457
4458 /* this buffer is not on any list */
4459 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
4460
4461 if (HDR_HAS_L2HDR(hdr)) {
4462 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4463
4464 /*
4465 * We have to recheck this conditional again now that
4466 * we're holding the l2ad_mtx to prevent a race with
4467 * another thread which might be concurrently calling
4468 * l2arc_evict(). In that case, l2arc_evict() might have
4469 * destroyed the header's L2 portion as we were waiting
4470 * to acquire the l2ad_mtx.
4471 */
4472 if (HDR_HAS_L2HDR(hdr))
4473 arc_hdr_l2hdr_destroy(hdr);
4474
4475 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4476 }
4477
4478 /*
4479 * Do we have more than one buf?
4480 */
4481 if (hdr->b_l1hdr.b_datacnt > 1) {
4482 arc_buf_hdr_t *nhdr;
4483 arc_buf_t **bufp;
4484 uint64_t blksz = hdr->b_size;
4485 uint64_t spa = hdr->b_spa;
4486 arc_buf_contents_t type = arc_buf_type(hdr);
4487 uint32_t flags = hdr->b_flags;
4488
4489 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
4490 /*
4491 * Pull the data off of this hdr and attach it to
4492 * a new anonymous hdr.
4493 */
4494 (void) remove_reference(hdr, hash_lock, tag);
4495 bufp = &hdr->b_l1hdr.b_buf;
4496 while (*bufp != buf)
4497 bufp = &(*bufp)->b_next;
4498 *bufp = buf->b_next;
4499 buf->b_next = NULL;
4500
4501 ASSERT3P(state, !=, arc_l2c_only);
4502
4503 (void) refcount_remove_many(
4504 &state->arcs_size, hdr->b_size, buf);
4505
4506 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
4507 ASSERT3P(state, !=, arc_l2c_only);
4508 uint64_t *size = &state->arcs_lsize[type];
4509 ASSERT3U(*size, >=, hdr->b_size);
4510 atomic_add_64(size, -hdr->b_size);
4511 }
4512
4513 /*
4514 * We're releasing a duplicate user data buffer, update
4515 * our statistics accordingly.
4516 */
4517 if (HDR_ISTYPE_DATA(hdr)) {
4518 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
4519 ARCSTAT_INCR(arcstat_duplicate_buffers_size,
4520 -hdr->b_size);
4521 }
4522 hdr->b_l1hdr.b_datacnt -= 1;
4523 arc_cksum_verify(buf);
4524 arc_buf_unwatch(buf);
4525
4526 mutex_exit(hash_lock);
4527
4528 nhdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
4529 nhdr->b_size = blksz;
4530 nhdr->b_spa = spa;
4531
4532 nhdr->b_flags = flags & ARC_FLAG_L2_WRITING;
4533 nhdr->b_flags |= arc_bufc_to_flags(type);
4534 nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
4535
4536 nhdr->b_l1hdr.b_buf = buf;
4537 nhdr->b_l1hdr.b_datacnt = 1;
4538 nhdr->b_l1hdr.b_state = arc_anon;
4539 nhdr->b_l1hdr.b_arc_access = 0;
4540 nhdr->b_l1hdr.b_tmp_cdata = NULL;
4541 nhdr->b_freeze_cksum = NULL;
4542
4543 (void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
4544 buf->b_hdr = nhdr;
4545 mutex_exit(&buf->b_evict_lock);
4546 (void) refcount_add_many(&arc_anon->arcs_size, blksz, buf);
4547 } else {
4548 mutex_exit(&buf->b_evict_lock);
4549 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
4550 /* protected by hash lock, or hdr is on arc_anon */
4551 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4552 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4553 arc_change_state(arc_anon, hdr, hash_lock);
4554 hdr->b_l1hdr.b_arc_access = 0;
4555 mutex_exit(hash_lock);
4556
4557 buf_discard_identity(hdr);
4558 arc_buf_thaw(buf);
4559 }
4560 buf->b_efunc = NULL;
4561 buf->b_private = NULL;
4562 }
4563
4564 int
4565 arc_released(arc_buf_t *buf)
4566 {
4567 int released;
4568
4569 mutex_enter(&buf->b_evict_lock);
4570 released = (buf->b_data != NULL &&
4571 buf->b_hdr->b_l1hdr.b_state == arc_anon);
4572 mutex_exit(&buf->b_evict_lock);
4573 return (released);
4574 }
4575
4576 #ifdef ZFS_DEBUG
4577 int
4578 arc_referenced(arc_buf_t *buf)
4579 {
4580 int referenced;
4581
4582 mutex_enter(&buf->b_evict_lock);
4583 referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
4584 mutex_exit(&buf->b_evict_lock);
4585 return (referenced);
4586 }
4587 #endif
4588
4589 static void
4590 arc_write_ready(zio_t *zio)
4591 {
4592 arc_write_callback_t *callback = zio->io_private;
4593 arc_buf_t *buf = callback->awcb_buf;
4594 arc_buf_hdr_t *hdr = buf->b_hdr;
4595
4596 ASSERT(HDR_HAS_L1HDR(hdr));
4597 ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
4598 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4599 callback->awcb_ready(zio, buf, callback->awcb_private);
4600
4601 /*
4602 * If the IO is already in progress, then this is a re-write
4603 * attempt, so we need to thaw and re-compute the cksum.
4604 * It is the responsibility of the callback to handle the
4605 * accounting for any re-write attempt.
4606 */
4607 if (HDR_IO_IN_PROGRESS(hdr)) {
4608 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
4609 if (hdr->b_freeze_cksum != NULL) {
4610 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
4611 hdr->b_freeze_cksum = NULL;
4612 }
4613 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
4614 }
4615 arc_cksum_compute(buf, B_FALSE);
4616 hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4617 }
4618
4619 /*
4620 * The SPA calls this callback for each physical write that happens on behalf
4621 * of a logical write. See the comment in dbuf_write_physdone() for details.
4622 */
4623 static void
4624 arc_write_physdone(zio_t *zio)
4625 {
4626 arc_write_callback_t *cb = zio->io_private;
4627 if (cb->awcb_physdone != NULL)
4628 cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
4629 }
4630
4631 static void
4632 arc_write_done(zio_t *zio)
4633 {
4634 arc_write_callback_t *callback = zio->io_private;
4635 arc_buf_t *buf = callback->awcb_buf;
4636 arc_buf_hdr_t *hdr = buf->b_hdr;
4637
4638 ASSERT(hdr->b_l1hdr.b_acb == NULL);
4639
4640 if (zio->io_error == 0) {
4641 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
4642 buf_discard_identity(hdr);
4643 } else {
4644 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
4645 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
4646 }
4647 } else {
4648 ASSERT(BUF_EMPTY(hdr));
4649 }
4650
4651 /*
4652 * If the block to be written was all-zero or compressed enough to be
4653 * embedded in the BP, no write was performed so there will be no
4654 * dva/birth/checksum. The buffer must therefore remain anonymous
4655 * (and uncached).
4656 */
4657 if (!BUF_EMPTY(hdr)) {
4658 arc_buf_hdr_t *exists;
4659 kmutex_t *hash_lock;
4660
4661 ASSERT(zio->io_error == 0);
4662
4663 arc_cksum_verify(buf);
4664
4665 exists = buf_hash_insert(hdr, &hash_lock);
4666 if (exists != NULL) {
4667 /*
4668 * This can only happen if we overwrite for
4669 * sync-to-convergence, because we remove
4670 * buffers from the hash table when we arc_free().
4671 */
4672 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
4673 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4674 panic("bad overwrite, hdr=%p exists=%p",
4675 (void *)hdr, (void *)exists);
4676 ASSERT(refcount_is_zero(
4677 &exists->b_l1hdr.b_refcnt));
4678 arc_change_state(arc_anon, exists, hash_lock);
4679 mutex_exit(hash_lock);
4680 arc_hdr_destroy(exists);
4681 exists = buf_hash_insert(hdr, &hash_lock);
4682 ASSERT3P(exists, ==, NULL);
4683 } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
4684 /* nopwrite */
4685 ASSERT(zio->io_prop.zp_nopwrite);
4686 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4687 panic("bad nopwrite, hdr=%p exists=%p",
4688 (void *)hdr, (void *)exists);
4689 } else {
4690 /* Dedup */
4691 ASSERT(hdr->b_l1hdr.b_datacnt == 1);
4692 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
4693 ASSERT(BP_GET_DEDUP(zio->io_bp));
4694 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
4695 }
4696 }
4697 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4698 /* if it's not anon, we are doing a scrub */
4699 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
4700 arc_access(hdr, hash_lock);
4701 mutex_exit(hash_lock);
4702 } else {
4703 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4704 }
4705
4706 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4707 callback->awcb_done(zio, buf, callback->awcb_private);
4708
4709 kmem_free(callback, sizeof (arc_write_callback_t));
4710 }
4711
4712 zio_t *
4713 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
4714 blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
4715 const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
4716 arc_done_func_t *done, void *private, zio_priority_t priority,
4717 int zio_flags, const zbookmark_phys_t *zb)
4718 {
4719 arc_buf_hdr_t *hdr = buf->b_hdr;
4720 arc_write_callback_t *callback;
4721 zio_t *zio;
4722
4723 ASSERT(ready != NULL);
4724 ASSERT(done != NULL);
4725 ASSERT(!HDR_IO_ERROR(hdr));
4726 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4727 ASSERT(hdr->b_l1hdr.b_acb == NULL);
4728 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4729 if (l2arc)
4730 hdr->b_flags |= ARC_FLAG_L2CACHE;
4731 if (l2arc_compress)
4732 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4733 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
4734 callback->awcb_ready = ready;
4735 callback->awcb_physdone = physdone;
4736 callback->awcb_done = done;
4737 callback->awcb_private = private;
4738 callback->awcb_buf = buf;
4739
4740 zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
4741 arc_write_ready, arc_write_physdone, arc_write_done, callback,
4742 priority, zio_flags, zb);
4743
4744 return (zio);
4745 }
4746
4747 static int
4748 arc_memory_throttle(uint64_t reserve, uint64_t txg)
4749 {
4750 #ifdef _KERNEL
4751 uint64_t available_memory = ptob(freemem);
4752 static uint64_t page_load = 0;
4753 static uint64_t last_txg = 0;
4754
4755 #if defined(__i386)
4756 available_memory =
4757 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
4758 #endif
4759
4760 if (freemem > physmem * arc_lotsfree_percent / 100)
4761 return (0);
4762
4763 if (txg > last_txg) {
4764 last_txg = txg;
4765 page_load = 0;
4766 }
4767 /*
4768 * If we are in pageout, we know that memory is already tight,
4769 * the arc is already going to be evicting, so we just want to
4770 * continue to let page writes occur as quickly as possible.
4771 */
4772 if (curproc == proc_pageout) {
4773 if (page_load > MAX(ptob(minfree), available_memory) / 4)
4774 return (SET_ERROR(ERESTART));
4775 /* Note: reserve is inflated, so we deflate */
4776 page_load += reserve / 8;
4777 return (0);
4778 } else if (page_load > 0 && arc_reclaim_needed()) {
4779 /* memory is low, delay before restarting */
4780 ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
4781 return (SET_ERROR(EAGAIN));
4782 }
4783 page_load = 0;
4784 #endif
4785 return (0);
4786 }
4787
4788 void
4789 arc_tempreserve_clear(uint64_t reserve)
4790 {
4791 atomic_add_64(&arc_tempreserve, -reserve);
4792 ASSERT((int64_t)arc_tempreserve >= 0);
4793 }
4794
4795 int
4796 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
4797 {
4798 int error;
4799 uint64_t anon_size;
4800
4801 if (reserve > arc_c/4 && !arc_no_grow)
4802 arc_c = MIN(arc_c_max, reserve * 4);
4803 if (reserve > arc_c)
4804 return (SET_ERROR(ENOMEM));
4805
4806 /*
4807 * Don't count loaned bufs as in flight dirty data to prevent long
4808 * network delays from blocking transactions that are ready to be
4809 * assigned to a txg.
4810 */
4811 anon_size = MAX((int64_t)(refcount_count(&arc_anon->arcs_size) -
4812 arc_loaned_bytes), 0);
4813
4814 /*
4815 * Writes will, almost always, require additional memory allocations
4816 * in order to compress/encrypt/etc the data. We therefore need to
4817 * make sure that there is sufficient available memory for this.
4818 */
4819 error = arc_memory_throttle(reserve, txg);
4820 if (error != 0)
4821 return (error);
4822
4823 /*
4824 * Throttle writes when the amount of dirty data in the cache
4825 * gets too large. We try to keep the cache less than half full
4826 * of dirty blocks so that our sync times don't grow too large.
4827 * Note: if two requests come in concurrently, we might let them
4828 * both succeed, when one of them should fail. Not a huge deal.
4829 */
4830
4831 if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
4832 anon_size > arc_c / 4) {
4833 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
4834 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
4835 arc_tempreserve>>10,
4836 arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
4837 arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
4838 reserve>>10, arc_c>>10);
4839 return (SET_ERROR(ERESTART));
4840 }
4841 atomic_add_64(&arc_tempreserve, reserve);
4842 return (0);
4843 }
4844
4845 static void
4846 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
4847 kstat_named_t *evict_data, kstat_named_t *evict_metadata)
4848 {
4849 size->value.ui64 = refcount_count(&state->arcs_size);
4850 evict_data->value.ui64 = state->arcs_lsize[ARC_BUFC_DATA];
4851 evict_metadata->value.ui64 = state->arcs_lsize[ARC_BUFC_METADATA];
4852 }
4853
4854 static int
4855 arc_kstat_update(kstat_t *ksp, int rw)
4856 {
4857 arc_stats_t *as = ksp->ks_data;
4858
4859 if (rw == KSTAT_WRITE) {
4860 return (EACCES);
4861 } else {
4862 arc_kstat_update_state(arc_anon,
4863 &as->arcstat_anon_size,
4864 &as->arcstat_anon_evictable_data,
4865 &as->arcstat_anon_evictable_metadata);
4866 arc_kstat_update_state(arc_mru,
4867 &as->arcstat_mru_size,
4868 &as->arcstat_mru_evictable_data,
4869 &as->arcstat_mru_evictable_metadata);
4870 arc_kstat_update_state(arc_mru_ghost,
4871 &as->arcstat_mru_ghost_size,
4872 &as->arcstat_mru_ghost_evictable_data,
4873 &as->arcstat_mru_ghost_evictable_metadata);
4874 arc_kstat_update_state(arc_mfu,
4875 &as->arcstat_mfu_size,
4876 &as->arcstat_mfu_evictable_data,
4877 &as->arcstat_mfu_evictable_metadata);
4878 arc_kstat_update_state(arc_mfu_ghost,
4879 &as->arcstat_mfu_ghost_size,
4880 &as->arcstat_mfu_ghost_evictable_data,
4881 &as->arcstat_mfu_ghost_evictable_metadata);
4882 }
4883
4884 return (0);
4885 }
4886
4887 /*
4888 * This function *must* return indices evenly distributed between all
4889 * sublists of the multilist. This is needed due to how the ARC eviction
4890 * code is laid out; arc_evict_state() assumes ARC buffers are evenly
4891 * distributed between all sublists and uses this assumption when
4892 * deciding which sublist to evict from and how much to evict from it.
4893 */
4894 unsigned int
4895 arc_state_multilist_index_func(multilist_t *ml, void *obj)
4896 {
4897 arc_buf_hdr_t *hdr = obj;
4898
4899 /*
4900 * We rely on b_dva to generate evenly distributed index
4901 * numbers using buf_hash below. So, as an added precaution,
4902 * let's make sure we never add empty buffers to the arc lists.
4903 */
4904 ASSERT(!BUF_EMPTY(hdr));
4905
4906 /*
4907 * The assumption here, is the hash value for a given
4908 * arc_buf_hdr_t will remain constant throughout it's lifetime
4909 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
4910 * Thus, we don't need to store the header's sublist index
4911 * on insertion, as this index can be recalculated on removal.
4912 *
4913 * Also, the low order bits of the hash value are thought to be
4914 * distributed evenly. Otherwise, in the case that the multilist
4915 * has a power of two number of sublists, each sublists' usage
4916 * would not be evenly distributed.
4917 */
4918 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
4919 multilist_get_num_sublists(ml));
4920 }
4921
4922 void
4923 arc_init(void)
4924 {
4925 /*
4926 * allmem is "all memory that we could possibly use".
4927 */
4928 #ifdef _KERNEL
4929 uint64_t allmem = ptob(physmem - swapfs_minfree);
4930 #else
4931 uint64_t allmem = (physmem * PAGESIZE) / 2;
4932 #endif
4933
4934 mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
4935 cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
4936 cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
4937
4938 mutex_init(&arc_user_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
4939 cv_init(&arc_user_evicts_cv, NULL, CV_DEFAULT, NULL);
4940
4941 /* Convert seconds to clock ticks */
4942 arc_min_prefetch_lifespan = 1 * hz;
4943
4944 /* Start out with 1/8 of all memory */
4945 arc_c = allmem / 8;
4946
4947 #ifdef _KERNEL
4948 /*
4949 * On architectures where the physical memory can be larger
4950 * than the addressable space (intel in 32-bit mode), we may
4951 * need to limit the cache to 1/8 of VM size.
4952 */
4953 arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
4954 #endif
4955
4956 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */
4957 arc_c_min = MAX(allmem / 32, 64 << 20);
4958 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */
4959 if (allmem >= 1 << 30)
4960 arc_c_max = allmem - (1 << 30);
4961 else
4962 arc_c_max = arc_c_min;
4963 arc_c_max = MAX(allmem * 3 / 4, arc_c_max);
4964
4965 /*
4966 * Allow the tunables to override our calculations if they are
4967 * reasonable (ie. over 64MB)
4968 */
4969 if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem)
4970 arc_c_max = zfs_arc_max;
4971 if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max)
4972 arc_c_min = zfs_arc_min;
4973
4974 arc_c = arc_c_max;
4975 arc_p = (arc_c >> 1);
4976
4977 /* limit meta-data to 1/4 of the arc capacity */
4978 arc_meta_limit = arc_c_max / 4;
4979
4980 /* Allow the tunable to override if it is reasonable */
4981 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
4982 arc_meta_limit = zfs_arc_meta_limit;
4983
4984 if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
4985 arc_c_min = arc_meta_limit / 2;
4986
4987 if (zfs_arc_meta_min > 0) {
4988 arc_meta_min = zfs_arc_meta_min;
4989 } else {
4990 arc_meta_min = arc_c_min / 2;
4991 }
4992
4993 if (zfs_arc_grow_retry > 0)
4994 arc_grow_retry = zfs_arc_grow_retry;
4995
4996 if (zfs_arc_shrink_shift > 0)
4997 arc_shrink_shift = zfs_arc_shrink_shift;
4998
4999 /*
5000 * Ensure that arc_no_grow_shift is less than arc_shrink_shift.
5001 */
5002 if (arc_no_grow_shift >= arc_shrink_shift)
5003 arc_no_grow_shift = arc_shrink_shift - 1;
5004
5005 if (zfs_arc_p_min_shift > 0)
5006 arc_p_min_shift = zfs_arc_p_min_shift;
5007
5008 if (zfs_arc_num_sublists_per_state < 1)
5009 zfs_arc_num_sublists_per_state = MAX(boot_ncpus, 1);
5010
5011 /* if kmem_flags are set, lets try to use less memory */
5012 if (kmem_debugging())
5013 arc_c = arc_c / 2;
5014 if (arc_c < arc_c_min)
5015 arc_c = arc_c_min;
5016
5017 arc_anon = &ARC_anon;
5018 arc_mru = &ARC_mru;
5019 arc_mru_ghost = &ARC_mru_ghost;
5020 arc_mfu = &ARC_mfu;
5021 arc_mfu_ghost = &ARC_mfu_ghost;
5022 arc_l2c_only = &ARC_l2c_only;
5023 arc_size = 0;
5024
5025 multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
5026 sizeof (arc_buf_hdr_t),
5027 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5028 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5029 multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
5030 sizeof (arc_buf_hdr_t),
5031 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5032 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5033 multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
5034 sizeof (arc_buf_hdr_t),
5035 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5036 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5037 multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
5038 sizeof (arc_buf_hdr_t),
5039 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5040 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5041 multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
5042 sizeof (arc_buf_hdr_t),
5043 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5044 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5045 multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
5046 sizeof (arc_buf_hdr_t),
5047 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5048 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5049 multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
5050 sizeof (arc_buf_hdr_t),
5051 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5052 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5053 multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
5054 sizeof (arc_buf_hdr_t),
5055 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5056 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5057 multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
5058 sizeof (arc_buf_hdr_t),
5059 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5060 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5061 multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
5062 sizeof (arc_buf_hdr_t),
5063 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5064 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5065
5066 refcount_create(&arc_anon->arcs_size);
5067 refcount_create(&arc_mru->arcs_size);
5068 refcount_create(&arc_mru_ghost->arcs_size);
5069 refcount_create(&arc_mfu->arcs_size);
5070 refcount_create(&arc_mfu_ghost->arcs_size);
5071 refcount_create(&arc_l2c_only->arcs_size);
5072
5073 buf_init();
5074
5075 arc_reclaim_thread_exit = FALSE;
5076 arc_user_evicts_thread_exit = FALSE;
5077 arc_eviction_list = NULL;
5078 bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
5079
5080 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
5081 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
5082
5083 if (arc_ksp != NULL) {
5084 arc_ksp->ks_data = &arc_stats;
5085 arc_ksp->ks_update = arc_kstat_update;
5086 kstat_install(arc_ksp);
5087 }
5088
5089 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
5090 TS_RUN, minclsyspri);
5091
5092 (void) thread_create(NULL, 0, arc_user_evicts_thread, NULL, 0, &p0,
5093 TS_RUN, minclsyspri);
5094
5095 arc_dead = FALSE;
5096 arc_warm = B_FALSE;
5097
5098 /*
5099 * Calculate maximum amount of dirty data per pool.
5100 *
5101 * If it has been set by /etc/system, take that.
5102 * Otherwise, use a percentage of physical memory defined by
5103 * zfs_dirty_data_max_percent (default 10%) with a cap at
5104 * zfs_dirty_data_max_max (default 4GB).
5105 */
5106 if (zfs_dirty_data_max == 0) {
5107 zfs_dirty_data_max = physmem * PAGESIZE *
5108 zfs_dirty_data_max_percent / 100;
5109 zfs_dirty_data_max = MIN(zfs_dirty_data_max,
5110 zfs_dirty_data_max_max);
5111 }
5112 }
5113
5114 void
5115 arc_fini(void)
5116 {
5117 mutex_enter(&arc_reclaim_lock);
5118 arc_reclaim_thread_exit = TRUE;
5119 /*
5120 * The reclaim thread will set arc_reclaim_thread_exit back to
5121 * FALSE when it is finished exiting; we're waiting for that.
5122 */
5123 while (arc_reclaim_thread_exit) {
5124 cv_signal(&arc_reclaim_thread_cv);
5125 cv_wait(&arc_reclaim_thread_cv, &arc_reclaim_lock);
5126 }
5127 mutex_exit(&arc_reclaim_lock);
5128
5129 mutex_enter(&arc_user_evicts_lock);
5130 arc_user_evicts_thread_exit = TRUE;
5131 /*
5132 * The user evicts thread will set arc_user_evicts_thread_exit
5133 * to FALSE when it is finished exiting; we're waiting for that.
5134 */
5135 while (arc_user_evicts_thread_exit) {
5136 cv_signal(&arc_user_evicts_cv);
5137 cv_wait(&arc_user_evicts_cv, &arc_user_evicts_lock);
5138 }
5139 mutex_exit(&arc_user_evicts_lock);
5140
5141 /* Use TRUE to ensure *all* buffers are evicted */
5142 arc_flush(NULL, TRUE);
5143
5144 arc_dead = TRUE;
5145
5146 if (arc_ksp != NULL) {
5147 kstat_delete(arc_ksp);
5148 arc_ksp = NULL;
5149 }
5150
5151 mutex_destroy(&arc_reclaim_lock);
5152 cv_destroy(&arc_reclaim_thread_cv);
5153 cv_destroy(&arc_reclaim_waiters_cv);
5154
5155 mutex_destroy(&arc_user_evicts_lock);
5156 cv_destroy(&arc_user_evicts_cv);
5157
5158 refcount_destroy(&arc_anon->arcs_size);
5159 refcount_destroy(&arc_mru->arcs_size);
5160 refcount_destroy(&arc_mru_ghost->arcs_size);
5161 refcount_destroy(&arc_mfu->arcs_size);
5162 refcount_destroy(&arc_mfu_ghost->arcs_size);
5163 refcount_destroy(&arc_l2c_only->arcs_size);
5164
5165 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
5166 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
5167 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
5168 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
5169 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
5170 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
5171 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
5172 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
5173
5174 buf_fini();
5175
5176 ASSERT0(arc_loaned_bytes);
5177 }
5178
5179 /*
5180 * Level 2 ARC
5181 *
5182 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
5183 * It uses dedicated storage devices to hold cached data, which are populated
5184 * using large infrequent writes. The main role of this cache is to boost
5185 * the performance of random read workloads. The intended L2ARC devices
5186 * include short-stroked disks, solid state disks, and other media with
5187 * substantially faster read latency than disk.
5188 *
5189 * +-----------------------+
5190 * | ARC |
5191 * +-----------------------+
5192 * | ^ ^
5193 * | | |
5194 * l2arc_feed_thread() arc_read()
5195 * | | |
5196 * | l2arc read |
5197 * V | |
5198 * +---------------+ |
5199 * | L2ARC | |
5200 * +---------------+ |
5201 * | ^ |
5202 * l2arc_write() | |
5203 * | | |
5204 * V | |
5205 * +-------+ +-------+
5206 * | vdev | | vdev |
5207 * | cache | | cache |
5208 * +-------+ +-------+
5209 * +=========+ .-----.
5210 * : L2ARC : |-_____-|
5211 * : devices : | Disks |
5212 * +=========+ `-_____-'
5213 *
5214 * Read requests are satisfied from the following sources, in order:
5215 *
5216 * 1) ARC
5217 * 2) vdev cache of L2ARC devices
5218 * 3) L2ARC devices
5219 * 4) vdev cache of disks
5220 * 5) disks
5221 *
5222 * Some L2ARC device types exhibit extremely slow write performance.
5223 * To accommodate for this there are some significant differences between
5224 * the L2ARC and traditional cache design:
5225 *
5226 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from
5227 * the ARC behave as usual, freeing buffers and placing headers on ghost
5228 * lists. The ARC does not send buffers to the L2ARC during eviction as
5229 * this would add inflated write latencies for all ARC memory pressure.
5230 *
5231 * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
5232 * It does this by periodically scanning buffers from the eviction-end of
5233 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
5234 * not already there. It scans until a headroom of buffers is satisfied,
5235 * which itself is a buffer for ARC eviction. If a compressible buffer is
5236 * found during scanning and selected for writing to an L2ARC device, we
5237 * temporarily boost scanning headroom during the next scan cycle to make
5238 * sure we adapt to compression effects (which might significantly reduce
5239 * the data volume we write to L2ARC). The thread that does this is
5240 * l2arc_feed_thread(), illustrated below; example sizes are included to
5241 * provide a better sense of ratio than this diagram:
5242 *
5243 * head --> tail
5244 * +---------------------+----------+
5245 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC
5246 * +---------------------+----------+ | o L2ARC eligible
5247 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer
5248 * +---------------------+----------+ |
5249 * 15.9 Gbytes ^ 32 Mbytes |
5250 * headroom |
5251 * l2arc_feed_thread()
5252 * |
5253 * l2arc write hand <--[oooo]--'
5254 * | 8 Mbyte
5255 * | write max
5256 * V
5257 * +==============================+
5258 * L2ARC dev |####|#|###|###| |####| ... |
5259 * +==============================+
5260 * 32 Gbytes
5261 *
5262 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
5263 * evicted, then the L2ARC has cached a buffer much sooner than it probably
5264 * needed to, potentially wasting L2ARC device bandwidth and storage. It is
5265 * safe to say that this is an uncommon case, since buffers at the end of
5266 * the ARC lists have moved there due to inactivity.
5267 *
5268 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
5269 * then the L2ARC simply misses copying some buffers. This serves as a
5270 * pressure valve to prevent heavy read workloads from both stalling the ARC
5271 * with waits and clogging the L2ARC with writes. This also helps prevent
5272 * the potential for the L2ARC to churn if it attempts to cache content too
5273 * quickly, such as during backups of the entire pool.
5274 *
5275 * 5. After system boot and before the ARC has filled main memory, there are
5276 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
5277 * lists can remain mostly static. Instead of searching from tail of these
5278 * lists as pictured, the l2arc_feed_thread() will search from the list heads
5279 * for eligible buffers, greatly increasing its chance of finding them.
5280 *
5281 * The L2ARC device write speed is also boosted during this time so that
5282 * the L2ARC warms up faster. Since there have been no ARC evictions yet,
5283 * there are no L2ARC reads, and no fear of degrading read performance
5284 * through increased writes.
5285 *
5286 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
5287 * the vdev queue can aggregate them into larger and fewer writes. Each
5288 * device is written to in a rotor fashion, sweeping writes through
5289 * available space then repeating.
5290 *
5291 * 7. The L2ARC does not store dirty content. It never needs to flush
5292 * write buffers back to disk based storage.
5293 *
5294 * 8. If an ARC buffer is written (and dirtied) which also exists in the
5295 * L2ARC, the now stale L2ARC buffer is immediately dropped.
5296 *
5297 * The performance of the L2ARC can be tweaked by a number of tunables, which
5298 * may be necessary for different workloads:
5299 *
5300 * l2arc_write_max max write bytes per interval
5301 * l2arc_write_boost extra write bytes during device warmup
5302 * l2arc_noprefetch skip caching prefetched buffers
5303 * l2arc_headroom number of max device writes to precache
5304 * l2arc_headroom_boost when we find compressed buffers during ARC
5305 * scanning, we multiply headroom by this
5306 * percentage factor for the next scan cycle,
5307 * since more compressed buffers are likely to
5308 * be present
5309 * l2arc_feed_secs seconds between L2ARC writing
5310 *
5311 * Tunables may be removed or added as future performance improvements are
5312 * integrated, and also may become zpool properties.
5313 *
5314 * There are three key functions that control how the L2ARC warms up:
5315 *
5316 * l2arc_write_eligible() check if a buffer is eligible to cache
5317 * l2arc_write_size() calculate how much to write
5318 * l2arc_write_interval() calculate sleep delay between writes
5319 *
5320 * These three functions determine what to write, how much, and how quickly
5321 * to send writes.
5322 */
5323
5324 static boolean_t
5325 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
5326 {
5327 /*
5328 * A buffer is *not* eligible for the L2ARC if it:
5329 * 1. belongs to a different spa.
5330 * 2. is already cached on the L2ARC.
5331 * 3. has an I/O in progress (it may be an incomplete read).
5332 * 4. is flagged not eligible (zfs property).
5333 */
5334 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
5335 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
5336 return (B_FALSE);
5337
5338 return (B_TRUE);
5339 }
5340
5341 static uint64_t
5342 l2arc_write_size(void)
5343 {
5344 uint64_t size;
5345
5346 /*
5347 * Make sure our globals have meaningful values in case the user
5348 * altered them.
5349 */
5350 size = l2arc_write_max;
5351 if (size == 0) {
5352 cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
5353 "be greater than zero, resetting it to the default (%d)",
5354 L2ARC_WRITE_SIZE);
5355 size = l2arc_write_max = L2ARC_WRITE_SIZE;
5356 }
5357
5358 if (arc_warm == B_FALSE)
5359 size += l2arc_write_boost;
5360
5361 return (size);
5362
5363 }
5364
5365 static clock_t
5366 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
5367 {
5368 clock_t interval, next, now;
5369
5370 /*
5371 * If the ARC lists are busy, increase our write rate; if the
5372 * lists are stale, idle back. This is achieved by checking
5373 * how much we previously wrote - if it was more than half of
5374 * what we wanted, schedule the next write much sooner.
5375 */
5376 if (l2arc_feed_again && wrote > (wanted / 2))
5377 interval = (hz * l2arc_feed_min_ms) / 1000;
5378 else
5379 interval = hz * l2arc_feed_secs;
5380
5381 now = ddi_get_lbolt();
5382 next = MAX(now, MIN(now + interval, began + interval));
5383
5384 return (next);
5385 }
5386
5387 /*
5388 * Cycle through L2ARC devices. This is how L2ARC load balances.
5389 * If a device is returned, this also returns holding the spa config lock.
5390 */
5391 static l2arc_dev_t *
5392 l2arc_dev_get_next(void)
5393 {
5394 l2arc_dev_t *first, *next = NULL;
5395
5396 /*
5397 * Lock out the removal of spas (spa_namespace_lock), then removal
5398 * of cache devices (l2arc_dev_mtx). Once a device has been selected,
5399 * both locks will be dropped and a spa config lock held instead.
5400 */
5401 mutex_enter(&spa_namespace_lock);
5402 mutex_enter(&l2arc_dev_mtx);
5403
5404 /* if there are no vdevs, there is nothing to do */
5405 if (l2arc_ndev == 0)
5406 goto out;
5407
5408 first = NULL;
5409 next = l2arc_dev_last;
5410 do {
5411 /* loop around the list looking for a non-faulted vdev */
5412 if (next == NULL) {
5413 next = list_head(l2arc_dev_list);
5414 } else {
5415 next = list_next(l2arc_dev_list, next);
5416 if (next == NULL)
5417 next = list_head(l2arc_dev_list);
5418 }
5419
5420 /* if we have come back to the start, bail out */
5421 if (first == NULL)
5422 first = next;
5423 else if (next == first)
5424 break;
5425
5426 } while (vdev_is_dead(next->l2ad_vdev));
5427
5428 /* if we were unable to find any usable vdevs, return NULL */
5429 if (vdev_is_dead(next->l2ad_vdev))
5430 next = NULL;
5431
5432 l2arc_dev_last = next;
5433
5434 out:
5435 mutex_exit(&l2arc_dev_mtx);
5436
5437 /*
5438 * Grab the config lock to prevent the 'next' device from being
5439 * removed while we are writing to it.
5440 */
5441 if (next != NULL)
5442 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
5443 mutex_exit(&spa_namespace_lock);
5444
5445 return (next);
5446 }
5447
5448 /*
5449 * Free buffers that were tagged for destruction.
5450 */
5451 static void
5452 l2arc_do_free_on_write()
5453 {
5454 list_t *buflist;
5455 l2arc_data_free_t *df, *df_prev;
5456
5457 mutex_enter(&l2arc_free_on_write_mtx);
5458 buflist = l2arc_free_on_write;
5459
5460 for (df = list_tail(buflist); df; df = df_prev) {
5461 df_prev = list_prev(buflist, df);
5462 ASSERT(df->l2df_data != NULL);
5463 ASSERT(df->l2df_func != NULL);
5464 df->l2df_func(df->l2df_data, df->l2df_size);
5465 list_remove(buflist, df);
5466 kmem_free(df, sizeof (l2arc_data_free_t));
5467 }
5468
5469 mutex_exit(&l2arc_free_on_write_mtx);
5470 }
5471
5472 /*
5473 * A write to a cache device has completed. Update all headers to allow
5474 * reads from these buffers to begin.
5475 */
5476 static void
5477 l2arc_write_done(zio_t *zio)
5478 {
5479 l2arc_write_callback_t *cb;
5480 l2arc_dev_t *dev;
5481 list_t *buflist;
5482 arc_buf_hdr_t *head, *hdr, *hdr_prev;
5483 kmutex_t *hash_lock;
5484 int64_t bytes_dropped = 0;
5485
5486 cb = zio->io_private;
5487 ASSERT(cb != NULL);
5488 dev = cb->l2wcb_dev;
5489 ASSERT(dev != NULL);
5490 head = cb->l2wcb_head;
5491 ASSERT(head != NULL);
5492 buflist = &dev->l2ad_buflist;
5493 ASSERT(buflist != NULL);
5494 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
5495 l2arc_write_callback_t *, cb);
5496
5497 if (zio->io_error != 0)
5498 ARCSTAT_BUMP(arcstat_l2_writes_error);
5499
5500 /*
5501 * All writes completed, or an error was hit.
5502 */
5503 top:
5504 mutex_enter(&dev->l2ad_mtx);
5505 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
5506 hdr_prev = list_prev(buflist, hdr);
5507
5508 hash_lock = HDR_LOCK(hdr);
5509
5510 /*
5511 * We cannot use mutex_enter or else we can deadlock
5512 * with l2arc_write_buffers (due to swapping the order
5513 * the hash lock and l2ad_mtx are taken).
5514 */
5515 if (!mutex_tryenter(hash_lock)) {
5516 /*
5517 * Missed the hash lock. We must retry so we
5518 * don't leave the ARC_FLAG_L2_WRITING bit set.
5519 */
5520 ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
5521
5522 /*
5523 * We don't want to rescan the headers we've
5524 * already marked as having been written out, so
5525 * we reinsert the head node so we can pick up
5526 * where we left off.
5527 */
5528 list_remove(buflist, head);
5529 list_insert_after(buflist, hdr, head);
5530
5531 mutex_exit(&dev->l2ad_mtx);
5532
5533 /*
5534 * We wait for the hash lock to become available
5535 * to try and prevent busy waiting, and increase
5536 * the chance we'll be able to acquire the lock
5537 * the next time around.
5538 */
5539 mutex_enter(hash_lock);
5540 mutex_exit(hash_lock);
5541 goto top;
5542 }
5543
5544 /*
5545 * We could not have been moved into the arc_l2c_only
5546 * state while in-flight due to our ARC_FLAG_L2_WRITING
5547 * bit being set. Let's just ensure that's being enforced.
5548 */
5549 ASSERT(HDR_HAS_L1HDR(hdr));
5550
5551 /*
5552 * We may have allocated a buffer for L2ARC compression,
5553 * we must release it to avoid leaking this data.
5554 */
5555 l2arc_release_cdata_buf(hdr);
5556
5557 if (zio->io_error != 0) {
5558 /*
5559 * Error - drop L2ARC entry.
5560 */
5561 list_remove(buflist, hdr);
5562 hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
5563
5564 ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
5565 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
5566
5567 bytes_dropped += hdr->b_l2hdr.b_asize;
5568 (void) refcount_remove_many(&dev->l2ad_alloc,
5569 hdr->b_l2hdr.b_asize, hdr);
5570 }
5571
5572 /*
5573 * Allow ARC to begin reads and ghost list evictions to
5574 * this L2ARC entry.
5575 */
5576 hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
5577
5578 mutex_exit(hash_lock);
5579 }
5580
5581 atomic_inc_64(&l2arc_writes_done);
5582 list_remove(buflist, head);
5583 ASSERT(!HDR_HAS_L1HDR(head));
5584 kmem_cache_free(hdr_l2only_cache, head);
5585 mutex_exit(&dev->l2ad_mtx);
5586
5587 vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
5588
5589 l2arc_do_free_on_write();
5590
5591 kmem_free(cb, sizeof (l2arc_write_callback_t));
5592 }
5593
5594 /*
5595 * A read to a cache device completed. Validate buffer contents before
5596 * handing over to the regular ARC routines.
5597 */
5598 static void
5599 l2arc_read_done(zio_t *zio)
5600 {
5601 l2arc_read_callback_t *cb;
5602 arc_buf_hdr_t *hdr;
5603 arc_buf_t *buf;
5604 kmutex_t *hash_lock;
5605 int equal;
5606
5607 ASSERT(zio->io_vd != NULL);
5608 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
5609
5610 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
5611
5612 cb = zio->io_private;
5613 ASSERT(cb != NULL);
5614 buf = cb->l2rcb_buf;
5615 ASSERT(buf != NULL);
5616
5617 hash_lock = HDR_LOCK(buf->b_hdr);
5618 mutex_enter(hash_lock);
5619 hdr = buf->b_hdr;
5620 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5621
5622 /*
5623 * If the buffer was compressed, decompress it first.
5624 */
5625 if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
5626 l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
5627 ASSERT(zio->io_data != NULL);
5628 ASSERT3U(zio->io_size, ==, hdr->b_size);
5629 ASSERT3U(BP_GET_LSIZE(&cb->l2rcb_bp), ==, hdr->b_size);
5630
5631 /*
5632 * Check this survived the L2ARC journey.
5633 */
5634 equal = arc_cksum_equal(buf);
5635 if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
5636 mutex_exit(hash_lock);
5637 zio->io_private = buf;
5638 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */
5639 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */
5640 arc_read_done(zio);
5641 } else {
5642 mutex_exit(hash_lock);
5643 /*
5644 * Buffer didn't survive caching. Increment stats and
5645 * reissue to the original storage device.
5646 */
5647 if (zio->io_error != 0) {
5648 ARCSTAT_BUMP(arcstat_l2_io_error);
5649 } else {
5650 zio->io_error = SET_ERROR(EIO);
5651 }
5652 if (!equal)
5653 ARCSTAT_BUMP(arcstat_l2_cksum_bad);
5654
5655 /*
5656 * If there's no waiter, issue an async i/o to the primary
5657 * storage now. If there *is* a waiter, the caller must
5658 * issue the i/o in a context where it's OK to block.
5659 */
5660 if (zio->io_waiter == NULL) {
5661 zio_t *pio = zio_unique_parent(zio);
5662
5663 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
5664
5665 zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
5666 buf->b_data, hdr->b_size, arc_read_done, buf,
5667 zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
5668 }
5669 }
5670
5671 kmem_free(cb, sizeof (l2arc_read_callback_t));
5672 }
5673
5674 /*
5675 * This is the list priority from which the L2ARC will search for pages to
5676 * cache. This is used within loops (0..3) to cycle through lists in the
5677 * desired order. This order can have a significant effect on cache
5678 * performance.
5679 *
5680 * Currently the metadata lists are hit first, MFU then MRU, followed by
5681 * the data lists. This function returns a locked list, and also returns
5682 * the lock pointer.
5683 */
5684 static multilist_sublist_t *
5685 l2arc_sublist_lock(int list_num)
5686 {
5687 multilist_t *ml = NULL;
5688 unsigned int idx;
5689
5690 ASSERT(list_num >= 0 && list_num <= 3);
5691
5692 switch (list_num) {
5693 case 0:
5694 ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
5695 break;
5696 case 1:
5697 ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
5698 break;
5699 case 2:
5700 ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
5701 break;
5702 case 3:
5703 ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
5704 break;
5705 }
5706
5707 /*
5708 * Return a randomly-selected sublist. This is acceptable
5709 * because the caller feeds only a little bit of data for each
5710 * call (8MB). Subsequent calls will result in different
5711 * sublists being selected.
5712 */
5713 idx = multilist_get_random_index(ml);
5714 return (multilist_sublist_lock(ml, idx));
5715 }
5716
5717 /*
5718 * Evict buffers from the device write hand to the distance specified in
5719 * bytes. This distance may span populated buffers, it may span nothing.
5720 * This is clearing a region on the L2ARC device ready for writing.
5721 * If the 'all' boolean is set, every buffer is evicted.
5722 */
5723 static void
5724 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
5725 {
5726 list_t *buflist;
5727 arc_buf_hdr_t *hdr, *hdr_prev;
5728 kmutex_t *hash_lock;
5729 uint64_t taddr;
5730
5731 buflist = &dev->l2ad_buflist;
5732
5733 if (!all && dev->l2ad_first) {
5734 /*
5735 * This is the first sweep through the device. There is
5736 * nothing to evict.
5737 */
5738 return;
5739 }
5740
5741 if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
5742 /*
5743 * When nearing the end of the device, evict to the end
5744 * before the device write hand jumps to the start.
5745 */
5746 taddr = dev->l2ad_end;
5747 } else {
5748 taddr = dev->l2ad_hand + distance;
5749 }
5750 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
5751 uint64_t, taddr, boolean_t, all);
5752
5753 top:
5754 mutex_enter(&dev->l2ad_mtx);
5755 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
5756 hdr_prev = list_prev(buflist, hdr);
5757
5758 hash_lock = HDR_LOCK(hdr);
5759
5760 /*
5761 * We cannot use mutex_enter or else we can deadlock
5762 * with l2arc_write_buffers (due to swapping the order
5763 * the hash lock and l2ad_mtx are taken).
5764 */
5765 if (!mutex_tryenter(hash_lock)) {
5766 /*
5767 * Missed the hash lock. Retry.
5768 */
5769 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
5770 mutex_exit(&dev->l2ad_mtx);
5771 mutex_enter(hash_lock);
5772 mutex_exit(hash_lock);
5773 goto top;
5774 }
5775
5776 if (HDR_L2_WRITE_HEAD(hdr)) {
5777 /*
5778 * We hit a write head node. Leave it for
5779 * l2arc_write_done().
5780 */
5781 list_remove(buflist, hdr);
5782 mutex_exit(hash_lock);
5783 continue;
5784 }
5785
5786 if (!all && HDR_HAS_L2HDR(hdr) &&
5787 (hdr->b_l2hdr.b_daddr > taddr ||
5788 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
5789 /*
5790 * We've evicted to the target address,
5791 * or the end of the device.
5792 */
5793 mutex_exit(hash_lock);
5794 break;
5795 }
5796
5797 ASSERT(HDR_HAS_L2HDR(hdr));
5798 if (!HDR_HAS_L1HDR(hdr)) {
5799 ASSERT(!HDR_L2_READING(hdr));
5800 /*
5801 * This doesn't exist in the ARC. Destroy.
5802 * arc_hdr_destroy() will call list_remove()
5803 * and decrement arcstat_l2_size.
5804 */
5805 arc_change_state(arc_anon, hdr, hash_lock);
5806 arc_hdr_destroy(hdr);
5807 } else {
5808 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
5809 ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
5810 /*
5811 * Invalidate issued or about to be issued
5812 * reads, since we may be about to write
5813 * over this location.
5814 */
5815 if (HDR_L2_READING(hdr)) {
5816 ARCSTAT_BUMP(arcstat_l2_evict_reading);
5817 hdr->b_flags |= ARC_FLAG_L2_EVICTED;
5818 }
5819
5820 /* Ensure this header has finished being written */
5821 ASSERT(!HDR_L2_WRITING(hdr));
5822 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
5823
5824 arc_hdr_l2hdr_destroy(hdr);
5825 }
5826 mutex_exit(hash_lock);
5827 }
5828 mutex_exit(&dev->l2ad_mtx);
5829 }
5830
5831 /*
5832 * Find and write ARC buffers to the L2ARC device.
5833 *
5834 * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
5835 * for reading until they have completed writing.
5836 * The headroom_boost is an in-out parameter used to maintain headroom boost
5837 * state between calls to this function.
5838 *
5839 * Returns the number of bytes actually written (which may be smaller than
5840 * the delta by which the device hand has changed due to alignment).
5841 */
5842 static uint64_t
5843 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
5844 boolean_t *headroom_boost)
5845 {
5846 arc_buf_hdr_t *hdr, *hdr_prev, *head;
5847 uint64_t write_asize, write_psize, write_sz, headroom,
5848 buf_compress_minsz;
5849 void *buf_data;
5850 boolean_t full;
5851 l2arc_write_callback_t *cb;
5852 zio_t *pio, *wzio;
5853 uint64_t guid = spa_load_guid(spa);
5854 const boolean_t do_headroom_boost = *headroom_boost;
5855
5856 ASSERT(dev->l2ad_vdev != NULL);
5857
5858 /* Lower the flag now, we might want to raise it again later. */
5859 *headroom_boost = B_FALSE;
5860
5861 pio = NULL;
5862 write_sz = write_asize = write_psize = 0;
5863 full = B_FALSE;
5864 head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
5865 head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
5866 head->b_flags |= ARC_FLAG_HAS_L2HDR;
5867
5868 /*
5869 * We will want to try to compress buffers that are at least 2x the
5870 * device sector size.
5871 */
5872 buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift;
5873
5874 /*
5875 * Copy buffers for L2ARC writing.
5876 */
5877 for (int try = 0; try <= 3; try++) {
5878 multilist_sublist_t *mls = l2arc_sublist_lock(try);
5879 uint64_t passed_sz = 0;
5880
5881 /*
5882 * L2ARC fast warmup.
5883 *
5884 * Until the ARC is warm and starts to evict, read from the
5885 * head of the ARC lists rather than the tail.
5886 */
5887 if (arc_warm == B_FALSE)
5888 hdr = multilist_sublist_head(mls);
5889 else
5890 hdr = multilist_sublist_tail(mls);
5891
5892 headroom = target_sz * l2arc_headroom;
5893 if (do_headroom_boost)
5894 headroom = (headroom * l2arc_headroom_boost) / 100;
5895
5896 for (; hdr; hdr = hdr_prev) {
5897 kmutex_t *hash_lock;
5898 uint64_t buf_sz;
5899
5900 if (arc_warm == B_FALSE)
5901 hdr_prev = multilist_sublist_next(mls, hdr);
5902 else
5903 hdr_prev = multilist_sublist_prev(mls, hdr);
5904
5905 hash_lock = HDR_LOCK(hdr);
5906 if (!mutex_tryenter(hash_lock)) {
5907 /*
5908 * Skip this buffer rather than waiting.
5909 */
5910 continue;
5911 }
5912
5913 passed_sz += hdr->b_size;
5914 if (passed_sz > headroom) {
5915 /*
5916 * Searched too far.
5917 */
5918 mutex_exit(hash_lock);
5919 break;
5920 }
5921
5922 if (!l2arc_write_eligible(guid, hdr)) {
5923 mutex_exit(hash_lock);
5924 continue;
5925 }
5926
5927 if ((write_sz + hdr->b_size) > target_sz) {
5928 full = B_TRUE;
5929 mutex_exit(hash_lock);
5930 break;
5931 }
5932
5933 if (pio == NULL) {
5934 /*
5935 * Insert a dummy header on the buflist so
5936 * l2arc_write_done() can find where the
5937 * write buffers begin without searching.
5938 */
5939 mutex_enter(&dev->l2ad_mtx);
5940 list_insert_head(&dev->l2ad_buflist, head);
5941 mutex_exit(&dev->l2ad_mtx);
5942
5943 cb = kmem_alloc(
5944 sizeof (l2arc_write_callback_t), KM_SLEEP);
5945 cb->l2wcb_dev = dev;
5946 cb->l2wcb_head = head;
5947 pio = zio_root(spa, l2arc_write_done, cb,
5948 ZIO_FLAG_CANFAIL);
5949 }
5950
5951 /*
5952 * Create and add a new L2ARC header.
5953 */
5954 hdr->b_l2hdr.b_dev = dev;
5955 hdr->b_flags |= ARC_FLAG_L2_WRITING;
5956 /*
5957 * Temporarily stash the data buffer in b_tmp_cdata.
5958 * The subsequent write step will pick it up from
5959 * there. This is because can't access b_l1hdr.b_buf
5960 * without holding the hash_lock, which we in turn
5961 * can't access without holding the ARC list locks
5962 * (which we want to avoid during compression/writing).
5963 */
5964 hdr->b_l2hdr.b_compress = ZIO_COMPRESS_OFF;
5965 hdr->b_l2hdr.b_asize = hdr->b_size;
5966 hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
5967
5968 /*
5969 * Explicitly set the b_daddr field to a known
5970 * value which means "invalid address". This
5971 * enables us to differentiate which stage of
5972 * l2arc_write_buffers() the particular header
5973 * is in (e.g. this loop, or the one below).
5974 * ARC_FLAG_L2_WRITING is not enough to make
5975 * this distinction, and we need to know in
5976 * order to do proper l2arc vdev accounting in
5977 * arc_release() and arc_hdr_destroy().
5978 *
5979 * Note, we can't use a new flag to distinguish
5980 * the two stages because we don't hold the
5981 * header's hash_lock below, in the second stage
5982 * of this function. Thus, we can't simply
5983 * change the b_flags field to denote that the
5984 * IO has been sent. We can change the b_daddr
5985 * field of the L2 portion, though, since we'll
5986 * be holding the l2ad_mtx; which is why we're
5987 * using it to denote the header's state change.
5988 */
5989 hdr->b_l2hdr.b_daddr = L2ARC_ADDR_UNSET;
5990
5991 buf_sz = hdr->b_size;
5992 hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
5993
5994 mutex_enter(&dev->l2ad_mtx);
5995 list_insert_head(&dev->l2ad_buflist, hdr);
5996 mutex_exit(&dev->l2ad_mtx);
5997
5998 /*
5999 * Compute and store the buffer cksum before
6000 * writing. On debug the cksum is verified first.
6001 */
6002 arc_cksum_verify(hdr->b_l1hdr.b_buf);
6003 arc_cksum_compute(hdr->b_l1hdr.b_buf, B_TRUE);
6004
6005 mutex_exit(hash_lock);
6006
6007 write_sz += buf_sz;
6008 }
6009
6010 multilist_sublist_unlock(mls);
6011
6012 if (full == B_TRUE)
6013 break;
6014 }
6015
6016 /* No buffers selected for writing? */
6017 if (pio == NULL) {
6018 ASSERT0(write_sz);
6019 ASSERT(!HDR_HAS_L1HDR(head));
6020 kmem_cache_free(hdr_l2only_cache, head);
6021 return (0);
6022 }
6023
6024 mutex_enter(&dev->l2ad_mtx);
6025
6026 /*
6027 * Now start writing the buffers. We're starting at the write head
6028 * and work backwards, retracing the course of the buffer selector
6029 * loop above.
6030 */
6031 for (hdr = list_prev(&dev->l2ad_buflist, head); hdr;
6032 hdr = list_prev(&dev->l2ad_buflist, hdr)) {
6033 uint64_t buf_sz;
6034
6035 /*
6036 * We rely on the L1 portion of the header below, so
6037 * it's invalid for this header to have been evicted out
6038 * of the ghost cache, prior to being written out. The
6039 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
6040 */
6041 ASSERT(HDR_HAS_L1HDR(hdr));
6042
6043 /*
6044 * We shouldn't need to lock the buffer here, since we flagged
6045 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
6046 * take care to only access its L2 cache parameters. In
6047 * particular, hdr->l1hdr.b_buf may be invalid by now due to
6048 * ARC eviction.
6049 */
6050 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
6051
6052 if ((HDR_L2COMPRESS(hdr)) &&
6053 hdr->b_l2hdr.b_asize >= buf_compress_minsz) {
6054 if (l2arc_compress_buf(hdr)) {
6055 /*
6056 * If compression succeeded, enable headroom
6057 * boost on the next scan cycle.
6058 */
6059 *headroom_boost = B_TRUE;
6060 }
6061 }
6062
6063 /*
6064 * Pick up the buffer data we had previously stashed away
6065 * (and now potentially also compressed).
6066 */
6067 buf_data = hdr->b_l1hdr.b_tmp_cdata;
6068 buf_sz = hdr->b_l2hdr.b_asize;
6069
6070 /*
6071 * We need to do this regardless if buf_sz is zero or
6072 * not, otherwise, when this l2hdr is evicted we'll
6073 * remove a reference that was never added.
6074 */
6075 (void) refcount_add_many(&dev->l2ad_alloc, buf_sz, hdr);
6076
6077 /* Compression may have squashed the buffer to zero length. */
6078 if (buf_sz != 0) {
6079 uint64_t buf_p_sz;
6080
6081 wzio = zio_write_phys(pio, dev->l2ad_vdev,
6082 dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
6083 NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
6084 ZIO_FLAG_CANFAIL, B_FALSE);
6085
6086 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
6087 zio_t *, wzio);
6088 (void) zio_nowait(wzio);
6089
6090 write_asize += buf_sz;
6091
6092 /*
6093 * Keep the clock hand suitably device-aligned.
6094 */
6095 buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
6096 write_psize += buf_p_sz;
6097 dev->l2ad_hand += buf_p_sz;
6098 }
6099 }
6100
6101 mutex_exit(&dev->l2ad_mtx);
6102
6103 ASSERT3U(write_asize, <=, target_sz);
6104 ARCSTAT_BUMP(arcstat_l2_writes_sent);
6105 ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
6106 ARCSTAT_INCR(arcstat_l2_size, write_sz);
6107 ARCSTAT_INCR(arcstat_l2_asize, write_asize);
6108 vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
6109
6110 /*
6111 * Bump device hand to the device start if it is approaching the end.
6112 * l2arc_evict() will already have evicted ahead for this case.
6113 */
6114 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
6115 dev->l2ad_hand = dev->l2ad_start;
6116 dev->l2ad_first = B_FALSE;
6117 }
6118
6119 dev->l2ad_writing = B_TRUE;
6120 (void) zio_wait(pio);
6121 dev->l2ad_writing = B_FALSE;
6122
6123 return (write_asize);
6124 }
6125
6126 /*
6127 * Compresses an L2ARC buffer.
6128 * The data to be compressed must be prefilled in l1hdr.b_tmp_cdata and its
6129 * size in l2hdr->b_asize. This routine tries to compress the data and
6130 * depending on the compression result there are three possible outcomes:
6131 * *) The buffer was incompressible. The original l2hdr contents were left
6132 * untouched and are ready for writing to an L2 device.
6133 * *) The buffer was all-zeros, so there is no need to write it to an L2
6134 * device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is
6135 * set to zero and b_compress is set to ZIO_COMPRESS_EMPTY.
6136 * *) Compression succeeded and b_tmp_cdata was replaced with a temporary
6137 * data buffer which holds the compressed data to be written, and b_asize
6138 * tells us how much data there is. b_compress is set to the appropriate
6139 * compression algorithm. Once writing is done, invoke
6140 * l2arc_release_cdata_buf on this l2hdr to free this temporary buffer.
6141 *
6142 * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the
6143 * buffer was incompressible).
6144 */
6145 static boolean_t
6146 l2arc_compress_buf(arc_buf_hdr_t *hdr)
6147 {
6148 void *cdata;
6149 size_t csize, len, rounded;
6150 ASSERT(HDR_HAS_L2HDR(hdr));
6151 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
6152
6153 ASSERT(HDR_HAS_L1HDR(hdr));
6154 ASSERT3S(l2hdr->b_compress, ==, ZIO_COMPRESS_OFF);
6155 ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
6156
6157 len = l2hdr->b_asize;
6158 cdata = zio_data_buf_alloc(len);
6159 ASSERT3P(cdata, !=, NULL);
6160 csize = zio_compress_data(ZIO_COMPRESS_LZ4, hdr->b_l1hdr.b_tmp_cdata,
6161 cdata, l2hdr->b_asize);
6162
6163 rounded = P2ROUNDUP(csize, (size_t)SPA_MINBLOCKSIZE);
6164 if (rounded > csize) {
6165 bzero((char *)cdata + csize, rounded - csize);
6166 csize = rounded;
6167 }
6168
6169 if (csize == 0) {
6170 /* zero block, indicate that there's nothing to write */
6171 zio_data_buf_free(cdata, len);
6172 l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
6173 l2hdr->b_asize = 0;
6174 hdr->b_l1hdr.b_tmp_cdata = NULL;
6175 ARCSTAT_BUMP(arcstat_l2_compress_zeros);
6176 return (B_TRUE);
6177 } else if (csize > 0 && csize < len) {
6178 /*
6179 * Compression succeeded, we'll keep the cdata around for
6180 * writing and release it afterwards.
6181 */
6182 l2hdr->b_compress = ZIO_COMPRESS_LZ4;
6183 l2hdr->b_asize = csize;
6184 hdr->b_l1hdr.b_tmp_cdata = cdata;
6185 ARCSTAT_BUMP(arcstat_l2_compress_successes);
6186 return (B_TRUE);
6187 } else {
6188 /*
6189 * Compression failed, release the compressed buffer.
6190 * l2hdr will be left unmodified.
6191 */
6192 zio_data_buf_free(cdata, len);
6193 ARCSTAT_BUMP(arcstat_l2_compress_failures);
6194 return (B_FALSE);
6195 }
6196 }
6197
6198 /*
6199 * Decompresses a zio read back from an l2arc device. On success, the
6200 * underlying zio's io_data buffer is overwritten by the uncompressed
6201 * version. On decompression error (corrupt compressed stream), the
6202 * zio->io_error value is set to signal an I/O error.
6203 *
6204 * Please note that the compressed data stream is not checksummed, so
6205 * if the underlying device is experiencing data corruption, we may feed
6206 * corrupt data to the decompressor, so the decompressor needs to be
6207 * able to handle this situation (LZ4 does).
6208 */
6209 static void
6210 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
6211 {
6212 ASSERT(L2ARC_IS_VALID_COMPRESS(c));
6213
6214 if (zio->io_error != 0) {
6215 /*
6216 * An io error has occured, just restore the original io
6217 * size in preparation for a main pool read.
6218 */
6219 zio->io_orig_size = zio->io_size = hdr->b_size;
6220 return;
6221 }
6222
6223 if (c == ZIO_COMPRESS_EMPTY) {
6224 /*
6225 * An empty buffer results in a null zio, which means we
6226 * need to fill its io_data after we're done restoring the
6227 * buffer's contents.
6228 */
6229 ASSERT(hdr->b_l1hdr.b_buf != NULL);
6230 bzero(hdr->b_l1hdr.b_buf->b_data, hdr->b_size);
6231 zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_buf->b_data;
6232 } else {
6233 ASSERT(zio->io_data != NULL);
6234 /*
6235 * We copy the compressed data from the start of the arc buffer
6236 * (the zio_read will have pulled in only what we need, the
6237 * rest is garbage which we will overwrite at decompression)
6238 * and then decompress back to the ARC data buffer. This way we
6239 * can minimize copying by simply decompressing back over the
6240 * original compressed data (rather than decompressing to an
6241 * aux buffer and then copying back the uncompressed buffer,
6242 * which is likely to be much larger).
6243 */
6244 uint64_t csize;
6245 void *cdata;
6246
6247 csize = zio->io_size;
6248 cdata = zio_data_buf_alloc(csize);
6249 bcopy(zio->io_data, cdata, csize);
6250 if (zio_decompress_data(c, cdata, zio->io_data, csize,
6251 hdr->b_size) != 0)
6252 zio->io_error = EIO;
6253 zio_data_buf_free(cdata, csize);
6254 }
6255
6256 /* Restore the expected uncompressed IO size. */
6257 zio->io_orig_size = zio->io_size = hdr->b_size;
6258 }
6259
6260 /*
6261 * Releases the temporary b_tmp_cdata buffer in an l2arc header structure.
6262 * This buffer serves as a temporary holder of compressed data while
6263 * the buffer entry is being written to an l2arc device. Once that is
6264 * done, we can dispose of it.
6265 */
6266 static void
6267 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
6268 {
6269 ASSERT(HDR_HAS_L2HDR(hdr));
6270 enum zio_compress comp = hdr->b_l2hdr.b_compress;
6271
6272 ASSERT(HDR_HAS_L1HDR(hdr));
6273 ASSERT(comp == ZIO_COMPRESS_OFF || L2ARC_IS_VALID_COMPRESS(comp));
6274
6275 if (comp == ZIO_COMPRESS_OFF) {
6276 /*
6277 * In this case, b_tmp_cdata points to the same buffer
6278 * as the arc_buf_t's b_data field. We don't want to
6279 * free it, since the arc_buf_t will handle that.
6280 */
6281 hdr->b_l1hdr.b_tmp_cdata = NULL;
6282 } else if (comp == ZIO_COMPRESS_EMPTY) {
6283 /*
6284 * In this case, b_tmp_cdata was compressed to an empty
6285 * buffer, thus there's nothing to free and b_tmp_cdata
6286 * should have been set to NULL in l2arc_write_buffers().
6287 */
6288 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
6289 } else {
6290 /*
6291 * If the data was compressed, then we've allocated a
6292 * temporary buffer for it, so now we need to release it.
6293 */
6294 ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
6295 zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
6296 hdr->b_size);
6297 hdr->b_l1hdr.b_tmp_cdata = NULL;
6298 }
6299
6300 }
6301
6302 /*
6303 * This thread feeds the L2ARC at regular intervals. This is the beating
6304 * heart of the L2ARC.
6305 */
6306 static void
6307 l2arc_feed_thread(void)
6308 {
6309 callb_cpr_t cpr;
6310 l2arc_dev_t *dev;
6311 spa_t *spa;
6312 uint64_t size, wrote;
6313 clock_t begin, next = ddi_get_lbolt();
6314 boolean_t headroom_boost = B_FALSE;
6315
6316 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
6317
6318 mutex_enter(&l2arc_feed_thr_lock);
6319
6320 while (l2arc_thread_exit == 0) {
6321 CALLB_CPR_SAFE_BEGIN(&cpr);
6322 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
6323 next);
6324 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
6325 next = ddi_get_lbolt() + hz;
6326
6327 /*
6328 * Quick check for L2ARC devices.
6329 */
6330 mutex_enter(&l2arc_dev_mtx);
6331 if (l2arc_ndev == 0) {
6332 mutex_exit(&l2arc_dev_mtx);
6333 continue;
6334 }
6335 mutex_exit(&l2arc_dev_mtx);
6336 begin = ddi_get_lbolt();
6337
6338 /*
6339 * This selects the next l2arc device to write to, and in
6340 * doing so the next spa to feed from: dev->l2ad_spa. This
6341 * will return NULL if there are now no l2arc devices or if
6342 * they are all faulted.
6343 *
6344 * If a device is returned, its spa's config lock is also
6345 * held to prevent device removal. l2arc_dev_get_next()
6346 * will grab and release l2arc_dev_mtx.
6347 */
6348 if ((dev = l2arc_dev_get_next()) == NULL)
6349 continue;
6350
6351 spa = dev->l2ad_spa;
6352 ASSERT(spa != NULL);
6353
6354 /*
6355 * If the pool is read-only then force the feed thread to
6356 * sleep a little longer.
6357 */
6358 if (!spa_writeable(spa)) {
6359 next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
6360 spa_config_exit(spa, SCL_L2ARC, dev);
6361 continue;
6362 }
6363
6364 /*
6365 * Avoid contributing to memory pressure.
6366 */
6367 if (arc_reclaim_needed()) {
6368 ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
6369 spa_config_exit(spa, SCL_L2ARC, dev);
6370 continue;
6371 }
6372
6373 ARCSTAT_BUMP(arcstat_l2_feeds);
6374
6375 size = l2arc_write_size();
6376
6377 /*
6378 * Evict L2ARC buffers that will be overwritten.
6379 */
6380 l2arc_evict(dev, size, B_FALSE);
6381
6382 /*
6383 * Write ARC buffers.
6384 */
6385 wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost);
6386
6387 /*
6388 * Calculate interval between writes.
6389 */
6390 next = l2arc_write_interval(begin, size, wrote);
6391 spa_config_exit(spa, SCL_L2ARC, dev);
6392 }
6393
6394 l2arc_thread_exit = 0;
6395 cv_broadcast(&l2arc_feed_thr_cv);
6396 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */
6397 thread_exit();
6398 }
6399
6400 boolean_t
6401 l2arc_vdev_present(vdev_t *vd)
6402 {
6403 l2arc_dev_t *dev;
6404
6405 mutex_enter(&l2arc_dev_mtx);
6406 for (dev = list_head(l2arc_dev_list); dev != NULL;
6407 dev = list_next(l2arc_dev_list, dev)) {
6408 if (dev->l2ad_vdev == vd)
6409 break;
6410 }
6411 mutex_exit(&l2arc_dev_mtx);
6412
6413 return (dev != NULL);
6414 }
6415
6416 /*
6417 * Add a vdev for use by the L2ARC. By this point the spa has already
6418 * validated the vdev and opened it.
6419 */
6420 void
6421 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
6422 {
6423 l2arc_dev_t *adddev;
6424
6425 ASSERT(!l2arc_vdev_present(vd));
6426
6427 /*
6428 * Create a new l2arc device entry.
6429 */
6430 adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
6431 adddev->l2ad_spa = spa;
6432 adddev->l2ad_vdev = vd;
6433 adddev->l2ad_start = VDEV_LABEL_START_SIZE;
6434 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
6435 adddev->l2ad_hand = adddev->l2ad_start;
6436 adddev->l2ad_first = B_TRUE;
6437 adddev->l2ad_writing = B_FALSE;
6438
6439 mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
6440 /*
6441 * This is a list of all ARC buffers that are still valid on the
6442 * device.
6443 */
6444 list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
6445 offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
6446
6447 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
6448 refcount_create(&adddev->l2ad_alloc);
6449
6450 /*
6451 * Add device to global list
6452 */
6453 mutex_enter(&l2arc_dev_mtx);
6454 list_insert_head(l2arc_dev_list, adddev);
6455 atomic_inc_64(&l2arc_ndev);
6456 mutex_exit(&l2arc_dev_mtx);
6457 }
6458
6459 /*
6460 * Remove a vdev from the L2ARC.
6461 */
6462 void
6463 l2arc_remove_vdev(vdev_t *vd)
6464 {
6465 l2arc_dev_t *dev, *nextdev, *remdev = NULL;
6466
6467 /*
6468 * Find the device by vdev
6469 */
6470 mutex_enter(&l2arc_dev_mtx);
6471 for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
6472 nextdev = list_next(l2arc_dev_list, dev);
6473 if (vd == dev->l2ad_vdev) {
6474 remdev = dev;
6475 break;
6476 }
6477 }
6478 ASSERT(remdev != NULL);
6479
6480 /*
6481 * Remove device from global list
6482 */
6483 list_remove(l2arc_dev_list, remdev);
6484 l2arc_dev_last = NULL; /* may have been invalidated */
6485 atomic_dec_64(&l2arc_ndev);
6486 mutex_exit(&l2arc_dev_mtx);
6487
6488 /*
6489 * Clear all buflists and ARC references. L2ARC device flush.
6490 */
6491 l2arc_evict(remdev, 0, B_TRUE);
6492 list_destroy(&remdev->l2ad_buflist);
6493 mutex_destroy(&remdev->l2ad_mtx);
6494 refcount_destroy(&remdev->l2ad_alloc);
6495 kmem_free(remdev, sizeof (l2arc_dev_t));
6496 }
6497
6498 void
6499 l2arc_init(void)
6500 {
6501 l2arc_thread_exit = 0;
6502 l2arc_ndev = 0;
6503 l2arc_writes_sent = 0;
6504 l2arc_writes_done = 0;
6505
6506 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
6507 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
6508 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
6509 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
6510
6511 l2arc_dev_list = &L2ARC_dev_list;
6512 l2arc_free_on_write = &L2ARC_free_on_write;
6513 list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
6514 offsetof(l2arc_dev_t, l2ad_node));
6515 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
6516 offsetof(l2arc_data_free_t, l2df_list_node));
6517 }
6518
6519 void
6520 l2arc_fini(void)
6521 {
6522 /*
6523 * This is called from dmu_fini(), which is called from spa_fini();
6524 * Because of this, we can assume that all l2arc devices have
6525 * already been removed when the pools themselves were removed.
6526 */
6527
6528 l2arc_do_free_on_write();
6529
6530 mutex_destroy(&l2arc_feed_thr_lock);
6531 cv_destroy(&l2arc_feed_thr_cv);
6532 mutex_destroy(&l2arc_dev_mtx);
6533 mutex_destroy(&l2arc_free_on_write_mtx);
6534
6535 list_destroy(l2arc_dev_list);
6536 list_destroy(l2arc_free_on_write);
6537 }
6538
6539 void
6540 l2arc_start(void)
6541 {
6542 if (!(spa_mode_global & FWRITE))
6543 return;
6544
6545 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
6546 TS_RUN, minclsyspri);
6547 }
6548
6549 void
6550 l2arc_stop(void)
6551 {
6552 if (!(spa_mode_global & FWRITE))
6553 return;
6554
6555 mutex_enter(&l2arc_feed_thr_lock);
6556 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */
6557 l2arc_thread_exit = 1;
6558 while (l2arc_thread_exit != 0)
6559 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
6560 mutex_exit(&l2arc_feed_thr_lock);
6561 }