Print this page
5219 l2arc_write_buffers() may write beyond target_sz
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Saso Kiselkov <skiselkov@gmail.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Steven Hartland <steven.hartland@multiplay.co.uk>
Reviewed by: Justin Gibbs <gibbs@FreeBSD.org>
Approved by: Matthew Ahrens <mahrens@delphix.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/arc.c
+++ new/usr/src/uts/common/fs/zfs/arc.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 * Copyright (c) 2012, 2013, Joyent, Inc. All rights reserved.
24 24 * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
25 25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 26 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
27 27 */
28 28
29 29 /*
30 30 * DVA-based Adjustable Replacement Cache
31 31 *
32 32 * While much of the theory of operation used here is
33 33 * based on the self-tuning, low overhead replacement cache
34 34 * presented by Megiddo and Modha at FAST 2003, there are some
35 35 * significant differences:
36 36 *
37 37 * 1. The Megiddo and Modha model assumes any page is evictable.
38 38 * Pages in its cache cannot be "locked" into memory. This makes
39 39 * the eviction algorithm simple: evict the last page in the list.
40 40 * This also make the performance characteristics easy to reason
41 41 * about. Our cache is not so simple. At any given moment, some
42 42 * subset of the blocks in the cache are un-evictable because we
43 43 * have handed out a reference to them. Blocks are only evictable
44 44 * when there are no external references active. This makes
45 45 * eviction far more problematic: we choose to evict the evictable
46 46 * blocks that are the "lowest" in the list.
47 47 *
48 48 * There are times when it is not possible to evict the requested
49 49 * space. In these circumstances we are unable to adjust the cache
50 50 * size. To prevent the cache growing unbounded at these times we
51 51 * implement a "cache throttle" that slows the flow of new data
52 52 * into the cache until we can make space available.
53 53 *
54 54 * 2. The Megiddo and Modha model assumes a fixed cache size.
55 55 * Pages are evicted when the cache is full and there is a cache
56 56 * miss. Our model has a variable sized cache. It grows with
57 57 * high use, but also tries to react to memory pressure from the
58 58 * operating system: decreasing its size when system memory is
59 59 * tight.
60 60 *
61 61 * 3. The Megiddo and Modha model assumes a fixed page size. All
62 62 * elements of the cache are therefore exactly the same size. So
63 63 * when adjusting the cache size following a cache miss, its simply
64 64 * a matter of choosing a single page to evict. In our model, we
65 65 * have variable sized cache blocks (rangeing from 512 bytes to
66 66 * 128K bytes). We therefore choose a set of blocks to evict to make
67 67 * space for a cache miss that approximates as closely as possible
68 68 * the space used by the new block.
69 69 *
70 70 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71 71 * by N. Megiddo & D. Modha, FAST 2003
72 72 */
73 73
74 74 /*
75 75 * The locking model:
76 76 *
77 77 * A new reference to a cache buffer can be obtained in two
78 78 * ways: 1) via a hash table lookup using the DVA as a key,
79 79 * or 2) via one of the ARC lists. The arc_read() interface
80 80 * uses method 1, while the internal arc algorithms for
81 81 * adjusting the cache use method 2. We therefore provide two
82 82 * types of locks: 1) the hash table lock array, and 2) the
83 83 * arc list locks.
84 84 *
85 85 * Buffers do not have their own mutexes, rather they rely on the
86 86 * hash table mutexes for the bulk of their protection (i.e. most
87 87 * fields in the arc_buf_hdr_t are protected by these mutexes).
88 88 *
89 89 * buf_hash_find() returns the appropriate mutex (held) when it
90 90 * locates the requested buffer in the hash table. It returns
91 91 * NULL for the mutex if the buffer was not in the table.
92 92 *
93 93 * buf_hash_remove() expects the appropriate hash mutex to be
94 94 * already held before it is invoked.
95 95 *
96 96 * Each arc state also has a mutex which is used to protect the
97 97 * buffer list associated with the state. When attempting to
98 98 * obtain a hash table lock while holding an arc list lock you
99 99 * must use: mutex_tryenter() to avoid deadlock. Also note that
100 100 * the active state mutex must be held before the ghost state mutex.
101 101 *
102 102 * Arc buffers may have an associated eviction callback function.
103 103 * This function will be invoked prior to removing the buffer (e.g.
104 104 * in arc_do_user_evicts()). Note however that the data associated
105 105 * with the buffer may be evicted prior to the callback. The callback
106 106 * must be made with *no locks held* (to prevent deadlock). Additionally,
107 107 * the users of callbacks must ensure that their private data is
108 108 * protected from simultaneous callbacks from arc_clear_callback()
109 109 * and arc_do_user_evicts().
110 110 *
111 111 * Note that the majority of the performance stats are manipulated
112 112 * with atomic operations.
113 113 *
114 114 * The L2ARC uses the l2ad_mtx on each vdev for the following:
115 115 *
116 116 * - L2ARC buflist creation
117 117 * - L2ARC buflist eviction
118 118 * - L2ARC write completion, which walks L2ARC buflists
119 119 * - ARC header destruction, as it removes from L2ARC buflists
120 120 * - ARC header release, as it removes from L2ARC buflists
121 121 */
122 122
123 123 #include <sys/spa.h>
124 124 #include <sys/zio.h>
125 125 #include <sys/zio_compress.h>
126 126 #include <sys/zfs_context.h>
127 127 #include <sys/arc.h>
128 128 #include <sys/refcount.h>
129 129 #include <sys/vdev.h>
130 130 #include <sys/vdev_impl.h>
131 131 #include <sys/dsl_pool.h>
132 132 #include <sys/zfs_zone.h>
133 133 #include <sys/multilist.h>
134 134 #ifdef _KERNEL
135 135 #include <sys/vmsystm.h>
136 136 #include <vm/anon.h>
137 137 #include <sys/fs/swapnode.h>
138 138 #include <sys/dnlc.h>
139 139 #endif
140 140 #include <sys/callb.h>
141 141 #include <sys/kstat.h>
142 142 #include <zfs_fletcher.h>
143 143
144 144 #ifndef _KERNEL
145 145 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
146 146 boolean_t arc_watch = B_FALSE;
147 147 int arc_procfd;
148 148 #endif
149 149
150 150 static kmutex_t arc_reclaim_lock;
151 151 static kcondvar_t arc_reclaim_thread_cv;
152 152 static boolean_t arc_reclaim_thread_exit;
153 153 static kcondvar_t arc_reclaim_waiters_cv;
154 154
155 155 static kmutex_t arc_user_evicts_lock;
156 156 static kcondvar_t arc_user_evicts_cv;
157 157 static boolean_t arc_user_evicts_thread_exit;
158 158
159 159 uint_t arc_reduce_dnlc_percent = 3;
160 160
161 161 /*
162 162 * The number of headers to evict in arc_evict_state_impl() before
163 163 * dropping the sublist lock and evicting from another sublist. A lower
164 164 * value means we're more likely to evict the "correct" header (i.e. the
165 165 * oldest header in the arc state), but comes with higher overhead
166 166 * (i.e. more invocations of arc_evict_state_impl()).
167 167 */
168 168 int zfs_arc_evict_batch_limit = 10;
169 169
170 170 /*
171 171 * The number of sublists used for each of the arc state lists. If this
172 172 * is not set to a suitable value by the user, it will be configured to
173 173 * the number of CPUs on the system in arc_init().
174 174 */
175 175 int zfs_arc_num_sublists_per_state = 0;
176 176
177 177 /* number of seconds before growing cache again */
178 178 static int arc_grow_retry = 60;
179 179
180 180 /* shift of arc_c for calculating overflow limit in arc_get_data_buf */
181 181 int zfs_arc_overflow_shift = 8;
182 182
183 183 /* shift of arc_c for calculating both min and max arc_p */
184 184 static int arc_p_min_shift = 4;
185 185
186 186 /* log2(fraction of arc to reclaim) */
187 187 static int arc_shrink_shift = 7;
188 188
189 189 /*
190 190 * log2(fraction of ARC which must be free to allow growing).
191 191 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
192 192 * when reading a new block into the ARC, we will evict an equal-sized block
193 193 * from the ARC.
194 194 *
195 195 * This must be less than arc_shrink_shift, so that when we shrink the ARC,
196 196 * we will still not allow it to grow.
197 197 */
198 198 int arc_no_grow_shift = 5;
199 199
200 200
201 201 /*
202 202 * minimum lifespan of a prefetch block in clock ticks
203 203 * (initialized in arc_init())
204 204 */
205 205 static int arc_min_prefetch_lifespan;
206 206
207 207 /*
208 208 * If this percent of memory is free, don't throttle.
209 209 */
210 210 int arc_lotsfree_percent = 10;
211 211
212 212 static int arc_dead;
213 213
214 214 /*
215 215 * The arc has filled available memory and has now warmed up.
216 216 */
217 217 static boolean_t arc_warm;
218 218
219 219 /*
220 220 * These tunables are for performance analysis.
221 221 */
222 222 uint64_t zfs_arc_max;
223 223 uint64_t zfs_arc_min;
224 224 uint64_t zfs_arc_meta_limit = 0;
225 225 uint64_t zfs_arc_meta_min = 0;
226 226 int zfs_arc_grow_retry = 0;
227 227 int zfs_arc_shrink_shift = 0;
228 228 int zfs_arc_p_min_shift = 0;
229 229 int zfs_disable_dup_eviction = 0;
230 230 int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
231 231
232 232 /*
233 233 * Note that buffers can be in one of 6 states:
234 234 * ARC_anon - anonymous (discussed below)
235 235 * ARC_mru - recently used, currently cached
236 236 * ARC_mru_ghost - recentely used, no longer in cache
237 237 * ARC_mfu - frequently used, currently cached
238 238 * ARC_mfu_ghost - frequently used, no longer in cache
239 239 * ARC_l2c_only - exists in L2ARC but not other states
240 240 * When there are no active references to the buffer, they are
241 241 * are linked onto a list in one of these arc states. These are
242 242 * the only buffers that can be evicted or deleted. Within each
243 243 * state there are multiple lists, one for meta-data and one for
244 244 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes,
245 245 * etc.) is tracked separately so that it can be managed more
246 246 * explicitly: favored over data, limited explicitly.
247 247 *
248 248 * Anonymous buffers are buffers that are not associated with
249 249 * a DVA. These are buffers that hold dirty block copies
250 250 * before they are written to stable storage. By definition,
251 251 * they are "ref'd" and are considered part of arc_mru
252 252 * that cannot be freed. Generally, they will aquire a DVA
253 253 * as they are written and migrate onto the arc_mru list.
254 254 *
255 255 * The ARC_l2c_only state is for buffers that are in the second
256 256 * level ARC but no longer in any of the ARC_m* lists. The second
257 257 * level ARC itself may also contain buffers that are in any of
258 258 * the ARC_m* states - meaning that a buffer can exist in two
259 259 * places. The reason for the ARC_l2c_only state is to keep the
260 260 * buffer header in the hash table, so that reads that hit the
261 261 * second level ARC benefit from these fast lookups.
262 262 */
263 263
264 264 typedef struct arc_state {
265 265 /*
266 266 * list of evictable buffers
267 267 */
268 268 multilist_t arcs_list[ARC_BUFC_NUMTYPES];
269 269 /*
270 270 * total amount of evictable data in this state
271 271 */
272 272 uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];
273 273 /*
274 274 * total amount of data in this state; this includes: evictable,
275 275 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
276 276 */
277 277 refcount_t arcs_size;
278 278 } arc_state_t;
279 279
280 280 /* The 6 states: */
281 281 static arc_state_t ARC_anon;
282 282 static arc_state_t ARC_mru;
283 283 static arc_state_t ARC_mru_ghost;
284 284 static arc_state_t ARC_mfu;
285 285 static arc_state_t ARC_mfu_ghost;
286 286 static arc_state_t ARC_l2c_only;
287 287
288 288 typedef struct arc_stats {
289 289 kstat_named_t arcstat_hits;
290 290 kstat_named_t arcstat_misses;
291 291 kstat_named_t arcstat_demand_data_hits;
292 292 kstat_named_t arcstat_demand_data_misses;
293 293 kstat_named_t arcstat_demand_metadata_hits;
294 294 kstat_named_t arcstat_demand_metadata_misses;
295 295 kstat_named_t arcstat_prefetch_data_hits;
296 296 kstat_named_t arcstat_prefetch_data_misses;
297 297 kstat_named_t arcstat_prefetch_metadata_hits;
298 298 kstat_named_t arcstat_prefetch_metadata_misses;
299 299 kstat_named_t arcstat_mru_hits;
300 300 kstat_named_t arcstat_mru_ghost_hits;
301 301 kstat_named_t arcstat_mfu_hits;
302 302 kstat_named_t arcstat_mfu_ghost_hits;
303 303 kstat_named_t arcstat_deleted;
304 304 /*
305 305 * Number of buffers that could not be evicted because the hash lock
306 306 * was held by another thread. The lock may not necessarily be held
307 307 * by something using the same buffer, since hash locks are shared
308 308 * by multiple buffers.
309 309 */
310 310 kstat_named_t arcstat_mutex_miss;
311 311 /*
312 312 * Number of buffers skipped because they have I/O in progress, are
313 313 * indrect prefetch buffers that have not lived long enough, or are
314 314 * not from the spa we're trying to evict from.
315 315 */
316 316 kstat_named_t arcstat_evict_skip;
317 317 /*
318 318 * Number of times arc_evict_state() was unable to evict enough
319 319 * buffers to reach it's target amount.
320 320 */
321 321 kstat_named_t arcstat_evict_not_enough;
322 322 kstat_named_t arcstat_evict_l2_cached;
323 323 kstat_named_t arcstat_evict_l2_eligible;
324 324 kstat_named_t arcstat_evict_l2_ineligible;
325 325 kstat_named_t arcstat_evict_l2_skip;
326 326 kstat_named_t arcstat_hash_elements;
327 327 kstat_named_t arcstat_hash_elements_max;
328 328 kstat_named_t arcstat_hash_collisions;
329 329 kstat_named_t arcstat_hash_chains;
330 330 kstat_named_t arcstat_hash_chain_max;
331 331 kstat_named_t arcstat_p;
332 332 kstat_named_t arcstat_c;
333 333 kstat_named_t arcstat_c_min;
334 334 kstat_named_t arcstat_c_max;
335 335 kstat_named_t arcstat_size;
336 336 /*
337 337 * Number of bytes consumed by internal ARC structures necessary
338 338 * for tracking purposes; these structures are not actually
339 339 * backed by ARC buffers. This includes arc_buf_hdr_t structures
340 340 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
341 341 * caches), and arc_buf_t structures (allocated via arc_buf_t
342 342 * cache).
343 343 */
344 344 kstat_named_t arcstat_hdr_size;
345 345 /*
346 346 * Number of bytes consumed by ARC buffers of type equal to
347 347 * ARC_BUFC_DATA. This is generally consumed by buffers backing
348 348 * on disk user data (e.g. plain file contents).
349 349 */
350 350 kstat_named_t arcstat_data_size;
351 351 /*
352 352 * Number of bytes consumed by ARC buffers of type equal to
353 353 * ARC_BUFC_METADATA. This is generally consumed by buffers
354 354 * backing on disk data that is used for internal ZFS
355 355 * structures (e.g. ZAP, dnode, indirect blocks, etc).
356 356 */
357 357 kstat_named_t arcstat_metadata_size;
358 358 /*
359 359 * Number of bytes consumed by various buffers and structures
360 360 * not actually backed with ARC buffers. This includes bonus
361 361 * buffers (allocated directly via zio_buf_* functions),
362 362 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
363 363 * cache), and dnode_t structures (allocated via dnode_t cache).
364 364 */
365 365 kstat_named_t arcstat_other_size;
366 366 /*
367 367 * Total number of bytes consumed by ARC buffers residing in the
368 368 * arc_anon state. This includes *all* buffers in the arc_anon
369 369 * state; e.g. data, metadata, evictable, and unevictable buffers
370 370 * are all included in this value.
371 371 */
372 372 kstat_named_t arcstat_anon_size;
373 373 /*
374 374 * Number of bytes consumed by ARC buffers that meet the
375 375 * following criteria: backing buffers of type ARC_BUFC_DATA,
376 376 * residing in the arc_anon state, and are eligible for eviction
377 377 * (e.g. have no outstanding holds on the buffer).
378 378 */
379 379 kstat_named_t arcstat_anon_evictable_data;
380 380 /*
381 381 * Number of bytes consumed by ARC buffers that meet the
382 382 * following criteria: backing buffers of type ARC_BUFC_METADATA,
383 383 * residing in the arc_anon state, and are eligible for eviction
384 384 * (e.g. have no outstanding holds on the buffer).
385 385 */
386 386 kstat_named_t arcstat_anon_evictable_metadata;
387 387 /*
388 388 * Total number of bytes consumed by ARC buffers residing in the
389 389 * arc_mru state. This includes *all* buffers in the arc_mru
390 390 * state; e.g. data, metadata, evictable, and unevictable buffers
391 391 * are all included in this value.
392 392 */
393 393 kstat_named_t arcstat_mru_size;
394 394 /*
395 395 * Number of bytes consumed by ARC buffers that meet the
396 396 * following criteria: backing buffers of type ARC_BUFC_DATA,
397 397 * residing in the arc_mru state, and are eligible for eviction
398 398 * (e.g. have no outstanding holds on the buffer).
399 399 */
400 400 kstat_named_t arcstat_mru_evictable_data;
401 401 /*
402 402 * Number of bytes consumed by ARC buffers that meet the
403 403 * following criteria: backing buffers of type ARC_BUFC_METADATA,
404 404 * residing in the arc_mru state, and are eligible for eviction
405 405 * (e.g. have no outstanding holds on the buffer).
406 406 */
407 407 kstat_named_t arcstat_mru_evictable_metadata;
408 408 /*
409 409 * Total number of bytes that *would have been* consumed by ARC
410 410 * buffers in the arc_mru_ghost state. The key thing to note
411 411 * here, is the fact that this size doesn't actually indicate
412 412 * RAM consumption. The ghost lists only consist of headers and
413 413 * don't actually have ARC buffers linked off of these headers.
414 414 * Thus, *if* the headers had associated ARC buffers, these
415 415 * buffers *would have* consumed this number of bytes.
416 416 */
417 417 kstat_named_t arcstat_mru_ghost_size;
418 418 /*
419 419 * Number of bytes that *would have been* consumed by ARC
420 420 * buffers that are eligible for eviction, of type
421 421 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
422 422 */
423 423 kstat_named_t arcstat_mru_ghost_evictable_data;
424 424 /*
425 425 * Number of bytes that *would have been* consumed by ARC
426 426 * buffers that are eligible for eviction, of type
427 427 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
428 428 */
429 429 kstat_named_t arcstat_mru_ghost_evictable_metadata;
430 430 /*
431 431 * Total number of bytes consumed by ARC buffers residing in the
432 432 * arc_mfu state. This includes *all* buffers in the arc_mfu
433 433 * state; e.g. data, metadata, evictable, and unevictable buffers
434 434 * are all included in this value.
435 435 */
436 436 kstat_named_t arcstat_mfu_size;
437 437 /*
438 438 * Number of bytes consumed by ARC buffers that are eligible for
439 439 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
440 440 * state.
441 441 */
442 442 kstat_named_t arcstat_mfu_evictable_data;
443 443 /*
444 444 * Number of bytes consumed by ARC buffers that are eligible for
445 445 * eviction, of type ARC_BUFC_METADATA, and reside in the
446 446 * arc_mfu state.
447 447 */
448 448 kstat_named_t arcstat_mfu_evictable_metadata;
449 449 /*
450 450 * Total number of bytes that *would have been* consumed by ARC
451 451 * buffers in the arc_mfu_ghost state. See the comment above
452 452 * arcstat_mru_ghost_size for more details.
453 453 */
454 454 kstat_named_t arcstat_mfu_ghost_size;
455 455 /*
456 456 * Number of bytes that *would have been* consumed by ARC
457 457 * buffers that are eligible for eviction, of type
458 458 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
459 459 */
460 460 kstat_named_t arcstat_mfu_ghost_evictable_data;
461 461 /*
462 462 * Number of bytes that *would have been* consumed by ARC
463 463 * buffers that are eligible for eviction, of type
464 464 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
465 465 */
466 466 kstat_named_t arcstat_mfu_ghost_evictable_metadata;
467 467 kstat_named_t arcstat_l2_hits;
468 468 kstat_named_t arcstat_l2_misses;
469 469 kstat_named_t arcstat_l2_feeds;
470 470 kstat_named_t arcstat_l2_rw_clash;
471 471 kstat_named_t arcstat_l2_read_bytes;
472 472 kstat_named_t arcstat_l2_write_bytes;
473 473 kstat_named_t arcstat_l2_writes_sent;
474 474 kstat_named_t arcstat_l2_writes_done;
475 475 kstat_named_t arcstat_l2_writes_error;
476 476 kstat_named_t arcstat_l2_writes_lock_retry;
477 477 kstat_named_t arcstat_l2_evict_lock_retry;
478 478 kstat_named_t arcstat_l2_evict_reading;
479 479 kstat_named_t arcstat_l2_evict_l1cached;
480 480 kstat_named_t arcstat_l2_free_on_write;
481 481 kstat_named_t arcstat_l2_cdata_free_on_write;
482 482 kstat_named_t arcstat_l2_abort_lowmem;
483 483 kstat_named_t arcstat_l2_cksum_bad;
484 484 kstat_named_t arcstat_l2_io_error;
485 485 kstat_named_t arcstat_l2_size;
486 486 kstat_named_t arcstat_l2_asize;
487 487 kstat_named_t arcstat_l2_hdr_size;
488 488 kstat_named_t arcstat_l2_compress_successes;
489 489 kstat_named_t arcstat_l2_compress_zeros;
490 490 kstat_named_t arcstat_l2_compress_failures;
491 491 kstat_named_t arcstat_memory_throttle_count;
492 492 kstat_named_t arcstat_duplicate_buffers;
493 493 kstat_named_t arcstat_duplicate_buffers_size;
494 494 kstat_named_t arcstat_duplicate_reads;
495 495 kstat_named_t arcstat_meta_used;
496 496 kstat_named_t arcstat_meta_limit;
497 497 kstat_named_t arcstat_meta_max;
498 498 kstat_named_t arcstat_meta_min;
499 499 } arc_stats_t;
500 500
501 501 static arc_stats_t arc_stats = {
502 502 { "hits", KSTAT_DATA_UINT64 },
503 503 { "misses", KSTAT_DATA_UINT64 },
504 504 { "demand_data_hits", KSTAT_DATA_UINT64 },
505 505 { "demand_data_misses", KSTAT_DATA_UINT64 },
506 506 { "demand_metadata_hits", KSTAT_DATA_UINT64 },
507 507 { "demand_metadata_misses", KSTAT_DATA_UINT64 },
508 508 { "prefetch_data_hits", KSTAT_DATA_UINT64 },
509 509 { "prefetch_data_misses", KSTAT_DATA_UINT64 },
510 510 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 },
511 511 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 },
512 512 { "mru_hits", KSTAT_DATA_UINT64 },
513 513 { "mru_ghost_hits", KSTAT_DATA_UINT64 },
514 514 { "mfu_hits", KSTAT_DATA_UINT64 },
515 515 { "mfu_ghost_hits", KSTAT_DATA_UINT64 },
516 516 { "deleted", KSTAT_DATA_UINT64 },
517 517 { "mutex_miss", KSTAT_DATA_UINT64 },
518 518 { "evict_skip", KSTAT_DATA_UINT64 },
519 519 { "evict_not_enough", KSTAT_DATA_UINT64 },
520 520 { "evict_l2_cached", KSTAT_DATA_UINT64 },
521 521 { "evict_l2_eligible", KSTAT_DATA_UINT64 },
522 522 { "evict_l2_ineligible", KSTAT_DATA_UINT64 },
523 523 { "evict_l2_skip", KSTAT_DATA_UINT64 },
524 524 { "hash_elements", KSTAT_DATA_UINT64 },
525 525 { "hash_elements_max", KSTAT_DATA_UINT64 },
526 526 { "hash_collisions", KSTAT_DATA_UINT64 },
527 527 { "hash_chains", KSTAT_DATA_UINT64 },
528 528 { "hash_chain_max", KSTAT_DATA_UINT64 },
529 529 { "p", KSTAT_DATA_UINT64 },
530 530 { "c", KSTAT_DATA_UINT64 },
531 531 { "c_min", KSTAT_DATA_UINT64 },
532 532 { "c_max", KSTAT_DATA_UINT64 },
533 533 { "size", KSTAT_DATA_UINT64 },
534 534 { "hdr_size", KSTAT_DATA_UINT64 },
535 535 { "data_size", KSTAT_DATA_UINT64 },
536 536 { "metadata_size", KSTAT_DATA_UINT64 },
537 537 { "other_size", KSTAT_DATA_UINT64 },
538 538 { "anon_size", KSTAT_DATA_UINT64 },
539 539 { "anon_evictable_data", KSTAT_DATA_UINT64 },
540 540 { "anon_evictable_metadata", KSTAT_DATA_UINT64 },
541 541 { "mru_size", KSTAT_DATA_UINT64 },
542 542 { "mru_evictable_data", KSTAT_DATA_UINT64 },
543 543 { "mru_evictable_metadata", KSTAT_DATA_UINT64 },
544 544 { "mru_ghost_size", KSTAT_DATA_UINT64 },
545 545 { "mru_ghost_evictable_data", KSTAT_DATA_UINT64 },
546 546 { "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
547 547 { "mfu_size", KSTAT_DATA_UINT64 },
548 548 { "mfu_evictable_data", KSTAT_DATA_UINT64 },
549 549 { "mfu_evictable_metadata", KSTAT_DATA_UINT64 },
550 550 { "mfu_ghost_size", KSTAT_DATA_UINT64 },
551 551 { "mfu_ghost_evictable_data", KSTAT_DATA_UINT64 },
552 552 { "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
553 553 { "l2_hits", KSTAT_DATA_UINT64 },
554 554 { "l2_misses", KSTAT_DATA_UINT64 },
555 555 { "l2_feeds", KSTAT_DATA_UINT64 },
556 556 { "l2_rw_clash", KSTAT_DATA_UINT64 },
557 557 { "l2_read_bytes", KSTAT_DATA_UINT64 },
558 558 { "l2_write_bytes", KSTAT_DATA_UINT64 },
559 559 { "l2_writes_sent", KSTAT_DATA_UINT64 },
560 560 { "l2_writes_done", KSTAT_DATA_UINT64 },
561 561 { "l2_writes_error", KSTAT_DATA_UINT64 },
562 562 { "l2_writes_lock_retry", KSTAT_DATA_UINT64 },
563 563 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 },
564 564 { "l2_evict_reading", KSTAT_DATA_UINT64 },
565 565 { "l2_evict_l1cached", KSTAT_DATA_UINT64 },
566 566 { "l2_free_on_write", KSTAT_DATA_UINT64 },
567 567 { "l2_cdata_free_on_write", KSTAT_DATA_UINT64 },
568 568 { "l2_abort_lowmem", KSTAT_DATA_UINT64 },
569 569 { "l2_cksum_bad", KSTAT_DATA_UINT64 },
570 570 { "l2_io_error", KSTAT_DATA_UINT64 },
571 571 { "l2_size", KSTAT_DATA_UINT64 },
572 572 { "l2_asize", KSTAT_DATA_UINT64 },
573 573 { "l2_hdr_size", KSTAT_DATA_UINT64 },
574 574 { "l2_compress_successes", KSTAT_DATA_UINT64 },
575 575 { "l2_compress_zeros", KSTAT_DATA_UINT64 },
576 576 { "l2_compress_failures", KSTAT_DATA_UINT64 },
577 577 { "memory_throttle_count", KSTAT_DATA_UINT64 },
578 578 { "duplicate_buffers", KSTAT_DATA_UINT64 },
579 579 { "duplicate_buffers_size", KSTAT_DATA_UINT64 },
580 580 { "duplicate_reads", KSTAT_DATA_UINT64 },
581 581 { "arc_meta_used", KSTAT_DATA_UINT64 },
582 582 { "arc_meta_limit", KSTAT_DATA_UINT64 },
583 583 { "arc_meta_max", KSTAT_DATA_UINT64 },
584 584 { "arc_meta_min", KSTAT_DATA_UINT64 }
585 585 };
586 586
587 587 #define ARCSTAT(stat) (arc_stats.stat.value.ui64)
588 588
589 589 #define ARCSTAT_INCR(stat, val) \
590 590 atomic_add_64(&arc_stats.stat.value.ui64, (val))
591 591
592 592 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1)
593 593 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1)
594 594
595 595 #define ARCSTAT_MAX(stat, val) { \
596 596 uint64_t m; \
597 597 while ((val) > (m = arc_stats.stat.value.ui64) && \
598 598 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \
599 599 continue; \
600 600 }
601 601
602 602 #define ARCSTAT_MAXSTAT(stat) \
603 603 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
604 604
605 605 /*
606 606 * We define a macro to allow ARC hits/misses to be easily broken down by
607 607 * two separate conditions, giving a total of four different subtypes for
608 608 * each of hits and misses (so eight statistics total).
609 609 */
610 610 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
611 611 if (cond1) { \
612 612 if (cond2) { \
613 613 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
614 614 } else { \
615 615 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
616 616 } \
617 617 } else { \
618 618 if (cond2) { \
619 619 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
620 620 } else { \
621 621 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
622 622 } \
623 623 }
624 624
625 625 kstat_t *arc_ksp;
626 626 static arc_state_t *arc_anon;
627 627 static arc_state_t *arc_mru;
628 628 static arc_state_t *arc_mru_ghost;
629 629 static arc_state_t *arc_mfu;
630 630 static arc_state_t *arc_mfu_ghost;
631 631 static arc_state_t *arc_l2c_only;
632 632
633 633 /*
634 634 * There are several ARC variables that are critical to export as kstats --
635 635 * but we don't want to have to grovel around in the kstat whenever we wish to
636 636 * manipulate them. For these variables, we therefore define them to be in
637 637 * terms of the statistic variable. This assures that we are not introducing
638 638 * the possibility of inconsistency by having shadow copies of the variables,
639 639 * while still allowing the code to be readable.
640 640 */
641 641 #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */
642 642 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */
643 643 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */
644 644 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */
645 645 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */
646 646 #define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */
647 647 #define arc_meta_min ARCSTAT(arcstat_meta_min) /* min size for metadata */
648 648 #define arc_meta_used ARCSTAT(arcstat_meta_used) /* size of metadata */
649 649 #define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */
650 650
651 651 #define L2ARC_IS_VALID_COMPRESS(_c_) \
652 652 ((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
653 653
654 654 static int arc_no_grow; /* Don't try to grow cache size */
655 655 static uint64_t arc_tempreserve;
656 656 static uint64_t arc_loaned_bytes;
657 657
658 658 typedef struct arc_callback arc_callback_t;
659 659
660 660 struct arc_callback {
661 661 void *acb_private;
662 662 arc_done_func_t *acb_done;
663 663 arc_buf_t *acb_buf;
664 664 zio_t *acb_zio_dummy;
665 665 arc_callback_t *acb_next;
666 666 };
667 667
668 668 typedef struct arc_write_callback arc_write_callback_t;
669 669
670 670 struct arc_write_callback {
671 671 void *awcb_private;
672 672 arc_done_func_t *awcb_ready;
673 673 arc_done_func_t *awcb_physdone;
674 674 arc_done_func_t *awcb_done;
675 675 arc_buf_t *awcb_buf;
676 676 };
677 677
678 678 /*
679 679 * ARC buffers are separated into multiple structs as a memory saving measure:
680 680 * - Common fields struct, always defined, and embedded within it:
681 681 * - L2-only fields, always allocated but undefined when not in L2ARC
682 682 * - L1-only fields, only allocated when in L1ARC
683 683 *
684 684 * Buffer in L1 Buffer only in L2
685 685 * +------------------------+ +------------------------+
686 686 * | arc_buf_hdr_t | | arc_buf_hdr_t |
687 687 * | | | |
688 688 * | | | |
689 689 * | | | |
690 690 * +------------------------+ +------------------------+
691 691 * | l2arc_buf_hdr_t | | l2arc_buf_hdr_t |
692 692 * | (undefined if L1-only) | | |
693 693 * +------------------------+ +------------------------+
694 694 * | l1arc_buf_hdr_t |
695 695 * | |
696 696 * | |
697 697 * | |
698 698 * | |
699 699 * +------------------------+
700 700 *
701 701 * Because it's possible for the L2ARC to become extremely large, we can wind
702 702 * up eating a lot of memory in L2ARC buffer headers, so the size of a header
703 703 * is minimized by only allocating the fields necessary for an L1-cached buffer
704 704 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
705 705 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
706 706 * words in pointers. arc_hdr_realloc() is used to switch a header between
707 707 * these two allocation states.
708 708 */
709 709 typedef struct l1arc_buf_hdr {
710 710 kmutex_t b_freeze_lock;
711 711 #ifdef ZFS_DEBUG
712 712 /*
713 713 * used for debugging wtih kmem_flags - by allocating and freeing
714 714 * b_thawed when the buffer is thawed, we get a record of the stack
715 715 * trace that thawed it.
716 716 */
717 717 void *b_thawed;
718 718 #endif
719 719
720 720 arc_buf_t *b_buf;
721 721 uint32_t b_datacnt;
722 722 /* for waiting on writes to complete */
723 723 kcondvar_t b_cv;
724 724
725 725 /* protected by arc state mutex */
726 726 arc_state_t *b_state;
727 727 multilist_node_t b_arc_node;
728 728
729 729 /* updated atomically */
730 730 clock_t b_arc_access;
731 731
732 732 /* self protecting */
733 733 refcount_t b_refcnt;
734 734
735 735 arc_callback_t *b_acb;
736 736 /* temporary buffer holder for in-flight compressed data */
737 737 void *b_tmp_cdata;
738 738 } l1arc_buf_hdr_t;
739 739
740 740 typedef struct l2arc_dev l2arc_dev_t;
741 741
742 742 typedef struct l2arc_buf_hdr {
743 743 /* protected by arc_buf_hdr mutex */
744 744 l2arc_dev_t *b_dev; /* L2ARC device */
745 745 uint64_t b_daddr; /* disk address, offset byte */
746 746 /* real alloc'd buffer size depending on b_compress applied */
747 747 int32_t b_asize;
748 748 uint8_t b_compress;
749 749
750 750 list_node_t b_l2node;
751 751 } l2arc_buf_hdr_t;
752 752
753 753 struct arc_buf_hdr {
754 754 /* protected by hash lock */
755 755 dva_t b_dva;
756 756 uint64_t b_birth;
757 757 /*
758 758 * Even though this checksum is only set/verified when a buffer is in
759 759 * the L1 cache, it needs to be in the set of common fields because it
760 760 * must be preserved from the time before a buffer is written out to
761 761 * L2ARC until after it is read back in.
762 762 */
763 763 zio_cksum_t *b_freeze_cksum;
764 764
765 765 arc_buf_hdr_t *b_hash_next;
766 766 arc_flags_t b_flags;
767 767
768 768 /* immutable */
769 769 int32_t b_size;
770 770 uint64_t b_spa;
771 771
772 772 /* L2ARC fields. Undefined when not in L2ARC. */
773 773 l2arc_buf_hdr_t b_l2hdr;
774 774 /* L1ARC fields. Undefined when in l2arc_only state */
775 775 l1arc_buf_hdr_t b_l1hdr;
776 776 };
777 777
778 778 static arc_buf_t *arc_eviction_list;
779 779 static arc_buf_hdr_t arc_eviction_hdr;
780 780
781 781 #define GHOST_STATE(state) \
782 782 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
783 783 (state) == arc_l2c_only)
784 784
785 785 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
786 786 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
787 787 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
788 788 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
789 789 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FLAG_FREED_IN_READ)
790 790 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE)
791 791
792 792 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
793 793 #define HDR_L2COMPRESS(hdr) ((hdr)->b_flags & ARC_FLAG_L2COMPRESS)
794 794 #define HDR_L2_READING(hdr) \
795 795 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
796 796 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
797 797 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
798 798 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
799 799 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
800 800
801 801 #define HDR_ISTYPE_METADATA(hdr) \
802 802 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
803 803 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr))
804 804
805 805 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
806 806 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
807 807
808 808 /*
809 809 * Other sizes
810 810 */
811 811
812 812 #define HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
813 813 #define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
814 814
815 815 /*
816 816 * Hash table routines
817 817 */
818 818
819 819 #define HT_LOCK_PAD 64
820 820
821 821 struct ht_lock {
822 822 kmutex_t ht_lock;
823 823 #ifdef _KERNEL
824 824 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
825 825 #endif
826 826 };
827 827
828 828 #define BUF_LOCKS 256
829 829 typedef struct buf_hash_table {
830 830 uint64_t ht_mask;
831 831 arc_buf_hdr_t **ht_table;
832 832 struct ht_lock ht_locks[BUF_LOCKS];
833 833 } buf_hash_table_t;
834 834
835 835 static buf_hash_table_t buf_hash_table;
836 836
837 837 #define BUF_HASH_INDEX(spa, dva, birth) \
838 838 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
839 839 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
840 840 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
841 841 #define HDR_LOCK(hdr) \
842 842 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
843 843
844 844 uint64_t zfs_crc64_table[256];
845 845
846 846 /*
847 847 * Level 2 ARC
848 848 */
849 849
850 850 #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */
851 851 #define L2ARC_HEADROOM 2 /* num of writes */
852 852 /*
853 853 * If we discover during ARC scan any buffers to be compressed, we boost
854 854 * our headroom for the next scanning cycle by this percentage multiple.
855 855 */
856 856 #define L2ARC_HEADROOM_BOOST 200
857 857 #define L2ARC_FEED_SECS 1 /* caching interval secs */
858 858 #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */
859 859
860 860 /*
861 861 * Used to distinguish headers that are being process by
862 862 * l2arc_write_buffers(), but have yet to be assigned to a l2arc disk
863 863 * address. This can happen when the header is added to the l2arc's list
864 864 * of buffers to write in the first stage of l2arc_write_buffers(), but
865 865 * has not yet been written out which happens in the second stage of
866 866 * l2arc_write_buffers().
867 867 */
868 868 #define L2ARC_ADDR_UNSET ((uint64_t)(-1))
869 869
870 870 #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent)
871 871 #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done)
872 872
873 873 /* L2ARC Performance Tunables */
874 874 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */
875 875 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */
876 876 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */
877 877 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
878 878 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */
879 879 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */
880 880 boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */
881 881 boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */
882 882 boolean_t l2arc_norw = B_TRUE; /* no reads during writes */
883 883
884 884 /*
885 885 * L2ARC Internals
886 886 */
887 887 struct l2arc_dev {
888 888 vdev_t *l2ad_vdev; /* vdev */
889 889 spa_t *l2ad_spa; /* spa */
890 890 uint64_t l2ad_hand; /* next write location */
891 891 uint64_t l2ad_start; /* first addr on device */
892 892 uint64_t l2ad_end; /* last addr on device */
893 893 boolean_t l2ad_first; /* first sweep through */
894 894 boolean_t l2ad_writing; /* currently writing */
895 895 kmutex_t l2ad_mtx; /* lock for buffer list */
896 896 list_t l2ad_buflist; /* buffer list */
897 897 list_node_t l2ad_node; /* device list node */
898 898 refcount_t l2ad_alloc; /* allocated bytes */
899 899 };
900 900
901 901 static list_t L2ARC_dev_list; /* device list */
902 902 static list_t *l2arc_dev_list; /* device list pointer */
903 903 static kmutex_t l2arc_dev_mtx; /* device list mutex */
904 904 static l2arc_dev_t *l2arc_dev_last; /* last device used */
905 905 static list_t L2ARC_free_on_write; /* free after write buf list */
906 906 static list_t *l2arc_free_on_write; /* free after write list ptr */
907 907 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */
908 908 static uint64_t l2arc_ndev; /* number of devices */
909 909
910 910 typedef struct l2arc_read_callback {
911 911 arc_buf_t *l2rcb_buf; /* read buffer */
912 912 spa_t *l2rcb_spa; /* spa */
913 913 blkptr_t l2rcb_bp; /* original blkptr */
914 914 zbookmark_phys_t l2rcb_zb; /* original bookmark */
915 915 int l2rcb_flags; /* original flags */
916 916 enum zio_compress l2rcb_compress; /* applied compress */
917 917 } l2arc_read_callback_t;
918 918
919 919 typedef struct l2arc_write_callback {
920 920 l2arc_dev_t *l2wcb_dev; /* device info */
921 921 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */
922 922 } l2arc_write_callback_t;
923 923
924 924 typedef struct l2arc_data_free {
925 925 /* protected by l2arc_free_on_write_mtx */
926 926 void *l2df_data;
927 927 size_t l2df_size;
928 928 void (*l2df_func)(void *, size_t);
929 929 list_node_t l2df_list_node;
930 930 } l2arc_data_free_t;
931 931
932 932 static kmutex_t l2arc_feed_thr_lock;
933 933 static kcondvar_t l2arc_feed_thr_cv;
934 934 static uint8_t l2arc_thread_exit;
935 935
936 936 static void arc_get_data_buf(arc_buf_t *);
937 937 static void arc_access(arc_buf_hdr_t *, kmutex_t *);
938 938 static boolean_t arc_is_overflowing();
939 939 static void arc_buf_watch(arc_buf_t *);
940 940
941 941 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
942 942 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
943 943
944 944 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
945 945 static void l2arc_read_done(zio_t *);
946 946
947 947 static boolean_t l2arc_compress_buf(arc_buf_hdr_t *);
948 948 static void l2arc_decompress_zio(zio_t *, arc_buf_hdr_t *, enum zio_compress);
949 949 static void l2arc_release_cdata_buf(arc_buf_hdr_t *);
950 950
951 951 static uint64_t
952 952 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
953 953 {
954 954 uint8_t *vdva = (uint8_t *)dva;
955 955 uint64_t crc = -1ULL;
956 956 int i;
957 957
958 958 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
959 959
960 960 for (i = 0; i < sizeof (dva_t); i++)
961 961 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
962 962
963 963 crc ^= (spa>>8) ^ birth;
964 964
965 965 return (crc);
966 966 }
967 967
968 968 #define BUF_EMPTY(buf) \
969 969 ((buf)->b_dva.dva_word[0] == 0 && \
970 970 (buf)->b_dva.dva_word[1] == 0)
971 971
972 972 #define BUF_EQUAL(spa, dva, birth, buf) \
973 973 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
974 974 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
975 975 ((buf)->b_birth == birth) && ((buf)->b_spa == spa)
976 976
977 977 static void
978 978 buf_discard_identity(arc_buf_hdr_t *hdr)
979 979 {
980 980 hdr->b_dva.dva_word[0] = 0;
981 981 hdr->b_dva.dva_word[1] = 0;
982 982 hdr->b_birth = 0;
983 983 }
984 984
985 985 static arc_buf_hdr_t *
986 986 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
987 987 {
988 988 const dva_t *dva = BP_IDENTITY(bp);
989 989 uint64_t birth = BP_PHYSICAL_BIRTH(bp);
990 990 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
991 991 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
992 992 arc_buf_hdr_t *hdr;
993 993
994 994 mutex_enter(hash_lock);
995 995 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
996 996 hdr = hdr->b_hash_next) {
997 997 if (BUF_EQUAL(spa, dva, birth, hdr)) {
998 998 *lockp = hash_lock;
999 999 return (hdr);
1000 1000 }
1001 1001 }
1002 1002 mutex_exit(hash_lock);
1003 1003 *lockp = NULL;
1004 1004 return (NULL);
1005 1005 }
1006 1006
1007 1007 /*
1008 1008 * Insert an entry into the hash table. If there is already an element
1009 1009 * equal to elem in the hash table, then the already existing element
1010 1010 * will be returned and the new element will not be inserted.
1011 1011 * Otherwise returns NULL.
1012 1012 * If lockp == NULL, the caller is assumed to already hold the hash lock.
1013 1013 */
1014 1014 static arc_buf_hdr_t *
1015 1015 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1016 1016 {
1017 1017 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1018 1018 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1019 1019 arc_buf_hdr_t *fhdr;
1020 1020 uint32_t i;
1021 1021
1022 1022 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1023 1023 ASSERT(hdr->b_birth != 0);
1024 1024 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1025 1025
1026 1026 if (lockp != NULL) {
1027 1027 *lockp = hash_lock;
1028 1028 mutex_enter(hash_lock);
1029 1029 } else {
1030 1030 ASSERT(MUTEX_HELD(hash_lock));
1031 1031 }
1032 1032
1033 1033 for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1034 1034 fhdr = fhdr->b_hash_next, i++) {
1035 1035 if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1036 1036 return (fhdr);
1037 1037 }
1038 1038
1039 1039 hdr->b_hash_next = buf_hash_table.ht_table[idx];
1040 1040 buf_hash_table.ht_table[idx] = hdr;
1041 1041 hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
1042 1042
1043 1043 /* collect some hash table performance data */
1044 1044 if (i > 0) {
1045 1045 ARCSTAT_BUMP(arcstat_hash_collisions);
1046 1046 if (i == 1)
1047 1047 ARCSTAT_BUMP(arcstat_hash_chains);
1048 1048
1049 1049 ARCSTAT_MAX(arcstat_hash_chain_max, i);
1050 1050 }
1051 1051
1052 1052 ARCSTAT_BUMP(arcstat_hash_elements);
1053 1053 ARCSTAT_MAXSTAT(arcstat_hash_elements);
1054 1054
1055 1055 return (NULL);
1056 1056 }
1057 1057
1058 1058 static void
1059 1059 buf_hash_remove(arc_buf_hdr_t *hdr)
1060 1060 {
1061 1061 arc_buf_hdr_t *fhdr, **hdrp;
1062 1062 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1063 1063
1064 1064 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1065 1065 ASSERT(HDR_IN_HASH_TABLE(hdr));
1066 1066
1067 1067 hdrp = &buf_hash_table.ht_table[idx];
1068 1068 while ((fhdr = *hdrp) != hdr) {
1069 1069 ASSERT(fhdr != NULL);
1070 1070 hdrp = &fhdr->b_hash_next;
1071 1071 }
1072 1072 *hdrp = hdr->b_hash_next;
1073 1073 hdr->b_hash_next = NULL;
1074 1074 hdr->b_flags &= ~ARC_FLAG_IN_HASH_TABLE;
1075 1075
1076 1076 /* collect some hash table performance data */
1077 1077 ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1078 1078
1079 1079 if (buf_hash_table.ht_table[idx] &&
1080 1080 buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1081 1081 ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1082 1082 }
1083 1083
1084 1084 /*
1085 1085 * Global data structures and functions for the buf kmem cache.
1086 1086 */
1087 1087 static kmem_cache_t *hdr_full_cache;
1088 1088 static kmem_cache_t *hdr_l2only_cache;
1089 1089 static kmem_cache_t *buf_cache;
1090 1090
1091 1091 static void
1092 1092 buf_fini(void)
1093 1093 {
1094 1094 int i;
1095 1095
1096 1096 kmem_free(buf_hash_table.ht_table,
1097 1097 (buf_hash_table.ht_mask + 1) * sizeof (void *));
1098 1098 for (i = 0; i < BUF_LOCKS; i++)
1099 1099 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1100 1100 kmem_cache_destroy(hdr_full_cache);
1101 1101 kmem_cache_destroy(hdr_l2only_cache);
1102 1102 kmem_cache_destroy(buf_cache);
1103 1103 }
1104 1104
1105 1105 /*
1106 1106 * Constructor callback - called when the cache is empty
1107 1107 * and a new buf is requested.
1108 1108 */
1109 1109 /* ARGSUSED */
1110 1110 static int
1111 1111 hdr_full_cons(void *vbuf, void *unused, int kmflag)
1112 1112 {
1113 1113 arc_buf_hdr_t *hdr = vbuf;
1114 1114
1115 1115 bzero(hdr, HDR_FULL_SIZE);
1116 1116 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1117 1117 refcount_create(&hdr->b_l1hdr.b_refcnt);
1118 1118 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1119 1119 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1120 1120 arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1121 1121
1122 1122 return (0);
1123 1123 }
1124 1124
1125 1125 /* ARGSUSED */
1126 1126 static int
1127 1127 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1128 1128 {
1129 1129 arc_buf_hdr_t *hdr = vbuf;
1130 1130
1131 1131 bzero(hdr, HDR_L2ONLY_SIZE);
1132 1132 arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1133 1133
1134 1134 return (0);
1135 1135 }
1136 1136
1137 1137 /* ARGSUSED */
1138 1138 static int
1139 1139 buf_cons(void *vbuf, void *unused, int kmflag)
1140 1140 {
1141 1141 arc_buf_t *buf = vbuf;
1142 1142
1143 1143 bzero(buf, sizeof (arc_buf_t));
1144 1144 mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1145 1145 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1146 1146
1147 1147 return (0);
1148 1148 }
1149 1149
1150 1150 /*
1151 1151 * Destructor callback - called when a cached buf is
1152 1152 * no longer required.
1153 1153 */
1154 1154 /* ARGSUSED */
1155 1155 static void
1156 1156 hdr_full_dest(void *vbuf, void *unused)
1157 1157 {
1158 1158 arc_buf_hdr_t *hdr = vbuf;
1159 1159
1160 1160 ASSERT(BUF_EMPTY(hdr));
1161 1161 cv_destroy(&hdr->b_l1hdr.b_cv);
1162 1162 refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1163 1163 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1164 1164 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1165 1165 arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1166 1166 }
1167 1167
1168 1168 /* ARGSUSED */
1169 1169 static void
1170 1170 hdr_l2only_dest(void *vbuf, void *unused)
1171 1171 {
1172 1172 arc_buf_hdr_t *hdr = vbuf;
1173 1173
1174 1174 ASSERT(BUF_EMPTY(hdr));
1175 1175 arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1176 1176 }
1177 1177
1178 1178 /* ARGSUSED */
1179 1179 static void
1180 1180 buf_dest(void *vbuf, void *unused)
1181 1181 {
1182 1182 arc_buf_t *buf = vbuf;
1183 1183
1184 1184 mutex_destroy(&buf->b_evict_lock);
1185 1185 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1186 1186 }
1187 1187
1188 1188 /*
1189 1189 * Reclaim callback -- invoked when memory is low.
1190 1190 */
1191 1191 /* ARGSUSED */
1192 1192 static void
1193 1193 hdr_recl(void *unused)
1194 1194 {
1195 1195 dprintf("hdr_recl called\n");
1196 1196 /*
1197 1197 * umem calls the reclaim func when we destroy the buf cache,
1198 1198 * which is after we do arc_fini().
1199 1199 */
1200 1200 if (!arc_dead)
1201 1201 cv_signal(&arc_reclaim_thread_cv);
1202 1202 }
1203 1203
1204 1204 static void
1205 1205 buf_init(void)
1206 1206 {
1207 1207 uint64_t *ct;
1208 1208 uint64_t hsize = 1ULL << 12;
1209 1209 int i, j;
1210 1210
1211 1211 /*
1212 1212 * The hash table is big enough to fill all of physical memory
1213 1213 * with an average block size of zfs_arc_average_blocksize (default 8K).
1214 1214 * By default, the table will take up
1215 1215 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1216 1216 */
1217 1217 while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE)
1218 1218 hsize <<= 1;
1219 1219 retry:
1220 1220 buf_hash_table.ht_mask = hsize - 1;
1221 1221 buf_hash_table.ht_table =
1222 1222 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1223 1223 if (buf_hash_table.ht_table == NULL) {
1224 1224 ASSERT(hsize > (1ULL << 8));
1225 1225 hsize >>= 1;
1226 1226 goto retry;
1227 1227 }
1228 1228
1229 1229 hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1230 1230 0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1231 1231 hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1232 1232 HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1233 1233 NULL, NULL, 0);
1234 1234 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1235 1235 0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1236 1236
1237 1237 for (i = 0; i < 256; i++)
1238 1238 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1239 1239 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1240 1240
1241 1241 for (i = 0; i < BUF_LOCKS; i++) {
1242 1242 mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1243 1243 NULL, MUTEX_DEFAULT, NULL);
1244 1244 }
1245 1245 }
1246 1246
1247 1247 /*
1248 1248 * Transition between the two allocation states for the arc_buf_hdr struct.
1249 1249 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
1250 1250 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
1251 1251 * version is used when a cache buffer is only in the L2ARC in order to reduce
1252 1252 * memory usage.
1253 1253 */
1254 1254 static arc_buf_hdr_t *
1255 1255 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
1256 1256 {
1257 1257 ASSERT(HDR_HAS_L2HDR(hdr));
1258 1258
1259 1259 arc_buf_hdr_t *nhdr;
1260 1260 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1261 1261
1262 1262 ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
1263 1263 (old == hdr_l2only_cache && new == hdr_full_cache));
1264 1264
1265 1265 nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
1266 1266
1267 1267 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
1268 1268 buf_hash_remove(hdr);
1269 1269
1270 1270 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
1271 1271
1272 1272 if (new == hdr_full_cache) {
1273 1273 nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1274 1274 /*
1275 1275 * arc_access and arc_change_state need to be aware that a
1276 1276 * header has just come out of L2ARC, so we set its state to
1277 1277 * l2c_only even though it's about to change.
1278 1278 */
1279 1279 nhdr->b_l1hdr.b_state = arc_l2c_only;
1280 1280
1281 1281 /* Verify previous threads set to NULL before freeing */
1282 1282 ASSERT3P(nhdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1283 1283 } else {
1284 1284 ASSERT(hdr->b_l1hdr.b_buf == NULL);
1285 1285 ASSERT0(hdr->b_l1hdr.b_datacnt);
1286 1286
1287 1287 /*
1288 1288 * If we've reached here, We must have been called from
1289 1289 * arc_evict_hdr(), as such we should have already been
1290 1290 * removed from any ghost list we were previously on
1291 1291 * (which protects us from racing with arc_evict_state),
1292 1292 * thus no locking is needed during this check.
1293 1293 */
1294 1294 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1295 1295
1296 1296 /*
1297 1297 * A buffer must not be moved into the arc_l2c_only
1298 1298 * state if it's not finished being written out to the
1299 1299 * l2arc device. Otherwise, the b_l1hdr.b_tmp_cdata field
1300 1300 * might try to be accessed, even though it was removed.
1301 1301 */
1302 1302 VERIFY(!HDR_L2_WRITING(hdr));
1303 1303 VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1304 1304
1305 1305 nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
1306 1306 }
1307 1307 /*
1308 1308 * The header has been reallocated so we need to re-insert it into any
1309 1309 * lists it was on.
1310 1310 */
1311 1311 (void) buf_hash_insert(nhdr, NULL);
1312 1312
1313 1313 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
1314 1314
1315 1315 mutex_enter(&dev->l2ad_mtx);
1316 1316
1317 1317 /*
1318 1318 * We must place the realloc'ed header back into the list at
1319 1319 * the same spot. Otherwise, if it's placed earlier in the list,
1320 1320 * l2arc_write_buffers() could find it during the function's
1321 1321 * write phase, and try to write it out to the l2arc.
1322 1322 */
1323 1323 list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
1324 1324 list_remove(&dev->l2ad_buflist, hdr);
1325 1325
1326 1326 mutex_exit(&dev->l2ad_mtx);
1327 1327
1328 1328 /*
1329 1329 * Since we're using the pointer address as the tag when
1330 1330 * incrementing and decrementing the l2ad_alloc refcount, we
1331 1331 * must remove the old pointer (that we're about to destroy) and
1332 1332 * add the new pointer to the refcount. Otherwise we'd remove
1333 1333 * the wrong pointer address when calling arc_hdr_destroy() later.
1334 1334 */
1335 1335
1336 1336 (void) refcount_remove_many(&dev->l2ad_alloc,
1337 1337 hdr->b_l2hdr.b_asize, hdr);
1338 1338
1339 1339 (void) refcount_add_many(&dev->l2ad_alloc,
1340 1340 nhdr->b_l2hdr.b_asize, nhdr);
1341 1341
1342 1342 buf_discard_identity(hdr);
1343 1343 hdr->b_freeze_cksum = NULL;
1344 1344 kmem_cache_free(old, hdr);
1345 1345
1346 1346 return (nhdr);
1347 1347 }
1348 1348
1349 1349
1350 1350 #define ARC_MINTIME (hz>>4) /* 62 ms */
1351 1351
1352 1352 static void
1353 1353 arc_cksum_verify(arc_buf_t *buf)
1354 1354 {
1355 1355 zio_cksum_t zc;
1356 1356
1357 1357 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1358 1358 return;
1359 1359
1360 1360 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1361 1361 if (buf->b_hdr->b_freeze_cksum == NULL || HDR_IO_ERROR(buf->b_hdr)) {
1362 1362 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1363 1363 return;
1364 1364 }
1365 1365 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1366 1366 if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
1367 1367 panic("buffer modified while frozen!");
1368 1368 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1369 1369 }
1370 1370
1371 1371 static int
1372 1372 arc_cksum_equal(arc_buf_t *buf)
1373 1373 {
1374 1374 zio_cksum_t zc;
1375 1375 int equal;
1376 1376
1377 1377 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1378 1378 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1379 1379 equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
1380 1380 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1381 1381
1382 1382 return (equal);
1383 1383 }
1384 1384
1385 1385 static void
1386 1386 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
1387 1387 {
1388 1388 if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
1389 1389 return;
1390 1390
1391 1391 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1392 1392 if (buf->b_hdr->b_freeze_cksum != NULL) {
1393 1393 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1394 1394 return;
1395 1395 }
1396 1396 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1397 1397 fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1398 1398 buf->b_hdr->b_freeze_cksum);
1399 1399 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1400 1400 arc_buf_watch(buf);
1401 1401 }
1402 1402
1403 1403 #ifndef _KERNEL
1404 1404 typedef struct procctl {
1405 1405 long cmd;
1406 1406 prwatch_t prwatch;
1407 1407 } procctl_t;
1408 1408 #endif
1409 1409
1410 1410 /* ARGSUSED */
1411 1411 static void
1412 1412 arc_buf_unwatch(arc_buf_t *buf)
1413 1413 {
1414 1414 #ifndef _KERNEL
1415 1415 if (arc_watch) {
1416 1416 int result;
1417 1417 procctl_t ctl;
1418 1418 ctl.cmd = PCWATCH;
1419 1419 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1420 1420 ctl.prwatch.pr_size = 0;
1421 1421 ctl.prwatch.pr_wflags = 0;
1422 1422 result = write(arc_procfd, &ctl, sizeof (ctl));
1423 1423 ASSERT3U(result, ==, sizeof (ctl));
1424 1424 }
1425 1425 #endif
1426 1426 }
1427 1427
1428 1428 /* ARGSUSED */
1429 1429 static void
1430 1430 arc_buf_watch(arc_buf_t *buf)
1431 1431 {
1432 1432 #ifndef _KERNEL
1433 1433 if (arc_watch) {
1434 1434 int result;
1435 1435 procctl_t ctl;
1436 1436 ctl.cmd = PCWATCH;
1437 1437 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1438 1438 ctl.prwatch.pr_size = buf->b_hdr->b_size;
1439 1439 ctl.prwatch.pr_wflags = WA_WRITE;
1440 1440 result = write(arc_procfd, &ctl, sizeof (ctl));
1441 1441 ASSERT3U(result, ==, sizeof (ctl));
1442 1442 }
1443 1443 #endif
1444 1444 }
1445 1445
1446 1446 static arc_buf_contents_t
1447 1447 arc_buf_type(arc_buf_hdr_t *hdr)
1448 1448 {
1449 1449 if (HDR_ISTYPE_METADATA(hdr)) {
1450 1450 return (ARC_BUFC_METADATA);
1451 1451 } else {
1452 1452 return (ARC_BUFC_DATA);
1453 1453 }
1454 1454 }
1455 1455
1456 1456 static uint32_t
1457 1457 arc_bufc_to_flags(arc_buf_contents_t type)
1458 1458 {
1459 1459 switch (type) {
1460 1460 case ARC_BUFC_DATA:
1461 1461 /* metadata field is 0 if buffer contains normal data */
1462 1462 return (0);
1463 1463 case ARC_BUFC_METADATA:
1464 1464 return (ARC_FLAG_BUFC_METADATA);
1465 1465 default:
1466 1466 break;
1467 1467 }
1468 1468 panic("undefined ARC buffer type!");
1469 1469 return ((uint32_t)-1);
1470 1470 }
1471 1471
1472 1472 void
1473 1473 arc_buf_thaw(arc_buf_t *buf)
1474 1474 {
1475 1475 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1476 1476 if (buf->b_hdr->b_l1hdr.b_state != arc_anon)
1477 1477 panic("modifying non-anon buffer!");
1478 1478 if (HDR_IO_IN_PROGRESS(buf->b_hdr))
1479 1479 panic("modifying buffer while i/o in progress!");
1480 1480 arc_cksum_verify(buf);
1481 1481 }
1482 1482
1483 1483 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1484 1484 if (buf->b_hdr->b_freeze_cksum != NULL) {
1485 1485 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1486 1486 buf->b_hdr->b_freeze_cksum = NULL;
1487 1487 }
1488 1488
1489 1489 #ifdef ZFS_DEBUG
1490 1490 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1491 1491 if (buf->b_hdr->b_l1hdr.b_thawed != NULL)
1492 1492 kmem_free(buf->b_hdr->b_l1hdr.b_thawed, 1);
1493 1493 buf->b_hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
1494 1494 }
1495 1495 #endif
1496 1496
1497 1497 mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1498 1498
1499 1499 arc_buf_unwatch(buf);
1500 1500 }
1501 1501
1502 1502 void
1503 1503 arc_buf_freeze(arc_buf_t *buf)
1504 1504 {
1505 1505 kmutex_t *hash_lock;
1506 1506
1507 1507 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1508 1508 return;
1509 1509
1510 1510 hash_lock = HDR_LOCK(buf->b_hdr);
1511 1511 mutex_enter(hash_lock);
1512 1512
1513 1513 ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1514 1514 buf->b_hdr->b_l1hdr.b_state == arc_anon);
1515 1515 arc_cksum_compute(buf, B_FALSE);
1516 1516 mutex_exit(hash_lock);
1517 1517
1518 1518 }
1519 1519
1520 1520 static void
1521 1521 add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1522 1522 {
1523 1523 ASSERT(HDR_HAS_L1HDR(hdr));
1524 1524 ASSERT(MUTEX_HELD(hash_lock));
1525 1525 arc_state_t *state = hdr->b_l1hdr.b_state;
1526 1526
1527 1527 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
1528 1528 (state != arc_anon)) {
1529 1529 /* We don't use the L2-only state list. */
1530 1530 if (state != arc_l2c_only) {
1531 1531 arc_buf_contents_t type = arc_buf_type(hdr);
1532 1532 uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
1533 1533 multilist_t *list = &state->arcs_list[type];
1534 1534 uint64_t *size = &state->arcs_lsize[type];
1535 1535
1536 1536 multilist_remove(list, hdr);
1537 1537
1538 1538 if (GHOST_STATE(state)) {
1539 1539 ASSERT0(hdr->b_l1hdr.b_datacnt);
1540 1540 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
1541 1541 delta = hdr->b_size;
1542 1542 }
1543 1543 ASSERT(delta > 0);
1544 1544 ASSERT3U(*size, >=, delta);
1545 1545 atomic_add_64(size, -delta);
1546 1546 }
1547 1547 /* remove the prefetch flag if we get a reference */
1548 1548 hdr->b_flags &= ~ARC_FLAG_PREFETCH;
1549 1549 }
1550 1550 }
1551 1551
1552 1552 static int
1553 1553 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1554 1554 {
1555 1555 int cnt;
1556 1556 arc_state_t *state = hdr->b_l1hdr.b_state;
1557 1557
1558 1558 ASSERT(HDR_HAS_L1HDR(hdr));
1559 1559 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1560 1560 ASSERT(!GHOST_STATE(state));
1561 1561
1562 1562 /*
1563 1563 * arc_l2c_only counts as a ghost state so we don't need to explicitly
1564 1564 * check to prevent usage of the arc_l2c_only list.
1565 1565 */
1566 1566 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
1567 1567 (state != arc_anon)) {
1568 1568 arc_buf_contents_t type = arc_buf_type(hdr);
1569 1569 multilist_t *list = &state->arcs_list[type];
1570 1570 uint64_t *size = &state->arcs_lsize[type];
1571 1571
1572 1572 multilist_insert(list, hdr);
1573 1573
1574 1574 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
1575 1575 atomic_add_64(size, hdr->b_size *
1576 1576 hdr->b_l1hdr.b_datacnt);
1577 1577 }
1578 1578 return (cnt);
1579 1579 }
1580 1580
1581 1581 /*
1582 1582 * Move the supplied buffer to the indicated state. The hash lock
1583 1583 * for the buffer must be held by the caller.
1584 1584 */
1585 1585 static void
1586 1586 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
1587 1587 kmutex_t *hash_lock)
1588 1588 {
1589 1589 arc_state_t *old_state;
1590 1590 int64_t refcnt;
1591 1591 uint32_t datacnt;
1592 1592 uint64_t from_delta, to_delta;
1593 1593 arc_buf_contents_t buftype = arc_buf_type(hdr);
1594 1594
1595 1595 /*
1596 1596 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
1597 1597 * in arc_read() when bringing a buffer out of the L2ARC. However, the
1598 1598 * L1 hdr doesn't always exist when we change state to arc_anon before
1599 1599 * destroying a header, in which case reallocating to add the L1 hdr is
1600 1600 * pointless.
1601 1601 */
1602 1602 if (HDR_HAS_L1HDR(hdr)) {
1603 1603 old_state = hdr->b_l1hdr.b_state;
1604 1604 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
1605 1605 datacnt = hdr->b_l1hdr.b_datacnt;
1606 1606 } else {
1607 1607 old_state = arc_l2c_only;
1608 1608 refcnt = 0;
1609 1609 datacnt = 0;
1610 1610 }
1611 1611
1612 1612 ASSERT(MUTEX_HELD(hash_lock));
1613 1613 ASSERT3P(new_state, !=, old_state);
1614 1614 ASSERT(refcnt == 0 || datacnt > 0);
1615 1615 ASSERT(!GHOST_STATE(new_state) || datacnt == 0);
1616 1616 ASSERT(old_state != arc_anon || datacnt <= 1);
1617 1617
1618 1618 from_delta = to_delta = datacnt * hdr->b_size;
1619 1619
1620 1620 /*
1621 1621 * If this buffer is evictable, transfer it from the
1622 1622 * old state list to the new state list.
1623 1623 */
1624 1624 if (refcnt == 0) {
1625 1625 if (old_state != arc_anon && old_state != arc_l2c_only) {
1626 1626 uint64_t *size = &old_state->arcs_lsize[buftype];
1627 1627
1628 1628 ASSERT(HDR_HAS_L1HDR(hdr));
1629 1629 multilist_remove(&old_state->arcs_list[buftype], hdr);
1630 1630
1631 1631 /*
1632 1632 * If prefetching out of the ghost cache,
1633 1633 * we will have a non-zero datacnt.
1634 1634 */
1635 1635 if (GHOST_STATE(old_state) && datacnt == 0) {
1636 1636 /* ghost elements have a ghost size */
1637 1637 ASSERT(hdr->b_l1hdr.b_buf == NULL);
1638 1638 from_delta = hdr->b_size;
1639 1639 }
1640 1640 ASSERT3U(*size, >=, from_delta);
1641 1641 atomic_add_64(size, -from_delta);
1642 1642 }
1643 1643 if (new_state != arc_anon && new_state != arc_l2c_only) {
1644 1644 uint64_t *size = &new_state->arcs_lsize[buftype];
1645 1645
1646 1646 /*
1647 1647 * An L1 header always exists here, since if we're
1648 1648 * moving to some L1-cached state (i.e. not l2c_only or
1649 1649 * anonymous), we realloc the header to add an L1hdr
1650 1650 * beforehand.
1651 1651 */
1652 1652 ASSERT(HDR_HAS_L1HDR(hdr));
1653 1653 multilist_insert(&new_state->arcs_list[buftype], hdr);
1654 1654
1655 1655 /* ghost elements have a ghost size */
1656 1656 if (GHOST_STATE(new_state)) {
1657 1657 ASSERT0(datacnt);
1658 1658 ASSERT(hdr->b_l1hdr.b_buf == NULL);
1659 1659 to_delta = hdr->b_size;
1660 1660 }
1661 1661 atomic_add_64(size, to_delta);
1662 1662 }
1663 1663 }
1664 1664
1665 1665 ASSERT(!BUF_EMPTY(hdr));
1666 1666 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
1667 1667 buf_hash_remove(hdr);
1668 1668
1669 1669 /* adjust state sizes (ignore arc_l2c_only) */
1670 1670
1671 1671 if (to_delta && new_state != arc_l2c_only) {
1672 1672 ASSERT(HDR_HAS_L1HDR(hdr));
1673 1673 if (GHOST_STATE(new_state)) {
1674 1674 ASSERT0(datacnt);
1675 1675
1676 1676 /*
1677 1677 * We moving a header to a ghost state, we first
1678 1678 * remove all arc buffers. Thus, we'll have a
1679 1679 * datacnt of zero, and no arc buffer to use for
1680 1680 * the reference. As a result, we use the arc
1681 1681 * header pointer for the reference.
1682 1682 */
1683 1683 (void) refcount_add_many(&new_state->arcs_size,
1684 1684 hdr->b_size, hdr);
1685 1685 } else {
1686 1686 ASSERT3U(datacnt, !=, 0);
1687 1687
1688 1688 /*
1689 1689 * Each individual buffer holds a unique reference,
1690 1690 * thus we must remove each of these references one
1691 1691 * at a time.
1692 1692 */
1693 1693 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
1694 1694 buf = buf->b_next) {
1695 1695 (void) refcount_add_many(&new_state->arcs_size,
1696 1696 hdr->b_size, buf);
1697 1697 }
1698 1698 }
1699 1699 }
1700 1700
1701 1701 if (from_delta && old_state != arc_l2c_only) {
1702 1702 ASSERT(HDR_HAS_L1HDR(hdr));
1703 1703 if (GHOST_STATE(old_state)) {
1704 1704 /*
1705 1705 * When moving a header off of a ghost state,
1706 1706 * there's the possibility for datacnt to be
1707 1707 * non-zero. This is because we first add the
1708 1708 * arc buffer to the header prior to changing
1709 1709 * the header's state. Since we used the header
1710 1710 * for the reference when putting the header on
1711 1711 * the ghost state, we must balance that and use
1712 1712 * the header when removing off the ghost state
1713 1713 * (even though datacnt is non zero).
1714 1714 */
1715 1715
1716 1716 IMPLY(datacnt == 0, new_state == arc_anon ||
1717 1717 new_state == arc_l2c_only);
1718 1718
1719 1719 (void) refcount_remove_many(&old_state->arcs_size,
1720 1720 hdr->b_size, hdr);
1721 1721 } else {
1722 1722 ASSERT3P(datacnt, !=, 0);
1723 1723
1724 1724 /*
1725 1725 * Each individual buffer holds a unique reference,
1726 1726 * thus we must remove each of these references one
1727 1727 * at a time.
1728 1728 */
1729 1729 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
1730 1730 buf = buf->b_next) {
1731 1731 (void) refcount_remove_many(
1732 1732 &old_state->arcs_size, hdr->b_size, buf);
1733 1733 }
1734 1734 }
1735 1735 }
1736 1736
1737 1737 if (HDR_HAS_L1HDR(hdr))
1738 1738 hdr->b_l1hdr.b_state = new_state;
1739 1739
1740 1740 /*
1741 1741 * L2 headers should never be on the L2 state list since they don't
1742 1742 * have L1 headers allocated.
1743 1743 */
1744 1744 ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
1745 1745 multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
1746 1746 }
1747 1747
1748 1748 void
1749 1749 arc_space_consume(uint64_t space, arc_space_type_t type)
1750 1750 {
1751 1751 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1752 1752
1753 1753 switch (type) {
1754 1754 case ARC_SPACE_DATA:
1755 1755 ARCSTAT_INCR(arcstat_data_size, space);
1756 1756 break;
1757 1757 case ARC_SPACE_META:
1758 1758 ARCSTAT_INCR(arcstat_metadata_size, space);
1759 1759 break;
1760 1760 case ARC_SPACE_OTHER:
1761 1761 ARCSTAT_INCR(arcstat_other_size, space);
1762 1762 break;
1763 1763 case ARC_SPACE_HDRS:
1764 1764 ARCSTAT_INCR(arcstat_hdr_size, space);
1765 1765 break;
1766 1766 case ARC_SPACE_L2HDRS:
1767 1767 ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1768 1768 break;
1769 1769 }
1770 1770
1771 1771 if (type != ARC_SPACE_DATA)
1772 1772 ARCSTAT_INCR(arcstat_meta_used, space);
1773 1773
1774 1774 atomic_add_64(&arc_size, space);
1775 1775 }
1776 1776
1777 1777 void
1778 1778 arc_space_return(uint64_t space, arc_space_type_t type)
1779 1779 {
1780 1780 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1781 1781
1782 1782 switch (type) {
1783 1783 case ARC_SPACE_DATA:
1784 1784 ARCSTAT_INCR(arcstat_data_size, -space);
1785 1785 break;
1786 1786 case ARC_SPACE_META:
1787 1787 ARCSTAT_INCR(arcstat_metadata_size, -space);
1788 1788 break;
1789 1789 case ARC_SPACE_OTHER:
1790 1790 ARCSTAT_INCR(arcstat_other_size, -space);
1791 1791 break;
1792 1792 case ARC_SPACE_HDRS:
1793 1793 ARCSTAT_INCR(arcstat_hdr_size, -space);
1794 1794 break;
1795 1795 case ARC_SPACE_L2HDRS:
1796 1796 ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1797 1797 break;
1798 1798 }
1799 1799
1800 1800 if (type != ARC_SPACE_DATA) {
1801 1801 ASSERT(arc_meta_used >= space);
1802 1802 if (arc_meta_max < arc_meta_used)
1803 1803 arc_meta_max = arc_meta_used;
1804 1804 ARCSTAT_INCR(arcstat_meta_used, -space);
1805 1805 }
1806 1806
1807 1807 ASSERT(arc_size >= space);
1808 1808 atomic_add_64(&arc_size, -space);
1809 1809 }
1810 1810
1811 1811 arc_buf_t *
1812 1812 arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
1813 1813 {
1814 1814 arc_buf_hdr_t *hdr;
1815 1815 arc_buf_t *buf;
1816 1816
1817 1817 ASSERT3U(size, >, 0);
1818 1818 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
1819 1819 ASSERT(BUF_EMPTY(hdr));
1820 1820 ASSERT3P(hdr->b_freeze_cksum, ==, NULL);
1821 1821 hdr->b_size = size;
1822 1822 hdr->b_spa = spa_load_guid(spa);
1823 1823
1824 1824 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1825 1825 buf->b_hdr = hdr;
1826 1826 buf->b_data = NULL;
1827 1827 buf->b_efunc = NULL;
1828 1828 buf->b_private = NULL;
1829 1829 buf->b_next = NULL;
1830 1830
1831 1831 hdr->b_flags = arc_bufc_to_flags(type);
1832 1832 hdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1833 1833
1834 1834 hdr->b_l1hdr.b_buf = buf;
1835 1835 hdr->b_l1hdr.b_state = arc_anon;
1836 1836 hdr->b_l1hdr.b_arc_access = 0;
1837 1837 hdr->b_l1hdr.b_datacnt = 1;
1838 1838 hdr->b_l1hdr.b_tmp_cdata = NULL;
1839 1839
1840 1840 arc_get_data_buf(buf);
1841 1841 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
1842 1842 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1843 1843
1844 1844 return (buf);
1845 1845 }
1846 1846
1847 1847 static char *arc_onloan_tag = "onloan";
1848 1848
1849 1849 /*
1850 1850 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1851 1851 * flight data by arc_tempreserve_space() until they are "returned". Loaned
1852 1852 * buffers must be returned to the arc before they can be used by the DMU or
1853 1853 * freed.
1854 1854 */
1855 1855 arc_buf_t *
1856 1856 arc_loan_buf(spa_t *spa, int size)
1857 1857 {
1858 1858 arc_buf_t *buf;
1859 1859
1860 1860 buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1861 1861
1862 1862 atomic_add_64(&arc_loaned_bytes, size);
1863 1863 return (buf);
1864 1864 }
1865 1865
1866 1866 /*
1867 1867 * Return a loaned arc buffer to the arc.
1868 1868 */
1869 1869 void
1870 1870 arc_return_buf(arc_buf_t *buf, void *tag)
1871 1871 {
1872 1872 arc_buf_hdr_t *hdr = buf->b_hdr;
1873 1873
1874 1874 ASSERT(buf->b_data != NULL);
1875 1875 ASSERT(HDR_HAS_L1HDR(hdr));
1876 1876 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1877 1877 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1878 1878
1879 1879 atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1880 1880 }
1881 1881
1882 1882 /* Detach an arc_buf from a dbuf (tag) */
1883 1883 void
1884 1884 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1885 1885 {
1886 1886 arc_buf_hdr_t *hdr = buf->b_hdr;
1887 1887
1888 1888 ASSERT(buf->b_data != NULL);
1889 1889 ASSERT(HDR_HAS_L1HDR(hdr));
1890 1890 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1891 1891 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
1892 1892 buf->b_efunc = NULL;
1893 1893 buf->b_private = NULL;
1894 1894
1895 1895 atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1896 1896 }
1897 1897
1898 1898 static arc_buf_t *
1899 1899 arc_buf_clone(arc_buf_t *from)
1900 1900 {
1901 1901 arc_buf_t *buf;
1902 1902 arc_buf_hdr_t *hdr = from->b_hdr;
1903 1903 uint64_t size = hdr->b_size;
1904 1904
1905 1905 ASSERT(HDR_HAS_L1HDR(hdr));
1906 1906 ASSERT(hdr->b_l1hdr.b_state != arc_anon);
1907 1907
1908 1908 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1909 1909 buf->b_hdr = hdr;
1910 1910 buf->b_data = NULL;
1911 1911 buf->b_efunc = NULL;
1912 1912 buf->b_private = NULL;
1913 1913 buf->b_next = hdr->b_l1hdr.b_buf;
1914 1914 hdr->b_l1hdr.b_buf = buf;
1915 1915 arc_get_data_buf(buf);
1916 1916 bcopy(from->b_data, buf->b_data, size);
1917 1917
1918 1918 /*
1919 1919 * This buffer already exists in the arc so create a duplicate
1920 1920 * copy for the caller. If the buffer is associated with user data
1921 1921 * then track the size and number of duplicates. These stats will be
1922 1922 * updated as duplicate buffers are created and destroyed.
1923 1923 */
1924 1924 if (HDR_ISTYPE_DATA(hdr)) {
1925 1925 ARCSTAT_BUMP(arcstat_duplicate_buffers);
1926 1926 ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1927 1927 }
1928 1928 hdr->b_l1hdr.b_datacnt += 1;
1929 1929 return (buf);
1930 1930 }
1931 1931
1932 1932 void
1933 1933 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1934 1934 {
1935 1935 arc_buf_hdr_t *hdr;
1936 1936 kmutex_t *hash_lock;
1937 1937
1938 1938 /*
1939 1939 * Check to see if this buffer is evicted. Callers
1940 1940 * must verify b_data != NULL to know if the add_ref
1941 1941 * was successful.
1942 1942 */
1943 1943 mutex_enter(&buf->b_evict_lock);
1944 1944 if (buf->b_data == NULL) {
1945 1945 mutex_exit(&buf->b_evict_lock);
1946 1946 return;
1947 1947 }
1948 1948 hash_lock = HDR_LOCK(buf->b_hdr);
1949 1949 mutex_enter(hash_lock);
1950 1950 hdr = buf->b_hdr;
1951 1951 ASSERT(HDR_HAS_L1HDR(hdr));
1952 1952 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1953 1953 mutex_exit(&buf->b_evict_lock);
1954 1954
1955 1955 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
1956 1956 hdr->b_l1hdr.b_state == arc_mfu);
1957 1957
1958 1958 add_reference(hdr, hash_lock, tag);
1959 1959 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1960 1960 arc_access(hdr, hash_lock);
1961 1961 mutex_exit(hash_lock);
1962 1962 ARCSTAT_BUMP(arcstat_hits);
1963 1963 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
1964 1964 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
1965 1965 data, metadata, hits);
1966 1966 }
1967 1967
1968 1968 static void
1969 1969 arc_buf_free_on_write(void *data, size_t size,
1970 1970 void (*free_func)(void *, size_t))
1971 1971 {
1972 1972 l2arc_data_free_t *df;
1973 1973
1974 1974 df = kmem_alloc(sizeof (*df), KM_SLEEP);
1975 1975 df->l2df_data = data;
1976 1976 df->l2df_size = size;
1977 1977 df->l2df_func = free_func;
1978 1978 mutex_enter(&l2arc_free_on_write_mtx);
1979 1979 list_insert_head(l2arc_free_on_write, df);
1980 1980 mutex_exit(&l2arc_free_on_write_mtx);
1981 1981 }
1982 1982
1983 1983 /*
1984 1984 * Free the arc data buffer. If it is an l2arc write in progress,
1985 1985 * the buffer is placed on l2arc_free_on_write to be freed later.
1986 1986 */
1987 1987 static void
1988 1988 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1989 1989 {
1990 1990 arc_buf_hdr_t *hdr = buf->b_hdr;
1991 1991
1992 1992 if (HDR_L2_WRITING(hdr)) {
1993 1993 arc_buf_free_on_write(buf->b_data, hdr->b_size, free_func);
1994 1994 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1995 1995 } else {
1996 1996 free_func(buf->b_data, hdr->b_size);
1997 1997 }
1998 1998 }
1999 1999
2000 2000 static void
2001 2001 arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
2002 2002 {
2003 2003 ASSERT(HDR_HAS_L2HDR(hdr));
2004 2004 ASSERT(MUTEX_HELD(&hdr->b_l2hdr.b_dev->l2ad_mtx));
2005 2005
2006 2006 /*
2007 2007 * The b_tmp_cdata field is linked off of the b_l1hdr, so if
2008 2008 * that doesn't exist, the header is in the arc_l2c_only state,
2009 2009 * and there isn't anything to free (it's already been freed).
2010 2010 */
2011 2011 if (!HDR_HAS_L1HDR(hdr))
2012 2012 return;
2013 2013
2014 2014 /*
2015 2015 * The header isn't being written to the l2arc device, thus it
2016 2016 * shouldn't have a b_tmp_cdata to free.
2017 2017 */
2018 2018 if (!HDR_L2_WRITING(hdr)) {
2019 2019 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
2020 2020 return;
2021 2021 }
2022 2022
2023 2023 /*
2024 2024 * The header does not have compression enabled. This can be due
2025 2025 * to the buffer not being compressible, or because we're
2026 2026 * freeing the buffer before the second phase of
2027 2027 * l2arc_write_buffer() has started (which does the compression
2028 2028 * step). In either case, b_tmp_cdata does not point to a
2029 2029 * separately compressed buffer, so there's nothing to free (it
2030 2030 * points to the same buffer as the arc_buf_t's b_data field).
2031 2031 */
2032 2032 if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_OFF) {
2033 2033 hdr->b_l1hdr.b_tmp_cdata = NULL;
2034 2034 return;
2035 2035 }
2036 2036
2037 2037 /*
2038 2038 * There's nothing to free since the buffer was all zero's and
2039 2039 * compressed to a zero length buffer.
2040 2040 */
2041 2041 if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_EMPTY) {
2042 2042 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
2043 2043 return;
2044 2044 }
2045 2045
2046 2046 ASSERT(L2ARC_IS_VALID_COMPRESS(hdr->b_l2hdr.b_compress));
2047 2047
2048 2048 arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata,
2049 2049 hdr->b_size, zio_data_buf_free);
2050 2050
2051 2051 ARCSTAT_BUMP(arcstat_l2_cdata_free_on_write);
2052 2052 hdr->b_l1hdr.b_tmp_cdata = NULL;
2053 2053 }
2054 2054
2055 2055 /*
2056 2056 * Free up buf->b_data and if 'remove' is set, then pull the
2057 2057 * arc_buf_t off of the the arc_buf_hdr_t's list and free it.
2058 2058 */
2059 2059 static void
2060 2060 arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
2061 2061 {
2062 2062 arc_buf_t **bufp;
2063 2063
2064 2064 /* free up data associated with the buf */
2065 2065 if (buf->b_data != NULL) {
2066 2066 arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
2067 2067 uint64_t size = buf->b_hdr->b_size;
2068 2068 arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
2069 2069
2070 2070 arc_cksum_verify(buf);
2071 2071 arc_buf_unwatch(buf);
2072 2072
2073 2073 if (type == ARC_BUFC_METADATA) {
2074 2074 arc_buf_data_free(buf, zio_buf_free);
2075 2075 arc_space_return(size, ARC_SPACE_META);
2076 2076 } else {
2077 2077 ASSERT(type == ARC_BUFC_DATA);
2078 2078 arc_buf_data_free(buf, zio_data_buf_free);
2079 2079 arc_space_return(size, ARC_SPACE_DATA);
2080 2080 }
2081 2081
2082 2082 /* protected by hash lock, if in the hash table */
2083 2083 if (multilist_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
2084 2084 uint64_t *cnt = &state->arcs_lsize[type];
2085 2085
2086 2086 ASSERT(refcount_is_zero(
2087 2087 &buf->b_hdr->b_l1hdr.b_refcnt));
2088 2088 ASSERT(state != arc_anon && state != arc_l2c_only);
2089 2089
2090 2090 ASSERT3U(*cnt, >=, size);
2091 2091 atomic_add_64(cnt, -size);
2092 2092 }
2093 2093
2094 2094 (void) refcount_remove_many(&state->arcs_size, size, buf);
2095 2095 buf->b_data = NULL;
2096 2096
2097 2097 /*
2098 2098 * If we're destroying a duplicate buffer make sure
2099 2099 * that the appropriate statistics are updated.
2100 2100 */
2101 2101 if (buf->b_hdr->b_l1hdr.b_datacnt > 1 &&
2102 2102 HDR_ISTYPE_DATA(buf->b_hdr)) {
2103 2103 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
2104 2104 ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
2105 2105 }
2106 2106 ASSERT(buf->b_hdr->b_l1hdr.b_datacnt > 0);
2107 2107 buf->b_hdr->b_l1hdr.b_datacnt -= 1;
2108 2108 }
2109 2109
2110 2110 /* only remove the buf if requested */
2111 2111 if (!remove)
2112 2112 return;
2113 2113
2114 2114 /* remove the buf from the hdr list */
2115 2115 for (bufp = &buf->b_hdr->b_l1hdr.b_buf; *bufp != buf;
2116 2116 bufp = &(*bufp)->b_next)
2117 2117 continue;
2118 2118 *bufp = buf->b_next;
2119 2119 buf->b_next = NULL;
2120 2120
2121 2121 ASSERT(buf->b_efunc == NULL);
2122 2122
2123 2123 /* clean up the buf */
2124 2124 buf->b_hdr = NULL;
2125 2125 kmem_cache_free(buf_cache, buf);
2126 2126 }
2127 2127
2128 2128 static void
2129 2129 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
2130 2130 {
2131 2131 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
2132 2132 l2arc_dev_t *dev = l2hdr->b_dev;
2133 2133
2134 2134 ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
2135 2135 ASSERT(HDR_HAS_L2HDR(hdr));
2136 2136
2137 2137 list_remove(&dev->l2ad_buflist, hdr);
2138 2138
2139 2139 /*
2140 2140 * We don't want to leak the b_tmp_cdata buffer that was
2141 2141 * allocated in l2arc_write_buffers()
2142 2142 */
2143 2143 arc_buf_l2_cdata_free(hdr);
2144 2144
2145 2145 /*
2146 2146 * If the l2hdr's b_daddr is equal to L2ARC_ADDR_UNSET, then
2147 2147 * this header is being processed by l2arc_write_buffers() (i.e.
2148 2148 * it's in the first stage of l2arc_write_buffers()).
2149 2149 * Re-affirming that truth here, just to serve as a reminder. If
2150 2150 * b_daddr does not equal L2ARC_ADDR_UNSET, then the header may or
2151 2151 * may not have its HDR_L2_WRITING flag set. (the write may have
2152 2152 * completed, in which case HDR_L2_WRITING will be false and the
2153 2153 * b_daddr field will point to the address of the buffer on disk).
2154 2154 */
2155 2155 IMPLY(l2hdr->b_daddr == L2ARC_ADDR_UNSET, HDR_L2_WRITING(hdr));
2156 2156
2157 2157 /*
2158 2158 * If b_daddr is equal to L2ARC_ADDR_UNSET, we're racing with
2159 2159 * l2arc_write_buffers(). Since we've just removed this header
2160 2160 * from the l2arc buffer list, this header will never reach the
2161 2161 * second stage of l2arc_write_buffers(), which increments the
2162 2162 * accounting stats for this header. Thus, we must be careful
2163 2163 * not to decrement them for this header either.
2164 2164 */
2165 2165 if (l2hdr->b_daddr != L2ARC_ADDR_UNSET) {
2166 2166 ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
2167 2167 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
2168 2168
2169 2169 vdev_space_update(dev->l2ad_vdev,
2170 2170 -l2hdr->b_asize, 0, 0);
2171 2171
2172 2172 (void) refcount_remove_many(&dev->l2ad_alloc,
2173 2173 l2hdr->b_asize, hdr);
2174 2174 }
2175 2175
2176 2176 hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
2177 2177 }
2178 2178
2179 2179 static void
2180 2180 arc_hdr_destroy(arc_buf_hdr_t *hdr)
2181 2181 {
2182 2182 if (HDR_HAS_L1HDR(hdr)) {
2183 2183 ASSERT(hdr->b_l1hdr.b_buf == NULL ||
2184 2184 hdr->b_l1hdr.b_datacnt > 0);
2185 2185 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2186 2186 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2187 2187 }
2188 2188 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2189 2189 ASSERT(!HDR_IN_HASH_TABLE(hdr));
2190 2190
2191 2191 if (HDR_HAS_L2HDR(hdr)) {
2192 2192 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
2193 2193 boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
2194 2194
2195 2195 if (!buflist_held)
2196 2196 mutex_enter(&dev->l2ad_mtx);
2197 2197
2198 2198 /*
2199 2199 * Even though we checked this conditional above, we
2200 2200 * need to check this again now that we have the
2201 2201 * l2ad_mtx. This is because we could be racing with
2202 2202 * another thread calling l2arc_evict() which might have
2203 2203 * destroyed this header's L2 portion as we were waiting
2204 2204 * to acquire the l2ad_mtx. If that happens, we don't
2205 2205 * want to re-destroy the header's L2 portion.
2206 2206 */
2207 2207 if (HDR_HAS_L2HDR(hdr))
2208 2208 arc_hdr_l2hdr_destroy(hdr);
2209 2209
2210 2210 if (!buflist_held)
2211 2211 mutex_exit(&dev->l2ad_mtx);
2212 2212 }
2213 2213
2214 2214 if (!BUF_EMPTY(hdr))
2215 2215 buf_discard_identity(hdr);
2216 2216
2217 2217 if (hdr->b_freeze_cksum != NULL) {
2218 2218 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
2219 2219 hdr->b_freeze_cksum = NULL;
2220 2220 }
2221 2221
2222 2222 if (HDR_HAS_L1HDR(hdr)) {
2223 2223 while (hdr->b_l1hdr.b_buf) {
2224 2224 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2225 2225
2226 2226 if (buf->b_efunc != NULL) {
2227 2227 mutex_enter(&arc_user_evicts_lock);
2228 2228 mutex_enter(&buf->b_evict_lock);
2229 2229 ASSERT(buf->b_hdr != NULL);
2230 2230 arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE);
2231 2231 hdr->b_l1hdr.b_buf = buf->b_next;
2232 2232 buf->b_hdr = &arc_eviction_hdr;
2233 2233 buf->b_next = arc_eviction_list;
2234 2234 arc_eviction_list = buf;
2235 2235 mutex_exit(&buf->b_evict_lock);
2236 2236 cv_signal(&arc_user_evicts_cv);
2237 2237 mutex_exit(&arc_user_evicts_lock);
2238 2238 } else {
2239 2239 arc_buf_destroy(hdr->b_l1hdr.b_buf, TRUE);
2240 2240 }
2241 2241 }
2242 2242 #ifdef ZFS_DEBUG
2243 2243 if (hdr->b_l1hdr.b_thawed != NULL) {
2244 2244 kmem_free(hdr->b_l1hdr.b_thawed, 1);
2245 2245 hdr->b_l1hdr.b_thawed = NULL;
2246 2246 }
2247 2247 #endif
2248 2248 }
2249 2249
2250 2250 ASSERT3P(hdr->b_hash_next, ==, NULL);
2251 2251 if (HDR_HAS_L1HDR(hdr)) {
2252 2252 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
2253 2253 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
2254 2254 kmem_cache_free(hdr_full_cache, hdr);
2255 2255 } else {
2256 2256 kmem_cache_free(hdr_l2only_cache, hdr);
2257 2257 }
2258 2258 }
2259 2259
2260 2260 void
2261 2261 arc_buf_free(arc_buf_t *buf, void *tag)
2262 2262 {
2263 2263 arc_buf_hdr_t *hdr = buf->b_hdr;
2264 2264 int hashed = hdr->b_l1hdr.b_state != arc_anon;
2265 2265
2266 2266 ASSERT(buf->b_efunc == NULL);
2267 2267 ASSERT(buf->b_data != NULL);
2268 2268
2269 2269 if (hashed) {
2270 2270 kmutex_t *hash_lock = HDR_LOCK(hdr);
2271 2271
2272 2272 mutex_enter(hash_lock);
2273 2273 hdr = buf->b_hdr;
2274 2274 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2275 2275
2276 2276 (void) remove_reference(hdr, hash_lock, tag);
2277 2277 if (hdr->b_l1hdr.b_datacnt > 1) {
2278 2278 arc_buf_destroy(buf, TRUE);
2279 2279 } else {
2280 2280 ASSERT(buf == hdr->b_l1hdr.b_buf);
2281 2281 ASSERT(buf->b_efunc == NULL);
2282 2282 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2283 2283 }
2284 2284 mutex_exit(hash_lock);
2285 2285 } else if (HDR_IO_IN_PROGRESS(hdr)) {
2286 2286 int destroy_hdr;
2287 2287 /*
2288 2288 * We are in the middle of an async write. Don't destroy
2289 2289 * this buffer unless the write completes before we finish
2290 2290 * decrementing the reference count.
2291 2291 */
2292 2292 mutex_enter(&arc_user_evicts_lock);
2293 2293 (void) remove_reference(hdr, NULL, tag);
2294 2294 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2295 2295 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
2296 2296 mutex_exit(&arc_user_evicts_lock);
2297 2297 if (destroy_hdr)
2298 2298 arc_hdr_destroy(hdr);
2299 2299 } else {
2300 2300 if (remove_reference(hdr, NULL, tag) > 0)
2301 2301 arc_buf_destroy(buf, TRUE);
2302 2302 else
2303 2303 arc_hdr_destroy(hdr);
2304 2304 }
2305 2305 }
2306 2306
2307 2307 boolean_t
2308 2308 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
2309 2309 {
2310 2310 arc_buf_hdr_t *hdr = buf->b_hdr;
2311 2311 kmutex_t *hash_lock = HDR_LOCK(hdr);
2312 2312 boolean_t no_callback = (buf->b_efunc == NULL);
2313 2313
2314 2314 if (hdr->b_l1hdr.b_state == arc_anon) {
2315 2315 ASSERT(hdr->b_l1hdr.b_datacnt == 1);
2316 2316 arc_buf_free(buf, tag);
2317 2317 return (no_callback);
2318 2318 }
2319 2319
2320 2320 mutex_enter(hash_lock);
2321 2321 hdr = buf->b_hdr;
2322 2322 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
2323 2323 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2324 2324 ASSERT(hdr->b_l1hdr.b_state != arc_anon);
2325 2325 ASSERT(buf->b_data != NULL);
2326 2326
2327 2327 (void) remove_reference(hdr, hash_lock, tag);
2328 2328 if (hdr->b_l1hdr.b_datacnt > 1) {
2329 2329 if (no_callback)
2330 2330 arc_buf_destroy(buf, TRUE);
2331 2331 } else if (no_callback) {
2332 2332 ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
2333 2333 ASSERT(buf->b_efunc == NULL);
2334 2334 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2335 2335 }
2336 2336 ASSERT(no_callback || hdr->b_l1hdr.b_datacnt > 1 ||
2337 2337 refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2338 2338 mutex_exit(hash_lock);
2339 2339 return (no_callback);
2340 2340 }
2341 2341
2342 2342 int32_t
2343 2343 arc_buf_size(arc_buf_t *buf)
2344 2344 {
2345 2345 return (buf->b_hdr->b_size);
2346 2346 }
2347 2347
2348 2348 /*
2349 2349 * Called from the DMU to determine if the current buffer should be
2350 2350 * evicted. In order to ensure proper locking, the eviction must be initiated
2351 2351 * from the DMU. Return true if the buffer is associated with user data and
2352 2352 * duplicate buffers still exist.
2353 2353 */
2354 2354 boolean_t
2355 2355 arc_buf_eviction_needed(arc_buf_t *buf)
2356 2356 {
2357 2357 arc_buf_hdr_t *hdr;
2358 2358 boolean_t evict_needed = B_FALSE;
2359 2359
2360 2360 if (zfs_disable_dup_eviction)
2361 2361 return (B_FALSE);
2362 2362
2363 2363 mutex_enter(&buf->b_evict_lock);
2364 2364 hdr = buf->b_hdr;
2365 2365 if (hdr == NULL) {
2366 2366 /*
2367 2367 * We are in arc_do_user_evicts(); let that function
2368 2368 * perform the eviction.
2369 2369 */
2370 2370 ASSERT(buf->b_data == NULL);
2371 2371 mutex_exit(&buf->b_evict_lock);
2372 2372 return (B_FALSE);
2373 2373 } else if (buf->b_data == NULL) {
2374 2374 /*
2375 2375 * We have already been added to the arc eviction list;
2376 2376 * recommend eviction.
2377 2377 */
2378 2378 ASSERT3P(hdr, ==, &arc_eviction_hdr);
2379 2379 mutex_exit(&buf->b_evict_lock);
2380 2380 return (B_TRUE);
2381 2381 }
2382 2382
2383 2383 if (hdr->b_l1hdr.b_datacnt > 1 && HDR_ISTYPE_DATA(hdr))
2384 2384 evict_needed = B_TRUE;
2385 2385
2386 2386 mutex_exit(&buf->b_evict_lock);
2387 2387 return (evict_needed);
2388 2388 }
2389 2389
2390 2390 /*
2391 2391 * Evict the arc_buf_hdr that is provided as a parameter. The resultant
2392 2392 * state of the header is dependent on it's state prior to entering this
2393 2393 * function. The following transitions are possible:
2394 2394 *
2395 2395 * - arc_mru -> arc_mru_ghost
2396 2396 * - arc_mfu -> arc_mfu_ghost
2397 2397 * - arc_mru_ghost -> arc_l2c_only
2398 2398 * - arc_mru_ghost -> deleted
2399 2399 * - arc_mfu_ghost -> arc_l2c_only
2400 2400 * - arc_mfu_ghost -> deleted
2401 2401 */
2402 2402 static int64_t
2403 2403 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
2404 2404 {
2405 2405 arc_state_t *evicted_state, *state;
2406 2406 int64_t bytes_evicted = 0;
2407 2407
2408 2408 ASSERT(MUTEX_HELD(hash_lock));
2409 2409 ASSERT(HDR_HAS_L1HDR(hdr));
2410 2410
2411 2411 state = hdr->b_l1hdr.b_state;
2412 2412 if (GHOST_STATE(state)) {
2413 2413 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2414 2414 ASSERT(hdr->b_l1hdr.b_buf == NULL);
2415 2415
2416 2416 /*
2417 2417 * l2arc_write_buffers() relies on a header's L1 portion
2418 2418 * (i.e. it's b_tmp_cdata field) during it's write phase.
2419 2419 * Thus, we cannot push a header onto the arc_l2c_only
2420 2420 * state (removing it's L1 piece) until the header is
2421 2421 * done being written to the l2arc.
2422 2422 */
2423 2423 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
2424 2424 ARCSTAT_BUMP(arcstat_evict_l2_skip);
2425 2425 return (bytes_evicted);
2426 2426 }
2427 2427
2428 2428 ARCSTAT_BUMP(arcstat_deleted);
2429 2429 bytes_evicted += hdr->b_size;
2430 2430
2431 2431 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
2432 2432
2433 2433 if (HDR_HAS_L2HDR(hdr)) {
2434 2434 /*
2435 2435 * This buffer is cached on the 2nd Level ARC;
2436 2436 * don't destroy the header.
2437 2437 */
2438 2438 arc_change_state(arc_l2c_only, hdr, hash_lock);
2439 2439 /*
2440 2440 * dropping from L1+L2 cached to L2-only,
2441 2441 * realloc to remove the L1 header.
2442 2442 */
2443 2443 hdr = arc_hdr_realloc(hdr, hdr_full_cache,
2444 2444 hdr_l2only_cache);
2445 2445 } else {
2446 2446 arc_change_state(arc_anon, hdr, hash_lock);
2447 2447 arc_hdr_destroy(hdr);
2448 2448 }
2449 2449 return (bytes_evicted);
2450 2450 }
2451 2451
2452 2452 ASSERT(state == arc_mru || state == arc_mfu);
2453 2453 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
2454 2454
2455 2455 /* prefetch buffers have a minimum lifespan */
2456 2456 if (HDR_IO_IN_PROGRESS(hdr) ||
2457 2457 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
2458 2458 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
2459 2459 arc_min_prefetch_lifespan)) {
2460 2460 ARCSTAT_BUMP(arcstat_evict_skip);
2461 2461 return (bytes_evicted);
2462 2462 }
2463 2463
2464 2464 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
2465 2465 ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
2466 2466 while (hdr->b_l1hdr.b_buf) {
2467 2467 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2468 2468 if (!mutex_tryenter(&buf->b_evict_lock)) {
2469 2469 ARCSTAT_BUMP(arcstat_mutex_miss);
2470 2470 break;
2471 2471 }
2472 2472 if (buf->b_data != NULL)
2473 2473 bytes_evicted += hdr->b_size;
2474 2474 if (buf->b_efunc != NULL) {
2475 2475 mutex_enter(&arc_user_evicts_lock);
2476 2476 arc_buf_destroy(buf, FALSE);
2477 2477 hdr->b_l1hdr.b_buf = buf->b_next;
2478 2478 buf->b_hdr = &arc_eviction_hdr;
2479 2479 buf->b_next = arc_eviction_list;
2480 2480 arc_eviction_list = buf;
2481 2481 cv_signal(&arc_user_evicts_cv);
2482 2482 mutex_exit(&arc_user_evicts_lock);
2483 2483 mutex_exit(&buf->b_evict_lock);
2484 2484 } else {
2485 2485 mutex_exit(&buf->b_evict_lock);
2486 2486 arc_buf_destroy(buf, TRUE);
2487 2487 }
2488 2488 }
2489 2489
2490 2490 if (HDR_HAS_L2HDR(hdr)) {
2491 2491 ARCSTAT_INCR(arcstat_evict_l2_cached, hdr->b_size);
2492 2492 } else {
2493 2493 if (l2arc_write_eligible(hdr->b_spa, hdr))
2494 2494 ARCSTAT_INCR(arcstat_evict_l2_eligible, hdr->b_size);
2495 2495 else
2496 2496 ARCSTAT_INCR(arcstat_evict_l2_ineligible, hdr->b_size);
2497 2497 }
2498 2498
2499 2499 if (hdr->b_l1hdr.b_datacnt == 0) {
2500 2500 arc_change_state(evicted_state, hdr, hash_lock);
2501 2501 ASSERT(HDR_IN_HASH_TABLE(hdr));
2502 2502 hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
2503 2503 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
2504 2504 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
2505 2505 }
2506 2506
2507 2507 return (bytes_evicted);
2508 2508 }
2509 2509
2510 2510 static uint64_t
2511 2511 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
2512 2512 uint64_t spa, int64_t bytes)
2513 2513 {
2514 2514 multilist_sublist_t *mls;
2515 2515 uint64_t bytes_evicted = 0;
2516 2516 arc_buf_hdr_t *hdr;
2517 2517 kmutex_t *hash_lock;
2518 2518 int evict_count = 0;
2519 2519
2520 2520 ASSERT3P(marker, !=, NULL);
2521 2521 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2522 2522
2523 2523 mls = multilist_sublist_lock(ml, idx);
2524 2524
2525 2525 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
2526 2526 hdr = multilist_sublist_prev(mls, marker)) {
2527 2527 if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
2528 2528 (evict_count >= zfs_arc_evict_batch_limit))
2529 2529 break;
2530 2530
2531 2531 /*
2532 2532 * To keep our iteration location, move the marker
2533 2533 * forward. Since we're not holding hdr's hash lock, we
2534 2534 * must be very careful and not remove 'hdr' from the
2535 2535 * sublist. Otherwise, other consumers might mistake the
2536 2536 * 'hdr' as not being on a sublist when they call the
2537 2537 * multilist_link_active() function (they all rely on
2538 2538 * the hash lock protecting concurrent insertions and
2539 2539 * removals). multilist_sublist_move_forward() was
2540 2540 * specifically implemented to ensure this is the case
2541 2541 * (only 'marker' will be removed and re-inserted).
2542 2542 */
2543 2543 multilist_sublist_move_forward(mls, marker);
2544 2544
2545 2545 /*
2546 2546 * The only case where the b_spa field should ever be
2547 2547 * zero, is the marker headers inserted by
2548 2548 * arc_evict_state(). It's possible for multiple threads
2549 2549 * to be calling arc_evict_state() concurrently (e.g.
2550 2550 * dsl_pool_close() and zio_inject_fault()), so we must
2551 2551 * skip any markers we see from these other threads.
2552 2552 */
2553 2553 if (hdr->b_spa == 0)
2554 2554 continue;
2555 2555
2556 2556 /* we're only interested in evicting buffers of a certain spa */
2557 2557 if (spa != 0 && hdr->b_spa != spa) {
2558 2558 ARCSTAT_BUMP(arcstat_evict_skip);
2559 2559 continue;
2560 2560 }
2561 2561
2562 2562 hash_lock = HDR_LOCK(hdr);
2563 2563
2564 2564 /*
2565 2565 * We aren't calling this function from any code path
2566 2566 * that would already be holding a hash lock, so we're
2567 2567 * asserting on this assumption to be defensive in case
2568 2568 * this ever changes. Without this check, it would be
2569 2569 * possible to incorrectly increment arcstat_mutex_miss
2570 2570 * below (e.g. if the code changed such that we called
2571 2571 * this function with a hash lock held).
2572 2572 */
2573 2573 ASSERT(!MUTEX_HELD(hash_lock));
2574 2574
2575 2575 if (mutex_tryenter(hash_lock)) {
2576 2576 uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
2577 2577 mutex_exit(hash_lock);
2578 2578
2579 2579 bytes_evicted += evicted;
2580 2580
2581 2581 /*
2582 2582 * If evicted is zero, arc_evict_hdr() must have
2583 2583 * decided to skip this header, don't increment
2584 2584 * evict_count in this case.
2585 2585 */
2586 2586 if (evicted != 0)
2587 2587 evict_count++;
2588 2588
2589 2589 /*
2590 2590 * If arc_size isn't overflowing, signal any
2591 2591 * threads that might happen to be waiting.
2592 2592 *
2593 2593 * For each header evicted, we wake up a single
2594 2594 * thread. If we used cv_broadcast, we could
2595 2595 * wake up "too many" threads causing arc_size
2596 2596 * to significantly overflow arc_c; since
2597 2597 * arc_get_data_buf() doesn't check for overflow
2598 2598 * when it's woken up (it doesn't because it's
2599 2599 * possible for the ARC to be overflowing while
2600 2600 * full of un-evictable buffers, and the
2601 2601 * function should proceed in this case).
2602 2602 *
2603 2603 * If threads are left sleeping, due to not
2604 2604 * using cv_broadcast, they will be woken up
2605 2605 * just before arc_reclaim_thread() sleeps.
2606 2606 */
2607 2607 mutex_enter(&arc_reclaim_lock);
2608 2608 if (!arc_is_overflowing())
2609 2609 cv_signal(&arc_reclaim_waiters_cv);
2610 2610 mutex_exit(&arc_reclaim_lock);
2611 2611 } else {
2612 2612 ARCSTAT_BUMP(arcstat_mutex_miss);
2613 2613 }
2614 2614 }
2615 2615
2616 2616 multilist_sublist_unlock(mls);
2617 2617
2618 2618 return (bytes_evicted);
2619 2619 }
2620 2620
2621 2621 /*
2622 2622 * Evict buffers from the given arc state, until we've removed the
2623 2623 * specified number of bytes. Move the removed buffers to the
2624 2624 * appropriate evict state.
2625 2625 *
2626 2626 * This function makes a "best effort". It skips over any buffers
2627 2627 * it can't get a hash_lock on, and so, may not catch all candidates.
2628 2628 * It may also return without evicting as much space as requested.
2629 2629 *
2630 2630 * If bytes is specified using the special value ARC_EVICT_ALL, this
2631 2631 * will evict all available (i.e. unlocked and evictable) buffers from
2632 2632 * the given arc state; which is used by arc_flush().
2633 2633 */
2634 2634 static uint64_t
2635 2635 arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
2636 2636 arc_buf_contents_t type)
2637 2637 {
2638 2638 uint64_t total_evicted = 0;
2639 2639 multilist_t *ml = &state->arcs_list[type];
2640 2640 int num_sublists;
2641 2641 arc_buf_hdr_t **markers;
2642 2642
2643 2643 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2644 2644
2645 2645 num_sublists = multilist_get_num_sublists(ml);
2646 2646
2647 2647 /*
2648 2648 * If we've tried to evict from each sublist, made some
2649 2649 * progress, but still have not hit the target number of bytes
2650 2650 * to evict, we want to keep trying. The markers allow us to
2651 2651 * pick up where we left off for each individual sublist, rather
2652 2652 * than starting from the tail each time.
2653 2653 */
2654 2654 markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
2655 2655 for (int i = 0; i < num_sublists; i++) {
2656 2656 markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
2657 2657
2658 2658 /*
2659 2659 * A b_spa of 0 is used to indicate that this header is
2660 2660 * a marker. This fact is used in arc_adjust_type() and
2661 2661 * arc_evict_state_impl().
2662 2662 */
2663 2663 markers[i]->b_spa = 0;
2664 2664
2665 2665 multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2666 2666 multilist_sublist_insert_tail(mls, markers[i]);
2667 2667 multilist_sublist_unlock(mls);
2668 2668 }
2669 2669
2670 2670 /*
2671 2671 * While we haven't hit our target number of bytes to evict, or
2672 2672 * we're evicting all available buffers.
2673 2673 */
2674 2674 while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
2675 2675 /*
2676 2676 * Start eviction using a randomly selected sublist,
2677 2677 * this is to try and evenly balance eviction across all
2678 2678 * sublists. Always starting at the same sublist
2679 2679 * (e.g. index 0) would cause evictions to favor certain
2680 2680 * sublists over others.
2681 2681 */
2682 2682 int sublist_idx = multilist_get_random_index(ml);
2683 2683 uint64_t scan_evicted = 0;
2684 2684
2685 2685 for (int i = 0; i < num_sublists; i++) {
2686 2686 uint64_t bytes_remaining;
2687 2687 uint64_t bytes_evicted;
2688 2688
2689 2689 if (bytes == ARC_EVICT_ALL)
2690 2690 bytes_remaining = ARC_EVICT_ALL;
2691 2691 else if (total_evicted < bytes)
2692 2692 bytes_remaining = bytes - total_evicted;
2693 2693 else
2694 2694 break;
2695 2695
2696 2696 bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
2697 2697 markers[sublist_idx], spa, bytes_remaining);
2698 2698
2699 2699 scan_evicted += bytes_evicted;
2700 2700 total_evicted += bytes_evicted;
2701 2701
2702 2702 /* we've reached the end, wrap to the beginning */
2703 2703 if (++sublist_idx >= num_sublists)
2704 2704 sublist_idx = 0;
2705 2705 }
2706 2706
2707 2707 /*
2708 2708 * If we didn't evict anything during this scan, we have
2709 2709 * no reason to believe we'll evict more during another
2710 2710 * scan, so break the loop.
2711 2711 */
2712 2712 if (scan_evicted == 0) {
2713 2713 /* This isn't possible, let's make that obvious */
2714 2714 ASSERT3S(bytes, !=, 0);
2715 2715
2716 2716 /*
2717 2717 * When bytes is ARC_EVICT_ALL, the only way to
2718 2718 * break the loop is when scan_evicted is zero.
2719 2719 * In that case, we actually have evicted enough,
2720 2720 * so we don't want to increment the kstat.
2721 2721 */
2722 2722 if (bytes != ARC_EVICT_ALL) {
2723 2723 ASSERT3S(total_evicted, <, bytes);
2724 2724 ARCSTAT_BUMP(arcstat_evict_not_enough);
2725 2725 }
2726 2726
2727 2727 break;
2728 2728 }
2729 2729 }
2730 2730
2731 2731 for (int i = 0; i < num_sublists; i++) {
2732 2732 multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2733 2733 multilist_sublist_remove(mls, markers[i]);
2734 2734 multilist_sublist_unlock(mls);
2735 2735
2736 2736 kmem_cache_free(hdr_full_cache, markers[i]);
2737 2737 }
2738 2738 kmem_free(markers, sizeof (*markers) * num_sublists);
2739 2739
2740 2740 return (total_evicted);
2741 2741 }
2742 2742
2743 2743 /*
2744 2744 * Flush all "evictable" data of the given type from the arc state
2745 2745 * specified. This will not evict any "active" buffers (i.e. referenced).
2746 2746 *
2747 2747 * When 'retry' is set to FALSE, the function will make a single pass
2748 2748 * over the state and evict any buffers that it can. Since it doesn't
2749 2749 * continually retry the eviction, it might end up leaving some buffers
2750 2750 * in the ARC due to lock misses.
2751 2751 *
2752 2752 * When 'retry' is set to TRUE, the function will continually retry the
2753 2753 * eviction until *all* evictable buffers have been removed from the
2754 2754 * state. As a result, if concurrent insertions into the state are
2755 2755 * allowed (e.g. if the ARC isn't shutting down), this function might
2756 2756 * wind up in an infinite loop, continually trying to evict buffers.
2757 2757 */
2758 2758 static uint64_t
2759 2759 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
2760 2760 boolean_t retry)
2761 2761 {
2762 2762 uint64_t evicted = 0;
2763 2763
2764 2764 while (state->arcs_lsize[type] != 0) {
2765 2765 evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
2766 2766
2767 2767 if (!retry)
2768 2768 break;
2769 2769 }
2770 2770
2771 2771 return (evicted);
2772 2772 }
2773 2773
2774 2774 /*
2775 2775 * Evict the specified number of bytes from the state specified,
2776 2776 * restricting eviction to the spa and type given. This function
2777 2777 * prevents us from trying to evict more from a state's list than
2778 2778 * is "evictable", and to skip evicting altogether when passed a
2779 2779 * negative value for "bytes". In contrast, arc_evict_state() will
2780 2780 * evict everything it can, when passed a negative value for "bytes".
2781 2781 */
2782 2782 static uint64_t
2783 2783 arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
2784 2784 arc_buf_contents_t type)
2785 2785 {
2786 2786 int64_t delta;
2787 2787
2788 2788 if (bytes > 0 && state->arcs_lsize[type] > 0) {
2789 2789 delta = MIN(state->arcs_lsize[type], bytes);
2790 2790 return (arc_evict_state(state, spa, delta, type));
2791 2791 }
2792 2792
2793 2793 return (0);
2794 2794 }
2795 2795
2796 2796 /*
2797 2797 * Evict metadata buffers from the cache, such that arc_meta_used is
2798 2798 * capped by the arc_meta_limit tunable.
2799 2799 */
2800 2800 static uint64_t
2801 2801 arc_adjust_meta(void)
2802 2802 {
2803 2803 uint64_t total_evicted = 0;
2804 2804 int64_t target;
2805 2805
2806 2806 /*
2807 2807 * If we're over the meta limit, we want to evict enough
2808 2808 * metadata to get back under the meta limit. We don't want to
2809 2809 * evict so much that we drop the MRU below arc_p, though. If
2810 2810 * we're over the meta limit more than we're over arc_p, we
2811 2811 * evict some from the MRU here, and some from the MFU below.
2812 2812 */
2813 2813 target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2814 2814 (int64_t)(refcount_count(&arc_anon->arcs_size) +
2815 2815 refcount_count(&arc_mru->arcs_size) - arc_p));
2816 2816
2817 2817 total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2818 2818
2819 2819 /*
2820 2820 * Similar to the above, we want to evict enough bytes to get us
2821 2821 * below the meta limit, but not so much as to drop us below the
2822 2822 * space alloted to the MFU (which is defined as arc_c - arc_p).
2823 2823 */
2824 2824 target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2825 2825 (int64_t)(refcount_count(&arc_mfu->arcs_size) - (arc_c - arc_p)));
2826 2826
2827 2827 total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2828 2828
2829 2829 return (total_evicted);
2830 2830 }
2831 2831
2832 2832 /*
2833 2833 * Return the type of the oldest buffer in the given arc state
2834 2834 *
2835 2835 * This function will select a random sublist of type ARC_BUFC_DATA and
2836 2836 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
2837 2837 * is compared, and the type which contains the "older" buffer will be
2838 2838 * returned.
2839 2839 */
2840 2840 static arc_buf_contents_t
2841 2841 arc_adjust_type(arc_state_t *state)
2842 2842 {
2843 2843 multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
2844 2844 multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
2845 2845 int data_idx = multilist_get_random_index(data_ml);
2846 2846 int meta_idx = multilist_get_random_index(meta_ml);
2847 2847 multilist_sublist_t *data_mls;
2848 2848 multilist_sublist_t *meta_mls;
2849 2849 arc_buf_contents_t type;
2850 2850 arc_buf_hdr_t *data_hdr;
2851 2851 arc_buf_hdr_t *meta_hdr;
2852 2852
2853 2853 /*
2854 2854 * We keep the sublist lock until we're finished, to prevent
2855 2855 * the headers from being destroyed via arc_evict_state().
2856 2856 */
2857 2857 data_mls = multilist_sublist_lock(data_ml, data_idx);
2858 2858 meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
2859 2859
2860 2860 /*
2861 2861 * These two loops are to ensure we skip any markers that
2862 2862 * might be at the tail of the lists due to arc_evict_state().
2863 2863 */
2864 2864
2865 2865 for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
2866 2866 data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
2867 2867 if (data_hdr->b_spa != 0)
2868 2868 break;
2869 2869 }
2870 2870
2871 2871 for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
2872 2872 meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
2873 2873 if (meta_hdr->b_spa != 0)
2874 2874 break;
2875 2875 }
2876 2876
2877 2877 if (data_hdr == NULL && meta_hdr == NULL) {
2878 2878 type = ARC_BUFC_DATA;
2879 2879 } else if (data_hdr == NULL) {
2880 2880 ASSERT3P(meta_hdr, !=, NULL);
2881 2881 type = ARC_BUFC_METADATA;
2882 2882 } else if (meta_hdr == NULL) {
2883 2883 ASSERT3P(data_hdr, !=, NULL);
2884 2884 type = ARC_BUFC_DATA;
2885 2885 } else {
2886 2886 ASSERT3P(data_hdr, !=, NULL);
2887 2887 ASSERT3P(meta_hdr, !=, NULL);
2888 2888
2889 2889 /* The headers can't be on the sublist without an L1 header */
2890 2890 ASSERT(HDR_HAS_L1HDR(data_hdr));
2891 2891 ASSERT(HDR_HAS_L1HDR(meta_hdr));
2892 2892
2893 2893 if (data_hdr->b_l1hdr.b_arc_access <
2894 2894 meta_hdr->b_l1hdr.b_arc_access) {
2895 2895 type = ARC_BUFC_DATA;
2896 2896 } else {
2897 2897 type = ARC_BUFC_METADATA;
2898 2898 }
2899 2899 }
2900 2900
2901 2901 multilist_sublist_unlock(meta_mls);
2902 2902 multilist_sublist_unlock(data_mls);
2903 2903
2904 2904 return (type);
2905 2905 }
2906 2906
2907 2907 /*
2908 2908 * Evict buffers from the cache, such that arc_size is capped by arc_c.
2909 2909 */
2910 2910 static uint64_t
2911 2911 arc_adjust(void)
2912 2912 {
2913 2913 uint64_t total_evicted = 0;
2914 2914 uint64_t bytes;
2915 2915 int64_t target;
2916 2916
2917 2917 /*
2918 2918 * If we're over arc_meta_limit, we want to correct that before
2919 2919 * potentially evicting data buffers below.
2920 2920 */
2921 2921 total_evicted += arc_adjust_meta();
2922 2922
2923 2923 /*
2924 2924 * Adjust MRU size
2925 2925 *
2926 2926 * If we're over the target cache size, we want to evict enough
2927 2927 * from the list to get back to our target size. We don't want
2928 2928 * to evict too much from the MRU, such that it drops below
2929 2929 * arc_p. So, if we're over our target cache size more than
2930 2930 * the MRU is over arc_p, we'll evict enough to get back to
2931 2931 * arc_p here, and then evict more from the MFU below.
2932 2932 */
2933 2933 target = MIN((int64_t)(arc_size - arc_c),
2934 2934 (int64_t)(refcount_count(&arc_anon->arcs_size) +
2935 2935 refcount_count(&arc_mru->arcs_size) + arc_meta_used - arc_p));
2936 2936
2937 2937 /*
2938 2938 * If we're below arc_meta_min, always prefer to evict data.
2939 2939 * Otherwise, try to satisfy the requested number of bytes to
2940 2940 * evict from the type which contains older buffers; in an
2941 2941 * effort to keep newer buffers in the cache regardless of their
2942 2942 * type. If we cannot satisfy the number of bytes from this
2943 2943 * type, spill over into the next type.
2944 2944 */
2945 2945 if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2946 2946 arc_meta_used > arc_meta_min) {
2947 2947 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2948 2948 total_evicted += bytes;
2949 2949
2950 2950 /*
2951 2951 * If we couldn't evict our target number of bytes from
2952 2952 * metadata, we try to get the rest from data.
2953 2953 */
2954 2954 target -= bytes;
2955 2955
2956 2956 total_evicted +=
2957 2957 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2958 2958 } else {
2959 2959 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2960 2960 total_evicted += bytes;
2961 2961
2962 2962 /*
2963 2963 * If we couldn't evict our target number of bytes from
2964 2964 * data, we try to get the rest from metadata.
2965 2965 */
2966 2966 target -= bytes;
2967 2967
2968 2968 total_evicted +=
2969 2969 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2970 2970 }
2971 2971
2972 2972 /*
2973 2973 * Adjust MFU size
2974 2974 *
2975 2975 * Now that we've tried to evict enough from the MRU to get its
2976 2976 * size back to arc_p, if we're still above the target cache
2977 2977 * size, we evict the rest from the MFU.
2978 2978 */
2979 2979 target = arc_size - arc_c;
2980 2980
2981 2981 if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2982 2982 arc_meta_used > arc_meta_min) {
2983 2983 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2984 2984 total_evicted += bytes;
2985 2985
2986 2986 /*
2987 2987 * If we couldn't evict our target number of bytes from
2988 2988 * metadata, we try to get the rest from data.
2989 2989 */
2990 2990 target -= bytes;
2991 2991
2992 2992 total_evicted +=
2993 2993 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2994 2994 } else {
2995 2995 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2996 2996 total_evicted += bytes;
2997 2997
2998 2998 /*
2999 2999 * If we couldn't evict our target number of bytes from
3000 3000 * data, we try to get the rest from data.
3001 3001 */
3002 3002 target -= bytes;
3003 3003
3004 3004 total_evicted +=
3005 3005 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
3006 3006 }
3007 3007
3008 3008 /*
3009 3009 * Adjust ghost lists
3010 3010 *
3011 3011 * In addition to the above, the ARC also defines target values
3012 3012 * for the ghost lists. The sum of the mru list and mru ghost
3013 3013 * list should never exceed the target size of the cache, and
3014 3014 * the sum of the mru list, mfu list, mru ghost list, and mfu
3015 3015 * ghost list should never exceed twice the target size of the
3016 3016 * cache. The following logic enforces these limits on the ghost
3017 3017 * caches, and evicts from them as needed.
3018 3018 */
3019 3019 target = refcount_count(&arc_mru->arcs_size) +
3020 3020 refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
3021 3021
3022 3022 bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
3023 3023 total_evicted += bytes;
3024 3024
3025 3025 target -= bytes;
3026 3026
3027 3027 total_evicted +=
3028 3028 arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
3029 3029
3030 3030 /*
3031 3031 * We assume the sum of the mru list and mfu list is less than
3032 3032 * or equal to arc_c (we enforced this above), which means we
3033 3033 * can use the simpler of the two equations below:
3034 3034 *
3035 3035 * mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
3036 3036 * mru ghost + mfu ghost <= arc_c
3037 3037 */
3038 3038 target = refcount_count(&arc_mru_ghost->arcs_size) +
3039 3039 refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
3040 3040
3041 3041 bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
3042 3042 total_evicted += bytes;
3043 3043
3044 3044 target -= bytes;
3045 3045
3046 3046 total_evicted +=
3047 3047 arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
3048 3048
3049 3049 return (total_evicted);
3050 3050 }
3051 3051
3052 3052 static void
3053 3053 arc_do_user_evicts(void)
3054 3054 {
3055 3055 mutex_enter(&arc_user_evicts_lock);
3056 3056 while (arc_eviction_list != NULL) {
3057 3057 arc_buf_t *buf = arc_eviction_list;
3058 3058 arc_eviction_list = buf->b_next;
3059 3059 mutex_enter(&buf->b_evict_lock);
3060 3060 buf->b_hdr = NULL;
3061 3061 mutex_exit(&buf->b_evict_lock);
3062 3062 mutex_exit(&arc_user_evicts_lock);
3063 3063
3064 3064 if (buf->b_efunc != NULL)
3065 3065 VERIFY0(buf->b_efunc(buf->b_private));
3066 3066
3067 3067 buf->b_efunc = NULL;
3068 3068 buf->b_private = NULL;
3069 3069 kmem_cache_free(buf_cache, buf);
3070 3070 mutex_enter(&arc_user_evicts_lock);
3071 3071 }
3072 3072 mutex_exit(&arc_user_evicts_lock);
3073 3073 }
3074 3074
3075 3075 void
3076 3076 arc_flush(spa_t *spa, boolean_t retry)
3077 3077 {
3078 3078 uint64_t guid = 0;
3079 3079
3080 3080 /*
3081 3081 * If retry is TRUE, a spa must not be specified since we have
3082 3082 * no good way to determine if all of a spa's buffers have been
3083 3083 * evicted from an arc state.
3084 3084 */
3085 3085 ASSERT(!retry || spa == 0);
3086 3086
3087 3087 if (spa != NULL)
3088 3088 guid = spa_load_guid(spa);
3089 3089
3090 3090 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
3091 3091 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
3092 3092
3093 3093 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
3094 3094 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
3095 3095
3096 3096 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
3097 3097 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
3098 3098
3099 3099 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
3100 3100 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
3101 3101
3102 3102 arc_do_user_evicts();
3103 3103 ASSERT(spa || arc_eviction_list == NULL);
3104 3104 }
3105 3105
3106 3106 void
3107 3107 arc_shrink(int64_t to_free)
3108 3108 {
3109 3109 if (arc_c > arc_c_min) {
3110 3110
3111 3111 if (arc_c > arc_c_min + to_free)
3112 3112 atomic_add_64(&arc_c, -to_free);
3113 3113 else
3114 3114 arc_c = arc_c_min;
3115 3115
3116 3116 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
3117 3117 if (arc_c > arc_size)
3118 3118 arc_c = MAX(arc_size, arc_c_min);
3119 3119 if (arc_p > arc_c)
3120 3120 arc_p = (arc_c >> 1);
3121 3121 ASSERT(arc_c >= arc_c_min);
3122 3122 ASSERT((int64_t)arc_p >= 0);
3123 3123 }
3124 3124
3125 3125 if (arc_size > arc_c)
3126 3126 (void) arc_adjust();
3127 3127 }
3128 3128
3129 3129 typedef enum free_memory_reason_t {
3130 3130 FMR_UNKNOWN,
3131 3131 FMR_NEEDFREE,
3132 3132 FMR_LOTSFREE,
3133 3133 FMR_SWAPFS_MINFREE,
3134 3134 FMR_PAGES_PP_MAXIMUM,
3135 3135 FMR_HEAP_ARENA,
3136 3136 FMR_ZIO_ARENA,
3137 3137 } free_memory_reason_t;
3138 3138
3139 3139 int64_t last_free_memory;
3140 3140 free_memory_reason_t last_free_reason;
3141 3141
3142 3142 /*
3143 3143 * Additional reserve of pages for pp_reserve.
3144 3144 */
3145 3145 int64_t arc_pages_pp_reserve = 64;
3146 3146
3147 3147 /*
3148 3148 * Additional reserve of pages for swapfs.
3149 3149 */
3150 3150 int64_t arc_swapfs_reserve = 64;
3151 3151
3152 3152 /*
3153 3153 * Return the amount of memory that can be consumed before reclaim will be
3154 3154 * needed. Positive if there is sufficient free memory, negative indicates
3155 3155 * the amount of memory that needs to be freed up.
3156 3156 */
3157 3157 static int64_t
3158 3158 arc_available_memory(void)
3159 3159 {
3160 3160 int64_t lowest = INT64_MAX;
3161 3161 int64_t n;
3162 3162 free_memory_reason_t r = FMR_UNKNOWN;
3163 3163
3164 3164 #ifdef _KERNEL
3165 3165 if (needfree > 0) {
3166 3166 n = PAGESIZE * (-needfree);
3167 3167 if (n < lowest) {
3168 3168 lowest = n;
3169 3169 r = FMR_NEEDFREE;
3170 3170 }
3171 3171 }
3172 3172
3173 3173 /*
3174 3174 * check that we're out of range of the pageout scanner. It starts to
3175 3175 * schedule paging if freemem is less than lotsfree and needfree.
3176 3176 * lotsfree is the high-water mark for pageout, and needfree is the
3177 3177 * number of needed free pages. We add extra pages here to make sure
3178 3178 * the scanner doesn't start up while we're freeing memory.
3179 3179 */
3180 3180 n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
3181 3181 if (n < lowest) {
3182 3182 lowest = n;
3183 3183 r = FMR_LOTSFREE;
3184 3184 }
3185 3185
3186 3186 /*
3187 3187 * check to make sure that swapfs has enough space so that anon
3188 3188 * reservations can still succeed. anon_resvmem() checks that the
3189 3189 * availrmem is greater than swapfs_minfree, and the number of reserved
3190 3190 * swap pages. We also add a bit of extra here just to prevent
3191 3191 * circumstances from getting really dire.
3192 3192 */
3193 3193 n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
3194 3194 desfree - arc_swapfs_reserve);
3195 3195 if (n < lowest) {
3196 3196 lowest = n;
3197 3197 r = FMR_SWAPFS_MINFREE;
3198 3198 }
3199 3199
3200 3200
3201 3201 /*
3202 3202 * Check that we have enough availrmem that memory locking (e.g., via
3203 3203 * mlock(3C) or memcntl(2)) can still succeed. (pages_pp_maximum
3204 3204 * stores the number of pages that cannot be locked; when availrmem
3205 3205 * drops below pages_pp_maximum, page locking mechanisms such as
3206 3206 * page_pp_lock() will fail.)
3207 3207 */
3208 3208 n = PAGESIZE * (availrmem - pages_pp_maximum -
3209 3209 arc_pages_pp_reserve);
3210 3210 if (n < lowest) {
3211 3211 lowest = n;
3212 3212 r = FMR_PAGES_PP_MAXIMUM;
3213 3213 }
3214 3214
3215 3215 #if defined(__i386)
3216 3216 /*
3217 3217 * If we're on an i386 platform, it's possible that we'll exhaust the
3218 3218 * kernel heap space before we ever run out of available physical
3219 3219 * memory. Most checks of the size of the heap_area compare against
3220 3220 * tune.t_minarmem, which is the minimum available real memory that we
3221 3221 * can have in the system. However, this is generally fixed at 25 pages
3222 3222 * which is so low that it's useless. In this comparison, we seek to
3223 3223 * calculate the total heap-size, and reclaim if more than 3/4ths of the
3224 3224 * heap is allocated. (Or, in the calculation, if less than 1/4th is
3225 3225 * free)
3226 3226 */
3227 3227 n = vmem_size(heap_arena, VMEM_FREE) -
3228 3228 (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2);
3229 3229 if (n < lowest) {
3230 3230 lowest = n;
3231 3231 r = FMR_HEAP_ARENA;
3232 3232 }
3233 3233 #endif
3234 3234
3235 3235 /*
3236 3236 * If zio data pages are being allocated out of a separate heap segment,
3237 3237 * then enforce that the size of available vmem for this arena remains
3238 3238 * above about 1/16th free.
3239 3239 *
3240 3240 * Note: The 1/16th arena free requirement was put in place
3241 3241 * to aggressively evict memory from the arc in order to avoid
3242 3242 * memory fragmentation issues.
3243 3243 */
3244 3244 if (zio_arena != NULL) {
3245 3245 n = vmem_size(zio_arena, VMEM_FREE) -
3246 3246 (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
3247 3247 if (n < lowest) {
3248 3248 lowest = n;
3249 3249 r = FMR_ZIO_ARENA;
3250 3250 }
3251 3251 }
3252 3252 #else
3253 3253 /* Every 100 calls, free a small amount */
3254 3254 if (spa_get_random(100) == 0)
3255 3255 lowest = -1024;
3256 3256 #endif
3257 3257
3258 3258 last_free_memory = lowest;
3259 3259 last_free_reason = r;
3260 3260
3261 3261 return (lowest);
3262 3262 }
3263 3263
3264 3264
3265 3265 /*
3266 3266 * Determine if the system is under memory pressure and is asking
3267 3267 * to reclaim memory. A return value of TRUE indicates that the system
3268 3268 * is under memory pressure and that the arc should adjust accordingly.
3269 3269 */
3270 3270 static boolean_t
3271 3271 arc_reclaim_needed(void)
3272 3272 {
3273 3273 return (arc_available_memory() < 0);
3274 3274 }
3275 3275
3276 3276 static void
3277 3277 arc_kmem_reap_now(void)
3278 3278 {
3279 3279 size_t i;
3280 3280 kmem_cache_t *prev_cache = NULL;
3281 3281 kmem_cache_t *prev_data_cache = NULL;
3282 3282 extern kmem_cache_t *zio_buf_cache[];
3283 3283 extern kmem_cache_t *zio_data_buf_cache[];
3284 3284 extern kmem_cache_t *range_seg_cache;
3285 3285
3286 3286 #ifdef _KERNEL
3287 3287 if (arc_meta_used >= arc_meta_limit) {
3288 3288 /*
3289 3289 * We are exceeding our meta-data cache limit.
3290 3290 * Purge some DNLC entries to release holds on meta-data.
3291 3291 */
3292 3292 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
3293 3293 }
3294 3294 #if defined(__i386)
3295 3295 /*
3296 3296 * Reclaim unused memory from all kmem caches.
3297 3297 */
3298 3298 kmem_reap();
3299 3299 #endif
3300 3300 #endif
3301 3301
3302 3302 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
3303 3303 if (zio_buf_cache[i] != prev_cache) {
3304 3304 prev_cache = zio_buf_cache[i];
3305 3305 kmem_cache_reap_now(zio_buf_cache[i]);
3306 3306 }
3307 3307 if (zio_data_buf_cache[i] != prev_data_cache) {
3308 3308 prev_data_cache = zio_data_buf_cache[i];
3309 3309 kmem_cache_reap_now(zio_data_buf_cache[i]);
3310 3310 }
3311 3311 }
3312 3312 kmem_cache_reap_now(buf_cache);
3313 3313 kmem_cache_reap_now(hdr_full_cache);
3314 3314 kmem_cache_reap_now(hdr_l2only_cache);
3315 3315 kmem_cache_reap_now(range_seg_cache);
3316 3316
3317 3317 if (zio_arena != NULL) {
3318 3318 /*
3319 3319 * Ask the vmem arena to reclaim unused memory from its
3320 3320 * quantum caches.
3321 3321 */
3322 3322 vmem_qcache_reap(zio_arena);
3323 3323 }
3324 3324 }
3325 3325
3326 3326 /*
3327 3327 * Threads can block in arc_get_data_buf() waiting for this thread to evict
3328 3328 * enough data and signal them to proceed. When this happens, the threads in
3329 3329 * arc_get_data_buf() are sleeping while holding the hash lock for their
3330 3330 * particular arc header. Thus, we must be careful to never sleep on a
3331 3331 * hash lock in this thread. This is to prevent the following deadlock:
3332 3332 *
3333 3333 * - Thread A sleeps on CV in arc_get_data_buf() holding hash lock "L",
3334 3334 * waiting for the reclaim thread to signal it.
3335 3335 *
3336 3336 * - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
3337 3337 * fails, and goes to sleep forever.
3338 3338 *
3339 3339 * This possible deadlock is avoided by always acquiring a hash lock
3340 3340 * using mutex_tryenter() from arc_reclaim_thread().
3341 3341 */
3342 3342 static void
3343 3343 arc_reclaim_thread(void)
3344 3344 {
3345 3345 clock_t growtime = 0;
3346 3346 callb_cpr_t cpr;
3347 3347
3348 3348 CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
3349 3349
3350 3350 mutex_enter(&arc_reclaim_lock);
3351 3351 while (!arc_reclaim_thread_exit) {
3352 3352 int64_t free_memory = arc_available_memory();
3353 3353 uint64_t evicted = 0;
3354 3354
3355 3355 mutex_exit(&arc_reclaim_lock);
3356 3356
3357 3357 if (free_memory < 0) {
3358 3358
3359 3359 arc_no_grow = B_TRUE;
3360 3360 arc_warm = B_TRUE;
3361 3361
3362 3362 /*
3363 3363 * Wait at least zfs_grow_retry (default 60) seconds
3364 3364 * before considering growing.
3365 3365 */
3366 3366 growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
3367 3367
3368 3368 arc_kmem_reap_now();
3369 3369
3370 3370 /*
3371 3371 * If we are still low on memory, shrink the ARC
3372 3372 * so that we have arc_shrink_min free space.
3373 3373 */
3374 3374 free_memory = arc_available_memory();
3375 3375
3376 3376 int64_t to_free =
3377 3377 (arc_c >> arc_shrink_shift) - free_memory;
3378 3378 if (to_free > 0) {
3379 3379 #ifdef _KERNEL
3380 3380 to_free = MAX(to_free, ptob(needfree));
3381 3381 #endif
3382 3382 arc_shrink(to_free);
3383 3383 }
3384 3384 } else if (free_memory < arc_c >> arc_no_grow_shift) {
3385 3385 arc_no_grow = B_TRUE;
3386 3386 } else if (ddi_get_lbolt() >= growtime) {
3387 3387 arc_no_grow = B_FALSE;
3388 3388 }
3389 3389
3390 3390 evicted = arc_adjust();
3391 3391
3392 3392 mutex_enter(&arc_reclaim_lock);
3393 3393
3394 3394 /*
3395 3395 * If evicted is zero, we couldn't evict anything via
3396 3396 * arc_adjust(). This could be due to hash lock
3397 3397 * collisions, but more likely due to the majority of
3398 3398 * arc buffers being unevictable. Therefore, even if
3399 3399 * arc_size is above arc_c, another pass is unlikely to
3400 3400 * be helpful and could potentially cause us to enter an
3401 3401 * infinite loop.
3402 3402 */
3403 3403 if (arc_size <= arc_c || evicted == 0) {
3404 3404 /*
3405 3405 * We're either no longer overflowing, or we
3406 3406 * can't evict anything more, so we should wake
3407 3407 * up any threads before we go to sleep.
3408 3408 */
3409 3409 cv_broadcast(&arc_reclaim_waiters_cv);
3410 3410
3411 3411 /*
3412 3412 * Block until signaled, or after one second (we
3413 3413 * might need to perform arc_kmem_reap_now()
3414 3414 * even if we aren't being signalled)
3415 3415 */
3416 3416 CALLB_CPR_SAFE_BEGIN(&cpr);
3417 3417 (void) cv_timedwait(&arc_reclaim_thread_cv,
3418 3418 &arc_reclaim_lock, ddi_get_lbolt() + hz);
3419 3419 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
3420 3420 }
3421 3421 }
3422 3422
3423 3423 arc_reclaim_thread_exit = FALSE;
3424 3424 cv_broadcast(&arc_reclaim_thread_cv);
3425 3425 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_lock */
3426 3426 thread_exit();
3427 3427 }
3428 3428
3429 3429 static void
3430 3430 arc_user_evicts_thread(void)
3431 3431 {
3432 3432 callb_cpr_t cpr;
3433 3433
3434 3434 CALLB_CPR_INIT(&cpr, &arc_user_evicts_lock, callb_generic_cpr, FTAG);
3435 3435
3436 3436 mutex_enter(&arc_user_evicts_lock);
3437 3437 while (!arc_user_evicts_thread_exit) {
3438 3438 mutex_exit(&arc_user_evicts_lock);
3439 3439
3440 3440 arc_do_user_evicts();
3441 3441
3442 3442 /*
3443 3443 * This is necessary in order for the mdb ::arc dcmd to
3444 3444 * show up to date information. Since the ::arc command
3445 3445 * does not call the kstat's update function, without
3446 3446 * this call, the command may show stale stats for the
3447 3447 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
3448 3448 * with this change, the data might be up to 1 second
3449 3449 * out of date; but that should suffice. The arc_state_t
3450 3450 * structures can be queried directly if more accurate
3451 3451 * information is needed.
3452 3452 */
3453 3453 if (arc_ksp != NULL)
3454 3454 arc_ksp->ks_update(arc_ksp, KSTAT_READ);
3455 3455
3456 3456 mutex_enter(&arc_user_evicts_lock);
3457 3457
3458 3458 /*
3459 3459 * Block until signaled, or after one second (we need to
3460 3460 * call the arc's kstat update function regularly).
3461 3461 */
3462 3462 CALLB_CPR_SAFE_BEGIN(&cpr);
3463 3463 (void) cv_timedwait(&arc_user_evicts_cv,
3464 3464 &arc_user_evicts_lock, ddi_get_lbolt() + hz);
3465 3465 CALLB_CPR_SAFE_END(&cpr, &arc_user_evicts_lock);
3466 3466 }
3467 3467
3468 3468 arc_user_evicts_thread_exit = FALSE;
3469 3469 cv_broadcast(&arc_user_evicts_cv);
3470 3470 CALLB_CPR_EXIT(&cpr); /* drops arc_user_evicts_lock */
3471 3471 thread_exit();
3472 3472 }
3473 3473
3474 3474 /*
3475 3475 * Adapt arc info given the number of bytes we are trying to add and
3476 3476 * the state that we are comming from. This function is only called
3477 3477 * when we are adding new content to the cache.
3478 3478 */
3479 3479 static void
3480 3480 arc_adapt(int bytes, arc_state_t *state)
3481 3481 {
3482 3482 int mult;
3483 3483 uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
3484 3484 int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size);
3485 3485 int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size);
3486 3486
3487 3487 if (state == arc_l2c_only)
3488 3488 return;
3489 3489
3490 3490 ASSERT(bytes > 0);
3491 3491 /*
3492 3492 * Adapt the target size of the MRU list:
3493 3493 * - if we just hit in the MRU ghost list, then increase
3494 3494 * the target size of the MRU list.
3495 3495 * - if we just hit in the MFU ghost list, then increase
3496 3496 * the target size of the MFU list by decreasing the
3497 3497 * target size of the MRU list.
3498 3498 */
3499 3499 if (state == arc_mru_ghost) {
3500 3500 mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
3501 3501 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
3502 3502
3503 3503 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
3504 3504 } else if (state == arc_mfu_ghost) {
3505 3505 uint64_t delta;
3506 3506
3507 3507 mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
3508 3508 mult = MIN(mult, 10);
3509 3509
3510 3510 delta = MIN(bytes * mult, arc_p);
3511 3511 arc_p = MAX(arc_p_min, arc_p - delta);
3512 3512 }
3513 3513 ASSERT((int64_t)arc_p >= 0);
3514 3514
3515 3515 if (arc_reclaim_needed()) {
3516 3516 cv_signal(&arc_reclaim_thread_cv);
3517 3517 return;
3518 3518 }
3519 3519
3520 3520 if (arc_no_grow)
3521 3521 return;
3522 3522
3523 3523 if (arc_c >= arc_c_max)
3524 3524 return;
3525 3525
3526 3526 /*
3527 3527 * If we're within (2 * maxblocksize) bytes of the target
3528 3528 * cache size, increment the target cache size
3529 3529 */
3530 3530 if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
3531 3531 atomic_add_64(&arc_c, (int64_t)bytes);
3532 3532 if (arc_c > arc_c_max)
3533 3533 arc_c = arc_c_max;
3534 3534 else if (state == arc_anon)
3535 3535 atomic_add_64(&arc_p, (int64_t)bytes);
3536 3536 if (arc_p > arc_c)
3537 3537 arc_p = arc_c;
3538 3538 }
3539 3539 ASSERT((int64_t)arc_p >= 0);
3540 3540 }
3541 3541
3542 3542 /*
3543 3543 * Check if arc_size has grown past our upper threshold, determined by
3544 3544 * zfs_arc_overflow_shift.
3545 3545 */
3546 3546 static boolean_t
3547 3547 arc_is_overflowing(void)
3548 3548 {
3549 3549 /* Always allow at least one block of overflow */
3550 3550 uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
3551 3551 arc_c >> zfs_arc_overflow_shift);
3552 3552
3553 3553 return (arc_size >= arc_c + overflow);
3554 3554 }
3555 3555
3556 3556 /*
3557 3557 * The buffer, supplied as the first argument, needs a data block. If we
3558 3558 * are hitting the hard limit for the cache size, we must sleep, waiting
3559 3559 * for the eviction thread to catch up. If we're past the target size
3560 3560 * but below the hard limit, we'll only signal the reclaim thread and
3561 3561 * continue on.
3562 3562 */
3563 3563 static void
3564 3564 arc_get_data_buf(arc_buf_t *buf)
3565 3565 {
3566 3566 arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
3567 3567 uint64_t size = buf->b_hdr->b_size;
3568 3568 arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
3569 3569
3570 3570 arc_adapt(size, state);
3571 3571
3572 3572 /*
3573 3573 * If arc_size is currently overflowing, and has grown past our
3574 3574 * upper limit, we must be adding data faster than the evict
3575 3575 * thread can evict. Thus, to ensure we don't compound the
3576 3576 * problem by adding more data and forcing arc_size to grow even
3577 3577 * further past it's target size, we halt and wait for the
3578 3578 * eviction thread to catch up.
3579 3579 *
3580 3580 * It's also possible that the reclaim thread is unable to evict
3581 3581 * enough buffers to get arc_size below the overflow limit (e.g.
3582 3582 * due to buffers being un-evictable, or hash lock collisions).
3583 3583 * In this case, we want to proceed regardless if we're
3584 3584 * overflowing; thus we don't use a while loop here.
3585 3585 */
3586 3586 if (arc_is_overflowing()) {
3587 3587 mutex_enter(&arc_reclaim_lock);
3588 3588
3589 3589 /*
3590 3590 * Now that we've acquired the lock, we may no longer be
3591 3591 * over the overflow limit, lets check.
3592 3592 *
3593 3593 * We're ignoring the case of spurious wake ups. If that
3594 3594 * were to happen, it'd let this thread consume an ARC
3595 3595 * buffer before it should have (i.e. before we're under
3596 3596 * the overflow limit and were signalled by the reclaim
3597 3597 * thread). As long as that is a rare occurrence, it
3598 3598 * shouldn't cause any harm.
3599 3599 */
3600 3600 if (arc_is_overflowing()) {
3601 3601 cv_signal(&arc_reclaim_thread_cv);
3602 3602 cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
3603 3603 }
3604 3604
3605 3605 mutex_exit(&arc_reclaim_lock);
3606 3606 }
3607 3607
3608 3608 if (type == ARC_BUFC_METADATA) {
3609 3609 buf->b_data = zio_buf_alloc(size);
3610 3610 arc_space_consume(size, ARC_SPACE_META);
3611 3611 } else {
3612 3612 ASSERT(type == ARC_BUFC_DATA);
3613 3613 buf->b_data = zio_data_buf_alloc(size);
3614 3614 arc_space_consume(size, ARC_SPACE_DATA);
3615 3615 }
3616 3616
3617 3617 /*
3618 3618 * Update the state size. Note that ghost states have a
3619 3619 * "ghost size" and so don't need to be updated.
3620 3620 */
3621 3621 if (!GHOST_STATE(buf->b_hdr->b_l1hdr.b_state)) {
3622 3622 arc_buf_hdr_t *hdr = buf->b_hdr;
3623 3623 arc_state_t *state = hdr->b_l1hdr.b_state;
3624 3624
3625 3625 (void) refcount_add_many(&state->arcs_size, size, buf);
3626 3626
3627 3627 /*
3628 3628 * If this is reached via arc_read, the link is
3629 3629 * protected by the hash lock. If reached via
3630 3630 * arc_buf_alloc, the header should not be accessed by
3631 3631 * any other thread. And, if reached via arc_read_done,
3632 3632 * the hash lock will protect it if it's found in the
3633 3633 * hash table; otherwise no other thread should be
3634 3634 * trying to [add|remove]_reference it.
3635 3635 */
3636 3636 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
3637 3637 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3638 3638 atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
3639 3639 size);
3640 3640 }
3641 3641 /*
3642 3642 * If we are growing the cache, and we are adding anonymous
3643 3643 * data, and we have outgrown arc_p, update arc_p
3644 3644 */
3645 3645 if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
3646 3646 (refcount_count(&arc_anon->arcs_size) +
3647 3647 refcount_count(&arc_mru->arcs_size) > arc_p))
3648 3648 arc_p = MIN(arc_c, arc_p + size);
3649 3649 }
3650 3650 }
3651 3651
3652 3652 /*
3653 3653 * This routine is called whenever a buffer is accessed.
3654 3654 * NOTE: the hash lock is dropped in this function.
3655 3655 */
3656 3656 static void
3657 3657 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3658 3658 {
3659 3659 clock_t now;
3660 3660
3661 3661 ASSERT(MUTEX_HELD(hash_lock));
3662 3662 ASSERT(HDR_HAS_L1HDR(hdr));
3663 3663
3664 3664 if (hdr->b_l1hdr.b_state == arc_anon) {
3665 3665 /*
3666 3666 * This buffer is not in the cache, and does not
3667 3667 * appear in our "ghost" list. Add the new buffer
3668 3668 * to the MRU state.
3669 3669 */
3670 3670
3671 3671 ASSERT0(hdr->b_l1hdr.b_arc_access);
3672 3672 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3673 3673 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3674 3674 arc_change_state(arc_mru, hdr, hash_lock);
3675 3675
3676 3676 } else if (hdr->b_l1hdr.b_state == arc_mru) {
3677 3677 now = ddi_get_lbolt();
3678 3678
3679 3679 /*
3680 3680 * If this buffer is here because of a prefetch, then either:
3681 3681 * - clear the flag if this is a "referencing" read
3682 3682 * (any subsequent access will bump this into the MFU state).
3683 3683 * or
3684 3684 * - move the buffer to the head of the list if this is
3685 3685 * another prefetch (to make it less likely to be evicted).
3686 3686 */
3687 3687 if (HDR_PREFETCH(hdr)) {
3688 3688 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
3689 3689 /* link protected by hash lock */
3690 3690 ASSERT(multilist_link_active(
3691 3691 &hdr->b_l1hdr.b_arc_node));
3692 3692 } else {
3693 3693 hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3694 3694 ARCSTAT_BUMP(arcstat_mru_hits);
3695 3695 }
3696 3696 hdr->b_l1hdr.b_arc_access = now;
3697 3697 return;
3698 3698 }
3699 3699
3700 3700 /*
3701 3701 * This buffer has been "accessed" only once so far,
3702 3702 * but it is still in the cache. Move it to the MFU
3703 3703 * state.
3704 3704 */
3705 3705 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
3706 3706 /*
3707 3707 * More than 125ms have passed since we
3708 3708 * instantiated this buffer. Move it to the
3709 3709 * most frequently used state.
3710 3710 */
3711 3711 hdr->b_l1hdr.b_arc_access = now;
3712 3712 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3713 3713 arc_change_state(arc_mfu, hdr, hash_lock);
3714 3714 }
3715 3715 ARCSTAT_BUMP(arcstat_mru_hits);
3716 3716 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
3717 3717 arc_state_t *new_state;
3718 3718 /*
3719 3719 * This buffer has been "accessed" recently, but
3720 3720 * was evicted from the cache. Move it to the
3721 3721 * MFU state.
3722 3722 */
3723 3723
3724 3724 if (HDR_PREFETCH(hdr)) {
3725 3725 new_state = arc_mru;
3726 3726 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
3727 3727 hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3728 3728 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3729 3729 } else {
3730 3730 new_state = arc_mfu;
3731 3731 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3732 3732 }
3733 3733
3734 3734 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3735 3735 arc_change_state(new_state, hdr, hash_lock);
3736 3736
3737 3737 ARCSTAT_BUMP(arcstat_mru_ghost_hits);
3738 3738 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
3739 3739 /*
3740 3740 * This buffer has been accessed more than once and is
3741 3741 * still in the cache. Keep it in the MFU state.
3742 3742 *
3743 3743 * NOTE: an add_reference() that occurred when we did
3744 3744 * the arc_read() will have kicked this off the list.
3745 3745 * If it was a prefetch, we will explicitly move it to
3746 3746 * the head of the list now.
3747 3747 */
3748 3748 if ((HDR_PREFETCH(hdr)) != 0) {
3749 3749 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3750 3750 /* link protected by hash_lock */
3751 3751 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3752 3752 }
3753 3753 ARCSTAT_BUMP(arcstat_mfu_hits);
3754 3754 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3755 3755 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
3756 3756 arc_state_t *new_state = arc_mfu;
3757 3757 /*
3758 3758 * This buffer has been accessed more than once but has
3759 3759 * been evicted from the cache. Move it back to the
3760 3760 * MFU state.
3761 3761 */
3762 3762
3763 3763 if (HDR_PREFETCH(hdr)) {
3764 3764 /*
3765 3765 * This is a prefetch access...
3766 3766 * move this block back to the MRU state.
3767 3767 */
3768 3768 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3769 3769 new_state = arc_mru;
3770 3770 }
3771 3771
3772 3772 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3773 3773 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3774 3774 arc_change_state(new_state, hdr, hash_lock);
3775 3775
3776 3776 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
3777 3777 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
3778 3778 /*
3779 3779 * This buffer is on the 2nd Level ARC.
3780 3780 */
3781 3781
3782 3782 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3783 3783 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3784 3784 arc_change_state(arc_mfu, hdr, hash_lock);
3785 3785 } else {
3786 3786 ASSERT(!"invalid arc state");
3787 3787 }
3788 3788 }
3789 3789
3790 3790 /* a generic arc_done_func_t which you can use */
3791 3791 /* ARGSUSED */
3792 3792 void
3793 3793 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
3794 3794 {
3795 3795 if (zio == NULL || zio->io_error == 0)
3796 3796 bcopy(buf->b_data, arg, buf->b_hdr->b_size);
3797 3797 VERIFY(arc_buf_remove_ref(buf, arg));
3798 3798 }
3799 3799
3800 3800 /* a generic arc_done_func_t */
3801 3801 void
3802 3802 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
3803 3803 {
3804 3804 arc_buf_t **bufp = arg;
3805 3805 if (zio && zio->io_error) {
3806 3806 VERIFY(arc_buf_remove_ref(buf, arg));
3807 3807 *bufp = NULL;
3808 3808 } else {
3809 3809 *bufp = buf;
3810 3810 ASSERT(buf->b_data);
3811 3811 }
3812 3812 }
3813 3813
3814 3814 static void
3815 3815 arc_read_done(zio_t *zio)
3816 3816 {
3817 3817 arc_buf_hdr_t *hdr;
3818 3818 arc_buf_t *buf;
3819 3819 arc_buf_t *abuf; /* buffer we're assigning to callback */
3820 3820 kmutex_t *hash_lock = NULL;
3821 3821 arc_callback_t *callback_list, *acb;
3822 3822 int freeable = FALSE;
3823 3823
3824 3824 buf = zio->io_private;
3825 3825 hdr = buf->b_hdr;
3826 3826
3827 3827 /*
3828 3828 * The hdr was inserted into hash-table and removed from lists
3829 3829 * prior to starting I/O. We should find this header, since
3830 3830 * it's in the hash table, and it should be legit since it's
3831 3831 * not possible to evict it during the I/O. The only possible
3832 3832 * reason for it not to be found is if we were freed during the
3833 3833 * read.
3834 3834 */
3835 3835 if (HDR_IN_HASH_TABLE(hdr)) {
3836 3836 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
3837 3837 ASSERT3U(hdr->b_dva.dva_word[0], ==,
3838 3838 BP_IDENTITY(zio->io_bp)->dva_word[0]);
3839 3839 ASSERT3U(hdr->b_dva.dva_word[1], ==,
3840 3840 BP_IDENTITY(zio->io_bp)->dva_word[1]);
3841 3841
3842 3842 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
3843 3843 &hash_lock);
3844 3844
3845 3845 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) &&
3846 3846 hash_lock == NULL) ||
3847 3847 (found == hdr &&
3848 3848 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
3849 3849 (found == hdr && HDR_L2_READING(hdr)));
3850 3850 }
3851 3851
3852 3852 hdr->b_flags &= ~ARC_FLAG_L2_EVICTED;
3853 3853 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
3854 3854 hdr->b_flags &= ~ARC_FLAG_L2CACHE;
3855 3855
3856 3856 /* byteswap if necessary */
3857 3857 callback_list = hdr->b_l1hdr.b_acb;
3858 3858 ASSERT(callback_list != NULL);
3859 3859 if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
3860 3860 dmu_object_byteswap_t bswap =
3861 3861 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
3862 3862 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
3863 3863 byteswap_uint64_array :
3864 3864 dmu_ot_byteswap[bswap].ob_func;
3865 3865 func(buf->b_data, hdr->b_size);
3866 3866 }
3867 3867
3868 3868 arc_cksum_compute(buf, B_FALSE);
3869 3869 arc_buf_watch(buf);
3870 3870
3871 3871 if (hash_lock && zio->io_error == 0 &&
3872 3872 hdr->b_l1hdr.b_state == arc_anon) {
3873 3873 /*
3874 3874 * Only call arc_access on anonymous buffers. This is because
3875 3875 * if we've issued an I/O for an evicted buffer, we've already
3876 3876 * called arc_access (to prevent any simultaneous readers from
3877 3877 * getting confused).
3878 3878 */
3879 3879 arc_access(hdr, hash_lock);
3880 3880 }
3881 3881
3882 3882 /* create copies of the data buffer for the callers */
3883 3883 abuf = buf;
3884 3884 for (acb = callback_list; acb; acb = acb->acb_next) {
3885 3885 if (acb->acb_done) {
3886 3886 if (abuf == NULL) {
3887 3887 ARCSTAT_BUMP(arcstat_duplicate_reads);
3888 3888 abuf = arc_buf_clone(buf);
3889 3889 }
3890 3890 acb->acb_buf = abuf;
3891 3891 abuf = NULL;
3892 3892 }
3893 3893 }
3894 3894 hdr->b_l1hdr.b_acb = NULL;
3895 3895 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
3896 3896 ASSERT(!HDR_BUF_AVAILABLE(hdr));
3897 3897 if (abuf == buf) {
3898 3898 ASSERT(buf->b_efunc == NULL);
3899 3899 ASSERT(hdr->b_l1hdr.b_datacnt == 1);
3900 3900 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
3901 3901 }
3902 3902
3903 3903 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
3904 3904 callback_list != NULL);
3905 3905
3906 3906 if (zio->io_error != 0) {
3907 3907 hdr->b_flags |= ARC_FLAG_IO_ERROR;
3908 3908 if (hdr->b_l1hdr.b_state != arc_anon)
3909 3909 arc_change_state(arc_anon, hdr, hash_lock);
3910 3910 if (HDR_IN_HASH_TABLE(hdr))
3911 3911 buf_hash_remove(hdr);
3912 3912 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3913 3913 }
3914 3914
3915 3915 /*
3916 3916 * Broadcast before we drop the hash_lock to avoid the possibility
3917 3917 * that the hdr (and hence the cv) might be freed before we get to
3918 3918 * the cv_broadcast().
3919 3919 */
3920 3920 cv_broadcast(&hdr->b_l1hdr.b_cv);
3921 3921
3922 3922 if (hash_lock != NULL) {
3923 3923 mutex_exit(hash_lock);
3924 3924 } else {
3925 3925 /*
3926 3926 * This block was freed while we waited for the read to
3927 3927 * complete. It has been removed from the hash table and
3928 3928 * moved to the anonymous state (so that it won't show up
3929 3929 * in the cache).
3930 3930 */
3931 3931 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3932 3932 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3933 3933 }
3934 3934
3935 3935 /* execute each callback and free its structure */
3936 3936 while ((acb = callback_list) != NULL) {
3937 3937 if (acb->acb_done)
3938 3938 acb->acb_done(zio, acb->acb_buf, acb->acb_private);
3939 3939
3940 3940 if (acb->acb_zio_dummy != NULL) {
3941 3941 acb->acb_zio_dummy->io_error = zio->io_error;
3942 3942 zio_nowait(acb->acb_zio_dummy);
3943 3943 }
3944 3944
3945 3945 callback_list = acb->acb_next;
3946 3946 kmem_free(acb, sizeof (arc_callback_t));
3947 3947 }
3948 3948
3949 3949 if (freeable)
3950 3950 arc_hdr_destroy(hdr);
3951 3951 }
3952 3952
3953 3953 /*
3954 3954 * "Read" the block at the specified DVA (in bp) via the
3955 3955 * cache. If the block is found in the cache, invoke the provided
3956 3956 * callback immediately and return. Note that the `zio' parameter
3957 3957 * in the callback will be NULL in this case, since no IO was
3958 3958 * required. If the block is not in the cache pass the read request
3959 3959 * on to the spa with a substitute callback function, so that the
3960 3960 * requested block will be added to the cache.
3961 3961 *
3962 3962 * If a read request arrives for a block that has a read in-progress,
3963 3963 * either wait for the in-progress read to complete (and return the
3964 3964 * results); or, if this is a read with a "done" func, add a record
3965 3965 * to the read to invoke the "done" func when the read completes,
3966 3966 * and return; or just return.
3967 3967 *
3968 3968 * arc_read_done() will invoke all the requested "done" functions
3969 3969 * for readers of this block.
3970 3970 */
3971 3971 int
3972 3972 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
3973 3973 void *private, zio_priority_t priority, int zio_flags,
3974 3974 arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
3975 3975 {
3976 3976 arc_buf_hdr_t *hdr = NULL;
3977 3977 arc_buf_t *buf = NULL;
3978 3978 kmutex_t *hash_lock = NULL;
3979 3979 zio_t *rzio;
3980 3980 uint64_t guid = spa_load_guid(spa);
3981 3981
3982 3982 ASSERT(!BP_IS_EMBEDDED(bp) ||
3983 3983 BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
3984 3984
3985 3985 top:
3986 3986 if (!BP_IS_EMBEDDED(bp)) {
3987 3987 /*
3988 3988 * Embedded BP's have no DVA and require no I/O to "read".
3989 3989 * Create an anonymous arc buf to back it.
3990 3990 */
3991 3991 hdr = buf_hash_find(guid, bp, &hash_lock);
3992 3992 }
3993 3993
3994 3994 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_datacnt > 0) {
3995 3995
3996 3996 *arc_flags |= ARC_FLAG_CACHED;
3997 3997
3998 3998 if (HDR_IO_IN_PROGRESS(hdr)) {
3999 3999
4000 4000 if (*arc_flags & ARC_FLAG_WAIT) {
4001 4001 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
4002 4002 mutex_exit(hash_lock);
4003 4003 goto top;
4004 4004 }
4005 4005 ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
4006 4006
4007 4007 if (done) {
4008 4008 arc_callback_t *acb = NULL;
4009 4009
4010 4010 acb = kmem_zalloc(sizeof (arc_callback_t),
4011 4011 KM_SLEEP);
4012 4012 acb->acb_done = done;
4013 4013 acb->acb_private = private;
4014 4014 if (pio != NULL)
4015 4015 acb->acb_zio_dummy = zio_null(pio,
4016 4016 spa, NULL, NULL, NULL, zio_flags);
4017 4017
4018 4018 ASSERT(acb->acb_done != NULL);
4019 4019 acb->acb_next = hdr->b_l1hdr.b_acb;
4020 4020 hdr->b_l1hdr.b_acb = acb;
4021 4021 add_reference(hdr, hash_lock, private);
4022 4022 mutex_exit(hash_lock);
4023 4023 return (0);
4024 4024 }
4025 4025 mutex_exit(hash_lock);
4026 4026 return (0);
4027 4027 }
4028 4028
4029 4029 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4030 4030 hdr->b_l1hdr.b_state == arc_mfu);
4031 4031
4032 4032 if (done) {
4033 4033 add_reference(hdr, hash_lock, private);
4034 4034 /*
4035 4035 * If this block is already in use, create a new
4036 4036 * copy of the data so that we will be guaranteed
4037 4037 * that arc_release() will always succeed.
4038 4038 */
4039 4039 buf = hdr->b_l1hdr.b_buf;
4040 4040 ASSERT(buf);
4041 4041 ASSERT(buf->b_data);
4042 4042 if (HDR_BUF_AVAILABLE(hdr)) {
4043 4043 ASSERT(buf->b_efunc == NULL);
4044 4044 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
4045 4045 } else {
4046 4046 buf = arc_buf_clone(buf);
4047 4047 }
4048 4048
4049 4049 } else if (*arc_flags & ARC_FLAG_PREFETCH &&
4050 4050 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4051 4051 hdr->b_flags |= ARC_FLAG_PREFETCH;
4052 4052 }
4053 4053 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
4054 4054 arc_access(hdr, hash_lock);
4055 4055 if (*arc_flags & ARC_FLAG_L2CACHE)
4056 4056 hdr->b_flags |= ARC_FLAG_L2CACHE;
4057 4057 if (*arc_flags & ARC_FLAG_L2COMPRESS)
4058 4058 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4059 4059 mutex_exit(hash_lock);
4060 4060 ARCSTAT_BUMP(arcstat_hits);
4061 4061 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4062 4062 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4063 4063 data, metadata, hits);
4064 4064
4065 4065 if (done)
4066 4066 done(NULL, buf, private);
4067 4067 } else {
4068 4068 uint64_t size = BP_GET_LSIZE(bp);
4069 4069 arc_callback_t *acb;
4070 4070 vdev_t *vd = NULL;
4071 4071 uint64_t addr = 0;
4072 4072 boolean_t devw = B_FALSE;
4073 4073 enum zio_compress b_compress = ZIO_COMPRESS_OFF;
4074 4074 int32_t b_asize = 0;
4075 4075
4076 4076 if (hdr == NULL) {
4077 4077 /* this block is not in the cache */
4078 4078 arc_buf_hdr_t *exists = NULL;
4079 4079 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
4080 4080 buf = arc_buf_alloc(spa, size, private, type);
4081 4081 hdr = buf->b_hdr;
4082 4082 if (!BP_IS_EMBEDDED(bp)) {
4083 4083 hdr->b_dva = *BP_IDENTITY(bp);
4084 4084 hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
4085 4085 exists = buf_hash_insert(hdr, &hash_lock);
4086 4086 }
4087 4087 if (exists != NULL) {
4088 4088 /* somebody beat us to the hash insert */
4089 4089 mutex_exit(hash_lock);
4090 4090 buf_discard_identity(hdr);
4091 4091 (void) arc_buf_remove_ref(buf, private);
4092 4092 goto top; /* restart the IO request */
4093 4093 }
4094 4094
4095 4095 /* if this is a prefetch, we don't have a reference */
4096 4096 if (*arc_flags & ARC_FLAG_PREFETCH) {
4097 4097 (void) remove_reference(hdr, hash_lock,
4098 4098 private);
4099 4099 hdr->b_flags |= ARC_FLAG_PREFETCH;
4100 4100 }
4101 4101 if (*arc_flags & ARC_FLAG_L2CACHE)
4102 4102 hdr->b_flags |= ARC_FLAG_L2CACHE;
4103 4103 if (*arc_flags & ARC_FLAG_L2COMPRESS)
4104 4104 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4105 4105 if (BP_GET_LEVEL(bp) > 0)
4106 4106 hdr->b_flags |= ARC_FLAG_INDIRECT;
4107 4107 } else {
4108 4108 /*
4109 4109 * This block is in the ghost cache. If it was L2-only
4110 4110 * (and thus didn't have an L1 hdr), we realloc the
4111 4111 * header to add an L1 hdr.
4112 4112 */
4113 4113 if (!HDR_HAS_L1HDR(hdr)) {
4114 4114 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
4115 4115 hdr_full_cache);
4116 4116 }
4117 4117
4118 4118 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
4119 4119 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4120 4120 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4121 4121 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
4122 4122
4123 4123 /* if this is a prefetch, we don't have a reference */
4124 4124 if (*arc_flags & ARC_FLAG_PREFETCH)
4125 4125 hdr->b_flags |= ARC_FLAG_PREFETCH;
4126 4126 else
4127 4127 add_reference(hdr, hash_lock, private);
4128 4128 if (*arc_flags & ARC_FLAG_L2CACHE)
4129 4129 hdr->b_flags |= ARC_FLAG_L2CACHE;
4130 4130 if (*arc_flags & ARC_FLAG_L2COMPRESS)
4131 4131 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4132 4132 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
4133 4133 buf->b_hdr = hdr;
4134 4134 buf->b_data = NULL;
4135 4135 buf->b_efunc = NULL;
4136 4136 buf->b_private = NULL;
4137 4137 buf->b_next = NULL;
4138 4138 hdr->b_l1hdr.b_buf = buf;
4139 4139 ASSERT0(hdr->b_l1hdr.b_datacnt);
4140 4140 hdr->b_l1hdr.b_datacnt = 1;
4141 4141 arc_get_data_buf(buf);
4142 4142 arc_access(hdr, hash_lock);
4143 4143 }
4144 4144
4145 4145 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
4146 4146
4147 4147 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
4148 4148 acb->acb_done = done;
4149 4149 acb->acb_private = private;
4150 4150
4151 4151 ASSERT(hdr->b_l1hdr.b_acb == NULL);
4152 4152 hdr->b_l1hdr.b_acb = acb;
4153 4153 hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4154 4154
4155 4155 if (HDR_HAS_L2HDR(hdr) &&
4156 4156 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
4157 4157 devw = hdr->b_l2hdr.b_dev->l2ad_writing;
4158 4158 addr = hdr->b_l2hdr.b_daddr;
4159 4159 b_compress = hdr->b_l2hdr.b_compress;
4160 4160 b_asize = hdr->b_l2hdr.b_asize;
4161 4161 /*
4162 4162 * Lock out device removal.
4163 4163 */
4164 4164 if (vdev_is_dead(vd) ||
4165 4165 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
4166 4166 vd = NULL;
4167 4167 }
4168 4168
4169 4169 if (hash_lock != NULL)
4170 4170 mutex_exit(hash_lock);
4171 4171
4172 4172 /*
4173 4173 * At this point, we have a level 1 cache miss. Try again in
4174 4174 * L2ARC if possible.
4175 4175 */
4176 4176 ASSERT3U(hdr->b_size, ==, size);
4177 4177 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
4178 4178 uint64_t, size, zbookmark_phys_t *, zb);
4179 4179 ARCSTAT_BUMP(arcstat_misses);
4180 4180 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4181 4181 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4182 4182 data, metadata, misses);
4183 4183
4184 4184 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
4185 4185 /*
4186 4186 * Read from the L2ARC if the following are true:
4187 4187 * 1. The L2ARC vdev was previously cached.
4188 4188 * 2. This buffer still has L2ARC metadata.
4189 4189 * 3. This buffer isn't currently writing to the L2ARC.
4190 4190 * 4. The L2ARC entry wasn't evicted, which may
4191 4191 * also have invalidated the vdev.
4192 4192 * 5. This isn't prefetch and l2arc_noprefetch is set.
4193 4193 */
4194 4194 if (HDR_HAS_L2HDR(hdr) &&
4195 4195 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
4196 4196 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
4197 4197 l2arc_read_callback_t *cb;
4198 4198
4199 4199 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
4200 4200 ARCSTAT_BUMP(arcstat_l2_hits);
4201 4201
4202 4202 cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
4203 4203 KM_SLEEP);
4204 4204 cb->l2rcb_buf = buf;
4205 4205 cb->l2rcb_spa = spa;
4206 4206 cb->l2rcb_bp = *bp;
4207 4207 cb->l2rcb_zb = *zb;
4208 4208 cb->l2rcb_flags = zio_flags;
4209 4209 cb->l2rcb_compress = b_compress;
4210 4210
4211 4211 ASSERT(addr >= VDEV_LABEL_START_SIZE &&
4212 4212 addr + size < vd->vdev_psize -
4213 4213 VDEV_LABEL_END_SIZE);
4214 4214
4215 4215 /*
4216 4216 * l2arc read. The SCL_L2ARC lock will be
4217 4217 * released by l2arc_read_done().
4218 4218 * Issue a null zio if the underlying buffer
4219 4219 * was squashed to zero size by compression.
4220 4220 */
4221 4221 if (b_compress == ZIO_COMPRESS_EMPTY) {
4222 4222 rzio = zio_null(pio, spa, vd,
4223 4223 l2arc_read_done, cb,
4224 4224 zio_flags | ZIO_FLAG_DONT_CACHE |
4225 4225 ZIO_FLAG_CANFAIL |
4226 4226 ZIO_FLAG_DONT_PROPAGATE |
4227 4227 ZIO_FLAG_DONT_RETRY);
4228 4228 } else {
4229 4229 rzio = zio_read_phys(pio, vd, addr,
4230 4230 b_asize, buf->b_data,
4231 4231 ZIO_CHECKSUM_OFF,
4232 4232 l2arc_read_done, cb, priority,
4233 4233 zio_flags | ZIO_FLAG_DONT_CACHE |
4234 4234 ZIO_FLAG_CANFAIL |
4235 4235 ZIO_FLAG_DONT_PROPAGATE |
4236 4236 ZIO_FLAG_DONT_RETRY, B_FALSE);
4237 4237 }
4238 4238 DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
4239 4239 zio_t *, rzio);
4240 4240 ARCSTAT_INCR(arcstat_l2_read_bytes, b_asize);
4241 4241
4242 4242 if (*arc_flags & ARC_FLAG_NOWAIT) {
4243 4243 zio_nowait(rzio);
4244 4244 return (0);
4245 4245 }
4246 4246
4247 4247 ASSERT(*arc_flags & ARC_FLAG_WAIT);
4248 4248 if (zio_wait(rzio) == 0)
4249 4249 return (0);
4250 4250
4251 4251 /* l2arc read error; goto zio_read() */
4252 4252 } else {
4253 4253 DTRACE_PROBE1(l2arc__miss,
4254 4254 arc_buf_hdr_t *, hdr);
4255 4255 ARCSTAT_BUMP(arcstat_l2_misses);
4256 4256 if (HDR_L2_WRITING(hdr))
4257 4257 ARCSTAT_BUMP(arcstat_l2_rw_clash);
4258 4258 spa_config_exit(spa, SCL_L2ARC, vd);
4259 4259 }
4260 4260 } else {
4261 4261 if (vd != NULL)
4262 4262 spa_config_exit(spa, SCL_L2ARC, vd);
4263 4263 if (l2arc_ndev != 0) {
4264 4264 DTRACE_PROBE1(l2arc__miss,
4265 4265 arc_buf_hdr_t *, hdr);
4266 4266 ARCSTAT_BUMP(arcstat_l2_misses);
4267 4267 }
4268 4268 }
4269 4269
4270 4270 rzio = zio_read(pio, spa, bp, buf->b_data, size,
4271 4271 arc_read_done, buf, priority, zio_flags, zb);
4272 4272
4273 4273 /*
4274 4274 * At this point, this read I/O has already missed in the ARC
4275 4275 * and will be going through to the disk. The I/O throttle
4276 4276 * should delay this I/O if this zone is using more than its I/O
4277 4277 * priority allows.
4278 4278 */
4279 4279 zfs_zone_io_throttle(ZFS_ZONE_IOP_READ);
4280 4280
4281 4281 if (*arc_flags & ARC_FLAG_WAIT)
4282 4282 return (zio_wait(rzio));
4283 4283
4284 4284 ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
4285 4285 zio_nowait(rzio);
4286 4286 }
4287 4287 return (0);
4288 4288 }
4289 4289
4290 4290 void
4291 4291 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
4292 4292 {
4293 4293 ASSERT(buf->b_hdr != NULL);
4294 4294 ASSERT(buf->b_hdr->b_l1hdr.b_state != arc_anon);
4295 4295 ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt) ||
4296 4296 func == NULL);
4297 4297 ASSERT(buf->b_efunc == NULL);
4298 4298 ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
4299 4299
4300 4300 buf->b_efunc = func;
4301 4301 buf->b_private = private;
4302 4302 }
4303 4303
4304 4304 /*
4305 4305 * Notify the arc that a block was freed, and thus will never be used again.
4306 4306 */
4307 4307 void
4308 4308 arc_freed(spa_t *spa, const blkptr_t *bp)
4309 4309 {
4310 4310 arc_buf_hdr_t *hdr;
4311 4311 kmutex_t *hash_lock;
4312 4312 uint64_t guid = spa_load_guid(spa);
4313 4313
4314 4314 ASSERT(!BP_IS_EMBEDDED(bp));
4315 4315
4316 4316 hdr = buf_hash_find(guid, bp, &hash_lock);
4317 4317 if (hdr == NULL)
4318 4318 return;
4319 4319 if (HDR_BUF_AVAILABLE(hdr)) {
4320 4320 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
4321 4321 add_reference(hdr, hash_lock, FTAG);
4322 4322 hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
4323 4323 mutex_exit(hash_lock);
4324 4324
4325 4325 arc_release(buf, FTAG);
4326 4326 (void) arc_buf_remove_ref(buf, FTAG);
4327 4327 } else {
4328 4328 mutex_exit(hash_lock);
4329 4329 }
4330 4330
4331 4331 }
4332 4332
4333 4333 /*
4334 4334 * Clear the user eviction callback set by arc_set_callback(), first calling
4335 4335 * it if it exists. Because the presence of a callback keeps an arc_buf cached
4336 4336 * clearing the callback may result in the arc_buf being destroyed. However,
4337 4337 * it will not result in the *last* arc_buf being destroyed, hence the data
4338 4338 * will remain cached in the ARC. We make a copy of the arc buffer here so
4339 4339 * that we can process the callback without holding any locks.
4340 4340 *
4341 4341 * It's possible that the callback is already in the process of being cleared
4342 4342 * by another thread. In this case we can not clear the callback.
4343 4343 *
4344 4344 * Returns B_TRUE if the callback was successfully called and cleared.
4345 4345 */
4346 4346 boolean_t
4347 4347 arc_clear_callback(arc_buf_t *buf)
4348 4348 {
4349 4349 arc_buf_hdr_t *hdr;
4350 4350 kmutex_t *hash_lock;
4351 4351 arc_evict_func_t *efunc = buf->b_efunc;
4352 4352 void *private = buf->b_private;
4353 4353
4354 4354 mutex_enter(&buf->b_evict_lock);
4355 4355 hdr = buf->b_hdr;
4356 4356 if (hdr == NULL) {
4357 4357 /*
4358 4358 * We are in arc_do_user_evicts().
4359 4359 */
4360 4360 ASSERT(buf->b_data == NULL);
4361 4361 mutex_exit(&buf->b_evict_lock);
4362 4362 return (B_FALSE);
4363 4363 } else if (buf->b_data == NULL) {
4364 4364 /*
4365 4365 * We are on the eviction list; process this buffer now
4366 4366 * but let arc_do_user_evicts() do the reaping.
4367 4367 */
4368 4368 buf->b_efunc = NULL;
4369 4369 mutex_exit(&buf->b_evict_lock);
4370 4370 VERIFY0(efunc(private));
4371 4371 return (B_TRUE);
4372 4372 }
4373 4373 hash_lock = HDR_LOCK(hdr);
4374 4374 mutex_enter(hash_lock);
4375 4375 hdr = buf->b_hdr;
4376 4376 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4377 4377
4378 4378 ASSERT3U(refcount_count(&hdr->b_l1hdr.b_refcnt), <,
4379 4379 hdr->b_l1hdr.b_datacnt);
4380 4380 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4381 4381 hdr->b_l1hdr.b_state == arc_mfu);
4382 4382
4383 4383 buf->b_efunc = NULL;
4384 4384 buf->b_private = NULL;
4385 4385
4386 4386 if (hdr->b_l1hdr.b_datacnt > 1) {
4387 4387 mutex_exit(&buf->b_evict_lock);
4388 4388 arc_buf_destroy(buf, TRUE);
4389 4389 } else {
4390 4390 ASSERT(buf == hdr->b_l1hdr.b_buf);
4391 4391 hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
4392 4392 mutex_exit(&buf->b_evict_lock);
4393 4393 }
4394 4394
4395 4395 mutex_exit(hash_lock);
4396 4396 VERIFY0(efunc(private));
4397 4397 return (B_TRUE);
4398 4398 }
4399 4399
4400 4400 /*
4401 4401 * Release this buffer from the cache, making it an anonymous buffer. This
4402 4402 * must be done after a read and prior to modifying the buffer contents.
4403 4403 * If the buffer has more than one reference, we must make
4404 4404 * a new hdr for the buffer.
4405 4405 */
4406 4406 void
4407 4407 arc_release(arc_buf_t *buf, void *tag)
4408 4408 {
4409 4409 arc_buf_hdr_t *hdr = buf->b_hdr;
4410 4410
4411 4411 /*
4412 4412 * It would be nice to assert that if it's DMU metadata (level >
4413 4413 * 0 || it's the dnode file), then it must be syncing context.
4414 4414 * But we don't know that information at this level.
4415 4415 */
4416 4416
4417 4417 mutex_enter(&buf->b_evict_lock);
4418 4418
4419 4419 ASSERT(HDR_HAS_L1HDR(hdr));
4420 4420
4421 4421 /*
4422 4422 * We don't grab the hash lock prior to this check, because if
4423 4423 * the buffer's header is in the arc_anon state, it won't be
4424 4424 * linked into the hash table.
4425 4425 */
4426 4426 if (hdr->b_l1hdr.b_state == arc_anon) {
4427 4427 mutex_exit(&buf->b_evict_lock);
4428 4428 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4429 4429 ASSERT(!HDR_IN_HASH_TABLE(hdr));
4430 4430 ASSERT(!HDR_HAS_L2HDR(hdr));
4431 4431 ASSERT(BUF_EMPTY(hdr));
4432 4432
4433 4433 ASSERT3U(hdr->b_l1hdr.b_datacnt, ==, 1);
4434 4434 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
4435 4435 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
4436 4436
4437 4437 ASSERT3P(buf->b_efunc, ==, NULL);
4438 4438 ASSERT3P(buf->b_private, ==, NULL);
4439 4439
4440 4440 hdr->b_l1hdr.b_arc_access = 0;
4441 4441 arc_buf_thaw(buf);
4442 4442
4443 4443 return;
4444 4444 }
4445 4445
4446 4446 kmutex_t *hash_lock = HDR_LOCK(hdr);
4447 4447 mutex_enter(hash_lock);
4448 4448
4449 4449 /*
4450 4450 * This assignment is only valid as long as the hash_lock is
4451 4451 * held, we must be careful not to reference state or the
4452 4452 * b_state field after dropping the lock.
4453 4453 */
4454 4454 arc_state_t *state = hdr->b_l1hdr.b_state;
4455 4455 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4456 4456 ASSERT3P(state, !=, arc_anon);
4457 4457
4458 4458 /* this buffer is not on any list */
4459 4459 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
4460 4460
4461 4461 if (HDR_HAS_L2HDR(hdr)) {
4462 4462 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4463 4463
4464 4464 /*
4465 4465 * We have to recheck this conditional again now that
4466 4466 * we're holding the l2ad_mtx to prevent a race with
4467 4467 * another thread which might be concurrently calling
4468 4468 * l2arc_evict(). In that case, l2arc_evict() might have
4469 4469 * destroyed the header's L2 portion as we were waiting
4470 4470 * to acquire the l2ad_mtx.
4471 4471 */
4472 4472 if (HDR_HAS_L2HDR(hdr))
4473 4473 arc_hdr_l2hdr_destroy(hdr);
4474 4474
4475 4475 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4476 4476 }
4477 4477
4478 4478 /*
4479 4479 * Do we have more than one buf?
4480 4480 */
4481 4481 if (hdr->b_l1hdr.b_datacnt > 1) {
4482 4482 arc_buf_hdr_t *nhdr;
4483 4483 arc_buf_t **bufp;
4484 4484 uint64_t blksz = hdr->b_size;
4485 4485 uint64_t spa = hdr->b_spa;
4486 4486 arc_buf_contents_t type = arc_buf_type(hdr);
4487 4487 uint32_t flags = hdr->b_flags;
4488 4488
4489 4489 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
4490 4490 /*
4491 4491 * Pull the data off of this hdr and attach it to
4492 4492 * a new anonymous hdr.
4493 4493 */
4494 4494 (void) remove_reference(hdr, hash_lock, tag);
4495 4495 bufp = &hdr->b_l1hdr.b_buf;
4496 4496 while (*bufp != buf)
4497 4497 bufp = &(*bufp)->b_next;
4498 4498 *bufp = buf->b_next;
4499 4499 buf->b_next = NULL;
4500 4500
4501 4501 ASSERT3P(state, !=, arc_l2c_only);
4502 4502
4503 4503 (void) refcount_remove_many(
4504 4504 &state->arcs_size, hdr->b_size, buf);
4505 4505
4506 4506 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
4507 4507 ASSERT3P(state, !=, arc_l2c_only);
4508 4508 uint64_t *size = &state->arcs_lsize[type];
4509 4509 ASSERT3U(*size, >=, hdr->b_size);
4510 4510 atomic_add_64(size, -hdr->b_size);
4511 4511 }
4512 4512
4513 4513 /*
4514 4514 * We're releasing a duplicate user data buffer, update
4515 4515 * our statistics accordingly.
4516 4516 */
4517 4517 if (HDR_ISTYPE_DATA(hdr)) {
4518 4518 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
4519 4519 ARCSTAT_INCR(arcstat_duplicate_buffers_size,
4520 4520 -hdr->b_size);
4521 4521 }
4522 4522 hdr->b_l1hdr.b_datacnt -= 1;
4523 4523 arc_cksum_verify(buf);
4524 4524 arc_buf_unwatch(buf);
4525 4525
4526 4526 mutex_exit(hash_lock);
4527 4527
4528 4528 nhdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
4529 4529 nhdr->b_size = blksz;
4530 4530 nhdr->b_spa = spa;
4531 4531
4532 4532 nhdr->b_flags = flags & ARC_FLAG_L2_WRITING;
4533 4533 nhdr->b_flags |= arc_bufc_to_flags(type);
4534 4534 nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
4535 4535
4536 4536 nhdr->b_l1hdr.b_buf = buf;
4537 4537 nhdr->b_l1hdr.b_datacnt = 1;
4538 4538 nhdr->b_l1hdr.b_state = arc_anon;
4539 4539 nhdr->b_l1hdr.b_arc_access = 0;
4540 4540 nhdr->b_l1hdr.b_tmp_cdata = NULL;
4541 4541 nhdr->b_freeze_cksum = NULL;
4542 4542
4543 4543 (void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
4544 4544 buf->b_hdr = nhdr;
4545 4545 mutex_exit(&buf->b_evict_lock);
4546 4546 (void) refcount_add_many(&arc_anon->arcs_size, blksz, buf);
4547 4547 } else {
4548 4548 mutex_exit(&buf->b_evict_lock);
4549 4549 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
4550 4550 /* protected by hash lock, or hdr is on arc_anon */
4551 4551 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4552 4552 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4553 4553 arc_change_state(arc_anon, hdr, hash_lock);
4554 4554 hdr->b_l1hdr.b_arc_access = 0;
4555 4555 mutex_exit(hash_lock);
4556 4556
4557 4557 buf_discard_identity(hdr);
4558 4558 arc_buf_thaw(buf);
4559 4559 }
4560 4560 buf->b_efunc = NULL;
4561 4561 buf->b_private = NULL;
4562 4562 }
4563 4563
4564 4564 int
4565 4565 arc_released(arc_buf_t *buf)
4566 4566 {
4567 4567 int released;
4568 4568
4569 4569 mutex_enter(&buf->b_evict_lock);
4570 4570 released = (buf->b_data != NULL &&
4571 4571 buf->b_hdr->b_l1hdr.b_state == arc_anon);
4572 4572 mutex_exit(&buf->b_evict_lock);
4573 4573 return (released);
4574 4574 }
4575 4575
4576 4576 #ifdef ZFS_DEBUG
4577 4577 int
4578 4578 arc_referenced(arc_buf_t *buf)
4579 4579 {
4580 4580 int referenced;
4581 4581
4582 4582 mutex_enter(&buf->b_evict_lock);
4583 4583 referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
4584 4584 mutex_exit(&buf->b_evict_lock);
4585 4585 return (referenced);
4586 4586 }
4587 4587 #endif
4588 4588
4589 4589 static void
4590 4590 arc_write_ready(zio_t *zio)
4591 4591 {
4592 4592 arc_write_callback_t *callback = zio->io_private;
4593 4593 arc_buf_t *buf = callback->awcb_buf;
4594 4594 arc_buf_hdr_t *hdr = buf->b_hdr;
4595 4595
4596 4596 ASSERT(HDR_HAS_L1HDR(hdr));
4597 4597 ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
4598 4598 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4599 4599 callback->awcb_ready(zio, buf, callback->awcb_private);
4600 4600
4601 4601 /*
4602 4602 * If the IO is already in progress, then this is a re-write
4603 4603 * attempt, so we need to thaw and re-compute the cksum.
4604 4604 * It is the responsibility of the callback to handle the
4605 4605 * accounting for any re-write attempt.
4606 4606 */
4607 4607 if (HDR_IO_IN_PROGRESS(hdr)) {
4608 4608 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
4609 4609 if (hdr->b_freeze_cksum != NULL) {
4610 4610 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
4611 4611 hdr->b_freeze_cksum = NULL;
4612 4612 }
4613 4613 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
4614 4614 }
4615 4615 arc_cksum_compute(buf, B_FALSE);
4616 4616 hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4617 4617 }
4618 4618
4619 4619 /*
4620 4620 * The SPA calls this callback for each physical write that happens on behalf
4621 4621 * of a logical write. See the comment in dbuf_write_physdone() for details.
4622 4622 */
4623 4623 static void
4624 4624 arc_write_physdone(zio_t *zio)
4625 4625 {
4626 4626 arc_write_callback_t *cb = zio->io_private;
4627 4627 if (cb->awcb_physdone != NULL)
4628 4628 cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
4629 4629 }
4630 4630
4631 4631 static void
4632 4632 arc_write_done(zio_t *zio)
4633 4633 {
4634 4634 arc_write_callback_t *callback = zio->io_private;
4635 4635 arc_buf_t *buf = callback->awcb_buf;
4636 4636 arc_buf_hdr_t *hdr = buf->b_hdr;
4637 4637
4638 4638 ASSERT(hdr->b_l1hdr.b_acb == NULL);
4639 4639
4640 4640 if (zio->io_error == 0) {
4641 4641 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
4642 4642 buf_discard_identity(hdr);
4643 4643 } else {
4644 4644 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
4645 4645 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
4646 4646 }
4647 4647 } else {
4648 4648 ASSERT(BUF_EMPTY(hdr));
4649 4649 }
4650 4650
4651 4651 /*
4652 4652 * If the block to be written was all-zero or compressed enough to be
4653 4653 * embedded in the BP, no write was performed so there will be no
4654 4654 * dva/birth/checksum. The buffer must therefore remain anonymous
4655 4655 * (and uncached).
4656 4656 */
4657 4657 if (!BUF_EMPTY(hdr)) {
4658 4658 arc_buf_hdr_t *exists;
4659 4659 kmutex_t *hash_lock;
4660 4660
4661 4661 ASSERT(zio->io_error == 0);
4662 4662
4663 4663 arc_cksum_verify(buf);
4664 4664
4665 4665 exists = buf_hash_insert(hdr, &hash_lock);
4666 4666 if (exists != NULL) {
4667 4667 /*
4668 4668 * This can only happen if we overwrite for
4669 4669 * sync-to-convergence, because we remove
4670 4670 * buffers from the hash table when we arc_free().
4671 4671 */
4672 4672 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
4673 4673 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4674 4674 panic("bad overwrite, hdr=%p exists=%p",
4675 4675 (void *)hdr, (void *)exists);
4676 4676 ASSERT(refcount_is_zero(
4677 4677 &exists->b_l1hdr.b_refcnt));
4678 4678 arc_change_state(arc_anon, exists, hash_lock);
4679 4679 mutex_exit(hash_lock);
4680 4680 arc_hdr_destroy(exists);
4681 4681 exists = buf_hash_insert(hdr, &hash_lock);
4682 4682 ASSERT3P(exists, ==, NULL);
4683 4683 } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
4684 4684 /* nopwrite */
4685 4685 ASSERT(zio->io_prop.zp_nopwrite);
4686 4686 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4687 4687 panic("bad nopwrite, hdr=%p exists=%p",
4688 4688 (void *)hdr, (void *)exists);
4689 4689 } else {
4690 4690 /* Dedup */
4691 4691 ASSERT(hdr->b_l1hdr.b_datacnt == 1);
4692 4692 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
4693 4693 ASSERT(BP_GET_DEDUP(zio->io_bp));
4694 4694 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
4695 4695 }
4696 4696 }
4697 4697 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4698 4698 /* if it's not anon, we are doing a scrub */
4699 4699 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
4700 4700 arc_access(hdr, hash_lock);
4701 4701 mutex_exit(hash_lock);
4702 4702 } else {
4703 4703 hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4704 4704 }
4705 4705
4706 4706 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4707 4707 callback->awcb_done(zio, buf, callback->awcb_private);
4708 4708
4709 4709 kmem_free(callback, sizeof (arc_write_callback_t));
4710 4710 }
4711 4711
4712 4712 zio_t *
4713 4713 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
4714 4714 blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
4715 4715 const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
4716 4716 arc_done_func_t *done, void *private, zio_priority_t priority,
4717 4717 int zio_flags, const zbookmark_phys_t *zb)
4718 4718 {
4719 4719 arc_buf_hdr_t *hdr = buf->b_hdr;
4720 4720 arc_write_callback_t *callback;
4721 4721 zio_t *zio;
4722 4722
4723 4723 ASSERT(ready != NULL);
4724 4724 ASSERT(done != NULL);
4725 4725 ASSERT(!HDR_IO_ERROR(hdr));
4726 4726 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4727 4727 ASSERT(hdr->b_l1hdr.b_acb == NULL);
4728 4728 ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4729 4729 if (l2arc)
4730 4730 hdr->b_flags |= ARC_FLAG_L2CACHE;
4731 4731 if (l2arc_compress)
4732 4732 hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4733 4733 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
4734 4734 callback->awcb_ready = ready;
4735 4735 callback->awcb_physdone = physdone;
4736 4736 callback->awcb_done = done;
4737 4737 callback->awcb_private = private;
4738 4738 callback->awcb_buf = buf;
4739 4739
4740 4740 zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
4741 4741 arc_write_ready, arc_write_physdone, arc_write_done, callback,
4742 4742 priority, zio_flags, zb);
4743 4743
4744 4744 return (zio);
4745 4745 }
4746 4746
4747 4747 static int
4748 4748 arc_memory_throttle(uint64_t reserve, uint64_t txg)
4749 4749 {
4750 4750 #ifdef _KERNEL
4751 4751 uint64_t available_memory = ptob(freemem);
4752 4752 static uint64_t page_load = 0;
4753 4753 static uint64_t last_txg = 0;
4754 4754
4755 4755 #if defined(__i386)
4756 4756 available_memory =
4757 4757 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
4758 4758 #endif
4759 4759
4760 4760 if (freemem > physmem * arc_lotsfree_percent / 100)
4761 4761 return (0);
4762 4762
4763 4763 if (txg > last_txg) {
4764 4764 last_txg = txg;
4765 4765 page_load = 0;
4766 4766 }
4767 4767 /*
4768 4768 * If we are in pageout, we know that memory is already tight,
4769 4769 * the arc is already going to be evicting, so we just want to
4770 4770 * continue to let page writes occur as quickly as possible.
4771 4771 */
4772 4772 if (curproc == proc_pageout) {
4773 4773 if (page_load > MAX(ptob(minfree), available_memory) / 4)
4774 4774 return (SET_ERROR(ERESTART));
4775 4775 /* Note: reserve is inflated, so we deflate */
4776 4776 page_load += reserve / 8;
4777 4777 return (0);
4778 4778 } else if (page_load > 0 && arc_reclaim_needed()) {
4779 4779 /* memory is low, delay before restarting */
4780 4780 ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
4781 4781 return (SET_ERROR(EAGAIN));
4782 4782 }
4783 4783 page_load = 0;
4784 4784 #endif
4785 4785 return (0);
4786 4786 }
4787 4787
4788 4788 void
4789 4789 arc_tempreserve_clear(uint64_t reserve)
4790 4790 {
4791 4791 atomic_add_64(&arc_tempreserve, -reserve);
4792 4792 ASSERT((int64_t)arc_tempreserve >= 0);
4793 4793 }
4794 4794
4795 4795 int
4796 4796 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
4797 4797 {
4798 4798 int error;
4799 4799 uint64_t anon_size;
4800 4800
4801 4801 if (reserve > arc_c/4 && !arc_no_grow)
4802 4802 arc_c = MIN(arc_c_max, reserve * 4);
4803 4803 if (reserve > arc_c)
4804 4804 return (SET_ERROR(ENOMEM));
4805 4805
4806 4806 /*
4807 4807 * Don't count loaned bufs as in flight dirty data to prevent long
4808 4808 * network delays from blocking transactions that are ready to be
4809 4809 * assigned to a txg.
4810 4810 */
4811 4811 anon_size = MAX((int64_t)(refcount_count(&arc_anon->arcs_size) -
4812 4812 arc_loaned_bytes), 0);
4813 4813
4814 4814 /*
4815 4815 * Writes will, almost always, require additional memory allocations
4816 4816 * in order to compress/encrypt/etc the data. We therefore need to
4817 4817 * make sure that there is sufficient available memory for this.
4818 4818 */
4819 4819 error = arc_memory_throttle(reserve, txg);
4820 4820 if (error != 0)
4821 4821 return (error);
4822 4822
4823 4823 /*
4824 4824 * Throttle writes when the amount of dirty data in the cache
4825 4825 * gets too large. We try to keep the cache less than half full
4826 4826 * of dirty blocks so that our sync times don't grow too large.
4827 4827 * Note: if two requests come in concurrently, we might let them
4828 4828 * both succeed, when one of them should fail. Not a huge deal.
4829 4829 */
4830 4830
4831 4831 if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
4832 4832 anon_size > arc_c / 4) {
4833 4833 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
4834 4834 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
4835 4835 arc_tempreserve>>10,
4836 4836 arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
4837 4837 arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
4838 4838 reserve>>10, arc_c>>10);
4839 4839 return (SET_ERROR(ERESTART));
4840 4840 }
4841 4841 atomic_add_64(&arc_tempreserve, reserve);
4842 4842 return (0);
4843 4843 }
4844 4844
4845 4845 static void
4846 4846 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
4847 4847 kstat_named_t *evict_data, kstat_named_t *evict_metadata)
4848 4848 {
4849 4849 size->value.ui64 = refcount_count(&state->arcs_size);
4850 4850 evict_data->value.ui64 = state->arcs_lsize[ARC_BUFC_DATA];
4851 4851 evict_metadata->value.ui64 = state->arcs_lsize[ARC_BUFC_METADATA];
4852 4852 }
4853 4853
4854 4854 static int
4855 4855 arc_kstat_update(kstat_t *ksp, int rw)
4856 4856 {
4857 4857 arc_stats_t *as = ksp->ks_data;
4858 4858
4859 4859 if (rw == KSTAT_WRITE) {
4860 4860 return (EACCES);
4861 4861 } else {
4862 4862 arc_kstat_update_state(arc_anon,
4863 4863 &as->arcstat_anon_size,
4864 4864 &as->arcstat_anon_evictable_data,
4865 4865 &as->arcstat_anon_evictable_metadata);
4866 4866 arc_kstat_update_state(arc_mru,
4867 4867 &as->arcstat_mru_size,
4868 4868 &as->arcstat_mru_evictable_data,
4869 4869 &as->arcstat_mru_evictable_metadata);
4870 4870 arc_kstat_update_state(arc_mru_ghost,
4871 4871 &as->arcstat_mru_ghost_size,
4872 4872 &as->arcstat_mru_ghost_evictable_data,
4873 4873 &as->arcstat_mru_ghost_evictable_metadata);
4874 4874 arc_kstat_update_state(arc_mfu,
4875 4875 &as->arcstat_mfu_size,
4876 4876 &as->arcstat_mfu_evictable_data,
4877 4877 &as->arcstat_mfu_evictable_metadata);
4878 4878 arc_kstat_update_state(arc_mfu_ghost,
4879 4879 &as->arcstat_mfu_ghost_size,
4880 4880 &as->arcstat_mfu_ghost_evictable_data,
4881 4881 &as->arcstat_mfu_ghost_evictable_metadata);
4882 4882 }
4883 4883
4884 4884 return (0);
4885 4885 }
4886 4886
4887 4887 /*
4888 4888 * This function *must* return indices evenly distributed between all
4889 4889 * sublists of the multilist. This is needed due to how the ARC eviction
4890 4890 * code is laid out; arc_evict_state() assumes ARC buffers are evenly
4891 4891 * distributed between all sublists and uses this assumption when
4892 4892 * deciding which sublist to evict from and how much to evict from it.
4893 4893 */
4894 4894 unsigned int
4895 4895 arc_state_multilist_index_func(multilist_t *ml, void *obj)
4896 4896 {
4897 4897 arc_buf_hdr_t *hdr = obj;
4898 4898
4899 4899 /*
4900 4900 * We rely on b_dva to generate evenly distributed index
4901 4901 * numbers using buf_hash below. So, as an added precaution,
4902 4902 * let's make sure we never add empty buffers to the arc lists.
4903 4903 */
4904 4904 ASSERT(!BUF_EMPTY(hdr));
4905 4905
4906 4906 /*
4907 4907 * The assumption here, is the hash value for a given
4908 4908 * arc_buf_hdr_t will remain constant throughout it's lifetime
4909 4909 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
4910 4910 * Thus, we don't need to store the header's sublist index
4911 4911 * on insertion, as this index can be recalculated on removal.
4912 4912 *
4913 4913 * Also, the low order bits of the hash value are thought to be
4914 4914 * distributed evenly. Otherwise, in the case that the multilist
4915 4915 * has a power of two number of sublists, each sublists' usage
4916 4916 * would not be evenly distributed.
4917 4917 */
4918 4918 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
4919 4919 multilist_get_num_sublists(ml));
4920 4920 }
4921 4921
4922 4922 void
4923 4923 arc_init(void)
4924 4924 {
4925 4925 /*
4926 4926 * allmem is "all memory that we could possibly use".
4927 4927 */
4928 4928 #ifdef _KERNEL
4929 4929 uint64_t allmem = ptob(physmem - swapfs_minfree);
4930 4930 #else
4931 4931 uint64_t allmem = (physmem * PAGESIZE) / 2;
4932 4932 #endif
4933 4933
4934 4934 mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
4935 4935 cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
4936 4936 cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
4937 4937
4938 4938 mutex_init(&arc_user_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
4939 4939 cv_init(&arc_user_evicts_cv, NULL, CV_DEFAULT, NULL);
4940 4940
4941 4941 /* Convert seconds to clock ticks */
4942 4942 arc_min_prefetch_lifespan = 1 * hz;
4943 4943
4944 4944 /* Start out with 1/8 of all memory */
4945 4945 arc_c = allmem / 8;
4946 4946
4947 4947 #ifdef _KERNEL
4948 4948 /*
4949 4949 * On architectures where the physical memory can be larger
4950 4950 * than the addressable space (intel in 32-bit mode), we may
4951 4951 * need to limit the cache to 1/8 of VM size.
4952 4952 */
4953 4953 arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
4954 4954 #endif
4955 4955
4956 4956 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */
4957 4957 arc_c_min = MAX(allmem / 32, 64 << 20);
4958 4958 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */
4959 4959 if (allmem >= 1 << 30)
4960 4960 arc_c_max = allmem - (1 << 30);
4961 4961 else
4962 4962 arc_c_max = arc_c_min;
4963 4963 arc_c_max = MAX(allmem * 3 / 4, arc_c_max);
4964 4964
4965 4965 /*
4966 4966 * Allow the tunables to override our calculations if they are
4967 4967 * reasonable (ie. over 64MB)
4968 4968 */
4969 4969 if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem)
4970 4970 arc_c_max = zfs_arc_max;
4971 4971 if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max)
4972 4972 arc_c_min = zfs_arc_min;
4973 4973
4974 4974 arc_c = arc_c_max;
4975 4975 arc_p = (arc_c >> 1);
4976 4976
4977 4977 /* limit meta-data to 1/4 of the arc capacity */
4978 4978 arc_meta_limit = arc_c_max / 4;
4979 4979
4980 4980 /* Allow the tunable to override if it is reasonable */
4981 4981 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
4982 4982 arc_meta_limit = zfs_arc_meta_limit;
4983 4983
4984 4984 if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
4985 4985 arc_c_min = arc_meta_limit / 2;
4986 4986
4987 4987 if (zfs_arc_meta_min > 0) {
4988 4988 arc_meta_min = zfs_arc_meta_min;
4989 4989 } else {
4990 4990 arc_meta_min = arc_c_min / 2;
4991 4991 }
4992 4992
4993 4993 if (zfs_arc_grow_retry > 0)
4994 4994 arc_grow_retry = zfs_arc_grow_retry;
4995 4995
4996 4996 if (zfs_arc_shrink_shift > 0)
4997 4997 arc_shrink_shift = zfs_arc_shrink_shift;
4998 4998
4999 4999 /*
5000 5000 * Ensure that arc_no_grow_shift is less than arc_shrink_shift.
5001 5001 */
5002 5002 if (arc_no_grow_shift >= arc_shrink_shift)
5003 5003 arc_no_grow_shift = arc_shrink_shift - 1;
5004 5004
5005 5005 if (zfs_arc_p_min_shift > 0)
5006 5006 arc_p_min_shift = zfs_arc_p_min_shift;
5007 5007
5008 5008 if (zfs_arc_num_sublists_per_state < 1)
5009 5009 zfs_arc_num_sublists_per_state = MAX(boot_ncpus, 1);
5010 5010
5011 5011 /* if kmem_flags are set, lets try to use less memory */
5012 5012 if (kmem_debugging())
5013 5013 arc_c = arc_c / 2;
5014 5014 if (arc_c < arc_c_min)
5015 5015 arc_c = arc_c_min;
5016 5016
5017 5017 arc_anon = &ARC_anon;
5018 5018 arc_mru = &ARC_mru;
5019 5019 arc_mru_ghost = &ARC_mru_ghost;
5020 5020 arc_mfu = &ARC_mfu;
5021 5021 arc_mfu_ghost = &ARC_mfu_ghost;
5022 5022 arc_l2c_only = &ARC_l2c_only;
5023 5023 arc_size = 0;
5024 5024
5025 5025 multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
5026 5026 sizeof (arc_buf_hdr_t),
5027 5027 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5028 5028 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5029 5029 multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
5030 5030 sizeof (arc_buf_hdr_t),
5031 5031 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5032 5032 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5033 5033 multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
5034 5034 sizeof (arc_buf_hdr_t),
5035 5035 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5036 5036 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5037 5037 multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
5038 5038 sizeof (arc_buf_hdr_t),
5039 5039 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5040 5040 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5041 5041 multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
5042 5042 sizeof (arc_buf_hdr_t),
5043 5043 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5044 5044 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5045 5045 multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
5046 5046 sizeof (arc_buf_hdr_t),
5047 5047 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5048 5048 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5049 5049 multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
5050 5050 sizeof (arc_buf_hdr_t),
5051 5051 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5052 5052 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5053 5053 multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
5054 5054 sizeof (arc_buf_hdr_t),
5055 5055 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5056 5056 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5057 5057 multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
5058 5058 sizeof (arc_buf_hdr_t),
5059 5059 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5060 5060 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5061 5061 multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
5062 5062 sizeof (arc_buf_hdr_t),
5063 5063 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
5064 5064 zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
5065 5065
5066 5066 refcount_create(&arc_anon->arcs_size);
5067 5067 refcount_create(&arc_mru->arcs_size);
5068 5068 refcount_create(&arc_mru_ghost->arcs_size);
5069 5069 refcount_create(&arc_mfu->arcs_size);
5070 5070 refcount_create(&arc_mfu_ghost->arcs_size);
5071 5071 refcount_create(&arc_l2c_only->arcs_size);
5072 5072
5073 5073 buf_init();
5074 5074
5075 5075 arc_reclaim_thread_exit = FALSE;
5076 5076 arc_user_evicts_thread_exit = FALSE;
5077 5077 arc_eviction_list = NULL;
5078 5078 bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
5079 5079
5080 5080 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
5081 5081 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
5082 5082
5083 5083 if (arc_ksp != NULL) {
5084 5084 arc_ksp->ks_data = &arc_stats;
5085 5085 arc_ksp->ks_update = arc_kstat_update;
5086 5086 kstat_install(arc_ksp);
5087 5087 }
5088 5088
5089 5089 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
5090 5090 TS_RUN, minclsyspri);
5091 5091
5092 5092 (void) thread_create(NULL, 0, arc_user_evicts_thread, NULL, 0, &p0,
5093 5093 TS_RUN, minclsyspri);
5094 5094
5095 5095 arc_dead = FALSE;
5096 5096 arc_warm = B_FALSE;
5097 5097
5098 5098 /*
5099 5099 * Calculate maximum amount of dirty data per pool.
5100 5100 *
5101 5101 * If it has been set by /etc/system, take that.
5102 5102 * Otherwise, use a percentage of physical memory defined by
5103 5103 * zfs_dirty_data_max_percent (default 10%) with a cap at
5104 5104 * zfs_dirty_data_max_max (default 4GB).
5105 5105 */
5106 5106 if (zfs_dirty_data_max == 0) {
5107 5107 zfs_dirty_data_max = physmem * PAGESIZE *
5108 5108 zfs_dirty_data_max_percent / 100;
5109 5109 zfs_dirty_data_max = MIN(zfs_dirty_data_max,
5110 5110 zfs_dirty_data_max_max);
5111 5111 }
5112 5112 }
5113 5113
5114 5114 void
5115 5115 arc_fini(void)
5116 5116 {
5117 5117 mutex_enter(&arc_reclaim_lock);
5118 5118 arc_reclaim_thread_exit = TRUE;
5119 5119 /*
5120 5120 * The reclaim thread will set arc_reclaim_thread_exit back to
5121 5121 * FALSE when it is finished exiting; we're waiting for that.
5122 5122 */
5123 5123 while (arc_reclaim_thread_exit) {
5124 5124 cv_signal(&arc_reclaim_thread_cv);
5125 5125 cv_wait(&arc_reclaim_thread_cv, &arc_reclaim_lock);
5126 5126 }
5127 5127 mutex_exit(&arc_reclaim_lock);
5128 5128
5129 5129 mutex_enter(&arc_user_evicts_lock);
5130 5130 arc_user_evicts_thread_exit = TRUE;
5131 5131 /*
5132 5132 * The user evicts thread will set arc_user_evicts_thread_exit
5133 5133 * to FALSE when it is finished exiting; we're waiting for that.
5134 5134 */
5135 5135 while (arc_user_evicts_thread_exit) {
5136 5136 cv_signal(&arc_user_evicts_cv);
5137 5137 cv_wait(&arc_user_evicts_cv, &arc_user_evicts_lock);
5138 5138 }
5139 5139 mutex_exit(&arc_user_evicts_lock);
5140 5140
5141 5141 /* Use TRUE to ensure *all* buffers are evicted */
5142 5142 arc_flush(NULL, TRUE);
5143 5143
5144 5144 arc_dead = TRUE;
5145 5145
5146 5146 if (arc_ksp != NULL) {
5147 5147 kstat_delete(arc_ksp);
5148 5148 arc_ksp = NULL;
5149 5149 }
5150 5150
5151 5151 mutex_destroy(&arc_reclaim_lock);
5152 5152 cv_destroy(&arc_reclaim_thread_cv);
5153 5153 cv_destroy(&arc_reclaim_waiters_cv);
5154 5154
5155 5155 mutex_destroy(&arc_user_evicts_lock);
5156 5156 cv_destroy(&arc_user_evicts_cv);
5157 5157
5158 5158 refcount_destroy(&arc_anon->arcs_size);
5159 5159 refcount_destroy(&arc_mru->arcs_size);
5160 5160 refcount_destroy(&arc_mru_ghost->arcs_size);
5161 5161 refcount_destroy(&arc_mfu->arcs_size);
5162 5162 refcount_destroy(&arc_mfu_ghost->arcs_size);
5163 5163 refcount_destroy(&arc_l2c_only->arcs_size);
5164 5164
5165 5165 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
5166 5166 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
5167 5167 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
5168 5168 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
5169 5169 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
5170 5170 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
5171 5171 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
5172 5172 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
5173 5173
5174 5174 buf_fini();
5175 5175
5176 5176 ASSERT0(arc_loaned_bytes);
5177 5177 }
5178 5178
5179 5179 /*
5180 5180 * Level 2 ARC
5181 5181 *
5182 5182 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
5183 5183 * It uses dedicated storage devices to hold cached data, which are populated
5184 5184 * using large infrequent writes. The main role of this cache is to boost
5185 5185 * the performance of random read workloads. The intended L2ARC devices
5186 5186 * include short-stroked disks, solid state disks, and other media with
5187 5187 * substantially faster read latency than disk.
5188 5188 *
5189 5189 * +-----------------------+
5190 5190 * | ARC |
5191 5191 * +-----------------------+
5192 5192 * | ^ ^
5193 5193 * | | |
5194 5194 * l2arc_feed_thread() arc_read()
5195 5195 * | | |
5196 5196 * | l2arc read |
5197 5197 * V | |
5198 5198 * +---------------+ |
5199 5199 * | L2ARC | |
5200 5200 * +---------------+ |
5201 5201 * | ^ |
5202 5202 * l2arc_write() | |
5203 5203 * | | |
5204 5204 * V | |
5205 5205 * +-------+ +-------+
5206 5206 * | vdev | | vdev |
5207 5207 * | cache | | cache |
5208 5208 * +-------+ +-------+
5209 5209 * +=========+ .-----.
5210 5210 * : L2ARC : |-_____-|
5211 5211 * : devices : | Disks |
5212 5212 * +=========+ `-_____-'
5213 5213 *
5214 5214 * Read requests are satisfied from the following sources, in order:
5215 5215 *
5216 5216 * 1) ARC
5217 5217 * 2) vdev cache of L2ARC devices
5218 5218 * 3) L2ARC devices
5219 5219 * 4) vdev cache of disks
5220 5220 * 5) disks
5221 5221 *
5222 5222 * Some L2ARC device types exhibit extremely slow write performance.
5223 5223 * To accommodate for this there are some significant differences between
5224 5224 * the L2ARC and traditional cache design:
5225 5225 *
5226 5226 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from
5227 5227 * the ARC behave as usual, freeing buffers and placing headers on ghost
5228 5228 * lists. The ARC does not send buffers to the L2ARC during eviction as
5229 5229 * this would add inflated write latencies for all ARC memory pressure.
5230 5230 *
5231 5231 * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
5232 5232 * It does this by periodically scanning buffers from the eviction-end of
5233 5233 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
5234 5234 * not already there. It scans until a headroom of buffers is satisfied,
5235 5235 * which itself is a buffer for ARC eviction. If a compressible buffer is
5236 5236 * found during scanning and selected for writing to an L2ARC device, we
5237 5237 * temporarily boost scanning headroom during the next scan cycle to make
5238 5238 * sure we adapt to compression effects (which might significantly reduce
5239 5239 * the data volume we write to L2ARC). The thread that does this is
5240 5240 * l2arc_feed_thread(), illustrated below; example sizes are included to
5241 5241 * provide a better sense of ratio than this diagram:
5242 5242 *
5243 5243 * head --> tail
5244 5244 * +---------------------+----------+
5245 5245 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC
5246 5246 * +---------------------+----------+ | o L2ARC eligible
5247 5247 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer
5248 5248 * +---------------------+----------+ |
5249 5249 * 15.9 Gbytes ^ 32 Mbytes |
5250 5250 * headroom |
5251 5251 * l2arc_feed_thread()
5252 5252 * |
5253 5253 * l2arc write hand <--[oooo]--'
5254 5254 * | 8 Mbyte
5255 5255 * | write max
5256 5256 * V
5257 5257 * +==============================+
5258 5258 * L2ARC dev |####|#|###|###| |####| ... |
5259 5259 * +==============================+
5260 5260 * 32 Gbytes
5261 5261 *
5262 5262 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
5263 5263 * evicted, then the L2ARC has cached a buffer much sooner than it probably
5264 5264 * needed to, potentially wasting L2ARC device bandwidth and storage. It is
5265 5265 * safe to say that this is an uncommon case, since buffers at the end of
5266 5266 * the ARC lists have moved there due to inactivity.
5267 5267 *
5268 5268 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
5269 5269 * then the L2ARC simply misses copying some buffers. This serves as a
5270 5270 * pressure valve to prevent heavy read workloads from both stalling the ARC
5271 5271 * with waits and clogging the L2ARC with writes. This also helps prevent
5272 5272 * the potential for the L2ARC to churn if it attempts to cache content too
5273 5273 * quickly, such as during backups of the entire pool.
5274 5274 *
5275 5275 * 5. After system boot and before the ARC has filled main memory, there are
5276 5276 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
5277 5277 * lists can remain mostly static. Instead of searching from tail of these
5278 5278 * lists as pictured, the l2arc_feed_thread() will search from the list heads
5279 5279 * for eligible buffers, greatly increasing its chance of finding them.
5280 5280 *
5281 5281 * The L2ARC device write speed is also boosted during this time so that
5282 5282 * the L2ARC warms up faster. Since there have been no ARC evictions yet,
5283 5283 * there are no L2ARC reads, and no fear of degrading read performance
5284 5284 * through increased writes.
5285 5285 *
5286 5286 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
5287 5287 * the vdev queue can aggregate them into larger and fewer writes. Each
5288 5288 * device is written to in a rotor fashion, sweeping writes through
5289 5289 * available space then repeating.
5290 5290 *
5291 5291 * 7. The L2ARC does not store dirty content. It never needs to flush
5292 5292 * write buffers back to disk based storage.
5293 5293 *
5294 5294 * 8. If an ARC buffer is written (and dirtied) which also exists in the
5295 5295 * L2ARC, the now stale L2ARC buffer is immediately dropped.
5296 5296 *
5297 5297 * The performance of the L2ARC can be tweaked by a number of tunables, which
5298 5298 * may be necessary for different workloads:
5299 5299 *
5300 5300 * l2arc_write_max max write bytes per interval
5301 5301 * l2arc_write_boost extra write bytes during device warmup
5302 5302 * l2arc_noprefetch skip caching prefetched buffers
5303 5303 * l2arc_headroom number of max device writes to precache
5304 5304 * l2arc_headroom_boost when we find compressed buffers during ARC
5305 5305 * scanning, we multiply headroom by this
5306 5306 * percentage factor for the next scan cycle,
5307 5307 * since more compressed buffers are likely to
5308 5308 * be present
5309 5309 * l2arc_feed_secs seconds between L2ARC writing
5310 5310 *
5311 5311 * Tunables may be removed or added as future performance improvements are
5312 5312 * integrated, and also may become zpool properties.
5313 5313 *
5314 5314 * There are three key functions that control how the L2ARC warms up:
5315 5315 *
5316 5316 * l2arc_write_eligible() check if a buffer is eligible to cache
5317 5317 * l2arc_write_size() calculate how much to write
5318 5318 * l2arc_write_interval() calculate sleep delay between writes
5319 5319 *
5320 5320 * These three functions determine what to write, how much, and how quickly
5321 5321 * to send writes.
5322 5322 */
5323 5323
5324 5324 static boolean_t
5325 5325 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
5326 5326 {
5327 5327 /*
5328 5328 * A buffer is *not* eligible for the L2ARC if it:
5329 5329 * 1. belongs to a different spa.
5330 5330 * 2. is already cached on the L2ARC.
5331 5331 * 3. has an I/O in progress (it may be an incomplete read).
5332 5332 * 4. is flagged not eligible (zfs property).
5333 5333 */
5334 5334 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
5335 5335 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
5336 5336 return (B_FALSE);
5337 5337
5338 5338 return (B_TRUE);
5339 5339 }
5340 5340
5341 5341 static uint64_t
5342 5342 l2arc_write_size(void)
5343 5343 {
5344 5344 uint64_t size;
5345 5345
5346 5346 /*
5347 5347 * Make sure our globals have meaningful values in case the user
5348 5348 * altered them.
5349 5349 */
5350 5350 size = l2arc_write_max;
5351 5351 if (size == 0) {
5352 5352 cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
5353 5353 "be greater than zero, resetting it to the default (%d)",
5354 5354 L2ARC_WRITE_SIZE);
5355 5355 size = l2arc_write_max = L2ARC_WRITE_SIZE;
5356 5356 }
5357 5357
5358 5358 if (arc_warm == B_FALSE)
5359 5359 size += l2arc_write_boost;
5360 5360
5361 5361 return (size);
5362 5362
5363 5363 }
5364 5364
5365 5365 static clock_t
5366 5366 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
5367 5367 {
5368 5368 clock_t interval, next, now;
5369 5369
5370 5370 /*
5371 5371 * If the ARC lists are busy, increase our write rate; if the
5372 5372 * lists are stale, idle back. This is achieved by checking
5373 5373 * how much we previously wrote - if it was more than half of
5374 5374 * what we wanted, schedule the next write much sooner.
5375 5375 */
5376 5376 if (l2arc_feed_again && wrote > (wanted / 2))
5377 5377 interval = (hz * l2arc_feed_min_ms) / 1000;
5378 5378 else
5379 5379 interval = hz * l2arc_feed_secs;
5380 5380
5381 5381 now = ddi_get_lbolt();
5382 5382 next = MAX(now, MIN(now + interval, began + interval));
5383 5383
5384 5384 return (next);
5385 5385 }
5386 5386
5387 5387 /*
5388 5388 * Cycle through L2ARC devices. This is how L2ARC load balances.
5389 5389 * If a device is returned, this also returns holding the spa config lock.
5390 5390 */
5391 5391 static l2arc_dev_t *
5392 5392 l2arc_dev_get_next(void)
5393 5393 {
5394 5394 l2arc_dev_t *first, *next = NULL;
5395 5395
5396 5396 /*
5397 5397 * Lock out the removal of spas (spa_namespace_lock), then removal
5398 5398 * of cache devices (l2arc_dev_mtx). Once a device has been selected,
5399 5399 * both locks will be dropped and a spa config lock held instead.
5400 5400 */
5401 5401 mutex_enter(&spa_namespace_lock);
5402 5402 mutex_enter(&l2arc_dev_mtx);
5403 5403
5404 5404 /* if there are no vdevs, there is nothing to do */
5405 5405 if (l2arc_ndev == 0)
5406 5406 goto out;
5407 5407
5408 5408 first = NULL;
5409 5409 next = l2arc_dev_last;
5410 5410 do {
5411 5411 /* loop around the list looking for a non-faulted vdev */
5412 5412 if (next == NULL) {
5413 5413 next = list_head(l2arc_dev_list);
5414 5414 } else {
5415 5415 next = list_next(l2arc_dev_list, next);
5416 5416 if (next == NULL)
5417 5417 next = list_head(l2arc_dev_list);
5418 5418 }
5419 5419
5420 5420 /* if we have come back to the start, bail out */
5421 5421 if (first == NULL)
5422 5422 first = next;
5423 5423 else if (next == first)
5424 5424 break;
5425 5425
5426 5426 } while (vdev_is_dead(next->l2ad_vdev));
5427 5427
5428 5428 /* if we were unable to find any usable vdevs, return NULL */
5429 5429 if (vdev_is_dead(next->l2ad_vdev))
5430 5430 next = NULL;
5431 5431
5432 5432 l2arc_dev_last = next;
5433 5433
5434 5434 out:
5435 5435 mutex_exit(&l2arc_dev_mtx);
5436 5436
5437 5437 /*
5438 5438 * Grab the config lock to prevent the 'next' device from being
5439 5439 * removed while we are writing to it.
5440 5440 */
5441 5441 if (next != NULL)
5442 5442 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
5443 5443 mutex_exit(&spa_namespace_lock);
5444 5444
5445 5445 return (next);
5446 5446 }
5447 5447
5448 5448 /*
5449 5449 * Free buffers that were tagged for destruction.
5450 5450 */
5451 5451 static void
5452 5452 l2arc_do_free_on_write()
5453 5453 {
5454 5454 list_t *buflist;
5455 5455 l2arc_data_free_t *df, *df_prev;
5456 5456
5457 5457 mutex_enter(&l2arc_free_on_write_mtx);
5458 5458 buflist = l2arc_free_on_write;
5459 5459
5460 5460 for (df = list_tail(buflist); df; df = df_prev) {
5461 5461 df_prev = list_prev(buflist, df);
5462 5462 ASSERT(df->l2df_data != NULL);
5463 5463 ASSERT(df->l2df_func != NULL);
5464 5464 df->l2df_func(df->l2df_data, df->l2df_size);
5465 5465 list_remove(buflist, df);
5466 5466 kmem_free(df, sizeof (l2arc_data_free_t));
5467 5467 }
5468 5468
5469 5469 mutex_exit(&l2arc_free_on_write_mtx);
5470 5470 }
5471 5471
5472 5472 /*
5473 5473 * A write to a cache device has completed. Update all headers to allow
5474 5474 * reads from these buffers to begin.
5475 5475 */
5476 5476 static void
5477 5477 l2arc_write_done(zio_t *zio)
5478 5478 {
5479 5479 l2arc_write_callback_t *cb;
5480 5480 l2arc_dev_t *dev;
5481 5481 list_t *buflist;
5482 5482 arc_buf_hdr_t *head, *hdr, *hdr_prev;
5483 5483 kmutex_t *hash_lock;
5484 5484 int64_t bytes_dropped = 0;
5485 5485
5486 5486 cb = zio->io_private;
5487 5487 ASSERT(cb != NULL);
5488 5488 dev = cb->l2wcb_dev;
5489 5489 ASSERT(dev != NULL);
5490 5490 head = cb->l2wcb_head;
5491 5491 ASSERT(head != NULL);
5492 5492 buflist = &dev->l2ad_buflist;
5493 5493 ASSERT(buflist != NULL);
5494 5494 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
5495 5495 l2arc_write_callback_t *, cb);
5496 5496
5497 5497 if (zio->io_error != 0)
5498 5498 ARCSTAT_BUMP(arcstat_l2_writes_error);
5499 5499
5500 5500 /*
5501 5501 * All writes completed, or an error was hit.
5502 5502 */
5503 5503 top:
5504 5504 mutex_enter(&dev->l2ad_mtx);
5505 5505 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
5506 5506 hdr_prev = list_prev(buflist, hdr);
5507 5507
5508 5508 hash_lock = HDR_LOCK(hdr);
5509 5509
5510 5510 /*
5511 5511 * We cannot use mutex_enter or else we can deadlock
5512 5512 * with l2arc_write_buffers (due to swapping the order
5513 5513 * the hash lock and l2ad_mtx are taken).
5514 5514 */
5515 5515 if (!mutex_tryenter(hash_lock)) {
5516 5516 /*
5517 5517 * Missed the hash lock. We must retry so we
5518 5518 * don't leave the ARC_FLAG_L2_WRITING bit set.
5519 5519 */
5520 5520 ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
5521 5521
5522 5522 /*
5523 5523 * We don't want to rescan the headers we've
5524 5524 * already marked as having been written out, so
5525 5525 * we reinsert the head node so we can pick up
5526 5526 * where we left off.
5527 5527 */
5528 5528 list_remove(buflist, head);
5529 5529 list_insert_after(buflist, hdr, head);
5530 5530
5531 5531 mutex_exit(&dev->l2ad_mtx);
5532 5532
5533 5533 /*
5534 5534 * We wait for the hash lock to become available
5535 5535 * to try and prevent busy waiting, and increase
5536 5536 * the chance we'll be able to acquire the lock
5537 5537 * the next time around.
5538 5538 */
5539 5539 mutex_enter(hash_lock);
5540 5540 mutex_exit(hash_lock);
5541 5541 goto top;
5542 5542 }
5543 5543
5544 5544 /*
5545 5545 * We could not have been moved into the arc_l2c_only
5546 5546 * state while in-flight due to our ARC_FLAG_L2_WRITING
5547 5547 * bit being set. Let's just ensure that's being enforced.
5548 5548 */
5549 5549 ASSERT(HDR_HAS_L1HDR(hdr));
5550 5550
5551 5551 /*
5552 5552 * We may have allocated a buffer for L2ARC compression,
5553 5553 * we must release it to avoid leaking this data.
5554 5554 */
5555 5555 l2arc_release_cdata_buf(hdr);
5556 5556
5557 5557 if (zio->io_error != 0) {
5558 5558 /*
5559 5559 * Error - drop L2ARC entry.
5560 5560 */
5561 5561 list_remove(buflist, hdr);
5562 5562 hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
5563 5563
5564 5564 ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
5565 5565 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
5566 5566
5567 5567 bytes_dropped += hdr->b_l2hdr.b_asize;
5568 5568 (void) refcount_remove_many(&dev->l2ad_alloc,
5569 5569 hdr->b_l2hdr.b_asize, hdr);
5570 5570 }
5571 5571
5572 5572 /*
5573 5573 * Allow ARC to begin reads and ghost list evictions to
5574 5574 * this L2ARC entry.
5575 5575 */
5576 5576 hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
5577 5577
5578 5578 mutex_exit(hash_lock);
5579 5579 }
5580 5580
5581 5581 atomic_inc_64(&l2arc_writes_done);
5582 5582 list_remove(buflist, head);
5583 5583 ASSERT(!HDR_HAS_L1HDR(head));
5584 5584 kmem_cache_free(hdr_l2only_cache, head);
5585 5585 mutex_exit(&dev->l2ad_mtx);
5586 5586
5587 5587 vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
5588 5588
5589 5589 l2arc_do_free_on_write();
5590 5590
5591 5591 kmem_free(cb, sizeof (l2arc_write_callback_t));
5592 5592 }
5593 5593
5594 5594 /*
5595 5595 * A read to a cache device completed. Validate buffer contents before
5596 5596 * handing over to the regular ARC routines.
5597 5597 */
5598 5598 static void
5599 5599 l2arc_read_done(zio_t *zio)
5600 5600 {
5601 5601 l2arc_read_callback_t *cb;
5602 5602 arc_buf_hdr_t *hdr;
5603 5603 arc_buf_t *buf;
5604 5604 kmutex_t *hash_lock;
5605 5605 int equal;
5606 5606
5607 5607 ASSERT(zio->io_vd != NULL);
5608 5608 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
5609 5609
5610 5610 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
5611 5611
5612 5612 cb = zio->io_private;
5613 5613 ASSERT(cb != NULL);
5614 5614 buf = cb->l2rcb_buf;
5615 5615 ASSERT(buf != NULL);
5616 5616
5617 5617 hash_lock = HDR_LOCK(buf->b_hdr);
5618 5618 mutex_enter(hash_lock);
5619 5619 hdr = buf->b_hdr;
5620 5620 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5621 5621
5622 5622 /*
5623 5623 * If the buffer was compressed, decompress it first.
5624 5624 */
5625 5625 if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
5626 5626 l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
5627 5627 ASSERT(zio->io_data != NULL);
5628 5628 ASSERT3U(zio->io_size, ==, hdr->b_size);
5629 5629 ASSERT3U(BP_GET_LSIZE(&cb->l2rcb_bp), ==, hdr->b_size);
5630 5630
5631 5631 /*
5632 5632 * Check this survived the L2ARC journey.
5633 5633 */
5634 5634 equal = arc_cksum_equal(buf);
5635 5635 if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
5636 5636 mutex_exit(hash_lock);
5637 5637 zio->io_private = buf;
5638 5638 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */
5639 5639 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */
5640 5640 arc_read_done(zio);
5641 5641 } else {
5642 5642 mutex_exit(hash_lock);
5643 5643 /*
5644 5644 * Buffer didn't survive caching. Increment stats and
5645 5645 * reissue to the original storage device.
5646 5646 */
5647 5647 if (zio->io_error != 0) {
5648 5648 ARCSTAT_BUMP(arcstat_l2_io_error);
5649 5649 } else {
5650 5650 zio->io_error = SET_ERROR(EIO);
5651 5651 }
5652 5652 if (!equal)
5653 5653 ARCSTAT_BUMP(arcstat_l2_cksum_bad);
5654 5654
5655 5655 /*
5656 5656 * If there's no waiter, issue an async i/o to the primary
5657 5657 * storage now. If there *is* a waiter, the caller must
5658 5658 * issue the i/o in a context where it's OK to block.
5659 5659 */
5660 5660 if (zio->io_waiter == NULL) {
5661 5661 zio_t *pio = zio_unique_parent(zio);
5662 5662
5663 5663 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
5664 5664
5665 5665 zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
5666 5666 buf->b_data, hdr->b_size, arc_read_done, buf,
5667 5667 zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
5668 5668 }
5669 5669 }
5670 5670
5671 5671 kmem_free(cb, sizeof (l2arc_read_callback_t));
5672 5672 }
5673 5673
5674 5674 /*
5675 5675 * This is the list priority from which the L2ARC will search for pages to
5676 5676 * cache. This is used within loops (0..3) to cycle through lists in the
5677 5677 * desired order. This order can have a significant effect on cache
5678 5678 * performance.
5679 5679 *
5680 5680 * Currently the metadata lists are hit first, MFU then MRU, followed by
5681 5681 * the data lists. This function returns a locked list, and also returns
5682 5682 * the lock pointer.
5683 5683 */
5684 5684 static multilist_sublist_t *
5685 5685 l2arc_sublist_lock(int list_num)
5686 5686 {
5687 5687 multilist_t *ml = NULL;
5688 5688 unsigned int idx;
5689 5689
5690 5690 ASSERT(list_num >= 0 && list_num <= 3);
5691 5691
5692 5692 switch (list_num) {
5693 5693 case 0:
5694 5694 ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
5695 5695 break;
5696 5696 case 1:
5697 5697 ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
5698 5698 break;
5699 5699 case 2:
5700 5700 ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
5701 5701 break;
5702 5702 case 3:
5703 5703 ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
5704 5704 break;
5705 5705 }
5706 5706
5707 5707 /*
5708 5708 * Return a randomly-selected sublist. This is acceptable
5709 5709 * because the caller feeds only a little bit of data for each
5710 5710 * call (8MB). Subsequent calls will result in different
5711 5711 * sublists being selected.
5712 5712 */
5713 5713 idx = multilist_get_random_index(ml);
5714 5714 return (multilist_sublist_lock(ml, idx));
5715 5715 }
5716 5716
5717 5717 /*
5718 5718 * Evict buffers from the device write hand to the distance specified in
5719 5719 * bytes. This distance may span populated buffers, it may span nothing.
5720 5720 * This is clearing a region on the L2ARC device ready for writing.
5721 5721 * If the 'all' boolean is set, every buffer is evicted.
5722 5722 */
5723 5723 static void
5724 5724 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
5725 5725 {
5726 5726 list_t *buflist;
5727 5727 arc_buf_hdr_t *hdr, *hdr_prev;
5728 5728 kmutex_t *hash_lock;
5729 5729 uint64_t taddr;
5730 5730
5731 5731 buflist = &dev->l2ad_buflist;
5732 5732
5733 5733 if (!all && dev->l2ad_first) {
5734 5734 /*
5735 5735 * This is the first sweep through the device. There is
5736 5736 * nothing to evict.
5737 5737 */
5738 5738 return;
5739 5739 }
5740 5740
5741 5741 if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
5742 5742 /*
5743 5743 * When nearing the end of the device, evict to the end
5744 5744 * before the device write hand jumps to the start.
5745 5745 */
5746 5746 taddr = dev->l2ad_end;
5747 5747 } else {
5748 5748 taddr = dev->l2ad_hand + distance;
5749 5749 }
5750 5750 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
5751 5751 uint64_t, taddr, boolean_t, all);
5752 5752
5753 5753 top:
5754 5754 mutex_enter(&dev->l2ad_mtx);
5755 5755 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
5756 5756 hdr_prev = list_prev(buflist, hdr);
5757 5757
5758 5758 hash_lock = HDR_LOCK(hdr);
5759 5759
5760 5760 /*
5761 5761 * We cannot use mutex_enter or else we can deadlock
5762 5762 * with l2arc_write_buffers (due to swapping the order
5763 5763 * the hash lock and l2ad_mtx are taken).
5764 5764 */
5765 5765 if (!mutex_tryenter(hash_lock)) {
5766 5766 /*
5767 5767 * Missed the hash lock. Retry.
5768 5768 */
5769 5769 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
5770 5770 mutex_exit(&dev->l2ad_mtx);
5771 5771 mutex_enter(hash_lock);
5772 5772 mutex_exit(hash_lock);
5773 5773 goto top;
5774 5774 }
5775 5775
5776 5776 if (HDR_L2_WRITE_HEAD(hdr)) {
5777 5777 /*
5778 5778 * We hit a write head node. Leave it for
5779 5779 * l2arc_write_done().
5780 5780 */
5781 5781 list_remove(buflist, hdr);
5782 5782 mutex_exit(hash_lock);
5783 5783 continue;
5784 5784 }
5785 5785
5786 5786 if (!all && HDR_HAS_L2HDR(hdr) &&
5787 5787 (hdr->b_l2hdr.b_daddr > taddr ||
5788 5788 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
5789 5789 /*
5790 5790 * We've evicted to the target address,
5791 5791 * or the end of the device.
5792 5792 */
5793 5793 mutex_exit(hash_lock);
5794 5794 break;
5795 5795 }
5796 5796
5797 5797 ASSERT(HDR_HAS_L2HDR(hdr));
5798 5798 if (!HDR_HAS_L1HDR(hdr)) {
5799 5799 ASSERT(!HDR_L2_READING(hdr));
5800 5800 /*
5801 5801 * This doesn't exist in the ARC. Destroy.
5802 5802 * arc_hdr_destroy() will call list_remove()
5803 5803 * and decrement arcstat_l2_size.
5804 5804 */
5805 5805 arc_change_state(arc_anon, hdr, hash_lock);
5806 5806 arc_hdr_destroy(hdr);
5807 5807 } else {
5808 5808 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
5809 5809 ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
5810 5810 /*
5811 5811 * Invalidate issued or about to be issued
5812 5812 * reads, since we may be about to write
5813 5813 * over this location.
5814 5814 */
5815 5815 if (HDR_L2_READING(hdr)) {
5816 5816 ARCSTAT_BUMP(arcstat_l2_evict_reading);
5817 5817 hdr->b_flags |= ARC_FLAG_L2_EVICTED;
5818 5818 }
5819 5819
5820 5820 /* Ensure this header has finished being written */
5821 5821 ASSERT(!HDR_L2_WRITING(hdr));
5822 5822 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
5823 5823
5824 5824 arc_hdr_l2hdr_destroy(hdr);
5825 5825 }
5826 5826 mutex_exit(hash_lock);
5827 5827 }
5828 5828 mutex_exit(&dev->l2ad_mtx);
5829 5829 }
5830 5830
5831 5831 /*
5832 5832 * Find and write ARC buffers to the L2ARC device.
5833 5833 *
5834 5834 * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
5835 5835 * for reading until they have completed writing.
5836 5836 * The headroom_boost is an in-out parameter used to maintain headroom boost
|
↓ open down ↓ |
5836 lines elided |
↑ open up ↑ |
5837 5837 * state between calls to this function.
5838 5838 *
5839 5839 * Returns the number of bytes actually written (which may be smaller than
5840 5840 * the delta by which the device hand has changed due to alignment).
5841 5841 */
5842 5842 static uint64_t
5843 5843 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
5844 5844 boolean_t *headroom_boost)
5845 5845 {
5846 5846 arc_buf_hdr_t *hdr, *hdr_prev, *head;
5847 - uint64_t write_asize, write_psize, write_sz, headroom,
5847 + uint64_t write_asize, write_sz, headroom,
5848 5848 buf_compress_minsz;
5849 5849 void *buf_data;
5850 5850 boolean_t full;
5851 5851 l2arc_write_callback_t *cb;
5852 5852 zio_t *pio, *wzio;
5853 5853 uint64_t guid = spa_load_guid(spa);
5854 5854 const boolean_t do_headroom_boost = *headroom_boost;
5855 5855
5856 5856 ASSERT(dev->l2ad_vdev != NULL);
5857 5857
5858 5858 /* Lower the flag now, we might want to raise it again later. */
5859 5859 *headroom_boost = B_FALSE;
5860 5860
5861 5861 pio = NULL;
5862 - write_sz = write_asize = write_psize = 0;
5862 + write_sz = write_asize = 0;
5863 5863 full = B_FALSE;
5864 5864 head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
5865 5865 head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
5866 5866 head->b_flags |= ARC_FLAG_HAS_L2HDR;
5867 5867
5868 5868 /*
5869 5869 * We will want to try to compress buffers that are at least 2x the
5870 5870 * device sector size.
5871 5871 */
5872 5872 buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift;
5873 5873
5874 5874 /*
5875 5875 * Copy buffers for L2ARC writing.
5876 5876 */
5877 5877 for (int try = 0; try <= 3; try++) {
5878 5878 multilist_sublist_t *mls = l2arc_sublist_lock(try);
5879 5879 uint64_t passed_sz = 0;
5880 5880
5881 5881 /*
5882 5882 * L2ARC fast warmup.
5883 5883 *
5884 5884 * Until the ARC is warm and starts to evict, read from the
5885 5885 * head of the ARC lists rather than the tail.
5886 5886 */
5887 5887 if (arc_warm == B_FALSE)
5888 5888 hdr = multilist_sublist_head(mls);
|
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
5889 5889 else
5890 5890 hdr = multilist_sublist_tail(mls);
5891 5891
5892 5892 headroom = target_sz * l2arc_headroom;
5893 5893 if (do_headroom_boost)
5894 5894 headroom = (headroom * l2arc_headroom_boost) / 100;
5895 5895
5896 5896 for (; hdr; hdr = hdr_prev) {
5897 5897 kmutex_t *hash_lock;
5898 5898 uint64_t buf_sz;
5899 + uint64_t buf_a_sz;
5899 5900
5900 5901 if (arc_warm == B_FALSE)
5901 5902 hdr_prev = multilist_sublist_next(mls, hdr);
5902 5903 else
5903 5904 hdr_prev = multilist_sublist_prev(mls, hdr);
5904 5905
5905 5906 hash_lock = HDR_LOCK(hdr);
5906 5907 if (!mutex_tryenter(hash_lock)) {
5907 5908 /*
5908 5909 * Skip this buffer rather than waiting.
5909 5910 */
5910 5911 continue;
5911 5912 }
5912 5913
5913 5914 passed_sz += hdr->b_size;
5914 5915 if (passed_sz > headroom) {
5915 5916 /*
5916 5917 * Searched too far.
|
↓ open down ↓ |
8 lines elided |
↑ open up ↑ |
5917 5918 */
5918 5919 mutex_exit(hash_lock);
5919 5920 break;
5920 5921 }
5921 5922
5922 5923 if (!l2arc_write_eligible(guid, hdr)) {
5923 5924 mutex_exit(hash_lock);
5924 5925 continue;
5925 5926 }
5926 5927
5927 - if ((write_sz + hdr->b_size) > target_sz) {
5928 + /*
5929 + * Assume that the buffer is not going to be compressed
5930 + * and could take more space on disk because of a larger
5931 + * disk block size.
5932 + */
5933 + buf_sz = hdr->b_size;
5934 + buf_a_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
5935 +
5936 + if ((write_asize + buf_a_sz) > target_sz) {
5928 5937 full = B_TRUE;
5929 5938 mutex_exit(hash_lock);
5930 5939 break;
5931 5940 }
5932 5941
5933 5942 if (pio == NULL) {
5934 5943 /*
5935 5944 * Insert a dummy header on the buflist so
5936 5945 * l2arc_write_done() can find where the
5937 5946 * write buffers begin without searching.
5938 5947 */
5939 5948 mutex_enter(&dev->l2ad_mtx);
5940 5949 list_insert_head(&dev->l2ad_buflist, head);
5941 5950 mutex_exit(&dev->l2ad_mtx);
5942 5951
5943 5952 cb = kmem_alloc(
5944 5953 sizeof (l2arc_write_callback_t), KM_SLEEP);
5945 5954 cb->l2wcb_dev = dev;
5946 5955 cb->l2wcb_head = head;
5947 5956 pio = zio_root(spa, l2arc_write_done, cb,
5948 5957 ZIO_FLAG_CANFAIL);
5949 5958 }
5950 5959
5951 5960 /*
5952 5961 * Create and add a new L2ARC header.
5953 5962 */
5954 5963 hdr->b_l2hdr.b_dev = dev;
5955 5964 hdr->b_flags |= ARC_FLAG_L2_WRITING;
5956 5965 /*
5957 5966 * Temporarily stash the data buffer in b_tmp_cdata.
5958 5967 * The subsequent write step will pick it up from
5959 5968 * there. This is because can't access b_l1hdr.b_buf
5960 5969 * without holding the hash_lock, which we in turn
5961 5970 * can't access without holding the ARC list locks
5962 5971 * (which we want to avoid during compression/writing).
5963 5972 */
5964 5973 hdr->b_l2hdr.b_compress = ZIO_COMPRESS_OFF;
5965 5974 hdr->b_l2hdr.b_asize = hdr->b_size;
5966 5975 hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
5967 5976
5968 5977 /*
5969 5978 * Explicitly set the b_daddr field to a known
5970 5979 * value which means "invalid address". This
5971 5980 * enables us to differentiate which stage of
5972 5981 * l2arc_write_buffers() the particular header
5973 5982 * is in (e.g. this loop, or the one below).
5974 5983 * ARC_FLAG_L2_WRITING is not enough to make
5975 5984 * this distinction, and we need to know in
5976 5985 * order to do proper l2arc vdev accounting in
5977 5986 * arc_release() and arc_hdr_destroy().
5978 5987 *
5979 5988 * Note, we can't use a new flag to distinguish
5980 5989 * the two stages because we don't hold the
|
↓ open down ↓ |
43 lines elided |
↑ open up ↑ |
5981 5990 * header's hash_lock below, in the second stage
5982 5991 * of this function. Thus, we can't simply
5983 5992 * change the b_flags field to denote that the
5984 5993 * IO has been sent. We can change the b_daddr
5985 5994 * field of the L2 portion, though, since we'll
5986 5995 * be holding the l2ad_mtx; which is why we're
5987 5996 * using it to denote the header's state change.
5988 5997 */
5989 5998 hdr->b_l2hdr.b_daddr = L2ARC_ADDR_UNSET;
5990 5999
5991 - buf_sz = hdr->b_size;
5992 6000 hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
5993 6001
5994 6002 mutex_enter(&dev->l2ad_mtx);
5995 6003 list_insert_head(&dev->l2ad_buflist, hdr);
5996 6004 mutex_exit(&dev->l2ad_mtx);
5997 6005
5998 6006 /*
5999 6007 * Compute and store the buffer cksum before
6000 6008 * writing. On debug the cksum is verified first.
6001 6009 */
6002 6010 arc_cksum_verify(hdr->b_l1hdr.b_buf);
6003 6011 arc_cksum_compute(hdr->b_l1hdr.b_buf, B_TRUE);
6004 6012
6005 6013 mutex_exit(hash_lock);
6006 6014
6007 6015 write_sz += buf_sz;
6016 + write_asize += buf_a_sz;
6008 6017 }
6009 6018
6010 6019 multilist_sublist_unlock(mls);
6011 6020
6012 6021 if (full == B_TRUE)
6013 6022 break;
6014 6023 }
6015 6024
6016 6025 /* No buffers selected for writing? */
6017 6026 if (pio == NULL) {
6018 6027 ASSERT0(write_sz);
6019 6028 ASSERT(!HDR_HAS_L1HDR(head));
6020 6029 kmem_cache_free(hdr_l2only_cache, head);
6021 6030 return (0);
6022 6031 }
6023 6032
6024 6033 mutex_enter(&dev->l2ad_mtx);
6025 6034
6026 6035 /*
6036 + * Note that elsewhere in this file arcstat_l2_asize
6037 + * and the used space on l2ad_vdev are updated using b_asize,
6038 + * which is not necessarily rounded up to the device block size.
6039 + * Too keep accounting consistent we do the same here as well:
6040 + * stats_size accumulates the sum of b_asize of the written buffers,
6041 + * while write_asize accumulates the sum of b_asize rounded up
6042 + * to the device block size.
6043 + * The latter sum is used only to validate the corectness of the code.
6044 + */
6045 + uint64_t stats_size = 0;
6046 + write_asize = 0;
6047 +
6048 + /*
6027 6049 * Now start writing the buffers. We're starting at the write head
6028 6050 * and work backwards, retracing the course of the buffer selector
6029 6051 * loop above.
6030 6052 */
6031 6053 for (hdr = list_prev(&dev->l2ad_buflist, head); hdr;
6032 6054 hdr = list_prev(&dev->l2ad_buflist, hdr)) {
6033 6055 uint64_t buf_sz;
6034 6056
6035 6057 /*
6036 6058 * We rely on the L1 portion of the header below, so
6037 6059 * it's invalid for this header to have been evicted out
6038 6060 * of the ghost cache, prior to being written out. The
6039 6061 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
6040 6062 */
6041 6063 ASSERT(HDR_HAS_L1HDR(hdr));
6042 6064
6043 6065 /*
6044 6066 * We shouldn't need to lock the buffer here, since we flagged
6045 6067 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
6046 6068 * take care to only access its L2 cache parameters. In
6047 6069 * particular, hdr->l1hdr.b_buf may be invalid by now due to
6048 6070 * ARC eviction.
6049 6071 */
6050 6072 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
6051 6073
6052 6074 if ((HDR_L2COMPRESS(hdr)) &&
6053 6075 hdr->b_l2hdr.b_asize >= buf_compress_minsz) {
6054 6076 if (l2arc_compress_buf(hdr)) {
6055 6077 /*
6056 6078 * If compression succeeded, enable headroom
6057 6079 * boost on the next scan cycle.
6058 6080 */
6059 6081 *headroom_boost = B_TRUE;
6060 6082 }
6061 6083 }
6062 6084
6063 6085 /*
6064 6086 * Pick up the buffer data we had previously stashed away
6065 6087 * (and now potentially also compressed).
6066 6088 */
6067 6089 buf_data = hdr->b_l1hdr.b_tmp_cdata;
6068 6090 buf_sz = hdr->b_l2hdr.b_asize;
|
↓ open down ↓ |
32 lines elided |
↑ open up ↑ |
6069 6091
6070 6092 /*
6071 6093 * We need to do this regardless if buf_sz is zero or
6072 6094 * not, otherwise, when this l2hdr is evicted we'll
6073 6095 * remove a reference that was never added.
6074 6096 */
6075 6097 (void) refcount_add_many(&dev->l2ad_alloc, buf_sz, hdr);
6076 6098
6077 6099 /* Compression may have squashed the buffer to zero length. */
6078 6100 if (buf_sz != 0) {
6079 - uint64_t buf_p_sz;
6101 + uint64_t buf_a_sz;
6080 6102
6081 6103 wzio = zio_write_phys(pio, dev->l2ad_vdev,
6082 6104 dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
6083 6105 NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
6084 6106 ZIO_FLAG_CANFAIL, B_FALSE);
6085 6107
6086 6108 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
6087 6109 zio_t *, wzio);
6088 6110 (void) zio_nowait(wzio);
6089 6111
6090 - write_asize += buf_sz;
6112 + stats_size += buf_sz;
6091 6113
6092 6114 /*
6093 6115 * Keep the clock hand suitably device-aligned.
6094 6116 */
6095 - buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
6096 - write_psize += buf_p_sz;
6097 - dev->l2ad_hand += buf_p_sz;
6117 + buf_a_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
6118 + write_asize += buf_a_sz;
6119 + dev->l2ad_hand += buf_a_sz;
6098 6120 }
6099 6121 }
6100 6122
6101 6123 mutex_exit(&dev->l2ad_mtx);
6102 6124
6103 6125 ASSERT3U(write_asize, <=, target_sz);
6104 6126 ARCSTAT_BUMP(arcstat_l2_writes_sent);
6105 6127 ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
6106 6128 ARCSTAT_INCR(arcstat_l2_size, write_sz);
6107 - ARCSTAT_INCR(arcstat_l2_asize, write_asize);
6108 - vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
6129 + ARCSTAT_INCR(arcstat_l2_asize, stats_size);
6130 + vdev_space_update(dev->l2ad_vdev, stats_size, 0, 0);
6109 6131
6110 6132 /*
6111 6133 * Bump device hand to the device start if it is approaching the end.
6112 6134 * l2arc_evict() will already have evicted ahead for this case.
6113 6135 */
6114 6136 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
6115 6137 dev->l2ad_hand = dev->l2ad_start;
6116 6138 dev->l2ad_first = B_FALSE;
6117 6139 }
6118 6140
6119 6141 dev->l2ad_writing = B_TRUE;
6120 6142 (void) zio_wait(pio);
6121 6143 dev->l2ad_writing = B_FALSE;
6122 6144
6123 6145 return (write_asize);
6124 6146 }
6125 6147
6126 6148 /*
6127 6149 * Compresses an L2ARC buffer.
6128 6150 * The data to be compressed must be prefilled in l1hdr.b_tmp_cdata and its
6129 6151 * size in l2hdr->b_asize. This routine tries to compress the data and
6130 6152 * depending on the compression result there are three possible outcomes:
6131 6153 * *) The buffer was incompressible. The original l2hdr contents were left
6132 6154 * untouched and are ready for writing to an L2 device.
6133 6155 * *) The buffer was all-zeros, so there is no need to write it to an L2
6134 6156 * device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is
6135 6157 * set to zero and b_compress is set to ZIO_COMPRESS_EMPTY.
6136 6158 * *) Compression succeeded and b_tmp_cdata was replaced with a temporary
6137 6159 * data buffer which holds the compressed data to be written, and b_asize
6138 6160 * tells us how much data there is. b_compress is set to the appropriate
6139 6161 * compression algorithm. Once writing is done, invoke
6140 6162 * l2arc_release_cdata_buf on this l2hdr to free this temporary buffer.
6141 6163 *
6142 6164 * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the
6143 6165 * buffer was incompressible).
6144 6166 */
6145 6167 static boolean_t
6146 6168 l2arc_compress_buf(arc_buf_hdr_t *hdr)
6147 6169 {
6148 6170 void *cdata;
6149 6171 size_t csize, len, rounded;
6150 6172 ASSERT(HDR_HAS_L2HDR(hdr));
6151 6173 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
6152 6174
6153 6175 ASSERT(HDR_HAS_L1HDR(hdr));
6154 6176 ASSERT3S(l2hdr->b_compress, ==, ZIO_COMPRESS_OFF);
6155 6177 ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
6156 6178
6157 6179 len = l2hdr->b_asize;
6158 6180 cdata = zio_data_buf_alloc(len);
6159 6181 ASSERT3P(cdata, !=, NULL);
6160 6182 csize = zio_compress_data(ZIO_COMPRESS_LZ4, hdr->b_l1hdr.b_tmp_cdata,
6161 6183 cdata, l2hdr->b_asize);
6162 6184
6163 6185 rounded = P2ROUNDUP(csize, (size_t)SPA_MINBLOCKSIZE);
6164 6186 if (rounded > csize) {
6165 6187 bzero((char *)cdata + csize, rounded - csize);
6166 6188 csize = rounded;
6167 6189 }
6168 6190
6169 6191 if (csize == 0) {
6170 6192 /* zero block, indicate that there's nothing to write */
6171 6193 zio_data_buf_free(cdata, len);
6172 6194 l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
6173 6195 l2hdr->b_asize = 0;
6174 6196 hdr->b_l1hdr.b_tmp_cdata = NULL;
6175 6197 ARCSTAT_BUMP(arcstat_l2_compress_zeros);
6176 6198 return (B_TRUE);
6177 6199 } else if (csize > 0 && csize < len) {
6178 6200 /*
6179 6201 * Compression succeeded, we'll keep the cdata around for
6180 6202 * writing and release it afterwards.
6181 6203 */
6182 6204 l2hdr->b_compress = ZIO_COMPRESS_LZ4;
6183 6205 l2hdr->b_asize = csize;
6184 6206 hdr->b_l1hdr.b_tmp_cdata = cdata;
6185 6207 ARCSTAT_BUMP(arcstat_l2_compress_successes);
6186 6208 return (B_TRUE);
6187 6209 } else {
6188 6210 /*
6189 6211 * Compression failed, release the compressed buffer.
6190 6212 * l2hdr will be left unmodified.
6191 6213 */
6192 6214 zio_data_buf_free(cdata, len);
6193 6215 ARCSTAT_BUMP(arcstat_l2_compress_failures);
6194 6216 return (B_FALSE);
6195 6217 }
6196 6218 }
6197 6219
6198 6220 /*
6199 6221 * Decompresses a zio read back from an l2arc device. On success, the
6200 6222 * underlying zio's io_data buffer is overwritten by the uncompressed
6201 6223 * version. On decompression error (corrupt compressed stream), the
6202 6224 * zio->io_error value is set to signal an I/O error.
6203 6225 *
6204 6226 * Please note that the compressed data stream is not checksummed, so
6205 6227 * if the underlying device is experiencing data corruption, we may feed
6206 6228 * corrupt data to the decompressor, so the decompressor needs to be
6207 6229 * able to handle this situation (LZ4 does).
6208 6230 */
6209 6231 static void
6210 6232 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
6211 6233 {
6212 6234 ASSERT(L2ARC_IS_VALID_COMPRESS(c));
6213 6235
6214 6236 if (zio->io_error != 0) {
6215 6237 /*
6216 6238 * An io error has occured, just restore the original io
6217 6239 * size in preparation for a main pool read.
6218 6240 */
6219 6241 zio->io_orig_size = zio->io_size = hdr->b_size;
6220 6242 return;
6221 6243 }
6222 6244
6223 6245 if (c == ZIO_COMPRESS_EMPTY) {
6224 6246 /*
6225 6247 * An empty buffer results in a null zio, which means we
6226 6248 * need to fill its io_data after we're done restoring the
6227 6249 * buffer's contents.
6228 6250 */
6229 6251 ASSERT(hdr->b_l1hdr.b_buf != NULL);
6230 6252 bzero(hdr->b_l1hdr.b_buf->b_data, hdr->b_size);
6231 6253 zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_buf->b_data;
6232 6254 } else {
6233 6255 ASSERT(zio->io_data != NULL);
6234 6256 /*
6235 6257 * We copy the compressed data from the start of the arc buffer
6236 6258 * (the zio_read will have pulled in only what we need, the
6237 6259 * rest is garbage which we will overwrite at decompression)
6238 6260 * and then decompress back to the ARC data buffer. This way we
6239 6261 * can minimize copying by simply decompressing back over the
6240 6262 * original compressed data (rather than decompressing to an
6241 6263 * aux buffer and then copying back the uncompressed buffer,
6242 6264 * which is likely to be much larger).
6243 6265 */
6244 6266 uint64_t csize;
6245 6267 void *cdata;
6246 6268
6247 6269 csize = zio->io_size;
6248 6270 cdata = zio_data_buf_alloc(csize);
6249 6271 bcopy(zio->io_data, cdata, csize);
6250 6272 if (zio_decompress_data(c, cdata, zio->io_data, csize,
6251 6273 hdr->b_size) != 0)
6252 6274 zio->io_error = EIO;
6253 6275 zio_data_buf_free(cdata, csize);
6254 6276 }
6255 6277
6256 6278 /* Restore the expected uncompressed IO size. */
6257 6279 zio->io_orig_size = zio->io_size = hdr->b_size;
6258 6280 }
6259 6281
6260 6282 /*
6261 6283 * Releases the temporary b_tmp_cdata buffer in an l2arc header structure.
6262 6284 * This buffer serves as a temporary holder of compressed data while
6263 6285 * the buffer entry is being written to an l2arc device. Once that is
6264 6286 * done, we can dispose of it.
6265 6287 */
6266 6288 static void
6267 6289 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
6268 6290 {
6269 6291 ASSERT(HDR_HAS_L2HDR(hdr));
6270 6292 enum zio_compress comp = hdr->b_l2hdr.b_compress;
6271 6293
6272 6294 ASSERT(HDR_HAS_L1HDR(hdr));
6273 6295 ASSERT(comp == ZIO_COMPRESS_OFF || L2ARC_IS_VALID_COMPRESS(comp));
6274 6296
6275 6297 if (comp == ZIO_COMPRESS_OFF) {
6276 6298 /*
6277 6299 * In this case, b_tmp_cdata points to the same buffer
6278 6300 * as the arc_buf_t's b_data field. We don't want to
6279 6301 * free it, since the arc_buf_t will handle that.
6280 6302 */
6281 6303 hdr->b_l1hdr.b_tmp_cdata = NULL;
6282 6304 } else if (comp == ZIO_COMPRESS_EMPTY) {
6283 6305 /*
6284 6306 * In this case, b_tmp_cdata was compressed to an empty
6285 6307 * buffer, thus there's nothing to free and b_tmp_cdata
6286 6308 * should have been set to NULL in l2arc_write_buffers().
6287 6309 */
6288 6310 ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
6289 6311 } else {
6290 6312 /*
6291 6313 * If the data was compressed, then we've allocated a
6292 6314 * temporary buffer for it, so now we need to release it.
6293 6315 */
6294 6316 ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
6295 6317 zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
6296 6318 hdr->b_size);
6297 6319 hdr->b_l1hdr.b_tmp_cdata = NULL;
6298 6320 }
6299 6321
6300 6322 }
6301 6323
6302 6324 /*
6303 6325 * This thread feeds the L2ARC at regular intervals. This is the beating
6304 6326 * heart of the L2ARC.
6305 6327 */
6306 6328 static void
6307 6329 l2arc_feed_thread(void)
6308 6330 {
6309 6331 callb_cpr_t cpr;
6310 6332 l2arc_dev_t *dev;
6311 6333 spa_t *spa;
6312 6334 uint64_t size, wrote;
6313 6335 clock_t begin, next = ddi_get_lbolt();
6314 6336 boolean_t headroom_boost = B_FALSE;
6315 6337
6316 6338 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
6317 6339
6318 6340 mutex_enter(&l2arc_feed_thr_lock);
6319 6341
6320 6342 while (l2arc_thread_exit == 0) {
6321 6343 CALLB_CPR_SAFE_BEGIN(&cpr);
6322 6344 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
6323 6345 next);
6324 6346 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
6325 6347 next = ddi_get_lbolt() + hz;
6326 6348
6327 6349 /*
6328 6350 * Quick check for L2ARC devices.
6329 6351 */
6330 6352 mutex_enter(&l2arc_dev_mtx);
6331 6353 if (l2arc_ndev == 0) {
6332 6354 mutex_exit(&l2arc_dev_mtx);
6333 6355 continue;
6334 6356 }
6335 6357 mutex_exit(&l2arc_dev_mtx);
6336 6358 begin = ddi_get_lbolt();
6337 6359
6338 6360 /*
6339 6361 * This selects the next l2arc device to write to, and in
6340 6362 * doing so the next spa to feed from: dev->l2ad_spa. This
6341 6363 * will return NULL if there are now no l2arc devices or if
6342 6364 * they are all faulted.
6343 6365 *
6344 6366 * If a device is returned, its spa's config lock is also
6345 6367 * held to prevent device removal. l2arc_dev_get_next()
6346 6368 * will grab and release l2arc_dev_mtx.
6347 6369 */
6348 6370 if ((dev = l2arc_dev_get_next()) == NULL)
6349 6371 continue;
6350 6372
6351 6373 spa = dev->l2ad_spa;
6352 6374 ASSERT(spa != NULL);
6353 6375
6354 6376 /*
6355 6377 * If the pool is read-only then force the feed thread to
6356 6378 * sleep a little longer.
6357 6379 */
6358 6380 if (!spa_writeable(spa)) {
6359 6381 next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
6360 6382 spa_config_exit(spa, SCL_L2ARC, dev);
6361 6383 continue;
6362 6384 }
6363 6385
6364 6386 /*
6365 6387 * Avoid contributing to memory pressure.
6366 6388 */
6367 6389 if (arc_reclaim_needed()) {
6368 6390 ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
6369 6391 spa_config_exit(spa, SCL_L2ARC, dev);
6370 6392 continue;
6371 6393 }
6372 6394
6373 6395 ARCSTAT_BUMP(arcstat_l2_feeds);
6374 6396
6375 6397 size = l2arc_write_size();
6376 6398
6377 6399 /*
6378 6400 * Evict L2ARC buffers that will be overwritten.
6379 6401 */
6380 6402 l2arc_evict(dev, size, B_FALSE);
6381 6403
6382 6404 /*
6383 6405 * Write ARC buffers.
6384 6406 */
6385 6407 wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost);
6386 6408
6387 6409 /*
6388 6410 * Calculate interval between writes.
6389 6411 */
6390 6412 next = l2arc_write_interval(begin, size, wrote);
6391 6413 spa_config_exit(spa, SCL_L2ARC, dev);
6392 6414 }
6393 6415
6394 6416 l2arc_thread_exit = 0;
6395 6417 cv_broadcast(&l2arc_feed_thr_cv);
6396 6418 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */
6397 6419 thread_exit();
6398 6420 }
6399 6421
6400 6422 boolean_t
6401 6423 l2arc_vdev_present(vdev_t *vd)
6402 6424 {
6403 6425 l2arc_dev_t *dev;
6404 6426
6405 6427 mutex_enter(&l2arc_dev_mtx);
6406 6428 for (dev = list_head(l2arc_dev_list); dev != NULL;
6407 6429 dev = list_next(l2arc_dev_list, dev)) {
6408 6430 if (dev->l2ad_vdev == vd)
6409 6431 break;
6410 6432 }
6411 6433 mutex_exit(&l2arc_dev_mtx);
6412 6434
6413 6435 return (dev != NULL);
6414 6436 }
6415 6437
6416 6438 /*
6417 6439 * Add a vdev for use by the L2ARC. By this point the spa has already
6418 6440 * validated the vdev and opened it.
6419 6441 */
6420 6442 void
6421 6443 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
6422 6444 {
6423 6445 l2arc_dev_t *adddev;
6424 6446
6425 6447 ASSERT(!l2arc_vdev_present(vd));
6426 6448
6427 6449 /*
6428 6450 * Create a new l2arc device entry.
6429 6451 */
6430 6452 adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
6431 6453 adddev->l2ad_spa = spa;
6432 6454 adddev->l2ad_vdev = vd;
6433 6455 adddev->l2ad_start = VDEV_LABEL_START_SIZE;
6434 6456 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
6435 6457 adddev->l2ad_hand = adddev->l2ad_start;
6436 6458 adddev->l2ad_first = B_TRUE;
6437 6459 adddev->l2ad_writing = B_FALSE;
6438 6460
6439 6461 mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
6440 6462 /*
6441 6463 * This is a list of all ARC buffers that are still valid on the
6442 6464 * device.
6443 6465 */
6444 6466 list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
6445 6467 offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
6446 6468
6447 6469 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
6448 6470 refcount_create(&adddev->l2ad_alloc);
6449 6471
6450 6472 /*
6451 6473 * Add device to global list
6452 6474 */
6453 6475 mutex_enter(&l2arc_dev_mtx);
6454 6476 list_insert_head(l2arc_dev_list, adddev);
6455 6477 atomic_inc_64(&l2arc_ndev);
6456 6478 mutex_exit(&l2arc_dev_mtx);
6457 6479 }
6458 6480
6459 6481 /*
6460 6482 * Remove a vdev from the L2ARC.
6461 6483 */
6462 6484 void
6463 6485 l2arc_remove_vdev(vdev_t *vd)
6464 6486 {
6465 6487 l2arc_dev_t *dev, *nextdev, *remdev = NULL;
6466 6488
6467 6489 /*
6468 6490 * Find the device by vdev
6469 6491 */
6470 6492 mutex_enter(&l2arc_dev_mtx);
6471 6493 for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
6472 6494 nextdev = list_next(l2arc_dev_list, dev);
6473 6495 if (vd == dev->l2ad_vdev) {
6474 6496 remdev = dev;
6475 6497 break;
6476 6498 }
6477 6499 }
6478 6500 ASSERT(remdev != NULL);
6479 6501
6480 6502 /*
6481 6503 * Remove device from global list
6482 6504 */
6483 6505 list_remove(l2arc_dev_list, remdev);
6484 6506 l2arc_dev_last = NULL; /* may have been invalidated */
6485 6507 atomic_dec_64(&l2arc_ndev);
6486 6508 mutex_exit(&l2arc_dev_mtx);
6487 6509
6488 6510 /*
6489 6511 * Clear all buflists and ARC references. L2ARC device flush.
6490 6512 */
6491 6513 l2arc_evict(remdev, 0, B_TRUE);
6492 6514 list_destroy(&remdev->l2ad_buflist);
6493 6515 mutex_destroy(&remdev->l2ad_mtx);
6494 6516 refcount_destroy(&remdev->l2ad_alloc);
6495 6517 kmem_free(remdev, sizeof (l2arc_dev_t));
6496 6518 }
6497 6519
6498 6520 void
6499 6521 l2arc_init(void)
6500 6522 {
6501 6523 l2arc_thread_exit = 0;
6502 6524 l2arc_ndev = 0;
6503 6525 l2arc_writes_sent = 0;
6504 6526 l2arc_writes_done = 0;
6505 6527
6506 6528 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
6507 6529 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
6508 6530 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
6509 6531 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
6510 6532
6511 6533 l2arc_dev_list = &L2ARC_dev_list;
6512 6534 l2arc_free_on_write = &L2ARC_free_on_write;
6513 6535 list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
6514 6536 offsetof(l2arc_dev_t, l2ad_node));
6515 6537 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
6516 6538 offsetof(l2arc_data_free_t, l2df_list_node));
6517 6539 }
6518 6540
6519 6541 void
6520 6542 l2arc_fini(void)
6521 6543 {
6522 6544 /*
6523 6545 * This is called from dmu_fini(), which is called from spa_fini();
6524 6546 * Because of this, we can assume that all l2arc devices have
6525 6547 * already been removed when the pools themselves were removed.
6526 6548 */
6527 6549
6528 6550 l2arc_do_free_on_write();
6529 6551
6530 6552 mutex_destroy(&l2arc_feed_thr_lock);
6531 6553 cv_destroy(&l2arc_feed_thr_cv);
6532 6554 mutex_destroy(&l2arc_dev_mtx);
6533 6555 mutex_destroy(&l2arc_free_on_write_mtx);
6534 6556
6535 6557 list_destroy(l2arc_dev_list);
6536 6558 list_destroy(l2arc_free_on_write);
6537 6559 }
6538 6560
6539 6561 void
6540 6562 l2arc_start(void)
6541 6563 {
6542 6564 if (!(spa_mode_global & FWRITE))
6543 6565 return;
6544 6566
6545 6567 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
6546 6568 TS_RUN, minclsyspri);
6547 6569 }
6548 6570
6549 6571 void
6550 6572 l2arc_stop(void)
6551 6573 {
6552 6574 if (!(spa_mode_global & FWRITE))
6553 6575 return;
6554 6576
6555 6577 mutex_enter(&l2arc_feed_thr_lock);
6556 6578 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */
6557 6579 l2arc_thread_exit = 1;
6558 6580 while (l2arc_thread_exit != 0)
6559 6581 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
6560 6582 mutex_exit(&l2arc_feed_thr_lock);
6561 6583 }
|
↓ open down ↓ |
443 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX