Print this page
10592 misc. metaslab and vdev related ZoL bug fixes
Portions contributed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: Giuseppe Di Natale <guss80@gmail.com>
Reviewed by: George Melikov <mail@gmelikov.ru>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed by: Pavel Zakharov <pavel.zakharov@delphix.com>
Reviewed by: Tony Hutter <hutter2@llnl.gov>
Reviewed by: Kody Kantor <kody.kantor@joyent.com>
Approved by: Dan McDonald <danmcd@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/vdev_indirect.c
+++ new/usr/src/uts/common/fs/zfs/vdev_indirect.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * This file and its contents are supplied under the terms of the
5 5 * Common Development and Distribution License ("CDDL"), version 1.0.
6 6 * You may only use this file in accordance with the terms of version
7 7 * 1.0 of the CDDL.
8 8 *
9 9 * A full copy of the text of the CDDL should have accompanied this
10 10 * source. A copy of the CDDL is also available via the Internet at
11 11 * http://www.illumos.org/license/CDDL.
12 12 *
13 13 * CDDL HEADER END
14 14 */
15 15
16 16 /*
17 17 * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
18 18 */
19 19
20 20 #include <sys/zfs_context.h>
21 21 #include <sys/spa.h>
22 22 #include <sys/spa_impl.h>
23 23 #include <sys/vdev_impl.h>
24 24 #include <sys/fs/zfs.h>
25 25 #include <sys/zio.h>
26 26 #include <sys/zio_checksum.h>
27 27 #include <sys/metaslab.h>
28 28 #include <sys/refcount.h>
29 29 #include <sys/dmu.h>
30 30 #include <sys/vdev_indirect_mapping.h>
31 31 #include <sys/dmu_tx.h>
32 32 #include <sys/dsl_synctask.h>
33 33 #include <sys/zap.h>
34 34 #include <sys/abd.h>
35 35 #include <sys/zthr.h>
36 36
37 37 /*
38 38 * An indirect vdev corresponds to a vdev that has been removed. Since
39 39 * we cannot rewrite block pointers of snapshots, etc., we keep a
40 40 * mapping from old location on the removed device to the new location
41 41 * on another device in the pool and use this mapping whenever we need
42 42 * to access the DVA. Unfortunately, this mapping did not respect
43 43 * logical block boundaries when it was first created, and so a DVA on
44 44 * this indirect vdev may be "split" into multiple sections that each
45 45 * map to a different location. As a consequence, not all DVAs can be
46 46 * translated to an equivalent new DVA. Instead we must provide a
47 47 * "vdev_remap" operation that executes a callback on each contiguous
48 48 * segment of the new location. This function is used in multiple ways:
49 49 *
50 50 * - i/os to this vdev use the callback to determine where the
51 51 * data is now located, and issue child i/os for each segment's new
52 52 * location.
53 53 *
54 54 * - frees and claims to this vdev use the callback to free or claim
55 55 * each mapped segment. (Note that we don't actually need to claim
56 56 * log blocks on indirect vdevs, because we don't allocate to
57 57 * removing vdevs. However, zdb uses zio_claim() for its leak
58 58 * detection.)
59 59 */
60 60
61 61 /*
62 62 * "Big theory statement" for how we mark blocks obsolete.
63 63 *
64 64 * When a block on an indirect vdev is freed or remapped, a section of
65 65 * that vdev's mapping may no longer be referenced (aka "obsolete"). We
66 66 * keep track of how much of each mapping entry is obsolete. When
67 67 * an entry becomes completely obsolete, we can remove it, thus reducing
68 68 * the memory used by the mapping. The complete picture of obsolescence
69 69 * is given by the following data structures, described below:
70 70 * - the entry-specific obsolete count
71 71 * - the vdev-specific obsolete spacemap
72 72 * - the pool-specific obsolete bpobj
73 73 *
74 74 * == On disk data structures used ==
75 75 *
76 76 * We track the obsolete space for the pool using several objects. Each
77 77 * of these objects is created on demand and freed when no longer
78 78 * needed, and is assumed to be empty if it does not exist.
79 79 * SPA_FEATURE_OBSOLETE_COUNTS includes the count of these objects.
80 80 *
81 81 * - Each vic_mapping_object (associated with an indirect vdev) can
82 82 * have a vimp_counts_object. This is an array of uint32_t's
83 83 * with the same number of entries as the vic_mapping_object. When
84 84 * the mapping is condensed, entries from the vic_obsolete_sm_object
85 85 * (see below) are folded into the counts. Therefore, each
86 86 * obsolete_counts entry tells us the number of bytes in the
87 87 * corresponding mapping entry that were not referenced when the
88 88 * mapping was last condensed.
89 89 *
90 90 * - Each indirect or removing vdev can have a vic_obsolete_sm_object.
91 91 * This is a space map containing an alloc entry for every DVA that
92 92 * has been obsoleted since the last time this indirect vdev was
93 93 * condensed. We use this object in order to improve performance
94 94 * when marking a DVA as obsolete. Instead of modifying an arbitrary
95 95 * offset of the vimp_counts_object, we only need to append an entry
96 96 * to the end of this object. When a DVA becomes obsolete, it is
97 97 * added to the obsolete space map. This happens when the DVA is
98 98 * freed, remapped and not referenced by a snapshot, or the last
99 99 * snapshot referencing it is destroyed.
100 100 *
101 101 * - Each dataset can have a ds_remap_deadlist object. This is a
102 102 * deadlist object containing all blocks that were remapped in this
103 103 * dataset but referenced in a previous snapshot. Blocks can *only*
104 104 * appear on this list if they were remapped (dsl_dataset_block_remapped);
105 105 * blocks that were killed in a head dataset are put on the normal
106 106 * ds_deadlist and marked obsolete when they are freed.
107 107 *
108 108 * - The pool can have a dp_obsolete_bpobj. This is a list of blocks
109 109 * in the pool that need to be marked obsolete. When a snapshot is
110 110 * destroyed, we move some of the ds_remap_deadlist to the obsolete
111 111 * bpobj (see dsl_destroy_snapshot_handle_remaps()). We then
112 112 * asynchronously process the obsolete bpobj, moving its entries to
113 113 * the specific vdevs' obsolete space maps.
114 114 *
115 115 * == Summary of how we mark blocks as obsolete ==
116 116 *
117 117 * - When freeing a block: if any DVA is on an indirect vdev, append to
118 118 * vic_obsolete_sm_object.
119 119 * - When remapping a block, add dva to ds_remap_deadlist (if prev snap
120 120 * references; otherwise append to vic_obsolete_sm_object).
121 121 * - When freeing a snapshot: move parts of ds_remap_deadlist to
122 122 * dp_obsolete_bpobj (same algorithm as ds_deadlist).
123 123 * - When syncing the spa: process dp_obsolete_bpobj, moving ranges to
124 124 * individual vdev's vic_obsolete_sm_object.
125 125 */
126 126
127 127 /*
128 128 * "Big theory statement" for how we condense indirect vdevs.
129 129 *
130 130 * Condensing an indirect vdev's mapping is the process of determining
131 131 * the precise counts of obsolete space for each mapping entry (by
132 132 * integrating the obsolete spacemap into the obsolete counts) and
133 133 * writing out a new mapping that contains only referenced entries.
134 134 *
135 135 * We condense a vdev when we expect the mapping to shrink (see
136 136 * vdev_indirect_should_condense()), but only perform one condense at a
137 137 * time to limit the memory usage. In addition, we use a separate
138 138 * open-context thread (spa_condense_indirect_thread) to incrementally
139 139 * create the new mapping object in a way that minimizes the impact on
140 140 * the rest of the system.
141 141 *
142 142 * == Generating a new mapping ==
143 143 *
144 144 * To generate a new mapping, we follow these steps:
145 145 *
146 146 * 1. Save the old obsolete space map and create a new mapping object
147 147 * (see spa_condense_indirect_start_sync()). This initializes the
148 148 * spa_condensing_indirect_phys with the "previous obsolete space map",
149 149 * which is now read only. Newly obsolete DVAs will be added to a
150 150 * new (initially empty) obsolete space map, and will not be
151 151 * considered as part of this condense operation.
152 152 *
153 153 * 2. Construct in memory the precise counts of obsolete space for each
154 154 * mapping entry, by incorporating the obsolete space map into the
155 155 * counts. (See vdev_indirect_mapping_load_obsolete_{counts,spacemap}().)
156 156 *
157 157 * 3. Iterate through each mapping entry, writing to the new mapping any
158 158 * entries that are not completely obsolete (i.e. which don't have
159 159 * obsolete count == mapping length). (See
160 160 * spa_condense_indirect_generate_new_mapping().)
161 161 *
162 162 * 4. Destroy the old mapping object and switch over to the new one
163 163 * (spa_condense_indirect_complete_sync).
164 164 *
165 165 * == Restarting from failure ==
166 166 *
167 167 * To restart the condense when we import/open the pool, we must start
168 168 * at the 2nd step above: reconstruct the precise counts in memory,
169 169 * based on the space map + counts. Then in the 3rd step, we start
170 170 * iterating where we left off: at vimp_max_offset of the new mapping
171 171 * object.
172 172 */
173 173
174 174 boolean_t zfs_condense_indirect_vdevs_enable = B_TRUE;
175 175
176 176 /*
177 177 * Condense if at least this percent of the bytes in the mapping is
178 178 * obsolete. With the default of 25%, the amount of space mapped
179 179 * will be reduced to 1% of its original size after at most 16
180 180 * condenses. Higher values will condense less often (causing less
181 181 * i/o); lower values will reduce the mapping size more quickly.
182 182 */
183 183 int zfs_indirect_condense_obsolete_pct = 25;
184 184
185 185 /*
186 186 * Condense if the obsolete space map takes up more than this amount of
187 187 * space on disk (logically). This limits the amount of disk space
188 188 * consumed by the obsolete space map; the default of 1GB is small enough
189 189 * that we typically don't mind "wasting" it.
190 190 */
191 191 uint64_t zfs_condense_max_obsolete_bytes = 1024 * 1024 * 1024;
192 192
193 193 /*
194 194 * Don't bother condensing if the mapping uses less than this amount of
195 195 * memory. The default of 128KB is considered a "trivial" amount of
196 196 * memory and not worth reducing.
197 197 */
198 198 uint64_t zfs_condense_min_mapping_bytes = 128 * 1024;
199 199
200 200 /*
201 201 * This is used by the test suite so that it can ensure that certain
202 202 * actions happen while in the middle of a condense (which might otherwise
203 203 * complete too quickly). If used to reduce the performance impact of
204 204 * condensing in production, a maximum value of 1 should be sufficient.
205 205 */
206 206 int zfs_condense_indirect_commit_entry_delay_ticks = 0;
207 207
208 208 /*
209 209 * If an indirect split block contains more than this many possible unique
210 210 * combinations when being reconstructed, consider it too computationally
211 211 * expensive to check them all. Instead, try at most 100 randomly-selected
212 212 * combinations each time the block is accessed. This allows all segment
213 213 * copies to participate fairly in the reconstruction when all combinations
214 214 * cannot be checked and prevents repeated use of one bad copy.
215 215 */
216 216 int zfs_reconstruct_indirect_combinations_max = 256;
217 217
218 218
219 219 /*
220 220 * Enable to simulate damaged segments and validate reconstruction.
221 221 * Used by ztest
222 222 */
223 223 unsigned long zfs_reconstruct_indirect_damage_fraction = 0;
224 224
225 225 /*
226 226 * The indirect_child_t represents the vdev that we will read from, when we
227 227 * need to read all copies of the data (e.g. for scrub or reconstruction).
228 228 * For plain (non-mirror) top-level vdevs (i.e. is_vdev is not a mirror),
229 229 * ic_vdev is the same as is_vdev. However, for mirror top-level vdevs,
230 230 * ic_vdev is a child of the mirror.
231 231 */
232 232 typedef struct indirect_child {
233 233 abd_t *ic_data;
234 234 vdev_t *ic_vdev;
235 235
236 236 /*
237 237 * ic_duplicate is NULL when the ic_data contents are unique, when it
238 238 * is determined to be a duplicate it references the primary child.
239 239 */
240 240 struct indirect_child *ic_duplicate;
241 241 list_node_t ic_node; /* node on is_unique_child */
242 242 } indirect_child_t;
243 243
244 244 /*
245 245 * The indirect_split_t represents one mapped segment of an i/o to the
246 246 * indirect vdev. For non-split (contiguously-mapped) blocks, there will be
247 247 * only one indirect_split_t, with is_split_offset==0 and is_size==io_size.
248 248 * For split blocks, there will be several of these.
249 249 */
250 250 typedef struct indirect_split {
251 251 list_node_t is_node; /* link on iv_splits */
252 252
253 253 /*
254 254 * is_split_offset is the offset into the i/o.
255 255 * This is the sum of the previous splits' is_size's.
256 256 */
257 257 uint64_t is_split_offset;
258 258
259 259 vdev_t *is_vdev; /* top-level vdev */
260 260 uint64_t is_target_offset; /* offset on is_vdev */
261 261 uint64_t is_size;
262 262 int is_children; /* number of entries in is_child[] */
263 263 int is_unique_children; /* number of entries in is_unique_child */
264 264 list_t is_unique_child;
265 265
266 266 /*
267 267 * is_good_child is the child that we are currently using to
268 268 * attempt reconstruction.
269 269 */
270 270 indirect_child_t *is_good_child;
271 271
272 272 indirect_child_t is_child[1]; /* variable-length */
273 273 } indirect_split_t;
274 274
275 275 /*
276 276 * The indirect_vsd_t is associated with each i/o to the indirect vdev.
277 277 * It is the "Vdev-Specific Data" in the zio_t's io_vsd.
278 278 */
279 279 typedef struct indirect_vsd {
280 280 boolean_t iv_split_block;
281 281 boolean_t iv_reconstruct;
282 282 uint64_t iv_unique_combinations;
283 283 uint64_t iv_attempts;
284 284 uint64_t iv_attempts_max;
285 285
286 286 list_t iv_splits; /* list of indirect_split_t's */
287 287 } indirect_vsd_t;
288 288
289 289 static void
290 290 vdev_indirect_map_free(zio_t *zio)
291 291 {
292 292 indirect_vsd_t *iv = zio->io_vsd;
293 293
294 294 indirect_split_t *is;
295 295 while ((is = list_head(&iv->iv_splits)) != NULL) {
296 296 for (int c = 0; c < is->is_children; c++) {
297 297 indirect_child_t *ic = &is->is_child[c];
298 298 if (ic->ic_data != NULL)
299 299 abd_free(ic->ic_data);
300 300 }
301 301 list_remove(&iv->iv_splits, is);
302 302
303 303 indirect_child_t *ic;
304 304 while ((ic = list_head(&is->is_unique_child)) != NULL)
305 305 list_remove(&is->is_unique_child, ic);
306 306
307 307 list_destroy(&is->is_unique_child);
308 308
309 309 kmem_free(is,
310 310 offsetof(indirect_split_t, is_child[is->is_children]));
311 311 }
312 312 kmem_free(iv, sizeof (*iv));
313 313 }
314 314
315 315 static const zio_vsd_ops_t vdev_indirect_vsd_ops = {
316 316 vdev_indirect_map_free,
317 317 zio_vsd_default_cksum_report
318 318 };
319 319 /*
320 320 * Mark the given offset and size as being obsolete.
321 321 */
322 322 void
323 323 vdev_indirect_mark_obsolete(vdev_t *vd, uint64_t offset, uint64_t size)
324 324 {
325 325 spa_t *spa = vd->vdev_spa;
326 326
327 327 ASSERT3U(vd->vdev_indirect_config.vic_mapping_object, !=, 0);
328 328 ASSERT(vd->vdev_removing || vd->vdev_ops == &vdev_indirect_ops);
329 329 ASSERT(size > 0);
330 330 VERIFY(vdev_indirect_mapping_entry_for_offset(
331 331 vd->vdev_indirect_mapping, offset) != NULL);
332 332
333 333 if (spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)) {
334 334 mutex_enter(&vd->vdev_obsolete_lock);
335 335 range_tree_add(vd->vdev_obsolete_segments, offset, size);
336 336 mutex_exit(&vd->vdev_obsolete_lock);
337 337 vdev_dirty(vd, 0, NULL, spa_syncing_txg(spa));
338 338 }
339 339 }
340 340
341 341 /*
342 342 * Mark the DVA vdev_id:offset:size as being obsolete in the given tx. This
343 343 * wrapper is provided because the DMU does not know about vdev_t's and
344 344 * cannot directly call vdev_indirect_mark_obsolete.
345 345 */
346 346 void
347 347 spa_vdev_indirect_mark_obsolete(spa_t *spa, uint64_t vdev_id, uint64_t offset,
348 348 uint64_t size, dmu_tx_t *tx)
349 349 {
350 350 vdev_t *vd = vdev_lookup_top(spa, vdev_id);
351 351 ASSERT(dmu_tx_is_syncing(tx));
352 352
353 353 /* The DMU can only remap indirect vdevs. */
354 354 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
355 355 vdev_indirect_mark_obsolete(vd, offset, size);
356 356 }
357 357
358 358 static spa_condensing_indirect_t *
359 359 spa_condensing_indirect_create(spa_t *spa)
360 360 {
361 361 spa_condensing_indirect_phys_t *scip =
362 362 &spa->spa_condensing_indirect_phys;
363 363 spa_condensing_indirect_t *sci = kmem_zalloc(sizeof (*sci), KM_SLEEP);
364 364 objset_t *mos = spa->spa_meta_objset;
365 365
366 366 for (int i = 0; i < TXG_SIZE; i++) {
367 367 list_create(&sci->sci_new_mapping_entries[i],
368 368 sizeof (vdev_indirect_mapping_entry_t),
369 369 offsetof(vdev_indirect_mapping_entry_t, vime_node));
370 370 }
371 371
372 372 sci->sci_new_mapping =
373 373 vdev_indirect_mapping_open(mos, scip->scip_next_mapping_object);
374 374
375 375 return (sci);
376 376 }
377 377
378 378 static void
379 379 spa_condensing_indirect_destroy(spa_condensing_indirect_t *sci)
380 380 {
381 381 for (int i = 0; i < TXG_SIZE; i++)
382 382 list_destroy(&sci->sci_new_mapping_entries[i]);
383 383
384 384 if (sci->sci_new_mapping != NULL)
385 385 vdev_indirect_mapping_close(sci->sci_new_mapping);
386 386
387 387 kmem_free(sci, sizeof (*sci));
388 388 }
389 389
390 390 boolean_t
391 391 vdev_indirect_should_condense(vdev_t *vd)
392 392 {
393 393 vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
394 394 spa_t *spa = vd->vdev_spa;
395 395
396 396 ASSERT(dsl_pool_sync_context(spa->spa_dsl_pool));
397 397
398 398 if (!zfs_condense_indirect_vdevs_enable)
399 399 return (B_FALSE);
400 400
401 401 /*
402 402 * We can only condense one indirect vdev at a time.
403 403 */
404 404 if (spa->spa_condensing_indirect != NULL)
405 405 return (B_FALSE);
406 406
407 407 if (spa_shutting_down(spa))
408 408 return (B_FALSE);
409 409
410 410 /*
411 411 * The mapping object size must not change while we are
412 412 * condensing, so we can only condense indirect vdevs
413 413 * (not vdevs that are still in the middle of being removed).
414 414 */
415 415 if (vd->vdev_ops != &vdev_indirect_ops)
416 416 return (B_FALSE);
417 417
418 418 /*
419 419 * If nothing new has been marked obsolete, there is no
420 420 * point in condensing.
421 421 */
422 422 if (vd->vdev_obsolete_sm == NULL) {
423 423 ASSERT0(vdev_obsolete_sm_object(vd));
424 424 return (B_FALSE);
425 425 }
426 426
427 427 ASSERT(vd->vdev_obsolete_sm != NULL);
428 428
429 429 ASSERT3U(vdev_obsolete_sm_object(vd), ==,
430 430 space_map_object(vd->vdev_obsolete_sm));
431 431
432 432 uint64_t bytes_mapped = vdev_indirect_mapping_bytes_mapped(vim);
433 433 uint64_t bytes_obsolete = space_map_allocated(vd->vdev_obsolete_sm);
434 434 uint64_t mapping_size = vdev_indirect_mapping_size(vim);
435 435 uint64_t obsolete_sm_size = space_map_length(vd->vdev_obsolete_sm);
436 436
437 437 ASSERT3U(bytes_obsolete, <=, bytes_mapped);
438 438
439 439 /*
440 440 * If a high percentage of the bytes that are mapped have become
441 441 * obsolete, condense (unless the mapping is already small enough).
442 442 * This has a good chance of reducing the amount of memory used
443 443 * by the mapping.
444 444 */
445 445 if (bytes_obsolete * 100 / bytes_mapped >=
446 446 zfs_indirect_condense_obsolete_pct &&
447 447 mapping_size > zfs_condense_min_mapping_bytes) {
448 448 zfs_dbgmsg("should condense vdev %llu because obsolete "
449 449 "spacemap covers %d%% of %lluMB mapping",
450 450 (u_longlong_t)vd->vdev_id,
451 451 (int)(bytes_obsolete * 100 / bytes_mapped),
452 452 (u_longlong_t)bytes_mapped / 1024 / 1024);
453 453 return (B_TRUE);
454 454 }
455 455
456 456 /*
457 457 * If the obsolete space map takes up too much space on disk,
458 458 * condense in order to free up this disk space.
459 459 */
460 460 if (obsolete_sm_size >= zfs_condense_max_obsolete_bytes) {
461 461 zfs_dbgmsg("should condense vdev %llu because obsolete sm "
462 462 "length %lluMB >= max size %lluMB",
463 463 (u_longlong_t)vd->vdev_id,
464 464 (u_longlong_t)obsolete_sm_size / 1024 / 1024,
465 465 (u_longlong_t)zfs_condense_max_obsolete_bytes /
466 466 1024 / 1024);
467 467 return (B_TRUE);
468 468 }
469 469
470 470 return (B_FALSE);
471 471 }
472 472
473 473 /*
474 474 * This sync task completes (finishes) a condense, deleting the old
475 475 * mapping and replacing it with the new one.
476 476 */
477 477 static void
478 478 spa_condense_indirect_complete_sync(void *arg, dmu_tx_t *tx)
479 479 {
480 480 spa_condensing_indirect_t *sci = arg;
481 481 spa_t *spa = dmu_tx_pool(tx)->dp_spa;
482 482 spa_condensing_indirect_phys_t *scip =
483 483 &spa->spa_condensing_indirect_phys;
484 484 vdev_t *vd = vdev_lookup_top(spa, scip->scip_vdev);
485 485 vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
486 486 objset_t *mos = spa->spa_meta_objset;
487 487 vdev_indirect_mapping_t *old_mapping = vd->vdev_indirect_mapping;
488 488 uint64_t old_count = vdev_indirect_mapping_num_entries(old_mapping);
489 489 uint64_t new_count =
490 490 vdev_indirect_mapping_num_entries(sci->sci_new_mapping);
491 491
492 492 ASSERT(dmu_tx_is_syncing(tx));
493 493 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
494 494 ASSERT3P(sci, ==, spa->spa_condensing_indirect);
495 495 for (int i = 0; i < TXG_SIZE; i++) {
496 496 ASSERT(list_is_empty(&sci->sci_new_mapping_entries[i]));
497 497 }
498 498 ASSERT(vic->vic_mapping_object != 0);
499 499 ASSERT3U(vd->vdev_id, ==, scip->scip_vdev);
500 500 ASSERT(scip->scip_next_mapping_object != 0);
501 501 ASSERT(scip->scip_prev_obsolete_sm_object != 0);
502 502
503 503 /*
504 504 * Reset vdev_indirect_mapping to refer to the new object.
505 505 */
506 506 rw_enter(&vd->vdev_indirect_rwlock, RW_WRITER);
507 507 vdev_indirect_mapping_close(vd->vdev_indirect_mapping);
508 508 vd->vdev_indirect_mapping = sci->sci_new_mapping;
509 509 rw_exit(&vd->vdev_indirect_rwlock);
510 510
511 511 sci->sci_new_mapping = NULL;
512 512 vdev_indirect_mapping_free(mos, vic->vic_mapping_object, tx);
513 513 vic->vic_mapping_object = scip->scip_next_mapping_object;
514 514 scip->scip_next_mapping_object = 0;
515 515
516 516 space_map_free_obj(mos, scip->scip_prev_obsolete_sm_object, tx);
517 517 spa_feature_decr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
518 518 scip->scip_prev_obsolete_sm_object = 0;
519 519
520 520 scip->scip_vdev = 0;
521 521
522 522 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT,
523 523 DMU_POOL_CONDENSING_INDIRECT, tx));
524 524 spa_condensing_indirect_destroy(spa->spa_condensing_indirect);
525 525 spa->spa_condensing_indirect = NULL;
526 526
527 527 zfs_dbgmsg("finished condense of vdev %llu in txg %llu: "
528 528 "new mapping object %llu has %llu entries "
529 529 "(was %llu entries)",
530 530 vd->vdev_id, dmu_tx_get_txg(tx), vic->vic_mapping_object,
531 531 new_count, old_count);
532 532
533 533 vdev_config_dirty(spa->spa_root_vdev);
534 534 }
535 535
536 536 /*
537 537 * This sync task appends entries to the new mapping object.
538 538 */
539 539 static void
540 540 spa_condense_indirect_commit_sync(void *arg, dmu_tx_t *tx)
541 541 {
542 542 spa_condensing_indirect_t *sci = arg;
543 543 uint64_t txg = dmu_tx_get_txg(tx);
544 544 spa_t *spa = dmu_tx_pool(tx)->dp_spa;
545 545
546 546 ASSERT(dmu_tx_is_syncing(tx));
547 547 ASSERT3P(sci, ==, spa->spa_condensing_indirect);
548 548
549 549 vdev_indirect_mapping_add_entries(sci->sci_new_mapping,
550 550 &sci->sci_new_mapping_entries[txg & TXG_MASK], tx);
551 551 ASSERT(list_is_empty(&sci->sci_new_mapping_entries[txg & TXG_MASK]));
552 552 }
553 553
554 554 /*
555 555 * Open-context function to add one entry to the new mapping. The new
556 556 * entry will be remembered and written from syncing context.
557 557 */
558 558 static void
559 559 spa_condense_indirect_commit_entry(spa_t *spa,
560 560 vdev_indirect_mapping_entry_phys_t *vimep, uint32_t count)
561 561 {
562 562 spa_condensing_indirect_t *sci = spa->spa_condensing_indirect;
563 563
564 564 ASSERT3U(count, <, DVA_GET_ASIZE(&vimep->vimep_dst));
565 565
566 566 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
567 567 dmu_tx_hold_space(tx, sizeof (*vimep) + sizeof (count));
568 568 VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
569 569 int txgoff = dmu_tx_get_txg(tx) & TXG_MASK;
570 570
571 571 /*
572 572 * If we are the first entry committed this txg, kick off the sync
573 573 * task to write to the MOS on our behalf.
574 574 */
575 575 if (list_is_empty(&sci->sci_new_mapping_entries[txgoff])) {
576 576 dsl_sync_task_nowait(dmu_tx_pool(tx),
577 577 spa_condense_indirect_commit_sync, sci,
578 578 0, ZFS_SPACE_CHECK_NONE, tx);
579 579 }
580 580
581 581 vdev_indirect_mapping_entry_t *vime =
582 582 kmem_alloc(sizeof (*vime), KM_SLEEP);
583 583 vime->vime_mapping = *vimep;
584 584 vime->vime_obsolete_count = count;
585 585 list_insert_tail(&sci->sci_new_mapping_entries[txgoff], vime);
586 586
587 587 dmu_tx_commit(tx);
588 588 }
589 589
590 590 static void
591 591 spa_condense_indirect_generate_new_mapping(vdev_t *vd,
592 592 uint32_t *obsolete_counts, uint64_t start_index, zthr_t *zthr)
593 593 {
594 594 spa_t *spa = vd->vdev_spa;
595 595 uint64_t mapi = start_index;
596 596 vdev_indirect_mapping_t *old_mapping = vd->vdev_indirect_mapping;
597 597 uint64_t old_num_entries =
598 598 vdev_indirect_mapping_num_entries(old_mapping);
599 599
600 600 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
601 601 ASSERT3U(vd->vdev_id, ==, spa->spa_condensing_indirect_phys.scip_vdev);
602 602
603 603 zfs_dbgmsg("starting condense of vdev %llu from index %llu",
604 604 (u_longlong_t)vd->vdev_id,
605 605 (u_longlong_t)mapi);
606 606
607 607 while (mapi < old_num_entries) {
608 608
609 609 if (zthr_iscancelled(zthr)) {
610 610 zfs_dbgmsg("pausing condense of vdev %llu "
611 611 "at index %llu", (u_longlong_t)vd->vdev_id,
612 612 (u_longlong_t)mapi);
613 613 break;
614 614 }
615 615
616 616 vdev_indirect_mapping_entry_phys_t *entry =
617 617 &old_mapping->vim_entries[mapi];
618 618 uint64_t entry_size = DVA_GET_ASIZE(&entry->vimep_dst);
619 619 ASSERT3U(obsolete_counts[mapi], <=, entry_size);
620 620 if (obsolete_counts[mapi] < entry_size) {
621 621 spa_condense_indirect_commit_entry(spa, entry,
622 622 obsolete_counts[mapi]);
623 623
624 624 /*
625 625 * This delay may be requested for testing, debugging,
626 626 * or performance reasons.
627 627 */
628 628 delay(zfs_condense_indirect_commit_entry_delay_ticks);
629 629 }
630 630
631 631 mapi++;
632 632 }
633 633 }
634 634
635 635 /* ARGSUSED */
636 636 static boolean_t
637 637 spa_condense_indirect_thread_check(void *arg, zthr_t *zthr)
638 638 {
639 639 spa_t *spa = arg;
640 640
641 641 return (spa->spa_condensing_indirect != NULL);
642 642 }
643 643
644 644 /* ARGSUSED */
645 645 static int
646 646 spa_condense_indirect_thread(void *arg, zthr_t *zthr)
647 647 {
648 648 spa_t *spa = arg;
649 649 vdev_t *vd;
650 650
651 651 ASSERT3P(spa->spa_condensing_indirect, !=, NULL);
652 652 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
653 653 vd = vdev_lookup_top(spa, spa->spa_condensing_indirect_phys.scip_vdev);
654 654 ASSERT3P(vd, !=, NULL);
655 655 spa_config_exit(spa, SCL_VDEV, FTAG);
656 656
657 657 spa_condensing_indirect_t *sci = spa->spa_condensing_indirect;
658 658 spa_condensing_indirect_phys_t *scip =
659 659 &spa->spa_condensing_indirect_phys;
660 660 uint32_t *counts;
661 661 uint64_t start_index;
662 662 vdev_indirect_mapping_t *old_mapping = vd->vdev_indirect_mapping;
663 663 space_map_t *prev_obsolete_sm = NULL;
664 664
665 665 ASSERT3U(vd->vdev_id, ==, scip->scip_vdev);
666 666 ASSERT(scip->scip_next_mapping_object != 0);
667 667 ASSERT(scip->scip_prev_obsolete_sm_object != 0);
668 668 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
669 669
670 670 for (int i = 0; i < TXG_SIZE; i++) {
671 671 /*
672 672 * The list must start out empty in order for the
|
↓ open down ↓ |
672 lines elided |
↑ open up ↑ |
673 673 * _commit_sync() sync task to be properly registered
674 674 * on the first call to _commit_entry(); so it's wise
675 675 * to double check and ensure we actually are starting
676 676 * with empty lists.
677 677 */
678 678 ASSERT(list_is_empty(&sci->sci_new_mapping_entries[i]));
679 679 }
680 680
681 681 VERIFY0(space_map_open(&prev_obsolete_sm, spa->spa_meta_objset,
682 682 scip->scip_prev_obsolete_sm_object, 0, vd->vdev_asize, 0));
683 - space_map_update(prev_obsolete_sm);
684 683 counts = vdev_indirect_mapping_load_obsolete_counts(old_mapping);
685 684 if (prev_obsolete_sm != NULL) {
686 685 vdev_indirect_mapping_load_obsolete_spacemap(old_mapping,
687 686 counts, prev_obsolete_sm);
688 687 }
689 688 space_map_close(prev_obsolete_sm);
690 689
691 690 /*
692 691 * Generate new mapping. Determine what index to continue from
693 692 * based on the max offset that we've already written in the
694 693 * new mapping.
695 694 */
696 695 uint64_t max_offset =
697 696 vdev_indirect_mapping_max_offset(sci->sci_new_mapping);
698 697 if (max_offset == 0) {
699 698 /* We haven't written anything to the new mapping yet. */
700 699 start_index = 0;
701 700 } else {
702 701 /*
703 702 * Pick up from where we left off. _entry_for_offset()
704 703 * returns a pointer into the vim_entries array. If
705 704 * max_offset is greater than any of the mappings
706 705 * contained in the table NULL will be returned and
707 706 * that indicates we've exhausted our iteration of the
708 707 * old_mapping.
709 708 */
710 709
711 710 vdev_indirect_mapping_entry_phys_t *entry =
712 711 vdev_indirect_mapping_entry_for_offset_or_next(old_mapping,
713 712 max_offset);
714 713
715 714 if (entry == NULL) {
716 715 /*
717 716 * We've already written the whole new mapping.
718 717 * This special value will cause us to skip the
719 718 * generate_new_mapping step and just do the sync
720 719 * task to complete the condense.
721 720 */
722 721 start_index = UINT64_MAX;
723 722 } else {
724 723 start_index = entry - old_mapping->vim_entries;
725 724 ASSERT3U(start_index, <,
726 725 vdev_indirect_mapping_num_entries(old_mapping));
727 726 }
728 727 }
729 728
730 729 spa_condense_indirect_generate_new_mapping(vd, counts,
731 730 start_index, zthr);
732 731
733 732 vdev_indirect_mapping_free_obsolete_counts(old_mapping, counts);
734 733
735 734 /*
736 735 * If the zthr has received a cancellation signal while running
737 736 * in generate_new_mapping() or at any point after that, then bail
738 737 * early. We don't want to complete the condense if the spa is
739 738 * shutting down.
740 739 */
741 740 if (zthr_iscancelled(zthr))
742 741 return (0);
743 742
744 743 VERIFY0(dsl_sync_task(spa_name(spa), NULL,
745 744 spa_condense_indirect_complete_sync, sci, 0,
746 745 ZFS_SPACE_CHECK_EXTRA_RESERVED));
747 746
748 747 return (0);
749 748 }
750 749
751 750 /*
752 751 * Sync task to begin the condensing process.
753 752 */
754 753 void
755 754 spa_condense_indirect_start_sync(vdev_t *vd, dmu_tx_t *tx)
756 755 {
757 756 spa_t *spa = vd->vdev_spa;
758 757 spa_condensing_indirect_phys_t *scip =
759 758 &spa->spa_condensing_indirect_phys;
760 759
761 760 ASSERT0(scip->scip_next_mapping_object);
762 761 ASSERT0(scip->scip_prev_obsolete_sm_object);
763 762 ASSERT0(scip->scip_vdev);
764 763 ASSERT(dmu_tx_is_syncing(tx));
765 764 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops);
766 765 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_OBSOLETE_COUNTS));
767 766 ASSERT(vdev_indirect_mapping_num_entries(vd->vdev_indirect_mapping));
768 767
769 768 uint64_t obsolete_sm_obj = vdev_obsolete_sm_object(vd);
770 769 ASSERT(obsolete_sm_obj != 0);
771 770
772 771 scip->scip_vdev = vd->vdev_id;
773 772 scip->scip_next_mapping_object =
774 773 vdev_indirect_mapping_alloc(spa->spa_meta_objset, tx);
775 774
776 775 scip->scip_prev_obsolete_sm_object = obsolete_sm_obj;
777 776
778 777 /*
779 778 * We don't need to allocate a new space map object, since
780 779 * vdev_indirect_sync_obsolete will allocate one when needed.
781 780 */
782 781 space_map_close(vd->vdev_obsolete_sm);
783 782 vd->vdev_obsolete_sm = NULL;
784 783 VERIFY0(zap_remove(spa->spa_meta_objset, vd->vdev_top_zap,
785 784 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM, tx));
786 785
787 786 VERIFY0(zap_add(spa->spa_dsl_pool->dp_meta_objset,
788 787 DMU_POOL_DIRECTORY_OBJECT,
789 788 DMU_POOL_CONDENSING_INDIRECT, sizeof (uint64_t),
790 789 sizeof (*scip) / sizeof (uint64_t), scip, tx));
791 790
792 791 ASSERT3P(spa->spa_condensing_indirect, ==, NULL);
793 792 spa->spa_condensing_indirect = spa_condensing_indirect_create(spa);
794 793
795 794 zfs_dbgmsg("starting condense of vdev %llu in txg %llu: "
796 795 "posm=%llu nm=%llu",
797 796 vd->vdev_id, dmu_tx_get_txg(tx),
798 797 (u_longlong_t)scip->scip_prev_obsolete_sm_object,
799 798 (u_longlong_t)scip->scip_next_mapping_object);
800 799
801 800 zthr_wakeup(spa->spa_condense_zthr);
802 801 }
803 802
804 803 /*
805 804 * Sync to the given vdev's obsolete space map any segments that are no longer
806 805 * referenced as of the given txg.
807 806 *
808 807 * If the obsolete space map doesn't exist yet, create and open it.
809 808 */
810 809 void
811 810 vdev_indirect_sync_obsolete(vdev_t *vd, dmu_tx_t *tx)
812 811 {
813 812 spa_t *spa = vd->vdev_spa;
814 813 vdev_indirect_config_t *vic = &vd->vdev_indirect_config;
815 814
816 815 ASSERT3U(vic->vic_mapping_object, !=, 0);
817 816 ASSERT(range_tree_space(vd->vdev_obsolete_segments) > 0);
818 817 ASSERT(vd->vdev_removing || vd->vdev_ops == &vdev_indirect_ops);
819 818 ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS));
820 819
821 820 if (vdev_obsolete_sm_object(vd) == 0) {
822 821 uint64_t obsolete_sm_object =
823 822 space_map_alloc(spa->spa_meta_objset,
824 823 vdev_standard_sm_blksz, tx);
825 824
|
↓ open down ↓ |
132 lines elided |
↑ open up ↑ |
826 825 ASSERT(vd->vdev_top_zap != 0);
827 826 VERIFY0(zap_add(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,
828 827 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM,
829 828 sizeof (obsolete_sm_object), 1, &obsolete_sm_object, tx));
830 829 ASSERT3U(vdev_obsolete_sm_object(vd), !=, 0);
831 830
832 831 spa_feature_incr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
833 832 VERIFY0(space_map_open(&vd->vdev_obsolete_sm,
834 833 spa->spa_meta_objset, obsolete_sm_object,
835 834 0, vd->vdev_asize, 0));
836 - space_map_update(vd->vdev_obsolete_sm);
837 835 }
838 836
839 837 ASSERT(vd->vdev_obsolete_sm != NULL);
840 838 ASSERT3U(vdev_obsolete_sm_object(vd), ==,
841 839 space_map_object(vd->vdev_obsolete_sm));
842 840
843 841 space_map_write(vd->vdev_obsolete_sm,
844 842 vd->vdev_obsolete_segments, SM_ALLOC, SM_NO_VDEVID, tx);
845 - space_map_update(vd->vdev_obsolete_sm);
846 843 range_tree_vacate(vd->vdev_obsolete_segments, NULL, NULL);
847 844 }
848 845
849 846 int
850 847 spa_condense_init(spa_t *spa)
851 848 {
852 849 int error = zap_lookup(spa->spa_meta_objset,
853 850 DMU_POOL_DIRECTORY_OBJECT,
854 851 DMU_POOL_CONDENSING_INDIRECT, sizeof (uint64_t),
855 852 sizeof (spa->spa_condensing_indirect_phys) / sizeof (uint64_t),
856 853 &spa->spa_condensing_indirect_phys);
857 854 if (error == 0) {
858 855 if (spa_writeable(spa)) {
859 856 spa->spa_condensing_indirect =
860 857 spa_condensing_indirect_create(spa);
861 858 }
862 859 return (0);
863 860 } else if (error == ENOENT) {
864 861 return (0);
865 862 } else {
866 863 return (error);
867 864 }
868 865 }
869 866
870 867 void
871 868 spa_condense_fini(spa_t *spa)
872 869 {
873 870 if (spa->spa_condensing_indirect != NULL) {
874 871 spa_condensing_indirect_destroy(spa->spa_condensing_indirect);
875 872 spa->spa_condensing_indirect = NULL;
876 873 }
877 874 }
878 875
879 876 void
880 877 spa_start_indirect_condensing_thread(spa_t *spa)
881 878 {
882 879 ASSERT3P(spa->spa_condense_zthr, ==, NULL);
883 880 spa->spa_condense_zthr = zthr_create(spa_condense_indirect_thread_check,
884 881 spa_condense_indirect_thread, spa);
885 882 }
886 883
887 884 /*
888 885 * Gets the obsolete spacemap object from the vdev's ZAP.
889 886 * Returns the spacemap object, or 0 if it wasn't in the ZAP or the ZAP doesn't
890 887 * exist yet.
891 888 */
892 889 int
893 890 vdev_obsolete_sm_object(vdev_t *vd)
894 891 {
895 892 ASSERT0(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
896 893 if (vd->vdev_top_zap == 0) {
897 894 return (0);
898 895 }
899 896
900 897 uint64_t sm_obj = 0;
901 898 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,
902 899 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM, sizeof (sm_obj), 1, &sm_obj);
903 900
904 901 ASSERT(err == 0 || err == ENOENT);
905 902
906 903 return (sm_obj);
907 904 }
908 905
909 906 boolean_t
910 907 vdev_obsolete_counts_are_precise(vdev_t *vd)
911 908 {
912 909 ASSERT0(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
913 910 if (vd->vdev_top_zap == 0) {
914 911 return (B_FALSE);
915 912 }
916 913
917 914 uint64_t val = 0;
918 915 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,
919 916 VDEV_TOP_ZAP_OBSOLETE_COUNTS_ARE_PRECISE, sizeof (val), 1, &val);
920 917
921 918 ASSERT(err == 0 || err == ENOENT);
922 919
923 920 return (val != 0);
924 921 }
925 922
926 923 /* ARGSUSED */
927 924 static void
928 925 vdev_indirect_close(vdev_t *vd)
929 926 {
930 927 }
931 928
932 929 /* ARGSUSED */
933 930 static int
934 931 vdev_indirect_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize,
935 932 uint64_t *ashift)
936 933 {
937 934 *psize = *max_psize = vd->vdev_asize +
938 935 VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
939 936 *ashift = vd->vdev_ashift;
940 937 return (0);
941 938 }
942 939
943 940 typedef struct remap_segment {
944 941 vdev_t *rs_vd;
945 942 uint64_t rs_offset;
946 943 uint64_t rs_asize;
947 944 uint64_t rs_split_offset;
948 945 list_node_t rs_node;
949 946 } remap_segment_t;
950 947
951 948 remap_segment_t *
952 949 rs_alloc(vdev_t *vd, uint64_t offset, uint64_t asize, uint64_t split_offset)
953 950 {
954 951 remap_segment_t *rs = kmem_alloc(sizeof (remap_segment_t), KM_SLEEP);
955 952 rs->rs_vd = vd;
956 953 rs->rs_offset = offset;
957 954 rs->rs_asize = asize;
958 955 rs->rs_split_offset = split_offset;
959 956 return (rs);
960 957 }
961 958
962 959 /*
963 960 * Given an indirect vdev and an extent on that vdev, it duplicates the
964 961 * physical entries of the indirect mapping that correspond to the extent
965 962 * to a new array and returns a pointer to it. In addition, copied_entries
966 963 * is populated with the number of mapping entries that were duplicated.
967 964 *
968 965 * Note that the function assumes that the caller holds vdev_indirect_rwlock.
969 966 * This ensures that the mapping won't change due to condensing as we
970 967 * copy over its contents.
971 968 *
972 969 * Finally, since we are doing an allocation, it is up to the caller to
973 970 * free the array allocated in this function.
974 971 */
975 972 vdev_indirect_mapping_entry_phys_t *
976 973 vdev_indirect_mapping_duplicate_adjacent_entries(vdev_t *vd, uint64_t offset,
977 974 uint64_t asize, uint64_t *copied_entries)
978 975 {
979 976 vdev_indirect_mapping_entry_phys_t *duplicate_mappings = NULL;
980 977 vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
981 978 uint64_t entries = 0;
982 979
983 980 ASSERT(RW_READ_HELD(&vd->vdev_indirect_rwlock));
984 981
985 982 vdev_indirect_mapping_entry_phys_t *first_mapping =
986 983 vdev_indirect_mapping_entry_for_offset(vim, offset);
987 984 ASSERT3P(first_mapping, !=, NULL);
988 985
989 986 vdev_indirect_mapping_entry_phys_t *m = first_mapping;
990 987 while (asize > 0) {
991 988 uint64_t size = DVA_GET_ASIZE(&m->vimep_dst);
992 989
993 990 ASSERT3U(offset, >=, DVA_MAPPING_GET_SRC_OFFSET(m));
994 991 ASSERT3U(offset, <, DVA_MAPPING_GET_SRC_OFFSET(m) + size);
995 992
996 993 uint64_t inner_offset = offset - DVA_MAPPING_GET_SRC_OFFSET(m);
997 994 uint64_t inner_size = MIN(asize, size - inner_offset);
998 995
999 996 offset += inner_size;
1000 997 asize -= inner_size;
1001 998 entries++;
1002 999 m++;
1003 1000 }
1004 1001
1005 1002 size_t copy_length = entries * sizeof (*first_mapping);
1006 1003 duplicate_mappings = kmem_alloc(copy_length, KM_SLEEP);
1007 1004 bcopy(first_mapping, duplicate_mappings, copy_length);
1008 1005 *copied_entries = entries;
1009 1006
1010 1007 return (duplicate_mappings);
1011 1008 }
1012 1009
1013 1010 /*
1014 1011 * Goes through the relevant indirect mappings until it hits a concrete vdev
1015 1012 * and issues the callback. On the way to the concrete vdev, if any other
1016 1013 * indirect vdevs are encountered, then the callback will also be called on
1017 1014 * each of those indirect vdevs. For example, if the segment is mapped to
1018 1015 * segment A on indirect vdev 1, and then segment A on indirect vdev 1 is
1019 1016 * mapped to segment B on concrete vdev 2, then the callback will be called on
1020 1017 * both vdev 1 and vdev 2.
1021 1018 *
1022 1019 * While the callback passed to vdev_indirect_remap() is called on every vdev
1023 1020 * the function encounters, certain callbacks only care about concrete vdevs.
1024 1021 * These types of callbacks should return immediately and explicitly when they
1025 1022 * are called on an indirect vdev.
1026 1023 *
1027 1024 * Because there is a possibility that a DVA section in the indirect device
1028 1025 * has been split into multiple sections in our mapping, we keep track
1029 1026 * of the relevant contiguous segments of the new location (remap_segment_t)
1030 1027 * in a stack. This way we can call the callback for each of the new sections
1031 1028 * created by a single section of the indirect device. Note though, that in
1032 1029 * this scenario the callbacks in each split block won't occur in-order in
1033 1030 * terms of offset, so callers should not make any assumptions about that.
1034 1031 *
1035 1032 * For callbacks that don't handle split blocks and immediately return when
1036 1033 * they encounter them (as is the case for remap_blkptr_cb), the caller can
1037 1034 * assume that its callback will be applied from the first indirect vdev
1038 1035 * encountered to the last one and then the concrete vdev, in that order.
1039 1036 */
1040 1037 static void
1041 1038 vdev_indirect_remap(vdev_t *vd, uint64_t offset, uint64_t asize,
1042 1039 void (*func)(uint64_t, vdev_t *, uint64_t, uint64_t, void *), void *arg)
1043 1040 {
1044 1041 list_t stack;
1045 1042 spa_t *spa = vd->vdev_spa;
1046 1043
1047 1044 list_create(&stack, sizeof (remap_segment_t),
1048 1045 offsetof(remap_segment_t, rs_node));
1049 1046
1050 1047 for (remap_segment_t *rs = rs_alloc(vd, offset, asize, 0);
1051 1048 rs != NULL; rs = list_remove_head(&stack)) {
1052 1049 vdev_t *v = rs->rs_vd;
1053 1050 uint64_t num_entries = 0;
1054 1051
1055 1052 ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
1056 1053 ASSERT(rs->rs_asize > 0);
1057 1054
1058 1055 /*
1059 1056 * Note: As this function can be called from open context
1060 1057 * (e.g. zio_read()), we need the following rwlock to
1061 1058 * prevent the mapping from being changed by condensing.
1062 1059 *
1063 1060 * So we grab the lock and we make a copy of the entries
1064 1061 * that are relevant to the extent that we are working on.
1065 1062 * Once that is done, we drop the lock and iterate over
1066 1063 * our copy of the mapping. Once we are done with the with
1067 1064 * the remap segment and we free it, we also free our copy
1068 1065 * of the indirect mapping entries that are relevant to it.
1069 1066 *
1070 1067 * This way we don't need to wait until the function is
1071 1068 * finished with a segment, to condense it. In addition, we
1072 1069 * don't need a recursive rwlock for the case that a call to
1073 1070 * vdev_indirect_remap() needs to call itself (through the
1074 1071 * codepath of its callback) for the same vdev in the middle
1075 1072 * of its execution.
1076 1073 */
1077 1074 rw_enter(&v->vdev_indirect_rwlock, RW_READER);
1078 1075 vdev_indirect_mapping_t *vim = v->vdev_indirect_mapping;
1079 1076 ASSERT3P(vim, !=, NULL);
1080 1077
1081 1078 vdev_indirect_mapping_entry_phys_t *mapping =
1082 1079 vdev_indirect_mapping_duplicate_adjacent_entries(v,
1083 1080 rs->rs_offset, rs->rs_asize, &num_entries);
1084 1081 ASSERT3P(mapping, !=, NULL);
1085 1082 ASSERT3U(num_entries, >, 0);
1086 1083 rw_exit(&v->vdev_indirect_rwlock);
1087 1084
1088 1085 for (uint64_t i = 0; i < num_entries; i++) {
1089 1086 /*
1090 1087 * Note: the vdev_indirect_mapping can not change
1091 1088 * while we are running. It only changes while the
1092 1089 * removal is in progress, and then only from syncing
1093 1090 * context. While a removal is in progress, this
1094 1091 * function is only called for frees, which also only
1095 1092 * happen from syncing context.
1096 1093 */
1097 1094 vdev_indirect_mapping_entry_phys_t *m = &mapping[i];
1098 1095
1099 1096 ASSERT3P(m, !=, NULL);
1100 1097 ASSERT3U(rs->rs_asize, >, 0);
1101 1098
1102 1099 uint64_t size = DVA_GET_ASIZE(&m->vimep_dst);
1103 1100 uint64_t dst_offset = DVA_GET_OFFSET(&m->vimep_dst);
1104 1101 uint64_t dst_vdev = DVA_GET_VDEV(&m->vimep_dst);
1105 1102
1106 1103 ASSERT3U(rs->rs_offset, >=,
1107 1104 DVA_MAPPING_GET_SRC_OFFSET(m));
1108 1105 ASSERT3U(rs->rs_offset, <,
1109 1106 DVA_MAPPING_GET_SRC_OFFSET(m) + size);
1110 1107 ASSERT3U(dst_vdev, !=, v->vdev_id);
1111 1108
1112 1109 uint64_t inner_offset = rs->rs_offset -
1113 1110 DVA_MAPPING_GET_SRC_OFFSET(m);
1114 1111 uint64_t inner_size =
1115 1112 MIN(rs->rs_asize, size - inner_offset);
1116 1113
1117 1114 vdev_t *dst_v = vdev_lookup_top(spa, dst_vdev);
1118 1115 ASSERT3P(dst_v, !=, NULL);
1119 1116
1120 1117 if (dst_v->vdev_ops == &vdev_indirect_ops) {
1121 1118 list_insert_head(&stack,
1122 1119 rs_alloc(dst_v, dst_offset + inner_offset,
1123 1120 inner_size, rs->rs_split_offset));
1124 1121
1125 1122 }
1126 1123
1127 1124 if ((zfs_flags & ZFS_DEBUG_INDIRECT_REMAP) &&
1128 1125 IS_P2ALIGNED(inner_size, 2 * SPA_MINBLOCKSIZE)) {
1129 1126 /*
1130 1127 * Note: This clause exists only solely for
1131 1128 * testing purposes. We use it to ensure that
1132 1129 * split blocks work and that the callbacks
1133 1130 * using them yield the same result if issued
1134 1131 * in reverse order.
1135 1132 */
1136 1133 uint64_t inner_half = inner_size / 2;
1137 1134
1138 1135 func(rs->rs_split_offset + inner_half, dst_v,
1139 1136 dst_offset + inner_offset + inner_half,
1140 1137 inner_half, arg);
1141 1138
1142 1139 func(rs->rs_split_offset, dst_v,
1143 1140 dst_offset + inner_offset,
1144 1141 inner_half, arg);
1145 1142 } else {
1146 1143 func(rs->rs_split_offset, dst_v,
1147 1144 dst_offset + inner_offset,
1148 1145 inner_size, arg);
1149 1146 }
1150 1147
1151 1148 rs->rs_offset += inner_size;
1152 1149 rs->rs_asize -= inner_size;
1153 1150 rs->rs_split_offset += inner_size;
1154 1151 }
1155 1152 VERIFY0(rs->rs_asize);
1156 1153
1157 1154 kmem_free(mapping, num_entries * sizeof (*mapping));
1158 1155 kmem_free(rs, sizeof (remap_segment_t));
1159 1156 }
1160 1157 list_destroy(&stack);
1161 1158 }
1162 1159
1163 1160 static void
1164 1161 vdev_indirect_child_io_done(zio_t *zio)
1165 1162 {
1166 1163 zio_t *pio = zio->io_private;
1167 1164
1168 1165 mutex_enter(&pio->io_lock);
1169 1166 pio->io_error = zio_worst_error(pio->io_error, zio->io_error);
1170 1167 mutex_exit(&pio->io_lock);
1171 1168
1172 1169 abd_put(zio->io_abd);
1173 1170 }
1174 1171
1175 1172 /*
1176 1173 * This is a callback for vdev_indirect_remap() which allocates an
1177 1174 * indirect_split_t for each split segment and adds it to iv_splits.
1178 1175 */
1179 1176 static void
1180 1177 vdev_indirect_gather_splits(uint64_t split_offset, vdev_t *vd, uint64_t offset,
1181 1178 uint64_t size, void *arg)
1182 1179 {
1183 1180 zio_t *zio = arg;
1184 1181 indirect_vsd_t *iv = zio->io_vsd;
1185 1182
1186 1183 ASSERT3P(vd, !=, NULL);
1187 1184
1188 1185 if (vd->vdev_ops == &vdev_indirect_ops)
1189 1186 return;
1190 1187
1191 1188 int n = 1;
1192 1189 if (vd->vdev_ops == &vdev_mirror_ops)
1193 1190 n = vd->vdev_children;
1194 1191
1195 1192 indirect_split_t *is =
1196 1193 kmem_zalloc(offsetof(indirect_split_t, is_child[n]), KM_SLEEP);
1197 1194
1198 1195 is->is_children = n;
1199 1196 is->is_size = size;
1200 1197 is->is_split_offset = split_offset;
1201 1198 is->is_target_offset = offset;
1202 1199 is->is_vdev = vd;
1203 1200 list_create(&is->is_unique_child, sizeof (indirect_child_t),
1204 1201 offsetof(indirect_child_t, ic_node));
1205 1202
1206 1203 /*
1207 1204 * Note that we only consider multiple copies of the data for
1208 1205 * *mirror* vdevs. We don't for "replacing" or "spare" vdevs, even
1209 1206 * though they use the same ops as mirror, because there's only one
1210 1207 * "good" copy under the replacing/spare.
1211 1208 */
1212 1209 if (vd->vdev_ops == &vdev_mirror_ops) {
1213 1210 for (int i = 0; i < n; i++) {
1214 1211 is->is_child[i].ic_vdev = vd->vdev_child[i];
1215 1212 list_link_init(&is->is_child[i].ic_node);
1216 1213 }
1217 1214 } else {
1218 1215 is->is_child[0].ic_vdev = vd;
1219 1216 }
1220 1217
1221 1218 list_insert_tail(&iv->iv_splits, is);
1222 1219 }
1223 1220
1224 1221 static void
1225 1222 vdev_indirect_read_split_done(zio_t *zio)
1226 1223 {
1227 1224 indirect_child_t *ic = zio->io_private;
1228 1225
1229 1226 if (zio->io_error != 0) {
1230 1227 /*
1231 1228 * Clear ic_data to indicate that we do not have data for this
1232 1229 * child.
1233 1230 */
1234 1231 abd_free(ic->ic_data);
1235 1232 ic->ic_data = NULL;
1236 1233 }
1237 1234 }
1238 1235
1239 1236 /*
1240 1237 * Issue reads for all copies (mirror children) of all splits.
1241 1238 */
1242 1239 static void
1243 1240 vdev_indirect_read_all(zio_t *zio)
1244 1241 {
1245 1242 indirect_vsd_t *iv = zio->io_vsd;
1246 1243
1247 1244 for (indirect_split_t *is = list_head(&iv->iv_splits);
1248 1245 is != NULL; is = list_next(&iv->iv_splits, is)) {
1249 1246 for (int i = 0; i < is->is_children; i++) {
1250 1247 indirect_child_t *ic = &is->is_child[i];
1251 1248
1252 1249 if (!vdev_readable(ic->ic_vdev))
1253 1250 continue;
1254 1251
1255 1252 /*
1256 1253 * Note, we may read from a child whose DTL
1257 1254 * indicates that the data may not be present here.
1258 1255 * While this might result in a few i/os that will
1259 1256 * likely return incorrect data, it simplifies the
1260 1257 * code since we can treat scrub and resilver
1261 1258 * identically. (The incorrect data will be
1262 1259 * detected and ignored when we verify the
1263 1260 * checksum.)
1264 1261 */
1265 1262
1266 1263 ic->ic_data = abd_alloc_sametype(zio->io_abd,
1267 1264 is->is_size);
1268 1265 ic->ic_duplicate = NULL;
1269 1266
1270 1267 zio_nowait(zio_vdev_child_io(zio, NULL,
1271 1268 ic->ic_vdev, is->is_target_offset, ic->ic_data,
1272 1269 is->is_size, zio->io_type, zio->io_priority, 0,
1273 1270 vdev_indirect_read_split_done, ic));
1274 1271 }
1275 1272 }
1276 1273 iv->iv_reconstruct = B_TRUE;
1277 1274 }
1278 1275
1279 1276 static void
1280 1277 vdev_indirect_io_start(zio_t *zio)
1281 1278 {
1282 1279 spa_t *spa = zio->io_spa;
1283 1280 indirect_vsd_t *iv = kmem_zalloc(sizeof (*iv), KM_SLEEP);
1284 1281 list_create(&iv->iv_splits,
1285 1282 sizeof (indirect_split_t), offsetof(indirect_split_t, is_node));
1286 1283
1287 1284 zio->io_vsd = iv;
1288 1285 zio->io_vsd_ops = &vdev_indirect_vsd_ops;
1289 1286
1290 1287 ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
1291 1288 if (zio->io_type != ZIO_TYPE_READ) {
1292 1289 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE);
1293 1290 /*
1294 1291 * Note: this code can handle other kinds of writes,
1295 1292 * but we don't expect them.
1296 1293 */
1297 1294 ASSERT((zio->io_flags & (ZIO_FLAG_SELF_HEAL |
1298 1295 ZIO_FLAG_RESILVER | ZIO_FLAG_INDUCE_DAMAGE)) != 0);
1299 1296 }
1300 1297
1301 1298 vdev_indirect_remap(zio->io_vd, zio->io_offset, zio->io_size,
1302 1299 vdev_indirect_gather_splits, zio);
1303 1300
1304 1301 indirect_split_t *first = list_head(&iv->iv_splits);
1305 1302 if (first->is_size == zio->io_size) {
1306 1303 /*
1307 1304 * This is not a split block; we are pointing to the entire
1308 1305 * data, which will checksum the same as the original data.
1309 1306 * Pass the BP down so that the child i/o can verify the
1310 1307 * checksum, and try a different location if available
1311 1308 * (e.g. on a mirror).
1312 1309 *
1313 1310 * While this special case could be handled the same as the
1314 1311 * general (split block) case, doing it this way ensures
1315 1312 * that the vast majority of blocks on indirect vdevs
1316 1313 * (which are not split) are handled identically to blocks
1317 1314 * on non-indirect vdevs. This allows us to be less strict
1318 1315 * about performance in the general (but rare) case.
1319 1316 */
1320 1317 ASSERT0(first->is_split_offset);
1321 1318 ASSERT3P(list_next(&iv->iv_splits, first), ==, NULL);
1322 1319 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
1323 1320 first->is_vdev, first->is_target_offset,
1324 1321 abd_get_offset(zio->io_abd, 0),
1325 1322 zio->io_size, zio->io_type, zio->io_priority, 0,
1326 1323 vdev_indirect_child_io_done, zio));
1327 1324 } else {
1328 1325 iv->iv_split_block = B_TRUE;
1329 1326 if (zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)) {
1330 1327 /*
1331 1328 * Read all copies. Note that for simplicity,
1332 1329 * we don't bother consulting the DTL in the
1333 1330 * resilver case.
1334 1331 */
1335 1332 vdev_indirect_read_all(zio);
1336 1333 } else {
1337 1334 /*
1338 1335 * Read one copy of each split segment, from the
1339 1336 * top-level vdev. Since we don't know the
1340 1337 * checksum of each split individually, the child
1341 1338 * zio can't ensure that we get the right data.
1342 1339 * E.g. if it's a mirror, it will just read from a
1343 1340 * random (healthy) leaf vdev. We have to verify
1344 1341 * the checksum in vdev_indirect_io_done().
1345 1342 */
1346 1343 for (indirect_split_t *is = list_head(&iv->iv_splits);
1347 1344 is != NULL; is = list_next(&iv->iv_splits, is)) {
1348 1345 zio_nowait(zio_vdev_child_io(zio, NULL,
1349 1346 is->is_vdev, is->is_target_offset,
1350 1347 abd_get_offset(zio->io_abd,
1351 1348 is->is_split_offset),
1352 1349 is->is_size, zio->io_type,
1353 1350 zio->io_priority, 0,
1354 1351 vdev_indirect_child_io_done, zio));
1355 1352 }
1356 1353 }
1357 1354 }
1358 1355
1359 1356 zio_execute(zio);
1360 1357 }
1361 1358
1362 1359 /*
1363 1360 * Report a checksum error for a child.
1364 1361 */
1365 1362 static void
1366 1363 vdev_indirect_checksum_error(zio_t *zio,
1367 1364 indirect_split_t *is, indirect_child_t *ic)
1368 1365 {
1369 1366 vdev_t *vd = ic->ic_vdev;
1370 1367
1371 1368 if (zio->io_flags & ZIO_FLAG_SPECULATIVE)
1372 1369 return;
1373 1370
1374 1371 mutex_enter(&vd->vdev_stat_lock);
1375 1372 vd->vdev_stat.vs_checksum_errors++;
1376 1373 mutex_exit(&vd->vdev_stat_lock);
1377 1374
1378 1375 zio_bad_cksum_t zbc = { 0 };
1379 1376 void *bad_buf = abd_borrow_buf_copy(ic->ic_data, is->is_size);
1380 1377 abd_t *good_abd = is->is_good_child->ic_data;
1381 1378 void *good_buf = abd_borrow_buf_copy(good_abd, is->is_size);
1382 1379 zfs_ereport_post_checksum(zio->io_spa, vd, zio,
1383 1380 is->is_target_offset, is->is_size, good_buf, bad_buf, &zbc);
1384 1381 abd_return_buf(ic->ic_data, bad_buf, is->is_size);
1385 1382 abd_return_buf(good_abd, good_buf, is->is_size);
1386 1383 }
1387 1384
1388 1385 /*
1389 1386 * Issue repair i/os for any incorrect copies. We do this by comparing
1390 1387 * each split segment's correct data (is_good_child's ic_data) with each
1391 1388 * other copy of the data. If they differ, then we overwrite the bad data
1392 1389 * with the good copy. Note that we do this without regard for the DTL's,
1393 1390 * which simplifies this code and also issues the optimal number of writes
1394 1391 * (based on which copies actually read bad data, as opposed to which we
1395 1392 * think might be wrong). For the same reason, we always use
1396 1393 * ZIO_FLAG_SELF_HEAL, to bypass the DTL check in zio_vdev_io_start().
1397 1394 */
1398 1395 static void
1399 1396 vdev_indirect_repair(zio_t *zio)
1400 1397 {
1401 1398 indirect_vsd_t *iv = zio->io_vsd;
1402 1399
1403 1400 enum zio_flag flags = ZIO_FLAG_IO_REPAIR;
1404 1401
1405 1402 if (!(zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)))
1406 1403 flags |= ZIO_FLAG_SELF_HEAL;
1407 1404
1408 1405 if (!spa_writeable(zio->io_spa))
1409 1406 return;
1410 1407
1411 1408 for (indirect_split_t *is = list_head(&iv->iv_splits);
1412 1409 is != NULL; is = list_next(&iv->iv_splits, is)) {
1413 1410 for (int c = 0; c < is->is_children; c++) {
1414 1411 indirect_child_t *ic = &is->is_child[c];
1415 1412 if (ic == is->is_good_child)
1416 1413 continue;
1417 1414 if (ic->ic_data == NULL)
1418 1415 continue;
1419 1416 if (ic->ic_duplicate == is->is_good_child)
1420 1417 continue;
1421 1418
1422 1419 zio_nowait(zio_vdev_child_io(zio, NULL,
1423 1420 ic->ic_vdev, is->is_target_offset,
1424 1421 is->is_good_child->ic_data, is->is_size,
1425 1422 ZIO_TYPE_WRITE, ZIO_PRIORITY_ASYNC_WRITE,
1426 1423 ZIO_FLAG_IO_REPAIR | ZIO_FLAG_SELF_HEAL,
1427 1424 NULL, NULL));
1428 1425
1429 1426 vdev_indirect_checksum_error(zio, is, ic);
1430 1427 }
1431 1428 }
1432 1429 }
1433 1430
1434 1431 /*
1435 1432 * Report checksum errors on all children that we read from.
1436 1433 */
1437 1434 static void
1438 1435 vdev_indirect_all_checksum_errors(zio_t *zio)
1439 1436 {
1440 1437 indirect_vsd_t *iv = zio->io_vsd;
1441 1438
1442 1439 if (zio->io_flags & ZIO_FLAG_SPECULATIVE)
1443 1440 return;
1444 1441
1445 1442 for (indirect_split_t *is = list_head(&iv->iv_splits);
1446 1443 is != NULL; is = list_next(&iv->iv_splits, is)) {
1447 1444 for (int c = 0; c < is->is_children; c++) {
1448 1445 indirect_child_t *ic = &is->is_child[c];
1449 1446
1450 1447 if (ic->ic_data == NULL)
1451 1448 continue;
1452 1449
1453 1450 vdev_t *vd = ic->ic_vdev;
1454 1451
1455 1452 mutex_enter(&vd->vdev_stat_lock);
1456 1453 vd->vdev_stat.vs_checksum_errors++;
1457 1454 mutex_exit(&vd->vdev_stat_lock);
1458 1455
1459 1456 zfs_ereport_post_checksum(zio->io_spa, vd, zio,
1460 1457 is->is_target_offset, is->is_size,
1461 1458 NULL, NULL, NULL);
1462 1459 }
1463 1460 }
1464 1461 }
1465 1462
1466 1463 /*
1467 1464 * Copy data from all the splits to a main zio then validate the checksum.
1468 1465 * If then checksum is successfully validated return success.
1469 1466 */
1470 1467 static int
1471 1468 vdev_indirect_splits_checksum_validate(indirect_vsd_t *iv, zio_t *zio)
1472 1469 {
1473 1470 zio_bad_cksum_t zbc;
1474 1471
1475 1472 for (indirect_split_t *is = list_head(&iv->iv_splits);
1476 1473 is != NULL; is = list_next(&iv->iv_splits, is)) {
1477 1474
1478 1475 ASSERT3P(is->is_good_child->ic_data, !=, NULL);
1479 1476 ASSERT3P(is->is_good_child->ic_duplicate, ==, NULL);
1480 1477
1481 1478 abd_copy_off(zio->io_abd, is->is_good_child->ic_data,
1482 1479 is->is_split_offset, 0, is->is_size);
1483 1480 }
1484 1481
1485 1482 return (zio_checksum_error(zio, &zbc));
1486 1483 }
1487 1484
1488 1485 /*
1489 1486 * There are relatively few possible combinations making it feasible to
1490 1487 * deterministically check them all. We do this by setting the good_child
1491 1488 * to the next unique split version. If we reach the end of the list then
1492 1489 * "carry over" to the next unique split version (like counting in base
1493 1490 * is_unique_children, but each digit can have a different base).
1494 1491 */
1495 1492 static int
1496 1493 vdev_indirect_splits_enumerate_all(indirect_vsd_t *iv, zio_t *zio)
1497 1494 {
1498 1495 boolean_t more = B_TRUE;
1499 1496
1500 1497 iv->iv_attempts = 0;
1501 1498
1502 1499 for (indirect_split_t *is = list_head(&iv->iv_splits);
1503 1500 is != NULL; is = list_next(&iv->iv_splits, is))
1504 1501 is->is_good_child = list_head(&is->is_unique_child);
1505 1502
1506 1503 while (more == B_TRUE) {
1507 1504 iv->iv_attempts++;
1508 1505 more = B_FALSE;
1509 1506
1510 1507 if (vdev_indirect_splits_checksum_validate(iv, zio) == 0)
1511 1508 return (0);
1512 1509
1513 1510 for (indirect_split_t *is = list_head(&iv->iv_splits);
1514 1511 is != NULL; is = list_next(&iv->iv_splits, is)) {
1515 1512 is->is_good_child = list_next(&is->is_unique_child,
1516 1513 is->is_good_child);
1517 1514 if (is->is_good_child != NULL) {
1518 1515 more = B_TRUE;
1519 1516 break;
1520 1517 }
1521 1518
1522 1519 is->is_good_child = list_head(&is->is_unique_child);
1523 1520 }
1524 1521 }
1525 1522
1526 1523 ASSERT3S(iv->iv_attempts, <=, iv->iv_unique_combinations);
1527 1524
1528 1525 return (SET_ERROR(ECKSUM));
1529 1526 }
1530 1527
1531 1528 /*
1532 1529 * There are too many combinations to try all of them in a reasonable amount
1533 1530 * of time. So try a fixed number of random combinations from the unique
1534 1531 * split versions, after which we'll consider the block unrecoverable.
1535 1532 */
1536 1533 static int
1537 1534 vdev_indirect_splits_enumerate_randomly(indirect_vsd_t *iv, zio_t *zio)
1538 1535 {
1539 1536 iv->iv_attempts = 0;
1540 1537
1541 1538 while (iv->iv_attempts < iv->iv_attempts_max) {
1542 1539 iv->iv_attempts++;
1543 1540
1544 1541 for (indirect_split_t *is = list_head(&iv->iv_splits);
1545 1542 is != NULL; is = list_next(&iv->iv_splits, is)) {
1546 1543 indirect_child_t *ic = list_head(&is->is_unique_child);
1547 1544 int children = is->is_unique_children;
1548 1545
1549 1546 for (int i = spa_get_random(children); i > 0; i--)
1550 1547 ic = list_next(&is->is_unique_child, ic);
1551 1548
1552 1549 ASSERT3P(ic, !=, NULL);
1553 1550 is->is_good_child = ic;
1554 1551 }
1555 1552
1556 1553 if (vdev_indirect_splits_checksum_validate(iv, zio) == 0)
1557 1554 return (0);
1558 1555 }
1559 1556
1560 1557 return (SET_ERROR(ECKSUM));
1561 1558 }
1562 1559
1563 1560 /*
1564 1561 * This is a validation function for reconstruction. It randomly selects
1565 1562 * a good combination, if one can be found, and then it intentionally
1566 1563 * damages all other segment copes by zeroing them. This forces the
1567 1564 * reconstruction algorithm to locate the one remaining known good copy.
1568 1565 */
1569 1566 static int
1570 1567 vdev_indirect_splits_damage(indirect_vsd_t *iv, zio_t *zio)
1571 1568 {
1572 1569 /* Presume all the copies are unique for initial selection. */
1573 1570 for (indirect_split_t *is = list_head(&iv->iv_splits);
1574 1571 is != NULL; is = list_next(&iv->iv_splits, is)) {
1575 1572 is->is_unique_children = 0;
1576 1573
1577 1574 for (int i = 0; i < is->is_children; i++) {
1578 1575 indirect_child_t *ic = &is->is_child[i];
1579 1576 if (ic->ic_data != NULL) {
1580 1577 is->is_unique_children++;
1581 1578 list_insert_tail(&is->is_unique_child, ic);
1582 1579 }
1583 1580 }
1584 1581 }
1585 1582
1586 1583 /*
1587 1584 * Set each is_good_child to a randomly-selected child which
1588 1585 * is known to contain validated data.
1589 1586 */
1590 1587 int error = vdev_indirect_splits_enumerate_randomly(iv, zio);
1591 1588 if (error)
1592 1589 goto out;
1593 1590
1594 1591 /*
1595 1592 * Damage all but the known good copy by zeroing it. This will
1596 1593 * result in two or less unique copies per indirect_child_t.
1597 1594 * Both may need to be checked in order to reconstruct the block.
1598 1595 * Set iv->iv_attempts_max such that all unique combinations will
1599 1596 * enumerated, but limit the damage to at most 16 indirect splits.
1600 1597 */
1601 1598 iv->iv_attempts_max = 1;
1602 1599
1603 1600 for (indirect_split_t *is = list_head(&iv->iv_splits);
1604 1601 is != NULL; is = list_next(&iv->iv_splits, is)) {
1605 1602 for (int c = 0; c < is->is_children; c++) {
1606 1603 indirect_child_t *ic = &is->is_child[c];
1607 1604
1608 1605 if (ic == is->is_good_child)
1609 1606 continue;
1610 1607 if (ic->ic_data == NULL)
1611 1608 continue;
1612 1609
1613 1610 abd_zero(ic->ic_data, ic->ic_data->abd_size);
1614 1611 }
1615 1612
1616 1613 iv->iv_attempts_max *= 2;
1617 1614 if (iv->iv_attempts_max > (1ULL << 16)) {
1618 1615 iv->iv_attempts_max = UINT64_MAX;
1619 1616 break;
1620 1617 }
1621 1618 }
1622 1619
1623 1620 out:
1624 1621 /* Empty the unique children lists so they can be reconstructed. */
1625 1622 for (indirect_split_t *is = list_head(&iv->iv_splits);
1626 1623 is != NULL; is = list_next(&iv->iv_splits, is)) {
1627 1624 indirect_child_t *ic;
1628 1625 while ((ic = list_head(&is->is_unique_child)) != NULL)
1629 1626 list_remove(&is->is_unique_child, ic);
1630 1627
1631 1628 is->is_unique_children = 0;
1632 1629 }
1633 1630
1634 1631 return (error);
1635 1632 }
1636 1633
1637 1634 /*
1638 1635 * This function is called when we have read all copies of the data and need
1639 1636 * to try to find a combination of copies that gives us the right checksum.
1640 1637 *
1641 1638 * If we pointed to any mirror vdevs, this effectively does the job of the
1642 1639 * mirror. The mirror vdev code can't do its own job because we don't know
1643 1640 * the checksum of each split segment individually.
1644 1641 *
1645 1642 * We have to try every unique combination of copies of split segments, until
1646 1643 * we find one that checksums correctly. Duplicate segment copies are first
1647 1644 * identified and latter skipped during reconstruction. This optimization
1648 1645 * reduces the search space and ensures that of the remaining combinations
1649 1646 * at most one is correct.
1650 1647 *
1651 1648 * When the total number of combinations is small they can all be checked.
1652 1649 * For example, if we have 3 segments in the split, and each points to a
1653 1650 * 2-way mirror with unique copies, we will have the following pieces of data:
1654 1651 *
1655 1652 * | mirror child
1656 1653 * split | [0] [1]
1657 1654 * ======|=====================
1658 1655 * A | data_A_0 data_A_1
1659 1656 * B | data_B_0 data_B_1
1660 1657 * C | data_C_0 data_C_1
1661 1658 *
1662 1659 * We will try the following (mirror children)^(number of splits) (2^3=8)
1663 1660 * combinations, which is similar to bitwise-little-endian counting in
1664 1661 * binary. In general each "digit" corresponds to a split segment, and the
1665 1662 * base of each digit is is_children, which can be different for each
1666 1663 * digit.
1667 1664 *
1668 1665 * "low bit" "high bit"
1669 1666 * v v
1670 1667 * data_A_0 data_B_0 data_C_0
1671 1668 * data_A_1 data_B_0 data_C_0
1672 1669 * data_A_0 data_B_1 data_C_0
1673 1670 * data_A_1 data_B_1 data_C_0
1674 1671 * data_A_0 data_B_0 data_C_1
1675 1672 * data_A_1 data_B_0 data_C_1
1676 1673 * data_A_0 data_B_1 data_C_1
1677 1674 * data_A_1 data_B_1 data_C_1
1678 1675 *
1679 1676 * Note that the split segments may be on the same or different top-level
1680 1677 * vdevs. In either case, we may need to try lots of combinations (see
1681 1678 * zfs_reconstruct_indirect_combinations_max). This ensures that if a mirror
1682 1679 * has small silent errors on all of its children, we can still reconstruct
1683 1680 * the correct data, as long as those errors are at sufficiently-separated
1684 1681 * offsets (specifically, separated by the largest block size - default of
1685 1682 * 128KB, but up to 16MB).
1686 1683 */
1687 1684 static void
1688 1685 vdev_indirect_reconstruct_io_done(zio_t *zio)
1689 1686 {
1690 1687 indirect_vsd_t *iv = zio->io_vsd;
1691 1688 boolean_t known_good = B_FALSE;
1692 1689 int error;
1693 1690
1694 1691 iv->iv_unique_combinations = 1;
1695 1692 iv->iv_attempts_max = UINT64_MAX;
1696 1693
1697 1694 if (zfs_reconstruct_indirect_combinations_max > 0)
1698 1695 iv->iv_attempts_max = zfs_reconstruct_indirect_combinations_max;
1699 1696
1700 1697 /*
1701 1698 * If nonzero, every 1/x blocks will be damaged, in order to validate
1702 1699 * reconstruction when there are split segments with damaged copies.
1703 1700 * Known_good will TRUE when reconstruction is known to be possible.
1704 1701 */
1705 1702 if (zfs_reconstruct_indirect_damage_fraction != 0 &&
1706 1703 spa_get_random(zfs_reconstruct_indirect_damage_fraction) == 0)
1707 1704 known_good = (vdev_indirect_splits_damage(iv, zio) == 0);
1708 1705
1709 1706 /*
1710 1707 * Determine the unique children for a split segment and add them
1711 1708 * to the is_unique_child list. By restricting reconstruction
1712 1709 * to these children, only unique combinations will be considered.
1713 1710 * This can vastly reduce the search space when there are a large
1714 1711 * number of indirect splits.
1715 1712 */
1716 1713 for (indirect_split_t *is = list_head(&iv->iv_splits);
1717 1714 is != NULL; is = list_next(&iv->iv_splits, is)) {
1718 1715 is->is_unique_children = 0;
1719 1716
1720 1717 for (int i = 0; i < is->is_children; i++) {
1721 1718 indirect_child_t *ic_i = &is->is_child[i];
1722 1719
1723 1720 if (ic_i->ic_data == NULL ||
1724 1721 ic_i->ic_duplicate != NULL)
1725 1722 continue;
1726 1723
1727 1724 for (int j = i + 1; j < is->is_children; j++) {
1728 1725 indirect_child_t *ic_j = &is->is_child[j];
1729 1726
1730 1727 if (ic_j->ic_data == NULL ||
1731 1728 ic_j->ic_duplicate != NULL)
1732 1729 continue;
1733 1730
1734 1731 if (abd_cmp(ic_i->ic_data, ic_j->ic_data,
1735 1732 is->is_size) == 0) {
1736 1733 ic_j->ic_duplicate = ic_i;
1737 1734 }
1738 1735 }
1739 1736
1740 1737 is->is_unique_children++;
1741 1738 list_insert_tail(&is->is_unique_child, ic_i);
1742 1739 }
1743 1740
1744 1741 /* Reconstruction is impossible, no valid children */
1745 1742 EQUIV(list_is_empty(&is->is_unique_child),
1746 1743 is->is_unique_children == 0);
1747 1744 if (list_is_empty(&is->is_unique_child)) {
1748 1745 zio->io_error = EIO;
1749 1746 vdev_indirect_all_checksum_errors(zio);
1750 1747 zio_checksum_verified(zio);
1751 1748 return;
1752 1749 }
1753 1750
1754 1751 iv->iv_unique_combinations *= is->is_unique_children;
1755 1752 }
1756 1753
1757 1754 if (iv->iv_unique_combinations <= iv->iv_attempts_max)
1758 1755 error = vdev_indirect_splits_enumerate_all(iv, zio);
1759 1756 else
1760 1757 error = vdev_indirect_splits_enumerate_randomly(iv, zio);
1761 1758
1762 1759 if (error != 0) {
1763 1760 /* All attempted combinations failed. */
1764 1761 ASSERT3B(known_good, ==, B_FALSE);
1765 1762 zio->io_error = error;
1766 1763 vdev_indirect_all_checksum_errors(zio);
1767 1764 } else {
1768 1765 /*
1769 1766 * The checksum has been successfully validated. Issue
1770 1767 * repair I/Os to any copies of splits which don't match
1771 1768 * the validated version.
1772 1769 */
1773 1770 ASSERT0(vdev_indirect_splits_checksum_validate(iv, zio));
1774 1771 vdev_indirect_repair(zio);
1775 1772 zio_checksum_verified(zio);
1776 1773 }
1777 1774 }
1778 1775
1779 1776 static void
1780 1777 vdev_indirect_io_done(zio_t *zio)
1781 1778 {
1782 1779 indirect_vsd_t *iv = zio->io_vsd;
1783 1780
1784 1781 if (iv->iv_reconstruct) {
1785 1782 /*
1786 1783 * We have read all copies of the data (e.g. from mirrors),
1787 1784 * either because this was a scrub/resilver, or because the
1788 1785 * one-copy read didn't checksum correctly.
1789 1786 */
1790 1787 vdev_indirect_reconstruct_io_done(zio);
1791 1788 return;
1792 1789 }
1793 1790
1794 1791 if (!iv->iv_split_block) {
1795 1792 /*
1796 1793 * This was not a split block, so we passed the BP down,
1797 1794 * and the checksum was handled by the (one) child zio.
1798 1795 */
1799 1796 return;
1800 1797 }
1801 1798
1802 1799 zio_bad_cksum_t zbc;
1803 1800 int ret = zio_checksum_error(zio, &zbc);
1804 1801 if (ret == 0) {
1805 1802 zio_checksum_verified(zio);
1806 1803 return;
1807 1804 }
1808 1805
1809 1806 /*
1810 1807 * The checksum didn't match. Read all copies of all splits, and
1811 1808 * then we will try to reconstruct. The next time
1812 1809 * vdev_indirect_io_done() is called, iv_reconstruct will be set.
1813 1810 */
1814 1811 vdev_indirect_read_all(zio);
1815 1812
1816 1813 zio_vdev_io_redone(zio);
1817 1814 }
1818 1815
1819 1816 vdev_ops_t vdev_indirect_ops = {
1820 1817 vdev_indirect_open,
1821 1818 vdev_indirect_close,
1822 1819 vdev_default_asize,
1823 1820 vdev_indirect_io_start,
1824 1821 vdev_indirect_io_done,
1825 1822 NULL,
1826 1823 NULL,
1827 1824 NULL,
1828 1825 vdev_indirect_remap,
1829 1826 NULL,
1830 1827 VDEV_TYPE_INDIRECT, /* name of this vdev type */
1831 1828 B_FALSE /* leaf vdev */
1832 1829 };
|
↓ open down ↓ |
977 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX