Print this page
NEX-6088 ZFS scrub/resilver take excessively long due to issuing lots of random IO
Reviewed by: Roman Strashkin <roman.strashkin@nexenta.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/lib/libzfs/common/libzfs_status.c
+++ new/usr/src/lib/libzfs/common/libzfs_status.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright (c) 2012 by Delphix. All rights reserved.
25 25 * Copyright (c) 2013 Steven Hartland. All rights reserved.
26 26 */
27 27
28 28 /*
29 29 * This file contains the functions which analyze the status of a pool. This
30 30 * include both the status of an active pool, as well as the status exported
31 31 * pools. Returns one of the ZPOOL_STATUS_* defines describing the status of
32 32 * the pool. This status is independent (to a certain degree) from the state of
33 33 * the pool. A pool's state describes only whether or not it is capable of
34 34 * providing the necessary fault tolerance for data. The status describes the
35 35 * overall status of devices. A pool that is online can still have a device
36 36 * that is experiencing errors.
37 37 *
38 38 * Only a subset of the possible faults can be detected using 'zpool status',
39 39 * and not all possible errors correspond to a FMA message ID. The explanation
40 40 * is left up to the caller, depending on whether it is a live pool or an
41 41 * import.
42 42 */
43 43
44 44 #include <libzfs.h>
45 45 #include <string.h>
46 46 #include <unistd.h>
47 47 #include "libzfs_impl.h"
48 48 #include "zfeature_common.h"
49 49
50 50 /*
51 51 * Message ID table. This must be kept in sync with the ZPOOL_STATUS_* defines
52 52 * in libzfs.h. Note that there are some status results which go past the end
53 53 * of this table, and hence have no associated message ID.
54 54 */
55 55 static char *zfs_msgid_table[] = {
56 56 "ZFS-8000-14",
57 57 "ZFS-8000-2Q",
58 58 "ZFS-8000-3C",
59 59 "ZFS-8000-4J",
60 60 "ZFS-8000-5E",
61 61 "ZFS-8000-6X",
62 62 "ZFS-8000-72",
63 63 "ZFS-8000-8A",
64 64 "ZFS-8000-9P",
65 65 "ZFS-8000-A5",
66 66 "ZFS-8000-EY",
67 67 "ZFS-8000-HC",
68 68 "ZFS-8000-JQ",
69 69 "ZFS-8000-K4",
70 70 };
71 71
72 72 #define NMSGID (sizeof (zfs_msgid_table) / sizeof (zfs_msgid_table[0]))
73 73
74 74 /* ARGSUSED */
75 75 static int
76 76 vdev_missing(uint64_t state, uint64_t aux, uint64_t errs)
77 77 {
78 78 return (state == VDEV_STATE_CANT_OPEN &&
79 79 aux == VDEV_AUX_OPEN_FAILED);
80 80 }
81 81
82 82 /* ARGSUSED */
83 83 static int
84 84 vdev_faulted(uint64_t state, uint64_t aux, uint64_t errs)
85 85 {
86 86 return (state == VDEV_STATE_FAULTED);
87 87 }
88 88
89 89 /* ARGSUSED */
90 90 static int
91 91 vdev_errors(uint64_t state, uint64_t aux, uint64_t errs)
92 92 {
93 93 return (state == VDEV_STATE_DEGRADED || errs != 0);
94 94 }
95 95
96 96 /* ARGSUSED */
97 97 static int
98 98 vdev_broken(uint64_t state, uint64_t aux, uint64_t errs)
99 99 {
100 100 return (state == VDEV_STATE_CANT_OPEN);
101 101 }
102 102
103 103 /* ARGSUSED */
104 104 static int
105 105 vdev_offlined(uint64_t state, uint64_t aux, uint64_t errs)
106 106 {
107 107 return (state == VDEV_STATE_OFFLINE);
108 108 }
109 109
110 110 /* ARGSUSED */
111 111 static int
112 112 vdev_removed(uint64_t state, uint64_t aux, uint64_t errs)
113 113 {
114 114 return (state == VDEV_STATE_REMOVED);
115 115 }
116 116
117 117 /*
118 118 * Detect if any leaf devices that have seen errors or could not be opened.
119 119 */
120 120 static boolean_t
121 121 find_vdev_problem(nvlist_t *vdev, int (*func)(uint64_t, uint64_t, uint64_t))
122 122 {
123 123 nvlist_t **child;
124 124 vdev_stat_t *vs;
125 125 uint_t c, children;
126 126 char *type;
127 127
128 128 /*
129 129 * Ignore problems within a 'replacing' vdev, since we're presumably in
130 130 * the process of repairing any such errors, and don't want to call them
131 131 * out again. We'll pick up the fact that a resilver is happening
132 132 * later.
133 133 */
134 134 verify(nvlist_lookup_string(vdev, ZPOOL_CONFIG_TYPE, &type) == 0);
135 135 if (strcmp(type, VDEV_TYPE_REPLACING) == 0)
136 136 return (B_FALSE);
137 137
138 138 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_CHILDREN, &child,
139 139 &children) == 0) {
140 140 for (c = 0; c < children; c++)
141 141 if (find_vdev_problem(child[c], func))
142 142 return (B_TRUE);
143 143 } else {
144 144 verify(nvlist_lookup_uint64_array(vdev, ZPOOL_CONFIG_VDEV_STATS,
145 145 (uint64_t **)&vs, &c) == 0);
146 146
147 147 if (func(vs->vs_state, vs->vs_aux,
148 148 vs->vs_read_errors +
149 149 vs->vs_write_errors +
150 150 vs->vs_checksum_errors))
151 151 return (B_TRUE);
152 152 }
153 153
154 154 /*
155 155 * Check any L2 cache devs
156 156 */
157 157 if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_L2CACHE, &child,
158 158 &children) == 0) {
159 159 for (c = 0; c < children; c++)
160 160 if (find_vdev_problem(child[c], func))
161 161 return (B_TRUE);
162 162 }
163 163
164 164 return (B_FALSE);
165 165 }
166 166
167 167 /*
168 168 * Active pool health status.
169 169 *
170 170 * To determine the status for a pool, we make several passes over the config,
171 171 * picking the most egregious error we find. In order of importance, we do the
172 172 * following:
173 173 *
174 174 * - Check for a complete and valid configuration
175 175 * - Look for any faulted or missing devices in a non-replicated config
176 176 * - Check for any data errors
177 177 * - Check for any faulted or missing devices in a replicated config
178 178 * - Look for any devices showing errors
179 179 * - Check for any resilvering devices
180 180 *
181 181 * There can obviously be multiple errors within a single pool, so this routine
182 182 * only picks the most damaging of all the current errors to report.
183 183 */
184 184 static zpool_status_t
185 185 check_status(nvlist_t *config, boolean_t isimport)
186 186 {
187 187 nvlist_t *nvroot;
188 188 vdev_stat_t *vs;
189 189 pool_scan_stat_t *ps = NULL;
190 190 uint_t vsc, psc;
191 191 uint64_t nerr;
192 192 uint64_t version;
193 193 uint64_t stateval;
194 194 uint64_t suspended;
195 195 uint64_t hostid = 0;
196 196
197 197 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
198 198 &version) == 0);
199 199 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
200 200 &nvroot) == 0);
201 201 verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
|
↓ open down ↓ |
201 lines elided |
↑ open up ↑ |
202 202 (uint64_t **)&vs, &vsc) == 0);
203 203 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
204 204 &stateval) == 0);
205 205
206 206 /*
207 207 * Currently resilvering a vdev
208 208 */
209 209 (void) nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
210 210 (uint64_t **)&ps, &psc);
211 211 if (ps && ps->pss_func == POOL_SCAN_RESILVER &&
212 - ps->pss_state == DSS_SCANNING)
212 + (ps->pss_state == DSS_SCANNING || ps->pss_state == DSS_FINISHING))
213 213 return (ZPOOL_STATUS_RESILVERING);
214 214
215 215 /*
216 216 * Pool last accessed by another system.
217 217 */
218 218 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
219 219 if (hostid != 0 && (unsigned long)hostid != gethostid() &&
220 220 stateval == POOL_STATE_ACTIVE)
221 221 return (ZPOOL_STATUS_HOSTID_MISMATCH);
222 222
223 223 /*
224 224 * Newer on-disk version.
225 225 */
226 226 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
227 227 vs->vs_aux == VDEV_AUX_VERSION_NEWER)
228 228 return (ZPOOL_STATUS_VERSION_NEWER);
229 229
230 230 /*
231 231 * Unsupported feature(s).
232 232 */
233 233 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
234 234 vs->vs_aux == VDEV_AUX_UNSUP_FEAT) {
235 235 nvlist_t *nvinfo;
236 236
237 237 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
238 238 &nvinfo) == 0);
239 239 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_CAN_RDONLY))
240 240 return (ZPOOL_STATUS_UNSUP_FEAT_WRITE);
241 241 return (ZPOOL_STATUS_UNSUP_FEAT_READ);
242 242 }
243 243
244 244 /*
245 245 * Check that the config is complete.
246 246 */
247 247 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
248 248 vs->vs_aux == VDEV_AUX_BAD_GUID_SUM)
249 249 return (ZPOOL_STATUS_BAD_GUID_SUM);
250 250
251 251 /*
252 252 * Check whether the pool has suspended due to failed I/O.
253 253 */
254 254 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED,
255 255 &suspended) == 0) {
256 256 if (suspended == ZIO_FAILURE_MODE_CONTINUE)
257 257 return (ZPOOL_STATUS_IO_FAILURE_CONTINUE);
258 258 return (ZPOOL_STATUS_IO_FAILURE_WAIT);
259 259 }
260 260
261 261 /*
262 262 * Could not read a log.
263 263 */
264 264 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
265 265 vs->vs_aux == VDEV_AUX_BAD_LOG) {
266 266 return (ZPOOL_STATUS_BAD_LOG);
267 267 }
268 268
269 269 /*
270 270 * Bad devices in non-replicated config.
271 271 */
272 272 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
273 273 find_vdev_problem(nvroot, vdev_faulted))
274 274 return (ZPOOL_STATUS_FAULTED_DEV_NR);
275 275
276 276 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
277 277 find_vdev_problem(nvroot, vdev_missing))
278 278 return (ZPOOL_STATUS_MISSING_DEV_NR);
279 279
280 280 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
281 281 find_vdev_problem(nvroot, vdev_broken))
282 282 return (ZPOOL_STATUS_CORRUPT_LABEL_NR);
283 283
284 284 /*
285 285 * Corrupted pool metadata
286 286 */
287 287 if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
288 288 vs->vs_aux == VDEV_AUX_CORRUPT_DATA)
289 289 return (ZPOOL_STATUS_CORRUPT_POOL);
290 290
291 291 /*
292 292 * Persistent data errors.
293 293 */
294 294 if (!isimport) {
295 295 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
296 296 &nerr) == 0 && nerr != 0)
297 297 return (ZPOOL_STATUS_CORRUPT_DATA);
298 298 }
299 299
300 300 /*
301 301 * Missing devices in a replicated config.
302 302 */
303 303 if (find_vdev_problem(nvroot, vdev_faulted))
304 304 return (ZPOOL_STATUS_FAULTED_DEV_R);
305 305 if (find_vdev_problem(nvroot, vdev_missing))
306 306 return (ZPOOL_STATUS_MISSING_DEV_R);
307 307 if (find_vdev_problem(nvroot, vdev_broken))
308 308 return (ZPOOL_STATUS_CORRUPT_LABEL_R);
309 309
310 310 /*
311 311 * Devices with errors
312 312 */
313 313 if (!isimport && find_vdev_problem(nvroot, vdev_errors))
314 314 return (ZPOOL_STATUS_FAILING_DEV);
315 315
316 316 /*
317 317 * Offlined devices
318 318 */
319 319 if (find_vdev_problem(nvroot, vdev_offlined))
320 320 return (ZPOOL_STATUS_OFFLINE_DEV);
321 321
322 322 /*
323 323 * Removed device
324 324 */
325 325 if (find_vdev_problem(nvroot, vdev_removed))
326 326 return (ZPOOL_STATUS_REMOVED_DEV);
327 327
328 328 /*
329 329 * Outdated, but usable, version
330 330 */
331 331 if (SPA_VERSION_IS_SUPPORTED(version) && version != SPA_VERSION)
332 332 return (ZPOOL_STATUS_VERSION_OLDER);
333 333
334 334 /*
335 335 * Usable pool with disabled features
336 336 */
337 337 if (version >= SPA_VERSION_FEATURES) {
338 338 int i;
339 339 nvlist_t *feat;
340 340
341 341 if (isimport) {
342 342 feat = fnvlist_lookup_nvlist(config,
343 343 ZPOOL_CONFIG_LOAD_INFO);
344 344 feat = fnvlist_lookup_nvlist(feat,
345 345 ZPOOL_CONFIG_ENABLED_FEAT);
346 346 } else {
347 347 feat = fnvlist_lookup_nvlist(config,
348 348 ZPOOL_CONFIG_FEATURE_STATS);
349 349 }
350 350
351 351 for (i = 0; i < SPA_FEATURES; i++) {
352 352 zfeature_info_t *fi = &spa_feature_table[i];
353 353 if (!nvlist_exists(feat, fi->fi_guid))
354 354 return (ZPOOL_STATUS_FEAT_DISABLED);
355 355 }
356 356 }
357 357
358 358 return (ZPOOL_STATUS_OK);
359 359 }
360 360
361 361 zpool_status_t
362 362 zpool_get_status(zpool_handle_t *zhp, char **msgid)
363 363 {
364 364 zpool_status_t ret = check_status(zhp->zpool_config, B_FALSE);
365 365
366 366 if (ret >= NMSGID)
367 367 *msgid = NULL;
368 368 else
369 369 *msgid = zfs_msgid_table[ret];
370 370
371 371 return (ret);
372 372 }
373 373
374 374 zpool_status_t
375 375 zpool_import_status(nvlist_t *config, char **msgid)
376 376 {
377 377 zpool_status_t ret = check_status(config, B_TRUE);
378 378
379 379 if (ret >= NMSGID)
380 380 *msgid = NULL;
381 381 else
382 382 *msgid = zfs_msgid_table[ret];
383 383
384 384 return (ret);
385 385 }
386 386
387 387 static void
388 388 dump_ddt_stat(const ddt_stat_t *dds, int h)
389 389 {
390 390 char refcnt[6];
391 391 char blocks[6], lsize[6], psize[6], dsize[6];
392 392 char ref_blocks[6], ref_lsize[6], ref_psize[6], ref_dsize[6];
393 393
394 394 if (dds == NULL || dds->dds_blocks == 0)
395 395 return;
396 396
397 397 if (h == -1)
398 398 (void) strcpy(refcnt, "Total");
399 399 else
400 400 zfs_nicenum(1ULL << h, refcnt, sizeof (refcnt));
401 401
402 402 zfs_nicenum(dds->dds_blocks, blocks, sizeof (blocks));
403 403 zfs_nicenum(dds->dds_lsize, lsize, sizeof (lsize));
404 404 zfs_nicenum(dds->dds_psize, psize, sizeof (psize));
405 405 zfs_nicenum(dds->dds_dsize, dsize, sizeof (dsize));
406 406 zfs_nicenum(dds->dds_ref_blocks, ref_blocks, sizeof (ref_blocks));
407 407 zfs_nicenum(dds->dds_ref_lsize, ref_lsize, sizeof (ref_lsize));
408 408 zfs_nicenum(dds->dds_ref_psize, ref_psize, sizeof (ref_psize));
409 409 zfs_nicenum(dds->dds_ref_dsize, ref_dsize, sizeof (ref_dsize));
410 410
411 411 (void) printf("%6s %6s %5s %5s %5s %6s %5s %5s %5s\n",
412 412 refcnt,
413 413 blocks, lsize, psize, dsize,
414 414 ref_blocks, ref_lsize, ref_psize, ref_dsize);
415 415 }
416 416
417 417 /*
418 418 * Print the DDT histogram and the column totals.
419 419 */
420 420 void
421 421 zpool_dump_ddt(const ddt_stat_t *dds_total, const ddt_histogram_t *ddh)
422 422 {
423 423 int h;
424 424
425 425 (void) printf("\n");
426 426
427 427 (void) printf("bucket "
428 428 " allocated "
429 429 " referenced \n");
430 430 (void) printf("______ "
431 431 "______________________________ "
432 432 "______________________________\n");
433 433
434 434 (void) printf("%6s %6s %5s %5s %5s %6s %5s %5s %5s\n",
435 435 "refcnt",
436 436 "blocks", "LSIZE", "PSIZE", "DSIZE",
437 437 "blocks", "LSIZE", "PSIZE", "DSIZE");
438 438
439 439 (void) printf("%6s %6s %5s %5s %5s %6s %5s %5s %5s\n",
440 440 "------",
441 441 "------", "-----", "-----", "-----",
442 442 "------", "-----", "-----", "-----");
443 443
444 444 for (h = 0; h < 64; h++)
445 445 dump_ddt_stat(&ddh->ddh_stat[h], h);
446 446
447 447 dump_ddt_stat(dds_total, -1);
448 448
449 449 (void) printf("\n");
450 450 }
|
↓ open down ↓ |
228 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX