Print this page
NEX-3288 fmd segfaults in zfs-diagnosis.so`zfs_fm_close
Reviewed by: Alek Pinchuk <alek.pinchuk@nexenta.com>
Reviewed by: Marcel Telka <marcel.telka@nexenta.com>
NEX-3011 zfs-diagnosis: Memory leak in zpool_find_load_time()
Reviewed by: Dan Fields <dan.fields@nexenta.com>
OS-70 remove zio timer code
re #12393 rb3935 Kerberos and smbd disagree about who is our AD server (fix elf runtime attributes check)
re #11612 rb3907 Failing vdev of a mirrored pool should not take zfs operations out of action for extended periods of time.
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/cmd/fm/modules/common/zfs-diagnosis/zfs_de.c
+++ new/usr/src/cmd/fm/modules/common/zfs-diagnosis/zfs_de.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 - * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
24 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
25 24 */
25 +/*
26 + * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
27 + */
26 28
27 29 #include <assert.h>
28 30 #include <stddef.h>
29 31 #include <strings.h>
30 32 #include <libuutil.h>
31 33 #include <libzfs.h>
32 34 #include <fm/fmd_api.h>
33 35 #include <fm/libtopo.h>
34 36 #include <sys/types.h>
35 37 #include <sys/time.h>
36 38 #include <sys/fs/zfs.h>
37 39 #include <sys/fm/protocol.h>
38 40 #include <sys/fm/fs/zfs.h>
39 41
40 42 /*
41 43 * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'. This
42 44 * #define reserves enough space for two 64-bit hex values plus the length of
43 45 * the longest string.
44 46 */
45 47 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
46 48
47 49 /*
48 50 * On-disk case structure. This must maintain backwards compatibility with
49 51 * previous versions of the DE. By default, any members appended to the end
50 52 * will be filled with zeros if they don't exist in a previous version.
51 53 */
52 54 typedef struct zfs_case_data {
53 55 uint64_t zc_version;
54 56 uint64_t zc_ena;
55 57 uint64_t zc_pool_guid;
56 58 uint64_t zc_vdev_guid;
57 59 int zc_has_timer; /* defunct */
58 60 int zc_pool_state;
59 61 char zc_serd_checksum[MAX_SERDLEN];
60 62 char zc_serd_io[MAX_SERDLEN];
61 63 int zc_has_remove_timer;
62 64 } zfs_case_data_t;
63 65
64 66 /*
65 67 * Time-of-day
66 68 */
67 69 typedef struct er_timeval {
68 70 uint64_t ertv_sec;
69 71 uint64_t ertv_nsec;
70 72 } er_timeval_t;
71 73
72 74 /*
73 75 * In-core case structure.
74 76 */
75 77 typedef struct zfs_case {
76 78 boolean_t zc_present;
77 79 uint32_t zc_version;
78 80 zfs_case_data_t zc_data;
79 81 fmd_case_t *zc_case;
80 82 uu_list_node_t zc_node;
81 83 id_t zc_remove_timer;
82 84 char *zc_fru;
83 85 er_timeval_t zc_when;
84 86 } zfs_case_t;
85 87
86 88 #define CASE_DATA "data"
87 89 #define CASE_FRU "fru"
88 90 #define CASE_DATA_VERSION_INITIAL 1
89 91 #define CASE_DATA_VERSION_SERD 2
90 92
91 93 typedef struct zfs_de_stats {
92 94 fmd_stat_t old_drops;
93 95 fmd_stat_t dev_drops;
94 96 fmd_stat_t vdev_drops;
95 97 fmd_stat_t import_drops;
96 98 fmd_stat_t resource_drops;
97 99 } zfs_de_stats_t;
98 100
99 101 zfs_de_stats_t zfs_stats = {
100 102 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
101 103 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
102 104 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
103 105 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
104 106 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
105 107 };
106 108
107 109 static hrtime_t zfs_remove_timeout;
108 110
109 111 uu_list_pool_t *zfs_case_pool;
110 112 uu_list_t *zfs_cases;
111 113
112 114 #define ZFS_MAKE_RSRC(type) \
113 115 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
114 116 #define ZFS_MAKE_EREPORT(type) \
115 117 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
116 118
117 119 /*
118 120 * Write out the persistent representation of an active case.
119 121 */
120 122 static void
121 123 zfs_case_serialize(fmd_hdl_t *hdl, zfs_case_t *zcp)
122 124 {
123 125 /*
124 126 * Always update cases to the latest version, even if they were the
125 127 * previous version when unserialized.
126 128 */
127 129 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
128 130 fmd_buf_write(hdl, zcp->zc_case, CASE_DATA, &zcp->zc_data,
129 131 sizeof (zcp->zc_data));
130 132
131 133 if (zcp->zc_fru != NULL)
132 134 fmd_buf_write(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
133 135 strlen(zcp->zc_fru));
134 136 }
135 137
136 138 /*
137 139 * Read back the persistent representation of an active case.
138 140 */
139 141 static zfs_case_t *
140 142 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
141 143 {
142 144 zfs_case_t *zcp;
143 145 size_t frulen;
144 146
145 147 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
146 148 zcp->zc_case = cp;
147 149
148 150 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
149 151 sizeof (zcp->zc_data));
150 152
151 153 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
152 154 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
153 155 return (NULL);
154 156 }
155 157
156 158 if ((frulen = fmd_buf_size(hdl, zcp->zc_case, CASE_FRU)) > 0) {
157 159 zcp->zc_fru = fmd_hdl_alloc(hdl, frulen + 1, FMD_SLEEP);
158 160 fmd_buf_read(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
159 161 frulen);
160 162 zcp->zc_fru[frulen] = '\0';
161 163 }
162 164
163 165 /*
164 166 * fmd_buf_read() will have already zeroed out the remainder of the
165 167 * buffer, so we don't have to do anything special if the version
166 168 * doesn't include the SERD engine name.
167 169 */
168 170
169 171 if (zcp->zc_data.zc_has_remove_timer)
170 172 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
171 173 NULL, zfs_remove_timeout);
172 174
173 175 (void) uu_list_insert_before(zfs_cases, NULL, zcp);
174 176
175 177 fmd_case_setspecific(hdl, cp, zcp);
176 178
177 179 return (zcp);
178 180 }
179 181
180 182 /*
181 183 * Iterate over any active cases. If any cases are associated with a pool or
182 184 * vdev which is no longer present on the system, close the associated case.
183 185 */
184 186 static void
185 187 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
186 188 {
187 189 uint64_t vdev_guid;
188 190 uint_t c, children;
189 191 nvlist_t **child;
190 192 zfs_case_t *zcp;
191 193 int ret;
192 194
193 195 ret = nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
194 196 assert(ret == 0);
195 197
196 198 /*
197 199 * Mark any cases associated with this (pool, vdev) pair.
198 200 */
199 201 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
200 202 zcp = uu_list_next(zfs_cases, zcp)) {
201 203 if (zcp->zc_data.zc_pool_guid == pool_guid &&
202 204 zcp->zc_data.zc_vdev_guid == vdev_guid) {
203 205 zcp->zc_present = B_TRUE;
204 206 zcp->zc_when = *loaded;
205 207 }
206 208 }
207 209
208 210 /*
209 211 * Iterate over all children.
210 212 */
211 213 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
212 214 &children) == 0) {
213 215 for (c = 0; c < children; c++)
214 216 zfs_mark_vdev(pool_guid, child[c], loaded);
215 217 }
216 218
217 219 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
218 220 &children) == 0) {
219 221 for (c = 0; c < children; c++)
220 222 zfs_mark_vdev(pool_guid, child[c], loaded);
221 223 }
222 224
223 225 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
224 226 &children) == 0) {
225 227 for (c = 0; c < children; c++)
226 228 zfs_mark_vdev(pool_guid, child[c], loaded);
227 229 }
228 230 }
229 231
230 232 /*ARGSUSED*/
231 233 static int
232 234 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
233 235 {
234 236 zfs_case_t *zcp;
235 237 uint64_t pool_guid;
236 238 uint64_t *tod;
237 239 er_timeval_t loaded = { 0 };
238 240 nvlist_t *config, *vd;
239 241 uint_t nelem = 0;
240 242 int ret;
241 243
242 244 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
243 245 /*
244 246 * Mark any cases associated with just this pool.
245 247 */
246 248 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
247 249 zcp = uu_list_next(zfs_cases, zcp)) {
248 250 if (zcp->zc_data.zc_pool_guid == pool_guid &&
249 251 zcp->zc_data.zc_vdev_guid == 0)
250 252 zcp->zc_present = B_TRUE;
251 253 }
252 254
253 255 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
254 256 zpool_close(zhp);
255 257 return (-1);
256 258 }
257 259
258 260 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
259 261 &tod, &nelem);
260 262 if (nelem == 2) {
261 263 loaded.ertv_sec = tod[0];
262 264 loaded.ertv_nsec = tod[1];
263 265 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
264 266 zcp = uu_list_next(zfs_cases, zcp)) {
265 267 if (zcp->zc_data.zc_pool_guid == pool_guid &&
266 268 zcp->zc_data.zc_vdev_guid == 0) {
267 269 zcp->zc_when = loaded;
268 270 }
269 271 }
270 272 }
271 273
272 274 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
273 275 assert(ret == 0);
274 276
275 277 zfs_mark_vdev(pool_guid, vd, &loaded);
276 278
277 279 zpool_close(zhp);
278 280
279 281 return (0);
280 282 }
281 283
282 284 struct load_time_arg {
283 285 uint64_t lt_guid;
284 286 er_timeval_t *lt_time;
285 287 boolean_t lt_found;
286 288 };
287 289
288 290 static int
289 291 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
290 292 {
291 293 struct load_time_arg *lta = arg;
292 294 uint64_t pool_guid;
293 295 uint64_t *tod;
294 296 nvlist_t *config;
295 297 uint_t nelem;
296 298
297 299 if (lta->lt_found) {
298 300 zpool_close(zhp);
299 301 return (0);
300 302 }
301 303
302 304 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
303 305 if (pool_guid != lta->lt_guid) {
304 306 zpool_close(zhp);
305 307 return (0);
306 308 }
307 309
308 310 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
309 311 zpool_close(zhp);
310 312 return (-1);
311 313 }
312 314
313 315 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
314 316 &tod, &nelem) == 0 && nelem == 2) {
315 317 lta->lt_found = B_TRUE;
316 318 lta->lt_time->ertv_sec = tod[0];
317 319 lta->lt_time->ertv_nsec = tod[1];
318 320 }
319 321
320 322 zpool_close(zhp);
321 323
322 324 return (0);
323 325 }
324 326
325 327 static void
326 328 zfs_purge_cases(fmd_hdl_t *hdl)
327 329 {
328 330 zfs_case_t *zcp;
329 331 uu_list_walk_t *walk;
330 332 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
331 333
332 334 /*
333 335 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
334 336 * matter what we do, we're going to have to stomach a O(vdevs * cases)
335 337 * algorithm. In reality, both quantities are likely so small that
336 338 * neither will matter. Given that iterating over pools is more
337 339 * expensive than iterating over the in-memory case list, we opt for a
338 340 * 'present' flag in each case that starts off cleared. We then iterate
339 341 * over all pools, marking those that are still present, and removing
340 342 * those that aren't found.
341 343 *
342 344 * Note that we could also construct an FMRI and rely on
343 345 * fmd_nvl_fmri_present(), but this would end up doing the same search.
344 346 */
345 347
346 348 /*
347 349 * Mark the cases an not present.
348 350 */
349 351 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
350 352 zcp = uu_list_next(zfs_cases, zcp))
351 353 zcp->zc_present = B_FALSE;
352 354
353 355 /*
354 356 * Iterate over all pools and mark the pools and vdevs found. If this
355 357 * fails (most probably because we're out of memory), then don't close
356 358 * any of the cases and we cannot be sure they are accurate.
357 359 */
358 360 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
359 361 return;
360 362
361 363 /*
362 364 * Remove those cases which were not found.
363 365 */
364 366 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
365 367 while ((zcp = uu_list_walk_next(walk)) != NULL) {
366 368 if (!zcp->zc_present)
367 369 fmd_case_close(hdl, zcp->zc_case);
368 370 }
369 371 uu_list_walk_end(walk);
370 372 }
371 373
372 374 /*
373 375 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
374 376 * checksum).
375 377 */
376 378 static void
377 379 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
378 380 const char *type)
379 381 {
380 382 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s", pool_guid,
381 383 vdev_guid, type);
382 384 }
383 385
384 386 /*
385 387 * Solve a given ZFS case. This first checks to make sure the diagnosis is
386 388 * still valid, as well as cleaning up any pending timer associated with the
387 389 * case.
388 390 */
389 391 static void
390 392 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname,
391 393 boolean_t checkunusable)
392 394 {
393 395 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
394 396 nvlist_t *detector, *fault;
395 397 boolean_t serialize;
396 398 nvlist_t *fmri, *fru;
397 399 topo_hdl_t *thp;
398 400 int err;
399 401
400 402 /*
401 403 * Construct the detector from the case data. The detector is in the
402 404 * ZFS scheme, and is either the pool or the vdev, depending on whether
403 405 * this is a vdev or pool fault.
404 406 */
405 407 detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
406 408
407 409 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
408 410 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
409 411 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
410 412 zcp->zc_data.zc_pool_guid);
411 413 if (zcp->zc_data.zc_vdev_guid != 0) {
412 414 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
413 415 zcp->zc_data.zc_vdev_guid);
414 416 }
415 417
416 418 /*
417 419 * We also want to make sure that the detector (pool or vdev) properly
418 420 * reflects the diagnosed state, when the fault corresponds to internal
419 421 * ZFS state (i.e. not checksum or I/O error-induced). Otherwise, a
420 422 * device which was unavailable early in boot (because the driver/file
421 423 * wasn't available) and is now healthy will be mis-diagnosed.
422 424 */
423 425 if (!fmd_nvl_fmri_present(hdl, detector) ||
424 426 (checkunusable && !fmd_nvl_fmri_unusable(hdl, detector))) {
425 427 fmd_case_close(hdl, zcp->zc_case);
426 428 nvlist_free(detector);
427 429 return;
428 430 }
429 431
430 432
431 433 fru = NULL;
432 434 if (zcp->zc_fru != NULL &&
433 435 (thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION)) != NULL) {
434 436 /*
435 437 * If the vdev had an associated FRU, then get the FRU nvlist
436 438 * from the topo handle and use that in the suspect list. We
437 439 * explicitly lookup the FRU because the fmri reported from the
438 440 * kernel may not have up to date details about the disk itself
439 441 * (serial, part, etc).
440 442 */
441 443 if (topo_fmri_str2nvl(thp, zcp->zc_fru, &fmri, &err) == 0) {
442 444 /*
443 445 * If the disk is part of the system chassis, but the
444 446 * FRU indicates a different chassis ID than our
445 447 * current system, then ignore the error. This
446 448 * indicates that the device was part of another
447 449 * cluster head, and for obvious reasons cannot be
448 450 * imported on this system.
449 451 */
450 452 if (libzfs_fru_notself(zhdl, zcp->zc_fru)) {
451 453 fmd_case_close(hdl, zcp->zc_case);
452 454 nvlist_free(fmri);
453 455 fmd_hdl_topo_rele(hdl, thp);
454 456 nvlist_free(detector);
455 457 return;
456 458 }
457 459
458 460 /*
459 461 * If the device is no longer present on the system, or
460 462 * topo_fmri_fru() fails for other reasons, then fall
461 463 * back to the fmri specified in the vdev.
462 464 */
463 465 if (topo_fmri_fru(thp, fmri, &fru, &err) != 0)
464 466 fru = fmd_nvl_dup(hdl, fmri, FMD_SLEEP);
465 467 nvlist_free(fmri);
466 468 }
467 469
468 470 fmd_hdl_topo_rele(hdl, thp);
469 471 }
470 472
471 473 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
472 474 fru, detector);
473 475 fmd_case_add_suspect(hdl, zcp->zc_case, fault);
474 476
475 477 nvlist_free(fru);
476 478
477 479 fmd_case_solve(hdl, zcp->zc_case);
478 480
479 481 serialize = B_FALSE;
480 482 if (zcp->zc_data.zc_has_remove_timer) {
481 483 fmd_timer_remove(hdl, zcp->zc_remove_timer);
482 484 zcp->zc_data.zc_has_remove_timer = 0;
483 485 serialize = B_TRUE;
484 486 }
485 487 if (serialize)
486 488 zfs_case_serialize(hdl, zcp);
487 489
488 490 nvlist_free(detector);
489 491 }
490 492
491 493 /*
492 494 * This #define and function access a private interface of the FMA
493 495 * framework. Ereports include a time-of-day upper bound.
494 496 * We want to look at that so we can compare it to when pools get
495 497 * loaded.
496 498 */
497 499 #define FMD_EVN_TOD "__tod"
498 500
499 501 static boolean_t
500 502 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
501 503 {
502 504 return (a->ertv_sec < b->ertv_sec ||
503 505 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
504 506 }
505 507
506 508 /*ARGSUSED*/
507 509 static void
508 510 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
509 511 {
510 512 uint64_t *tod;
511 513 uint_t nelem;
512 514
513 515 if (nvlist_lookup_uint64_array(nvl, FMD_EVN_TOD, &tod, &nelem) == 0 &&
514 516 nelem == 2) {
515 517 when->ertv_sec = tod[0];
516 518 when->ertv_nsec = tod[1];
517 519 } else {
518 520 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
519 521 }
520 522 }
521 523
522 524 /*
523 525 * Main fmd entry point.
524 526 */
525 527 /*ARGSUSED*/
526 528 static void
527 529 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
528 530 {
529 531 zfs_case_t *zcp, *dcp;
530 532 int32_t pool_state;
531 533 uint64_t ena, pool_guid, vdev_guid;
532 534 er_timeval_t pool_load;
533 535 er_timeval_t er_when;
534 536 nvlist_t *detector;
535 537 boolean_t pool_found = B_FALSE;
536 538 boolean_t isresource;
537 539 char *fru, *type;
538 540
539 541 /*
540 542 * We subscribe to notifications for vdev or pool removal. In these
541 543 * cases, there may be cases that no longer apply. Purge any cases
542 544 * that no longer apply.
543 545 */
544 546 if (fmd_nvl_class_match(hdl, nvl, "resource.sysevent.EC_zfs.*")) {
545 547 zfs_purge_cases(hdl);
546 548 zfs_stats.resource_drops.fmds_value.ui64++;
547 549 return;
548 550 }
549 551
550 552 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
551 553
552 554 if (isresource) {
553 555 /*
554 556 * For resources, we don't have a normal payload.
555 557 */
556 558 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
557 559 &vdev_guid) != 0)
558 560 pool_state = SPA_LOAD_OPEN;
559 561 else
560 562 pool_state = SPA_LOAD_NONE;
561 563 detector = NULL;
562 564 } else {
563 565 (void) nvlist_lookup_nvlist(nvl,
564 566 FM_EREPORT_DETECTOR, &detector);
565 567 (void) nvlist_lookup_int32(nvl,
566 568 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
567 569 }
568 570
569 571 /*
570 572 * We also ignore all ereports generated during an import of a pool,
571 573 * since the only possible fault (.pool) would result in import failure,
572 574 * and hence no persistent fault. Some day we may want to do something
573 575 * with these ereports, so we continue generating them internally.
574 576 */
575 577 if (pool_state == SPA_LOAD_IMPORT) {
576 578 zfs_stats.import_drops.fmds_value.ui64++;
577 579 return;
578 580 }
579 581
580 582 /*
581 583 * Device I/O errors are ignored during pool open.
582 584 */
583 585 if (pool_state == SPA_LOAD_OPEN &&
584 586 (fmd_nvl_class_match(hdl, nvl,
585 587 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
586 588 fmd_nvl_class_match(hdl, nvl,
587 589 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
588 590 fmd_nvl_class_match(hdl, nvl,
589 591 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
590 592 zfs_stats.dev_drops.fmds_value.ui64++;
591 593 return;
592 594 }
593 595
594 596 /*
595 597 * We ignore ereports for anything except disks and files.
596 598 */
597 599 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
598 600 &type) == 0) {
599 601 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
600 602 strcmp(type, VDEV_TYPE_FILE) != 0) {
601 603 zfs_stats.vdev_drops.fmds_value.ui64++;
602 604 return;
603 605 }
604 606 }
605 607
606 608 /*
607 609 * Determine if this ereport corresponds to an open case. Previous
608 610 * incarnations of this DE used the ENA to chain events together as
609 611 * part of the same case. The problem with this is that we rely on
610 612 * global uniqueness of cases based on (pool_guid, vdev_guid) pair when
611 613 * generating SERD engines. Instead, we have a case for each vdev or
612 614 * pool, regardless of the ENA.
613 615 */
614 616 (void) nvlist_lookup_uint64(nvl,
615 617 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
616 618 if (nvlist_lookup_uint64(nvl,
617 619 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
618 620 vdev_guid = 0;
619 621 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
620 622 ena = 0;
621 623
622 624 zfs_ereport_when(hdl, nvl, &er_when);
623 625
624 626 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
625 627 zcp = uu_list_next(zfs_cases, zcp)) {
626 628 if (zcp->zc_data.zc_pool_guid == pool_guid) {
627 629 pool_found = B_TRUE;
628 630 pool_load = zcp->zc_when;
629 631 }
630 632 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
631 633 break;
632 634 }
633 635
634 636 if (pool_found) {
635 637 fmd_hdl_debug(hdl, "pool %llx, "
636 638 "ereport time %lld.%lld, pool load time = %lld.%lld\n",
637 639 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
638 640 pool_load.ertv_sec, pool_load.ertv_nsec);
639 641 }
640 642
641 643 /*
642 644 * Avoid falsely accusing a pool of being faulty. Do so by
643 645 * not replaying ereports that were generated prior to the
644 646 * current import. If the failure that generated them was
645 647 * transient because the device was actually removed but we
646 648 * didn't receive the normal asynchronous notification, we
647 649 * don't want to mark it as faulted and potentially panic. If
648 650 * there is still a problem we'd expect not to be able to
649 651 * import the pool, or that new ereports will be generated
650 652 * once the pool is used.
651 653 */
652 654 if (pool_found && timeval_earlier(&er_when, &pool_load)) {
653 655 zfs_stats.old_drops.fmds_value.ui64++;
654 656 return;
655 657 }
656 658
657 659 if (!pool_found) {
658 660 /*
659 661 * Haven't yet seen this pool, but same situation
660 662 * may apply.
661 663 */
662 664 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
663 665 struct load_time_arg la;
664 666
665 667 la.lt_guid = pool_guid;
666 668 la.lt_time = &pool_load;
667 669 la.lt_found = B_FALSE;
668 670
669 671 if (zhdl != NULL &&
670 672 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
671 673 la.lt_found == B_TRUE) {
672 674 pool_found = B_TRUE;
673 675 fmd_hdl_debug(hdl, "pool %llx, "
674 676 "ereport time %lld.%lld, "
675 677 "pool load time = %lld.%lld\n",
676 678 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
677 679 pool_load.ertv_sec, pool_load.ertv_nsec);
678 680 if (timeval_earlier(&er_when, &pool_load)) {
679 681 zfs_stats.old_drops.fmds_value.ui64++;
680 682 return;
681 683 }
682 684 }
683 685 }
684 686
685 687 if (zcp == NULL) {
686 688 fmd_case_t *cs;
687 689 zfs_case_data_t data = { 0 };
688 690
689 691 /*
690 692 * If this is one of our 'fake' resource ereports, and there is
691 693 * no case open, simply discard it.
692 694 */
693 695 if (isresource) {
694 696 zfs_stats.resource_drops.fmds_value.ui64++;
695 697 return;
696 698 }
697 699
698 700 /*
699 701 * Open a new case.
700 702 */
701 703 cs = fmd_case_open(hdl, NULL);
702 704
703 705 /*
704 706 * Initialize the case buffer. To commonize code, we actually
705 707 * create the buffer with existing data, and then call
706 708 * zfs_case_unserialize() to instantiate the in-core structure.
707 709 */
708 710 fmd_buf_create(hdl, cs, CASE_DATA,
709 711 sizeof (zfs_case_data_t));
710 712
711 713 data.zc_version = CASE_DATA_VERSION_SERD;
712 714 data.zc_ena = ena;
713 715 data.zc_pool_guid = pool_guid;
714 716 data.zc_vdev_guid = vdev_guid;
715 717 data.zc_pool_state = (int)pool_state;
716 718
717 719 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
718 720
719 721 zcp = zfs_case_unserialize(hdl, cs);
720 722 assert(zcp != NULL);
721 723 if (pool_found)
722 724 zcp->zc_when = pool_load;
723 725 }
724 726
725 727
726 728 /*
727 729 * If this is an ereport for a case with an associated vdev FRU, make
728 730 * sure it is accurate and up to date.
729 731 */
730 732 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU,
731 733 &fru) == 0) {
732 734 topo_hdl_t *thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
733 735 if (zcp->zc_fru == NULL ||
734 736 !topo_fmri_strcmp(thp, zcp->zc_fru, fru)) {
735 737 if (zcp->zc_fru != NULL) {
736 738 fmd_hdl_strfree(hdl, zcp->zc_fru);
737 739 fmd_buf_destroy(hdl, zcp->zc_case, CASE_FRU);
738 740 }
739 741 zcp->zc_fru = fmd_hdl_strdup(hdl, fru, FMD_SLEEP);
740 742 zfs_case_serialize(hdl, zcp);
741 743 }
742 744 fmd_hdl_topo_rele(hdl, thp);
743 745 }
744 746
745 747 if (isresource) {
746 748 if (fmd_nvl_class_match(hdl, nvl,
747 749 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
748 750 /*
749 751 * The 'resource.fs.zfs.autoreplace' event indicates
750 752 * that the pool was loaded with the 'autoreplace'
751 753 * property set. In this case, any pending device
752 754 * failures should be ignored, as the asynchronous
753 755 * autoreplace handling will take care of them.
754 756 */
755 757 fmd_case_close(hdl, zcp->zc_case);
756 758 } else if (fmd_nvl_class_match(hdl, nvl,
757 759 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
758 760 /*
759 761 * The 'resource.fs.zfs.removed' event indicates that
760 762 * device removal was detected, and the device was
761 763 * closed asynchronously. If this is the case, we
762 764 * assume that any recent I/O errors were due to the
763 765 * device removal, not any fault of the device itself.
764 766 * We reset the SERD engine, and cancel any pending
765 767 * timers.
766 768 */
767 769 if (zcp->zc_data.zc_has_remove_timer) {
768 770 fmd_timer_remove(hdl, zcp->zc_remove_timer);
769 771 zcp->zc_data.zc_has_remove_timer = 0;
770 772 zfs_case_serialize(hdl, zcp);
771 773 }
772 774 if (zcp->zc_data.zc_serd_io[0] != '\0')
773 775 fmd_serd_reset(hdl,
774 776 zcp->zc_data.zc_serd_io);
775 777 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
776 778 fmd_serd_reset(hdl,
777 779 zcp->zc_data.zc_serd_checksum);
778 780 }
779 781 zfs_stats.resource_drops.fmds_value.ui64++;
780 782 return;
781 783 }
782 784
783 785 /*
784 786 * Associate the ereport with this case.
785 787 */
786 788 fmd_case_add_ereport(hdl, zcp->zc_case, ep);
787 789
788 790 /*
789 791 * Don't do anything else if this case is already solved.
790 792 */
791 793 if (fmd_case_solved(hdl, zcp->zc_case))
792 794 return;
793 795
794 796 /*
795 797 * Determine if we should solve the case and generate a fault. We solve
796 798 * a case if:
797 799 *
798 800 * a. A pool failed to open (ereport.fs.zfs.pool)
799 801 * b. A device failed to open (ereport.fs.zfs.pool) while a pool
800 802 * was up and running.
801 803 *
802 804 * We may see a series of ereports associated with a pool open, all
803 805 * chained together by the same ENA. If the pool open succeeds, then
804 806 * we'll see no further ereports. To detect when a pool open has
805 807 * succeeded, we associate a timer with the event. When it expires, we
806 808 * close the case.
807 809 */
808 810 if (fmd_nvl_class_match(hdl, nvl,
809 811 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
810 812 /*
811 813 * Pool level fault. Before solving the case, go through and
812 814 * close any open device cases that may be pending.
813 815 */
814 816 for (dcp = uu_list_first(zfs_cases); dcp != NULL;
815 817 dcp = uu_list_next(zfs_cases, dcp)) {
816 818 if (dcp->zc_data.zc_pool_guid ==
817 819 zcp->zc_data.zc_pool_guid &&
818 820 dcp->zc_data.zc_vdev_guid != 0)
819 821 fmd_case_close(hdl, dcp->zc_case);
820 822 }
821 823
822 824 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool", B_TRUE);
823 825 } else if (fmd_nvl_class_match(hdl, nvl,
824 826 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
825 827 /*
826 828 * Pool level fault for reading the intent logs.
827 829 */
828 830 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay", B_TRUE);
829 831 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
830 832 /*
831 833 * Device fault.
832 834 */
833 835 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device", B_TRUE);
834 836 } else if (fmd_nvl_class_match(hdl, nvl,
835 837 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
836 838 fmd_nvl_class_match(hdl, nvl,
837 839 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
838 840 fmd_nvl_class_match(hdl, nvl,
839 841 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
840 842 fmd_nvl_class_match(hdl, nvl,
841 843 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
842 844 char *failmode = NULL;
843 845 boolean_t checkremove = B_FALSE;
844 846
845 847 /*
846 848 * If this is a checksum or I/O error, then toss it into the
847 849 * appropriate SERD engine and check to see if it has fired.
848 850 * Ideally, we want to do something more sophisticated,
849 851 * (persistent errors for a single data block, etc). For now,
850 852 * a single SERD engine is sufficient.
851 853 */
852 854 if (fmd_nvl_class_match(hdl, nvl,
853 855 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
854 856 if (zcp->zc_data.zc_serd_io[0] == '\0') {
855 857 zfs_serd_name(zcp->zc_data.zc_serd_io,
856 858 pool_guid, vdev_guid, "io");
857 859 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
858 860 fmd_prop_get_int32(hdl, "io_N"),
859 861 fmd_prop_get_int64(hdl, "io_T"));
860 862 zfs_case_serialize(hdl, zcp);
861 863 }
862 864 if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep))
863 865 checkremove = B_TRUE;
864 866 } else if (fmd_nvl_class_match(hdl, nvl,
865 867 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
866 868 if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
867 869 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
868 870 pool_guid, vdev_guid, "checksum");
869 871 fmd_serd_create(hdl,
870 872 zcp->zc_data.zc_serd_checksum,
871 873 fmd_prop_get_int32(hdl, "checksum_N"),
872 874 fmd_prop_get_int64(hdl, "checksum_T"));
873 875 zfs_case_serialize(hdl, zcp);
874 876 }
875 877 if (fmd_serd_record(hdl,
876 878 zcp->zc_data.zc_serd_checksum, ep)) {
877 879 zfs_case_solve(hdl, zcp,
878 880 "fault.fs.zfs.vdev.checksum", B_FALSE);
879 881 }
880 882 } else if (fmd_nvl_class_match(hdl, nvl,
881 883 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
882 884 (nvlist_lookup_string(nvl,
883 885 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
884 886 failmode != NULL) {
885 887 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
886 888 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
887 889 zfs_case_solve(hdl, zcp,
888 890 "fault.fs.zfs.io_failure_continue",
889 891 B_FALSE);
890 892 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
891 893 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
892 894 zfs_case_solve(hdl, zcp,
893 895 "fault.fs.zfs.io_failure_wait", B_FALSE);
894 896 }
895 897 } else if (fmd_nvl_class_match(hdl, nvl,
896 898 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
897 899 checkremove = B_TRUE;
898 900 }
899 901
900 902 /*
901 903 * Because I/O errors may be due to device removal, we postpone
902 904 * any diagnosis until we're sure that we aren't about to
903 905 * receive a 'resource.fs.zfs.removed' event.
904 906 */
905 907 if (checkremove) {
906 908 if (zcp->zc_data.zc_has_remove_timer)
907 909 fmd_timer_remove(hdl, zcp->zc_remove_timer);
908 910 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
909 911 zfs_remove_timeout);
910 912 if (!zcp->zc_data.zc_has_remove_timer) {
911 913 zcp->zc_data.zc_has_remove_timer = 1;
912 914 zfs_case_serialize(hdl, zcp);
913 915 }
914 916 }
915 917 }
916 918 }
917 919
918 920 /*
919 921 * The timeout is fired when we diagnosed an I/O error, and it was not due to
920 922 * device removal (which would cause the timeout to be cancelled).
921 923 */
922 924 /* ARGSUSED */
923 925 static void
924 926 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
|
↓ open down ↓ |
889 lines elided |
↑ open up ↑ |
925 927 {
926 928 zfs_case_t *zcp = data;
927 929
928 930 if (id == zcp->zc_remove_timer)
929 931 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io", B_FALSE);
930 932 }
931 933
932 934 static void
933 935 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
934 936 {
935 - zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
937 + zfs_case_t *zcp;
936 938
939 + if ((zcp = fmd_case_getspecific(hdl, cs)) == NULL)
940 + return;
941 +
937 942 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
938 943 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
939 944 if (zcp->zc_data.zc_serd_io[0] != '\0')
940 945 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
941 946 if (zcp->zc_data.zc_has_remove_timer)
942 947 fmd_timer_remove(hdl, zcp->zc_remove_timer);
943 948 uu_list_remove(zfs_cases, zcp);
944 949 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
945 950 }
946 951
947 952 /*
948 953 * We use the fmd gc entry point to look for old cases that no longer apply.
949 954 * This allows us to keep our set of case data small in a long running system.
950 955 */
951 956 static void
952 957 zfs_fm_gc(fmd_hdl_t *hdl)
953 958 {
954 959 zfs_purge_cases(hdl);
955 960 }
956 961
957 962 static const fmd_hdl_ops_t fmd_ops = {
958 963 zfs_fm_recv, /* fmdo_recv */
959 964 zfs_fm_timeout, /* fmdo_timeout */
960 965 zfs_fm_close, /* fmdo_close */
961 966 NULL, /* fmdo_stats */
962 967 zfs_fm_gc, /* fmdo_gc */
963 968 };
964 969
|
↓ open down ↓ |
18 lines elided |
↑ open up ↑ |
965 970 static const fmd_prop_t fmd_props[] = {
966 971 { "checksum_N", FMD_TYPE_UINT32, "10" },
967 972 { "checksum_T", FMD_TYPE_TIME, "10min" },
968 973 { "io_N", FMD_TYPE_UINT32, "10" },
969 974 { "io_T", FMD_TYPE_TIME, "10min" },
970 975 { "remove_timeout", FMD_TYPE_TIME, "15sec" },
971 976 { NULL, 0, NULL }
972 977 };
973 978
974 979 static const fmd_hdl_info_t fmd_info = {
975 - "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
980 + "ZFS Diagnosis Engine", "1.1", &fmd_ops, fmd_props
976 981 };
977 982
978 983 void
979 984 _fmd_init(fmd_hdl_t *hdl)
980 985 {
981 986 fmd_case_t *cp;
982 987 libzfs_handle_t *zhdl;
983 988
984 989 if ((zhdl = libzfs_init()) == NULL)
985 990 return;
986 991
987 992 if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
988 993 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
989 994 NULL, 0)) == NULL) {
990 995 libzfs_fini(zhdl);
991 996 return;
992 997 }
993 998
994 999 if ((zfs_cases = uu_list_create(zfs_case_pool, NULL, 0)) == NULL) {
995 1000 uu_list_pool_destroy(zfs_case_pool);
996 1001 libzfs_fini(zhdl);
997 1002 return;
998 1003 }
999 1004
1000 1005 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1001 1006 uu_list_destroy(zfs_cases);
1002 1007 uu_list_pool_destroy(zfs_case_pool);
1003 1008 libzfs_fini(zhdl);
1004 1009 return;
1005 1010 }
1006 1011
1007 1012 fmd_hdl_setspecific(hdl, zhdl);
1008 1013
1009 1014 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1010 1015 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1011 1016
1012 1017 /*
1013 1018 * Iterate over all active cases and unserialize the associated buffers,
1014 1019 * adding them to our list of open cases.
1015 1020 */
1016 1021 for (cp = fmd_case_next(hdl, NULL);
1017 1022 cp != NULL; cp = fmd_case_next(hdl, cp))
1018 1023 (void) zfs_case_unserialize(hdl, cp);
1019 1024
1020 1025 /*
1021 1026 * Clear out any old cases that are no longer valid.
1022 1027 */
1023 1028 zfs_purge_cases(hdl);
1024 1029
1025 1030 zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout");
1026 1031 }
1027 1032
1028 1033 void
1029 1034 _fmd_fini(fmd_hdl_t *hdl)
1030 1035 {
1031 1036 zfs_case_t *zcp;
1032 1037 uu_list_walk_t *walk;
1033 1038 libzfs_handle_t *zhdl;
1034 1039
1035 1040 /*
1036 1041 * Remove all active cases.
1037 1042 */
1038 1043 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
1039 1044 while ((zcp = uu_list_walk_next(walk)) != NULL) {
1040 1045 uu_list_remove(zfs_cases, zcp);
1041 1046 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1042 1047 }
1043 1048 uu_list_walk_end(walk);
1044 1049
1045 1050 uu_list_destroy(zfs_cases);
1046 1051 uu_list_pool_destroy(zfs_case_pool);
1047 1052
1048 1053 zhdl = fmd_hdl_getspecific(hdl);
1049 1054 libzfs_fini(zhdl);
1050 1055 }
|
↓ open down ↓ |
65 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX