1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /*
26 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
27 */
28
29 #include <assert.h>
30 #include <stddef.h>
31 #include <strings.h>
32 #include <libuutil.h>
33 #include <libzfs.h>
34 #include <fm/fmd_api.h>
35 #include <fm/libtopo.h>
36 #include <sys/types.h>
37 #include <sys/time.h>
38 #include <sys/fs/zfs.h>
39 #include <sys/fm/protocol.h>
40 #include <sys/fm/fs/zfs.h>
41
42 /*
43 * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'. This
44 * #define reserves enough space for two 64-bit hex values plus the length of
45 * the longest string.
46 */
47 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
48
49 /*
50 * On-disk case structure. This must maintain backwards compatibility with
51 * previous versions of the DE. By default, any members appended to the end
52 * will be filled with zeros if they don't exist in a previous version.
53 */
54 typedef struct zfs_case_data {
55 uint64_t zc_version;
56 uint64_t zc_ena;
57 uint64_t zc_pool_guid;
58 uint64_t zc_vdev_guid;
59 int zc_has_timer; /* defunct */
60 int zc_pool_state;
61 char zc_serd_checksum[MAX_SERDLEN];
62 char zc_serd_io[MAX_SERDLEN];
63 int zc_has_remove_timer;
64 } zfs_case_data_t;
65
66 /*
67 * Time-of-day
68 */
69 typedef struct er_timeval {
70 uint64_t ertv_sec;
71 uint64_t ertv_nsec;
72 } er_timeval_t;
73
74 /*
75 * In-core case structure.
76 */
77 typedef struct zfs_case {
78 boolean_t zc_present;
79 uint32_t zc_version;
80 zfs_case_data_t zc_data;
81 fmd_case_t *zc_case;
82 uu_list_node_t zc_node;
83 id_t zc_remove_timer;
84 char *zc_fru;
85 er_timeval_t zc_when;
86 } zfs_case_t;
87
88 #define CASE_DATA "data"
89 #define CASE_FRU "fru"
90 #define CASE_DATA_VERSION_INITIAL 1
91 #define CASE_DATA_VERSION_SERD 2
92
93 typedef struct zfs_de_stats {
94 fmd_stat_t old_drops;
95 fmd_stat_t dev_drops;
96 fmd_stat_t vdev_drops;
97 fmd_stat_t import_drops;
98 fmd_stat_t resource_drops;
99 } zfs_de_stats_t;
100
101 zfs_de_stats_t zfs_stats = {
102 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
103 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
104 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
105 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
106 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
107 };
108
109 static hrtime_t zfs_remove_timeout;
110
111 uu_list_pool_t *zfs_case_pool;
112 uu_list_t *zfs_cases;
113
114 #define ZFS_MAKE_RSRC(type) \
115 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
116 #define ZFS_MAKE_EREPORT(type) \
117 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
118
119 /*
120 * Write out the persistent representation of an active case.
121 */
122 static void
123 zfs_case_serialize(fmd_hdl_t *hdl, zfs_case_t *zcp)
124 {
125 /*
126 * Always update cases to the latest version, even if they were the
127 * previous version when unserialized.
128 */
129 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
130 fmd_buf_write(hdl, zcp->zc_case, CASE_DATA, &zcp->zc_data,
131 sizeof (zcp->zc_data));
132
133 if (zcp->zc_fru != NULL)
134 fmd_buf_write(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
135 strlen(zcp->zc_fru));
136 }
137
138 /*
139 * Read back the persistent representation of an active case.
140 */
141 static zfs_case_t *
142 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
143 {
144 zfs_case_t *zcp;
145 size_t frulen;
146
147 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
148 zcp->zc_case = cp;
149
150 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
151 sizeof (zcp->zc_data));
152
153 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
154 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
155 return (NULL);
156 }
157
158 if ((frulen = fmd_buf_size(hdl, zcp->zc_case, CASE_FRU)) > 0) {
159 zcp->zc_fru = fmd_hdl_alloc(hdl, frulen + 1, FMD_SLEEP);
160 fmd_buf_read(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
161 frulen);
162 zcp->zc_fru[frulen] = '\0';
163 }
164
165 /*
166 * fmd_buf_read() will have already zeroed out the remainder of the
167 * buffer, so we don't have to do anything special if the version
168 * doesn't include the SERD engine name.
169 */
170
171 if (zcp->zc_data.zc_has_remove_timer)
172 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
173 NULL, zfs_remove_timeout);
174
175 (void) uu_list_insert_before(zfs_cases, NULL, zcp);
176
177 fmd_case_setspecific(hdl, cp, zcp);
178
179 return (zcp);
180 }
181
182 /*
183 * Iterate over any active cases. If any cases are associated with a pool or
184 * vdev which is no longer present on the system, close the associated case.
185 */
186 static void
187 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
188 {
189 uint64_t vdev_guid;
190 uint_t c, children;
191 nvlist_t **child;
192 zfs_case_t *zcp;
193 int ret;
194
195 ret = nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
196 assert(ret == 0);
197
198 /*
199 * Mark any cases associated with this (pool, vdev) pair.
200 */
201 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
202 zcp = uu_list_next(zfs_cases, zcp)) {
203 if (zcp->zc_data.zc_pool_guid == pool_guid &&
204 zcp->zc_data.zc_vdev_guid == vdev_guid) {
205 zcp->zc_present = B_TRUE;
206 zcp->zc_when = *loaded;
207 }
208 }
209
210 /*
211 * Iterate over all children.
212 */
213 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
214 &children) == 0) {
215 for (c = 0; c < children; c++)
216 zfs_mark_vdev(pool_guid, child[c], loaded);
217 }
218
219 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
220 &children) == 0) {
221 for (c = 0; c < children; c++)
222 zfs_mark_vdev(pool_guid, child[c], loaded);
223 }
224
225 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
226 &children) == 0) {
227 for (c = 0; c < children; c++)
228 zfs_mark_vdev(pool_guid, child[c], loaded);
229 }
230 }
231
232 /*ARGSUSED*/
233 static int
234 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
235 {
236 zfs_case_t *zcp;
237 uint64_t pool_guid;
238 uint64_t *tod;
239 er_timeval_t loaded = { 0 };
240 nvlist_t *config, *vd;
241 uint_t nelem = 0;
242 int ret;
243
244 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
245 /*
246 * Mark any cases associated with just this pool.
247 */
248 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
249 zcp = uu_list_next(zfs_cases, zcp)) {
250 if (zcp->zc_data.zc_pool_guid == pool_guid &&
251 zcp->zc_data.zc_vdev_guid == 0)
252 zcp->zc_present = B_TRUE;
253 }
254
255 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
256 zpool_close(zhp);
257 return (-1);
258 }
259
260 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
261 &tod, &nelem);
262 if (nelem == 2) {
263 loaded.ertv_sec = tod[0];
264 loaded.ertv_nsec = tod[1];
265 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
266 zcp = uu_list_next(zfs_cases, zcp)) {
267 if (zcp->zc_data.zc_pool_guid == pool_guid &&
268 zcp->zc_data.zc_vdev_guid == 0) {
269 zcp->zc_when = loaded;
270 }
271 }
272 }
273
274 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
275 assert(ret == 0);
276
277 zfs_mark_vdev(pool_guid, vd, &loaded);
278
279 zpool_close(zhp);
280
281 return (0);
282 }
283
284 struct load_time_arg {
285 uint64_t lt_guid;
286 er_timeval_t *lt_time;
287 boolean_t lt_found;
288 };
289
290 static int
291 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
292 {
293 struct load_time_arg *lta = arg;
294 uint64_t pool_guid;
295 uint64_t *tod;
296 nvlist_t *config;
297 uint_t nelem;
298
299 if (lta->lt_found) {
300 zpool_close(zhp);
301 return (0);
302 }
303
304 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
305 if (pool_guid != lta->lt_guid) {
306 zpool_close(zhp);
307 return (0);
308 }
309
310 if ((config = zpool_get_config(zhp, NULL)) == NULL) {
311 zpool_close(zhp);
312 return (-1);
313 }
314
315 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
316 &tod, &nelem) == 0 && nelem == 2) {
317 lta->lt_found = B_TRUE;
318 lta->lt_time->ertv_sec = tod[0];
319 lta->lt_time->ertv_nsec = tod[1];
320 }
321
322 zpool_close(zhp);
323
324 return (0);
325 }
326
327 static void
328 zfs_purge_cases(fmd_hdl_t *hdl)
329 {
330 zfs_case_t *zcp;
331 uu_list_walk_t *walk;
332 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
333
334 /*
335 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
336 * matter what we do, we're going to have to stomach a O(vdevs * cases)
337 * algorithm. In reality, both quantities are likely so small that
338 * neither will matter. Given that iterating over pools is more
339 * expensive than iterating over the in-memory case list, we opt for a
340 * 'present' flag in each case that starts off cleared. We then iterate
341 * over all pools, marking those that are still present, and removing
342 * those that aren't found.
343 *
344 * Note that we could also construct an FMRI and rely on
345 * fmd_nvl_fmri_present(), but this would end up doing the same search.
346 */
347
348 /*
349 * Mark the cases an not present.
350 */
351 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
352 zcp = uu_list_next(zfs_cases, zcp))
353 zcp->zc_present = B_FALSE;
354
355 /*
356 * Iterate over all pools and mark the pools and vdevs found. If this
357 * fails (most probably because we're out of memory), then don't close
358 * any of the cases and we cannot be sure they are accurate.
359 */
360 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
361 return;
362
363 /*
364 * Remove those cases which were not found.
365 */
366 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
367 while ((zcp = uu_list_walk_next(walk)) != NULL) {
368 if (!zcp->zc_present)
369 fmd_case_close(hdl, zcp->zc_case);
370 }
371 uu_list_walk_end(walk);
372 }
373
374 /*
375 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
376 * checksum).
377 */
378 static void
379 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
380 const char *type)
381 {
382 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s", pool_guid,
383 vdev_guid, type);
384 }
385
386 /*
387 * Solve a given ZFS case. This first checks to make sure the diagnosis is
388 * still valid, as well as cleaning up any pending timer associated with the
389 * case.
390 */
391 static void
392 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname,
393 boolean_t checkunusable)
394 {
395 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
396 nvlist_t *detector, *fault;
397 boolean_t serialize;
398 nvlist_t *fmri, *fru;
399 topo_hdl_t *thp;
400 int err;
401
402 /*
403 * Construct the detector from the case data. The detector is in the
404 * ZFS scheme, and is either the pool or the vdev, depending on whether
405 * this is a vdev or pool fault.
406 */
407 detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
408
409 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
410 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
411 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
412 zcp->zc_data.zc_pool_guid);
413 if (zcp->zc_data.zc_vdev_guid != 0) {
414 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
415 zcp->zc_data.zc_vdev_guid);
416 }
417
418 /*
419 * We also want to make sure that the detector (pool or vdev) properly
420 * reflects the diagnosed state, when the fault corresponds to internal
421 * ZFS state (i.e. not checksum or I/O error-induced). Otherwise, a
422 * device which was unavailable early in boot (because the driver/file
423 * wasn't available) and is now healthy will be mis-diagnosed.
424 */
425 if (!fmd_nvl_fmri_present(hdl, detector) ||
426 (checkunusable && !fmd_nvl_fmri_unusable(hdl, detector))) {
427 fmd_case_close(hdl, zcp->zc_case);
428 nvlist_free(detector);
429 return;
430 }
431
432
433 fru = NULL;
434 if (zcp->zc_fru != NULL &&
435 (thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION)) != NULL) {
436 /*
437 * If the vdev had an associated FRU, then get the FRU nvlist
438 * from the topo handle and use that in the suspect list. We
439 * explicitly lookup the FRU because the fmri reported from the
440 * kernel may not have up to date details about the disk itself
441 * (serial, part, etc).
442 */
443 if (topo_fmri_str2nvl(thp, zcp->zc_fru, &fmri, &err) == 0) {
444 /*
445 * If the disk is part of the system chassis, but the
446 * FRU indicates a different chassis ID than our
447 * current system, then ignore the error. This
448 * indicates that the device was part of another
449 * cluster head, and for obvious reasons cannot be
450 * imported on this system.
451 */
452 if (libzfs_fru_notself(zhdl, zcp->zc_fru)) {
453 fmd_case_close(hdl, zcp->zc_case);
454 nvlist_free(fmri);
455 fmd_hdl_topo_rele(hdl, thp);
456 nvlist_free(detector);
457 return;
458 }
459
460 /*
461 * If the device is no longer present on the system, or
462 * topo_fmri_fru() fails for other reasons, then fall
463 * back to the fmri specified in the vdev.
464 */
465 if (topo_fmri_fru(thp, fmri, &fru, &err) != 0)
466 fru = fmd_nvl_dup(hdl, fmri, FMD_SLEEP);
467 nvlist_free(fmri);
468 }
469
470 fmd_hdl_topo_rele(hdl, thp);
471 }
472
473 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
474 fru, detector);
475 fmd_case_add_suspect(hdl, zcp->zc_case, fault);
476
477 nvlist_free(fru);
478
479 fmd_case_solve(hdl, zcp->zc_case);
480
481 serialize = B_FALSE;
482 if (zcp->zc_data.zc_has_remove_timer) {
483 fmd_timer_remove(hdl, zcp->zc_remove_timer);
484 zcp->zc_data.zc_has_remove_timer = 0;
485 serialize = B_TRUE;
486 }
487 if (serialize)
488 zfs_case_serialize(hdl, zcp);
489
490 nvlist_free(detector);
491 }
492
493 /*
494 * This #define and function access a private interface of the FMA
495 * framework. Ereports include a time-of-day upper bound.
496 * We want to look at that so we can compare it to when pools get
497 * loaded.
498 */
499 #define FMD_EVN_TOD "__tod"
500
501 static boolean_t
502 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
503 {
504 return (a->ertv_sec < b->ertv_sec ||
505 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
506 }
507
508 /*ARGSUSED*/
509 static void
510 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
511 {
512 uint64_t *tod;
513 uint_t nelem;
514
515 if (nvlist_lookup_uint64_array(nvl, FMD_EVN_TOD, &tod, &nelem) == 0 &&
516 nelem == 2) {
517 when->ertv_sec = tod[0];
518 when->ertv_nsec = tod[1];
519 } else {
520 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
521 }
522 }
523
524 /*
525 * Main fmd entry point.
526 */
527 /*ARGSUSED*/
528 static void
529 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
530 {
531 zfs_case_t *zcp, *dcp;
532 int32_t pool_state;
533 uint64_t ena, pool_guid, vdev_guid;
534 er_timeval_t pool_load;
535 er_timeval_t er_when;
536 nvlist_t *detector;
537 boolean_t pool_found = B_FALSE;
538 boolean_t isresource;
539 char *fru, *type;
540
541 /*
542 * We subscribe to notifications for vdev or pool removal. In these
543 * cases, there may be cases that no longer apply. Purge any cases
544 * that no longer apply.
545 */
546 if (fmd_nvl_class_match(hdl, nvl, "resource.sysevent.EC_zfs.*")) {
547 zfs_purge_cases(hdl);
548 zfs_stats.resource_drops.fmds_value.ui64++;
549 return;
550 }
551
552 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
553
554 if (isresource) {
555 /*
556 * For resources, we don't have a normal payload.
557 */
558 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
559 &vdev_guid) != 0)
560 pool_state = SPA_LOAD_OPEN;
561 else
562 pool_state = SPA_LOAD_NONE;
563 detector = NULL;
564 } else {
565 (void) nvlist_lookup_nvlist(nvl,
566 FM_EREPORT_DETECTOR, &detector);
567 (void) nvlist_lookup_int32(nvl,
568 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
569 }
570
571 /*
572 * We also ignore all ereports generated during an import of a pool,
573 * since the only possible fault (.pool) would result in import failure,
574 * and hence no persistent fault. Some day we may want to do something
575 * with these ereports, so we continue generating them internally.
576 */
577 if (pool_state == SPA_LOAD_IMPORT) {
578 zfs_stats.import_drops.fmds_value.ui64++;
579 return;
580 }
581
582 /*
583 * Device I/O errors are ignored during pool open.
584 */
585 if (pool_state == SPA_LOAD_OPEN &&
586 (fmd_nvl_class_match(hdl, nvl,
587 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
588 fmd_nvl_class_match(hdl, nvl,
589 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
590 fmd_nvl_class_match(hdl, nvl,
591 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
592 zfs_stats.dev_drops.fmds_value.ui64++;
593 return;
594 }
595
596 /*
597 * We ignore ereports for anything except disks and files.
598 */
599 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
600 &type) == 0) {
601 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
602 strcmp(type, VDEV_TYPE_FILE) != 0) {
603 zfs_stats.vdev_drops.fmds_value.ui64++;
604 return;
605 }
606 }
607
608 /*
609 * Determine if this ereport corresponds to an open case. Previous
610 * incarnations of this DE used the ENA to chain events together as
611 * part of the same case. The problem with this is that we rely on
612 * global uniqueness of cases based on (pool_guid, vdev_guid) pair when
613 * generating SERD engines. Instead, we have a case for each vdev or
614 * pool, regardless of the ENA.
615 */
616 (void) nvlist_lookup_uint64(nvl,
617 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
618 if (nvlist_lookup_uint64(nvl,
619 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
620 vdev_guid = 0;
621 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
622 ena = 0;
623
624 zfs_ereport_when(hdl, nvl, &er_when);
625
626 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
627 zcp = uu_list_next(zfs_cases, zcp)) {
628 if (zcp->zc_data.zc_pool_guid == pool_guid) {
629 pool_found = B_TRUE;
630 pool_load = zcp->zc_when;
631 }
632 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
633 break;
634 }
635
636 if (pool_found) {
637 fmd_hdl_debug(hdl, "pool %llx, "
638 "ereport time %lld.%lld, pool load time = %lld.%lld\n",
639 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
640 pool_load.ertv_sec, pool_load.ertv_nsec);
641 }
642
643 /*
644 * Avoid falsely accusing a pool of being faulty. Do so by
645 * not replaying ereports that were generated prior to the
646 * current import. If the failure that generated them was
647 * transient because the device was actually removed but we
648 * didn't receive the normal asynchronous notification, we
649 * don't want to mark it as faulted and potentially panic. If
650 * there is still a problem we'd expect not to be able to
651 * import the pool, or that new ereports will be generated
652 * once the pool is used.
653 */
654 if (pool_found && timeval_earlier(&er_when, &pool_load)) {
655 zfs_stats.old_drops.fmds_value.ui64++;
656 return;
657 }
658
659 if (!pool_found) {
660 /*
661 * Haven't yet seen this pool, but same situation
662 * may apply.
663 */
664 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
665 struct load_time_arg la;
666
667 la.lt_guid = pool_guid;
668 la.lt_time = &pool_load;
669 la.lt_found = B_FALSE;
670
671 if (zhdl != NULL &&
672 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
673 la.lt_found == B_TRUE) {
674 pool_found = B_TRUE;
675 fmd_hdl_debug(hdl, "pool %llx, "
676 "ereport time %lld.%lld, "
677 "pool load time = %lld.%lld\n",
678 pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
679 pool_load.ertv_sec, pool_load.ertv_nsec);
680 if (timeval_earlier(&er_when, &pool_load)) {
681 zfs_stats.old_drops.fmds_value.ui64++;
682 return;
683 }
684 }
685 }
686
687 if (zcp == NULL) {
688 fmd_case_t *cs;
689 zfs_case_data_t data = { 0 };
690
691 /*
692 * If this is one of our 'fake' resource ereports, and there is
693 * no case open, simply discard it.
694 */
695 if (isresource) {
696 zfs_stats.resource_drops.fmds_value.ui64++;
697 return;
698 }
699
700 /*
701 * Open a new case.
702 */
703 cs = fmd_case_open(hdl, NULL);
704
705 /*
706 * Initialize the case buffer. To commonize code, we actually
707 * create the buffer with existing data, and then call
708 * zfs_case_unserialize() to instantiate the in-core structure.
709 */
710 fmd_buf_create(hdl, cs, CASE_DATA,
711 sizeof (zfs_case_data_t));
712
713 data.zc_version = CASE_DATA_VERSION_SERD;
714 data.zc_ena = ena;
715 data.zc_pool_guid = pool_guid;
716 data.zc_vdev_guid = vdev_guid;
717 data.zc_pool_state = (int)pool_state;
718
719 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
720
721 zcp = zfs_case_unserialize(hdl, cs);
722 assert(zcp != NULL);
723 if (pool_found)
724 zcp->zc_when = pool_load;
725 }
726
727
728 /*
729 * If this is an ereport for a case with an associated vdev FRU, make
730 * sure it is accurate and up to date.
731 */
732 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU,
733 &fru) == 0) {
734 topo_hdl_t *thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
735 if (zcp->zc_fru == NULL ||
736 !topo_fmri_strcmp(thp, zcp->zc_fru, fru)) {
737 if (zcp->zc_fru != NULL) {
738 fmd_hdl_strfree(hdl, zcp->zc_fru);
739 fmd_buf_destroy(hdl, zcp->zc_case, CASE_FRU);
740 }
741 zcp->zc_fru = fmd_hdl_strdup(hdl, fru, FMD_SLEEP);
742 zfs_case_serialize(hdl, zcp);
743 }
744 fmd_hdl_topo_rele(hdl, thp);
745 }
746
747 if (isresource) {
748 if (fmd_nvl_class_match(hdl, nvl,
749 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
750 /*
751 * The 'resource.fs.zfs.autoreplace' event indicates
752 * that the pool was loaded with the 'autoreplace'
753 * property set. In this case, any pending device
754 * failures should be ignored, as the asynchronous
755 * autoreplace handling will take care of them.
756 */
757 fmd_case_close(hdl, zcp->zc_case);
758 } else if (fmd_nvl_class_match(hdl, nvl,
759 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
760 /*
761 * The 'resource.fs.zfs.removed' event indicates that
762 * device removal was detected, and the device was
763 * closed asynchronously. If this is the case, we
764 * assume that any recent I/O errors were due to the
765 * device removal, not any fault of the device itself.
766 * We reset the SERD engine, and cancel any pending
767 * timers.
768 */
769 if (zcp->zc_data.zc_has_remove_timer) {
770 fmd_timer_remove(hdl, zcp->zc_remove_timer);
771 zcp->zc_data.zc_has_remove_timer = 0;
772 zfs_case_serialize(hdl, zcp);
773 }
774 if (zcp->zc_data.zc_serd_io[0] != '\0')
775 fmd_serd_reset(hdl,
776 zcp->zc_data.zc_serd_io);
777 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
778 fmd_serd_reset(hdl,
779 zcp->zc_data.zc_serd_checksum);
780 }
781 zfs_stats.resource_drops.fmds_value.ui64++;
782 return;
783 }
784
785 /*
786 * Associate the ereport with this case.
787 */
788 fmd_case_add_ereport(hdl, zcp->zc_case, ep);
789
790 /*
791 * Don't do anything else if this case is already solved.
792 */
793 if (fmd_case_solved(hdl, zcp->zc_case))
794 return;
795
796 /*
797 * Determine if we should solve the case and generate a fault. We solve
798 * a case if:
799 *
800 * a. A pool failed to open (ereport.fs.zfs.pool)
801 * b. A device failed to open (ereport.fs.zfs.pool) while a pool
802 * was up and running.
803 *
804 * We may see a series of ereports associated with a pool open, all
805 * chained together by the same ENA. If the pool open succeeds, then
806 * we'll see no further ereports. To detect when a pool open has
807 * succeeded, we associate a timer with the event. When it expires, we
808 * close the case.
809 */
810 if (fmd_nvl_class_match(hdl, nvl,
811 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
812 /*
813 * Pool level fault. Before solving the case, go through and
814 * close any open device cases that may be pending.
815 */
816 for (dcp = uu_list_first(zfs_cases); dcp != NULL;
817 dcp = uu_list_next(zfs_cases, dcp)) {
818 if (dcp->zc_data.zc_pool_guid ==
819 zcp->zc_data.zc_pool_guid &&
820 dcp->zc_data.zc_vdev_guid != 0)
821 fmd_case_close(hdl, dcp->zc_case);
822 }
823
824 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool", B_TRUE);
825 } else if (fmd_nvl_class_match(hdl, nvl,
826 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
827 /*
828 * Pool level fault for reading the intent logs.
829 */
830 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay", B_TRUE);
831 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
832 /*
833 * Device fault.
834 */
835 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device", B_TRUE);
836 } else if (fmd_nvl_class_match(hdl, nvl,
837 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
838 fmd_nvl_class_match(hdl, nvl,
839 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
840 fmd_nvl_class_match(hdl, nvl,
841 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
842 fmd_nvl_class_match(hdl, nvl,
843 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
844 char *failmode = NULL;
845 boolean_t checkremove = B_FALSE;
846
847 /*
848 * If this is a checksum or I/O error, then toss it into the
849 * appropriate SERD engine and check to see if it has fired.
850 * Ideally, we want to do something more sophisticated,
851 * (persistent errors for a single data block, etc). For now,
852 * a single SERD engine is sufficient.
853 */
854 if (fmd_nvl_class_match(hdl, nvl,
855 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
856 if (zcp->zc_data.zc_serd_io[0] == '\0') {
857 zfs_serd_name(zcp->zc_data.zc_serd_io,
858 pool_guid, vdev_guid, "io");
859 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
860 fmd_prop_get_int32(hdl, "io_N"),
861 fmd_prop_get_int64(hdl, "io_T"));
862 zfs_case_serialize(hdl, zcp);
863 }
864 if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep))
865 checkremove = B_TRUE;
866 } else if (fmd_nvl_class_match(hdl, nvl,
867 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
868 if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
869 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
870 pool_guid, vdev_guid, "checksum");
871 fmd_serd_create(hdl,
872 zcp->zc_data.zc_serd_checksum,
873 fmd_prop_get_int32(hdl, "checksum_N"),
874 fmd_prop_get_int64(hdl, "checksum_T"));
875 zfs_case_serialize(hdl, zcp);
876 }
877 if (fmd_serd_record(hdl,
878 zcp->zc_data.zc_serd_checksum, ep)) {
879 zfs_case_solve(hdl, zcp,
880 "fault.fs.zfs.vdev.checksum", B_FALSE);
881 }
882 } else if (fmd_nvl_class_match(hdl, nvl,
883 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
884 (nvlist_lookup_string(nvl,
885 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
886 failmode != NULL) {
887 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
888 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
889 zfs_case_solve(hdl, zcp,
890 "fault.fs.zfs.io_failure_continue",
891 B_FALSE);
892 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
893 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
894 zfs_case_solve(hdl, zcp,
895 "fault.fs.zfs.io_failure_wait", B_FALSE);
896 }
897 } else if (fmd_nvl_class_match(hdl, nvl,
898 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
899 checkremove = B_TRUE;
900 }
901
902 /*
903 * Because I/O errors may be due to device removal, we postpone
904 * any diagnosis until we're sure that we aren't about to
905 * receive a 'resource.fs.zfs.removed' event.
906 */
907 if (checkremove) {
908 if (zcp->zc_data.zc_has_remove_timer)
909 fmd_timer_remove(hdl, zcp->zc_remove_timer);
910 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
911 zfs_remove_timeout);
912 if (!zcp->zc_data.zc_has_remove_timer) {
913 zcp->zc_data.zc_has_remove_timer = 1;
914 zfs_case_serialize(hdl, zcp);
915 }
916 }
917 }
918 }
919
920 /*
921 * The timeout is fired when we diagnosed an I/O error, and it was not due to
922 * device removal (which would cause the timeout to be cancelled).
923 */
924 /* ARGSUSED */
925 static void
926 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
927 {
928 zfs_case_t *zcp = data;
929
930 if (id == zcp->zc_remove_timer)
931 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io", B_FALSE);
932 }
933
934 static void
935 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
936 {
937 zfs_case_t *zcp;
938
939 if ((zcp = fmd_case_getspecific(hdl, cs)) == NULL)
940 return;
941
942 if (zcp->zc_data.zc_serd_checksum[0] != '\0')
943 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
944 if (zcp->zc_data.zc_serd_io[0] != '\0')
945 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
946 if (zcp->zc_data.zc_has_remove_timer)
947 fmd_timer_remove(hdl, zcp->zc_remove_timer);
948 uu_list_remove(zfs_cases, zcp);
949 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
950 }
951
952 /*
953 * We use the fmd gc entry point to look for old cases that no longer apply.
954 * This allows us to keep our set of case data small in a long running system.
955 */
956 static void
957 zfs_fm_gc(fmd_hdl_t *hdl)
958 {
959 zfs_purge_cases(hdl);
960 }
961
962 static const fmd_hdl_ops_t fmd_ops = {
963 zfs_fm_recv, /* fmdo_recv */
964 zfs_fm_timeout, /* fmdo_timeout */
965 zfs_fm_close, /* fmdo_close */
966 NULL, /* fmdo_stats */
967 zfs_fm_gc, /* fmdo_gc */
968 };
969
970 static const fmd_prop_t fmd_props[] = {
971 { "checksum_N", FMD_TYPE_UINT32, "10" },
972 { "checksum_T", FMD_TYPE_TIME, "10min" },
973 { "io_N", FMD_TYPE_UINT32, "10" },
974 { "io_T", FMD_TYPE_TIME, "10min" },
975 { "remove_timeout", FMD_TYPE_TIME, "15sec" },
976 { NULL, 0, NULL }
977 };
978
979 static const fmd_hdl_info_t fmd_info = {
980 "ZFS Diagnosis Engine", "1.1", &fmd_ops, fmd_props
981 };
982
983 void
984 _fmd_init(fmd_hdl_t *hdl)
985 {
986 fmd_case_t *cp;
987 libzfs_handle_t *zhdl;
988
989 if ((zhdl = libzfs_init()) == NULL)
990 return;
991
992 if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
993 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
994 NULL, 0)) == NULL) {
995 libzfs_fini(zhdl);
996 return;
997 }
998
999 if ((zfs_cases = uu_list_create(zfs_case_pool, NULL, 0)) == NULL) {
1000 uu_list_pool_destroy(zfs_case_pool);
1001 libzfs_fini(zhdl);
1002 return;
1003 }
1004
1005 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1006 uu_list_destroy(zfs_cases);
1007 uu_list_pool_destroy(zfs_case_pool);
1008 libzfs_fini(zhdl);
1009 return;
1010 }
1011
1012 fmd_hdl_setspecific(hdl, zhdl);
1013
1014 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1015 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1016
1017 /*
1018 * Iterate over all active cases and unserialize the associated buffers,
1019 * adding them to our list of open cases.
1020 */
1021 for (cp = fmd_case_next(hdl, NULL);
1022 cp != NULL; cp = fmd_case_next(hdl, cp))
1023 (void) zfs_case_unserialize(hdl, cp);
1024
1025 /*
1026 * Clear out any old cases that are no longer valid.
1027 */
1028 zfs_purge_cases(hdl);
1029
1030 zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout");
1031 }
1032
1033 void
1034 _fmd_fini(fmd_hdl_t *hdl)
1035 {
1036 zfs_case_t *zcp;
1037 uu_list_walk_t *walk;
1038 libzfs_handle_t *zhdl;
1039
1040 /*
1041 * Remove all active cases.
1042 */
1043 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
1044 while ((zcp = uu_list_walk_next(walk)) != NULL) {
1045 uu_list_remove(zfs_cases, zcp);
1046 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1047 }
1048 uu_list_walk_end(walk);
1049
1050 uu_list_destroy(zfs_cases);
1051 uu_list_pool_destroy(zfs_case_pool);
1052
1053 zhdl = fmd_hdl_getspecific(hdl);
1054 libzfs_fini(zhdl);
1055 }