Print this page
NEX-18763 net-snmp service could go into maintenance, if there are hundreds of FMA cases (fix lint)
NEX-18763 net-snmp service could go into maintenance, if there are hundreds of FMA cases
Reviewed by: Roman Strashkin <roman.strashkin@nexenta.com>
NEX-18680 snmpd core dumped
Reviewed by: Dan Fields <dan.fields@nexenta.com>
Reviewed by: Alexander Eremin <alexander.eremin@nexenta.com>
NEX-18695 libfmd_snmp is unable to handle multiple OIDs in GET request
Reviewed by: Joyce McIntosh <joyce.mcintosh@nexenta.com>
Reviewed by: Cynthia Eastham <cynthia.eastham@nexenta.com>
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
NEX-17796 libfmd_snmp performance is awful
Reviewed by: Cynthia Eastham <cynthia.eastham@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
NEX-17938 libfmd_snmp dumps core in problem_lookup_uuid_exact()
Reviewed by: Cynthia Eastham <cynthia.eastham@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
NEX-17829 libfmd_snmp and snmp-notify should provide FMRIs for all fault types
Reviewed by: Cynthia Eastham <cynthia.eastham@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
NEX-17777 snmptable produces bogus output for FM tables
Reviewed by: Cynthia Eastham <cynthia.eastham@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
NEX-17772 libfmd_snmp should learn about new FmProblem fields
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Roman Strashkin <roman.strashkin@nexenta.com>
NEX-3125 libfmd_snmp should compile with newer net-snmp
Reviewed by: Gordon Ross <gwr@nexenta.com>
Reviewed by: Jean McCormack <jean.mccormack@nexenta.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>

@@ -22,58 +22,77 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright 2019 Nexenta Systems, Inc.
+ */
+
 #include <sys/fm/protocol.h>
+
 #include <fm/fmd_adm.h>
 #include <fm/fmd_snmp.h>
+#include <fm/libfmevent.h>
+#include <fm/libtopo.h>
+
 #include <net-snmp/net-snmp-config.h>
 #include <net-snmp/net-snmp-includes.h>
 #include <net-snmp/agent/net-snmp-agent-includes.h>
-#include <pthread.h>
-#include <stddef.h>
-#include <errno.h>
+
 #include <alloca.h>
-#include <locale.h>
-#include <libuutil.h>
+#include <errno.h>
 #include <libnvpair.h>
+#include <libuutil.h>
+#include <locale.h>
+#include <netdb.h>
+#include <pthread.h>
+#include <stddef.h>
+
 #include "sunFM_impl.h"
 #include "problem.h"
 
 /*
  * We assume that the number of suspect fault events associated with a
  * particular case will generally be sufficiently small that the overhead
  * associated with indexing them in a tree would exceed the gain from
  * not traversing the fault list for each request.
  */
-static uu_avl_pool_t    *problem_uuid_avl_pool;
-static uu_avl_t         *problem_uuid_avl;
+static uu_avl_pool_t    *problem_uuid_avl_pool = NULL;
+static uu_avl_t         *problem_uuid_avl = NULL;
 
 #define VALID_AVL_STATE (problem_uuid_avl_pool != NULL &&       \
         problem_uuid_avl != NULL)
 
-#define UPDATE_WAIT_MILLIS      10      /* poll interval in milliseconds */
-
-/*
- * Update types.  Single-index and all are mutually exclusive.
- */
-#define UCT_INDEX       0x1
-#define UCT_ALL         0x2
-#define UCT_FLAGS       0x3
-
-/*
- * Locking strategy is described in module.c.
- */
 static int              valid_stamp;
 static pthread_mutex_t  update_lock;
 static pthread_cond_t   update_cv;
-static volatile enum { US_QUIET, US_NEEDED, US_INPROGRESS } update_status;
+static fmev_shdl_t      evhdl;
 
 static Netsnmp_Node_Handler     sunFmProblemTable_handler;
 static Netsnmp_Node_Handler     sunFmFaultEventTable_handler;
 
+static char *
+nvl2fmri(nvlist_t *nvl)
+{
+        topo_hdl_t *thp;
+        int topoerr;
+        char *fmri, *ret = NULL;
+
+        thp = topo_open(TOPO_VERSION, NULL, &topoerr);
+        if (thp == NULL)
+                return (NULL);
+
+        if (topo_fmri_nvl2str(thp, nvl, &fmri, &topoerr) == 0) {
+                ret = strdup(fmri);
+                topo_hdl_strfree(thp, fmri);
+        }
+
+        topo_close(thp);
+        return (ret);
+}
+
 static sunFmProblem_data_t *
 problem_key_build(const char *uuid)
 {
         static sunFmProblem_data_t      key;
 

@@ -138,54 +157,105 @@
                 return (0);
 
         return (data->d_statuses[index - 1]);
 }
 
+#define FM_SUSPECT_SKIP \
+        (FM_SUSPECT_NOT_PRESENT | FM_SUSPECT_REPAIRED | \
+        FM_SUSPECT_REPLACED | FM_SUSPECT_ACQUITTED)
+
 /*ARGSUSED*/
 static int
 problem_update_one(const fmd_adm_caseinfo_t *acp, void *arg)
 {
         sunFmProblem_data_t             *data;
         nvlist_t                        *nvl;
         int64_t                         *diag_time;
         uint_t                          nelem;
-        uint32_t                        nsusp;
         int                             err;
+        int                             i;
+        int                             cr = 0;
+        uint8_t                         *statuses;
 
-        DEBUGMSGTL((MODNAME_STR, "update_one\n"));
-
         ASSERT(acp->aci_uuid != NULL);
 
         if ((data = problem_lookup_uuid_exact(acp->aci_uuid)) == NULL) {
                 uu_avl_index_t idx;
+                nvlist_t **fnvl;
+                nvlist_t *snvl;
+                uint_t nnvl;
 
+                /* Lookup statuses early so we could skip resolved problems */
+                if (nvlist_lookup_uint8_array(acp->aci_event,
+                    FM_SUSPECT_FAULT_STATUS, &statuses, &nelem) != 0)
+                        return (0);
+
+                for (i = 0; i < nelem; i++) {
+                        if (statuses[i] & FM_SUSPECT_SKIP)
+                                cr++;
+                }
+                if (cr == nelem) {
+                        DEBUGMSGTL((MODNAME_STR,
+                            "problem %s is resolved, skipping\n",
+                            acp->aci_uuid));
+                        return (0);
+                }
+
                 DEBUGMSGTL((MODNAME_STR, "found new problem %s\n",
                     acp->aci_uuid));
                 if ((data = SNMP_MALLOC_TYPEDEF(sunFmProblem_data_t)) == NULL) {
-                        (void) snmp_log(LOG_ERR, MODNAME_STR ": Out of memory "
-                            "for new problem data at %s:%d\n", __FILE__,
-                            __LINE__);
+                        (void) snmp_log(LOG_ERR, MODNAME_STR
+                            ": out of memory for new problem data\n");
                         return (0);
                 }
                 if ((err = nvlist_dup(acp->aci_event, &data->d_aci_event, 0))
                     != 0) {
-                        (void) snmp_log(LOG_ERR, MODNAME_STR ": Problem data "
-                            "setup failed: %s\n", strerror(err));
+                        (void) snmp_log(LOG_ERR, MODNAME_STR
+                            ": problem data setup failed: %s\n", strerror(err));
                         SNMP_FREE(data);
                         return (0);
                 }
 
-                data->d_aci_uuid = data->d_aci_code = data->d_aci_url = "-";
+                data->d_aci_uuid = data->d_aci_code = data->d_aci_type =
+                    data->d_aci_severity = data->d_aci_url =
+                    data->d_aci_desc = "-";
                 (void) nvlist_lookup_string(data->d_aci_event, FM_SUSPECT_UUID,
                     (char **)&data->d_aci_uuid);
                 (void) nvlist_lookup_string(data->d_aci_event,
                     FM_SUSPECT_DIAG_CODE, (char **)&data->d_aci_code);
+                (void) nvlist_lookup_string(data->d_aci_event,
+                    FM_SUSPECT_TYPE, (char **)&data->d_aci_type);
+                (void) nvlist_lookup_string(data->d_aci_event,
+                    FM_SUSPECT_SEVERITY, (char **)&data->d_aci_severity);
+                if (acp->aci_url != NULL)
                 data->d_aci_url = strdup(acp->aci_url);
+                (void) nvlist_lookup_string(data->d_aci_event,
+                    FM_SUSPECT_DESC, (char **)&data->d_aci_desc);
 
+                /*
+                 * NOTE: This should match the logic in libfmnotify.
+                 *
+                 * Extract the fault-list, and use the following order
+                 * of nested nvlists from its first element to make up FMRI:
+                 * - FRU
+                 * - ASRU
+                 * - resource
+                 */
+                if (nvlist_lookup_nvlist_array(data->d_aci_event,
+                    FM_SUSPECT_FAULT_LIST, &fnvl, &nnvl) == 0 && nnvl == 1 &&
+                    (nvlist_lookup_nvlist(fnvl[0], FM_FAULT_FRU, &snvl) == 0 ||
+                    nvlist_lookup_nvlist(fnvl[0], FM_FAULT_ASRU, &snvl) == 0 ||
+                    nvlist_lookup_nvlist(fnvl[0], FM_FAULT_RESOURCE,
+                    &snvl) == 0))
+                        data->d_aci_fmri = nvl2fmri(snvl);
+                if (data->d_aci_fmri == NULL)
+                        data->d_aci_fmri = "-";
+
                 if (nvlist_lookup_nvlist(data->d_aci_event, FM_SUSPECT_DE,
                     &nvl) == 0)
-                        if ((data->d_diag_engine = sunFm_nvl2str(nvl)) == NULL)
+                        data->d_diag_engine = nvl2fmri(nvl);
+                if (data->d_diag_engine == NULL)
                                 data->d_diag_engine = "-";
 
                 if (nvlist_lookup_int64_array(data->d_aci_event,
                     FM_SUSPECT_DIAG_TIME, &diag_time, &nelem) == 0 &&
                     nelem >= 2) {

@@ -192,23 +262,16 @@
                         data->d_diag_time.tv_sec = (long)diag_time[0];
                         data->d_diag_time.tv_usec = (long)diag_time[1];
                 }
 
                 (void) nvlist_lookup_uint32(data->d_aci_event,
-                    FM_SUSPECT_FAULT_SZ, &nsusp);
-                data->d_nsuspects = (ulong_t)nsusp;
-
+                    FM_SUSPECT_FAULT_SZ, &data->d_nsuspects);
                 (void) nvlist_lookup_nvlist_array(data->d_aci_event,
                     FM_SUSPECT_FAULT_LIST, &data->d_suspects, &nelem);
-
-                ASSERT(nelem == data->d_nsuspects);
-
                 (void) nvlist_lookup_uint8_array(data->d_aci_event,
                     FM_SUSPECT_FAULT_STATUS, &data->d_statuses, &nelem);
 
-                ASSERT(nelem == data->d_nsuspects);
-
                 uu_avl_node_init(data, &data->d_uuid_avl,
                     problem_uuid_avl_pool);
                 (void) uu_avl_find(problem_uuid_avl, data, NULL, &idx);
                 uu_avl_insert(problem_uuid_avl, data, idx);
 

@@ -215,106 +278,102 @@
                 data->d_valid = valid_stamp;
 
                 DEBUGMSGTL((MODNAME_STR, "completed new problem %s@%p\n",
                     data->d_aci_uuid, data));
         } else {
-                uint8_t *statuses;
-                int i;
+                if (nvlist_lookup_uint8_array(acp->aci_event,
+                    FM_SUSPECT_FAULT_STATUS, &statuses, &nelem) != 0)
+                        return (0);
 
-                (void) nvlist_lookup_uint8_array(acp->aci_event,
-                    FM_SUSPECT_FAULT_STATUS, &statuses, &nelem);
+                if (nelem != data->d_nsuspects) {
+                        DEBUGMSGTL((MODNAME_STR,
+                            "problem %s is malformed; deleting\n",
+                            data->d_aci_uuid));
+                        goto delete;
+                }
 
-                ASSERT(nelem == data->d_nsuspects);
-
-                for (i = 0; i < nelem; i++)
+                for (i = 0; i < nelem; i++) {
+                        if (statuses[i] & FM_SUSPECT_SKIP)
+                                cr++;
                         data->d_statuses[i] = statuses[i];
-
+                }
+                if (cr == nelem) {
+                        DEBUGMSGTL((MODNAME_STR,
+                            "problem %s is now resolved; deleting\n",
+                            data->d_aci_uuid));
+                        goto delete;
+                } else {
                 data->d_valid = valid_stamp;
         }
+        }
 
-        /*
-         * We don't touch problems we've seen before; they shouldn't change
-         * in any way we care about, since they've already been solved.  The
-         * state, however, could change, and if we later expose that to the
-         * client we need to update it here.
-         */
+        return (0);
 
+delete:
+        uu_avl_remove(problem_uuid_avl, data);
+        uu_avl_node_fini(data, &data->d_uuid_avl,
+            problem_uuid_avl_pool);
+        nvlist_free(data->d_aci_event);
+        SNMP_FREE(data);
         return (0);
 }
 
-static int
-problem_update(sunFmProblem_update_ctx_t *update_ctx)
+/*ARGSUSED*/
+static void *
+update_thread(void *arg)
 {
         fmd_adm_t *adm;
+        static struct timespec tv;
 
-        ASSERT(update_ctx != NULL);
-        ASSERT((update_ctx->uc_type & (UCT_INDEX|UCT_ALL)) !=
-            (UCT_INDEX|UCT_ALL));
-        ASSERT((update_ctx->uc_type & ~UCT_FLAGS) == 0);
+        /* Do a 1-minute checks for changes */
+        tv.tv_sec = 60;
+        tv.tv_nsec = 0;
+
+        for (;;) {
         ASSERT(VALID_AVL_STATE);
 
-        if ((adm = fmd_adm_open(update_ctx->uc_host, update_ctx->uc_prog,
-            update_ctx->uc_version)) == NULL) {
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": Communication with fmd "
-                    "failed: %s\n", strerror(errno));
-                return (SNMP_ERR_RESOURCEUNAVAILABLE);
+                (void) pthread_mutex_lock(&update_lock);
+                /* We don't care if we were awaken explicitly or by timeout */
+                (void) pthread_cond_reltimedwait_np(&update_cv, &update_lock,
+                    &tv);
+                if ((adm = fmd_adm_open(NULL, FMD_ADM_PROGRAM,
+                    FMD_ADM_VERSION)) == NULL) {
+                        (void) pthread_mutex_unlock(&update_lock);
+                        (void) snmp_log(LOG_ERR, MODNAME_STR
+                            ": communication with fmd failed: %s\n",
+                            strerror(errno));
+                        continue;
         }
 
-        ++valid_stamp;
+                valid_stamp++;
+
+                DEBUGMSGTL((MODNAME_STR, "case iteration started\n"));
         if (fmd_adm_case_iter(adm, SNMP_URL_MSG, problem_update_one,
-            update_ctx) != 0) {
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": fmd case information "
-                    "update failed: %s\n", fmd_adm_errmsg(adm));
+                    NULL) != 0) {
+                        (void) pthread_mutex_unlock(&update_lock);
+                        (void) snmp_log(LOG_ERR, MODNAME_STR
+                            ": fmd case information update failed: %s\n",
+                            fmd_adm_errmsg(adm));
                 fmd_adm_close(adm);
-                return (SNMP_ERR_RESOURCEUNAVAILABLE);
+                        continue;
         }
 
+                fmd_adm_close(adm);
+                (void) pthread_mutex_unlock(&update_lock);
+
         DEBUGMSGTL((MODNAME_STR, "case iteration completed\n"));
+        }
 
-        fmd_adm_close(adm);
-        return (SNMP_ERR_NOERROR);
+        /*NOTREACHED*/
+        return (NULL);
 }
 
 /*ARGSUSED*/
 static void
-update_thread(void *arg)
+event_cb(fmev_t ev, const char *class, nvlist_t *nvl, void *arg)
 {
-        /*
-         * The current problem_update implementation offers minimal savings
-         * for the use of index-only updates; therefore we always do a full
-         * update.  If it becomes advantageous to limit updates to a single
-         * index, the contexts can be queued by the handler instead.
-         */
-        sunFmProblem_update_ctx_t       uc;
-
-        uc.uc_host = NULL;
-        uc.uc_prog = FMD_ADM_PROGRAM;
-        uc.uc_version = FMD_ADM_VERSION;
-
-        uc.uc_index = NULL;
-        uc.uc_type = UCT_ALL;
-
-        for (;;) {
                 (void) pthread_mutex_lock(&update_lock);
-                update_status = US_QUIET;
-                while (update_status == US_QUIET)
-                        (void) pthread_cond_wait(&update_cv, &update_lock);
-                update_status = US_INPROGRESS;
-                (void) pthread_mutex_unlock(&update_lock);
-                (void) problem_update(&uc);
-        }
-}
-
-static void
-request_update(void)
-{
-        (void) pthread_mutex_lock(&update_lock);
-        if (update_status != US_QUIET) {
-                (void) pthread_mutex_unlock(&update_lock);
-                return;
-        }
-        update_status = US_NEEDED;
         (void) pthread_cond_signal(&update_cv);
         (void) pthread_mutex_unlock(&update_lock);
 }
 
 /*ARGSUSED*/

@@ -327,144 +386,135 @@
         ASSERT(l_data != NULL && r_data != NULL);
 
         return (strcmp(l_data->d_aci_uuid, r_data->d_aci_uuid));
 }
 
-int
-sunFmProblemTable_init(void)
+/* ARGSUSED */
+void *
+pid_thread(void *arg)
 {
-        static oid sunFmProblemTable_oid[] = { SUNFMPROBLEMTABLE_OID };
-        netsnmp_table_registration_info *table_info;
-        netsnmp_handler_registration *handler;
-        int err;
+        pid_t pid = getpid();
+        int wait = 0;
 
-        if ((err = pthread_mutex_init(&update_lock, NULL)) != 0) {
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": mutex_init failure: "
-                    "%s\n", strerror(err));
-                return (MIB_REGISTRATION_FAILED);
-        }
-        if ((err = pthread_cond_init(&update_cv, NULL)) != 0) {
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": cond_init failure: "
-                    "%s\n", strerror(err));
-                return (MIB_REGISTRATION_FAILED);
-        }
-
-        if ((err = pthread_create(NULL, NULL, (void *(*)(void *))update_thread,
-            NULL)) != 0) {
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": error creating update "
-                    "thread: %s\n", strerror(err));
-                return (MIB_REGISTRATION_FAILED);
-        }
-
-        if ((table_info =
-            SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info)) == NULL)
-                return (MIB_REGISTRATION_FAILED);
-
-        if ((handler = netsnmp_create_handler_registration("sunFmProblemTable",
-            sunFmProblemTable_handler, sunFmProblemTable_oid,
-            OID_LENGTH(sunFmProblemTable_oid), HANDLER_CAN_RONLY)) == NULL) {
-                SNMP_FREE(table_info);
-                return (MIB_REGISTRATION_FAILED);
-        }
-
         /*
-         * The Net-SNMP template uses add_indexes here, but that
-         * function is unsafe because it does not check for failure.
+         * Workaround the forking madness in net-snmp -- we need to
+         * subscribe from the *forked* process so that event notifications
+         * get our PID correctly.
+         *
+         * We also limit the wait to arbitrary long time of 10 seconds so that
+         * we subscribe to event notifications when running with -f (don't fork)
+         * specified.
          */
-        if (netsnmp_table_helper_add_index(table_info, ASN_OCTET_STR) == NULL) {
-                SNMP_FREE(table_info);
-                SNMP_FREE(handler);
-                return (MIB_REGISTRATION_FAILED);
+        for (;;) {
+                if (getpid() != pid || wait == 10) {
+                        /* Subscribe to fault event notifications */
+                        evhdl = fmev_shdl_init(LIBFMEVENT_VERSION_2, NULL, NULL,
+                            NULL);
+                        (void) fmev_shdl_subscribe(evhdl, "list.*", event_cb,
+                            NULL);
+                        break;
         }
-
-        table_info->min_column = SUNFMPROBLEM_COLMIN;
-        table_info->max_column = SUNFMPROBLEM_COLMAX;
-
-        if ((problem_uuid_avl_pool = uu_avl_pool_create("problem_uuid",
-            sizeof (sunFmProblem_data_t),
-            offsetof(sunFmProblem_data_t, d_uuid_avl), problem_compare_uuid,
-            UU_AVL_DEBUG)) == NULL) {
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": problem_uuid avl pool "
-                    "creation failed: %s\n", uu_strerror(uu_error()));
-                snmp_free_varbind(table_info->indexes);
-                SNMP_FREE(table_info);
-                SNMP_FREE(handler);
-                return (MIB_REGISTRATION_FAILED);
+                wait++;
+                (void) sleep(1);
         }
 
-        if ((problem_uuid_avl = uu_avl_create(problem_uuid_avl_pool, NULL,
-            UU_AVL_DEBUG)) == NULL) {
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": problem_uuid avl "
-                    "creation failed: %s\n", uu_strerror(uu_error()));
-                snmp_free_varbind(table_info->indexes);
-                SNMP_FREE(table_info);
-                SNMP_FREE(handler);
-                uu_avl_pool_destroy(problem_uuid_avl_pool);
-                return (MIB_REGISTRATION_FAILED);
-        }
-
-        if ((err = netsnmp_register_table(handler, table_info)) !=
-            MIB_REGISTERED_OK) {
-                snmp_free_varbind(table_info->indexes);
-                SNMP_FREE(table_info);
-                SNMP_FREE(handler);
-                uu_avl_destroy(problem_uuid_avl);
-                uu_avl_pool_destroy(problem_uuid_avl_pool);
-                return (err);
-        }
-
-        return (MIB_REGISTERED_OK);
+        return (NULL);
 }
 
 int
-sunFmFaultEventTable_init(void)
+sunFmProblemTable_init(void)
 {
         static oid sunFmFaultEventTable_oid[] = { SUNFMFAULTEVENTTABLE_OID };
-        netsnmp_table_registration_info *table_info;
-        netsnmp_handler_registration *handler;
-        int err;
+        netsnmp_table_registration_info *ftinfo = NULL;
+        netsnmp_handler_registration *fhandler = NULL;
+        static oid sunFmProblemTable_oid[] = { SUNFMPROBLEMTABLE_OID };
+        netsnmp_table_registration_info *ptinfo = NULL;
+        netsnmp_handler_registration *phandler = NULL;
+        pthread_t ptid;
+        pthread_t utid;
+        int ret = MIB_REGISTRATION_FAILED;
 
-        if ((table_info =
-            SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info)) == NULL)
-                return (MIB_REGISTRATION_FAILED);
-
-        if ((handler =
+        /* Create fault event table and handler */
+        if ((ftinfo =
+            SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info)) == NULL ||
+            netsnmp_table_helper_add_index(ftinfo, ASN_OCTET_STR) == NULL ||
+            netsnmp_table_helper_add_index(ftinfo, ASN_UNSIGNED) == NULL ||
+            (fhandler =
             netsnmp_create_handler_registration("sunFmFaultEventTable",
             sunFmFaultEventTable_handler, sunFmFaultEventTable_oid,
-            OID_LENGTH(sunFmFaultEventTable_oid), HANDLER_CAN_RONLY)) == NULL) {
-                SNMP_FREE(table_info);
-                return (MIB_REGISTRATION_FAILED);
-        }
+            OID_LENGTH(sunFmFaultEventTable_oid), HANDLER_CAN_RONLY)) == NULL)
+                goto fail;
 
-        /*
-         * The Net-SNMP template uses add_indexes here, but that
-         * function is unsafe because it does not check for failure.
-         */
-        if (netsnmp_table_helper_add_index(table_info, ASN_OCTET_STR) == NULL) {
-                SNMP_FREE(table_info);
-                SNMP_FREE(handler);
-                return (MIB_REGISTRATION_FAILED);
-        }
-        if (netsnmp_table_helper_add_index(table_info, ASN_UNSIGNED) == NULL) {
-                snmp_free_varbind(table_info->indexes);
-                SNMP_FREE(table_info);
-                SNMP_FREE(handler);
-                return (MIB_REGISTRATION_FAILED);
-        }
+        ftinfo->min_column = SUNFMFAULTEVENT_COLMIN;
+        ftinfo->max_column = SUNFMFAULTEVENT_COLMAX;
 
-        table_info->min_column = SUNFMFAULTEVENT_COLMIN;
-        table_info->max_column = SUNFMFAULTEVENT_COLMAX;
+        /* Register fault event handler */
+        if ((ret = netsnmp_register_table(fhandler, ftinfo)) !=
+            MIB_REGISTERED_OK)
+                goto fail;
 
-        if ((err = netsnmp_register_table(handler, table_info)) !=
-            MIB_REGISTERED_OK) {
-                snmp_free_varbind(table_info->indexes);
-                SNMP_FREE(table_info);
-                SNMP_FREE(handler);
-                return (err);
+        /* Create problem table, data pool, and handler */
+        if ((problem_uuid_avl_pool = uu_avl_pool_create("problem_uuid",
+            sizeof (sunFmProblem_data_t), offsetof(sunFmProblem_data_t,
+            d_uuid_avl), problem_compare_uuid, UU_AVL_DEBUG)) == NULL ||
+            (problem_uuid_avl = uu_avl_create(problem_uuid_avl_pool, NULL,
+            UU_AVL_DEBUG)) == NULL ||
+            (ptinfo =
+            SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info)) == NULL ||
+            netsnmp_table_helper_add_index(ptinfo, ASN_OCTET_STR) == NULL ||
+            (phandler =
+            netsnmp_create_handler_registration("sunFmProblemTable",
+            sunFmProblemTable_handler, sunFmProblemTable_oid,
+            OID_LENGTH(sunFmProblemTable_oid), HANDLER_CAN_RONLY)) == NULL)
+                goto fail;
+
+        ptinfo->min_column = SUNFMPROBLEM_COLMIN;
+        ptinfo->max_column = SUNFMPROBLEM_COLMAX;
+
+        /* Register problem handler */
+        if ((ret = netsnmp_register_table(phandler, ptinfo)) !=
+            MIB_REGISTERED_OK)
+                goto fail;
+
+        /* Create PID change waiter thread */
+        if (pthread_create(&ptid, NULL, pid_thread, 0) != 0) {
+                (void) snmp_log(LOG_ERR, MODNAME_STR
+                    ": failed to create pid thread: %s\n", strerror(ret));
+                goto fail;
         }
 
+        /* Create update thread */
+        if ((ret = pthread_mutex_init(&update_lock, NULL)) != 0 ||
+            (ret = pthread_cond_init(&update_cv, NULL)) != 0 ||
+            (ret = pthread_create(&utid, NULL, update_thread, 0)) != 0) {
+                (void) snmp_log(LOG_ERR, MODNAME_STR
+                    ": failed to create update thread: %s\n", strerror(ret));
+                goto fail;
+        }
+
         return (MIB_REGISTERED_OK);
+
+fail:
+        (void) pthread_mutex_destroy(&update_lock);
+        if (problem_uuid_avl != NULL)
+                uu_avl_destroy(problem_uuid_avl);
+        if (problem_uuid_avl_pool != NULL)
+                uu_avl_pool_destroy(problem_uuid_avl_pool);
+        if (ftinfo->indexes != NULL)
+                snmp_free_varbind(ftinfo->indexes);
+        if (ftinfo != NULL)
+                SNMP_FREE(ftinfo);
+        if (ptinfo->indexes != NULL)
+                snmp_free_varbind(ptinfo->indexes);
+        if (ptinfo != NULL)
+                SNMP_FREE(ptinfo);
+        if (fhandler != NULL)
+                SNMP_FREE(fhandler);
+        if (phandler != NULL)
+                SNMP_FREE(phandler);
+
+        return (ret);
 }
 
 /*
  * Returns the problem data for the problem whose uuid is next according
  * to ASN.1 lexical ordering after the request in table_info.  Indexes are

@@ -499,11 +549,11 @@
                 table_info->number_indexes = 1;
                 table_info->index_oid_len = table_info->indexes->name_length;
                 (void) memcpy(table_info->index_oid, table_info->indexes->name,
                     table_info->indexes->name_length);
 
-                DEBUGMSGTL((MODNAME_STR, "nextpr: built fake index:\n"));
+                DEBUGMSGTL((MODNAME_STR, "nextpr: built fake index: "));
                 DEBUGMSGVAR((MODNAME_STR, table_info->indexes));
                 DEBUGMSG((MODNAME_STR, "\n"));
         } else {
                 /*
                  * Construct the next possible UUID to look for.  We can

@@ -647,12 +697,12 @@
                                 }
                                 snmp_free_varbind(
                                     table_info->indexes->next_variable);
                                 table_info->indexes->next_variable = var;
                                 table_info->number_indexes = 2;
-                                DEBUGMSGTL((MODNAME_STR, "nextfe: built fake "
-                                    "index:\n"));
+                                DEBUGMSGTL((MODNAME_STR,
+                                    "nextfe: built fake index: "));
                                 DEBUGMSGVAR((MODNAME_STR, table_info->indexes));
                                 DEBUGMSG((MODNAME_STR, "\n"));
                                 DEBUGMSGVAR((MODNAME_STR,
                                     table_info->indexes->next_variable));
                                 DEBUGMSG((MODNAME_STR, "\n"));

@@ -689,45 +739,32 @@
         return (faultevent_lookup_index_exact(data,
             *(ulong_t *)table_info->indexes->next_variable->val.integer));
 }
 
 /*ARGSUSED*/
-static void
-sunFmProblemTable_return(unsigned int reg, void *arg)
+static int
+sunFmProblemTable_handler(netsnmp_mib_handler *handler,
+    netsnmp_handler_registration *reginfo, netsnmp_agent_request_info *reqinfo,
+    netsnmp_request_info *request)
 {
-        netsnmp_delegated_cache         *cache = (netsnmp_delegated_cache *)arg;
-        netsnmp_request_info            *request;
-        netsnmp_agent_request_info      *reqinfo;
-        netsnmp_handler_registration    *reginfo;
         netsnmp_table_request_info      *table_info;
         sunFmProblem_data_t             *data;
+        int                             ret = SNMP_ERR_NOERROR;
 
-        ASSERT(netsnmp_handler_check_cache(cache) != NULL);
+        /*
+         * We don't support MODE_GETBULK directly, so all bulk requests should
+         * come through bulk_to_next helper.  Make sure it stays that way.
+         */
+        ASSERT(reqinfo->mode == MODE_GET || reqinfo->mode == MODE_GETNEXT);
 
         (void) pthread_mutex_lock(&update_lock);
-        if (update_status != US_QUIET) {
-                struct timeval                  tv;
 
-                tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
-                tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
-
-                (void) snmp_alarm_register_hr(tv, 0, sunFmProblemTable_return,
-                    cache);
-                (void) pthread_mutex_unlock(&update_lock);
-                return;
-        }
-
-        request = cache->requests;
-        reqinfo = cache->reqinfo;
-        reginfo = cache->reginfo;
-
+        for (; request != NULL; request = request->next) {
         table_info = netsnmp_extract_table_info(request);
-        request->delegated = 0;
+                if (table_info == NULL)
+                        continue;
 
-        ASSERT(table_info->colnum >= SUNFMPROBLEM_COLMIN);
-        ASSERT(table_info->colnum <= SUNFMPROBLEM_COLMAX);
-
         /*
          * table_info->colnum contains the column number requested.
          * table_info->indexes contains a linked list of snmp variable
          * bindings for the indexes of the table.  Values in the list
          * have been set corresponding to the indexes of the

@@ -738,160 +775,143 @@
          * - We will never receive requests outside our table nor
          *   those with the first subid anything other than 1 (Entry)
          *   nor those without a column number.  This is true even
          *   for GETNEXT requests.
          */
-
         switch (reqinfo->mode) {
         case MODE_GET:
-                if ((data = sunFmProblemTable_pr(reginfo, table_info)) ==
-                    NULL) {
-                        netsnmp_free_delegated_cache(cache);
-                        (void) pthread_mutex_unlock(&update_lock);
-                        return;
-                }
+                        data = sunFmProblemTable_pr(reginfo, table_info);
+                        if (data == NULL)
+                                goto out;
                 break;
         case MODE_GETNEXT:
-        case MODE_GETBULK:
-                if ((data = sunFmProblemTable_nextpr(reginfo, table_info)) ==
-                    NULL) {
-                        netsnmp_free_delegated_cache(cache);
-                        (void) pthread_mutex_unlock(&update_lock);
-                        return;
-                }
+                        data = sunFmProblemTable_nextpr(reginfo, table_info);
+                        if (data == NULL)
+                                goto out;
                 break;
         default:
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": Unsupported request "
-                    "mode %d\n", reqinfo->mode);
-                netsnmp_free_delegated_cache(cache);
-                (void) pthread_mutex_unlock(&update_lock);
-                return;
+                        (void) snmp_log(LOG_ERR, MODNAME_STR
+                            ": unsupported request mode: %d\n", reqinfo->mode);
+                        ret = SNMP_ERR_GENERR;
+                        goto out;
         }
 
         switch (table_info->colnum) {
         case SUNFMPROBLEM_COL_UUID:
-        {
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)data->d_aci_uuid,
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_aci_uuid,
                     strlen(data->d_aci_uuid));
                 break;
+                case SUNFMPROBLEM_COL_HOSTNAME: {
+                        char hostname[MAXHOSTNAMELEN+1];
+
+                        (void) gethostname(hostname, sizeof (hostname) - 1);
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, (uchar_t *)hostname,
+                            strlen(hostname));
+                        break;
         }
         case SUNFMPROBLEM_COL_CODE:
-        {
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)data->d_aci_code,
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_aci_code,
                     strlen(data->d_aci_code));
                 break;
-        }
+                case SUNFMPROBLEM_COL_TYPE:
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_aci_type,
+                            strlen(data->d_aci_type));
+                        break;
+                case SUNFMPROBLEM_COL_SEVERITY:
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_aci_severity,
+                            strlen(data->d_aci_severity));
+                        break;
         case SUNFMPROBLEM_COL_URL:
-        {
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)data->d_aci_url,
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_aci_url,
                     strlen(data->d_aci_url));
                 break;
-        }
+                case SUNFMPROBLEM_COL_DESC:
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_aci_desc,
+                            strlen(data->d_aci_desc));
+                        break;
+                case SUNFMPROBLEM_COL_FMRI:
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_aci_fmri,
+                            strlen(data->d_aci_fmri));
+                        break;
         case SUNFMPROBLEM_COL_DIAGENGINE:
-        {
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)data->d_diag_engine,
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)data->d_diag_engine,
                     strlen(data->d_diag_engine));
                 break;
-        }
-        case SUNFMPROBLEM_COL_DIAGTIME:
-        {
+                case SUNFMPROBLEM_COL_DIAGTIME: {
                 /*
                  * The date_n_time function is not Y2038-safe; this may
-                 * need to be updated when a suitable Y2038-safe Net-SNMP
-                 * API is available.
+                         * need to be updated when a suitable Y2038-safe
+                         * Net-SNMP API is available.
                  */
                 size_t  dt_size;
                 time_t  dt_time = (time_t)data->d_diag_time.tv_sec;
                 uchar_t *dt = date_n_time(&dt_time, &dt_size);
 
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, dt, dt_size);
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, dt, dt_size);
                 break;
         }
         case SUNFMPROBLEM_COL_SUSPECTCOUNT:
-        {
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_UNSIGNED, (uchar_t *)&data->d_nsuspects,
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_UNSIGNED,
+                            (uchar_t *)&data->d_nsuspects,
                     sizeof (data->d_nsuspects));
                 break;
-        }
         default:
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, (uchar_t *)"-",
+                            strlen("-"));
                 break;
         }
+        }
 
-        netsnmp_free_delegated_cache(cache);
+out:
         (void) pthread_mutex_unlock(&update_lock);
+        return (ret);
 }
 
+/*ARGSUSED*/
 static int
-sunFmProblemTable_handler(netsnmp_mib_handler *handler,
+sunFmFaultEventTable_handler(netsnmp_mib_handler *handler,
     netsnmp_handler_registration *reginfo, netsnmp_agent_request_info *reqinfo,
-    netsnmp_request_info *requests)
+    netsnmp_request_info *request)
 {
-        netsnmp_request_info            *request;
-        struct timeval                  tv;
-
-        tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
-        tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
-
-        request_update();
-
-        for (request = requests; request; request = request->next) {
-                if (request->processed != 0)
-                        continue;
-
-                if (netsnmp_extract_table_info(request) == NULL)
-                        continue;
-
-                request->delegated = 1;
-                (void) snmp_alarm_register_hr(tv, 0,
-                    sunFmProblemTable_return,
-                    (void *) netsnmp_create_delegated_cache(handler, reginfo,
-                    reqinfo, request, NULL));
-        }
-
-        return (SNMP_ERR_NOERROR);
-}
-
-/*ARGSUSED*/
-static void
-sunFmFaultEventTable_return(unsigned int reg, void *arg)
-{
-        netsnmp_delegated_cache         *cache = (netsnmp_delegated_cache *)arg;
-        netsnmp_request_info            *request;
-        netsnmp_agent_request_info      *reqinfo;
-        netsnmp_handler_registration    *reginfo;
         netsnmp_table_request_info      *table_info;
-        sunFmProblem_data_t             *pdata;
         sunFmFaultEvent_data_t          *data;
         sunFmFaultStatus_data_t         status;
+        sunFmProblem_data_t             *pdata;
+        int                             ret = SNMP_ERR_NOERROR;
 
-        ASSERT(netsnmp_handler_check_cache(cache) != NULL);
+        /*
+         * We don't support MODE_GETBULK directly, so all bulk requests should
+         * come through bulk_to_next helper.  Make sure it stays that way.
+         */
+        ASSERT(reqinfo->mode == MODE_GET || reqinfo->mode == MODE_GETNEXT);
 
         (void) pthread_mutex_lock(&update_lock);
-        if (update_status != US_QUIET) {
-                struct timeval                  tv;
 
-                tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
-                tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
-
-                (void) snmp_alarm_register_hr(tv, 0,
-                    sunFmFaultEventTable_return, cache);
-                (void) pthread_mutex_unlock(&update_lock);
-                return;
-        }
-
-        request = cache->requests;
-        reqinfo = cache->reqinfo;
-        reginfo = cache->reginfo;
-
+        for (; request != NULL; request = request->next) {
         table_info = netsnmp_extract_table_info(request);
-        request->delegated = 0;
+                if (table_info == NULL)
+                        continue;
 
         ASSERT(table_info->colnum >= SUNFMFAULTEVENT_COLMIN);
         ASSERT(table_info->colnum <= SUNFMFAULTEVENT_COLMAX);
 
         /*

@@ -906,122 +926,109 @@
          * - We will never receive requests outside our table nor
          *   those with the first subid anything other than 1 (Entry)
          *   nor those without a column number.  This is true even
          *   for GETNEXT requests.
          */
-
         switch (reqinfo->mode) {
         case MODE_GET:
-                if ((data = sunFmFaultEventTable_fe(reginfo, table_info,
-                    &status)) == NULL) {
-                        netsnmp_free_delegated_cache(cache);
-                        (void) pthread_mutex_unlock(&update_lock);
-                        return;
-                }
+                        data = sunFmFaultEventTable_fe(reginfo, table_info,
+                            &status);
+                        if (data == NULL)
+                                goto out;
                 break;
         case MODE_GETNEXT:
-        case MODE_GETBULK:
-                if ((data = sunFmFaultEventTable_nextfe(reginfo, table_info,
-                    &status)) == NULL) {
-                        netsnmp_free_delegated_cache(cache);
-                        (void) pthread_mutex_unlock(&update_lock);
-                        return;
-                }
+                        data = sunFmFaultEventTable_nextfe(reginfo, table_info,
+                            &status);
+                        if (data == NULL)
+                                goto out;
                 break;
         default:
-                (void) snmp_log(LOG_ERR, MODNAME_STR ": Unsupported request "
-                    "mode %d\n", reqinfo->mode);
-                netsnmp_free_delegated_cache(cache);
-                (void) pthread_mutex_unlock(&update_lock);
-                return;
+                        (void) snmp_log(LOG_ERR, MODNAME_STR
+                            ": unsupported request mode: %d\n", reqinfo->mode);
+                        ret = SNMP_ERR_GENERR;
+                        goto out;
         }
 
         switch (table_info->colnum) {
         case SUNFMFAULTEVENT_COL_PROBLEMUUID:
-        {
                 if ((pdata = sunFmProblemTable_pr(reginfo, table_info))
                     == NULL) {
-                        (void) netsnmp_table_build_result(reginfo, request,
-                            table_info, ASN_OCTET_STR, NULL, 0);
+                                (void) netsnmp_table_build_result(reginfo,
+                                    request, table_info, ASN_OCTET_STR,
+                                    NULL, 0);
                         break;
                 }
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)pdata->d_aci_uuid,
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR,
+                            (uchar_t *)pdata->d_aci_uuid,
                     strlen(pdata->d_aci_uuid));
                 break;
-        }
-        case SUNFMFAULTEVENT_COL_CLASS:
-        {
+                case SUNFMFAULTEVENT_COL_CLASS: {
                 char    *class = "-";
 
                 (void) nvlist_lookup_string(data, FM_CLASS, &class);
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)class, strlen(class));
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, (uchar_t *)class,
+                            strlen(class));
                 break;
         }
-        case SUNFMFAULTEVENT_COL_CERTAINTY:
-        {
+                case SUNFMFAULTEVENT_COL_CERTAINTY: {
                 uint8_t pct = 0;
                 ulong_t pl;
 
                 (void) nvlist_lookup_uint8(data, FM_FAULT_CERTAINTY,
                     &pct);
                 pl = (ulong_t)pct;
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_UNSIGNED, (uchar_t *)&pl, sizeof (pl));
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_UNSIGNED, (uchar_t *)&pl,
+                            sizeof (pl));
                 break;
         }
-        case SUNFMFAULTEVENT_COL_ASRU:
-        {
+                case SUNFMFAULTEVENT_COL_ASRU: {
                 nvlist_t        *asru = NULL;
-                char            *fmri, *str;
+                        char            *fmri = "-", *str;
 
                 (void) nvlist_lookup_nvlist(data, FM_FAULT_ASRU, &asru);
-                if ((str = sunFm_nvl2str(asru)) == NULL)
-                        fmri = "-";
-                else
+                        if ((str = nvl2fmri(asru)) != NULL)
                         fmri = str;
 
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)fmri, strlen(fmri));
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, (uchar_t *)fmri,
+                            strlen(fmri));
                 free(str);
                 break;
         }
-        case SUNFMFAULTEVENT_COL_FRU:
-        {
+                case SUNFMFAULTEVENT_COL_FRU: {
                 nvlist_t        *fru = NULL;
-                char            *fmri, *str;
+                        char            *fmri = "-", *str;
 
                 (void) nvlist_lookup_nvlist(data, FM_FAULT_FRU, &fru);
-                if ((str = sunFm_nvl2str(fru)) == NULL)
-                        fmri = "-";
-                else
+                        if ((str = nvl2fmri(fru)) != NULL)
                         fmri = str;
 
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)fmri, strlen(fmri));
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, (uchar_t *)fmri,
+                            strlen(fmri));
                 free(str);
                 break;
         }
-        case SUNFMFAULTEVENT_COL_RESOURCE:
-        {
+                case SUNFMFAULTEVENT_COL_RESOURCE: {
                 nvlist_t        *rsrc = NULL;
-                char            *fmri, *str;
+                        char            *fmri = "-", *str;
 
-                (void) nvlist_lookup_nvlist(data, FM_FAULT_RESOURCE, &rsrc);
-                if ((str = sunFm_nvl2str(rsrc)) == NULL)
-                        fmri = "-";
-                else
+                        (void) nvlist_lookup_nvlist(data, FM_FAULT_RESOURCE,
+                            &rsrc);
+                        if ((str = nvl2fmri(rsrc)) != NULL)
                         fmri = str;
 
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)fmri, strlen(fmri));
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, (uchar_t *)fmri,
+                            strlen(fmri));
                 free(str);
                 break;
         }
-        case SUNFMFAULTEVENT_COL_STATUS:
-        {
+                case SUNFMFAULTEVENT_COL_STATUS: {
                 ulong_t pl = SUNFMFAULTEVENT_STATE_OTHER;
 
                 if (status & FM_SUSPECT_FAULTY)
                         pl = SUNFMFAULTEVENT_STATE_FAULTY;
                 else if (status & FM_SUSPECT_NOT_PRESENT)

@@ -1030,55 +1037,29 @@
                         pl = SUNFMFAULTEVENT_STATE_REPLACED;
                 else if (status & FM_SUSPECT_REPAIRED)
                         pl = SUNFMFAULTEVENT_STATE_REPAIRED;
                 else if (status & FM_SUSPECT_ACQUITTED)
                         pl = SUNFMFAULTEVENT_STATE_ACQUITTED;
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_INTEGER, (uchar_t *)&pl, sizeof (pl));
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_INTEGER, (uchar_t *)&pl,
+                            sizeof (pl));
                 break;
         }
-        case SUNFMFAULTEVENT_COL_LOCATION:
-        {
+                case SUNFMFAULTEVENT_COL_LOCATION: {
                 char    *location = "-";
 
-                (void) nvlist_lookup_string(data, FM_FAULT_LOCATION, &location);
-                (void) netsnmp_table_build_result(reginfo, request, table_info,
-                    ASN_OCTET_STR, (uchar_t *)location, strlen(location));
+                        (void) nvlist_lookup_string(data, FM_FAULT_LOCATION,
+                            &location);
+                        (void) netsnmp_table_build_result(reginfo, request,
+                            table_info, ASN_OCTET_STR, (uchar_t *)location,
+                            strlen(location));
                 break;
         }
         default:
                 break;
         }
-
-        netsnmp_free_delegated_cache(cache);
-        (void) pthread_mutex_unlock(&update_lock);
-}
-
-static int
-sunFmFaultEventTable_handler(netsnmp_mib_handler *handler,
-    netsnmp_handler_registration *reginfo, netsnmp_agent_request_info *reqinfo,
-    netsnmp_request_info *requests)
-{
-        netsnmp_request_info            *request;
-        struct timeval                  tv;
-
-        tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
-        tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
-
-        request_update();
-
-        for (request = requests; request; request = request->next) {
-                if (request->processed != 0)
-                        continue;
-
-                if (netsnmp_extract_table_info(request) == NULL)
-                        continue;
-
-                request->delegated = 1;
-                (void) snmp_alarm_register_hr(tv, 0,
-                    sunFmFaultEventTable_return,
-                    (void *) netsnmp_create_delegated_cache(handler, reginfo,
-                    reqinfo, request, NULL));
         }
 
-        return (SNMP_ERR_NOERROR);
+out:
+        (void) pthread_mutex_unlock(&update_lock);
+        return (ret);
 }