Print this page
NEX-18417 bring back UUID-based OIDs for FM traps
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Roman Strashkin <roman.strashkin@nexenta.com>
NEX-17772 libfmd_snmp should learn about new FmProblem fields
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Roman Strashkin <roman.strashkin@nexenta.com>
NEX-16536 SUN-IREPORT-MIB is broken
NEX-16537 enhance FM traps
NEX-16545 SMF dict should have obsolete entries removed
Reviewed by: Dan Fields <dan.fields@nexenta.com>
Reviewed by: Cynthia Eastham <cynthia.eastham@nexenta.com>
Reviewed by: Alexander Eremin <alexander.eremin@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
NEX-14494 FMA related SNMP traps should add description
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
        
*** 21,48 ****
  
  /*
   * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
   */
  
  #include <sys/fm/protocol.h>
  #include <fm/fmd_snmp.h>
  #include <fm/fmd_msg.h>
  #include <fm/libfmevent.h>
  #include <net-snmp/net-snmp-config.h>
  #include <net-snmp/net-snmp-includes.h>
  #include <net-snmp/agent/net-snmp-agent-includes.h>
  #include <errno.h>
  #include <locale.h>
  #include <netdb.h>
  #include <signal.h>
- #include <strings.h>
  #include <stdlib.h>
  #include <unistd.h>
- #include <limits.h>
- #include <alloca.h>
- #include <priv_utils.h>
  #include <zone.h>
  #include "libfmnotify.h"
  
  /*
   * Debug messages can be enabled by setting the debug property to true
   *
--- 21,56 ----
  
  /*
   * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
   */
  
+ /*
+  * Copyright 2018 Nexenta Systems, Inc.
+  */
+ 
  #include <sys/fm/protocol.h>
+ 
  #include <fm/fmd_snmp.h>
  #include <fm/fmd_msg.h>
  #include <fm/libfmevent.h>
+ 
  #include <net-snmp/net-snmp-config.h>
  #include <net-snmp/net-snmp-includes.h>
  #include <net-snmp/agent/net-snmp-agent-includes.h>
+ 
+ #include <alloca.h>
  #include <errno.h>
+ #include <limits.h>
  #include <locale.h>
  #include <netdb.h>
+ #include <priv_utils.h>
  #include <signal.h>
  #include <stdlib.h>
+ #include <strings.h>
  #include <unistd.h>
  #include <zone.h>
+ 
  #include "libfmnotify.h"
  
  /*
   * Debug messages can be enabled by setting the debug property to true
   *
*** 49,69 ****
   * # svccfg -s svc:/system/fm/snmp-notify setprop config/debug=true
   */
  #define SVCNAME         "system/fm/snmp-notify"
  
  typedef struct ireport_trap {
          char *host;
          char *msgid;
          char *desc;
-         long long tstamp;
          char *fmri;
          uint32_t from_state;
          uint32_t to_state;
          char *reason;
          boolean_t is_stn_event;
  } ireport_trap_t;
  
  static nd_hdl_t *nhdl;
  static const char optstr[] = "dfR:";
  static const char SNMP_SUPPCONF[] = "fmd-trapgen";
  static char hostname[MAXHOSTNAMELEN + 1];
  
--- 57,89 ----
   * # svccfg -s svc:/system/fm/snmp-notify setprop config/debug=true
   */
  #define SVCNAME         "system/fm/snmp-notify"
  
  typedef struct ireport_trap {
+         long long tstamp;
          char *host;
          char *msgid;
+         char *severity;
          char *desc;
          char *fmri;
          uint32_t from_state;
          uint32_t to_state;
          char *reason;
          boolean_t is_stn_event;
  } ireport_trap_t;
  
+ typedef struct fmproblem_trap {
+         char *uuid;
+         char *host;
+         char *code;
+         char *type;
+         char *severity;
+         char *url;
+         char *descr;
+         char *fmri;
+ } fmproblem_trap_t;
+ 
  static nd_hdl_t *nhdl;
  static const char optstr[] = "dfR:";
  static const char SNMP_SUPPCONF[] = "fmd-trapgen";
  static char hostname[MAXHOSTNAMELEN + 1];
  
*** 163,172 ****
--- 183,194 ----
  
          static const oid sunIreportHostname_oid[] =
              { SUNIREPORTHOSTNAME_OID };
          static const oid sunIreportMsgid_oid[] =
              { SUNIREPORTMSGID_OID };
+         static const oid sunIreportSeverity_oid[] =
+             { SUNIREPORTSEVERITY_OID };
          static const oid sunIreportDescription_oid[] =
              { SUNIREPORTDESCRIPTION_OID };
          static const oid sunIreportTime_oid[] =
              { SUNIREPORTTIME_OID };
  
*** 179,190 ****
          static const oid sunIreportSmfTransitionReason_oid[] =
              { SUNIREPORTTRANSITIONREASON_OID };
          const size_t
              sunIreport_base_len = OID_LENGTH(sunIreportHostname_oid);
  
          size_t var_len = sunIreport_base_len + 1;
!         oid var_name[MAX_OID_LEN];
  
          netsnmp_variable_list *notification_vars = NULL;
  
          size_t dt_len;
          uchar_t dt[11], *tdt;
--- 201,213 ----
          static const oid sunIreportSmfTransitionReason_oid[] =
              { SUNIREPORTTRANSITIONREASON_OID };
          const size_t
              sunIreport_base_len = OID_LENGTH(sunIreportHostname_oid);
  
+         size_t oid_len = sunIreport_base_len * sizeof (oid);
          size_t var_len = sunIreport_base_len + 1;
!         oid var_name[MAX_OID_LEN] = { 0 };
  
          netsnmp_variable_list *notification_vars = NULL;
  
          size_t dt_len;
          uchar_t dt[11], *tdt;
*** 202,258 ****
                  nd_error(nhdl, "var_len %d > MAX_OID_LEN %d\n", var_len,
                      MAX_OID_LEN);
                  return;
          }
  
!         (void) memcpy(var_name, sunIreportHostname_oid, sunIreport_base_len *
!             sizeof (oid));
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             sunIreport_base_len + 1, ASN_OCTET_STR, (uchar_t *)t->host,
!             strlen(t->host));
  
!         (void) memcpy(var_name, sunIreportMsgid_oid,
!             sunIreport_base_len * sizeof (oid));
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             sunIreport_base_len + 1, ASN_OCTET_STR, (uchar_t *)t->msgid,
!             strlen(t->msgid));
  
!         (void) memcpy(var_name, sunIreportDescription_oid,
!             sunIreport_base_len * sizeof (oid));
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             sunIreport_base_len + 1, ASN_OCTET_STR, (uchar_t *)t->desc,
!             strlen(t->desc));
  
!         (void) memcpy(var_name, sunIreportTime_oid, sunIreport_base_len *
!             sizeof (oid));
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             sunIreport_base_len + 1, ASN_OCTET_STR, dt, dt_len);
  
          if (t->is_stn_event) {
!                 (void) memcpy(var_name, sunIreportSmfFmri_oid,
!                     sunIreport_base_len * sizeof (oid));
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     sunIreport_base_len + 1, ASN_OCTET_STR, (uchar_t *)t->fmri,
                      strlen(t->fmri));
  
!                 (void) memcpy(var_name, sunIreportSmfFromState_oid,
!                     sunIreport_base_len * sizeof (oid));
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     sunIreport_base_len + 1, ASN_INTEGER,
!                     (uchar_t *)&t->from_state, sizeof (uint32_t));
  
!                 (void) memcpy(var_name, sunIreportSmfToState_oid,
!                     sunIreport_base_len * sizeof (oid));
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     sunIreport_base_len + 1, ASN_INTEGER,
!                     (uchar_t *)&t->to_state, sizeof (uint32_t));
  
                  (void) memcpy(var_name, sunIreportSmfTransitionReason_oid,
!                     sunIreport_base_len * sizeof (oid));
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     sunIreport_base_len + 1, ASN_OCTET_STR,
!                     (uchar_t *)t->reason, strlen(t->reason));
          }
  
          /*
           * This function is capable of sending both v1 and v2/v3 traps.
           * Which is sent to a specific destination is determined by the
--- 225,276 ----
                  nd_error(nhdl, "var_len %d > MAX_OID_LEN %d\n", var_len,
                      MAX_OID_LEN);
                  return;
          }
  
!         (void) memcpy(var_name, sunIreportHostname_oid, oid_len);
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             var_len, ASN_OCTET_STR, (uchar_t *)t->host, strlen(t->host));
  
!         (void) memcpy(var_name, sunIreportMsgid_oid, oid_len);
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             var_len, ASN_OCTET_STR, (uchar_t *)t->msgid, strlen(t->msgid));
  
!         (void) memcpy(var_name, sunIreportSeverity_oid, oid_len);
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             var_len, ASN_OCTET_STR, (uchar_t *)t->severity,
!             strlen(t->severity));
  
!         (void) memcpy(var_name, sunIreportDescription_oid, oid_len);
          (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!             var_len, ASN_OCTET_STR, (uchar_t *)t->desc, strlen(t->desc));
  
+         (void) memcpy(var_name, sunIreportTime_oid, oid_len);
+         (void) snmp_varlist_add_variable(¬ification_vars, var_name,
+             var_len, ASN_OCTET_STR, dt, dt_len);
+ 
          if (t->is_stn_event) {
!                 (void) memcpy(var_name, sunIreportSmfFmri_oid, oid_len);
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     var_len, ASN_OCTET_STR, (uchar_t *)t->fmri,
                      strlen(t->fmri));
  
!                 (void) memcpy(var_name, sunIreportSmfFromState_oid, oid_len);
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     var_len, ASN_INTEGER, (uchar_t *)&t->from_state,
!                     sizeof (uint32_t));
  
!                 (void) memcpy(var_name, sunIreportSmfToState_oid, oid_len);
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     var_len, ASN_INTEGER, (uchar_t *)&t->to_state,
!                     sizeof (uint32_t));
  
                  (void) memcpy(var_name, sunIreportSmfTransitionReason_oid,
!                     oid_len);
                  (void) snmp_varlist_add_variable(¬ification_vars, var_name,
!                     var_len, ASN_OCTET_STR, (uchar_t *)t->reason,
!                     strlen(t->reason));
          }
  
          /*
           * This function is capable of sending both v1 and v2/v3 traps.
           * Which is sent to a specific destination is determined by the
*** 263,292 ****
              (oid *)sunIreportTrap_oid, sunIreportTrap_len - 2,
              notification_vars);
          nd_debug(nhdl, "Sent SNMP trap for %s", t->msgid);
  
          snmp_free_varbind(notification_vars);
- 
  }
  
- /*ARGSUSED*/
  static void
! send_fm_trap(const char *uuid, const char *code, const char *url)
  {
          static const oid sunFmProblemTrap_oid[] = { SUNFMPROBLEMTRAP_OID };
          const size_t sunFmProblemTrap_len = OID_LENGTH(sunFmProblemTrap_oid);
  
          static const oid sunFmProblemUUID_oid[] =
              { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_UUID };
          static const oid sunFmProblemCode_oid[] =
              { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_CODE };
          static const oid sunFmProblemURL_oid[] =
              { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_URL };
  
          const size_t sunFmProblem_base_len = OID_LENGTH(sunFmProblemUUID_oid);
  
!         size_t uuid_len = strlen(uuid);
          size_t var_len = sunFmProblem_base_len + 1 + uuid_len;
          oid var_name[MAX_OID_LEN];
  
          netsnmp_variable_list *notification_vars = NULL;
  
--- 281,319 ----
              (oid *)sunIreportTrap_oid, sunIreportTrap_len - 2,
              notification_vars);
          nd_debug(nhdl, "Sent SNMP trap for %s", t->msgid);
  
          snmp_free_varbind(notification_vars);
  }
  
  static void
! send_fm_trap(fmproblem_trap_t *t)
  {
          static const oid sunFmProblemTrap_oid[] = { SUNFMPROBLEMTRAP_OID };
          const size_t sunFmProblemTrap_len = OID_LENGTH(sunFmProblemTrap_oid);
  
          static const oid sunFmProblemUUID_oid[] =
              { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_UUID };
+         static const oid sunFmProblemHostname_oid[] =
+             { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_HOSTNAME };
          static const oid sunFmProblemCode_oid[] =
              { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_CODE };
+         static const oid sunFmProblemType_oid[] =
+             { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_TYPE };
+         static const oid sunFmProblemSeverity_oid[] =
+             { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_SEVERITY };
          static const oid sunFmProblemURL_oid[] =
              { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_URL };
+         static const oid sunFmProblemDescr_oid[] =
+             { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_DESC };
+         static const oid sunFmProblemFMRI_oid[] =
+             { SUNFMPROBLEMTABLE_OID, 1, SUNFMPROBLEM_COL_FMRI };
  
          const size_t sunFmProblem_base_len = OID_LENGTH(sunFmProblemUUID_oid);
  
!         size_t oid_len = sunFmProblem_base_len * sizeof (oid);
!         size_t uuid_len = strlen(t->uuid);
          size_t var_len = sunFmProblem_base_len + 1 + uuid_len;
          oid var_name[MAX_OID_LEN];
  
          netsnmp_variable_list *notification_vars = NULL;
  
*** 308,350 ****
          if (var_len > MAX_OID_LEN)
                  return;
  
          var_name[sunFmProblem_base_len] = (oid)uuid_len;
          for (int i = 0; i < uuid_len; i++)
!                 var_name[i + sunFmProblem_base_len + 1] = (oid)uuid[i];
  
          /*
           * Ordinarily, we would need to add the OID of the trap itself
           * to the head of the variable list; this is required by SNMP v2.
           * However, send_enterprise_trap_vars does this for us as a part
           * of converting between v1 and v2 traps, so we skip directly to
           * the objects we're sending.
           */
  
!         (void) memcpy(var_name, sunFmProblemUUID_oid,
!             sunFmProblem_base_len * sizeof (oid));
          (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
!             ASN_OCTET_STR, (uchar_t *)uuid, strlen(uuid));
!         (void) memcpy(var_name, sunFmProblemCode_oid,
!             sunFmProblem_base_len * sizeof (oid));
          (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
!             ASN_OCTET_STR, (uchar_t *)code, strlen(code));
!         (void) memcpy(var_name, sunFmProblemURL_oid,
!             sunFmProblem_base_len * sizeof (oid));
          (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
!             ASN_OCTET_STR, (uchar_t *)url, strlen(url));
  
          /*
           * This function is capable of sending both v1 and v2/v3 traps.
           * Which is sent to a specific destination is determined by the
           * configuration file(s).
           */
          send_enterprise_trap_vars(SNMP_TRAP_ENTERPRISESPECIFIC,
              sunFmProblemTrap_oid[sunFmProblemTrap_len - 1],
              (oid *)sunFmProblemTrap_oid, sunFmProblemTrap_len - 2,
              notification_vars);
!         nd_debug(nhdl, "Sent SNMP trap for %s", code);
  
          snmp_free_varbind(notification_vars);
  }
  
  /*
--- 335,399 ----
          if (var_len > MAX_OID_LEN)
                  return;
  
          var_name[sunFmProblem_base_len] = (oid)uuid_len;
          for (int i = 0; i < uuid_len; i++)
!                 var_name[i + sunFmProblem_base_len + 1] = (oid)t->uuid[i];
  
          /*
           * Ordinarily, we would need to add the OID of the trap itself
           * to the head of the variable list; this is required by SNMP v2.
           * However, send_enterprise_trap_vars does this for us as a part
           * of converting between v1 and v2 traps, so we skip directly to
           * the objects we're sending.
           */
  
!         (void) memcpy(var_name, sunFmProblemUUID_oid, oid_len);
          (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
!             ASN_OCTET_STR, (uchar_t *)t->uuid, strlen(t->uuid));
! 
!         (void) memcpy(var_name, sunFmProblemHostname_oid, oid_len);
          (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
!             ASN_OCTET_STR, (uchar_t *)t->host, strlen(t->host));
! 
!         (void) memcpy(var_name, sunFmProblemCode_oid, oid_len);
          (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
!             ASN_OCTET_STR, (uchar_t *)t->code, strlen(t->code));
  
+         (void) memcpy(var_name, sunFmProblemType_oid, oid_len);
+         (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
+             ASN_OCTET_STR, (uchar_t *)t->type, strlen(t->type));
+ 
+         (void) memcpy(var_name, sunFmProblemSeverity_oid, oid_len);
+         (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
+             ASN_OCTET_STR, (uchar_t *)t->severity, strlen(t->severity));
+ 
+         (void) memcpy(var_name, sunFmProblemURL_oid, oid_len);
+         (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
+             ASN_OCTET_STR, (uchar_t *)t->url, strlen(t->url));
+ 
+         (void) memcpy(var_name, sunFmProblemDescr_oid, oid_len);
+         (void) snmp_varlist_add_variable(¬ification_vars, var_name, var_len,
+             ASN_OCTET_STR, (uchar_t *)t->descr, strlen(t->descr));
+ 
+         if (strcmp(t->fmri, ND_UNKNOWN) != 0) {
+                 (void) memcpy(var_name, sunFmProblemFMRI_oid, oid_len);
+                 (void) snmp_varlist_add_variable(¬ification_vars, var_name,
+                     var_len, ASN_OCTET_STR, (uchar_t *)t->fmri,
+                     strlen(t->fmri));
+         }
+ 
          /*
           * This function is capable of sending both v1 and v2/v3 traps.
           * Which is sent to a specific destination is determined by the
           * configuration file(s).
           */
          send_enterprise_trap_vars(SNMP_TRAP_ENTERPRISESPECIFIC,
              sunFmProblemTrap_oid[sunFmProblemTrap_len - 1],
              (oid *)sunFmProblemTrap_oid, sunFmProblemTrap_len - 2,
              notification_vars);
!         nd_debug(nhdl, "Sent SNMP trap for %s", t->code);
  
          snmp_free_varbind(notification_vars);
  }
  
  /*
*** 353,363 ****
   *
   * offline(0), online(1), degraded(2), disabled(3), maintenance(4),
   * uninitialized(5)
   *
   * This function converts a string representation of an SMF service state
!  * to it's corresponding enum val.
   */
  static int
  state_to_val(char *statestr, uint32_t *stateval)
  {
          if (strcmp(statestr, "offline") == 0)
--- 402,412 ----
   *
   * offline(0), online(1), degraded(2), disabled(3), maintenance(4),
   * uninitialized(5)
   *
   * This function converts a string representation of an SMF service state
!  * to its corresponding enum val.
   */
  static int
  state_to_val(char *statestr, uint32_t *stateval)
  {
          if (strcmp(statestr, "offline") == 0)
*** 408,417 ****
--- 457,467 ----
          if (nd_get_event_info(nhdl, class, ev, &ev_info) != 0)
                  goto irpt_done;
  
          swtrap.host = hostname;
          swtrap.msgid = ev_info->ei_diagcode;
+         swtrap.severity = ev_info->ei_severity;
          swtrap.desc = ev_info->ei_descr;
          swtrap.tstamp = (time_t)fmev_time_sec(ev);
  
          if (strncmp(class, "ireport.os.smf", 14) == 0) {
                  swtrap.fmri = ev_info->ei_fmri;
*** 434,447 ****
  
  /*ARGSUSED*/
  static void
  list_cb(fmev_t ev, const char *class, nvlist_t *nvl, void *arg)
  {
-         char *uuid;
          uint8_t version;
          nd_ev_info_t *ev_info = NULL;
          nvlist_t **pref_nvl = NULL;
          uint_t npref;
          int ret;
          boolean_t domsg;
  
          nd_debug(nhdl, "Received event of class %s", class);
--- 484,497 ----
  
  /*ARGSUSED*/
  static void
  list_cb(fmev_t ev, const char *class, nvlist_t *nvl, void *arg)
  {
          uint8_t version;
          nd_ev_info_t *ev_info = NULL;
          nvlist_t **pref_nvl = NULL;
+         fmproblem_trap_t fmtrap;
          uint_t npref;
          int ret;
          boolean_t domsg;
  
          nd_debug(nhdl, "Received event of class %s", class);
*** 473,495 ****
              &domsg) == 0 && !domsg) {
                  nd_debug(nhdl, "Messaging suppressed for this event");
                  goto listcb_done;
          }
  
!         if (nvlist_lookup_uint8(ev_info->ei_payload, FM_VERSION, &version)
!             != 0 || version > FM_SUSPECT_VERSION) {
                  nd_error(nhdl, "invalid event version: %u", version);
                  goto listcb_done;
          }
  
!         (void) nvlist_lookup_string(ev_info->ei_payload, FM_SUSPECT_UUID,
!             &uuid);
  
!         if (strcmp(ev_info->ei_url, ND_UNKNOWN) != 0)
!                 send_fm_trap(uuid, ev_info->ei_diagcode, ev_info->ei_url);
!         else
!                 nd_error(nhdl, "failed to format url for %s", uuid);
  listcb_done:
          nd_free_nvlarray(pref_nvl, npref);
          if (ev_info)
                  nd_free_event_info(ev_info);
  }
--- 523,548 ----
              &domsg) == 0 && !domsg) {
                  nd_debug(nhdl, "Messaging suppressed for this event");
                  goto listcb_done;
          }
  
!         if (nvlist_lookup_uint8(ev_info->ei_payload, FM_VERSION,
!             &version) != 0 || version > FM_SUSPECT_VERSION) {
                  nd_error(nhdl, "invalid event version: %u", version);
                  goto listcb_done;
          }
  
!         fmtrap.uuid = ev_info->ei_uuid;
!         fmtrap.host = hostname;
!         fmtrap.code = ev_info->ei_diagcode;
!         fmtrap.type = ev_info->ei_type;
!         fmtrap.severity = ev_info->ei_severity;
!         fmtrap.url = ev_info->ei_url;
!         fmtrap.descr = ev_info->ei_descr;
!         fmtrap.fmri = ev_info->ei_fmri;
  
!         send_fm_trap(&fmtrap);
  listcb_done:
          nd_free_nvlarray(pref_nvl, npref);
          if (ev_info)
                  nd_free_event_info(ev_info);
  }