1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
  24  */
  25 
  26 
  27 /*
  28  * main() of idmapd(1M)
  29  */
  30 
  31 #include "idmapd.h"
  32 #include <atomic.h>
  33 #include <signal.h>
  34 #include <rpc/pmap_clnt.h> /* for pmap_unset */
  35 #include <string.h> /* strcmp */
  36 #include <unistd.h> /* setsid */
  37 #include <sys/types.h>
  38 #include <memory.h>
  39 #include <stropts.h>
  40 #include <netconfig.h>
  41 #include <sys/resource.h> /* rlimit */
  42 #include <rpcsvc/daemon_utils.h> /* DAEMON_UID and DAEMON_GID */
  43 #include <priv_utils.h> /* privileges */
  44 #include <locale.h>
  45 #include <sys/systeminfo.h>
  46 #include <errno.h>
  47 #include <sys/wait.h>
  48 #include <sys/time.h>
  49 #include <zone.h>
  50 #include <door.h>
  51 #include <port.h>
  52 #include <tsol/label.h>
  53 #include <sys/resource.h>
  54 #include <sys/sid.h>
  55 #include <sys/idmap.h>
  56 #include <pthread.h>
  57 #include <stdarg.h>
  58 #include <assert.h>
  59 #include <note.h>
  60 
  61 #define CBUFSIZ 26      /* ctime(3c) */
  62 
  63 static void     term_handler(int);
  64 static void     init_idmapd();
  65 static void     fini_idmapd();
  66 
  67 /* The DC Locator lives inside idmap (for now). */
  68 extern void     init_dc_locator(void);
  69 extern void     fini_dc_locator(void);
  70 
  71 idmapd_state_t  _idmapdstate;
  72 
  73 SVCXPRT *xprt = NULL;
  74 
  75 static int dfd = -1;            /* our door server fildes, for unregistration */
  76 static boolean_t degraded = B_FALSE;
  77 
  78 
  79 static uint32_t         num_threads = 0;
  80 static pthread_key_t    create_threads_key;
  81 static uint32_t         max_threads = 40;
  82 
  83 /*
  84  * Server door thread start routine.
  85  *
  86  * Set a TSD value to the door thread. This enables the destructor to
  87  * be called when this thread exits. Note that we need a non-NULL
  88  * value for this or the TSD destructor is not called.
  89  */
  90 /*ARGSUSED*/
  91 static void *
  92 idmapd_door_thread_start(void *arg)
  93 {
  94         static void *value = "NON-NULL TSD";
  95 
  96         /*
  97          * Disable cancellation to avoid memory leaks from not running
  98          * the thread cleanup code.
  99          */
 100         (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
 101         (void) pthread_setspecific(create_threads_key, value);
 102         (void) door_return(NULL, 0, NULL, 0);
 103 
 104         /* make lint happy */
 105         return (NULL);
 106 }
 107 
 108 /*
 109  * Server door threads creation
 110  */
 111 /*ARGSUSED*/
 112 static void
 113 idmapd_door_thread_create(door_info_t *dip)
 114 {
 115         int             num;
 116         pthread_t       thread_id;
 117 
 118         if ((num = atomic_inc_32_nv(&num_threads)) > max_threads) {
 119                 atomic_dec_32(&num_threads);
 120                 idmapdlog(LOG_DEBUG,
 121                     "thread creation refused - %d threads currently active",
 122                     num - 1);
 123                 return;
 124         }
 125         (void) pthread_create(&thread_id, NULL, idmapd_door_thread_start, NULL);
 126         idmapdlog(LOG_DEBUG,
 127             "created thread ID %d - %d threads currently active",
 128             thread_id, num);
 129 }
 130 
 131 /*
 132  * Server door thread cleanup
 133  */
 134 /*ARGSUSED*/
 135 static void
 136 idmapd_door_thread_cleanup(void *arg)
 137 {
 138         int num;
 139 
 140         /* set TSD to NULL so we don't loop infinitely */
 141         (void) pthread_setspecific(create_threads_key, NULL);
 142         num = atomic_dec_32_nv(&num_threads);
 143         idmapdlog(LOG_DEBUG,
 144             "exiting thread ID %d - %d threads currently active",
 145             pthread_self(), num);
 146 }
 147 
 148 /*
 149  * This is needed for mech_krb5 -- we run as daemon, yes, but we want
 150  * mech_krb5 to think we're root so it can get host/nodename.fqdn
 151  * tickets for us so we can authenticate to AD as the machine account
 152  * that we are.  For more details look at the entry point in mech_krb5
 153  * corresponding to gss_init_sec_context().
 154  *
 155  * As a side effect of faking our effective UID to mech_krb5 we will use
 156  * root's default ccache (/tmp/krb5cc_0).  But if that's created by
 157  * another process then we won't have access to it: we run as daemon and
 158  * keep PRIV_FILE_DAC_READ, which is insufficient to share the ccache
 159  * with others.  We putenv("KRB5CCNAME=/var/run/idmap/ccache") in main()
 160  * to avoid this issue; see main().
 161  *
 162  * Someday we'll have gss/mech_krb5 extensions for acquiring initiator
 163  * creds with keytabs/raw keys, and someday we'll have extensions to
 164  * libsasl to specify creds/name to use on the initiator side, and
 165  * someday we'll have extensions to libldap to pass those through to
 166  * libsasl.  Until then this interposer will have to do.
 167  *
 168  * Also, we have to tell lint to shut up: it thinks app_krb5_user_uid()
 169  * is defined but not used.
 170  */
 171 /*LINTLIBRARY*/
 172 uid_t
 173 app_krb5_user_uid(void)
 174 {
 175         return (0);
 176 }
 177 
 178 /*ARGSUSED*/
 179 static void
 180 term_handler(int sig)
 181 {
 182         idmapdlog(LOG_INFO, "Terminating.");
 183         fini_dc_locator();
 184         fini_idmapd();
 185         _exit(0);
 186 }
 187 
 188 /*ARGSUSED*/
 189 static void
 190 usr1_handler(int sig)
 191 {
 192         NOTE(ARGUNUSED(sig))
 193         print_idmapdstate();
 194 }
 195 
 196 static int pipe_fd = -1;
 197 
 198 static void
 199 daemonize_ready(void)
 200 {
 201         char data = '\0';
 202         /*
 203          * wake the parent
 204          */
 205         (void) write(pipe_fd, &data, 1);
 206         (void) close(pipe_fd);
 207 }
 208 
 209 static int
 210 daemonize_start(void)
 211 {
 212         char    data;
 213         int     status;
 214         int     devnull;
 215         int     filedes[2];
 216         pid_t   pid;
 217 
 218         (void) sigset(SIGPIPE, SIG_IGN);
 219         devnull = open("/dev/null", O_RDONLY);
 220         if (devnull < 0)
 221                 return (-1);
 222         (void) dup2(devnull, 0);
 223         (void) dup2(2, 1);      /* stderr only */
 224         if (pipe(filedes) < 0)
 225                 return (-1);
 226         if ((pid = fork1()) < 0)
 227                 return (-1);
 228         if (pid != 0) {
 229                 /*
 230                  * parent
 231                  */
 232                 (void) close(filedes[1]);
 233                 if (read(filedes[0], &data, 1) == 1) {
 234                         /* presume success */
 235                         _exit(0);
 236                 }
 237                 status = -1;
 238                 (void) wait4(pid, &status, 0, NULL);
 239                 if (WIFEXITED(status))
 240                         _exit(WEXITSTATUS(status));
 241                 else
 242                         _exit(-1);
 243         }
 244 
 245         /*
 246          * child
 247          */
 248         pipe_fd = filedes[1];
 249         (void) close(filedes[0]);
 250         (void) setsid();
 251         (void) umask(0077);
 252         openlog("idmap", LOG_PID, LOG_DAEMON);
 253 
 254         return (0);
 255 }
 256 
 257 
 258 int
 259 main(int argc, char **argv)
 260 {
 261         int c;
 262         struct rlimit rl;
 263 
 264         if (rwlock_init(&_idmapdstate.rwlk_cfg, USYNC_THREAD, NULL) != 0)
 265                 return (-1);
 266         if (mutex_init(&_idmapdstate.addisc_lk, USYNC_THREAD, NULL) != 0)
 267                 return (-1);
 268         if (cond_init(&_idmapdstate.addisc_cv, USYNC_THREAD, NULL) != 0)
 269                 return (-1);
 270 
 271         _idmapdstate.daemon_mode = TRUE;
 272         while ((c = getopt(argc, argv, "d")) != -1) {
 273                 switch (c) {
 274                         case 'd':
 275                                 _idmapdstate.daemon_mode = FALSE;
 276                                 break;
 277                         default:
 278                                 (void) fprintf(stderr,
 279                                     "Usage: /usr/lib/idmapd [-d]\n");
 280                                 return (SMF_EXIT_ERR_CONFIG);
 281                 }
 282         }
 283 
 284         /* set locale and domain for internationalization */
 285         (void) setlocale(LC_ALL, "");
 286         (void) textdomain(TEXT_DOMAIN);
 287 
 288         idmap_set_logger(idmapdlog);
 289         adutils_set_logger(idmapdlog);
 290 
 291         if (is_system_labeled() && getzoneid() != GLOBAL_ZONEID) {
 292                 idmapdlog(LOG_ERR,
 293                     "with Trusted Extensions idmapd runs only in the "
 294                     "global zone");
 295                 exit(1);
 296         }
 297 
 298         /*
 299          * Raise the fd limit to max
 300          */
 301         if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
 302                 idmapdlog(LOG_ERR, "getrlimit failed");
 303         } else if (rl.rlim_cur < rl.rlim_max) {
 304                 rl.rlim_cur = rl.rlim_max;
 305                 if (setrlimit(RLIMIT_NOFILE, &rl) != 0)
 306                         idmapdlog(LOG_ERR,
 307                             "Unable to raise RLIMIT_NOFILE to %d",
 308                             rl.rlim_cur);
 309         }
 310 
 311         (void) mutex_init(&_svcstate_lock, USYNC_THREAD, NULL);
 312 
 313         if (_idmapdstate.daemon_mode == TRUE) {
 314                 if (daemonize_start() < 0) {
 315                         idmapdlog(LOG_ERR, "unable to daemonize");
 316                         exit(-1);
 317                 }
 318         } else
 319                 (void) umask(0077);
 320 
 321         idmap_init_tsd_key();
 322 
 323         init_idmapd();
 324         init_dc_locator();
 325 
 326         /* signal handlers that should run only after we're initialized */
 327         (void) sigset(SIGTERM, term_handler);
 328         (void) sigset(SIGUSR1, usr1_handler);
 329         (void) sigset(SIGHUP, idmap_cfg_hup_handler);
 330 
 331         if (__init_daemon_priv(PU_RESETGROUPS|PU_CLEARLIMITSET,
 332             DAEMON_UID, DAEMON_GID,
 333             PRIV_PROC_AUDIT, PRIV_FILE_DAC_READ,
 334             (char *)NULL) == -1) {
 335                 idmapdlog(LOG_ERR, "unable to drop privileges");
 336                 exit(1);
 337         }
 338 
 339         __fini_daemon_priv(PRIV_PROC_FORK, PRIV_PROC_EXEC, PRIV_PROC_SESSION,
 340             PRIV_FILE_LINK_ANY, PRIV_PROC_INFO, (char *)NULL);
 341 
 342         if (_idmapdstate.daemon_mode == TRUE)
 343                 daemonize_ready();
 344 
 345         /* With doors RPC this just wastes this thread, oh well */
 346         svc_run();
 347         return (0);
 348 }
 349 
 350 static void
 351 init_idmapd()
 352 {
 353         int     error;
 354         int     connmaxrec = IDMAP_MAX_DOOR_RPC;
 355 
 356 
 357         /* create directories as root and chown to daemon uid */
 358         if (create_directory(IDMAP_DBDIR, DAEMON_UID, DAEMON_GID) < 0)
 359                 exit(1);
 360         if (create_directory(IDMAP_CACHEDIR, DAEMON_UID, DAEMON_GID) < 0)
 361                 exit(1);
 362 
 363         /*
 364          * Set KRB5CCNAME in the environment.  See app_krb5_user_uid()
 365          * for more details.  We blow away the existing one, if there is
 366          * one.
 367          */
 368         (void) unlink(IDMAP_CACHEDIR "/ccache");
 369         (void) putenv("KRB5CCNAME=" IDMAP_CACHEDIR "/ccache");
 370         (void) putenv("MS_INTEROP=1");
 371 
 372         if (sysinfo(SI_HOSTNAME, _idmapdstate.hostname,
 373             sizeof (_idmapdstate.hostname)) == -1) {
 374                 error = errno;
 375                 idmapdlog(LOG_ERR, "unable to determine hostname, error: %d",
 376                     error);
 377                 exit(1);
 378         }
 379 
 380         if ((error = init_mapping_system()) < 0) {
 381                 idmapdlog(LOG_ERR, "unable to initialize mapping system");
 382                 exit(error < -2 ? SMF_EXIT_ERR_CONFIG : 1);
 383         }
 384 
 385         /*
 386          * This means max_threads can't be updated without restarting idmap.
 387          */
 388         RDLOCK_CONFIG();
 389         max_threads = _idmapdstate.cfg->pgcfg.max_threads;
 390         UNLOCK_CONFIG();
 391 
 392         (void) door_server_create(idmapd_door_thread_create);
 393         if ((error = pthread_key_create(&create_threads_key,
 394             idmapd_door_thread_cleanup)) != 0) {
 395                 idmapdlog(LOG_ERR, "unable to create threads key (%s)",
 396                     strerror(error));
 397                 goto errout;
 398         }
 399 
 400         xprt = svc_door_create(idmap_prog_1, IDMAP_PROG, IDMAP_V1, connmaxrec);
 401         if (xprt == NULL) {
 402                 idmapdlog(LOG_ERR, "unable to create door RPC service");
 403                 goto errout;
 404         }
 405 
 406         if (!svc_control(xprt, SVCSET_CONNMAXREC, &connmaxrec)) {
 407                 idmapdlog(LOG_ERR, "unable to limit RPC request size");
 408                 goto errout;
 409         }
 410 
 411         dfd = xprt->xp_fd;
 412 
 413         if (dfd == -1) {
 414                 idmapdlog(LOG_ERR, "unable to register door");
 415                 goto errout;
 416         }
 417         if ((error = __idmap_reg(dfd)) != 0) {
 418                 idmapdlog(LOG_ERR, "unable to register door (%s)",
 419                     strerror(errno));
 420                 goto errout;
 421         }
 422 
 423         if ((error = allocids(_idmapdstate.new_eph_db,
 424             8192, &_idmapdstate.next_uid,
 425             8192, &_idmapdstate.next_gid)) != 0) {
 426                 idmapdlog(LOG_ERR, "unable to allocate ephemeral IDs (%s)",
 427                     strerror(errno));
 428                 _idmapdstate.next_uid = IDMAP_SENTINEL_PID;
 429                 _idmapdstate.limit_uid = IDMAP_SENTINEL_PID;
 430                 _idmapdstate.next_gid = IDMAP_SENTINEL_PID;
 431                 _idmapdstate.limit_gid = IDMAP_SENTINEL_PID;
 432         } else {
 433                 _idmapdstate.limit_uid = _idmapdstate.next_uid + 8192;
 434                 _idmapdstate.limit_gid = _idmapdstate.next_gid + 8192;
 435         }
 436 
 437         if (DBG(CONFIG, 1))
 438                 print_idmapdstate();
 439 
 440         return;
 441 
 442 errout:
 443         fini_idmapd();
 444         exit(1);
 445 }
 446 
 447 static void
 448 fini_idmapd()
 449 {
 450         (void) __idmap_unreg(dfd);
 451         fini_mapping_system();
 452         if (xprt != NULL)
 453                 svc_destroy(xprt);
 454 }
 455 
 456 static
 457 const char *
 458 get_fmri(void)
 459 {
 460         static char *fmri = NULL;
 461         static char buf[60];
 462         char *s;
 463 
 464         membar_consumer();
 465         s = fmri;
 466         if (s != NULL && *s == '\0')
 467                 return (NULL);
 468         else if (s != NULL)
 469                 return (s);
 470 
 471         if ((s = getenv("SMF_FMRI")) == NULL || strlen(s) >= sizeof (buf))
 472                 buf[0] = '\0';
 473         else
 474                 (void) strlcpy(buf, s, sizeof (buf));
 475 
 476         membar_producer();
 477         fmri = buf;
 478 
 479         return (get_fmri());
 480 }
 481 
 482 /*
 483  * Wrappers for smf_degrade/restore_instance()
 484  *
 485  * smf_restore_instance() is too heavy duty to be calling every time we
 486  * have a successful AD name<->SID lookup.
 487  */
 488 void
 489 degrade_svc(int poke_discovery, const char *reason)
 490 {
 491         const char *fmri;
 492 
 493         membar_consumer();
 494         if (degraded)
 495                 return;
 496 
 497         idmapdlog(LOG_ERR, "Degraded operation (%s).", reason);
 498 
 499         membar_producer();
 500         degraded = B_TRUE;
 501 
 502         if ((fmri = get_fmri()) != NULL)
 503                 (void) smf_degrade_instance(fmri, 0);
 504 
 505         /*
 506          * If the config update thread is in a state where auto-discovery could
 507          * be re-tried, then this will make it try it -- a sort of auto-refresh.
 508          */
 509         if (poke_discovery)
 510                 idmap_cfg_poke_updates();
 511 }
 512 
 513 void
 514 restore_svc(void)
 515 {
 516         const char *fmri;
 517 
 518         membar_consumer();
 519         if (!degraded)
 520                 return;
 521 
 522         if ((fmri = get_fmri()) == NULL)
 523                 (void) smf_restore_instance(fmri);
 524 
 525         membar_producer();
 526         degraded = B_FALSE;
 527 
 528         idmapdlog(LOG_NOTICE, "Normal operation restored");
 529 }
 530 
 531 
 532 /* printflike */
 533 void
 534 idmapdlog(int pri, const char *format, ...) {
 535         static time_t prev_ts;
 536         va_list args;
 537         char cbuf[CBUFSIZ];
 538         time_t ts;
 539 
 540         ts = time(NULL);
 541         if (prev_ts != ts) {
 542                 prev_ts = ts;
 543                 /* NB: cbuf has \n */
 544                 (void) fprintf(stderr, "@ %s",
 545                     ctime_r(&ts, cbuf, sizeof (cbuf)));
 546         }
 547 
 548         va_start(args, format);
 549         (void) vfprintf(stderr, format, args);
 550         (void) fprintf(stderr, "\n");
 551         va_end(args);
 552 
 553         /*
 554          * We don't want to fill up the logs with useless messages when
 555          * we're degraded, but we still want to log.
 556          */
 557         if (degraded)
 558                 pri = LOG_DEBUG;
 559 
 560         va_start(args, format);
 561         vsyslog(pri, format, args);
 562         va_end(args);
 563 }
 564 
 565 static void
 566 trace_str(nvlist_t *entry, char *n1, char *n2, char *str)
 567 {
 568         char name[IDMAP_TRACE_NAME_MAX+1];      /* Max used is only about 11 */
 569 
 570         (void) strlcpy(name, n1, sizeof (name));
 571         if (n2 != NULL)
 572                 (void) strlcat(name, n2, sizeof (name));
 573 
 574         (void) nvlist_add_string(entry, name, str);
 575 }
 576 
 577 static void
 578 trace_int(nvlist_t *entry, char *n1, char *n2, int64_t i)
 579 {
 580         char name[IDMAP_TRACE_NAME_MAX+1];      /* Max used is only about 11 */
 581 
 582         (void) strlcpy(name, n1, sizeof (name));
 583         if (n2 != NULL)
 584                 (void) strlcat(name, n2, sizeof (name));
 585 
 586         (void) nvlist_add_int64(entry, name, i);
 587 }
 588 
 589 static void
 590 trace_sid(nvlist_t *entry, char *n1, char *n2, idmap_sid *sid)
 591 {
 592         char *str;
 593 
 594         (void) asprintf(&str, "%s-%u", sid->prefix, sid->rid);
 595         if (str == NULL)
 596                 return;
 597 
 598         trace_str(entry, n1, n2, str);
 599         free(str);
 600 }
 601 
 602 static void
 603 trace_id(nvlist_t *entry, char *fromto, idmap_id *id, char *name, char *domain)
 604 {
 605         trace_int(entry, fromto, IDMAP_TRACE_TYPE, (int64_t)id->idtype);
 606         if (IS_ID_SID(*id)) {
 607                 if (name != NULL) {
 608                         char *str;
 609 
 610                         (void) asprintf(&str, "%s%s%s", name,
 611                             domain == NULL ? "" : "@",
 612                             domain == NULL ? "" : domain);
 613                         if (str != NULL) {
 614                                 trace_str(entry, fromto, IDMAP_TRACE_NAME, str);
 615                                 free(str);
 616                         }
 617                 }
 618                 if (id->idmap_id_u.sid.prefix != NULL) {
 619                         trace_sid(entry, fromto, IDMAP_TRACE_SID,
 620                             &id->idmap_id_u.sid);
 621                 }
 622         } else if (IS_ID_POSIX(*id)) {
 623                 if (name != NULL)
 624                         trace_str(entry, fromto, IDMAP_TRACE_NAME, name);
 625                 if (id->idmap_id_u.uid != IDMAP_SENTINEL_PID) {
 626                         trace_int(entry, fromto, IDMAP_TRACE_UNIXID,
 627                             (int64_t)id->idmap_id_u.uid);
 628                 }
 629         }
 630 }
 631 
 632 /*
 633  * Record a trace event.  TRACE() has already decided whether or not
 634  * tracing is required; what we do here is collect the data and send it
 635  * to its destination - to the trace log in the response, if
 636  * IDMAP_REQ_FLG_TRACE is set, and to the SMF service log, if debug/mapping
 637  * is greater than zero.
 638  */
 639 int
 640 trace(idmap_mapping *req, idmap_id_res *res, char *fmt, ...)
 641 {
 642         va_list va;
 643         char *buf;
 644         int err;
 645         nvlist_t *entry;
 646 
 647         assert(req != NULL);
 648         assert(res != NULL);
 649 
 650         err = nvlist_alloc(&entry, NV_UNIQUE_NAME, 0);
 651         if (err != 0) {
 652                 (void) fprintf(stderr, "trace nvlist_alloc(entry):  %s\n",
 653                     strerror(err));
 654                 return (0);
 655         }
 656 
 657         trace_id(entry, "from", &req->id1, req->id1name, req->id1domain);
 658         trace_id(entry, "to", &res->id, req->id2name, req->id2domain);
 659 
 660         if (IDMAP_ERROR(res->retcode)) {
 661                 trace_int(entry, IDMAP_TRACE_ERROR, NULL,
 662                     (int64_t)res->retcode);
 663         }
 664 
 665         va_start(va, fmt);
 666         (void) vasprintf(&buf, fmt, va);
 667         va_end(va);
 668         if (buf != NULL) {
 669                 trace_str(entry, IDMAP_TRACE_MESSAGE, NULL, buf);
 670                 free(buf);
 671         }
 672 
 673         if (DBG(MAPPING, 1))
 674                 idmap_trace_print_1(stderr, "", entry);
 675 
 676         if (req->flag & IDMAP_REQ_FLG_TRACE) {
 677                 /* Lazily allocate the trace list */
 678                 if (res->info.trace == NULL) {
 679                         err = nvlist_alloc(&res->info.trace, 0, 0);
 680                         if (err != 0) {
 681                                 res->info.trace = NULL; /* just in case */
 682                                 (void) fprintf(stderr,
 683                                     "trace nvlist_alloc(trace):  %s\n",
 684                                     strerror(err));
 685                                 nvlist_free(entry);
 686                                 return (0);
 687                         }
 688                 }
 689                 (void) nvlist_add_nvlist(res->info.trace, "", entry);
 690                 /* Note that entry is copied, so we must still free our copy */
 691         }
 692 
 693         nvlist_free(entry);
 694 
 695         return (0);
 696 }