1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #pragma ident   "%Z%%M% %I%     %E% SMI"
  27 
  28 #include <string.h>
  29 #include <errno.h>
  30 #include <syslog.h>
  31 #include <procfs.h>
  32 #include <unistd.h>
  33 #include <fcntl.h>
  34 #include <libintl.h>
  35 #include <atomic.h>
  36 #include <pthread.h>
  37 #include <sys/mman.h>
  38 #include <time.h>
  39 #include "solaris-int.h"
  40 #include "ns_connmgmt.h"
  41 #include "ns_cache_door.h"
  42 #include "ns_internal.h"
  43 
  44 /*
  45  * Access (reference, shutdown, or reload) the current connection
  46  * management control structure conn_mgmt_t.
  47  */
  48 #define NS_CONN_MGMT_OP_REF             1
  49 #define NS_CONN_MGMT_OP_SHUTDOWN        2
  50 #define NS_CONN_MGMT_OP_RELOAD_CONFIG   3
  51 #define NS_CONN_MGMT_OP_NEW_CONFIG      4
  52 #define NS_CONN_MGMT_OP_LIB_INIT        5
  53 
  54 static ns_conn_mgmt_t *access_conn_mgmt(int);
  55 static ns_conn_mgmt_t *release_conn_mgmt(ns_conn_mgmt_t *, boolean_t);
  56 static int close_conn_mt(ns_conn_mt_t *, int, ns_ldap_error_t **,
  57         ns_conn_user_t *);
  58 static int close_conn_mt_when_nouser(ns_conn_mt_t *cm);
  59 void shutdown_all_conn_mt(ns_conn_mgmt_t *cmg);
  60 static int conn_signal(ns_conn_mt_t *);
  61 static int conn_wait(ns_conn_mt_t *, ns_conn_user_t *);
  62 static void close_conn_mt_by_procchg(ns_conn_mt_t *cm, int rc, char *errmsg);
  63 static ns_conn_mgmt_t *proc_server_change(ns_server_status_change_t *chg,
  64         ns_conn_mgmt_t  *cmg);
  65 static void get_preferred_servers(boolean_t, boolean_t, ns_conn_mgmt_t *);
  66 static void start_thread();
  67 
  68 static ns_conn_mgmt_t   *ns_connmgmt = NULL;
  69 static ns_conn_mgmt_t   *ns_connmgmt_parent = NULL;
  70 static mutex_t          ns_connmgmt_lock = DEFAULTMUTEX;
  71 static boolean_t        ns_connmgmt_shutting_down = B_FALSE;
  72 
  73 #define NS_CONN_MSG_NO_CONN_MGMT gettext( \
  74         "libsldap: unable to allocate the connection management control")
  75 #define NS_CONN_MSG_NO_MTC_KEY gettext( \
  76         "libsldap: unable to allocate the TSD key for per-thread ldap error")
  77 #define NS_CONN_MSG_NO_CMG_KEY gettext( \
  78         "libsldap: unable to allocate the TSD key for connection management")
  79 #define NS_CONN_MSG_SHUTDOWN gettext("libsldap: library is being unloaded")
  80 #define NS_CONN_MSG_RELOADED gettext( \
  81         "libsldap: configuration has been reloaded")
  82 #define NS_CONN_MSG_SHUTDOWN_RELOADED gettext( \
  83         "libsldap: library unloaded or configuration has been reloaded")
  84 #define NS_CONN_MSG_BAD_CACHEMGR_DATA gettext( \
  85         "libsldap: received incorrect data from ldap_cachemgr")
  86 #define NS_CONN_MSG_MEMORY_ERROR gettext( \
  87         "libsldap: unable to allocate memory")
  88 #define NS_CONN_MSG_NO_PROCCHG_THREAD gettext( \
  89         "libsldap: unable to start the server monitor thread (%s)")
  90 #define NS_CONN_MSG_DOWN_FROM_CACHEMGR gettext( \
  91         "libsldap: server down reported by ldap_cachemgr")
  92 
  93 static int ns_conn_free = 1;
  94 #define NS_CONN_UNLOCK_AND_FREE(free, cm, cmg)  \
  95 { \
  96         (void) mutex_unlock(&(cm)->lock);        \
  97         if (free == 1)  \
  98                 cmg = free_conn_mt((cm), 1); \
  99         if (cmg != NULL) \
 100                 (void) mutex_unlock(&(cmg)->lock); \
 101 }
 102 
 103 #define NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, errp) \
 104 { \
 105         char *msg = NULL; \
 106         (void) mutex_lock(&(cmg)->lock); \
 107         if ((cmg)->shutting_down == B_TRUE) \
 108                 msg = NS_CONN_MSG_SHUTDOWN; \
 109         else if ((cmg)->cfg_reloaded == B_TRUE)  \
 110                 msg = NS_CONN_MSG_RELOADED; \
 111         if (msg != NULL) { \
 112                 (*errp) = __s_api_make_error(NS_LDAP_OP_FAILED, msg); \
 113                 (void) mutex_unlock(&(cmg)->lock); \
 114                 return (NS_LDAP_OP_FAILED); \
 115         } \
 116 }
 117 
 118 /*
 119  * TSD keys ns_mtckey and ns_cmgkey are for sharing ldap connections
 120  * and their associated connection management structure among
 121  * multiple threads. The pointers to the per-thread ldap error
 122  * information and the connection management structure are
 123  * saved in ns_mtckey and ns_cmgkey.
 124  */
 125 thread_key_t ns_mtckey = THR_ONCE_KEY;
 126 thread_key_t ns_cmgkey = THR_ONCE_KEY;
 127 
 128 /* Per thread LDAP error resides in thread-specific data (ns_mtckey) */
 129 struct ldap_error {
 130         int     le_errno;
 131         char    *le_matched;
 132         char    *le_errmsg;
 133 };
 134 
 135 /* NULL struct ldap_error */
 136 static struct ldap_error ldap_error_NULL = { LDAP_SUCCESS, NULL, NULL};
 137 
 138 /* destructor: free the ldap error data in the thread specific area */
 139 static void
 140 ns_mtckey_cleanup(void *key) {
 141         struct ldap_error *le = (struct ldap_error *)key;
 142 
 143         if (le == NULL)
 144                 return;
 145         if (le->le_matched != NULL) {
 146                 ldap_memfree(le->le_matched);
 147         }
 148         if (le->le_errmsg != NULL) {
 149                 ldap_memfree(le->le_errmsg);
 150         }
 151         free(le);
 152 }
 153 
 154 /* Free/detach the thread specific data structures */
 155 static void
 156 conn_tsd_free() {
 157         void    *tsd = NULL;
 158         int     rc;
 159 
 160         /* free the per-thread ldap error info */
 161         rc = thr_getspecific(ns_mtckey, &tsd);
 162         if (rc == 0 && tsd != NULL)
 163                 ns_mtckey_cleanup(tsd);
 164         (void) thr_setspecific(ns_mtckey, NULL);
 165 
 166         /* detach the connection management control */
 167         (void) thr_setspecific(ns_cmgkey, NULL);
 168 }
 169 
 170 /* per-thread callback function for allocating a mutex */
 171 static void *
 172 ns_mutex_alloc(void)
 173 {
 174         mutex_t *mutexp = NULL;
 175 
 176         if ((mutexp = malloc(sizeof (mutex_t))) != NULL) {
 177                 if (mutex_init(mutexp, USYNC_THREAD, NULL) != 0) {
 178                         free(mutexp);
 179                         mutexp = NULL;
 180                 }
 181         }
 182         return (mutexp);
 183 }
 184 
 185 /* per-thread callback function for freeing a mutex */
 186 static void
 187 ns_mutex_free(void *mutexp)
 188 {
 189         (void) mutex_destroy((mutex_t *)mutexp);
 190         free(mutexp);
 191 }
 192 
 193 /*
 194  * Function for setting up thread-specific data
 195  * where per thread LDAP error and the pointer
 196  * to the active connection management control
 197  * are stored.
 198  */
 199 static int
 200 conn_tsd_setup(ns_conn_mgmt_t *cmg)
 201 {
 202         void    *tsd;
 203         int     rc;
 204 
 205         rc = thr_setspecific(ns_cmgkey, cmg);
 206         if (rc != 0) /* must be ENOMEM */
 207                 return (-1);
 208 
 209         /* return success if the ns_mtckey TSD is already set */
 210         rc = thr_getspecific(ns_mtckey, &tsd);
 211         if (rc == 0 && tsd != NULL)
 212                 return (0);
 213 
 214         /* allocate and set the ns_mtckey TSD */
 215         tsd = (void *) calloc(1, sizeof (struct ldap_error));
 216         if (tsd == NULL)
 217                 return (-1);
 218         rc = thr_setspecific(ns_mtckey, tsd);
 219         if (rc != 0) { /* must be ENOMEM */
 220                 free(tsd);
 221                 return (-1);
 222         }
 223         return (0);
 224 }
 225 
 226 /* Callback function for setting the per thread LDAP error */
 227 /*ARGSUSED*/
 228 static void
 229 set_ld_error(int err, char *matched, char *errmsg, void *dummy)
 230 {
 231         struct ldap_error       *le;
 232         int                     eno;
 233 
 234         if ((eno = thr_getspecific(ns_mtckey, (void **)&le)) != 0) {
 235                 syslog(LOG_ERR, gettext(
 236                     "libsldap: set_ld_error: thr_getspecific failed (%s)."),
 237                     strerror(eno));
 238                 return;
 239         }
 240 
 241         /* play safe, do nothing if TSD pointer is NULL */
 242         if (le == NULL) {
 243                 syslog(LOG_INFO, gettext(
 244                     "libsldap: set_ld_error: TSD pointer is NULL."));
 245                 return;
 246         }
 247 
 248         le->le_errno = err;
 249 
 250         if (le->le_matched != NULL) {
 251                 ldap_memfree(le->le_matched);
 252                 le->le_matched = NULL;
 253         }
 254         le->le_matched = matched;
 255 
 256         if (le->le_errmsg != NULL) {
 257                 ldap_memfree(le->le_errmsg);
 258                 le->le_errmsg = NULL;
 259         }
 260         le->le_errmsg = errmsg;
 261 }
 262 
 263 /* check and allocate the thread-specific data for using a MT connection */
 264 static int
 265 conn_tsd_check(ns_conn_mgmt_t *cmg)
 266 {
 267         if (conn_tsd_setup(cmg) != 0)
 268                 return (NS_LDAP_MEMORY);
 269 
 270         return (NS_LDAP_SUCCESS);
 271 }
 272 
 273 /* Callback function for getting the per thread LDAP error */
 274 /*ARGSUSED*/
 275 static int
 276 get_ld_error(char **matched, char **errmsg, void *dummy)
 277 {
 278         struct ldap_error       *le;
 279         int                     eno;
 280 
 281         if ((eno = thr_getspecific(ns_mtckey, (void **)&le)) != 0) {
 282                 syslog(LOG_ERR, gettext(
 283                     "libsldap: get_ld_error: thr_getspecific failed (%s)"),
 284                     strerror(eno));
 285                 return (eno);
 286         }
 287 
 288         /* play safe, return NULL error data, if TSD pointer is NULL */
 289         if (le == NULL)
 290                 le = &ldap_error_NULL;
 291 
 292         if (matched != NULL) {
 293                 *matched = le->le_matched;
 294         }
 295         if (errmsg != NULL) {
 296                 *errmsg = le->le_errmsg;
 297         }
 298         return (le->le_errno);
 299 }
 300 
 301 /* Callback function for setting per thread errno */
 302 static void
 303 set_errno(int err)
 304 {
 305         errno = err;
 306 }
 307 
 308 /* Callback function for getting per thread errno */
 309 static int
 310 get_errno(void)
 311 {
 312         return (errno);
 313 }
 314 
 315 /* set up an ldap session 'ld' for sharing among multiple threads */
 316 static int
 317 setup_mt_conn(LDAP *ld)
 318 {
 319 
 320         struct ldap_thread_fns          tfns;
 321         struct ldap_extra_thread_fns    extrafns;
 322         int                             rc;
 323 
 324         /*
 325          * Set the function pointers for dealing with mutexes
 326          * and error information
 327          */
 328         (void) memset(&tfns, '\0', sizeof (struct ldap_thread_fns));
 329         tfns.ltf_mutex_alloc = (void *(*)(void)) ns_mutex_alloc;
 330         tfns.ltf_mutex_free = (void (*)(void *)) ns_mutex_free;
 331         tfns.ltf_mutex_lock = (int (*)(void *)) mutex_lock;
 332         tfns.ltf_mutex_unlock = (int (*)(void *)) mutex_unlock;
 333         tfns.ltf_get_errno = get_errno;
 334         tfns.ltf_set_errno = set_errno;
 335         tfns.ltf_get_lderrno = get_ld_error;
 336         tfns.ltf_set_lderrno = set_ld_error;
 337         tfns.ltf_lderrno_arg = NULL;
 338 
 339         /*
 340          * Set up the ld to use those function pointers
 341          */
 342         rc = ldap_set_option(ld, LDAP_OPT_THREAD_FN_PTRS,
 343             (void *) &tfns);
 344         if (rc < 0) {
 345                 syslog(LOG_INFO, gettext("libsldap: ldap_set_option "
 346                 "(LDAP_OPT_THREAD_FN_PTRS)"));
 347                 return (0);
 348         }
 349 
 350         /*
 351          * Set the function pointers for working with semaphores
 352          */
 353         (void) memset(&extrafns, '\0',
 354             sizeof (struct ldap_extra_thread_fns));
 355         extrafns.ltf_threadid_fn = (void * (*)(void))thr_self;
 356         extrafns.ltf_mutex_trylock = NULL;
 357         extrafns.ltf_sema_alloc = NULL;
 358         extrafns.ltf_sema_free = NULL;
 359         extrafns.ltf_sema_wait = NULL;
 360         extrafns.ltf_sema_post = NULL;
 361 
 362         /* Set up the ld to use those function pointers */
 363         rc = ldap_set_option(ld, LDAP_OPT_EXTRA_THREAD_FN_PTRS,
 364             (void *) &extrafns);
 365         if (rc < 0) {
 366                 syslog(LOG_INFO, gettext("libsldap: ldap_set_option "
 367                 "(LDAP_OPT_EXTRA_THREAD_FN_PTRS)"));
 368                 return (0);
 369         }
 370 
 371         return (1);
 372 }
 373 
 374 /* set up an MT connection for sharing among multiple threads */
 375 static int
 376 setup_mt_ld(LDAP *ld, ns_conn_mgmt_t *cmg)
 377 {
 378         thread_t        t = thr_self();
 379 
 380         /* set up the per-thread data for using the MT connection */
 381         if (conn_tsd_setup(cmg) == -1) {
 382                 syslog(LOG_WARNING,
 383                     gettext("libsldap: tid= %d: unable to set up TSD\n"), t);
 384                 return (-1);
 385         }
 386 
 387         if (setup_mt_conn(ld) == 0) {
 388                 /* multiple threads per connection not supported */
 389                 syslog(LOG_WARNING, gettext("libsldap: tid= %d: multiple "
 390                     "threads per connection not supported\n"), t);
 391                 conn_tsd_free();
 392                 return (-1);
 393         }
 394         return (0);
 395 }
 396 
 397 /*
 398  * Check name and UID of process, if it is nscd.
 399  *
 400  * Input:
 401  *   pid        : PID of checked process
 402  *   check_uid  : check if UID == 0
 403  * Output:
 404  *   B_TRUE     : nscd detected
 405  *   B_FALSE    : nscd not confirmed
 406  */
 407 static boolean_t
 408 check_nscd_proc(pid_t pid, boolean_t check_uid)
 409 {
 410         psinfo_t        pinfo;
 411         char            fname[MAXPATHLEN];
 412         ssize_t         ret;
 413         int             fd;
 414 
 415         if (snprintf(fname, MAXPATHLEN, "/proc/%d/psinfo", pid) > 0) {
 416                 if ((fd = open(fname,  O_RDONLY)) >= 0) {
 417                         ret = read(fd, &pinfo, sizeof (psinfo_t));
 418                         (void) close(fd);
 419                         if ((ret == sizeof (psinfo_t)) &&
 420                             (strcmp(pinfo.pr_fname, "nscd") == 0)) {
 421                                 if (check_uid && (pinfo.pr_uid != 0))
 422                                         return (B_FALSE);
 423                                 return (B_TRUE);
 424                         }
 425                 }
 426         }
 427         return (B_FALSE);
 428 }
 429 
 430 /*
 431  * Check if this process is peruser nscd.
 432  */
 433 boolean_t
 434 __s_api_peruser_proc(void)
 435 {
 436         pid_t           my_ppid;
 437         static mutex_t  nscdLock = DEFAULTMUTEX;
 438         static pid_t    checkedPpid = (pid_t)-1;
 439         static boolean_t isPeruserNscd = B_FALSE;
 440 
 441         my_ppid = getppid();
 442 
 443         /*
 444          * Already checked before for this process? If yes, return cached
 445          * response.
 446          */
 447         if (my_ppid == checkedPpid) {
 448                 return (isPeruserNscd);
 449         }
 450 
 451         (void) mutex_lock(&nscdLock);
 452 
 453         /* Check once more incase another thread has just complete this. */
 454         if (my_ppid == checkedPpid) {
 455                 (void) mutex_unlock(&nscdLock);
 456                 return (isPeruserNscd);
 457         }
 458 
 459         /* Reinitialize to be sure there is no residue after fork. */
 460         isPeruserNscd = B_FALSE;
 461 
 462         /* Am I the nscd process? */
 463         if (check_nscd_proc(getpid(), B_FALSE)) {
 464                 /* Is my parent the nscd process with UID == 0. */
 465                 isPeruserNscd = check_nscd_proc(my_ppid, B_TRUE);
 466         }
 467 
 468         /* Remember for whom isPeruserNscd is. */
 469         checkedPpid = my_ppid;
 470 
 471         (void) mutex_unlock(&nscdLock);
 472         return (isPeruserNscd);
 473 }
 474 
 475 /*
 476  * Check if this process is main nscd.
 477  */
 478 boolean_t
 479 __s_api_nscd_proc(void)
 480 {
 481         pid_t           my_pid;
 482         static mutex_t  nscdLock = DEFAULTMUTEX;
 483         static pid_t    checkedPid = (pid_t)-1;
 484         static boolean_t isMainNscd = B_FALSE;
 485 
 486         /*
 487          * Don't bother checking if this process isn't root, this cannot
 488          * be main nscd.
 489          */
 490         if (getuid() != 0)
 491                 return (B_FALSE);
 492 
 493         my_pid = getpid();
 494 
 495         /*
 496          * Already checked before for this process? If yes, return cached
 497          * response.
 498          */
 499         if (my_pid == checkedPid) {
 500                 return (isMainNscd);
 501         }
 502 
 503         (void) mutex_lock(&nscdLock);
 504 
 505         /* Check once more incase another thread has just done this. */
 506         if (my_pid == checkedPid) {
 507                 (void) mutex_unlock(&nscdLock);
 508                 return (isMainNscd);
 509         }
 510 
 511         /*
 512          * Am I the nscd process? UID is already checked, not needed from
 513          * psinfo.
 514          */
 515         isMainNscd = check_nscd_proc(my_pid, B_FALSE);
 516 
 517         /* Remember for whom isMainNscd is. */
 518         checkedPid = my_pid;
 519 
 520         (void) mutex_unlock(&nscdLock);
 521         return (isMainNscd);
 522 }
 523 
 524 /*
 525  * initialize a connection management control structure conn_mgmt_t
 526  */
 527 ns_conn_mgmt_t *
 528 init_conn_mgmt()
 529 {
 530         ns_conn_mgmt_t  *cmg;
 531 
 532         cmg = (ns_conn_mgmt_t *)calloc(1, sizeof (*cmg));
 533         if (cmg == NULL) {
 534                 syslog(LOG_ERR, NS_CONN_MSG_NO_CONN_MGMT);
 535                 return (NULL);
 536         }
 537 
 538         /* is this process nscd or peruser nscd ? */
 539         cmg->is_nscd = __s_api_nscd_proc();
 540         cmg->is_peruser_nscd = __s_api_peruser_proc();
 541 
 542         /*
 543          * assume the underlying libldap allows multiple threads sharing
 544          * the same ldap connection (MT connection)
 545          */
 546         cmg->ldap_mt = B_TRUE;
 547         /* state is inactive until MT connection is required/requested */
 548         cmg->state = NS_CONN_MGMT_INACTIVE;
 549 
 550         (void) mutex_init(&cmg->lock, USYNC_THREAD, NULL);
 551         (void) mutex_init(&cmg->cfg_lock, USYNC_THREAD, NULL);
 552         cmg->pid = getpid();
 553 
 554         /* for nscd or peruser nscd, MT connection is required */
 555         if (cmg->is_nscd == B_TRUE || cmg->is_peruser_nscd == B_TRUE)
 556                 cmg->state = NS_CONN_MGMT_ACTIVE;
 557 
 558         /*
 559          * reference (or initialize) the current Native LDAP configuration and
 560          * if in nscd process, make it never refreshed
 561          */
 562         cmg->config = __s_api_get_default_config_global();
 563         if (cmg->config == NULL)
 564                 cmg->config = __s_api_loadrefresh_config_global();
 565         if (cmg->config != NULL) {
 566                 /*
 567                  * main nscd get config change notice from ldap_cachemgr
 568                  * so won't times out and refresh the config
 569                  */
 570                 if (cmg->is_nscd  == B_TRUE)
 571                         (cmg->config)->paramList[NS_LDAP_EXP_P].ns_tm = 0;
 572                 cmg->cfg_cookie = cmg->config->config_cookie;
 573         }
 574 
 575         return (cmg);
 576 }
 577 
 578 static void
 579 mark_shutdown_or_reloaded(int op)
 580 {
 581         ns_conn_mgmt_t  *cmg = ns_connmgmt;
 582 
 583         (void) mutex_lock(&cmg->lock);
 584         if (op == NS_CONN_MGMT_OP_SHUTDOWN)
 585                 cmg->shutting_down = B_TRUE;
 586         else
 587                 cmg->cfg_reloaded = B_TRUE;
 588         atomic_inc_uint(&cmg->ref_cnt);
 589         cmg->state = NS_CONN_MGMT_DETACHED;
 590 
 591         if (op == NS_CONN_MGMT_OP_RELOAD_CONFIG)
 592                 __s_api_init_config_global(NULL);
 593 
 594         (void) mutex_unlock(&cmg->lock);
 595 }
 596 
 597 /*
 598  * Return a pointer to the current connection management. If
 599  * it has not been created, or is requested to recreate, then
 600  * create and return the pointer. It is possible, the current
 601  * one is created by the parent before fork, create a new
 602  * one too in such a case.
 603  */
 604 static ns_conn_mgmt_t *
 605 get_current_conn_mgmt(int op)
 606 {
 607         ns_conn_mgmt_t  *cmg = ns_connmgmt;
 608         static pid_t    checked_pid = (pid_t)-1;
 609         pid_t           mypid;
 610 
 611         mypid = getpid();
 612         if (cmg == NULL || checked_pid != mypid) {
 613                 checked_pid = mypid;
 614 
 615                 /*
 616                  * if current conn_mgmt not created yet or is from parent
 617                  * or is requested to recreate, create it
 618                  */
 619                 if (cmg == NULL || cmg->pid != mypid) {
 620                         if (cmg != NULL) {
 621                                 /*
 622                                  * We don't want to free the conn_mgmt
 623                                  * allocated by the parent, since
 624                                  * there may be ldap connections
 625                                  * still being used. So leave it
 626                                  * alone but keep it referenced,
 627                                  * so that it will not be flagged
 628                                  * as a piece of leaked memory.
 629                                  */
 630                                 ns_connmgmt_parent = cmg;
 631                                 /*
 632                                  * avoid lint warning; does not
 633                                  * change the conn_mgmt in parent
 634                                  */
 635                                 ns_connmgmt_parent->state =
 636                                     NS_CONN_MGMT_DETACHED;
 637                         }
 638                         ns_connmgmt = init_conn_mgmt();
 639                         cmg = ns_connmgmt;
 640                         /*
 641                          * ensure it will not be destroyed until explicitly
 642                          * shut down or reloaded
 643                          */
 644                         if (op == NS_CONN_MGMT_OP_REF)
 645                                 atomic_inc_uint(&cmg->ref_cnt);
 646                 }
 647         }
 648 
 649         return (cmg);
 650 }
 651 
 652 static ns_conn_mgmt_t *
 653 access_conn_mgmt(int op)
 654 {
 655         ns_conn_mgmt_t  *cmg = NULL;
 656         ns_conn_mgmt_t  *cmg_prev;
 657 
 658         (void) mutex_lock(&ns_connmgmt_lock);
 659 
 660         /*
 661          * connection management is not available when the libsldap is being
 662          * unloaded or shut down
 663          */
 664         if (ns_connmgmt_shutting_down == B_TRUE) {
 665                 (void) mutex_unlock(&ns_connmgmt_lock);
 666                 return (NULL);
 667         }
 668 
 669         if (op == NS_CONN_MGMT_OP_SHUTDOWN) {
 670                 ns_connmgmt_shutting_down = B_TRUE;
 671                 if (ns_connmgmt != NULL) {
 672                         cmg = ns_connmgmt;
 673                         mark_shutdown_or_reloaded(op);
 674                         ns_connmgmt = NULL;
 675                 }
 676                 (void) mutex_unlock(&ns_connmgmt_lock);
 677                 return (cmg);
 678         }
 679 
 680         if (op == NS_CONN_MGMT_OP_RELOAD_CONFIG ||
 681             op == NS_CONN_MGMT_OP_NEW_CONFIG) {
 682                 cmg_prev = ns_connmgmt;
 683                 mark_shutdown_or_reloaded(op);
 684                 /*
 685                  * the previous cmg (cmg_prev) will be freed later
 686                  * when its ref count reaches zero
 687                  */
 688                 ns_connmgmt = NULL;
 689         }
 690 
 691         cmg = get_current_conn_mgmt(op);
 692         if (cmg == NULL) {
 693                 (void) mutex_unlock(&ns_connmgmt_lock);
 694                 return (NULL);
 695         }
 696 
 697         atomic_inc_uint(&cmg->ref_cnt);
 698         if (op == NS_CONN_MGMT_OP_RELOAD_CONFIG ||
 699             op == NS_CONN_MGMT_OP_NEW_CONFIG)
 700                 cmg = cmg_prev;
 701         else { /* op is  NS_CONN_MGMT_OP_REF  or NS_CONN_MGMT_OP_LIB_INIT */
 702                 if (cmg->config == NULL)
 703                         cmg->config = __s_api_get_default_config();
 704         }
 705 
 706         (void) mutex_unlock(&ns_connmgmt_lock);
 707         return (cmg);
 708 }
 709 
 710 /*
 711  * free a connection management control
 712  */
 713 static void
 714 free_conn_mgmt(ns_conn_mgmt_t *cmg)
 715 {
 716         union {
 717                 ldap_data_t     s_d;
 718                 char            s_b[1024];
 719         } space;
 720         ldap_data_t     *sptr;
 721         int             ndata;
 722         int             adata;
 723         int             rc;
 724         ldap_get_chg_cookie_t cookie;
 725 
 726         if (cmg == NULL)
 727                 return;
 728         cookie = cmg->cfg_cookie;
 729 
 730         __s_api_free2dArray(cmg->pservers);
 731         /* destroy the previous config or release the current one */
 732         if (cmg->config != NULL) {
 733                 if (cmg->state == NS_CONN_MGMT_DETACHED)
 734                         __s_api_destroy_config(cmg->config);
 735                 else
 736                         __s_api_release_config(cmg->config);
 737         }
 738 
 739         /* stop the server status/config-change monitor thread */
 740         if (cmg->procchg_started == B_TRUE) {
 741                 if (cmg->procchg_tid != thr_self()) {
 742                         if (cmg->procchg_door_call == B_TRUE) {
 743                                 adata = sizeof (ldap_call_t) + 1;
 744                                 ndata = sizeof (space);
 745                                 space.s_d.ldap_call.ldap_callnumber =
 746                                     GETSTATUSCHANGE;
 747                                 space.s_d.ldap_call.ldap_u.get_change.op =
 748                                     NS_STATUS_CHANGE_OP_STOP;
 749                                 space.s_d.ldap_call.ldap_u.get_change.cookie =
 750                                     cookie;
 751                                 sptr = &space.s_d;
 752                                 rc = __ns_ldap_trydoorcall(&sptr, &ndata,
 753                                     &adata);
 754                                 if (rc != NS_CACHE_SUCCESS)
 755                                         syslog(LOG_INFO,
 756                                             gettext("libsldap: "
 757                                             "free_conn_mgmt():"
 758                                             " stopping door call "
 759                                             " GETSTATUSCHANGE failed "
 760                                             " (rc = %d)"), rc);
 761                         }
 762                         (void) pthread_cancel(cmg->procchg_tid);
 763                         cmg->procchg_started = B_FALSE;
 764                 }
 765         }
 766 
 767         free(cmg);
 768 }
 769 
 770 static ns_conn_mgmt_t *
 771 release_conn_mgmt(ns_conn_mgmt_t *cmg, boolean_t unlock_cmg)
 772 {
 773         if (cmg == NULL)
 774                 return (NULL);
 775         if (atomic_dec_uint_nv(&cmg->ref_cnt) == 0) {
 776                 if (cmg->state == NS_CONN_MGMT_DETACHED) {
 777                         if (unlock_cmg == B_TRUE)
 778                                 (void) mutex_unlock(&cmg->lock);
 779                         free_conn_mgmt(cmg);
 780                         __s_api_free_sessionPool();
 781                         return (NULL);
 782                 } else {
 783                         syslog(LOG_WARNING,
 784                             gettext("libsldap: connection management "
 785                             " has a refcount of zero but the state "
 786                             " is not DETACHED (%d)"), cmg->state);
 787                         cmg = NULL;
 788                 }
 789         }
 790         return (cmg);
 791 }
 792 
 793 /*
 794  * exposed function for initializing a connection management control structure
 795  */
 796 ns_conn_mgmt_t *
 797 __s_api_conn_mgmt_init()
 798 {
 799         if (thr_keycreate_once(&ns_mtckey, ns_mtckey_cleanup) != 0) {
 800                 syslog(LOG_WARNING, NS_CONN_MSG_NO_MTC_KEY);
 801                 return (NULL);
 802         }
 803 
 804         if (thr_keycreate_once(&ns_cmgkey, NULL) != 0) {
 805                 syslog(LOG_WARNING, NS_CONN_MSG_NO_CMG_KEY);
 806                 return (NULL);
 807         }
 808 
 809         return (access_conn_mgmt(NS_CONN_MGMT_OP_LIB_INIT));
 810 }
 811 
 812 /* initialize a connection user */
 813 ns_conn_user_t *
 814 __s_api_conn_user_init(int type, void *userinfo, boolean_t referral)
 815 {
 816         ns_conn_user_t  *cu;
 817         ns_conn_mgmt_t  *cmg;
 818 
 819         /* delete the reference to the previously used conn_mgmt */
 820         (void) thr_setspecific(ns_cmgkey, NULL);
 821 
 822         cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
 823         if (cmg == NULL)
 824                 return (NULL);
 825 
 826         if (cmg->state != NS_CONN_MGMT_ACTIVE &&
 827             cmg->state != NS_CONN_MGMT_INACTIVE) {
 828                 atomic_dec_uint(&cmg->ref_cnt);
 829                 return (NULL);
 830         }
 831 
 832         cu = (ns_conn_user_t *)calloc(1, sizeof (*cu));
 833         if (cu == NULL) {
 834                 atomic_dec_uint(&cmg->ref_cnt);
 835                 return (NULL);
 836         }
 837 
 838         cu->type = type;
 839         cu->state = NS_CONN_USER_ALLOCATED;
 840         cu->tid = thr_self();
 841         cu->userinfo = userinfo;
 842         cu->referral = referral;
 843         cu->ns_rc = NS_LDAP_SUCCESS;
 844         cu->conn_mgmt = cmg;
 845 
 846         (void) conn_tsd_setup(cmg);
 847 
 848         return (cu);
 849 }
 850 
 851 /*
 852  * Free the resources used by a connection user.
 853  * The caller should ensure this conn_user is
 854  * not associated with any conn_mt, i.e.,
 855  * not in any conn_mt's linked list of conn_users.
 856  * The caller needs to free the userinfo member
 857  * as well.
 858  */
 859 void
 860 __s_api_conn_user_free(ns_conn_user_t *cu)
 861 {
 862         ns_conn_mgmt_t  *cmg;
 863 
 864         if (cu == NULL)
 865                 return;
 866 
 867         cu->state = NS_CONN_USER_FREED;
 868         if (cu->ns_error != NULL)
 869                 (void) __ns_ldap_freeError(&cu->ns_error);
 870 
 871         cmg = cu->conn_mgmt;
 872         conn_tsd_free();
 873         (void) release_conn_mgmt(cmg, B_FALSE);
 874         (void) free(cu);
 875 }
 876 
 877 /*
 878  * Initialize an MT connection control structure
 879  * that will be used to represent an ldap connection
 880  * to be shared among multiple threads and to hold
 881  * and manage all the conn_users using the ldap
 882  * connection.
 883  */
 884 static ns_conn_mt_t *
 885 init_conn_mt(ns_conn_mgmt_t *cmg, ns_ldap_error_t **ep)
 886 {
 887         ns_conn_mt_t    *cm;
 888         ns_conn_mgmt_t  *cmg_a;
 889 
 890         cm = (ns_conn_mt_t *)calloc(1, sizeof (*cm));
 891         if (cm == NULL) {
 892                 if (ep != NULL)
 893                         *ep = __s_api_make_error(NS_LDAP_MEMORY, NULL);
 894                 return (NULL);
 895         }
 896 
 897         cmg_a = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
 898         if (cmg_a != cmg) {
 899                 if (cmg_a != NULL) {
 900                         (void) release_conn_mgmt(cmg_a, B_FALSE);
 901                         if (ep != NULL)
 902                                 *ep = __s_api_make_error(NS_LDAP_OP_FAILED,
 903                                     NS_CONN_MSG_SHUTDOWN_RELOADED);
 904                 }
 905                 return (NULL);
 906         }
 907 
 908         (void) mutex_init(&cm->lock, USYNC_THREAD, NULL);
 909         cm->state = NS_CONN_MT_CONNECTING;
 910         cm->tid = thr_self();
 911         cm->pid = getpid();
 912         cm->next = NULL;
 913         cm->cu_head = NULL;
 914         cm->cu_tail = NULL;
 915         cm->conn = NULL;
 916         cm->conn_mgmt = cmg;
 917 
 918         return (cm);
 919 }
 920 
 921 /*
 922  * Free an MT connection control structure, assume conn_mgmt is locked.
 923  * 'unlock_cmg' is passed to release_conn_mgmt() to indicate the
 924  * cmg needs to be unlocked or not.
 925  */
 926 static ns_conn_mgmt_t *
 927 free_conn_mt(ns_conn_mt_t *cm, int unlock_cmg)
 928 {
 929         ns_conn_mgmt_t  *cmg;
 930 
 931         if (cm == NULL)
 932                 return (NULL);
 933         if (cm->ns_error != NULL)
 934                 (void) __ns_ldap_freeError(&cm->ns_error);
 935         if (cm->conn != NULL) {
 936                 if (cm->conn->ld != NULL)
 937                         (void) ldap_unbind(cm->conn->ld);
 938                 __s_api_freeConnection(cm->conn);
 939         }
 940         cmg = cm->conn_mgmt;
 941         free(cm);
 942         return (release_conn_mgmt(cmg, unlock_cmg));
 943 }
 944 
 945 /* add a connection user to an MT connection */
 946 static void
 947 add_cu2cm(ns_conn_user_t *cu, ns_conn_mt_t *cm)
 948 {
 949 
 950         if (cm->cu_head == NULL) {
 951                 cm->cu_head = cu;
 952                 cm->cu_tail = cu;
 953         } else {
 954                 cm->cu_tail->next = cu;
 955                 cm->cu_tail = cu;
 956         }
 957         cm->cu_cnt++;
 958 }
 959 
 960 /* add an MT connection to the connection management */
 961 static void
 962 add_cm2cmg(ns_conn_mt_t *cm, ns_conn_mgmt_t *cmg)
 963 {
 964         /*
 965          * add connection opened for WRITE to top of list
 966          * for garbage collection purpose. This is to
 967          * ensure the connection will be closed after a
 968          * certain amount of time (60 seconds).
 969          */
 970         if (cmg->cm_head == NULL) {
 971                 cmg->cm_head = cm;
 972                 cmg->cm_tail = cm;
 973         } else {
 974                 if (cm->opened_for == NS_CONN_USER_WRITE) {
 975                         cm->next = cmg->cm_head;
 976                         cmg->cm_head = cm;
 977                 } else {
 978                         cmg->cm_tail->next = cm;
 979                         cmg->cm_tail = cm;
 980                 }
 981         }
 982         cmg->cm_cnt++;
 983 }
 984 
 985 /* delete a connection user from an MT connection */
 986 static void
 987 del_cu4cm(ns_conn_user_t *cu, ns_conn_mt_t *cm)
 988 {
 989         ns_conn_user_t *pu, *u;
 990 
 991         if (cu == NULL || cm->cu_head == NULL || cm->cu_cnt == 0)
 992                 return;
 993 
 994         /* only one conn_user on list */
 995         if (cm->cu_head == cm->cu_tail) {
 996                 if (cu == cm->cu_head) {
 997                         cm->cu_head = cm->cu_tail = NULL;
 998                         cm->cu_cnt = 0;
 999                         cu->next = NULL;
1000                 }
1001                 return;
1002         }
1003 
1004         /* more than one and cu is the first one */
1005         if (cu == cm->cu_head) {
1006                 cm->cu_head = cu->next;
1007                 cm->cu_cnt--;
1008                 cu->next = NULL;
1009                 return;
1010         }
1011 
1012         pu = cm->cu_head;
1013         for (u = cm->cu_head->next; u; u = u->next) {
1014                 if (cu == u)
1015                         break;
1016                 pu = u;
1017         }
1018         if (pu != cm->cu_tail) {
1019                 pu->next = cu->next;
1020                 if (pu->next == NULL)
1021                         cm->cu_tail = pu;
1022                 cm->cu_cnt--;
1023                 cu->next = NULL;
1024         } else {
1025                 syslog(LOG_INFO, gettext(
1026                     "libsldap: del_cu4cm(): connection user not found"));
1027         }
1028 }
1029 
1030 /* delete an MT connection from the connection management control structure */
1031 static void
1032 del_cm4cmg(ns_conn_mt_t *cm, ns_conn_mgmt_t *cmg)
1033 {
1034         ns_conn_mt_t *pm, *m;
1035 
1036         if (cm == NULL || cmg->cm_head == NULL || cmg->cm_cnt == 0)
1037                 return;
1038 
1039         /* only one conn_mt on list */
1040         if (cmg->cm_head == cmg->cm_tail) {
1041                 if (cm == cmg->cm_head) {
1042                         cmg->cm_head = cmg->cm_tail = NULL;
1043                         cmg->cm_cnt = 0;
1044                         cm->next = NULL;
1045                 }
1046                 return;
1047         }
1048 
1049         /* more than one and cm is the first one */
1050         if (cm == cmg->cm_head) {
1051                 cmg->cm_head = cm->next;
1052                 cmg->cm_cnt--;
1053                 cm->next = NULL;
1054                 return;
1055         }
1056 
1057         pm = cmg->cm_head;
1058         for (m = cmg->cm_head->next; m; m = m->next) {
1059                 if (cm == m)
1060                         break;
1061                 pm = m;
1062         }
1063         if (pm != cmg->cm_tail) {
1064                 pm->next = cm->next;
1065                 if (pm->next == NULL)
1066                         cmg->cm_tail = pm;
1067                 cmg->cm_cnt--;
1068                 cm->next = NULL;
1069         } else {
1070                 syslog(LOG_INFO, gettext(
1071                     "libsldap: del_cm4cmg(): MT connection not found"));
1072         }
1073 }
1074 
1075 /*
1076  * compare to see if the server and credential for authentication match
1077  * those used by an MT connection
1078  */
1079 static boolean_t
1080 is_server_cred_matched(const char *server, const ns_cred_t *cred,
1081         ns_conn_mt_t *cm)
1082 {
1083         Connection      *cp = cm->conn;
1084 
1085         /* check server first */
1086         if (server != NULL && *server != 0) {
1087                 if (strcasecmp(server, cp->serverAddr) != 0)
1088                         return (B_FALSE);
1089         }
1090 
1091         if (cred == NULL)
1092                 return (B_TRUE);
1093 
1094         /* then check cred */
1095         return (__s_api_is_auth_matched(cp->auth, cred));
1096 }
1097 
1098 /*
1099  * Wait until a pending MT connection becomes available.
1100  * Return 1 if so, 0 if error.
1101  *
1102  * Assume the current conn_mgmt and the input conn_mt
1103  * are locked.
1104  */
1105 static int
1106 wait_for_conn_mt(ns_conn_user_t *cu, ns_conn_mt_t *cm)
1107 {
1108 
1109         cu->state = NS_CONN_USER_WAITING;
1110         add_cu2cm(cu, cm);
1111         cu->conn_mt = cm;
1112 
1113         (void) mutex_unlock(&cm->lock);
1114         /*
1115          * It could take some time so we don't want to hold
1116          * cm->conn_mgmt across the wait
1117          */
1118         (void) mutex_unlock(&(cm->conn_mgmt)->lock);
1119 
1120         (void) mutex_lock(&cm->lock);
1121         /* check one more time see if need to wait */
1122         if (cm->state == NS_CONN_MT_CONNECTING) {
1123                 (void) conn_wait(cm, cu);
1124 
1125                 /* cm->lock is locked again at this point */
1126 
1127                 cu->state = NS_CONN_USER_WOKEUP;
1128         }
1129 
1130         if (cm->state == NS_CONN_MT_CONNECTED)
1131                 return (1);
1132         else {
1133                 del_cu4cm(cu, cm);
1134                 cu->conn_mt = NULL;
1135                 cu->bad_mt_conn = B_FALSE;
1136                 return (0);
1137         }
1138 }
1139 
1140 /*
1141  * Check and see if the input MT connection '*cm' should be closed.
1142  * In two cases, it should be closed. If a preferred server is
1143  * found to be up when ldap_cachemgr is queried and reported back.
1144  * Or when the server being used for the connection is found to
1145  * be down. Return B_FALSE if the connection is not closed (or not marked
1146  * to be closed), otherwise unlock mutex (*cm)->lock and return B_TRUE.
1147  * This function assumes conn_mgmt cmg and conn_mt *cm are locked.
1148  */
1149 static boolean_t
1150 check_and_close_conn(ns_conn_mgmt_t *cmg, ns_conn_mt_t **cm,
1151         ns_conn_user_t *cu) {
1152 
1153         int rc;
1154         int j;
1155         int svridx = -1;
1156         int upidx = -1;
1157         int free_cm;
1158         ns_server_info_t sinfo;
1159         ns_ldap_error_t *errorp = NULL;
1160 
1161         /*
1162          * check only if preferred servers are defined
1163          */
1164         if (cmg->pservers_loaded == B_FALSE)
1165                 get_preferred_servers(B_FALSE, B_FALSE, cmg);
1166         if (cmg->pservers == NULL)
1167                 return (B_FALSE);
1168 
1169         /*
1170          * ask ldap_cachemgr for the first available server
1171          */
1172         rc = __s_api_requestServer(NS_CACHE_NEW, NULL,
1173             &sinfo, &errorp, NS_CACHE_ADDR_IP);
1174         if (rc != NS_LDAP_SUCCESS || sinfo.server == NULL) {
1175                 (void) __ns_ldap_freeError(&errorp);
1176                 return (B_FALSE);
1177         }
1178 
1179         /*
1180          * Did ldap_cachemgr return a preferred server ?
1181          */
1182         for (j = 0; cmg->pservers[j] != NULL; j++) {
1183                 if (strcasecmp(sinfo.server, cmg->pservers[j]) != 0)
1184                         continue;
1185                 upidx = j;
1186                 break;
1187         }
1188 
1189         /*
1190          * Is the server being used a preferred one ?
1191          */
1192         for (j = 0; cmg->pservers[j] != NULL; j++) {
1193                 if (strcasecmp(cmg->pservers[j], (*cm)->conn->serverAddr) != 0)
1194                         continue;
1195                 svridx = j;
1196                 break;
1197         }
1198 
1199         /*
1200          * Need to fall back to a down-but-now-up preferred server ?
1201          * A preferred server falls back to a more preferred one.
1202          * A regular one falls back to any preferred ones. So if
1203          * both are preferred ones and same index, or both
1204          * are not preferred ones, then no need to close the
1205          * connection.
1206          */
1207         if ((upidx == -1 && svridx == -1) ||
1208             (upidx != -1 && svridx != -1 && upidx == svridx)) {
1209                 __s_api_free_server_info(&sinfo);
1210                 return (B_FALSE);
1211         }
1212 
1213         /*
1214          * otherwise, 4 cases, all may need to close the connection:
1215          * For case 1 and 2, both servers are preferred ones:
1216          * 1. ldap_cachemgr returned a better one to use (upidx < svridx)
1217          * 2. the server being used is down (upidx > svridx)
1218          * 3. ldap_cachemgr returned a preferred one, but the server
1219          *    being used is not, so need to fall back to the preferred server
1220          * 4. ldap_cachemgr returned a non-preferred one, but the server
1221          *    being used is a preferred one, so it must be down (since
1222          *    ldap_cachemgr always returns a preferred one when possible).
1223          * For case 1 & 3, close the READ connection when no user uses it.
1224          * For 2 and 4, close the connection with error rc, LDAP_SERVER_DOWN.
1225          */
1226         if (upidx != -1 && (svridx == -1 || upidx < svridx)) { /* case 1 & 3 */
1227                 /* fallback does not make sense for WRITE/referred connection */
1228                 if ((*cm)->opened_for == NS_CONN_USER_WRITE ||
1229                     (*cm)->referral == B_TRUE) {
1230                         __s_api_free_server_info(&sinfo);
1231                         return (B_FALSE);
1232                 }
1233                 free_cm = close_conn_mt_when_nouser(*cm);
1234                 if (cmg->shutting_down == B_FALSE)
1235                         cu->retry = B_TRUE;
1236         } else {
1237                 ns_ldap_error_t *ep;
1238                 ep = __s_api_make_error(LDAP_SERVER_DOWN,
1239                     NS_CONN_MSG_DOWN_FROM_CACHEMGR);
1240                 /* cu has not been attached to cm yet, use NULL as cu pointer */
1241                 free_cm = close_conn_mt(*cm, LDAP_SERVER_DOWN, &ep, NULL);
1242                 if (cmg->shutting_down == B_FALSE)
1243                         cu->retry = B_TRUE;
1244                 (void) __ns_ldap_freeError(&ep);
1245         }
1246 
1247         (void) mutex_unlock(&(*cm)->lock);
1248         if (free_cm == 1) {
1249                 (void) free_conn_mt(*cm, 0);
1250                 *cm = NULL;
1251         }
1252 
1253         __s_api_free_server_info(&sinfo);
1254 
1255         return (B_TRUE);
1256 }
1257 
1258 /*
1259  * Check to see if a conn_mt matches the connection criteria from
1260  * a conn_user. Return B_TRUE if yes, B_FALSE, otherwise. The input
1261  * conn_mt pointer (*cmt) may be freed and *cmt will be set to NULL
1262  * to indicate so.
1263  * conn_mt *cmt and conn_mgmt cm->conn_mgmt are assumed locked.
1264  * cm->lock is unlocked at exit if rc is B_FALSE.
1265  */
1266 static boolean_t
1267 match_conn_mt(ns_conn_user_t *cu, ns_conn_mt_t **cmt,
1268         ns_conn_mt_state_t st, const char *server,
1269         const ns_cred_t *cred)
1270 {
1271         boolean_t       matched = B_FALSE;
1272         boolean_t       drop_conn;
1273         int             free_cm = 0;
1274         ns_conn_mt_t    *cm = *cmt;
1275         ns_conn_mgmt_t  *cmg = cm->conn_mgmt;
1276 
1277         if (cm->state != st || cm->close_when_nouser  == B_TRUE ||
1278             cm->detached == B_TRUE || cm->pid != getpid() ||
1279             cm->referral != cu->referral) {
1280                 (void) mutex_unlock(&cm->lock);
1281                 return (B_FALSE);
1282         }
1283 
1284         /*
1285          * if a conn_mt opened for WRITE is idle
1286          * long enough, then close it. To improve
1287          * the performance of applications, such
1288          * as ldapaddent, a WRITE connection is
1289          * given a short time to live in the
1290          * connection pool, expecting the write
1291          * requests to come in a quick succession.
1292          * To save resource, the connection will
1293          * be closed if idle more than 60 seconds.
1294          */
1295         if (cm->opened_for == NS_CONN_USER_WRITE &&
1296             cu->type != NS_CONN_USER_WRITE && cm->cu_cnt == 0 &&
1297             ((time(NULL) - cm->access_time) > 60)) {
1298                 /*
1299                  * NS_LDAP_INTERNAL is irrelevant here. There no
1300                  * conn_user to consume the rc
1301                  */
1302                 free_cm = close_conn_mt(cm, NS_LDAP_INTERNAL, NULL, NULL);
1303                 (void) mutex_unlock(&cm->lock);
1304                 if (free_cm == 1) {
1305                         (void) free_conn_mt(cm, 0);
1306                         *cmt = NULL;
1307                 }
1308                 return (B_FALSE);
1309         }
1310 
1311         switch (cu->type) {
1312         case NS_CONN_USER_SEARCH:
1313         case NS_CONN_USER_GETENT:
1314                 if (cm->opened_for == NS_CONN_USER_SEARCH ||
1315                     cm->opened_for == NS_CONN_USER_GETENT)
1316                         matched = B_TRUE;
1317                 break;
1318 
1319         case NS_CONN_USER_WRITE:
1320                 if (cm->opened_for == NS_CONN_USER_WRITE)
1321                         matched = B_TRUE;
1322                 break;
1323 
1324         default:
1325                 matched = B_FALSE;
1326                 break;
1327         }
1328 
1329         if (matched == B_TRUE && ((server != NULL || cred != NULL) &&
1330             is_server_cred_matched(server, cred, cm) == B_FALSE))
1331                 matched = B_FALSE;
1332 
1333         if (matched != B_FALSE) {
1334                 /*
1335                  * Check and drop the 'connected' connection if
1336                  * necessary. Main nscd gets status changes from
1337                  * the ldap_cachemgr daemon directly via the
1338                  * GETSTATUSCHANGE door call, the standalone
1339                  * function works in a no ldap_cachemgr environment,
1340                  * so no need to check and drop connections.
1341                  */
1342                 if (cm->state == NS_CONN_MT_CONNECTED &&
1343                     cmg->is_nscd == B_FALSE && !__s_api_isStandalone()) {
1344                         drop_conn = check_and_close_conn(cmg, &cm, cu);
1345                         if (drop_conn == B_TRUE) {
1346                                 if (cm == NULL)
1347                                         *cmt = NULL;
1348                                 return (B_FALSE);
1349                         }
1350                 }
1351 
1352                 /* check if max. users using or waiting for the connection */
1353                 if ((cm->state == NS_CONN_MT_CONNECTED &&
1354                     cm->cu_max != NS_CONN_MT_USER_NO_MAX &&
1355                     cm->cu_cnt >= cm->cu_max) ||
1356                     (cm->state == NS_CONN_MT_CONNECTING &&
1357                     cm->cu_max != NS_CONN_MT_USER_NO_MAX &&
1358                     cm->waiter_cnt >= cm->cu_max - 1))
1359                         matched = B_FALSE;
1360         }
1361 
1362         if (matched == B_FALSE)
1363                 (void) mutex_unlock(&cm->lock);
1364 
1365         return (matched);
1366 }
1367 
1368 /*
1369  * obtain an MT connection from the connection management for a conn_user
1370  *
1371  * Input:
1372  *   server     : server name or IP address
1373  *   flags      : libsldap API flags
1374  *   cred       : pointer to the user credential
1375  *   cu         : pointer to the conn_user structure
1376  * Output:
1377  *   session    : hold pointer to the Connection structure
1378  *   errorp     : hold pointer to error info (ns_ldap_error_t)
1379  */
1380 int
1381 __s_api_conn_mt_get(const char *server, const int flags, const ns_cred_t *cred,
1382         Connection **session, ns_ldap_error_t **errorp, ns_conn_user_t *cu)
1383 {
1384         int             rc;
1385         int             i;
1386         ns_conn_mt_t    *cn;
1387         ns_conn_mt_state_t st;
1388         ns_conn_mgmt_t  *cmg;
1389 
1390         if (errorp == NULL || cu == NULL || session == NULL)
1391                 return (NS_LDAP_INVALID_PARAM);
1392 
1393         *session = NULL;
1394         cmg = cu->conn_mgmt;
1395 
1396         /*
1397          * for pam_ldap, always try opening a new connection
1398          */
1399         if (cu->type == NS_CONN_USER_AUTH)
1400                 return (NS_LDAP_NOTFOUND);
1401 
1402         /* if need a new conn, then don't reuse */
1403         if (flags & NS_LDAP_NEW_CONN)
1404                 return (NS_LDAP_NOTFOUND);
1405 
1406         if (flags & NS_LDAP_KEEP_CONN)
1407                 cu->keep_conn = B_TRUE;
1408 
1409         /*
1410          * We want to use MT connection only if keep-connection flag is
1411          * set or if MT was requested (or active)
1412          */
1413         if (!((cmg->state == NS_CONN_MGMT_INACTIVE &&
1414             cu->keep_conn == B_TRUE) || cmg->state == NS_CONN_MGMT_ACTIVE))
1415                 return (NS_LDAP_NOTFOUND);
1416 
1417         /* MT connection will be used now (if possible/available) */
1418         cu->use_mt_conn = B_TRUE;
1419 
1420         NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, errorp);
1421 
1422         /* first look for a connection already open */
1423         st = NS_CONN_MT_CONNECTED;
1424         cu->state = NS_CONN_USER_FINDING;
1425         for (i = 0; i < 2; i++) {
1426                 for (cn = cmg->cm_head; cn; cn = cn->next) {
1427                         (void) mutex_lock(&cn->lock);
1428                         rc = match_conn_mt(cu, &cn, st, server, cred);
1429                         if (rc == B_FALSE && cn != NULL) /* not found */
1430                                 continue;
1431                         if (cn == NULL) { /* not found and cn freed */
1432                                 /*
1433                                  * as the conn_mt list could
1434                                  * be different due to cn's
1435                                  * deletion, scan the entire
1436                                  * conn_mt list again
1437                                  */
1438                                 st = NS_CONN_MT_CONNECTED;
1439                                 i = -1;
1440                                 break;
1441                         }
1442 
1443                         /* return a connected one if found */
1444                         if (cn->state == NS_CONN_MT_CONNECTED) {
1445                                 *session = cn->conn;
1446                                 add_cu2cm(cu, cn);
1447                                 cu->conn_mt = cn;
1448                                 cu->state = NS_CONN_USER_CONNECTED;
1449                                 (void) mutex_unlock(&cn->lock);
1450                                 (void) mutex_unlock(&cmg->lock);
1451                                 return (NS_LDAP_SUCCESS);
1452                         }
1453 
1454                         /*
1455                          * if cn is not connecting, or allow only
1456                          * one user, skip it
1457                          */
1458                         if (cn->state != NS_CONN_MT_CONNECTING ||
1459                             cn->cu_max == 1) {
1460                                 (void) mutex_unlock(&cn->lock);
1461                                 continue;
1462                         }
1463 
1464                         /* wait for the connecting conn_mt */
1465                         if (wait_for_conn_mt(cu, cn) != 1) {
1466                                 /*
1467                                  * NS_LDAP_NOTFOUND signals that the function
1468                                  * __s_api_check_libldap_MT_conn_support()
1469                                  * detected that the lower libldap library
1470                                  * does not support MT connection, so return
1471                                  * NS_LDAP_NOTFOUND to let the caller to
1472                                  * open a non-MT conneciton. Otherwise,
1473                                  * connect error occurred, return
1474                                  * NS_CONN_USER_CONNECT_ERROR
1475                                  */
1476                                 if (cn->ns_rc != NS_LDAP_NOTFOUND)
1477                                         cu->state = NS_CONN_USER_CONNECT_ERROR;
1478                                 else {
1479                                         cu->state = NS_CONN_USER_FINDING;
1480                                         cu->use_mt_conn = B_FALSE;
1481                                 }
1482                                 (void) mutex_unlock(&cn->lock);
1483 
1484                                 /* cmg->lock unlocked by wait_for_conn_mt() */
1485 
1486                                 return (cn->ns_rc);
1487                         }
1488 
1489                         /* return the newly available conn_mt */
1490                         *session = cn->conn;
1491                         cu->state = NS_CONN_USER_CONNECTED;
1492                         (void) mutex_unlock(&cn->lock);
1493 
1494                         /* cmg->lock unlocked by wait_for_conn_mt() */
1495 
1496                         return (NS_LDAP_SUCCESS);
1497                 }
1498 
1499                 /* next, look for a connecting conn_mt */
1500                 if (i == 0)
1501                         st = NS_CONN_MT_CONNECTING;
1502         }
1503 
1504         /* no connection found, start opening one */
1505         cn = init_conn_mt(cmg, errorp);
1506         if (cn == NULL) {
1507                 (void) mutex_unlock(&cmg->lock);
1508                 return ((*errorp)->status);
1509         }
1510         cu->conn_mt = cn;
1511         cn->opened_for = cu->type;
1512         cn->referral = cu->referral;
1513         if (cmg->ldap_mt == B_TRUE)
1514                 cn->cu_max = NS_CONN_MT_USER_MAX;
1515         else
1516                 cn->cu_max = 1;
1517         add_cm2cmg(cn, cmg);
1518         (void) mutex_unlock(&cmg->lock);
1519 
1520         return (NS_LDAP_NOTFOUND);
1521 }
1522 
1523 
1524 /*
1525  * add an MT connection to the connection management
1526  *
1527  * Input:
1528  *   con        : pointer to the Connection info
1529  *   cu         : pointer to the conn_user structure
1530  * Output:
1531  *   ep         : hold pointer to error info (ns_ldap_error_t)
1532  */
1533 int
1534 __s_api_conn_mt_add(Connection *con, ns_conn_user_t *cu, ns_ldap_error_t **ep)
1535 {
1536         ns_conn_mgmt_t  *cmg = cu->conn_mgmt;
1537         ns_conn_mt_t    *cm = cu->conn_mt;
1538 
1539         /* if the conn_mgmt is being shut down, return error */
1540         NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, ep);
1541 
1542         /*
1543          * start the change monitor thread only if it
1544          * hasn't been started and the process is the
1545          * main nscd (not peruser nscd)
1546          */
1547         if (cmg->procchg_started == B_FALSE && cmg->is_nscd == B_TRUE) {
1548                 start_thread(cmg);
1549                 cmg->procchg_started = B_TRUE;
1550         }
1551         (void) mutex_lock(&cm->lock);
1552         cm->conn = con;
1553         cm->state = NS_CONN_MT_CONNECTED;
1554         cm->pid = getpid();
1555         cm->create_time = time(NULL);
1556         cm->access_time = cm->create_time;
1557         cm->opened_for = cu->type;
1558         add_cu2cm(cu, cm);
1559         cu->conn_mt = cm;
1560         cu->state = NS_CONN_USER_CONNECTED;
1561         if (cmg->ldap_mt == B_TRUE)
1562                 cm->cu_max = NS_CONN_MT_USER_MAX;
1563         else
1564                 cm->cu_max = 1;
1565 
1566         /* wake up the waiters if any */
1567         (void) conn_signal(cm);
1568 
1569         (void) mutex_unlock(&cm->lock);
1570         (void) mutex_unlock(&cmg->lock);
1571 
1572         return (NS_LDAP_SUCCESS);
1573 }
1574 
1575 /*
1576  * return an MT connection to the pool when a conn user is done using it
1577  *
1578  * Input:
1579  *   cu         : pointer to the conn_user structure
1580  * Output:      NONE
1581  */
1582 void
1583 __s_api_conn_mt_return(ns_conn_user_t *cu)
1584 {
1585         ns_conn_mt_t    *cm;
1586         ns_conn_mgmt_t  *cmg;
1587 
1588         if (cu == NULL || cu->use_mt_conn == B_FALSE)
1589                 return;
1590         cm = cu->conn_mt;
1591         if (cm == NULL)
1592                 return;
1593         cmg = cu->conn_mgmt;
1594 
1595         (void) mutex_lock(&cm->lock);
1596         del_cu4cm(cu, cm);
1597         cu->state = NS_CONN_USER_DISCONNECTED;
1598         cu->conn_mt = NULL;
1599         cu->bad_mt_conn = B_FALSE;
1600 
1601         /*
1602          *  if this MT connection is no longer needed, or not usable, and
1603          * no more conn_user uses it, then close it.
1604          */
1605 
1606         if ((cm->close_when_nouser == B_TRUE ||
1607             cm->state != NS_CONN_MT_CONNECTED) && cm->cu_cnt == 0) {
1608                 (void) mutex_unlock(&cm->lock);
1609                 (void) mutex_lock(&cmg->lock);
1610                 (void) mutex_lock(&cm->lock);
1611                 del_cm4cmg(cm, cmg);
1612                 /* use ns_conn_free (instead of 1) to avoid lint warning */
1613                 NS_CONN_UNLOCK_AND_FREE(ns_conn_free, cm, cmg);
1614         } else {
1615                 if (cm->state == NS_CONN_MT_CONNECTED && cm->cu_cnt == 0 &&
1616                     cm->conn != NULL && cm->conn->ld != NULL) {
1617                         struct timeval  zerotime;
1618                         LDAPMessage     *res;
1619 
1620                         zerotime.tv_sec = zerotime.tv_usec = 0L;
1621                         /* clean up remaining results just in case */
1622                         while (ldap_result(cm->conn->ld, LDAP_RES_ANY,
1623                             LDAP_MSG_ALL, &zerotime, &res) > 0) {
1624                                 if (res != NULL)
1625                                         (void) ldap_msgfree(res);
1626                         }
1627                 }
1628                 (void) mutex_unlock(&cm->lock);
1629         }
1630 }
1631 
1632 /* save error info (rc and ns_ldap_error_t) in the conn_mt */
1633 static void
1634 err2cm(ns_conn_mt_t *cm, int rc, ns_ldap_error_t **errorp) {
1635         ns_ldap_error_t *ep;
1636 
1637         cm->ns_rc = rc;
1638         cm->ns_error = NULL;
1639         if (errorp != NULL && *errorp != NULL) {
1640                 ep = __s_api_copy_error(*errorp);
1641                 if (ep == NULL)
1642                         cm->ns_rc = NS_LDAP_MEMORY;
1643                 else
1644                         cm->ns_error = ep;
1645         }
1646 }
1647 
1648 /* copy error info (rc and ns_ldap_error_t) from conn_mt to conn_user */
1649 static void
1650 err_from_cm(ns_conn_user_t *cu, ns_conn_mt_t *cm) {
1651         ns_ldap_error_t *ep;
1652 
1653         cu->ns_rc = cm->ns_rc;
1654         if (cu->ns_error != NULL)
1655                 (void) __ns_ldap_freeError(&cu->ns_error);
1656         cu->ns_error = NULL;
1657         if (cm->ns_rc != NS_LDAP_SUCCESS && cm->ns_error != NULL) {
1658                 ep = __s_api_copy_error(cm->ns_error);
1659                 if (ep == NULL)
1660                         cu->ns_rc = NS_LDAP_MEMORY;
1661                 else
1662                         cu->ns_error = ep;
1663         }
1664 }
1665 
1666 /* copy error info (rc and ns_ldap_error_t) from caller to conn_user */
1667 static void
1668 err_from_caller(ns_conn_user_t *cu, int rc, ns_ldap_error_t **errorp) {
1669 
1670         cu->ns_rc = rc;
1671         if (errorp != NULL) {
1672                 if (cu->ns_error != NULL)
1673                         (void) __ns_ldap_freeError(&cu->ns_error);
1674                 cu->ns_error = *errorp;
1675                 *errorp = NULL;
1676         } else
1677                 cu->ns_error = NULL;
1678 }
1679 
1680 /*
1681  * remove an MT connection from the connection management when failed to open
1682  *
1683  * Input:
1684  *   cu         : pointer to the conn_user structure
1685  *   rc         : error code
1686  *   errorp     : pointer to pointer to error info (ns_ldap_error_t)
1687  * Output:
1688  *   errorp     : set to NULL, if none NULL cm, callers do not need to free it
1689  */
1690 void
1691 __s_api_conn_mt_remove(ns_conn_user_t *cu, int rc, ns_ldap_error_t **errorp)
1692 {
1693         ns_conn_mgmt_t  *cmg;
1694         ns_conn_mt_t    *cm;
1695         int             free_cm = 0;
1696 
1697         if (cu == NULL || cu->use_mt_conn == B_FALSE)
1698                 return;
1699         if ((cm = cu->conn_mt) == NULL)
1700                 return;
1701         cmg = cu->conn_mgmt;
1702 
1703         (void) mutex_lock(&cmg->lock);
1704         (void) mutex_lock(&cm->lock);
1705         if (cm->state != NS_CONN_MT_CONNECT_ERROR) {
1706                 cm->state = NS_CONN_MT_CONNECT_ERROR;
1707                 cm->ns_rc = rc;
1708                 if (errorp != NULL) {
1709                         cm->ns_error = *errorp;
1710                         *errorp = NULL;
1711                 }
1712         }
1713 
1714         /* all the conn_users share the same error rc and ns_ldap_error_t */
1715         err_from_cm(cu, cm);
1716         /* wake up the waiters if any */
1717         (void) conn_signal(cm);
1718 
1719         del_cu4cm(cu, cm);
1720         cu->conn_mt = NULL;
1721         cu->bad_mt_conn = B_FALSE;
1722         if (cm->cu_cnt == 0) {
1723                 del_cm4cmg(cm, cmg);
1724                 free_cm = 1;
1725         }
1726 
1727         NS_CONN_UNLOCK_AND_FREE(free_cm, cm, cmg);
1728 }
1729 
1730 /*
1731  * check to see if the underlying libldap supports multi-threaded client
1732  * (MT connections)
1733  */
1734 int
1735 __s_api_check_libldap_MT_conn_support(ns_conn_user_t *cu, LDAP *ld,
1736         ns_ldap_error_t **ep)
1737 {
1738         int             rc;
1739         ns_conn_mgmt_t  *cmg;
1740 
1741         /* if no need to check, just return success */
1742         if (cu->conn_mt == NULL || cu->use_mt_conn == B_FALSE)
1743                 return (NS_LDAP_SUCCESS);
1744 
1745         cmg = cu->conn_mgmt;
1746         rc = setup_mt_ld(ld, cmg);
1747 
1748         if (cmg->do_mt_conn == B_FALSE) {
1749                 /*
1750                  * If the conn_mgmt is being shut down, return error.
1751                  * if cmg is usable, cmg->lock will be locked. Otherwise,
1752                  * this function will return with rc NS_LDAP_OP_FAILED.
1753                  */
1754                 NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, ep);
1755                 if (cmg->do_mt_conn == B_FALSE) {
1756                         if (rc < 0)
1757                                 cmg->ldap_mt = B_FALSE;
1758                         else {
1759                                 cmg->ldap_mt = B_TRUE;
1760                                 if (cmg->is_nscd  == B_TRUE ||
1761                                     cmg->is_peruser_nscd == B_TRUE) {
1762                                         cmg->do_mt_conn = B_TRUE;
1763                                         cmg->state = NS_CONN_MGMT_ACTIVE;
1764                                 }
1765                         }
1766                 }
1767                 (void) mutex_unlock(&cmg->lock);
1768         }
1769 
1770         if (rc < 0)
1771                 __s_api_conn_mt_remove(cu, NS_LDAP_NOTFOUND, NULL);
1772         return (NS_LDAP_SUCCESS);
1773 }
1774 
1775 /*
1776  * Close an MT connection.
1777  * Assume cm not null and locked, assume conn_mgmt is also locked.
1778  * Return -1 if error, 1 if the cm should be freed, otherwise 0.
1779  */
1780 static int
1781 close_conn_mt(ns_conn_mt_t *cm, int rc, ns_ldap_error_t **errorp,
1782         ns_conn_user_t *cu)
1783 {
1784         ns_conn_mgmt_t  *cmg = cm->conn_mgmt;
1785         ns_conn_mt_t    *m;
1786         ns_conn_user_t  *u;
1787 
1788         if ((cm->state != NS_CONN_MT_CONNECTED && cm->state !=
1789             NS_CONN_MT_CLOSING) || cmg->cm_head == NULL || cmg->cm_cnt == 0)
1790                 return (-1);
1791 
1792         /* if the conn_mt is not in the MT connection pool, nothing to do */
1793         for (m = cmg->cm_head; m; m = m->next) {
1794                 if (cm == m)
1795                         break;
1796         }
1797         if (m == NULL)
1798                 return (-1);
1799 
1800         if (cm->state == NS_CONN_MT_CONNECTED) { /* first time in here */
1801                 cm->state = NS_CONN_MT_CLOSING;
1802                 /*
1803                  * If more cu exist to consume the error info, copy
1804                  * it to the cm. If the caller calls on behalf of
1805                  * a cu, cu won't be NULL. Check to see if there's
1806                  * more cu that needs the error info. If caller does
1807                  * not have a specific cu attached to it (e.g.,
1808                  * shutdown_all_conn_mt()), cu is NULL, check if at
1809                  * least one cu exists.
1810                  */
1811                 if ((cu != NULL && cm->cu_cnt > 1) ||
1812                     (cu == NULL && cm->cu_cnt > 0)) {
1813                         err2cm(cm, rc, errorp);
1814                         /* wake up waiter (conn_user) if any */
1815                         (void) conn_signal(cm);
1816                 }
1817 
1818                 /* for each conn_user using the conn_mt, set bad_mt_conn flag */
1819                 if (cm->cu_head != NULL) {
1820                         for (u = cm->cu_head; u; u = u->next) {
1821                                 u->bad_mt_conn = B_TRUE;
1822                                 if (cmg->shutting_down == B_FALSE)
1823                                         u->retry = B_TRUE;
1824                         }
1825                 }
1826         }
1827 
1828         /* detach the conn_mt if no more conn_user left */
1829         if ((cu != NULL && cm->cu_cnt == 1) ||
1830             (cu == NULL && cm->cu_cnt ==  0)) {
1831                 del_cm4cmg(cm, cmg);
1832                 cm->detached = B_TRUE;
1833                 return (1);
1834         }
1835 
1836         return (0);
1837 }
1838 
1839 /*
1840  * An MT connection becomes bad, close it and free resources.
1841  * This function is called with a ns_conn_user_t representing
1842  * a user of the MT connection.
1843  *
1844  * Input:
1845  *   cu         : pointer to the conn_user structure
1846  *   rc         : error code
1847  *   errorp     : pointer to pointer to error info (ns_ldap_error_t)
1848  * Output:
1849  *   errorp     : set to NULL (if no error), callers do not need to free it
1850  */
1851 void
1852 __s_api_conn_mt_close(ns_conn_user_t *cu, int rc, ns_ldap_error_t **errorp)
1853 {
1854         ns_conn_mgmt_t  *cmg;
1855         ns_conn_mt_t    *cm;
1856         int             free_cm = 0;
1857 
1858         if (cu == NULL || cu->use_mt_conn == B_FALSE)
1859                 return;
1860 
1861         if (cu->state != NS_CONN_USER_CONNECTED || (cm = cu->conn_mt) == NULL)
1862                 return;
1863         cmg = cu->conn_mgmt;
1864 
1865         (void) mutex_lock(&cmg->lock);
1866         (void) mutex_lock(&cm->lock);
1867 
1868         /* close the MT connection if possible */
1869         free_cm = close_conn_mt(cm, rc, errorp, cu);
1870         if (free_cm == -1) { /* error case */
1871                 (void) mutex_unlock(&cm->lock);
1872                 (void) mutex_unlock(&cmg->lock);
1873                 return;
1874         }
1875 
1876         if (rc != NS_LDAP_SUCCESS) { /* error info passed in, use it */
1877                 err_from_caller(cu, rc, errorp);
1878         } else { /* error not passed in, use those saved in the conn_mt */
1879                 err_from_cm(cu, cm);
1880         }
1881 
1882         /* detach the conn_user from the conn_mt */
1883         del_cu4cm(cu, cm);
1884         cu->conn_mt = NULL;
1885         cu->bad_mt_conn = B_FALSE;
1886         if (cmg->shutting_down == B_FALSE)
1887                 cu->retry = B_TRUE;
1888         NS_CONN_UNLOCK_AND_FREE(free_cm, cm, cmg);
1889 }
1890 
1891 /*
1892  * Close an MT connection when the associated server is known to be
1893  * down. This function is called with a ns_conn_mt_t representing
1894  * the MT connection. That is, the caller is not a conn_user
1895  * thread but rather the procchg thread.
1896  */
1897 static void
1898 close_conn_mt_by_procchg(ns_conn_mt_t *cm, int rc, char *errmsg)
1899 {
1900         ns_conn_mgmt_t  *cmg;
1901         int             free_cm = 0;
1902         ns_ldap_error_t *ep;
1903 
1904         if (cm == NULL)
1905                 return;
1906         cmg = cm->conn_mgmt;
1907 
1908         ep = (ns_ldap_error_t *)calloc(1, sizeof (*ep));
1909         if (ep != NULL) {
1910                 ep->status = rc;
1911                 if (errmsg != NULL)
1912                         ep->message =  strdup(errmsg); /* OK if returns NULL */
1913         }
1914 
1915         (void) mutex_lock(&cmg->lock);
1916         (void) mutex_lock(&cm->lock);
1917 
1918         /* close the MT connection if possible */
1919         free_cm = close_conn_mt(cm, LDAP_SERVER_DOWN, &ep, NULL);
1920         if (free_cm == -1) { /* error case */
1921                 (void) mutex_unlock(&cm->lock);
1922                 (void) mutex_unlock(&cmg->lock);
1923                 return;
1924         }
1925         (void) __ns_ldap_freeError(&ep);
1926 
1927         NS_CONN_UNLOCK_AND_FREE(free_cm, cm, cmg);
1928 }
1929 
1930 /*
1931  * Close an MT connection when there is a better server to connect to.
1932  * Mark the connection as to-be-closed-when-no-one-using so that
1933  * any outstanding ldap operations can run to completion.
1934  * Assume that both the conn_mt and conn_mgmt are locked.
1935  * Return 1 if the conn_mt should be freed.
1936  */
1937 static int
1938 close_conn_mt_when_nouser(ns_conn_mt_t *cm)
1939 {
1940         int             free_cm = 0;
1941 
1942         if (cm->cu_cnt == 0) {
1943                 del_cm4cmg(cm, cm->conn_mgmt);
1944                 free_cm = 1;
1945         } else {
1946                 cm->close_when_nouser = B_TRUE;
1947         }
1948 
1949         return (free_cm);
1950 }
1951 
1952 /*
1953  * Retrieve the configured preferred server list.
1954  * This function locked the conn_mgmt and does not
1955  * unlock at exit.
1956  */
1957 static void
1958 get_preferred_servers(boolean_t lock, boolean_t reload, ns_conn_mgmt_t *cmg)
1959 {
1960         ns_ldap_error_t *errorp = NULL;
1961         void            **pservers = NULL;
1962 
1963         if (lock == B_TRUE)
1964                 (void) mutex_lock(&cmg->lock);
1965 
1966         /* if already done, and no reload, then return */
1967         if (cmg->pservers_loaded == B_TRUE && reload == B_FALSE)
1968                 return;
1969 
1970         if (cmg->pservers != NULL) {
1971                 (void) __ns_ldap_freeParam((void ***)&cmg->pservers);
1972                 cmg->pservers = NULL;
1973         }
1974 
1975         if (__ns_ldap_getParam(NS_LDAP_SERVER_PREF_P,
1976             &pservers, &errorp) == NS_LDAP_SUCCESS) {
1977                 cmg->pservers = (char **)pservers;
1978                 cmg->pservers_loaded = B_TRUE;
1979         } else {
1980                 (void) __ns_ldap_freeError(&errorp);
1981                 (void) __ns_ldap_freeParam(&pservers);
1982         }
1983 }
1984 
1985 /*
1986  * This function handles the config or server status change notification
1987  * from the ldap_cachemgr.
1988  */
1989 static ns_conn_mgmt_t *
1990 proc_server_change(ns_server_status_change_t *chg, ns_conn_mgmt_t  *cmg)
1991 {
1992         int             cnt, i, j, k, n;
1993         boolean_t       loop = B_TRUE;
1994         boolean_t       cmg_locked = B_FALSE;
1995         char            *s;
1996         ns_conn_mt_t    *cm;
1997         ns_conn_mgmt_t  *ocmg;
1998 
1999         /* if config changed, reload the configuration */
2000         if (chg->config_changed == B_TRUE) {
2001                 /* reload the conn_mgmt and Native LDAP config */
2002                 ocmg = access_conn_mgmt(NS_CONN_MGMT_OP_RELOAD_CONFIG);
2003                 shutdown_all_conn_mt(ocmg);
2004                 /* release the one obtained from access_conn_mgmt(RELOAD) */
2005                 (void) release_conn_mgmt(ocmg, B_FALSE);
2006                 /* release the one obtained when ocmg was created */
2007                 (void) release_conn_mgmt(ocmg, B_FALSE);
2008                 return (ocmg);
2009         }
2010 
2011         if ((cnt = chg->num_server) == 0)
2012                 return (cmg);
2013 
2014         /* handle down servers first */
2015         for (i = 0; i < cnt; i++) {
2016 
2017                 if (chg->changes[i] != NS_SERVER_DOWN)
2018                         continue;
2019                 s = chg->servers[i];
2020 
2021                 /*
2022                  * look for a CONNECTED MT connection using
2023                  * the same server s, and close it
2024                  */
2025                 while (loop) {
2026                         if (cmg_locked == B_FALSE) {
2027                                 (void) mutex_lock(&cmg->lock);
2028                                 cmg_locked = B_TRUE;
2029                         }
2030                         for (cm = cmg->cm_head; cm; cm = cm->next) {
2031                                 (void) mutex_lock(&cm->lock);
2032 
2033                                 if (cm->state == NS_CONN_MT_CONNECTED &&
2034                                     cm->conn != NULL &&
2035                                     strcasecmp(cm->conn->serverAddr, s) == 0) {
2036                                         (void) mutex_unlock(&cm->lock);
2037                                         break;
2038                                 }
2039 
2040                                 (void) mutex_unlock(&cm->lock);
2041                         }
2042                         if (cm != NULL) {
2043                                 (void) mutex_unlock(&cmg->lock);
2044                                 cmg_locked = B_FALSE;
2045                                 close_conn_mt_by_procchg(cm, LDAP_SERVER_DOWN,
2046                                     NS_CONN_MSG_DOWN_FROM_CACHEMGR);
2047                                 /*
2048                                  * Process the next cm using server s.
2049                                  * Start from the head of the cm linked
2050                                  * list again, as the cm list may change
2051                                  * after close_conn_mt_by_procchg() is done.
2052                                  */
2053                                 continue;
2054                         }
2055 
2056                         /*
2057                          * No (more) MT connection using the down server s.
2058                          * Process the next server on the list.
2059                          */
2060                         break;
2061                 } /* while loop */
2062         }
2063 
2064         /*
2065          * Next handle servers whose status changed to up.
2066          * Get the preferred server list first if not done yet.
2067          * get_preferred_servers() leaves conn_mgmt locked.
2068          */
2069         get_preferred_servers(cmg_locked == B_FALSE ? B_TRUE : B_FALSE,
2070             B_FALSE, cmg);
2071         cmg_locked = B_TRUE;
2072         /*
2073          * if no preferred server configured, we don't switch MT connection
2074          * to a more preferred server (i.e., fallback), so just return
2075          */
2076         if (cmg->pservers == NULL) {
2077                 (void) mutex_unlock(&cmg->lock);
2078                 return (cmg);
2079         }
2080 
2081         /* for each server that is up now */
2082         for (i = 0; i < cnt; i++) {
2083                 if (chg->changes[i] != NS_SERVER_UP)
2084                         continue;
2085                 s = chg->servers[i];
2086 
2087                 /*
2088                  * look for a CONNECTED MT connection which uses
2089                  * a server less preferred than s, and treat it
2090                  * as 'fallback needed' by calling
2091                  * close_conn_mt_when_nouser()
2092                  */
2093                 k = -1;
2094                 loop = B_TRUE;
2095                 while (loop) {
2096                         if (cmg_locked == B_FALSE) {
2097                                 (void) mutex_lock(&cmg->lock);
2098                                 cmg_locked = B_TRUE;
2099                         }
2100 
2101                         /* Is s a preferred server ? */
2102                         if (k == -1) {
2103                                 for (j = 0; cmg->pservers[j] != NULL; j++) {
2104                                         if (strcasecmp(cmg->pservers[j],
2105                                             s) == 0) {
2106                                                 k = j;
2107                                                 break;
2108                                         }
2109                                 }
2110                         }
2111                         /* skip s if not a preferred server */
2112                         if (k == -1) {
2113                                 break;
2114                         }
2115 
2116                         /* check each MT connection */
2117                         for (cm = cmg->cm_head; cm; cm = cm->next) {
2118                                 (void) mutex_lock(&cm->lock);
2119                                 /*
2120                                  * Find an MT connection that is connected and
2121                                  * not marked, but leave WRITE or REFERRAL
2122                                  * connections alone, since fallback does not
2123                                  * make sense for them.
2124                                  */
2125                                 if (cm->state == NS_CONN_MT_CONNECTED &&
2126                                     cm->close_when_nouser == B_FALSE &&
2127                                     cm->conn != NULL && cm->opened_for !=
2128                                     NS_CONN_USER_WRITE &&
2129                                     cm->referral == B_FALSE) {
2130                                         n = -1;
2131                                         /*
2132                                          * j < k ??? should we close
2133                                          * an active MT that is using s ?
2134                                          * ie could s went down and up
2135                                          * again, but cm is bound prior to
2136                                          * the down ? Play safe here,
2137                                          * and check j <= k.
2138                                          */
2139                                         for (j = 0; j <= k; j++) {
2140                                                 if (strcasecmp(
2141                                                     cm->conn->serverAddr,
2142                                                     cmg->pservers[j]) == 0) {
2143                                                         n = j;
2144                                                         break;
2145                                                 }
2146                                         }
2147                                         /*
2148                                          * s is preferred, if its location
2149                                          * in the preferred server list is
2150                                          * ahead of that of the server
2151                                          * used by the cm (i.e., no match
2152                                          * found before s)
2153                                          */
2154                                         if (n == -1) { /* s is preferred */
2155                                                 int fr = 0;
2156                                                 fr = close_conn_mt_when_nouser(
2157                                                     cm);
2158                                                 NS_CONN_UNLOCK_AND_FREE(fr,
2159                                                     cm, cmg);
2160                                                 cmg_locked = B_FALSE;
2161                                                 /*
2162                                                  * break, not continue,
2163                                                  * because we need to
2164                                                  * check the entire cm
2165                                                  * list again. The call
2166                                                  * above may change the
2167                                                  * cm list.
2168                                                  */
2169                                                 break;
2170                                         }
2171                                 }
2172                                 (void) mutex_unlock(&cm->lock);
2173                         }
2174                         /* if no (more) cm using s, check next server */
2175                         if (cm == NULL)
2176                                 loop = B_FALSE;
2177                 } /* while loop */
2178         }
2179         if (cmg_locked == B_TRUE)
2180                 (void) mutex_unlock(&cmg->lock);
2181         return (cmg);
2182 }
2183 
2184 /* Shut down all MT connection managed by the connection management */
2185 void
2186 shutdown_all_conn_mt(ns_conn_mgmt_t  *cmg)
2187 {
2188         ns_ldap_error_t *ep;
2189         ns_conn_mt_t    *cm;
2190         int             free_cm = 0;
2191         boolean_t       done = B_FALSE;
2192 
2193         ep = (ns_ldap_error_t *)calloc(1, sizeof (*ep));
2194         if (ep != NULL) { /* if NULL, not a problem */
2195                 /* OK if returns NULL */
2196                 ep->message = strdup(NS_CONN_MSG_SHUTDOWN_RELOADED);
2197         }
2198 
2199         (void) mutex_lock(&cmg->lock);
2200         while (cmg->cm_head != NULL && done == B_FALSE) {
2201                 for (cm = cmg->cm_head; cm; cm = cm->next) {
2202                         (void) mutex_lock(&cm->lock);
2203                         if (cm->next == NULL)
2204                                 done = B_TRUE;
2205                         /* shut down each conn_mt, ignore errors */
2206                         free_cm = close_conn_mt(cm, LDAP_OTHER, &ep, NULL);
2207                         (void) mutex_unlock(&cm->lock);
2208                         if (free_cm == 1) {
2209                                 (void) free_conn_mt(cm, 0);
2210                                 /*
2211                                  * conn_mt may change, so start from
2212                                  * top of list again
2213                                  */
2214                                 break;
2215                         }
2216                 }
2217         }
2218         (void) mutex_unlock(&cmg->lock);
2219         (void) __ns_ldap_freeError(&ep);
2220 }
2221 
2222 /* free all the resources used by the connection management */
2223 void
2224 __s_api_shutdown_conn_mgmt()
2225 {
2226         ns_conn_mgmt_t  *cmg;
2227 
2228         cmg = access_conn_mgmt(NS_CONN_MGMT_OP_SHUTDOWN);
2229         if (cmg == NULL) /* already being SHUT done */
2230                 return;
2231 
2232         (void) shutdown_all_conn_mt(cmg);
2233         (void) release_conn_mgmt(cmg, B_FALSE);
2234 
2235         /* then destroy the conn_mgmt */
2236         (void) release_conn_mgmt(cmg, B_FALSE);
2237 }
2238 
2239 
2240 /*
2241  * Reinitialize the libsldap connection management after
2242  * a new native LDAP configuration is received.
2243  */
2244 void
2245 __s_api_reinit_conn_mgmt_new_config(ns_config_t *new_cfg)
2246 {
2247         ns_conn_mgmt_t  *cmg;
2248         ns_conn_mgmt_t  *ocmg;
2249 
2250         cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
2251         if (cmg == NULL)
2252                 return;
2253         if (cmg->config == new_cfg || cmg->state == NS_CONN_MGMT_DETACHED) {
2254                 (void) release_conn_mgmt(cmg, B_FALSE);
2255                 return;
2256         }
2257 
2258         /* reload the conn_mgmt and native LDAP config */
2259         ocmg = access_conn_mgmt(NS_CONN_MGMT_OP_NEW_CONFIG);
2260         if (ocmg == cmg)
2261                 shutdown_all_conn_mt(ocmg);
2262         /* release the one obtained from access_conn_mgmt(RELOAD) */
2263         (void) release_conn_mgmt(ocmg, B_FALSE);
2264         /* release the one obtained when ocmg was created */
2265         (void) release_conn_mgmt(ocmg, B_FALSE);
2266         /* release the one obtained when this function is entered */
2267         (void) release_conn_mgmt(cmg, B_FALSE);
2268 }
2269 
2270 /*
2271  * Prepare to retry ldap search operation if needed.
2272  * Return 1 if retry is needed, otherwise 0.
2273  * If first time in, return 1. If not, return 1 if:
2274  * - not a NS_CONN_USER_GETENT conn_user AND
2275  * - have not retried 3 times yet AND
2276  * - previous search failed AND
2277  * - the retry flag is set in the ns_conn_user_t or config was reloaded
2278  */
2279 int
2280 __s_api_setup_retry_search(ns_conn_user_t **conn_user,
2281         ns_conn_user_type_t type, int *try_cnt, int *rc,
2282         ns_ldap_error_t **errorp)
2283 {
2284         boolean_t       retry;
2285         ns_conn_user_t  *cu = *conn_user;
2286         ns_conn_mgmt_t  *cmg;
2287 
2288         if (*try_cnt > 0 && cu != NULL) {
2289                 /*
2290                  * if called from firstEntry(), keep conn_mt for
2291                  * the subsequent getnext requests
2292                  */
2293                 if (cu->type == NS_CONN_USER_GETENT && *rc == NS_LDAP_SUCCESS)
2294                         return (0);
2295                 cmg = cu->conn_mgmt;
2296                 retry = cu->retry;
2297                 if (cu->conn_mt != NULL)
2298                         __s_api_conn_mt_return(cu);
2299                 if (cmg != NULL && cmg->cfg_reloaded == B_TRUE)
2300                         retry = B_TRUE;
2301                 __s_api_conn_user_free(cu);
2302                 *conn_user = NULL;
2303 
2304                 if (*rc == NS_LDAP_SUCCESS || retry != B_TRUE)
2305                         return (0);
2306         }
2307 
2308         *try_cnt = *try_cnt + 1;
2309         if (*try_cnt > NS_LIST_TRY_MAX)
2310                 return (0);
2311 
2312         *conn_user = __s_api_conn_user_init(type, NULL, B_FALSE);
2313         if (*conn_user == NULL) {
2314                 if (*try_cnt == 1) { /* first call before any retry */
2315                         *rc = NS_LDAP_MEMORY;
2316                         *errorp = NULL;
2317                 }
2318                 /* for 1+ try, use previous rc and errorp */
2319                 return (0);
2320         }
2321 
2322         /* free ldap_error_t from previous search */
2323         if (*try_cnt > 1 && rc != NS_LDAP_SUCCESS && *errorp != NULL)
2324                 (void) __ns_ldap_freeError(errorp);
2325 
2326         return (1);
2327 }
2328 
2329 /* prepare to get the next entry for an enumeration */
2330 int
2331 __s_api_setup_getnext(ns_conn_user_t *cu, int *ns_err,
2332         ns_ldap_error_t **errorp)
2333 {
2334         int rc;
2335         ns_conn_mgmt_t  *cmg;
2336 
2337         /*
2338          * if using an MT connection, ensure the thread-specific data are set,
2339          * but if the MT connection is no longer good, return the error saved.
2340          */
2341         if (cu->conn_mt != NULL && (cmg = cu->conn_mgmt) != NULL) {
2342 
2343                 if (cu->bad_mt_conn ==  B_TRUE) {
2344                         __s_api_conn_mt_close(cu, 0, NULL);
2345                         *ns_err = cu->ns_rc;
2346                         *errorp = cu->ns_error;
2347                         cu->ns_error = NULL;
2348                         return (*ns_err);
2349                 }
2350 
2351                 rc = conn_tsd_check(cmg);
2352                 if (rc != NS_LDAP_SUCCESS) {
2353                         *errorp = NULL;
2354                         return (rc);
2355                 }
2356         }
2357 
2358         return (NS_LDAP_SUCCESS);
2359 }
2360 
2361 /* wait for an MT connection to become available */
2362 static int
2363 conn_wait(ns_conn_mt_t *conn_mt, ns_conn_user_t *conn_user)
2364 {
2365         ns_conn_waiter_t        mywait;
2366         ns_conn_waiter_t        *head = &conn_mt->waiter;
2367 
2368         (void) cond_init(&(mywait.waitcv), USYNC_THREAD, 0);
2369         mywait.key = conn_user;
2370         mywait.signaled = 0;
2371         mywait.next = head->next;
2372         mywait.prev = head;
2373         if (mywait.next)
2374                 mywait.next->prev = &mywait;
2375         head->next = &mywait;
2376         atomic_inc_uint(&conn_mt->waiter_cnt);
2377 
2378         while (!mywait.signaled)
2379                 (void) cond_wait(&(mywait.waitcv), &conn_mt->lock);
2380         if (mywait.prev)
2381                 mywait.prev->next = mywait.next;
2382         if (mywait.next)
2383                 mywait.next->prev = mywait.prev;
2384         return (0);
2385 }
2386 
2387 /* signal that an MT connection is now available */
2388 static int
2389 conn_signal(ns_conn_mt_t *conn_mt)
2390 {
2391         int                     c = 0;
2392         ns_conn_waiter_t        *head = &conn_mt->waiter;
2393         ns_conn_waiter_t        *tmp = head->next;
2394 
2395         while (tmp) {
2396                 (void) cond_signal(&(tmp->waitcv));
2397                 tmp->signaled = 1;
2398                 atomic_dec_uint(&conn_mt->waiter_cnt);
2399                 c++;
2400                 tmp = tmp->next;
2401         }
2402 
2403         return (c);
2404 }
2405 
2406 /*
2407  * wait and process the server status and/or config change notification
2408  * from ldap_cachemgr
2409  */
2410 static void *
2411 get_server_change(void *arg)
2412 {
2413         union {
2414                 ldap_data_t     s_d;
2415                 char            s_b[DOORBUFFERSIZE];
2416         } space;
2417         ldap_data_t     *sptr = &space.s_d;
2418         int             ndata;
2419         int             adata;
2420         char            *ptr;
2421         int             ds_cnt;
2422         int             door_rc;
2423         int             which;
2424         int             retry = 0;
2425         boolean_t       loop = B_TRUE;
2426         char            *c, *oc;
2427         int             dslen = strlen(DOORLINESEP);
2428         char            dsep = DOORLINESEP_CHR;
2429         char            chg_data[DOORBUFFERSIZE];
2430         char            **servers = NULL;
2431         boolean_t       getchg_not_supported = B_FALSE;
2432         ns_conn_mgmt_t  *ocmg = (ns_conn_mgmt_t *)arg;
2433         ns_conn_mgmt_t  *cmg;
2434         ns_server_status_t *status = NULL;
2435         ns_server_status_change_t chg = { 0 };
2436         ldap_get_change_out_t *get_chg;
2437         ldap_get_chg_cookie_t cookie;
2438         ldap_get_chg_cookie_t new_cookie;
2439 
2440         cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
2441         if (cmg != ocmg)
2442                 thr_exit(NULL);
2443         /* cmg is locked before called */
2444         cmg->procchg_tid = thr_self();
2445 
2446         /* make sure the thread specific data are set */
2447         (void) conn_tsd_setup(cmg);
2448         cookie = cmg->cfg_cookie;
2449 
2450         while (loop) {
2451 
2452                 if (chg.servers != NULL)
2453                         free(chg.servers);
2454                 if (chg.changes != NULL)
2455                         free(chg.changes);
2456                 if (sptr != &space.s_d)
2457                         (void) munmap((char *)sptr, sizeof (space));
2458 
2459                 /*
2460                  * If the attached conn_mgmt has been deleted,
2461                  * then exit. The new conn_mgmt will starts it
2462                  * own monitor thread later. If libsldap is being
2463                  * unloaded or configuration reloaded, OR
2464                  * ldap_cachemgr rejected the GETSTATUSCHANGE door
2465                  * call, then exit as well.
2466                  */
2467                 if (cmg == NULL || cmg->state == NS_CONN_MGMT_DETACHED ||
2468                     getchg_not_supported == B_TRUE) {
2469 
2470                         if (cmg != NULL) {
2471                                 cmg->procchg_started = B_FALSE;
2472                                 (void) release_conn_mgmt(cmg, B_FALSE);
2473                         }
2474 
2475                         conn_tsd_free();
2476                         thr_exit(NULL);
2477                 }
2478 
2479                 (void) memset(space.s_b, 0, DOORBUFFERSIZE);
2480                 (void) memset(&chg, 0, sizeof (chg));
2481                 adata = sizeof (ldap_call_t) + 1;
2482                 ndata = sizeof (space);
2483                 space.s_d.ldap_call.ldap_callnumber = GETSTATUSCHANGE;
2484                 space.s_d.ldap_call.ldap_u.get_change.op =
2485                     NS_STATUS_CHANGE_OP_START;
2486                 space.s_d.ldap_call.ldap_u.get_change.cookie = cookie;
2487                 sptr = &space.s_d;
2488                 door_rc = __ns_ldap_trydoorcall_getfd();
2489                 cmg->procchg_door_call = B_TRUE;
2490                 if (release_conn_mgmt(cmg, B_FALSE) == NULL) {
2491                         conn_tsd_free();
2492                         thr_exit(NULL);
2493                 }
2494 
2495                 if (door_rc == NS_CACHE_SUCCESS)
2496                         door_rc = __ns_ldap_trydoorcall_send(&sptr, &ndata,
2497                             &adata);
2498 
2499                 /*
2500                  * Check and see if the conn_mgmt is still current.
2501                  * If not, no need to continue.
2502                  */
2503                 cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
2504                 if (cmg != NULL)
2505                         cmg->procchg_door_call = B_FALSE;
2506                 if (cmg != ocmg) {
2507                         if (cmg != NULL) {
2508                                 cmg->procchg_started = B_FALSE;
2509                                 (void) release_conn_mgmt(cmg, B_FALSE);
2510                         }
2511                         conn_tsd_free();
2512                         thr_exit(NULL);
2513                 }
2514 
2515                 if (door_rc != NS_CACHE_SUCCESS) {
2516                         if (door_rc == NS_CACHE_NOSERVER) {
2517                                 if (retry++ > 10)
2518                                         getchg_not_supported = B_TRUE;
2519                                 else {
2520                                         /*
2521                                          * ldap_cachemgr may be down, give
2522                                          * it time to restart
2523                                          */
2524                                         (void) sleep(2);
2525                                 }
2526                         } else if (door_rc == NS_CACHE_NOTFOUND)
2527                                 getchg_not_supported = B_TRUE;
2528                         continue;
2529                 } else
2530                         retry = 0;
2531 
2532                 /* copy info from door call return structure */
2533                 get_chg =  &sptr->ldap_ret.ldap_u.changes;
2534                 ptr = get_chg->data;
2535                 /* configuration change ? */
2536                 if (get_chg->type == NS_STATUS_CHANGE_TYPE_CONFIG) {
2537                         chg.config_changed = B_TRUE;
2538                         cmg = proc_server_change(&chg, cmg);
2539                         continue;
2540                 }
2541 
2542                 /* server status changes ? */
2543                 if (get_chg->type == NS_STATUS_CHANGE_TYPE_SERVER) {
2544                         /*
2545                          * first check cookies, if don't match, config
2546                          * has changed
2547                          */
2548                         new_cookie = get_chg->cookie;
2549                         if (new_cookie.mgr_pid != cookie.mgr_pid ||
2550                             new_cookie.seq_num != cookie.seq_num) {
2551                                 chg.config_changed = B_TRUE;
2552                                 cmg = proc_server_change(&chg, cmg);
2553                                 continue;
2554                         }
2555 
2556                         (void) strlcpy(chg_data, ptr, sizeof (chg_data));
2557                         chg.num_server = get_chg->server_count;
2558 
2559                         servers = (char **)calloc(chg.num_server,
2560                             sizeof (char *));
2561                         if (servers == NULL) {
2562                                 syslog(LOG_INFO, NS_CONN_MSG_MEMORY_ERROR);
2563                                 continue;
2564                         }
2565                         status = (ns_server_status_t *)calloc(chg.num_server,
2566                             sizeof (int));
2567                         if (status == NULL) {
2568                                 syslog(LOG_INFO, NS_CONN_MSG_MEMORY_ERROR);
2569                                 free(servers);
2570                                 continue;
2571                         }
2572                         ds_cnt = 0;
2573                         which = 0;
2574                         oc = ptr;
2575                         for (c = ptr; which != 2; c++) {
2576                                 /* look for DOORLINESEP or end of string */
2577                                 if (*c != dsep && *c != '\0')
2578                                         continue;
2579                                 if (*c == dsep) { /* DOORLINESEP */
2580                                         *c = '\0'; /* current value */
2581                                         c += dslen; /* skip to next value */
2582                                 }
2583                                 if (which == 0) { /* get server info */
2584                                         servers[ds_cnt] = oc;
2585                                         oc = c;
2586                                         which = 1; /* get status next */
2587                                         continue;
2588                                 }
2589                                 /* which == 1, get up/down status */
2590                                 if (strcmp(NS_SERVER_CHANGE_UP, oc) == 0) {
2591                                         status[ds_cnt] = NS_SERVER_UP;
2592                                 } else if (strcmp(NS_SERVER_CHANGE_DOWN,
2593                                     oc) == 0)
2594                                         status[ds_cnt] = NS_SERVER_DOWN;
2595                                 else {
2596                                         syslog(LOG_INFO,
2597                                             NS_CONN_MSG_BAD_CACHEMGR_DATA);
2598                                         continue;
2599                                 }
2600                                 oc = c;
2601                                 ds_cnt++;
2602                                 if (*c == '\0')
2603                                         which = 2; /* exit the loop */
2604                                 else
2605                                         which = 0; /* get server info next */
2606                         }
2607                         chg.servers = servers;
2608                         chg.changes = status;
2609                         cmg = proc_server_change(&chg, cmg);
2610                         continue;
2611                 }
2612         }
2613 
2614         return (NULL);
2615 }
2616 
2617 /* start the thread handling the change notification from ldap_cachemgr */
2618 static void
2619 start_thread(ns_conn_mgmt_t *cmg) {
2620 
2621         int             errnum;
2622 
2623         /*
2624          * start a thread to get and process config and server status changes
2625          */
2626         if (thr_create(NULL, NULL, get_server_change,
2627             (void *)cmg, THR_DETACHED, NULL) != 0) {
2628                 errnum = errno;
2629                 syslog(LOG_WARNING, NS_CONN_MSG_NO_PROCCHG_THREAD,
2630                     strerror(errnum));
2631         }
2632 }