Print this page
    
OS-5192 need faster clock_gettime
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Joshua M. Clulow <jmc@joyent.com>
Reviewed by: Ryan Zezeski <ryan@zinascii.com>
OS-2844 lx brand should support 64-bit user-land
OS-3280 need a way to specify the root of a native system in the lx brand
OS-3279 lx brand should allow delegated datasets
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
    
      
        | Split | 
	Close | 
      
      | Expand all | 
      | Collapse all | 
    
    
          --- old/usr/src/lib/libc/port/threads/thr.c
          +++ new/usr/src/lib/libc/port/threads/thr.c
   1    1  /*
   2    2   * CDDL HEADER START
   3    3   *
   4    4   * The contents of this file are subject to the terms of the
   5    5   * Common Development and Distribution License (the "License").
   6    6   * You may not use this file except in compliance with the License.
   7    7   *
   8    8   * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9    9   * or http://www.opensolaris.org/os/licensing.
  10   10   * See the License for the specific language governing permissions
  11   11   * and limitations under the License.
  12   12   *
  13   13   * When distributing Covered Code, include this CDDL HEADER in each
  14   14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15   15   * If applicable, add the following below this CDDL HEADER, with the
  16   16   * fields enclosed by brackets "[]" replaced with your own identifying
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
  21   21  
  22   22  /*
  23   23   * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24   24   */
  25   25  /*
  26   26   * Copyright 2016 Joyent, Inc.
  27   27   */
  28   28  
  29   29  #include "lint.h"
  30   30  #include "thr_uberdata.h"
  31   31  #include <pthread.h>
  32   32  #include <procfs.h>
  33   33  #include <sys/uio.h>
  34   34  #include <ctype.h>
  35   35  #include "libc.h"
  36   36  
  37   37  /*
  38   38   * These symbols should not be exported from libc, but
  39   39   * /lib/libm.so.2 references _thr_main.  libm needs to be fixed.
  40   40   * Also, some older versions of the Studio compiler/debugger
  41   41   * components reference them.  These need to be fixed, too.
  42   42   */
  43   43  #pragma weak _thr_main = thr_main
  44   44  #pragma weak _thr_create = thr_create
  45   45  #pragma weak _thr_join = thr_join
  46   46  #pragma weak _thr_self = thr_self
  47   47  
  48   48  #undef errno
  49   49  extern int errno;
  50   50  
  51   51  /*
  52   52   * Between Solaris 2.5 and Solaris 9, __threaded was used to indicate
  53   53   * "we are linked with libthread".  The Sun Workshop 6 update 1 compilation
  54   54   * system used it illegally (it is a consolidation private symbol).
  55   55   * To accommodate this and possibly other abusers of the symbol,
  56   56   * we make it always equal to 1 now that libthread has been folded
  57   57   * into libc.  The new __libc_threaded symbol is used to indicate
  58   58   * the new meaning, "more than one thread exists".
  59   59   */
  60   60  int __threaded = 1;             /* always equal to 1 */
  61   61  int __libc_threaded = 0;        /* zero until first thr_create() */
  62   62  
  63   63  /*
  64   64   * thr_concurrency and pthread_concurrency are not used by the library.
  65   65   * They exist solely to hold and return the values set by calls to
  66   66   * thr_setconcurrency() and pthread_setconcurrency().
  67   67   * Because thr_concurrency is affected by the THR_NEW_LWP flag
  68   68   * to thr_create(), thr_concurrency is protected by link_lock.
  69   69   */
  70   70  static  int     thr_concurrency = 1;
  71   71  static  int     pthread_concurrency;
  72   72  
  73   73  #define HASHTBLSZ       1024    /* must be a power of two */
  74   74  #define TIDHASH(tid, udp)       (tid & (udp)->hash_mask)
  75   75  
  76   76  /* initial allocation, just enough for one lwp */
  77   77  #pragma align 64(init_hash_table)
  78   78  thr_hash_table_t init_hash_table[1] = {
  79   79          { DEFAULTMUTEX, DEFAULTCV, NULL },
  80   80  };
  81   81  
  82   82  extern const Lc_interface rtld_funcs[];
  83   83  
  84   84  /*
  85   85   * The weak version is known to libc_db and mdb.
  86   86   */
  87   87  #pragma weak _uberdata = __uberdata
  88   88  uberdata_t __uberdata = {
  89   89          { DEFAULTMUTEX, NULL, 0 },      /* link_lock */
  90   90          { RECURSIVEMUTEX, NULL, 0 },    /* ld_lock */
  91   91          { RECURSIVEMUTEX, NULL, 0 },    /* fork_lock */
  92   92          { RECURSIVEMUTEX, NULL, 0 },    /* atfork_lock */
  93   93          { RECURSIVEMUTEX, NULL, 0 },    /* callout_lock */
  94   94          { DEFAULTMUTEX, NULL, 0 },      /* tdb_hash_lock */
  95   95          { 0, },                         /* tdb_hash_lock_stats */
  96   96          { { 0 }, },                     /* siguaction[NSIG] */
  97   97          {{ DEFAULTMUTEX, NULL, 0 },             /* bucket[NBUCKETS] */
  98   98          { DEFAULTMUTEX, NULL, 0 },
  99   99          { DEFAULTMUTEX, NULL, 0 },
 100  100          { DEFAULTMUTEX, NULL, 0 },
 101  101          { DEFAULTMUTEX, NULL, 0 },
 102  102          { DEFAULTMUTEX, NULL, 0 },
 103  103          { DEFAULTMUTEX, NULL, 0 },
 104  104          { DEFAULTMUTEX, NULL, 0 },
 105  105          { DEFAULTMUTEX, NULL, 0 },
 106  106          { DEFAULTMUTEX, NULL, 0 }},
 107  107          { RECURSIVEMUTEX, NULL, NULL },         /* atexit_root */
 108  108          { RECURSIVEMUTEX, NULL },               /* quickexit_root */
 109  109          { DEFAULTMUTEX, 0, 0, NULL },           /* tsd_metadata */
 110  110          { DEFAULTMUTEX, {0, 0}, {0, 0} },       /* tls_metadata */
 111  111          0,                      /* primary_map */
 112  112          0,                      /* bucket_init */
 113  113          0,                      /* pad[0] */
 114  114          0,                      /* pad[1] */
 115  115          { 0 },                  /* uberflags */
 116  116          NULL,                   /* queue_head */
 117  117          init_hash_table,        /* thr_hash_table */
  
    | 
      ↓ open down ↓ | 
    117 lines elided | 
    
      ↑ open up ↑ | 
  
 118  118          1,                      /* hash_size: size of the hash table */
 119  119          0,                      /* hash_mask: hash_size - 1 */
 120  120          NULL,                   /* ulwp_one */
 121  121          NULL,                   /* all_lwps */
 122  122          NULL,                   /* all_zombies */
 123  123          0,                      /* nthreads */
 124  124          0,                      /* nzombies */
 125  125          0,                      /* ndaemons */
 126  126          0,                      /* pid */
 127  127          sigacthandler,          /* sigacthandler */
      128 +        __setcontext,           /* setctxt */
 128  129          NULL,                   /* lwp_stacks */
 129  130          NULL,                   /* lwp_laststack */
 130  131          0,                      /* nfreestack */
 131  132          10,                     /* thread_stack_cache */
 132  133          NULL,                   /* ulwp_freelist */
 133  134          NULL,                   /* ulwp_lastfree */
 134  135          NULL,                   /* ulwp_replace_free */
 135  136          NULL,                   /* ulwp_replace_last */
 136  137          NULL,                   /* atforklist */
 137  138          NULL,                   /* robustlocks */
 138  139          NULL,                   /* robustlist */
 139  140          NULL,                   /* progname */
      141 +        NULL,                   /* ub_broot */
      142 +        NULL,                   /* ub_comm_page */
 140  143          NULL,                   /* __tdb_bootstrap */
 141  144          {                       /* tdb */
 142  145                  NULL,           /* tdb_sync_addr_hash */
 143  146                  0,              /* tdb_register_count */
 144  147                  0,              /* tdb_hash_alloc_failed */
 145  148                  NULL,           /* tdb_sync_addr_free */
 146  149                  NULL,           /* tdb_sync_addr_last */
 147  150                  0,              /* tdb_sync_alloc */
 148  151                  { 0, 0 },       /* tdb_ev_global_mask */
 149  152                  tdb_events,     /* tdb_events array */
 150  153          },
 151  154  };
 152  155  
 153  156  /*
 154  157   * The weak version is known to libc_db and mdb.
 155  158   */
 156  159  #pragma weak _tdb_bootstrap = __tdb_bootstrap
 157  160  uberdata_t **__tdb_bootstrap = NULL;
 158  161  
 159  162  int     thread_queue_fifo = 4;
 160  163  int     thread_queue_dump = 0;
 161  164  int     thread_cond_wait_defer = 0;
 162  165  int     thread_error_detection = 0;
 163  166  int     thread_async_safe = 0;
 164  167  int     thread_stack_cache = 10;
 165  168  int     thread_door_noreserve = 0;
 166  169  int     thread_locks_misaligned = 0;
 167  170  
 168  171  static  ulwp_t  *ulwp_alloc(void);
 169  172  static  void    ulwp_free(ulwp_t *);
 170  173  
 171  174  /*
 172  175   * Insert the lwp into the hash table.
 173  176   */
 174  177  void
 175  178  hash_in_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
 176  179  {
 177  180          ulwp->ul_hash = udp->thr_hash_table[ix].hash_bucket;
 178  181          udp->thr_hash_table[ix].hash_bucket = ulwp;
 179  182          ulwp->ul_ix = ix;
 180  183  }
 181  184  
 182  185  void
 183  186  hash_in(ulwp_t *ulwp, uberdata_t *udp)
 184  187  {
 185  188          int ix = TIDHASH(ulwp->ul_lwpid, udp);
 186  189          mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
 187  190  
 188  191          lmutex_lock(mp);
 189  192          hash_in_unlocked(ulwp, ix, udp);
 190  193          lmutex_unlock(mp);
 191  194  }
 192  195  
 193  196  /*
 194  197   * Delete the lwp from the hash table.
 195  198   */
 196  199  void
 197  200  hash_out_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
 198  201  {
 199  202          ulwp_t **ulwpp;
 200  203  
 201  204          for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
 202  205              ulwp != *ulwpp;
 203  206              ulwpp = &(*ulwpp)->ul_hash)
 204  207                  ;
 205  208          *ulwpp = ulwp->ul_hash;
 206  209          ulwp->ul_hash = NULL;
 207  210          ulwp->ul_ix = -1;
 208  211  }
 209  212  
 210  213  void
 211  214  hash_out(ulwp_t *ulwp, uberdata_t *udp)
 212  215  {
 213  216          int ix;
 214  217  
 215  218          if ((ix = ulwp->ul_ix) >= 0) {
 216  219                  mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
 217  220  
 218  221                  lmutex_lock(mp);
 219  222                  hash_out_unlocked(ulwp, ix, udp);
 220  223                  lmutex_unlock(mp);
 221  224          }
 222  225  }
 223  226  
 224  227  /*
 225  228   * Retain stack information for thread structures that are being recycled for
 226  229   * new threads.  All other members of the thread structure should be zeroed.
 227  230   */
 228  231  static void
 229  232  ulwp_clean(ulwp_t *ulwp)
 230  233  {
 231  234          caddr_t stk = ulwp->ul_stk;
 232  235          size_t mapsiz = ulwp->ul_mapsiz;
 233  236          size_t guardsize = ulwp->ul_guardsize;
 234  237          uintptr_t stktop = ulwp->ul_stktop;
 235  238          size_t stksiz = ulwp->ul_stksiz;
 236  239  
 237  240          (void) memset(ulwp, 0, sizeof (*ulwp));
 238  241  
 239  242          ulwp->ul_stk = stk;
 240  243          ulwp->ul_mapsiz = mapsiz;
 241  244          ulwp->ul_guardsize = guardsize;
 242  245          ulwp->ul_stktop = stktop;
 243  246          ulwp->ul_stksiz = stksiz;
 244  247  }
 245  248  
 246  249  static int stackprot;
 247  250  
 248  251  /*
 249  252   * Answer the question, "Is the lwp in question really dead?"
 250  253   * We must inquire of the operating system to be really sure
 251  254   * because the lwp may have called lwp_exit() but it has not
 252  255   * yet completed the exit.
 253  256   */
 254  257  static int
 255  258  dead_and_buried(ulwp_t *ulwp)
 256  259  {
 257  260          if (ulwp->ul_lwpid == (lwpid_t)(-1))
 258  261                  return (1);
 259  262          if (ulwp->ul_dead && ulwp->ul_detached &&
 260  263              _lwp_kill(ulwp->ul_lwpid, 0) == ESRCH) {
 261  264                  ulwp->ul_lwpid = (lwpid_t)(-1);
 262  265                  return (1);
 263  266          }
 264  267          return (0);
 265  268  }
 266  269  
 267  270  /*
 268  271   * Attempt to keep the stack cache within the specified cache limit.
 269  272   */
 270  273  static void
 271  274  trim_stack_cache(int cache_limit)
 272  275  {
 273  276          ulwp_t *self = curthread;
 274  277          uberdata_t *udp = self->ul_uberdata;
 275  278          ulwp_t *prev = NULL;
 276  279          ulwp_t **ulwpp = &udp->lwp_stacks;
 277  280          ulwp_t *ulwp;
 278  281  
 279  282          ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, self));
 280  283  
 281  284          while (udp->nfreestack > cache_limit && (ulwp = *ulwpp) != NULL) {
 282  285                  if (dead_and_buried(ulwp)) {
 283  286                          *ulwpp = ulwp->ul_next;
 284  287                          if (ulwp == udp->lwp_laststack)
 285  288                                  udp->lwp_laststack = prev;
 286  289                          hash_out(ulwp, udp);
 287  290                          udp->nfreestack--;
 288  291                          (void) munmap(ulwp->ul_stk, ulwp->ul_mapsiz);
 289  292                          /*
 290  293                           * Now put the free ulwp on the ulwp freelist.
 291  294                           */
 292  295                          ulwp->ul_mapsiz = 0;
 293  296                          ulwp->ul_next = NULL;
 294  297                          if (udp->ulwp_freelist == NULL)
 295  298                                  udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
 296  299                          else {
 297  300                                  udp->ulwp_lastfree->ul_next = ulwp;
 298  301                                  udp->ulwp_lastfree = ulwp;
 299  302                          }
 300  303                  } else {
 301  304                          prev = ulwp;
 302  305                          ulwpp = &ulwp->ul_next;
 303  306                  }
 304  307          }
 305  308  }
 306  309  
 307  310  /*
 308  311   * Find an unused stack of the requested size
 309  312   * or create a new stack of the requested size.
 310  313   * Return a pointer to the ulwp_t structure referring to the stack, or NULL.
 311  314   * thr_exit() stores 1 in the ul_dead member.
 312  315   * thr_join() stores -1 in the ul_lwpid member.
 313  316   */
 314  317  static ulwp_t *
 315  318  find_stack(size_t stksize, size_t guardsize)
 316  319  {
 317  320          static size_t pagesize = 0;
 318  321  
 319  322          uberdata_t *udp = curthread->ul_uberdata;
 320  323          size_t mapsize;
 321  324          ulwp_t *prev;
 322  325          ulwp_t *ulwp;
 323  326          ulwp_t **ulwpp;
 324  327          void *stk;
 325  328  
 326  329          /*
 327  330           * The stack is allocated PROT_READ|PROT_WRITE|PROT_EXEC
 328  331           * unless overridden by the system's configuration.
 329  332           */
 330  333          if (stackprot == 0) {   /* do this once */
 331  334                  long lprot = _sysconf(_SC_STACK_PROT);
 332  335                  if (lprot <= 0)
 333  336                          lprot = (PROT_READ|PROT_WRITE|PROT_EXEC);
 334  337                  stackprot = (int)lprot;
 335  338          }
 336  339          if (pagesize == 0)      /* do this once */
 337  340                  pagesize = _sysconf(_SC_PAGESIZE);
 338  341  
 339  342          /*
 340  343           * One megabyte stacks by default, but subtract off
 341  344           * two pages for the system-created red zones.
 342  345           * Round up a non-zero stack size to a pagesize multiple.
 343  346           */
 344  347          if (stksize == 0)
 345  348                  stksize = DEFAULTSTACK - 2 * pagesize;
 346  349          else
 347  350                  stksize = ((stksize + pagesize - 1) & -pagesize);
 348  351  
 349  352          /*
 350  353           * Round up the mapping size to a multiple of pagesize.
 351  354           * Note: mmap() provides at least one page of red zone
 352  355           * so we deduct that from the value of guardsize.
 353  356           */
 354  357          if (guardsize != 0)
 355  358                  guardsize = ((guardsize + pagesize - 1) & -pagesize) - pagesize;
 356  359          mapsize = stksize + guardsize;
 357  360  
 358  361          lmutex_lock(&udp->link_lock);
 359  362          for (prev = NULL, ulwpp = &udp->lwp_stacks;
 360  363              (ulwp = *ulwpp) != NULL;
 361  364              prev = ulwp, ulwpp = &ulwp->ul_next) {
 362  365                  if (ulwp->ul_mapsiz == mapsize &&
 363  366                      ulwp->ul_guardsize == guardsize &&
 364  367                      dead_and_buried(ulwp)) {
 365  368                          /*
 366  369                           * The previous lwp is gone; reuse the stack.
 367  370                           * Remove the ulwp from the stack list.
 368  371                           */
 369  372                          *ulwpp = ulwp->ul_next;
 370  373                          ulwp->ul_next = NULL;
 371  374                          if (ulwp == udp->lwp_laststack)
 372  375                                  udp->lwp_laststack = prev;
 373  376                          hash_out(ulwp, udp);
 374  377                          udp->nfreestack--;
 375  378                          lmutex_unlock(&udp->link_lock);
 376  379                          ulwp_clean(ulwp);
 377  380                          return (ulwp);
 378  381                  }
 379  382          }
 380  383  
 381  384          /*
 382  385           * None of the cached stacks matched our mapping size.
 383  386           * Reduce the stack cache to get rid of possibly
 384  387           * very old stacks that will never be reused.
 385  388           */
 386  389          if (udp->nfreestack > udp->thread_stack_cache)
 387  390                  trim_stack_cache(udp->thread_stack_cache);
 388  391          else if (udp->nfreestack > 0)
 389  392                  trim_stack_cache(udp->nfreestack - 1);
 390  393          lmutex_unlock(&udp->link_lock);
 391  394  
 392  395          /*
 393  396           * Create a new stack.
 394  397           */
 395  398          if ((stk = mmap(NULL, mapsize, stackprot,
 396  399              MAP_PRIVATE|MAP_NORESERVE|MAP_ANON, -1, (off_t)0)) != MAP_FAILED) {
 397  400                  /*
 398  401                   * We have allocated our stack.  Now allocate the ulwp.
 399  402                   */
 400  403                  ulwp = ulwp_alloc();
 401  404                  if (ulwp == NULL)
 402  405                          (void) munmap(stk, mapsize);
 403  406                  else {
 404  407                          ulwp->ul_stk = stk;
 405  408                          ulwp->ul_mapsiz = mapsize;
 406  409                          ulwp->ul_guardsize = guardsize;
 407  410                          ulwp->ul_stktop = (uintptr_t)stk + mapsize;
 408  411                          ulwp->ul_stksiz = stksize;
 409  412                          if (guardsize)  /* protect the extra red zone */
 410  413                                  (void) mprotect(stk, guardsize, PROT_NONE);
 411  414                  }
 412  415          }
 413  416          return (ulwp);
 414  417  }
 415  418  
 416  419  /*
 417  420   * Get a ulwp_t structure from the free list or allocate a new one.
 418  421   * Such ulwp_t's do not have a stack allocated by the library.
 419  422   */
 420  423  static ulwp_t *
 421  424  ulwp_alloc(void)
 422  425  {
 423  426          ulwp_t *self = curthread;
 424  427          uberdata_t *udp = self->ul_uberdata;
 425  428          size_t tls_size;
 426  429          ulwp_t *prev;
 427  430          ulwp_t *ulwp;
 428  431          ulwp_t **ulwpp;
 429  432          caddr_t data;
 430  433  
 431  434          lmutex_lock(&udp->link_lock);
 432  435          for (prev = NULL, ulwpp = &udp->ulwp_freelist;
 433  436              (ulwp = *ulwpp) != NULL;
 434  437              prev = ulwp, ulwpp = &ulwp->ul_next) {
 435  438                  if (dead_and_buried(ulwp)) {
 436  439                          *ulwpp = ulwp->ul_next;
 437  440                          ulwp->ul_next = NULL;
 438  441                          if (ulwp == udp->ulwp_lastfree)
 439  442                                  udp->ulwp_lastfree = prev;
 440  443                          hash_out(ulwp, udp);
 441  444                          lmutex_unlock(&udp->link_lock);
 442  445                          ulwp_clean(ulwp);
 443  446                          return (ulwp);
 444  447                  }
 445  448          }
 446  449          lmutex_unlock(&udp->link_lock);
 447  450  
 448  451          tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
 449  452          data = lmalloc(sizeof (*ulwp) + tls_size);
 450  453          if (data != NULL) {
 451  454                  /* LINTED pointer cast may result in improper alignment */
 452  455                  ulwp = (ulwp_t *)(data + tls_size);
 453  456          }
 454  457          return (ulwp);
 455  458  }
 456  459  
 457  460  /*
 458  461   * Free a ulwp structure.
 459  462   * If there is an associated stack, put it on the stack list and
 460  463   * munmap() previously freed stacks up to the residual cache limit.
 461  464   * Else put it on the ulwp free list and never call lfree() on it.
 462  465   */
 463  466  static void
 464  467  ulwp_free(ulwp_t *ulwp)
 465  468  {
 466  469          uberdata_t *udp = curthread->ul_uberdata;
 467  470  
 468  471          ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, curthread));
 469  472          ulwp->ul_next = NULL;
 470  473          if (ulwp == udp->ulwp_one)      /* don't reuse the primoridal stack */
 471  474                  /*EMPTY*/;
 472  475          else if (ulwp->ul_mapsiz != 0) {
 473  476                  if (udp->lwp_stacks == NULL)
 474  477                          udp->lwp_stacks = udp->lwp_laststack = ulwp;
 475  478                  else {
 476  479                          udp->lwp_laststack->ul_next = ulwp;
 477  480                          udp->lwp_laststack = ulwp;
 478  481                  }
 479  482                  if (++udp->nfreestack > udp->thread_stack_cache)
 480  483                          trim_stack_cache(udp->thread_stack_cache);
 481  484          } else {
 482  485                  if (udp->ulwp_freelist == NULL)
 483  486                          udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
 484  487                  else {
 485  488                          udp->ulwp_lastfree->ul_next = ulwp;
 486  489                          udp->ulwp_lastfree = ulwp;
 487  490                  }
 488  491          }
 489  492  }
 490  493  
 491  494  /*
 492  495   * Find a named lwp and return a pointer to its hash list location.
 493  496   * On success, returns with the hash lock held.
 494  497   */
 495  498  ulwp_t **
 496  499  find_lwpp(thread_t tid)
 497  500  {
 498  501          uberdata_t *udp = curthread->ul_uberdata;
 499  502          int ix = TIDHASH(tid, udp);
 500  503          mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
 501  504          ulwp_t *ulwp;
 502  505          ulwp_t **ulwpp;
 503  506  
 504  507          if (tid == 0)
 505  508                  return (NULL);
 506  509  
 507  510          lmutex_lock(mp);
 508  511          for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
 509  512              (ulwp = *ulwpp) != NULL;
 510  513              ulwpp = &ulwp->ul_hash) {
 511  514                  if (ulwp->ul_lwpid == tid)
 512  515                          return (ulwpp);
 513  516          }
 514  517          lmutex_unlock(mp);
 515  518          return (NULL);
 516  519  }
 517  520  
 518  521  /*
 519  522   * Wake up all lwps waiting on this lwp for some reason.
 520  523   */
 521  524  void
 522  525  ulwp_broadcast(ulwp_t *ulwp)
 523  526  {
 524  527          ulwp_t *self = curthread;
 525  528          uberdata_t *udp = self->ul_uberdata;
 526  529  
 527  530          ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
 528  531          (void) cond_broadcast(ulwp_condvar(ulwp, udp));
 529  532  }
 530  533  
 531  534  /*
 532  535   * Find a named lwp and return a pointer to it.
 533  536   * Returns with the hash lock held.
 534  537   */
 535  538  ulwp_t *
 536  539  find_lwp(thread_t tid)
 537  540  {
 538  541          ulwp_t *self = curthread;
 539  542          uberdata_t *udp = self->ul_uberdata;
 540  543          ulwp_t *ulwp = NULL;
 541  544          ulwp_t **ulwpp;
 542  545  
 543  546          if (self->ul_lwpid == tid) {
 544  547                  ulwp = self;
 545  548                  ulwp_lock(ulwp, udp);
 546  549          } else if ((ulwpp = find_lwpp(tid)) != NULL) {
 547  550                  ulwp = *ulwpp;
 548  551          }
 549  552  
 550  553          if (ulwp && ulwp->ul_dead) {
 551  554                  ulwp_unlock(ulwp, udp);
 552  555                  ulwp = NULL;
 553  556          }
 554  557  
 555  558          return (ulwp);
 556  559  }
 557  560  
 558  561  int
 559  562  _thrp_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
 560  563      long flags, thread_t *new_thread, size_t guardsize)
 561  564  {
 562  565          ulwp_t *self = curthread;
 563  566          uberdata_t *udp = self->ul_uberdata;
 564  567          ucontext_t uc;
 565  568          uint_t lwp_flags;
 566  569          thread_t tid;
 567  570          int error;
 568  571          ulwp_t *ulwp;
 569  572  
 570  573          /*
 571  574           * Enforce the restriction of not creating any threads
 572  575           * until the primary link map has been initialized.
 573  576           * Also, disallow thread creation to a child of vfork().
 574  577           */
 575  578          if (!self->ul_primarymap || self->ul_vfork)
 576  579                  return (ENOTSUP);
 577  580  
 578  581          if (udp->hash_size == 1)
 579  582                  finish_init();
 580  583  
 581  584          if ((stk || stksize) && stksize < MINSTACK)
 582  585                  return (EINVAL);
 583  586  
 584  587          if (stk == NULL) {
 585  588                  if ((ulwp = find_stack(stksize, guardsize)) == NULL)
 586  589                          return (ENOMEM);
 587  590                  stksize = ulwp->ul_mapsiz - ulwp->ul_guardsize;
 588  591          } else {
 589  592                  /* initialize the private stack */
 590  593                  if ((ulwp = ulwp_alloc()) == NULL)
 591  594                          return (ENOMEM);
 592  595                  ulwp->ul_stk = stk;
 593  596                  ulwp->ul_stktop = (uintptr_t)stk + stksize;
 594  597                  ulwp->ul_stksiz = stksize;
 595  598          }
 596  599          /* ulwp is not in the hash table; make sure hash_out() doesn't fail */
 597  600          ulwp->ul_ix = -1;
 598  601          ulwp->ul_errnop = &ulwp->ul_errno;
 599  602  
 600  603          lwp_flags = LWP_SUSPENDED;
 601  604          if (flags & (THR_DETACHED|THR_DAEMON)) {
 602  605                  flags |= THR_DETACHED;
 603  606                  lwp_flags |= LWP_DETACHED;
 604  607          }
 605  608          if (flags & THR_DAEMON)
 606  609                  lwp_flags |= LWP_DAEMON;
 607  610  
 608  611          /* creating a thread: enforce mt-correctness in mutex_lock() */
 609  612          self->ul_async_safe = 1;
 610  613  
 611  614          /* per-thread copies of global variables, for speed */
 612  615          ulwp->ul_queue_fifo = self->ul_queue_fifo;
 613  616          ulwp->ul_cond_wait_defer = self->ul_cond_wait_defer;
 614  617          ulwp->ul_error_detection = self->ul_error_detection;
 615  618          ulwp->ul_async_safe = self->ul_async_safe;
 616  619          ulwp->ul_max_spinners = self->ul_max_spinners;
 617  620          ulwp->ul_adaptive_spin = self->ul_adaptive_spin;
 618  621          ulwp->ul_queue_spin = self->ul_queue_spin;
 619  622          ulwp->ul_door_noreserve = self->ul_door_noreserve;
 620  623          ulwp->ul_misaligned = self->ul_misaligned;
 621  624  
 622  625          /* new thread inherits creating thread's scheduling parameters */
 623  626          ulwp->ul_policy = self->ul_policy;
 624  627          ulwp->ul_pri = (self->ul_epri? self->ul_epri : self->ul_pri);
 625  628          ulwp->ul_cid = self->ul_cid;
 626  629          ulwp->ul_rtclassid = self->ul_rtclassid;
 627  630  
 628  631          ulwp->ul_primarymap = self->ul_primarymap;
 629  632          ulwp->ul_self = ulwp;
 630  633          ulwp->ul_uberdata = udp;
 631  634  
 632  635          /* debugger support */
 633  636          ulwp->ul_usropts = flags;
 634  637  
 635  638  #ifdef __sparc
 636  639          /*
 637  640           * We cache several instructions in the thread structure for use
 638  641           * by the fasttrap DTrace provider. When changing this, read the
 639  642           * comment in fasttrap.h for the all the other places that must
 640  643           * be changed.
 641  644           */
 642  645          ulwp->ul_dsave = 0x9de04000;    /* save %g1, %g0, %sp */
 643  646          ulwp->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
 644  647          ulwp->ul_dftret = 0x91d0203a;   /* ta 0x3a */
 645  648          ulwp->ul_dreturn = 0x81ca0000;  /* return %o0 */
 646  649  #endif
 647  650  
 648  651          ulwp->ul_startpc = func;
 649  652          ulwp->ul_startarg = arg;
 650  653          _fpinherit(ulwp);
 651  654          /*
 652  655           * Defer signals on the new thread until its TLS constructors
 653  656           * have been called.  _thrp_setup() will call sigon() after
 654  657           * it has called tls_setup().
 655  658           */
 656  659          ulwp->ul_sigdefer = 1;
 657  660  
 658  661          error = setup_context(&uc, _thrp_setup, ulwp,
 659  662              (caddr_t)ulwp->ul_stk + ulwp->ul_guardsize, stksize);
 660  663          if (error != 0 && stk != NULL)  /* inaccessible stack */
 661  664                  error = EFAULT;
 662  665  
 663  666          /*
 664  667           * Call enter_critical() to avoid being suspended until we
 665  668           * have linked the new thread into the proper lists.
 666  669           * This is necessary because forkall() and fork1() must
 667  670           * suspend all threads and they must see a complete list.
 668  671           */
 669  672          enter_critical(self);
 670  673          uc.uc_sigmask = ulwp->ul_sigmask = self->ul_sigmask;
 671  674          if (error != 0 ||
 672  675              (error = __lwp_create(&uc, lwp_flags, &tid)) != 0) {
 673  676                  exit_critical(self);
 674  677                  ulwp->ul_lwpid = (lwpid_t)(-1);
 675  678                  ulwp->ul_dead = 1;
 676  679                  ulwp->ul_detached = 1;
 677  680                  lmutex_lock(&udp->link_lock);
 678  681                  ulwp_free(ulwp);
 679  682                  lmutex_unlock(&udp->link_lock);
 680  683                  return (error);
 681  684          }
 682  685          self->ul_nocancel = 0;  /* cancellation is now possible */
 683  686          udp->uberflags.uf_mt = 1;
 684  687          if (new_thread)
 685  688                  *new_thread = tid;
 686  689          if (flags & THR_DETACHED)
 687  690                  ulwp->ul_detached = 1;
 688  691          ulwp->ul_lwpid = tid;
 689  692          ulwp->ul_stop = TSTP_REGULAR;
 690  693          if (flags & THR_SUSPENDED)
 691  694                  ulwp->ul_created = 1;
 692  695  
 693  696          lmutex_lock(&udp->link_lock);
 694  697          ulwp->ul_forw = udp->all_lwps;
 695  698          ulwp->ul_back = udp->all_lwps->ul_back;
 696  699          ulwp->ul_back->ul_forw = ulwp;
 697  700          ulwp->ul_forw->ul_back = ulwp;
 698  701          hash_in(ulwp, udp);
 699  702          udp->nthreads++;
 700  703          if (flags & THR_DAEMON)
 701  704                  udp->ndaemons++;
 702  705          if (flags & THR_NEW_LWP)
 703  706                  thr_concurrency++;
 704  707          __libc_threaded = 1;            /* inform stdio */
 705  708          lmutex_unlock(&udp->link_lock);
 706  709  
 707  710          if (__td_event_report(self, TD_CREATE, udp)) {
 708  711                  self->ul_td_evbuf.eventnum = TD_CREATE;
 709  712                  self->ul_td_evbuf.eventdata = (void *)(uintptr_t)tid;
 710  713                  tdb_event(TD_CREATE, udp);
 711  714          }
 712  715  
 713  716          exit_critical(self);
 714  717  
 715  718          if (!(flags & THR_SUSPENDED))
 716  719                  (void) _thrp_continue(tid, TSTP_REGULAR);
 717  720  
 718  721          return (0);
 719  722  }
 720  723  
 721  724  int
 722  725  thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
 723  726      long flags, thread_t *new_thread)
 724  727  {
 725  728          return (_thrp_create(stk, stksize, func, arg, flags, new_thread, 0));
 726  729  }
 727  730  
 728  731  /*
 729  732   * A special cancellation cleanup hook for DCE.
 730  733   * cleanuphndlr, when it is not NULL, will contain a callback
 731  734   * function to be called before a thread is terminated in
 732  735   * thr_exit() as a result of being cancelled.
 733  736   */
 734  737  static void (*cleanuphndlr)(void) = NULL;
 735  738  
 736  739  /*
 737  740   * _pthread_setcleanupinit: sets the cleanup hook.
 738  741   */
 739  742  int
 740  743  _pthread_setcleanupinit(void (*func)(void))
 741  744  {
 742  745          cleanuphndlr = func;
 743  746          return (0);
 744  747  }
 745  748  
 746  749  void
 747  750  _thrp_exit()
 748  751  {
 749  752          ulwp_t *self = curthread;
 750  753          uberdata_t *udp = self->ul_uberdata;
 751  754          ulwp_t *replace = NULL;
 752  755  
 753  756          if (__td_event_report(self, TD_DEATH, udp)) {
 754  757                  self->ul_td_evbuf.eventnum = TD_DEATH;
 755  758                  tdb_event(TD_DEATH, udp);
 756  759          }
 757  760  
 758  761          ASSERT(self->ul_sigdefer != 0);
 759  762  
 760  763          lmutex_lock(&udp->link_lock);
 761  764          udp->nthreads--;
 762  765          if (self->ul_usropts & THR_NEW_LWP)
 763  766                  thr_concurrency--;
 764  767          if (self->ul_usropts & THR_DAEMON)
 765  768                  udp->ndaemons--;
 766  769          else if (udp->nthreads == udp->ndaemons) {
 767  770                  /*
 768  771                   * We are the last non-daemon thread exiting.
 769  772                   * Exit the process.  We retain our TSD and TLS so
 770  773                   * that atexit() application functions can use them.
 771  774                   */
 772  775                  lmutex_unlock(&udp->link_lock);
 773  776                  exit(0);
 774  777                  thr_panic("_thrp_exit(): exit(0) returned");
 775  778          }
 776  779          lmutex_unlock(&udp->link_lock);
 777  780  
 778  781          tmem_exit();            /* deallocate tmem allocations */
 779  782          tsd_exit();             /* deallocate thread-specific data */
 780  783          tls_exit();             /* deallocate thread-local storage */
 781  784          heldlock_exit();        /* deal with left-over held locks */
 782  785  
 783  786          /* block all signals to finish exiting */
 784  787          block_all_signals(self);
 785  788          /* also prevent ourself from being suspended */
 786  789          enter_critical(self);
 787  790          rwl_free(self);
 788  791          lmutex_lock(&udp->link_lock);
 789  792          ulwp_free(self);
 790  793          (void) ulwp_lock(self, udp);
 791  794  
 792  795          if (self->ul_mapsiz && !self->ul_detached) {
 793  796                  /*
 794  797                   * We want to free the stack for reuse but must keep
 795  798                   * the ulwp_t struct for the benefit of thr_join().
 796  799                   * For this purpose we allocate a replacement ulwp_t.
 797  800                   */
 798  801                  if ((replace = udp->ulwp_replace_free) == NULL)
 799  802                          replace = lmalloc(REPLACEMENT_SIZE);
 800  803                  else if ((udp->ulwp_replace_free = replace->ul_next) == NULL)
 801  804                          udp->ulwp_replace_last = NULL;
 802  805          }
 803  806  
 804  807          if (udp->all_lwps == self)
 805  808                  udp->all_lwps = self->ul_forw;
 806  809          if (udp->all_lwps == self)
 807  810                  udp->all_lwps = NULL;
 808  811          else {
 809  812                  self->ul_forw->ul_back = self->ul_back;
 810  813                  self->ul_back->ul_forw = self->ul_forw;
 811  814          }
 812  815          self->ul_forw = self->ul_back = NULL;
 813  816  #if defined(THREAD_DEBUG)
 814  817          /* collect queue lock statistics before marking ourself dead */
 815  818          record_spin_locks(self);
 816  819  #endif
 817  820          self->ul_dead = 1;
 818  821          self->ul_pleasestop = 0;
 819  822          if (replace != NULL) {
 820  823                  int ix = self->ul_ix;           /* the hash index */
 821  824                  (void) memcpy(replace, self, REPLACEMENT_SIZE);
 822  825                  replace->ul_self = replace;
 823  826                  replace->ul_next = NULL;        /* clone not on stack list */
 824  827                  replace->ul_mapsiz = 0;         /* allows clone to be freed */
 825  828                  replace->ul_replace = 1;        /* requires clone to be freed */
 826  829                  hash_out_unlocked(self, ix, udp);
 827  830                  hash_in_unlocked(replace, ix, udp);
 828  831                  ASSERT(!(self->ul_detached));
 829  832                  self->ul_detached = 1;          /* this frees the stack */
 830  833                  self->ul_schedctl = NULL;
 831  834                  self->ul_schedctl_called = &udp->uberflags;
 832  835                  set_curthread(self = replace);
 833  836                  /*
 834  837                   * Having just changed the address of curthread, we
 835  838                   * must reset the ownership of the locks we hold so
 836  839                   * that assertions will not fire when we release them.
 837  840                   */
 838  841                  udp->link_lock.mutex_owner = (uintptr_t)self;
 839  842                  ulwp_mutex(self, udp)->mutex_owner = (uintptr_t)self;
 840  843                  /*
 841  844                   * NOTE:
 842  845                   * On i386, %gs still references the original, not the
 843  846                   * replacement, ulwp structure.  Fetching the replacement
 844  847                   * curthread pointer via %gs:0 works correctly since the
 845  848                   * original ulwp structure will not be reallocated until
 846  849                   * this lwp has completed its lwp_exit() system call (see
 847  850                   * dead_and_buried()), but from here on out, we must make
 848  851                   * no references to %gs:<offset> other than %gs:0.
 849  852                   */
 850  853          }
 851  854          /*
 852  855           * Put non-detached terminated threads in the all_zombies list.
 853  856           */
 854  857          if (!self->ul_detached) {
 855  858                  udp->nzombies++;
 856  859                  if (udp->all_zombies == NULL) {
 857  860                          ASSERT(udp->nzombies == 1);
 858  861                          udp->all_zombies = self->ul_forw = self->ul_back = self;
 859  862                  } else {
 860  863                          self->ul_forw = udp->all_zombies;
 861  864                          self->ul_back = udp->all_zombies->ul_back;
 862  865                          self->ul_back->ul_forw = self;
 863  866                          self->ul_forw->ul_back = self;
 864  867                  }
 865  868          }
 866  869          /*
 867  870           * Notify everyone waiting for this thread.
 868  871           */
 869  872          ulwp_broadcast(self);
 870  873          (void) ulwp_unlock(self, udp);
 871  874          /*
 872  875           * Prevent any more references to the schedctl data.
 873  876           * We are exiting and continue_fork() may not find us.
 874  877           * Do this just before dropping link_lock, since fork
 875  878           * serializes on link_lock.
 876  879           */
 877  880          self->ul_schedctl = NULL;
 878  881          self->ul_schedctl_called = &udp->uberflags;
 879  882          lmutex_unlock(&udp->link_lock);
 880  883  
 881  884          ASSERT(self->ul_critical == 1);
 882  885          ASSERT(self->ul_preempt == 0);
 883  886          _lwp_terminate();       /* never returns */
 884  887          thr_panic("_thrp_exit(): _lwp_terminate() returned");
 885  888  }
 886  889  
 887  890  #if defined(THREAD_DEBUG)
 888  891  void
 889  892  collect_queue_statistics()
 890  893  {
 891  894          uberdata_t *udp = curthread->ul_uberdata;
 892  895          ulwp_t *ulwp;
 893  896  
 894  897          if (thread_queue_dump) {
 895  898                  lmutex_lock(&udp->link_lock);
 896  899                  if ((ulwp = udp->all_lwps) != NULL) {
 897  900                          do {
 898  901                                  record_spin_locks(ulwp);
 899  902                          } while ((ulwp = ulwp->ul_forw) != udp->all_lwps);
 900  903                  }
 901  904                  lmutex_unlock(&udp->link_lock);
 902  905          }
 903  906  }
 904  907  #endif
 905  908  
 906  909  static void __NORETURN
 907  910  _thrp_exit_common(void *status, int unwind)
 908  911  {
 909  912          ulwp_t *self = curthread;
 910  913          int cancelled = (self->ul_cancel_pending && status == PTHREAD_CANCELED);
 911  914  
 912  915          ASSERT(self->ul_critical == 0 && self->ul_preempt == 0);
 913  916  
 914  917          /*
 915  918           * Disable cancellation and call the special DCE cancellation
 916  919           * cleanup hook if it is enabled.  Do nothing else before calling
 917  920           * the DCE cancellation cleanup hook; it may call longjmp() and
 918  921           * never return here.
 919  922           */
 920  923          self->ul_cancel_disabled = 1;
 921  924          self->ul_cancel_async = 0;
 922  925          self->ul_save_async = 0;
 923  926          self->ul_cancelable = 0;
 924  927          self->ul_cancel_pending = 0;
 925  928          set_cancel_pending_flag(self, 1);
 926  929          if (cancelled && cleanuphndlr != NULL)
 927  930                  (*cleanuphndlr)();
 928  931  
 929  932          /*
 930  933           * Block application signals while we are exiting.
 931  934           * We call out to C++, TSD, and TLS destructors while exiting
 932  935           * and these are application-defined, so we cannot be assured
 933  936           * that they won't reset the signal mask.  We use sigoff() to
 934  937           * defer any signals that may be received as a result of this
 935  938           * bad behavior.  Such signals will be lost to the process
 936  939           * when the thread finishes exiting.
 937  940           */
 938  941          (void) thr_sigsetmask(SIG_SETMASK, &maskset, NULL);
 939  942          sigoff(self);
 940  943  
 941  944          self->ul_rval = status;
 942  945  
 943  946          /*
 944  947           * If thr_exit is being called from the places where
 945  948           * C++ destructors are to be called such as cancellation
 946  949           * points, then set this flag. It is checked in _t_cancel()
 947  950           * to decide whether _ex_unwind() is to be called or not.
 948  951           */
 949  952          if (unwind)
 950  953                  self->ul_unwind = 1;
 951  954  
 952  955          /*
 953  956           * _thrp_unwind() will eventually call _thrp_exit().
 954  957           * It never returns.
 955  958           */
 956  959          _thrp_unwind(NULL);
 957  960          thr_panic("_thrp_exit_common(): _thrp_unwind() returned");
 958  961  
 959  962          for (;;)        /* to shut the compiler up about __NORETURN */
 960  963                  continue;
 961  964  }
 962  965  
 963  966  /*
 964  967   * Called when a thread returns from its start function.
 965  968   * We are at the top of the stack; no unwinding is necessary.
 966  969   */
 967  970  void
 968  971  _thrp_terminate(void *status)
 969  972  {
 970  973          _thrp_exit_common(status, 0);
 971  974  }
 972  975  
 973  976  #pragma weak pthread_exit = thr_exit
 974  977  #pragma weak _thr_exit = thr_exit
 975  978  void
 976  979  thr_exit(void *status)
 977  980  {
 978  981          _thrp_exit_common(status, 1);
 979  982  }
 980  983  
 981  984  int
 982  985  _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
 983  986  {
 984  987          uberdata_t *udp = curthread->ul_uberdata;
 985  988          mutex_t *mp;
 986  989          void *rval;
 987  990          thread_t found;
 988  991          ulwp_t *ulwp;
 989  992          ulwp_t **ulwpp;
 990  993          int replace;
 991  994          int error;
 992  995  
 993  996          if (do_cancel)
 994  997                  error = lwp_wait(tid, &found);
 995  998          else {
 996  999                  while ((error = __lwp_wait(tid, &found)) == EINTR)
 997 1000                          ;
 998 1001          }
 999 1002          if (error)
1000 1003                  return (error);
1001 1004  
1002 1005          /*
1003 1006           * We must hold link_lock to avoid a race condition with find_stack().
1004 1007           */
1005 1008          lmutex_lock(&udp->link_lock);
1006 1009          if ((ulwpp = find_lwpp(found)) == NULL) {
1007 1010                  /*
1008 1011                   * lwp_wait() found an lwp that the library doesn't know
1009 1012                   * about.  It must have been created with _lwp_create().
1010 1013                   * Just return its lwpid; we can't know its status.
1011 1014                   */
1012 1015                  lmutex_unlock(&udp->link_lock);
1013 1016                  rval = NULL;
1014 1017          } else {
1015 1018                  /*
1016 1019                   * Remove ulwp from the hash table.
1017 1020                   */
1018 1021                  ulwp = *ulwpp;
1019 1022                  *ulwpp = ulwp->ul_hash;
1020 1023                  ulwp->ul_hash = NULL;
1021 1024                  /*
1022 1025                   * Remove ulwp from all_zombies list.
1023 1026                   */
1024 1027                  ASSERT(udp->nzombies >= 1);
1025 1028                  if (udp->all_zombies == ulwp)
1026 1029                          udp->all_zombies = ulwp->ul_forw;
1027 1030                  if (udp->all_zombies == ulwp)
1028 1031                          udp->all_zombies = NULL;
1029 1032                  else {
1030 1033                          ulwp->ul_forw->ul_back = ulwp->ul_back;
1031 1034                          ulwp->ul_back->ul_forw = ulwp->ul_forw;
1032 1035                  }
1033 1036                  ulwp->ul_forw = ulwp->ul_back = NULL;
1034 1037                  udp->nzombies--;
1035 1038                  ASSERT(ulwp->ul_dead && !ulwp->ul_detached &&
1036 1039                      !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
1037 1040                  /*
1038 1041                   * We can't call ulwp_unlock(ulwp) after we set
1039 1042                   * ulwp->ul_ix = -1 so we have to get a pointer to the
1040 1043                   * ulwp's hash table mutex now in order to unlock it below.
1041 1044                   */
1042 1045                  mp = ulwp_mutex(ulwp, udp);
1043 1046                  ulwp->ul_lwpid = (lwpid_t)(-1);
1044 1047                  ulwp->ul_ix = -1;
1045 1048                  rval = ulwp->ul_rval;
1046 1049                  replace = ulwp->ul_replace;
1047 1050                  lmutex_unlock(mp);
1048 1051                  if (replace) {
1049 1052                          ulwp->ul_next = NULL;
1050 1053                          if (udp->ulwp_replace_free == NULL)
1051 1054                                  udp->ulwp_replace_free =
1052 1055                                      udp->ulwp_replace_last = ulwp;
1053 1056                          else {
1054 1057                                  udp->ulwp_replace_last->ul_next = ulwp;
1055 1058                                  udp->ulwp_replace_last = ulwp;
1056 1059                          }
1057 1060                  }
1058 1061                  lmutex_unlock(&udp->link_lock);
1059 1062          }
1060 1063  
1061 1064          if (departed != NULL)
1062 1065                  *departed = found;
1063 1066          if (status != NULL)
1064 1067                  *status = rval;
1065 1068          return (0);
1066 1069  }
1067 1070  
1068 1071  int
1069 1072  thr_join(thread_t tid, thread_t *departed, void **status)
1070 1073  {
1071 1074          int error = _thrp_join(tid, departed, status, 1);
1072 1075          return ((error == EINVAL)? ESRCH : error);
1073 1076  }
1074 1077  
1075 1078  /*
1076 1079   * pthread_join() differs from Solaris thr_join():
1077 1080   * It does not return the departed thread's id
1078 1081   * and hence does not have a "departed" argument.
1079 1082   * It returns EINVAL if tid refers to a detached thread.
1080 1083   */
1081 1084  #pragma weak _pthread_join = pthread_join
1082 1085  int
1083 1086  pthread_join(pthread_t tid, void **status)
1084 1087  {
1085 1088          return ((tid == 0)? ESRCH : _thrp_join(tid, NULL, status, 1));
1086 1089  }
1087 1090  
1088 1091  int
1089 1092  pthread_detach(pthread_t tid)
1090 1093  {
1091 1094          uberdata_t *udp = curthread->ul_uberdata;
1092 1095          ulwp_t *ulwp;
1093 1096          ulwp_t **ulwpp;
1094 1097          int error = 0;
1095 1098  
1096 1099          if ((ulwpp = find_lwpp(tid)) == NULL)
1097 1100                  return (ESRCH);
1098 1101          ulwp = *ulwpp;
1099 1102  
1100 1103          if (ulwp->ul_dead) {
1101 1104                  ulwp_unlock(ulwp, udp);
1102 1105                  error = _thrp_join(tid, NULL, NULL, 0);
1103 1106          } else {
1104 1107                  error = __lwp_detach(tid);
1105 1108                  ulwp->ul_detached = 1;
1106 1109                  ulwp->ul_usropts |= THR_DETACHED;
1107 1110                  ulwp_unlock(ulwp, udp);
1108 1111          }
1109 1112          return (error);
1110 1113  }
1111 1114  
1112 1115  static const char *
1113 1116  ematch(const char *ev, const char *match)
1114 1117  {
1115 1118          int c;
1116 1119  
1117 1120          while ((c = *match++) != '\0') {
1118 1121                  if (*ev++ != c)
1119 1122                          return (NULL);
1120 1123          }
1121 1124          if (*ev++ != '=')
1122 1125                  return (NULL);
1123 1126          return (ev);
1124 1127  }
1125 1128  
1126 1129  static int
1127 1130  envvar(const char *ev, const char *match, int limit)
1128 1131  {
1129 1132          int val = -1;
1130 1133          const char *ename;
1131 1134  
1132 1135          if ((ename = ematch(ev, match)) != NULL) {
1133 1136                  int c;
1134 1137                  for (val = 0; (c = *ename) != '\0'; ename++) {
1135 1138                          if (!isdigit(c)) {
1136 1139                                  val = -1;
1137 1140                                  break;
1138 1141                          }
1139 1142                          val = val * 10 + (c - '0');
1140 1143                          if (val > limit) {
1141 1144                                  val = limit;
1142 1145                                  break;
1143 1146                          }
1144 1147                  }
1145 1148          }
1146 1149          return (val);
1147 1150  }
1148 1151  
1149 1152  static void
1150 1153  etest(const char *ev)
1151 1154  {
1152 1155          int value;
1153 1156  
1154 1157          if ((value = envvar(ev, "QUEUE_SPIN", 1000000)) >= 0)
1155 1158                  thread_queue_spin = value;
1156 1159          if ((value = envvar(ev, "ADAPTIVE_SPIN", 1000000)) >= 0)
1157 1160                  thread_adaptive_spin = value;
1158 1161          if ((value = envvar(ev, "MAX_SPINNERS", 255)) >= 0)
1159 1162                  thread_max_spinners = value;
1160 1163          if ((value = envvar(ev, "QUEUE_FIFO", 8)) >= 0)
1161 1164                  thread_queue_fifo = value;
1162 1165  #if defined(THREAD_DEBUG)
1163 1166          if ((value = envvar(ev, "QUEUE_VERIFY", 1)) >= 0)
1164 1167                  thread_queue_verify = value;
1165 1168          if ((value = envvar(ev, "QUEUE_DUMP", 1)) >= 0)
1166 1169                  thread_queue_dump = value;
1167 1170  #endif
1168 1171          if ((value = envvar(ev, "STACK_CACHE", 10000)) >= 0)
1169 1172                  thread_stack_cache = value;
1170 1173          if ((value = envvar(ev, "COND_WAIT_DEFER", 1)) >= 0)
1171 1174                  thread_cond_wait_defer = value;
1172 1175          if ((value = envvar(ev, "ERROR_DETECTION", 2)) >= 0)
1173 1176                  thread_error_detection = value;
1174 1177          if ((value = envvar(ev, "ASYNC_SAFE", 1)) >= 0)
1175 1178                  thread_async_safe = value;
1176 1179          if ((value = envvar(ev, "DOOR_NORESERVE", 1)) >= 0)
1177 1180                  thread_door_noreserve = value;
1178 1181          if ((value = envvar(ev, "LOCKS_MISALIGNED", 1)) >= 0)
1179 1182                  thread_locks_misaligned = value;
1180 1183  }
1181 1184  
1182 1185  /*
1183 1186   * Look for and evaluate environment variables of the form "_THREAD_*".
1184 1187   * For compatibility with the past, we also look for environment
1185 1188   * names of the form "LIBTHREAD_*".
1186 1189   */
1187 1190  static void
1188 1191  set_thread_vars()
1189 1192  {
1190 1193          extern const char **_environ;
1191 1194          const char **pev;
1192 1195          const char *ev;
1193 1196          char c;
1194 1197  
1195 1198          if ((pev = _environ) == NULL)
1196 1199                  return;
1197 1200          while ((ev = *pev++) != NULL) {
1198 1201                  c = *ev;
1199 1202                  if (c == '_' && strncmp(ev, "_THREAD_", 8) == 0)
1200 1203                          etest(ev + 8);
1201 1204                  if (c == 'L' && strncmp(ev, "LIBTHREAD_", 10) == 0)
1202 1205                          etest(ev + 10);
1203 1206          }
1204 1207  }
1205 1208  
1206 1209  /* PROBE_SUPPORT begin */
1207 1210  #pragma weak __tnf_probe_notify
1208 1211  extern void __tnf_probe_notify(void);
1209 1212  /* PROBE_SUPPORT end */
1210 1213  
1211 1214  /* same as atexit() but private to the library */
1212 1215  extern int _atexit(void (*)(void));
  
    | 
      ↓ open down ↓ | 
    1063 lines elided | 
    
      ↑ open up ↑ | 
  
1213 1216  
1214 1217  /* same as _cleanup() but private to the library */
1215 1218  extern void __cleanup(void);
1216 1219  
1217 1220  extern void atfork_init(void);
1218 1221  
1219 1222  #ifdef __amd64
1220 1223  extern void __proc64id(void);
1221 1224  #endif
1222 1225  
     1226 +static void
     1227 +init_auxv_data(uberdata_t *udp)
     1228 +{
     1229 +        Dl_argsinfo_t args;
     1230 +
     1231 +        udp->ub_broot = NULL;
     1232 +        udp->ub_comm_page = NULL;
     1233 +        if (dlinfo(RTLD_SELF, RTLD_DI_ARGSINFO, &args) < 0)
     1234 +                return;
     1235 +
     1236 +        while (args.dla_auxv->a_type != AT_NULL) {
     1237 +                switch (args.dla_auxv->a_type) {
     1238 +                case AT_SUN_BRAND_NROOT:
     1239 +                        udp->ub_broot = args.dla_auxv->a_un.a_ptr;
     1240 +                        break;
     1241 +                case AT_SUN_COMMPAGE:
     1242 +                        udp->ub_comm_page = args.dla_auxv->a_un.a_ptr;
     1243 +                        break;
     1244 +                }
     1245 +                args.dla_auxv++;
     1246 +        }
     1247 +}
     1248 +
1223 1249  /*
1224 1250   * libc_init() is called by ld.so.1 for library initialization.
1225 1251   * We perform minimal initialization; enough to work with the main thread.
1226 1252   */
1227 1253  void
1228 1254  libc_init(void)
1229 1255  {
1230 1256          uberdata_t *udp = &__uberdata;
1231 1257          ulwp_t *oldself = __curthread();
1232 1258          ucontext_t uc;
1233 1259          ulwp_t *self;
1234 1260          struct rlimit rl;
1235 1261          caddr_t data;
1236 1262          size_t tls_size;
1237 1263          int setmask;
1238 1264  
1239 1265          /*
1240 1266           * For the initial stage of initialization, we must be careful
1241 1267           * not to call any function that could possibly call _cerror().
1242 1268           * For this purpose, we call only the raw system call wrappers.
1243 1269           */
1244 1270  
1245 1271  #ifdef __amd64
1246 1272          /*
1247 1273           * Gather information about cache layouts for optimized
1248 1274           * AMD and Intel assembler strfoo() and memfoo() functions.
  
    | 
      ↓ open down ↓ | 
    16 lines elided | 
    
      ↑ open up ↑ | 
  
1249 1275           */
1250 1276          __proc64id();
1251 1277  #endif
1252 1278  
1253 1279          /*
1254 1280           * Every libc, regardless of which link map, must register __cleanup().
1255 1281           */
1256 1282          (void) _atexit(__cleanup);
1257 1283  
1258 1284          /*
     1285 +         * Every libc, regardless of link map, needs to go through and check
     1286 +         * its aux vectors.  Doing so will indicate whether or not this has
     1287 +         * been given a brand root (used to qualify various other data) or a
     1288 +         * comm page (to optimize certain system actions).
     1289 +         */
     1290 +        init_auxv_data(udp);
     1291 +
     1292 +        /*
1259 1293           * We keep our uberdata on one of (a) the first alternate link map
1260 1294           * or (b) the primary link map.  We switch to the primary link map
1261 1295           * and stay there once we see it.  All intermediate link maps are
1262 1296           * subject to being unloaded at any time.
1263 1297           */
1264 1298          if (oldself != NULL && (oldself->ul_primarymap || !primary_link_map)) {
1265 1299                  __tdb_bootstrap = oldself->ul_uberdata->tdb_bootstrap;
1266 1300                  mutex_setup();
1267 1301                  atfork_init();  /* every link map needs atfork() processing */
1268 1302                  init_progname();
1269 1303                  return;
1270 1304          }
1271 1305  
1272 1306          /*
1273 1307           * To establish the main stack information, we have to get our context.
1274 1308           * This is also convenient to use for getting our signal mask.
1275 1309           */
1276 1310          uc.uc_flags = UC_ALL;
1277 1311          (void) __getcontext(&uc);
1278 1312          ASSERT(uc.uc_link == NULL);
1279 1313  
1280 1314          tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
1281 1315          ASSERT(primary_link_map || tls_size == 0);
1282 1316          data = lmalloc(sizeof (ulwp_t) + tls_size);
1283 1317          if (data == NULL)
1284 1318                  thr_panic("cannot allocate thread structure for main thread");
1285 1319          /* LINTED pointer cast may result in improper alignment */
1286 1320          self = (ulwp_t *)(data + tls_size);
1287 1321          init_hash_table[0].hash_bucket = self;
1288 1322  
1289 1323          self->ul_sigmask = uc.uc_sigmask;
1290 1324          delete_reserved_signals(&self->ul_sigmask);
1291 1325          /*
1292 1326           * Are the old and new sets different?
1293 1327           * (This can happen if we are currently blocking SIGCANCEL.)
1294 1328           * If so, we must explicitly set our signal mask, below.
1295 1329           */
1296 1330          setmask =
1297 1331              ((self->ul_sigmask.__sigbits[0] ^ uc.uc_sigmask.__sigbits[0]) |
1298 1332              (self->ul_sigmask.__sigbits[1] ^ uc.uc_sigmask.__sigbits[1]) |
1299 1333              (self->ul_sigmask.__sigbits[2] ^ uc.uc_sigmask.__sigbits[2]) |
1300 1334              (self->ul_sigmask.__sigbits[3] ^ uc.uc_sigmask.__sigbits[3]));
1301 1335  
1302 1336  #ifdef __sparc
1303 1337          /*
1304 1338           * We cache several instructions in the thread structure for use
1305 1339           * by the fasttrap DTrace provider. When changing this, read the
1306 1340           * comment in fasttrap.h for the all the other places that must
1307 1341           * be changed.
1308 1342           */
1309 1343          self->ul_dsave = 0x9de04000;    /* save %g1, %g0, %sp */
1310 1344          self->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
1311 1345          self->ul_dftret = 0x91d0203a;   /* ta 0x3a */
1312 1346          self->ul_dreturn = 0x81ca0000;  /* return %o0 */
1313 1347  #endif
1314 1348  
1315 1349          self->ul_stktop = (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
1316 1350          (void) getrlimit(RLIMIT_STACK, &rl);
1317 1351          self->ul_stksiz = rl.rlim_cur;
1318 1352          self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz);
1319 1353  
1320 1354          self->ul_forw = self->ul_back = self;
1321 1355          self->ul_hash = NULL;
1322 1356          self->ul_ix = 0;
1323 1357          self->ul_lwpid = 1; /* _lwp_self() */
1324 1358          self->ul_main = 1;
1325 1359          self->ul_self = self;
1326 1360          self->ul_policy = -1;           /* initialize only when needed */
1327 1361          self->ul_pri = 0;
1328 1362          self->ul_cid = 0;
1329 1363          self->ul_rtclassid = -1;
1330 1364          self->ul_uberdata = udp;
1331 1365          if (oldself != NULL) {
1332 1366                  int i;
1333 1367  
1334 1368                  ASSERT(primary_link_map);
1335 1369                  ASSERT(oldself->ul_main == 1);
1336 1370                  self->ul_stsd = oldself->ul_stsd;
1337 1371                  for (i = 0; i < TSD_NFAST; i++)
1338 1372                          self->ul_ftsd[i] = oldself->ul_ftsd[i];
1339 1373                  self->ul_tls = oldself->ul_tls;
1340 1374                  /*
1341 1375                   * Retrieve all pointers to uberdata allocated
1342 1376                   * while running on previous link maps.
1343 1377                   * We would like to do a structure assignment here, but
1344 1378                   * gcc turns structure assignments into calls to memcpy(),
1345 1379                   * a function exported from libc.  We can't call any such
1346 1380                   * external functions until we establish curthread, below,
1347 1381                   * so we just call our private version of memcpy().
1348 1382                   */
1349 1383                  (void) memcpy(udp, oldself->ul_uberdata, sizeof (*udp));
1350 1384                  /*
1351 1385                   * These items point to global data on the primary link map.
1352 1386                   */
1353 1387                  udp->thr_hash_table = init_hash_table;
1354 1388                  udp->sigacthandler = sigacthandler;
1355 1389                  udp->tdb.tdb_events = tdb_events;
1356 1390                  ASSERT(udp->nthreads == 1 && !udp->uberflags.uf_mt);
1357 1391                  ASSERT(udp->lwp_stacks == NULL);
1358 1392                  ASSERT(udp->ulwp_freelist == NULL);
1359 1393                  ASSERT(udp->ulwp_replace_free == NULL);
1360 1394                  ASSERT(udp->hash_size == 1);
1361 1395          }
1362 1396          udp->all_lwps = self;
1363 1397          udp->ulwp_one = self;
1364 1398          udp->pid = getpid();
1365 1399          udp->nthreads = 1;
1366 1400          /*
1367 1401           * In every link map, tdb_bootstrap points to the same piece of
1368 1402           * allocated memory.  When the primary link map is initialized,
1369 1403           * the allocated memory is assigned a pointer to the one true
1370 1404           * uberdata.  This allows libc_db to initialize itself regardless
1371 1405           * of which instance of libc it finds in the address space.
1372 1406           */
1373 1407          if (udp->tdb_bootstrap == NULL)
1374 1408                  udp->tdb_bootstrap = lmalloc(sizeof (uberdata_t *));
1375 1409          __tdb_bootstrap = udp->tdb_bootstrap;
1376 1410          if (primary_link_map) {
1377 1411                  self->ul_primarymap = 1;
1378 1412                  udp->primary_map = 1;
1379 1413                  *udp->tdb_bootstrap = udp;
1380 1414          }
1381 1415          /*
1382 1416           * Cancellation can't happen until:
1383 1417           *      pthread_cancel() is called
1384 1418           * or:
1385 1419           *      another thread is created
1386 1420           * For now, as a single-threaded process, set the flag that tells
1387 1421           * PROLOGUE/EPILOGUE (in scalls.c) that cancellation can't happen.
1388 1422           */
1389 1423          self->ul_nocancel = 1;
1390 1424  
1391 1425  #if defined(__amd64)
1392 1426          (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_FSBASE, self);
1393 1427  #elif defined(__i386)
1394 1428          (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_GSBASE, self);
1395 1429  #endif  /* __i386 || __amd64 */
1396 1430          set_curthread(self);            /* redundant on i386 */
1397 1431          /*
1398 1432           * Now curthread is established and it is safe to call any
1399 1433           * function in libc except one that uses thread-local storage.
1400 1434           */
1401 1435          self->ul_errnop = &errno;
1402 1436          if (oldself != NULL) {
1403 1437                  /* tls_size was zero when oldself was allocated */
1404 1438                  lfree(oldself, sizeof (ulwp_t));
1405 1439          }
1406 1440          mutex_setup();
1407 1441          atfork_init();
1408 1442          signal_init();
1409 1443  
1410 1444          /*
1411 1445           * If the stack is unlimited, we set the size to zero to disable
1412 1446           * stack checking.
1413 1447           * XXX: Work harder here.  Get the stack size from /proc/self/rmap
1414 1448           */
1415 1449          if (self->ul_stksiz == RLIM_INFINITY) {
1416 1450                  self->ul_ustack.ss_sp = (void *)self->ul_stktop;
1417 1451                  self->ul_ustack.ss_size = 0;
1418 1452          } else {
1419 1453                  self->ul_ustack.ss_sp = self->ul_stk;
1420 1454                  self->ul_ustack.ss_size = self->ul_stksiz;
1421 1455          }
1422 1456          self->ul_ustack.ss_flags = 0;
1423 1457          (void) setustack(&self->ul_ustack);
1424 1458  
1425 1459          /*
1426 1460           * Get the variables that affect thread behavior from the environment.
1427 1461           */
1428 1462          set_thread_vars();
1429 1463          udp->uberflags.uf_thread_error_detection = (char)thread_error_detection;
1430 1464          udp->thread_stack_cache = thread_stack_cache;
1431 1465  
1432 1466          /*
1433 1467           * Make per-thread copies of global variables, for speed.
1434 1468           */
1435 1469          self->ul_queue_fifo = (char)thread_queue_fifo;
1436 1470          self->ul_cond_wait_defer = (char)thread_cond_wait_defer;
1437 1471          self->ul_error_detection = (char)thread_error_detection;
1438 1472          self->ul_async_safe = (char)thread_async_safe;
1439 1473          self->ul_door_noreserve = (char)thread_door_noreserve;
1440 1474          self->ul_misaligned = (char)thread_locks_misaligned;
1441 1475          self->ul_max_spinners = (uint8_t)thread_max_spinners;
1442 1476          self->ul_adaptive_spin = thread_adaptive_spin;
1443 1477          self->ul_queue_spin = thread_queue_spin;
1444 1478  
1445 1479  #if defined(__sparc) && !defined(_LP64)
1446 1480          if (self->ul_misaligned) {
1447 1481                  /*
1448 1482                   * Tell the kernel to fix up ldx/stx instructions that
1449 1483                   * refer to non-8-byte aligned data instead of giving
1450 1484                   * the process an alignment trap and generating SIGBUS.
1451 1485                   *
1452 1486                   * Programs compiled for 32-bit sparc with the Studio SS12
1453 1487                   * compiler get this done for them automatically (in _init()).
1454 1488                   * We do it here for the benefit of programs compiled with
1455 1489                   * other compilers, like gcc.
1456 1490                   *
1457 1491                   * This is necessary for the _THREAD_LOCKS_MISALIGNED=1
1458 1492                   * environment variable horrible hack to work.
1459 1493                   */
1460 1494                  extern void _do_fix_align(void);
1461 1495                  _do_fix_align();
1462 1496          }
1463 1497  #endif
1464 1498  
1465 1499          /*
1466 1500           * When we have initialized the primary link map, inform
1467 1501           * the dynamic linker about our interface functions.
1468 1502           * Set up our pointer to the program name.
1469 1503           */
1470 1504          if (self->ul_primarymap)
1471 1505                  _ld_libc((void *)rtld_funcs);
1472 1506          init_progname();
1473 1507  
1474 1508          /*
1475 1509           * Defer signals until TLS constructors have been called.
1476 1510           */
1477 1511          sigoff(self);
1478 1512          tls_setup();
1479 1513          sigon(self);
1480 1514          if (setmask)
1481 1515                  (void) restore_signals(self);
1482 1516  
1483 1517          /*
1484 1518           * Make private copies of __xpg4 and __xpg6 so libc can test
1485 1519           * them after this point without invoking the dynamic linker.
1486 1520           */
1487 1521          libc__xpg4 = __xpg4;
1488 1522          libc__xpg6 = __xpg6;
1489 1523  
1490 1524          /* PROBE_SUPPORT begin */
1491 1525          if (self->ul_primarymap && __tnf_probe_notify != NULL)
1492 1526                  __tnf_probe_notify();
1493 1527          /* PROBE_SUPPORT end */
1494 1528  
1495 1529          init_sigev_thread();
1496 1530          init_aio();
1497 1531  
1498 1532          /*
1499 1533           * We need to reset __threaded dynamically at runtime, so that
1500 1534           * __threaded can be bound to __threaded outside libc which may not
1501 1535           * have initial value of 1 (without a copy relocation in a.out).
1502 1536           */
1503 1537          __threaded = 1;
1504 1538  }
1505 1539  
1506 1540  #pragma fini(libc_fini)
1507 1541  void
1508 1542  libc_fini()
1509 1543  {
1510 1544          /*
1511 1545           * If we are doing fini processing for the instance of libc
1512 1546           * on the first alternate link map (this happens only when
1513 1547           * the dynamic linker rejects a bad audit library), then clear
1514 1548           * __curthread().  We abandon whatever memory was allocated by
1515 1549           * lmalloc() while running on this alternate link-map but we
1516 1550           * don't care (and can't find the memory in any case); we just
1517 1551           * want to protect the application from this bad audit library.
1518 1552           * No fini processing is done by libc in the normal case.
1519 1553           */
1520 1554  
1521 1555          uberdata_t *udp = curthread->ul_uberdata;
1522 1556  
1523 1557          if (udp->primary_map == 0 && udp == &__uberdata)
1524 1558                  set_curthread(NULL);
1525 1559  }
1526 1560  
1527 1561  /*
1528 1562   * finish_init is called when we are about to become multi-threaded,
1529 1563   * that is, on the first call to thr_create().
1530 1564   */
1531 1565  void
1532 1566  finish_init()
1533 1567  {
1534 1568          ulwp_t *self = curthread;
1535 1569          uberdata_t *udp = self->ul_uberdata;
1536 1570          thr_hash_table_t *htp;
1537 1571          void *data;
1538 1572          int i;
1539 1573  
1540 1574          /*
1541 1575           * No locks needed here; we are single-threaded on the first call.
1542 1576           * We can be called only after the primary link map has been set up.
1543 1577           */
1544 1578          ASSERT(self->ul_primarymap);
1545 1579          ASSERT(self == udp->ulwp_one);
1546 1580          ASSERT(!udp->uberflags.uf_mt);
1547 1581          ASSERT(udp->hash_size == 1);
1548 1582  
1549 1583          /*
1550 1584           * Initialize self->ul_policy, self->ul_cid, and self->ul_pri.
1551 1585           */
1552 1586          update_sched(self);
1553 1587  
1554 1588          /*
1555 1589           * Allocate the queue_head array if not already allocated.
1556 1590           */
1557 1591          if (udp->queue_head == NULL)
1558 1592                  queue_alloc();
1559 1593  
1560 1594          /*
1561 1595           * Now allocate the thread hash table.
1562 1596           */
1563 1597          if ((data = mmap(NULL, HASHTBLSZ * sizeof (thr_hash_table_t),
1564 1598              PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0))
1565 1599              == MAP_FAILED)
1566 1600                  thr_panic("cannot allocate thread hash table");
1567 1601  
1568 1602          udp->thr_hash_table = htp = (thr_hash_table_t *)data;
1569 1603          udp->hash_size = HASHTBLSZ;
1570 1604          udp->hash_mask = HASHTBLSZ - 1;
1571 1605  
1572 1606          for (i = 0; i < HASHTBLSZ; i++, htp++) {
1573 1607                  htp->hash_lock.mutex_flag = LOCK_INITED;
1574 1608                  htp->hash_lock.mutex_magic = MUTEX_MAGIC;
1575 1609                  htp->hash_cond.cond_magic = COND_MAGIC;
1576 1610          }
1577 1611          hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1578 1612  
1579 1613          /*
1580 1614           * Set up the SIGCANCEL handler for threads cancellation.
1581 1615           */
1582 1616          setup_cancelsig(SIGCANCEL);
1583 1617  
1584 1618          /*
1585 1619           * Arrange to do special things on exit --
1586 1620           * - collect queue statistics from all remaining active threads.
1587 1621           * - dump queue statistics to stderr if _THREAD_QUEUE_DUMP is set.
1588 1622           * - grab assert_lock to ensure that assertion failures
1589 1623           *   and a core dump take precedence over _exit().
1590 1624           * (Functions are called in the reverse order of their registration.)
1591 1625           */
1592 1626          (void) _atexit(grab_assert_lock);
1593 1627  #if defined(THREAD_DEBUG)
1594 1628          (void) _atexit(dump_queue_statistics);
1595 1629          (void) _atexit(collect_queue_statistics);
1596 1630  #endif
1597 1631  }
1598 1632  
1599 1633  /*
1600 1634   * Used only by postfork1_child(), below.
1601 1635   */
1602 1636  static void
1603 1637  mark_dead_and_buried(ulwp_t *ulwp)
1604 1638  {
1605 1639          ulwp->ul_dead = 1;
1606 1640          ulwp->ul_lwpid = (lwpid_t)(-1);
1607 1641          ulwp->ul_hash = NULL;
1608 1642          ulwp->ul_ix = -1;
1609 1643          ulwp->ul_schedctl = NULL;
1610 1644          ulwp->ul_schedctl_called = NULL;
1611 1645  }
1612 1646  
1613 1647  /*
1614 1648   * This is called from fork1() in the child.
1615 1649   * Reset our data structures to reflect one lwp.
1616 1650   */
1617 1651  void
1618 1652  postfork1_child()
1619 1653  {
1620 1654          ulwp_t *self = curthread;
1621 1655          uberdata_t *udp = self->ul_uberdata;
1622 1656          queue_head_t *qp;
1623 1657          ulwp_t *next;
1624 1658          ulwp_t *ulwp;
1625 1659          int i;
1626 1660  
1627 1661          /* daemon threads shouldn't call fork1(), but oh well... */
1628 1662          self->ul_usropts &= ~THR_DAEMON;
1629 1663          udp->nthreads = 1;
1630 1664          udp->ndaemons = 0;
1631 1665          udp->uberflags.uf_mt = 0;
1632 1666          __libc_threaded = 0;
1633 1667          for (i = 0; i < udp->hash_size; i++)
1634 1668                  udp->thr_hash_table[i].hash_bucket = NULL;
1635 1669          self->ul_lwpid = _lwp_self();
1636 1670          hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1637 1671  
1638 1672          /*
1639 1673           * Some thread in the parent might have been suspended
1640 1674           * while holding udp->callout_lock or udp->ld_lock.
1641 1675           * Reinitialize the child's copies.
1642 1676           */
1643 1677          (void) mutex_init(&udp->callout_lock,
1644 1678              USYNC_THREAD | LOCK_RECURSIVE, NULL);
1645 1679          (void) mutex_init(&udp->ld_lock,
1646 1680              USYNC_THREAD | LOCK_RECURSIVE, NULL);
1647 1681  
1648 1682          /* no one in the child is on a sleep queue; reinitialize */
1649 1683          if ((qp = udp->queue_head) != NULL) {
1650 1684                  (void) memset(qp, 0, 2 * QHASHSIZE * sizeof (queue_head_t));
1651 1685                  for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
1652 1686                          qp->qh_type = (i < QHASHSIZE)? MX : CV;
1653 1687                          qp->qh_lock.mutex_flag = LOCK_INITED;
1654 1688                          qp->qh_lock.mutex_magic = MUTEX_MAGIC;
1655 1689                          qp->qh_hlist = &qp->qh_def_root;
1656 1690  #if defined(THREAD_DEBUG)
1657 1691                          qp->qh_hlen = 1;
1658 1692                          qp->qh_hmax = 1;
1659 1693  #endif
1660 1694                  }
1661 1695          }
1662 1696  
1663 1697          /*
1664 1698           * Do post-fork1 processing for subsystems that need it.
1665 1699           * We need to do this before unmapping all of the abandoned
1666 1700           * threads' stacks, below(), because the post-fork1 actions
1667 1701           * might require access to those stacks.
1668 1702           */
1669 1703          postfork1_child_sigev_aio();
1670 1704          postfork1_child_sigev_mq();
1671 1705          postfork1_child_sigev_timer();
1672 1706          postfork1_child_aio();
1673 1707          /*
1674 1708           * The above subsystems use thread pools, so this action
1675 1709           * must be performed after those actions.
1676 1710           */
1677 1711          postfork1_child_tpool();
1678 1712  
1679 1713          /*
1680 1714           * All lwps except ourself are gone.  Mark them so.
1681 1715           * First mark all of the lwps that have already been freed.
1682 1716           * Then mark and free all of the active lwps except ourself.
1683 1717           * Since we are single-threaded, no locks are required here.
1684 1718           */
1685 1719          for (ulwp = udp->lwp_stacks; ulwp != NULL; ulwp = ulwp->ul_next)
1686 1720                  mark_dead_and_buried(ulwp);
1687 1721          for (ulwp = udp->ulwp_freelist; ulwp != NULL; ulwp = ulwp->ul_next)
1688 1722                  mark_dead_and_buried(ulwp);
1689 1723          for (ulwp = self->ul_forw; ulwp != self; ulwp = next) {
1690 1724                  next = ulwp->ul_forw;
1691 1725                  ulwp->ul_forw = ulwp->ul_back = NULL;
1692 1726                  mark_dead_and_buried(ulwp);
1693 1727                  tsd_free(ulwp);
1694 1728                  tls_free(ulwp);
1695 1729                  rwl_free(ulwp);
1696 1730                  heldlock_free(ulwp);
1697 1731                  ulwp_free(ulwp);
1698 1732          }
1699 1733          self->ul_forw = self->ul_back = udp->all_lwps = self;
1700 1734          if (self != udp->ulwp_one)
1701 1735                  mark_dead_and_buried(udp->ulwp_one);
1702 1736          if ((ulwp = udp->all_zombies) != NULL) {
1703 1737                  ASSERT(udp->nzombies != 0);
1704 1738                  do {
1705 1739                          next = ulwp->ul_forw;
1706 1740                          ulwp->ul_forw = ulwp->ul_back = NULL;
1707 1741                          mark_dead_and_buried(ulwp);
1708 1742                          udp->nzombies--;
1709 1743                          if (ulwp->ul_replace) {
1710 1744                                  ulwp->ul_next = NULL;
1711 1745                                  if (udp->ulwp_replace_free == NULL) {
1712 1746                                          udp->ulwp_replace_free =
1713 1747                                              udp->ulwp_replace_last = ulwp;
1714 1748                                  } else {
1715 1749                                          udp->ulwp_replace_last->ul_next = ulwp;
1716 1750                                          udp->ulwp_replace_last = ulwp;
1717 1751                                  }
1718 1752                          }
1719 1753                  } while ((ulwp = next) != udp->all_zombies);
1720 1754                  ASSERT(udp->nzombies == 0);
1721 1755                  udp->all_zombies = NULL;
1722 1756                  udp->nzombies = 0;
1723 1757          }
1724 1758          trim_stack_cache(0);
1725 1759  }
1726 1760  
1727 1761  lwpid_t
1728 1762  lwp_self(void)
1729 1763  {
1730 1764          return (curthread->ul_lwpid);
1731 1765  }
1732 1766  
1733 1767  #pragma weak _ti_thr_self = thr_self
1734 1768  #pragma weak pthread_self = thr_self
1735 1769  thread_t
1736 1770  thr_self()
1737 1771  {
1738 1772          return (curthread->ul_lwpid);
1739 1773  }
1740 1774  
1741 1775  int
1742 1776  thr_main()
1743 1777  {
1744 1778          ulwp_t *self = __curthread();
1745 1779  
1746 1780          return ((self == NULL)? -1 : self->ul_main);
1747 1781  }
1748 1782  
1749 1783  int
1750 1784  _thrp_cancelled(void)
1751 1785  {
1752 1786          return (curthread->ul_rval == PTHREAD_CANCELED);
1753 1787  }
1754 1788  
1755 1789  int
1756 1790  _thrp_stksegment(ulwp_t *ulwp, stack_t *stk)
1757 1791  {
1758 1792          stk->ss_sp = (void *)ulwp->ul_stktop;
1759 1793          stk->ss_size = ulwp->ul_stksiz;
1760 1794          stk->ss_flags = 0;
1761 1795          return (0);
1762 1796  }
1763 1797  
1764 1798  #pragma weak _thr_stksegment = thr_stksegment
1765 1799  int
1766 1800  thr_stksegment(stack_t *stk)
1767 1801  {
1768 1802          return (_thrp_stksegment(curthread, stk));
1769 1803  }
1770 1804  
1771 1805  void
1772 1806  force_continue(ulwp_t *ulwp)
1773 1807  {
1774 1808  #if defined(THREAD_DEBUG)
1775 1809          ulwp_t *self = curthread;
1776 1810          uberdata_t *udp = self->ul_uberdata;
1777 1811  #endif
1778 1812          int error;
1779 1813          timespec_t ts;
1780 1814  
1781 1815          ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1782 1816          ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
1783 1817  
1784 1818          for (;;) {
1785 1819                  error = _lwp_continue(ulwp->ul_lwpid);
1786 1820                  if (error != 0 && error != EINTR)
1787 1821                          break;
1788 1822                  error = 0;
1789 1823                  if (ulwp->ul_stopping) {        /* he is stopping himself */
1790 1824                          ts.tv_sec = 0;          /* give him a chance to run */
1791 1825                          ts.tv_nsec = 100000;    /* 100 usecs or clock tick */
1792 1826                          (void) __nanosleep(&ts, NULL);
1793 1827                  }
1794 1828                  if (!ulwp->ul_stopping)         /* he is running now */
1795 1829                          break;                  /* so we are done */
1796 1830                  /*
1797 1831                   * He is marked as being in the process of stopping
1798 1832                   * himself.  Loop around and continue him again.
1799 1833                   * He may not have been stopped the first time.
1800 1834                   */
1801 1835          }
1802 1836  }
1803 1837  
1804 1838  /*
1805 1839   * Suspend an lwp with lwp_suspend(), then move it to a safe point,
1806 1840   * that is, to a point where ul_critical and ul_rtld are both zero.
1807 1841   * On return, the ulwp_lock() is dropped as with ulwp_unlock().
1808 1842   * If 'link_dropped' is non-NULL, then 'link_lock' is held on entry.
1809 1843   * If we have to drop link_lock, we store 1 through link_dropped.
1810 1844   * If the lwp exits before it can be suspended, we return ESRCH.
1811 1845   */
1812 1846  int
1813 1847  safe_suspend(ulwp_t *ulwp, uchar_t whystopped, int *link_dropped)
1814 1848  {
1815 1849          ulwp_t *self = curthread;
1816 1850          uberdata_t *udp = self->ul_uberdata;
1817 1851          cond_t *cvp = ulwp_condvar(ulwp, udp);
1818 1852          mutex_t *mp = ulwp_mutex(ulwp, udp);
1819 1853          thread_t tid = ulwp->ul_lwpid;
1820 1854          int ix = ulwp->ul_ix;
1821 1855          int error = 0;
1822 1856  
1823 1857          ASSERT(whystopped == TSTP_REGULAR ||
1824 1858              whystopped == TSTP_MUTATOR ||
1825 1859              whystopped == TSTP_FORK);
1826 1860          ASSERT(ulwp != self);
1827 1861          ASSERT(!ulwp->ul_stop);
1828 1862          ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1829 1863          ASSERT(MUTEX_OWNED(mp, self));
1830 1864  
1831 1865          if (link_dropped != NULL)
1832 1866                  *link_dropped = 0;
1833 1867  
1834 1868          /*
1835 1869           * We must grab the target's spin lock before suspending it.
1836 1870           * See the comments below and in _thrp_suspend() for why.
1837 1871           */
1838 1872          spin_lock_set(&ulwp->ul_spinlock);
1839 1873          (void) ___lwp_suspend(tid);
1840 1874          spin_lock_clear(&ulwp->ul_spinlock);
1841 1875  
1842 1876  top:
1843 1877          if ((ulwp->ul_critical == 0 && ulwp->ul_rtld == 0) ||
1844 1878              ulwp->ul_stopping) {
1845 1879                  /* thread is already safe */
1846 1880                  ulwp->ul_stop |= whystopped;
1847 1881          } else {
1848 1882                  /*
1849 1883                   * Setting ul_pleasestop causes the target thread to stop
1850 1884                   * itself in _thrp_suspend(), below, after we drop its lock.
1851 1885                   * We must continue the critical thread before dropping
1852 1886                   * link_lock because the critical thread may be holding
1853 1887                   * the queue lock for link_lock.  This is delicate.
1854 1888                   */
1855 1889                  ulwp->ul_pleasestop |= whystopped;
1856 1890                  force_continue(ulwp);
1857 1891                  if (link_dropped != NULL) {
1858 1892                          *link_dropped = 1;
1859 1893                          lmutex_unlock(&udp->link_lock);
1860 1894                          /* be sure to drop link_lock only once */
1861 1895                          link_dropped = NULL;
1862 1896                  }
1863 1897  
1864 1898                  /*
1865 1899                   * The thread may disappear by calling thr_exit() so we
1866 1900                   * cannot rely on the ulwp pointer after dropping the lock.
1867 1901                   * Instead, we search the hash table to find it again.
1868 1902                   * When we return, we may find that the thread has been
1869 1903                   * continued by some other thread.  The suspend/continue
1870 1904                   * interfaces are prone to such race conditions by design.
1871 1905                   */
1872 1906                  while (ulwp && !ulwp->ul_dead && !ulwp->ul_stop &&
1873 1907                      (ulwp->ul_pleasestop & whystopped)) {
1874 1908                          (void) __cond_wait(cvp, mp);
1875 1909                          for (ulwp = udp->thr_hash_table[ix].hash_bucket;
1876 1910                              ulwp != NULL; ulwp = ulwp->ul_hash) {
1877 1911                                  if (ulwp->ul_lwpid == tid)
1878 1912                                          break;
1879 1913                          }
1880 1914                  }
1881 1915  
1882 1916                  if (ulwp == NULL || ulwp->ul_dead)
1883 1917                          error = ESRCH;
1884 1918                  else {
1885 1919                          /*
1886 1920                           * Do another lwp_suspend() to make sure we don't
1887 1921                           * return until the target thread is fully stopped
1888 1922                           * in the kernel.  Don't apply lwp_suspend() until
1889 1923                           * we know that the target is not holding any
1890 1924                           * queue locks, that is, that it has completed
1891 1925                           * ulwp_unlock(self) and has, or at least is
1892 1926                           * about to, call lwp_suspend() on itself.  We do
1893 1927                           * this by grabbing the target's spin lock.
1894 1928                           */
1895 1929                          ASSERT(ulwp->ul_lwpid == tid);
1896 1930                          spin_lock_set(&ulwp->ul_spinlock);
1897 1931                          (void) ___lwp_suspend(tid);
1898 1932                          spin_lock_clear(&ulwp->ul_spinlock);
1899 1933                          /*
1900 1934                           * If some other thread did a thr_continue()
1901 1935                           * on the target thread we have to start over.
1902 1936                           */
1903 1937                          if (!ulwp->ul_stopping || !(ulwp->ul_stop & whystopped))
1904 1938                                  goto top;
1905 1939                  }
1906 1940          }
1907 1941  
1908 1942          (void) cond_broadcast(cvp);
1909 1943          lmutex_unlock(mp);
1910 1944          return (error);
1911 1945  }
1912 1946  
1913 1947  int
1914 1948  _thrp_suspend(thread_t tid, uchar_t whystopped)
1915 1949  {
1916 1950          ulwp_t *self = curthread;
1917 1951          uberdata_t *udp = self->ul_uberdata;
1918 1952          ulwp_t *ulwp;
1919 1953          int error = 0;
1920 1954  
1921 1955          ASSERT((whystopped & (TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) != 0);
1922 1956          ASSERT((whystopped & ~(TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) == 0);
1923 1957  
1924 1958          /*
1925 1959           * We can't suspend anyone except ourself while
1926 1960           * some other thread is performing a fork.
1927 1961           * This also allows only one suspension at a time.
1928 1962           */
1929 1963          if (tid != self->ul_lwpid)
1930 1964                  fork_lock_enter();
1931 1965  
1932 1966          if ((ulwp = find_lwp(tid)) == NULL)
1933 1967                  error = ESRCH;
1934 1968          else if (whystopped == TSTP_MUTATOR && !ulwp->ul_mutator) {
1935 1969                  ulwp_unlock(ulwp, udp);
1936 1970                  error = EINVAL;
1937 1971          } else if (ulwp->ul_stop) {     /* already stopped */
1938 1972                  ulwp->ul_stop |= whystopped;
1939 1973                  ulwp_broadcast(ulwp);
1940 1974                  ulwp_unlock(ulwp, udp);
1941 1975          } else if (ulwp != self) {
1942 1976                  /*
1943 1977                   * After suspending the other thread, move it out of a
1944 1978                   * critical section and deal with the schedctl mappings.
1945 1979                   * safe_suspend() suspends the other thread, calls
1946 1980                   * ulwp_broadcast(ulwp) and drops the ulwp lock.
1947 1981                   */
1948 1982                  error = safe_suspend(ulwp, whystopped, NULL);
1949 1983          } else {
1950 1984                  int schedctl_after_fork = 0;
1951 1985  
1952 1986                  /*
1953 1987                   * We are suspending ourself.  We must not take a signal
1954 1988                   * until we return from lwp_suspend() and clear ul_stopping.
1955 1989                   * This is to guard against siglongjmp().
1956 1990                   */
1957 1991                  enter_critical(self);
1958 1992                  self->ul_sp = stkptr();
1959 1993                  _flush_windows();       /* sparc */
1960 1994                  self->ul_pleasestop = 0;
1961 1995                  self->ul_stop |= whystopped;
1962 1996                  /*
1963 1997                   * Grab our spin lock before dropping ulwp_mutex(self).
1964 1998                   * This prevents the suspending thread from applying
1965 1999                   * lwp_suspend() to us before we emerge from
1966 2000                   * lmutex_unlock(mp) and have dropped mp's queue lock.
1967 2001                   */
1968 2002                  spin_lock_set(&self->ul_spinlock);
1969 2003                  self->ul_stopping = 1;
1970 2004                  ulwp_broadcast(self);
1971 2005                  ulwp_unlock(self, udp);
1972 2006                  /*
1973 2007                   * From this point until we return from lwp_suspend(),
1974 2008                   * we must not call any function that might invoke the
1975 2009                   * dynamic linker, that is, we can only call functions
1976 2010                   * private to the library.
1977 2011                   *
1978 2012                   * Also, this is a nasty race condition for a process
1979 2013                   * that is undergoing a forkall() operation:
1980 2014                   * Once we clear our spinlock (below), we are vulnerable
1981 2015                   * to being suspended by the forkall() thread before
1982 2016                   * we manage to suspend ourself in ___lwp_suspend().
1983 2017                   * See safe_suspend() and force_continue().
1984 2018                   *
1985 2019                   * To avoid a SIGSEGV due to the disappearance
1986 2020                   * of the schedctl mappings in the child process,
1987 2021                   * which can happen in spin_lock_clear() if we
1988 2022                   * are suspended while we are in the middle of
1989 2023                   * its call to preempt(), we preemptively clear
1990 2024                   * our own schedctl pointer before dropping our
1991 2025                   * spinlock.  We reinstate it, in both the parent
1992 2026                   * and (if this really is a forkall()) the child.
1993 2027                   */
1994 2028                  if (whystopped & TSTP_FORK) {
1995 2029                          schedctl_after_fork = 1;
1996 2030                          self->ul_schedctl = NULL;
1997 2031                          self->ul_schedctl_called = &udp->uberflags;
1998 2032                  }
1999 2033                  spin_lock_clear(&self->ul_spinlock);
2000 2034                  (void) ___lwp_suspend(tid);
2001 2035                  /*
2002 2036                   * Somebody else continued us.
2003 2037                   * We can't grab ulwp_lock(self)
2004 2038                   * until after clearing ul_stopping.
2005 2039                   * force_continue() relies on this.
2006 2040                   */
2007 2041                  self->ul_stopping = 0;
2008 2042                  self->ul_sp = 0;
2009 2043                  if (schedctl_after_fork) {
2010 2044                          self->ul_schedctl_called = NULL;
2011 2045                          self->ul_schedctl = NULL;
2012 2046                          (void) setup_schedctl();
2013 2047                  }
2014 2048                  ulwp_lock(self, udp);
2015 2049                  ulwp_broadcast(self);
2016 2050                  ulwp_unlock(self, udp);
2017 2051                  exit_critical(self);
2018 2052          }
2019 2053  
2020 2054          if (tid != self->ul_lwpid)
2021 2055                  fork_lock_exit();
2022 2056  
2023 2057          return (error);
2024 2058  }
2025 2059  
2026 2060  /*
2027 2061   * Suspend all lwps other than ourself in preparation for fork.
2028 2062   */
2029 2063  void
2030 2064  suspend_fork()
2031 2065  {
2032 2066          ulwp_t *self = curthread;
2033 2067          uberdata_t *udp = self->ul_uberdata;
2034 2068          ulwp_t *ulwp;
2035 2069          int link_dropped;
2036 2070  
2037 2071          ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2038 2072  top:
2039 2073          lmutex_lock(&udp->link_lock);
2040 2074  
2041 2075          for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2042 2076                  ulwp_lock(ulwp, udp);
2043 2077                  if (ulwp->ul_stop) {    /* already stopped */
2044 2078                          ulwp->ul_stop |= TSTP_FORK;
2045 2079                          ulwp_broadcast(ulwp);
2046 2080                          ulwp_unlock(ulwp, udp);
2047 2081                  } else {
2048 2082                          /*
2049 2083                           * Move the stopped lwp out of a critical section.
2050 2084                           */
2051 2085                          if (safe_suspend(ulwp, TSTP_FORK, &link_dropped) ||
2052 2086                              link_dropped)
2053 2087                                  goto top;
2054 2088                  }
2055 2089          }
2056 2090  
2057 2091          lmutex_unlock(&udp->link_lock);
2058 2092  }
2059 2093  
2060 2094  void
2061 2095  continue_fork(int child)
2062 2096  {
2063 2097          ulwp_t *self = curthread;
2064 2098          uberdata_t *udp = self->ul_uberdata;
2065 2099          ulwp_t *ulwp;
2066 2100  
2067 2101          ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2068 2102  
2069 2103          /*
2070 2104           * Clear the schedctl pointers in the child of forkall().
2071 2105           */
2072 2106          if (child) {
2073 2107                  for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2074 2108                          ulwp->ul_schedctl_called =
2075 2109                              ulwp->ul_dead? &udp->uberflags : NULL;
2076 2110                          ulwp->ul_schedctl = NULL;
2077 2111                  }
2078 2112          }
2079 2113  
2080 2114          /*
2081 2115           * Set all lwps that were stopped for fork() running again.
2082 2116           */
2083 2117          lmutex_lock(&udp->link_lock);
2084 2118          for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2085 2119                  mutex_t *mp = ulwp_mutex(ulwp, udp);
2086 2120                  lmutex_lock(mp);
2087 2121                  ASSERT(ulwp->ul_stop & TSTP_FORK);
2088 2122                  ulwp->ul_stop &= ~TSTP_FORK;
2089 2123                  ulwp_broadcast(ulwp);
2090 2124                  if (!ulwp->ul_stop)
2091 2125                          force_continue(ulwp);
2092 2126                  lmutex_unlock(mp);
2093 2127          }
2094 2128          lmutex_unlock(&udp->link_lock);
2095 2129  }
2096 2130  
2097 2131  int
2098 2132  _thrp_continue(thread_t tid, uchar_t whystopped)
2099 2133  {
2100 2134          uberdata_t *udp = curthread->ul_uberdata;
2101 2135          ulwp_t *ulwp;
2102 2136          mutex_t *mp;
2103 2137          int error = 0;
2104 2138  
2105 2139          ASSERT(whystopped == TSTP_REGULAR ||
2106 2140              whystopped == TSTP_MUTATOR);
2107 2141  
2108 2142          /*
2109 2143           * We single-thread the entire thread suspend/continue mechanism.
2110 2144           */
2111 2145          fork_lock_enter();
2112 2146  
2113 2147          if ((ulwp = find_lwp(tid)) == NULL) {
2114 2148                  fork_lock_exit();
2115 2149                  return (ESRCH);
2116 2150          }
2117 2151  
2118 2152          mp = ulwp_mutex(ulwp, udp);
2119 2153          if ((whystopped == TSTP_MUTATOR && !ulwp->ul_mutator)) {
2120 2154                  error = EINVAL;
2121 2155          } else if (ulwp->ul_stop & whystopped) {
2122 2156                  ulwp->ul_stop &= ~whystopped;
2123 2157                  ulwp_broadcast(ulwp);
2124 2158                  if (!ulwp->ul_stop) {
2125 2159                          if (whystopped == TSTP_REGULAR && ulwp->ul_created) {
2126 2160                                  ulwp->ul_sp = 0;
2127 2161                                  ulwp->ul_created = 0;
2128 2162                          }
2129 2163                          force_continue(ulwp);
2130 2164                  }
2131 2165          }
2132 2166          lmutex_unlock(mp);
2133 2167  
2134 2168          fork_lock_exit();
2135 2169          return (error);
2136 2170  }
2137 2171  
2138 2172  int
2139 2173  thr_suspend(thread_t tid)
2140 2174  {
2141 2175          return (_thrp_suspend(tid, TSTP_REGULAR));
2142 2176  }
2143 2177  
2144 2178  int
2145 2179  thr_continue(thread_t tid)
2146 2180  {
2147 2181          return (_thrp_continue(tid, TSTP_REGULAR));
2148 2182  }
2149 2183  
2150 2184  void
2151 2185  thr_yield()
2152 2186  {
2153 2187          yield();
2154 2188  }
2155 2189  
2156 2190  #pragma weak pthread_kill = thr_kill
2157 2191  #pragma weak _thr_kill = thr_kill
2158 2192  int
2159 2193  thr_kill(thread_t tid, int sig)
2160 2194  {
2161 2195          if (sig == SIGCANCEL)
2162 2196                  return (EINVAL);
2163 2197          return (_lwp_kill(tid, sig));
2164 2198  }
2165 2199  
2166 2200  /*
2167 2201   * Exit a critical section, take deferred actions if necessary.
2168 2202   * Called from exit_critical() and from sigon().
2169 2203   */
2170 2204  void
2171 2205  do_exit_critical()
2172 2206  {
2173 2207          ulwp_t *self = curthread;
2174 2208          int sig;
2175 2209  
2176 2210          ASSERT(self->ul_critical == 0);
2177 2211  
2178 2212          /*
2179 2213           * Don't suspend ourself or take a deferred signal while dying
2180 2214           * or while executing inside the dynamic linker (ld.so.1).
2181 2215           */
2182 2216          if (self->ul_dead || self->ul_rtld)
2183 2217                  return;
2184 2218  
2185 2219          while (self->ul_pleasestop ||
2186 2220              (self->ul_cursig != 0 && self->ul_sigdefer == 0)) {
2187 2221                  /*
2188 2222                   * Avoid a recursive call to exit_critical() in _thrp_suspend()
2189 2223                   * by keeping self->ul_critical == 1 here.
2190 2224                   */
2191 2225                  self->ul_critical++;
2192 2226                  while (self->ul_pleasestop) {
2193 2227                          /*
2194 2228                           * Guard against suspending ourself while on a sleep
2195 2229                           * queue.  See the comments in call_user_handler().
2196 2230                           */
2197 2231                          unsleep_self();
2198 2232                          set_parking_flag(self, 0);
2199 2233                          (void) _thrp_suspend(self->ul_lwpid,
2200 2234                              self->ul_pleasestop);
2201 2235                  }
2202 2236                  self->ul_critical--;
2203 2237  
2204 2238                  if ((sig = self->ul_cursig) != 0 && self->ul_sigdefer == 0) {
2205 2239                          /*
2206 2240                           * Clear ul_cursig before proceeding.
2207 2241                           * This protects us from the dynamic linker's
2208 2242                           * calls to bind_guard()/bind_clear() in the
2209 2243                           * event that it is invoked to resolve a symbol
2210 2244                           * like take_deferred_signal() below.
2211 2245                           */
2212 2246                          self->ul_cursig = 0;
2213 2247                          take_deferred_signal(sig);
2214 2248                          ASSERT(self->ul_cursig == 0);
2215 2249                  }
2216 2250          }
2217 2251          ASSERT(self->ul_critical == 0);
2218 2252  }
2219 2253  
2220 2254  /*
2221 2255   * _ti_bind_guard() and _ti_bind_clear() are called by the dynamic linker
2222 2256   * (ld.so.1) when it has do do something, like resolve a symbol to be called
2223 2257   * by the application or one of its libraries.  _ti_bind_guard() is called
2224 2258   * on entry to ld.so.1, _ti_bind_clear() on exit from ld.so.1 back to the
2225 2259   * application.  The dynamic linker gets special dispensation from libc to
2226 2260   * run in a critical region (all signals deferred and no thread suspension
2227 2261   * or forking allowed), and to be immune from cancellation for the duration.
2228 2262   */
2229 2263  int
2230 2264  _ti_bind_guard(int flags)
2231 2265  {
2232 2266          ulwp_t *self = curthread;
2233 2267          uberdata_t *udp = self->ul_uberdata;
2234 2268          int bindflag = (flags & THR_FLG_RTLD);
2235 2269  
2236 2270          if ((self->ul_bindflags & bindflag) == bindflag)
2237 2271                  return (0);
2238 2272          self->ul_bindflags |= bindflag;
2239 2273          if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2240 2274                  sigoff(self);   /* see no signals while holding ld_lock */
2241 2275                  self->ul_rtld++;        /* don't suspend while in ld.so.1 */
2242 2276                  (void) mutex_lock(&udp->ld_lock);
2243 2277          }
2244 2278          enter_critical(self);
2245 2279          self->ul_save_state = self->ul_cancel_disabled;
2246 2280          self->ul_cancel_disabled = 1;
2247 2281          set_cancel_pending_flag(self, 0);
2248 2282          return (1);
2249 2283  }
2250 2284  
2251 2285  int
2252 2286  _ti_bind_clear(int flags)
2253 2287  {
2254 2288          ulwp_t *self = curthread;
2255 2289          uberdata_t *udp = self->ul_uberdata;
2256 2290          int bindflag = (flags & THR_FLG_RTLD);
2257 2291  
2258 2292          if ((self->ul_bindflags & bindflag) == 0)
2259 2293                  return (self->ul_bindflags);
2260 2294          self->ul_bindflags &= ~bindflag;
2261 2295          self->ul_cancel_disabled = self->ul_save_state;
2262 2296          set_cancel_pending_flag(self, 0);
2263 2297          exit_critical(self);
2264 2298          if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2265 2299                  if (MUTEX_OWNED(&udp->ld_lock, self)) {
2266 2300                          (void) mutex_unlock(&udp->ld_lock);
2267 2301                          self->ul_rtld--;
2268 2302                          sigon(self);    /* reenable signals */
2269 2303                  }
2270 2304          }
2271 2305          return (self->ul_bindflags);
2272 2306  }
2273 2307  
2274 2308  /*
2275 2309   * Tell the dynamic linker (ld.so.1) whether or not it was entered from
2276 2310   * a critical region in libc.  Return zero if not, else return non-zero.
2277 2311   */
2278 2312  int
2279 2313  _ti_critical(void)
2280 2314  {
2281 2315          ulwp_t *self = curthread;
2282 2316          int level = self->ul_critical;
2283 2317  
2284 2318          if ((self->ul_bindflags & THR_FLG_RTLD) == 0 || level == 0)
2285 2319                  return (level); /* ld.so.1 hasn't (yet) called enter() */
2286 2320          return (level - 1);
2287 2321  }
2288 2322  
2289 2323  /*
2290 2324   * sigoff() and sigon() enable cond_wait() to behave (optionally) like
2291 2325   * it does in the old libthread (see the comments in cond_wait_queue()).
2292 2326   * Also, signals are deferred at thread startup until TLS constructors
2293 2327   * have all been called, at which time _thrp_setup() calls sigon().
2294 2328   *
2295 2329   * _sigoff() and _sigon() are external consolidation-private interfaces to
2296 2330   * sigoff() and sigon(), respectively, in libc.  These are used in libnsl.
2297 2331   * Also, _sigoff() and _sigon() are called from dbx's run-time checking
2298 2332   * (librtc.so) to defer signals during its critical sections (not to be
2299 2333   * confused with libc critical sections [see exit_critical() above]).
2300 2334   */
2301 2335  void
2302 2336  _sigoff(void)
2303 2337  {
2304 2338          ulwp_t *self = curthread;
2305 2339  
2306 2340          sigoff(self);
2307 2341  }
2308 2342  
2309 2343  void
2310 2344  _sigon(void)
2311 2345  {
2312 2346          ulwp_t *self = curthread;
2313 2347  
2314 2348          ASSERT(self->ul_sigdefer > 0);
2315 2349          sigon(self);
2316 2350  }
2317 2351  
2318 2352  int
2319 2353  thr_getconcurrency()
2320 2354  {
2321 2355          return (thr_concurrency);
2322 2356  }
2323 2357  
2324 2358  int
2325 2359  pthread_getconcurrency()
2326 2360  {
2327 2361          return (pthread_concurrency);
2328 2362  }
2329 2363  
2330 2364  int
2331 2365  thr_setconcurrency(int new_level)
2332 2366  {
2333 2367          uberdata_t *udp = curthread->ul_uberdata;
2334 2368  
2335 2369          if (new_level < 0)
2336 2370                  return (EINVAL);
2337 2371          if (new_level > 65536)          /* 65536 is totally arbitrary */
2338 2372                  return (EAGAIN);
2339 2373          lmutex_lock(&udp->link_lock);
2340 2374          if (new_level > thr_concurrency)
2341 2375                  thr_concurrency = new_level;
2342 2376          lmutex_unlock(&udp->link_lock);
2343 2377          return (0);
2344 2378  }
2345 2379  
2346 2380  int
2347 2381  pthread_setconcurrency(int new_level)
2348 2382  {
2349 2383          if (new_level < 0)
2350 2384                  return (EINVAL);
2351 2385          if (new_level > 65536)          /* 65536 is totally arbitrary */
2352 2386                  return (EAGAIN);
2353 2387          pthread_concurrency = new_level;
2354 2388          return (0);
2355 2389  }
2356 2390  
2357 2391  size_t
2358 2392  thr_min_stack(void)
2359 2393  {
2360 2394          return (MINSTACK);
2361 2395  }
2362 2396  
2363 2397  int
2364 2398  __nthreads(void)
2365 2399  {
2366 2400          return (curthread->ul_uberdata->nthreads);
2367 2401  }
2368 2402  
2369 2403  /*
2370 2404   * XXX
2371 2405   * The remainder of this file implements the private interfaces to java for
2372 2406   * garbage collection.  It is no longer used, at least by java 1.2.
2373 2407   * It can all go away once all old JVMs have disappeared.
2374 2408   */
2375 2409  
2376 2410  int     suspendingallmutators;  /* when non-zero, suspending all mutators. */
2377 2411  int     suspendedallmutators;   /* when non-zero, all mutators suspended. */
2378 2412  int     mutatorsbarrier;        /* when non-zero, mutators barrier imposed. */
2379 2413  mutex_t mutatorslock = DEFAULTMUTEX;    /* used to enforce mutators barrier. */
2380 2414  cond_t  mutatorscv = DEFAULTCV;         /* where non-mutators sleep. */
2381 2415  
2382 2416  /*
2383 2417   * Get the available register state for the target thread.
2384 2418   * Return non-volatile registers: TRS_NONVOLATILE
2385 2419   */
2386 2420  #pragma weak _thr_getstate = thr_getstate
2387 2421  int
2388 2422  thr_getstate(thread_t tid, int *flag, lwpid_t *lwp, stack_t *ss, gregset_t rs)
2389 2423  {
2390 2424          ulwp_t *self = curthread;
2391 2425          uberdata_t *udp = self->ul_uberdata;
2392 2426          ulwp_t **ulwpp;
2393 2427          ulwp_t *ulwp;
2394 2428          int error = 0;
2395 2429          int trs_flag = TRS_LWPID;
2396 2430  
2397 2431          if (tid == 0 || self->ul_lwpid == tid) {
2398 2432                  ulwp = self;
2399 2433                  ulwp_lock(ulwp, udp);
2400 2434          } else if ((ulwpp = find_lwpp(tid)) != NULL) {
2401 2435                  ulwp = *ulwpp;
2402 2436          } else {
2403 2437                  if (flag)
2404 2438                          *flag = TRS_INVALID;
2405 2439                  return (ESRCH);
2406 2440          }
2407 2441  
2408 2442          if (ulwp->ul_dead) {
2409 2443                  trs_flag = TRS_INVALID;
2410 2444          } else if (!ulwp->ul_stop && !suspendedallmutators) {
2411 2445                  error = EINVAL;
2412 2446                  trs_flag = TRS_INVALID;
2413 2447          } else if (ulwp->ul_stop) {
2414 2448                  trs_flag = TRS_NONVOLATILE;
2415 2449                  getgregs(ulwp, rs);
2416 2450          }
2417 2451  
2418 2452          if (flag)
2419 2453                  *flag = trs_flag;
2420 2454          if (lwp)
2421 2455                  *lwp = tid;
2422 2456          if (ss != NULL)
2423 2457                  (void) _thrp_stksegment(ulwp, ss);
2424 2458  
2425 2459          ulwp_unlock(ulwp, udp);
2426 2460          return (error);
2427 2461  }
2428 2462  
2429 2463  /*
2430 2464   * Set the appropriate register state for the target thread.
2431 2465   * This is not used by java.  It exists solely for the MSTC test suite.
2432 2466   */
2433 2467  #pragma weak _thr_setstate = thr_setstate
2434 2468  int
2435 2469  thr_setstate(thread_t tid, int flag, gregset_t rs)
2436 2470  {
2437 2471          uberdata_t *udp = curthread->ul_uberdata;
2438 2472          ulwp_t *ulwp;
2439 2473          int error = 0;
2440 2474  
2441 2475          if ((ulwp = find_lwp(tid)) == NULL)
2442 2476                  return (ESRCH);
2443 2477  
2444 2478          if (!ulwp->ul_stop && !suspendedallmutators)
2445 2479                  error = EINVAL;
2446 2480          else if (rs != NULL) {
2447 2481                  switch (flag) {
2448 2482                  case TRS_NONVOLATILE:
2449 2483                          /* do /proc stuff here? */
2450 2484                          if (ulwp->ul_stop)
2451 2485                                  setgregs(ulwp, rs);
2452 2486                          else
2453 2487                                  error = EINVAL;
2454 2488                          break;
2455 2489                  case TRS_LWPID:         /* do /proc stuff here? */
2456 2490                  default:
2457 2491                          error = EINVAL;
2458 2492                          break;
2459 2493                  }
2460 2494          }
2461 2495  
2462 2496          ulwp_unlock(ulwp, udp);
2463 2497          return (error);
2464 2498  }
2465 2499  
2466 2500  int
2467 2501  getlwpstatus(thread_t tid, struct lwpstatus *sp)
2468 2502  {
2469 2503          extern ssize_t __pread(int, void *, size_t, off_t);
2470 2504          char buf[100];
2471 2505          int fd;
2472 2506  
2473 2507          /* "/proc/self/lwp/%u/lwpstatus" w/o stdio */
2474 2508          (void) strcpy(buf, "/proc/self/lwp/");
2475 2509          ultos((uint64_t)tid, 10, buf + strlen(buf));
2476 2510          (void) strcat(buf, "/lwpstatus");
2477 2511          if ((fd = __open(buf, O_RDONLY, 0)) >= 0) {
2478 2512                  while (__pread(fd, sp, sizeof (*sp), 0) == sizeof (*sp)) {
2479 2513                          if (sp->pr_flags & PR_STOPPED) {
2480 2514                                  (void) __close(fd);
2481 2515                                  return (0);
2482 2516                          }
2483 2517                          yield();        /* give him a chance to stop */
2484 2518                  }
2485 2519                  (void) __close(fd);
2486 2520          }
2487 2521          return (-1);
2488 2522  }
2489 2523  
2490 2524  int
2491 2525  putlwpregs(thread_t tid, prgregset_t prp)
2492 2526  {
2493 2527          extern ssize_t __writev(int, const struct iovec *, int);
2494 2528          char buf[100];
2495 2529          int fd;
2496 2530          long dstop_sreg[2];
2497 2531          long run_null[2];
2498 2532          iovec_t iov[3];
2499 2533  
2500 2534          /* "/proc/self/lwp/%u/lwpctl" w/o stdio */
2501 2535          (void) strcpy(buf, "/proc/self/lwp/");
2502 2536          ultos((uint64_t)tid, 10, buf + strlen(buf));
2503 2537          (void) strcat(buf, "/lwpctl");
2504 2538          if ((fd = __open(buf, O_WRONLY, 0)) >= 0) {
2505 2539                  dstop_sreg[0] = PCDSTOP;        /* direct it to stop */
2506 2540                  dstop_sreg[1] = PCSREG;         /* set the registers */
2507 2541                  iov[0].iov_base = (caddr_t)dstop_sreg;
2508 2542                  iov[0].iov_len = sizeof (dstop_sreg);
2509 2543                  iov[1].iov_base = (caddr_t)prp; /* from the register set */
2510 2544                  iov[1].iov_len = sizeof (prgregset_t);
2511 2545                  run_null[0] = PCRUN;            /* make it runnable again */
2512 2546                  run_null[1] = 0;
2513 2547                  iov[2].iov_base = (caddr_t)run_null;
2514 2548                  iov[2].iov_len = sizeof (run_null);
2515 2549                  if (__writev(fd, iov, 3) >= 0) {
2516 2550                          (void) __close(fd);
2517 2551                          return (0);
2518 2552                  }
2519 2553                  (void) __close(fd);
2520 2554          }
2521 2555          return (-1);
2522 2556  }
2523 2557  
2524 2558  static ulong_t
2525 2559  gettsp_slow(thread_t tid)
2526 2560  {
2527 2561          char buf[100];
2528 2562          struct lwpstatus status;
2529 2563  
2530 2564          if (getlwpstatus(tid, &status) != 0) {
2531 2565                  /* "__gettsp(%u): can't read lwpstatus" w/o stdio */
2532 2566                  (void) strcpy(buf, "__gettsp(");
2533 2567                  ultos((uint64_t)tid, 10, buf + strlen(buf));
2534 2568                  (void) strcat(buf, "): can't read lwpstatus");
2535 2569                  thr_panic(buf);
2536 2570          }
2537 2571          return (status.pr_reg[R_SP]);
2538 2572  }
2539 2573  
2540 2574  ulong_t
2541 2575  __gettsp(thread_t tid)
2542 2576  {
2543 2577          uberdata_t *udp = curthread->ul_uberdata;
2544 2578          ulwp_t *ulwp;
2545 2579          ulong_t result;
2546 2580  
2547 2581          if ((ulwp = find_lwp(tid)) == NULL)
2548 2582                  return (0);
2549 2583  
2550 2584          if (ulwp->ul_stop && (result = ulwp->ul_sp) != 0) {
2551 2585                  ulwp_unlock(ulwp, udp);
2552 2586                  return (result);
2553 2587          }
2554 2588  
2555 2589          result = gettsp_slow(tid);
2556 2590          ulwp_unlock(ulwp, udp);
2557 2591          return (result);
2558 2592  }
2559 2593  
2560 2594  /*
2561 2595   * This tells java stack walkers how to find the ucontext
2562 2596   * structure passed to signal handlers.
2563 2597   */
2564 2598  #pragma weak _thr_sighndlrinfo = thr_sighndlrinfo
2565 2599  void
2566 2600  thr_sighndlrinfo(void (**func)(), int *funcsize)
2567 2601  {
2568 2602          *func = &__sighndlr;
2569 2603          *funcsize = (char *)&__sighndlrend - (char *)&__sighndlr;
2570 2604  }
2571 2605  
2572 2606  /*
2573 2607   * Mark a thread a mutator or reset a mutator to being a default,
2574 2608   * non-mutator thread.
2575 2609   */
2576 2610  #pragma weak _thr_setmutator = thr_setmutator
2577 2611  int
2578 2612  thr_setmutator(thread_t tid, int enabled)
2579 2613  {
2580 2614          ulwp_t *self = curthread;
2581 2615          uberdata_t *udp = self->ul_uberdata;
2582 2616          ulwp_t *ulwp;
2583 2617          int error;
2584 2618          int cancel_state;
2585 2619  
2586 2620          enabled = enabled? 1 : 0;
2587 2621  top:
2588 2622          if (tid == 0) {
2589 2623                  ulwp = self;
2590 2624                  ulwp_lock(ulwp, udp);
2591 2625          } else if ((ulwp = find_lwp(tid)) == NULL) {
2592 2626                  return (ESRCH);
2593 2627          }
2594 2628  
2595 2629          /*
2596 2630           * The target thread should be the caller itself or a suspended thread.
2597 2631           * This prevents the target from also changing its ul_mutator field.
2598 2632           */
2599 2633          error = 0;
2600 2634          if (ulwp != self && !ulwp->ul_stop && enabled)
2601 2635                  error = EINVAL;
2602 2636          else if (ulwp->ul_mutator != enabled) {
2603 2637                  lmutex_lock(&mutatorslock);
2604 2638                  if (mutatorsbarrier) {
2605 2639                          ulwp_unlock(ulwp, udp);
2606 2640                          (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
2607 2641                              &cancel_state);
2608 2642                          while (mutatorsbarrier)
2609 2643                                  (void) cond_wait(&mutatorscv, &mutatorslock);
2610 2644                          (void) pthread_setcancelstate(cancel_state, NULL);
2611 2645                          lmutex_unlock(&mutatorslock);
2612 2646                          goto top;
2613 2647                  }
2614 2648                  ulwp->ul_mutator = enabled;
2615 2649                  lmutex_unlock(&mutatorslock);
2616 2650          }
2617 2651  
2618 2652          ulwp_unlock(ulwp, udp);
2619 2653          return (error);
2620 2654  }
2621 2655  
2622 2656  /*
2623 2657   * Establish a barrier against new mutators.  Any non-mutator trying
2624 2658   * to become a mutator is suspended until the barrier is removed.
2625 2659   */
2626 2660  #pragma weak _thr_mutators_barrier = thr_mutators_barrier
2627 2661  void
2628 2662  thr_mutators_barrier(int enabled)
2629 2663  {
2630 2664          int oldvalue;
2631 2665          int cancel_state;
2632 2666  
2633 2667          lmutex_lock(&mutatorslock);
2634 2668  
2635 2669          /*
2636 2670           * Wait if trying to set the barrier while it is already set.
2637 2671           */
2638 2672          (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2639 2673          while (mutatorsbarrier && enabled)
2640 2674                  (void) cond_wait(&mutatorscv, &mutatorslock);
2641 2675          (void) pthread_setcancelstate(cancel_state, NULL);
2642 2676  
2643 2677          oldvalue = mutatorsbarrier;
2644 2678          mutatorsbarrier = enabled;
2645 2679          /*
2646 2680           * Wakeup any blocked non-mutators when barrier is removed.
2647 2681           */
2648 2682          if (oldvalue && !enabled)
2649 2683                  (void) cond_broadcast(&mutatorscv);
2650 2684          lmutex_unlock(&mutatorslock);
2651 2685  }
2652 2686  
2653 2687  /*
2654 2688   * Suspend the set of all mutators except for the caller.  The list
2655 2689   * of actively running threads is searched and only the mutators
2656 2690   * in this list are suspended.  Actively running non-mutators remain
2657 2691   * running.  Any other thread is suspended.
2658 2692   */
2659 2693  #pragma weak _thr_suspend_allmutators = thr_suspend_allmutators
2660 2694  int
2661 2695  thr_suspend_allmutators(void)
2662 2696  {
2663 2697          ulwp_t *self = curthread;
2664 2698          uberdata_t *udp = self->ul_uberdata;
2665 2699          ulwp_t *ulwp;
2666 2700          int link_dropped;
2667 2701  
2668 2702          /*
2669 2703           * We single-thread the entire thread suspend/continue mechanism.
2670 2704           */
2671 2705          fork_lock_enter();
2672 2706  
2673 2707  top:
2674 2708          lmutex_lock(&udp->link_lock);
2675 2709  
2676 2710          if (suspendingallmutators || suspendedallmutators) {
2677 2711                  lmutex_unlock(&udp->link_lock);
2678 2712                  fork_lock_exit();
2679 2713                  return (EINVAL);
2680 2714          }
2681 2715          suspendingallmutators = 1;
2682 2716  
2683 2717          for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2684 2718                  ulwp_lock(ulwp, udp);
2685 2719                  if (!ulwp->ul_mutator) {
2686 2720                          ulwp_unlock(ulwp, udp);
2687 2721                  } else if (ulwp->ul_stop) {     /* already stopped */
2688 2722                          ulwp->ul_stop |= TSTP_MUTATOR;
2689 2723                          ulwp_broadcast(ulwp);
2690 2724                          ulwp_unlock(ulwp, udp);
2691 2725                  } else {
2692 2726                          /*
2693 2727                           * Move the stopped lwp out of a critical section.
2694 2728                           */
2695 2729                          if (safe_suspend(ulwp, TSTP_MUTATOR, &link_dropped) ||
2696 2730                              link_dropped) {
2697 2731                                  suspendingallmutators = 0;
2698 2732                                  goto top;
2699 2733                          }
2700 2734                  }
2701 2735          }
2702 2736  
2703 2737          suspendedallmutators = 1;
2704 2738          suspendingallmutators = 0;
2705 2739          lmutex_unlock(&udp->link_lock);
2706 2740          fork_lock_exit();
2707 2741          return (0);
2708 2742  }
2709 2743  
2710 2744  /*
2711 2745   * Suspend the target mutator.  The caller is permitted to suspend
2712 2746   * itself.  If a mutator barrier is enabled, the caller will suspend
2713 2747   * itself as though it had been suspended by thr_suspend_allmutators().
2714 2748   * When the barrier is removed, this thread will be resumed.  Any
2715 2749   * suspended mutator, whether suspended by thr_suspend_mutator(), or by
2716 2750   * thr_suspend_allmutators(), can be resumed by thr_continue_mutator().
2717 2751   */
2718 2752  #pragma weak _thr_suspend_mutator = thr_suspend_mutator
2719 2753  int
2720 2754  thr_suspend_mutator(thread_t tid)
2721 2755  {
2722 2756          if (tid == 0)
2723 2757                  tid = curthread->ul_lwpid;
2724 2758          return (_thrp_suspend(tid, TSTP_MUTATOR));
2725 2759  }
2726 2760  
2727 2761  /*
2728 2762   * Resume the set of all suspended mutators.
2729 2763   */
2730 2764  #pragma weak _thr_continue_allmutators = thr_continue_allmutators
2731 2765  int
2732 2766  thr_continue_allmutators()
2733 2767  {
2734 2768          ulwp_t *self = curthread;
2735 2769          uberdata_t *udp = self->ul_uberdata;
2736 2770          ulwp_t *ulwp;
2737 2771  
2738 2772          /*
2739 2773           * We single-thread the entire thread suspend/continue mechanism.
2740 2774           */
2741 2775          fork_lock_enter();
2742 2776  
2743 2777          lmutex_lock(&udp->link_lock);
2744 2778          if (!suspendedallmutators) {
2745 2779                  lmutex_unlock(&udp->link_lock);
2746 2780                  fork_lock_exit();
2747 2781                  return (EINVAL);
2748 2782          }
2749 2783          suspendedallmutators = 0;
2750 2784  
2751 2785          for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2752 2786                  mutex_t *mp = ulwp_mutex(ulwp, udp);
2753 2787                  lmutex_lock(mp);
2754 2788                  if (ulwp->ul_stop & TSTP_MUTATOR) {
2755 2789                          ulwp->ul_stop &= ~TSTP_MUTATOR;
2756 2790                          ulwp_broadcast(ulwp);
2757 2791                          if (!ulwp->ul_stop)
2758 2792                                  force_continue(ulwp);
2759 2793                  }
2760 2794                  lmutex_unlock(mp);
2761 2795          }
2762 2796  
2763 2797          lmutex_unlock(&udp->link_lock);
2764 2798          fork_lock_exit();
2765 2799          return (0);
2766 2800  }
2767 2801  
2768 2802  /*
2769 2803   * Resume a suspended mutator.
2770 2804   */
2771 2805  #pragma weak _thr_continue_mutator = thr_continue_mutator
2772 2806  int
2773 2807  thr_continue_mutator(thread_t tid)
2774 2808  {
2775 2809          return (_thrp_continue(tid, TSTP_MUTATOR));
2776 2810  }
2777 2811  
2778 2812  #pragma weak _thr_wait_mutator = thr_wait_mutator
2779 2813  int
2780 2814  thr_wait_mutator(thread_t tid, int dontwait)
2781 2815  {
2782 2816          uberdata_t *udp = curthread->ul_uberdata;
2783 2817          ulwp_t *ulwp;
2784 2818          int cancel_state;
2785 2819          int error = 0;
2786 2820  
2787 2821          (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2788 2822  top:
2789 2823          if ((ulwp = find_lwp(tid)) == NULL) {
2790 2824                  (void) pthread_setcancelstate(cancel_state, NULL);
2791 2825                  return (ESRCH);
2792 2826          }
2793 2827  
2794 2828          if (!ulwp->ul_mutator)
2795 2829                  error = EINVAL;
2796 2830          else if (dontwait) {
2797 2831                  if (!(ulwp->ul_stop & TSTP_MUTATOR))
2798 2832                          error = EWOULDBLOCK;
2799 2833          } else if (!(ulwp->ul_stop & TSTP_MUTATOR)) {
2800 2834                  cond_t *cvp = ulwp_condvar(ulwp, udp);
2801 2835                  mutex_t *mp = ulwp_mutex(ulwp, udp);
2802 2836  
2803 2837                  (void) cond_wait(cvp, mp);
2804 2838                  (void) lmutex_unlock(mp);
2805 2839                  goto top;
2806 2840          }
2807 2841  
2808 2842          ulwp_unlock(ulwp, udp);
2809 2843          (void) pthread_setcancelstate(cancel_state, NULL);
2810 2844          return (error);
2811 2845  }
2812 2846  
2813 2847  /* PROBE_SUPPORT begin */
2814 2848  
2815 2849  void
2816 2850  thr_probe_setup(void *data)
2817 2851  {
2818 2852          curthread->ul_tpdp = data;
2819 2853  }
2820 2854  
2821 2855  static void *
2822 2856  _thread_probe_getfunc()
2823 2857  {
2824 2858          return (curthread->ul_tpdp);
2825 2859  }
2826 2860  
2827 2861  void * (*thr_probe_getfunc_addr)(void) = _thread_probe_getfunc;
2828 2862  
2829 2863  /* ARGSUSED */
2830 2864  void
2831 2865  _resume(ulwp_t *ulwp, caddr_t sp, int dontsave)
2832 2866  {
2833 2867          /* never called */
2834 2868  }
2835 2869  
2836 2870  /* ARGSUSED */
2837 2871  void
2838 2872  _resume_ret(ulwp_t *oldlwp)
2839 2873  {
2840 2874          /* never called */
2841 2875  }
2842 2876  
2843 2877  /* PROBE_SUPPORT end */
  
    | 
      ↓ open down ↓ | 
    1575 lines elided | 
    
      ↑ open up ↑ | 
  
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX