1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /*
26 * Copyright 2016 Joyent, Inc.
27 */
28
29 #include "lint.h"
30 #include "thr_uberdata.h"
31 #include <pthread.h>
32 #include <procfs.h>
33 #include <sys/uio.h>
34 #include <ctype.h>
35 #include "libc.h"
36
37 /*
38 * These symbols should not be exported from libc, but
39 * /lib/libm.so.2 references _thr_main. libm needs to be fixed.
40 * Also, some older versions of the Studio compiler/debugger
41 * components reference them. These need to be fixed, too.
42 */
43 #pragma weak _thr_main = thr_main
44 #pragma weak _thr_create = thr_create
45 #pragma weak _thr_join = thr_join
46 #pragma weak _thr_self = thr_self
47
48 #undef errno
49 extern int errno;
50
51 /*
52 * Between Solaris 2.5 and Solaris 9, __threaded was used to indicate
53 * "we are linked with libthread". The Sun Workshop 6 update 1 compilation
54 * system used it illegally (it is a consolidation private symbol).
55 * To accommodate this and possibly other abusers of the symbol,
56 * we make it always equal to 1 now that libthread has been folded
57 * into libc. The new __libc_threaded symbol is used to indicate
58 * the new meaning, "more than one thread exists".
59 */
60 int __threaded = 1; /* always equal to 1 */
61 int __libc_threaded = 0; /* zero until first thr_create() */
62
63 /*
64 * thr_concurrency and pthread_concurrency are not used by the library.
65 * They exist solely to hold and return the values set by calls to
66 * thr_setconcurrency() and pthread_setconcurrency().
67 * Because thr_concurrency is affected by the THR_NEW_LWP flag
68 * to thr_create(), thr_concurrency is protected by link_lock.
69 */
70 static int thr_concurrency = 1;
71 static int pthread_concurrency;
72
73 #define HASHTBLSZ 1024 /* must be a power of two */
74 #define TIDHASH(tid, udp) (tid & (udp)->hash_mask)
75
76 /* initial allocation, just enough for one lwp */
77 #pragma align 64(init_hash_table)
78 thr_hash_table_t init_hash_table[1] = {
79 { DEFAULTMUTEX, DEFAULTCV, NULL },
80 };
81
82 extern const Lc_interface rtld_funcs[];
83
84 /*
85 * The weak version is known to libc_db and mdb.
86 */
87 #pragma weak _uberdata = __uberdata
88 uberdata_t __uberdata = {
89 { DEFAULTMUTEX, NULL, 0 }, /* link_lock */
90 { RECURSIVEMUTEX, NULL, 0 }, /* ld_lock */
91 { RECURSIVEMUTEX, NULL, 0 }, /* fork_lock */
92 { RECURSIVEMUTEX, NULL, 0 }, /* atfork_lock */
93 { RECURSIVEMUTEX, NULL, 0 }, /* callout_lock */
94 { DEFAULTMUTEX, NULL, 0 }, /* tdb_hash_lock */
95 { 0, }, /* tdb_hash_lock_stats */
96 { { 0 }, }, /* siguaction[NSIG] */
97 {{ DEFAULTMUTEX, NULL, 0 }, /* bucket[NBUCKETS] */
98 { DEFAULTMUTEX, NULL, 0 },
99 { DEFAULTMUTEX, NULL, 0 },
100 { DEFAULTMUTEX, NULL, 0 },
101 { DEFAULTMUTEX, NULL, 0 },
102 { DEFAULTMUTEX, NULL, 0 },
103 { DEFAULTMUTEX, NULL, 0 },
104 { DEFAULTMUTEX, NULL, 0 },
105 { DEFAULTMUTEX, NULL, 0 },
106 { DEFAULTMUTEX, NULL, 0 }},
107 { RECURSIVEMUTEX, NULL, NULL }, /* atexit_root */
108 { RECURSIVEMUTEX, NULL }, /* quickexit_root */
109 { DEFAULTMUTEX, 0, 0, NULL }, /* tsd_metadata */
110 { DEFAULTMUTEX, {0, 0}, {0, 0} }, /* tls_metadata */
111 0, /* primary_map */
112 0, /* bucket_init */
113 0, /* pad[0] */
114 0, /* pad[1] */
115 { 0 }, /* uberflags */
116 NULL, /* queue_head */
117 init_hash_table, /* thr_hash_table */
118 1, /* hash_size: size of the hash table */
119 0, /* hash_mask: hash_size - 1 */
120 NULL, /* ulwp_one */
121 NULL, /* all_lwps */
122 NULL, /* all_zombies */
123 0, /* nthreads */
124 0, /* nzombies */
125 0, /* ndaemons */
126 0, /* pid */
127 sigacthandler, /* sigacthandler */
128 __setcontext, /* setctxt */
129 NULL, /* lwp_stacks */
130 NULL, /* lwp_laststack */
131 0, /* nfreestack */
132 10, /* thread_stack_cache */
133 NULL, /* ulwp_freelist */
134 NULL, /* ulwp_lastfree */
135 NULL, /* ulwp_replace_free */
136 NULL, /* ulwp_replace_last */
137 NULL, /* atforklist */
138 NULL, /* robustlocks */
139 NULL, /* robustlist */
140 NULL, /* progname */
141 NULL, /* ub_broot */
142 NULL, /* ub_comm_page */
143 NULL, /* __tdb_bootstrap */
144 { /* tdb */
145 NULL, /* tdb_sync_addr_hash */
146 0, /* tdb_register_count */
147 0, /* tdb_hash_alloc_failed */
148 NULL, /* tdb_sync_addr_free */
149 NULL, /* tdb_sync_addr_last */
150 0, /* tdb_sync_alloc */
151 { 0, 0 }, /* tdb_ev_global_mask */
152 tdb_events, /* tdb_events array */
153 },
154 };
155
156 /*
157 * The weak version is known to libc_db and mdb.
158 */
159 #pragma weak _tdb_bootstrap = __tdb_bootstrap
160 uberdata_t **__tdb_bootstrap = NULL;
161
162 int thread_queue_fifo = 4;
163 int thread_queue_dump = 0;
164 int thread_cond_wait_defer = 0;
165 int thread_error_detection = 0;
166 int thread_async_safe = 0;
167 int thread_stack_cache = 10;
168 int thread_door_noreserve = 0;
169 int thread_locks_misaligned = 0;
170
171 static ulwp_t *ulwp_alloc(void);
172 static void ulwp_free(ulwp_t *);
173
174 /*
175 * Insert the lwp into the hash table.
176 */
177 void
178 hash_in_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
179 {
180 ulwp->ul_hash = udp->thr_hash_table[ix].hash_bucket;
181 udp->thr_hash_table[ix].hash_bucket = ulwp;
182 ulwp->ul_ix = ix;
183 }
184
185 void
186 hash_in(ulwp_t *ulwp, uberdata_t *udp)
187 {
188 int ix = TIDHASH(ulwp->ul_lwpid, udp);
189 mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
190
191 lmutex_lock(mp);
192 hash_in_unlocked(ulwp, ix, udp);
193 lmutex_unlock(mp);
194 }
195
196 /*
197 * Delete the lwp from the hash table.
198 */
199 void
200 hash_out_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
201 {
202 ulwp_t **ulwpp;
203
204 for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
205 ulwp != *ulwpp;
206 ulwpp = &(*ulwpp)->ul_hash)
207 ;
208 *ulwpp = ulwp->ul_hash;
209 ulwp->ul_hash = NULL;
210 ulwp->ul_ix = -1;
211 }
212
213 void
214 hash_out(ulwp_t *ulwp, uberdata_t *udp)
215 {
216 int ix;
217
218 if ((ix = ulwp->ul_ix) >= 0) {
219 mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
220
221 lmutex_lock(mp);
222 hash_out_unlocked(ulwp, ix, udp);
223 lmutex_unlock(mp);
224 }
225 }
226
227 /*
228 * Retain stack information for thread structures that are being recycled for
229 * new threads. All other members of the thread structure should be zeroed.
230 */
231 static void
232 ulwp_clean(ulwp_t *ulwp)
233 {
234 caddr_t stk = ulwp->ul_stk;
235 size_t mapsiz = ulwp->ul_mapsiz;
236 size_t guardsize = ulwp->ul_guardsize;
237 uintptr_t stktop = ulwp->ul_stktop;
238 size_t stksiz = ulwp->ul_stksiz;
239
240 (void) memset(ulwp, 0, sizeof (*ulwp));
241
242 ulwp->ul_stk = stk;
243 ulwp->ul_mapsiz = mapsiz;
244 ulwp->ul_guardsize = guardsize;
245 ulwp->ul_stktop = stktop;
246 ulwp->ul_stksiz = stksiz;
247 }
248
249 static int stackprot;
250
251 /*
252 * Answer the question, "Is the lwp in question really dead?"
253 * We must inquire of the operating system to be really sure
254 * because the lwp may have called lwp_exit() but it has not
255 * yet completed the exit.
256 */
257 static int
258 dead_and_buried(ulwp_t *ulwp)
259 {
260 if (ulwp->ul_lwpid == (lwpid_t)(-1))
261 return (1);
262 if (ulwp->ul_dead && ulwp->ul_detached &&
263 _lwp_kill(ulwp->ul_lwpid, 0) == ESRCH) {
264 ulwp->ul_lwpid = (lwpid_t)(-1);
265 return (1);
266 }
267 return (0);
268 }
269
270 /*
271 * Attempt to keep the stack cache within the specified cache limit.
272 */
273 static void
274 trim_stack_cache(int cache_limit)
275 {
276 ulwp_t *self = curthread;
277 uberdata_t *udp = self->ul_uberdata;
278 ulwp_t *prev = NULL;
279 ulwp_t **ulwpp = &udp->lwp_stacks;
280 ulwp_t *ulwp;
281
282 ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, self));
283
284 while (udp->nfreestack > cache_limit && (ulwp = *ulwpp) != NULL) {
285 if (dead_and_buried(ulwp)) {
286 *ulwpp = ulwp->ul_next;
287 if (ulwp == udp->lwp_laststack)
288 udp->lwp_laststack = prev;
289 hash_out(ulwp, udp);
290 udp->nfreestack--;
291 (void) munmap(ulwp->ul_stk, ulwp->ul_mapsiz);
292 /*
293 * Now put the free ulwp on the ulwp freelist.
294 */
295 ulwp->ul_mapsiz = 0;
296 ulwp->ul_next = NULL;
297 if (udp->ulwp_freelist == NULL)
298 udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
299 else {
300 udp->ulwp_lastfree->ul_next = ulwp;
301 udp->ulwp_lastfree = ulwp;
302 }
303 } else {
304 prev = ulwp;
305 ulwpp = &ulwp->ul_next;
306 }
307 }
308 }
309
310 /*
311 * Find an unused stack of the requested size
312 * or create a new stack of the requested size.
313 * Return a pointer to the ulwp_t structure referring to the stack, or NULL.
314 * thr_exit() stores 1 in the ul_dead member.
315 * thr_join() stores -1 in the ul_lwpid member.
316 */
317 static ulwp_t *
318 find_stack(size_t stksize, size_t guardsize)
319 {
320 static size_t pagesize = 0;
321
322 uberdata_t *udp = curthread->ul_uberdata;
323 size_t mapsize;
324 ulwp_t *prev;
325 ulwp_t *ulwp;
326 ulwp_t **ulwpp;
327 void *stk;
328
329 /*
330 * The stack is allocated PROT_READ|PROT_WRITE|PROT_EXEC
331 * unless overridden by the system's configuration.
332 */
333 if (stackprot == 0) { /* do this once */
334 long lprot = _sysconf(_SC_STACK_PROT);
335 if (lprot <= 0)
336 lprot = (PROT_READ|PROT_WRITE|PROT_EXEC);
337 stackprot = (int)lprot;
338 }
339 if (pagesize == 0) /* do this once */
340 pagesize = _sysconf(_SC_PAGESIZE);
341
342 /*
343 * One megabyte stacks by default, but subtract off
344 * two pages for the system-created red zones.
345 * Round up a non-zero stack size to a pagesize multiple.
346 */
347 if (stksize == 0)
348 stksize = DEFAULTSTACK - 2 * pagesize;
349 else
350 stksize = ((stksize + pagesize - 1) & -pagesize);
351
352 /*
353 * Round up the mapping size to a multiple of pagesize.
354 * Note: mmap() provides at least one page of red zone
355 * so we deduct that from the value of guardsize.
356 */
357 if (guardsize != 0)
358 guardsize = ((guardsize + pagesize - 1) & -pagesize) - pagesize;
359 mapsize = stksize + guardsize;
360
361 lmutex_lock(&udp->link_lock);
362 for (prev = NULL, ulwpp = &udp->lwp_stacks;
363 (ulwp = *ulwpp) != NULL;
364 prev = ulwp, ulwpp = &ulwp->ul_next) {
365 if (ulwp->ul_mapsiz == mapsize &&
366 ulwp->ul_guardsize == guardsize &&
367 dead_and_buried(ulwp)) {
368 /*
369 * The previous lwp is gone; reuse the stack.
370 * Remove the ulwp from the stack list.
371 */
372 *ulwpp = ulwp->ul_next;
373 ulwp->ul_next = NULL;
374 if (ulwp == udp->lwp_laststack)
375 udp->lwp_laststack = prev;
376 hash_out(ulwp, udp);
377 udp->nfreestack--;
378 lmutex_unlock(&udp->link_lock);
379 ulwp_clean(ulwp);
380 return (ulwp);
381 }
382 }
383
384 /*
385 * None of the cached stacks matched our mapping size.
386 * Reduce the stack cache to get rid of possibly
387 * very old stacks that will never be reused.
388 */
389 if (udp->nfreestack > udp->thread_stack_cache)
390 trim_stack_cache(udp->thread_stack_cache);
391 else if (udp->nfreestack > 0)
392 trim_stack_cache(udp->nfreestack - 1);
393 lmutex_unlock(&udp->link_lock);
394
395 /*
396 * Create a new stack.
397 */
398 if ((stk = mmap(NULL, mapsize, stackprot,
399 MAP_PRIVATE|MAP_NORESERVE|MAP_ANON, -1, (off_t)0)) != MAP_FAILED) {
400 /*
401 * We have allocated our stack. Now allocate the ulwp.
402 */
403 ulwp = ulwp_alloc();
404 if (ulwp == NULL)
405 (void) munmap(stk, mapsize);
406 else {
407 ulwp->ul_stk = stk;
408 ulwp->ul_mapsiz = mapsize;
409 ulwp->ul_guardsize = guardsize;
410 ulwp->ul_stktop = (uintptr_t)stk + mapsize;
411 ulwp->ul_stksiz = stksize;
412 if (guardsize) /* protect the extra red zone */
413 (void) mprotect(stk, guardsize, PROT_NONE);
414 }
415 }
416 return (ulwp);
417 }
418
419 /*
420 * Get a ulwp_t structure from the free list or allocate a new one.
421 * Such ulwp_t's do not have a stack allocated by the library.
422 */
423 static ulwp_t *
424 ulwp_alloc(void)
425 {
426 ulwp_t *self = curthread;
427 uberdata_t *udp = self->ul_uberdata;
428 size_t tls_size;
429 ulwp_t *prev;
430 ulwp_t *ulwp;
431 ulwp_t **ulwpp;
432 caddr_t data;
433
434 lmutex_lock(&udp->link_lock);
435 for (prev = NULL, ulwpp = &udp->ulwp_freelist;
436 (ulwp = *ulwpp) != NULL;
437 prev = ulwp, ulwpp = &ulwp->ul_next) {
438 if (dead_and_buried(ulwp)) {
439 *ulwpp = ulwp->ul_next;
440 ulwp->ul_next = NULL;
441 if (ulwp == udp->ulwp_lastfree)
442 udp->ulwp_lastfree = prev;
443 hash_out(ulwp, udp);
444 lmutex_unlock(&udp->link_lock);
445 ulwp_clean(ulwp);
446 return (ulwp);
447 }
448 }
449 lmutex_unlock(&udp->link_lock);
450
451 tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
452 data = lmalloc(sizeof (*ulwp) + tls_size);
453 if (data != NULL) {
454 /* LINTED pointer cast may result in improper alignment */
455 ulwp = (ulwp_t *)(data + tls_size);
456 }
457 return (ulwp);
458 }
459
460 /*
461 * Free a ulwp structure.
462 * If there is an associated stack, put it on the stack list and
463 * munmap() previously freed stacks up to the residual cache limit.
464 * Else put it on the ulwp free list and never call lfree() on it.
465 */
466 static void
467 ulwp_free(ulwp_t *ulwp)
468 {
469 uberdata_t *udp = curthread->ul_uberdata;
470
471 ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, curthread));
472 ulwp->ul_next = NULL;
473 if (ulwp == udp->ulwp_one) /* don't reuse the primoridal stack */
474 /*EMPTY*/;
475 else if (ulwp->ul_mapsiz != 0) {
476 if (udp->lwp_stacks == NULL)
477 udp->lwp_stacks = udp->lwp_laststack = ulwp;
478 else {
479 udp->lwp_laststack->ul_next = ulwp;
480 udp->lwp_laststack = ulwp;
481 }
482 if (++udp->nfreestack > udp->thread_stack_cache)
483 trim_stack_cache(udp->thread_stack_cache);
484 } else {
485 if (udp->ulwp_freelist == NULL)
486 udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
487 else {
488 udp->ulwp_lastfree->ul_next = ulwp;
489 udp->ulwp_lastfree = ulwp;
490 }
491 }
492 }
493
494 /*
495 * Find a named lwp and return a pointer to its hash list location.
496 * On success, returns with the hash lock held.
497 */
498 ulwp_t **
499 find_lwpp(thread_t tid)
500 {
501 uberdata_t *udp = curthread->ul_uberdata;
502 int ix = TIDHASH(tid, udp);
503 mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
504 ulwp_t *ulwp;
505 ulwp_t **ulwpp;
506
507 if (tid == 0)
508 return (NULL);
509
510 lmutex_lock(mp);
511 for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
512 (ulwp = *ulwpp) != NULL;
513 ulwpp = &ulwp->ul_hash) {
514 if (ulwp->ul_lwpid == tid)
515 return (ulwpp);
516 }
517 lmutex_unlock(mp);
518 return (NULL);
519 }
520
521 /*
522 * Wake up all lwps waiting on this lwp for some reason.
523 */
524 void
525 ulwp_broadcast(ulwp_t *ulwp)
526 {
527 ulwp_t *self = curthread;
528 uberdata_t *udp = self->ul_uberdata;
529
530 ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
531 (void) cond_broadcast(ulwp_condvar(ulwp, udp));
532 }
533
534 /*
535 * Find a named lwp and return a pointer to it.
536 * Returns with the hash lock held.
537 */
538 ulwp_t *
539 find_lwp(thread_t tid)
540 {
541 ulwp_t *self = curthread;
542 uberdata_t *udp = self->ul_uberdata;
543 ulwp_t *ulwp = NULL;
544 ulwp_t **ulwpp;
545
546 if (self->ul_lwpid == tid) {
547 ulwp = self;
548 ulwp_lock(ulwp, udp);
549 } else if ((ulwpp = find_lwpp(tid)) != NULL) {
550 ulwp = *ulwpp;
551 }
552
553 if (ulwp && ulwp->ul_dead) {
554 ulwp_unlock(ulwp, udp);
555 ulwp = NULL;
556 }
557
558 return (ulwp);
559 }
560
561 int
562 _thrp_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
563 long flags, thread_t *new_thread, size_t guardsize)
564 {
565 ulwp_t *self = curthread;
566 uberdata_t *udp = self->ul_uberdata;
567 ucontext_t uc;
568 uint_t lwp_flags;
569 thread_t tid;
570 int error;
571 ulwp_t *ulwp;
572
573 /*
574 * Enforce the restriction of not creating any threads
575 * until the primary link map has been initialized.
576 * Also, disallow thread creation to a child of vfork().
577 */
578 if (!self->ul_primarymap || self->ul_vfork)
579 return (ENOTSUP);
580
581 if (udp->hash_size == 1)
582 finish_init();
583
584 if ((stk || stksize) && stksize < MINSTACK)
585 return (EINVAL);
586
587 if (stk == NULL) {
588 if ((ulwp = find_stack(stksize, guardsize)) == NULL)
589 return (ENOMEM);
590 stksize = ulwp->ul_mapsiz - ulwp->ul_guardsize;
591 } else {
592 /* initialize the private stack */
593 if ((ulwp = ulwp_alloc()) == NULL)
594 return (ENOMEM);
595 ulwp->ul_stk = stk;
596 ulwp->ul_stktop = (uintptr_t)stk + stksize;
597 ulwp->ul_stksiz = stksize;
598 }
599 /* ulwp is not in the hash table; make sure hash_out() doesn't fail */
600 ulwp->ul_ix = -1;
601 ulwp->ul_errnop = &ulwp->ul_errno;
602
603 lwp_flags = LWP_SUSPENDED;
604 if (flags & (THR_DETACHED|THR_DAEMON)) {
605 flags |= THR_DETACHED;
606 lwp_flags |= LWP_DETACHED;
607 }
608 if (flags & THR_DAEMON)
609 lwp_flags |= LWP_DAEMON;
610
611 /* creating a thread: enforce mt-correctness in mutex_lock() */
612 self->ul_async_safe = 1;
613
614 /* per-thread copies of global variables, for speed */
615 ulwp->ul_queue_fifo = self->ul_queue_fifo;
616 ulwp->ul_cond_wait_defer = self->ul_cond_wait_defer;
617 ulwp->ul_error_detection = self->ul_error_detection;
618 ulwp->ul_async_safe = self->ul_async_safe;
619 ulwp->ul_max_spinners = self->ul_max_spinners;
620 ulwp->ul_adaptive_spin = self->ul_adaptive_spin;
621 ulwp->ul_queue_spin = self->ul_queue_spin;
622 ulwp->ul_door_noreserve = self->ul_door_noreserve;
623 ulwp->ul_misaligned = self->ul_misaligned;
624
625 /* new thread inherits creating thread's scheduling parameters */
626 ulwp->ul_policy = self->ul_policy;
627 ulwp->ul_pri = (self->ul_epri? self->ul_epri : self->ul_pri);
628 ulwp->ul_cid = self->ul_cid;
629 ulwp->ul_rtclassid = self->ul_rtclassid;
630
631 ulwp->ul_primarymap = self->ul_primarymap;
632 ulwp->ul_self = ulwp;
633 ulwp->ul_uberdata = udp;
634
635 /* debugger support */
636 ulwp->ul_usropts = flags;
637
638 #ifdef __sparc
639 /*
640 * We cache several instructions in the thread structure for use
641 * by the fasttrap DTrace provider. When changing this, read the
642 * comment in fasttrap.h for the all the other places that must
643 * be changed.
644 */
645 ulwp->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
646 ulwp->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
647 ulwp->ul_dftret = 0x91d0203a; /* ta 0x3a */
648 ulwp->ul_dreturn = 0x81ca0000; /* return %o0 */
649 #endif
650
651 ulwp->ul_startpc = func;
652 ulwp->ul_startarg = arg;
653 _fpinherit(ulwp);
654 /*
655 * Defer signals on the new thread until its TLS constructors
656 * have been called. _thrp_setup() will call sigon() after
657 * it has called tls_setup().
658 */
659 ulwp->ul_sigdefer = 1;
660
661 error = setup_context(&uc, _thrp_setup, ulwp,
662 (caddr_t)ulwp->ul_stk + ulwp->ul_guardsize, stksize);
663 if (error != 0 && stk != NULL) /* inaccessible stack */
664 error = EFAULT;
665
666 /*
667 * Call enter_critical() to avoid being suspended until we
668 * have linked the new thread into the proper lists.
669 * This is necessary because forkall() and fork1() must
670 * suspend all threads and they must see a complete list.
671 */
672 enter_critical(self);
673 uc.uc_sigmask = ulwp->ul_sigmask = self->ul_sigmask;
674 if (error != 0 ||
675 (error = __lwp_create(&uc, lwp_flags, &tid)) != 0) {
676 exit_critical(self);
677 ulwp->ul_lwpid = (lwpid_t)(-1);
678 ulwp->ul_dead = 1;
679 ulwp->ul_detached = 1;
680 lmutex_lock(&udp->link_lock);
681 ulwp_free(ulwp);
682 lmutex_unlock(&udp->link_lock);
683 return (error);
684 }
685 self->ul_nocancel = 0; /* cancellation is now possible */
686 udp->uberflags.uf_mt = 1;
687 if (new_thread)
688 *new_thread = tid;
689 if (flags & THR_DETACHED)
690 ulwp->ul_detached = 1;
691 ulwp->ul_lwpid = tid;
692 ulwp->ul_stop = TSTP_REGULAR;
693 if (flags & THR_SUSPENDED)
694 ulwp->ul_created = 1;
695
696 lmutex_lock(&udp->link_lock);
697 ulwp->ul_forw = udp->all_lwps;
698 ulwp->ul_back = udp->all_lwps->ul_back;
699 ulwp->ul_back->ul_forw = ulwp;
700 ulwp->ul_forw->ul_back = ulwp;
701 hash_in(ulwp, udp);
702 udp->nthreads++;
703 if (flags & THR_DAEMON)
704 udp->ndaemons++;
705 if (flags & THR_NEW_LWP)
706 thr_concurrency++;
707 __libc_threaded = 1; /* inform stdio */
708 lmutex_unlock(&udp->link_lock);
709
710 if (__td_event_report(self, TD_CREATE, udp)) {
711 self->ul_td_evbuf.eventnum = TD_CREATE;
712 self->ul_td_evbuf.eventdata = (void *)(uintptr_t)tid;
713 tdb_event(TD_CREATE, udp);
714 }
715
716 exit_critical(self);
717
718 if (!(flags & THR_SUSPENDED))
719 (void) _thrp_continue(tid, TSTP_REGULAR);
720
721 return (0);
722 }
723
724 int
725 thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
726 long flags, thread_t *new_thread)
727 {
728 return (_thrp_create(stk, stksize, func, arg, flags, new_thread, 0));
729 }
730
731 /*
732 * A special cancellation cleanup hook for DCE.
733 * cleanuphndlr, when it is not NULL, will contain a callback
734 * function to be called before a thread is terminated in
735 * thr_exit() as a result of being cancelled.
736 */
737 static void (*cleanuphndlr)(void) = NULL;
738
739 /*
740 * _pthread_setcleanupinit: sets the cleanup hook.
741 */
742 int
743 _pthread_setcleanupinit(void (*func)(void))
744 {
745 cleanuphndlr = func;
746 return (0);
747 }
748
749 void
750 _thrp_exit()
751 {
752 ulwp_t *self = curthread;
753 uberdata_t *udp = self->ul_uberdata;
754 ulwp_t *replace = NULL;
755
756 if (__td_event_report(self, TD_DEATH, udp)) {
757 self->ul_td_evbuf.eventnum = TD_DEATH;
758 tdb_event(TD_DEATH, udp);
759 }
760
761 ASSERT(self->ul_sigdefer != 0);
762
763 lmutex_lock(&udp->link_lock);
764 udp->nthreads--;
765 if (self->ul_usropts & THR_NEW_LWP)
766 thr_concurrency--;
767 if (self->ul_usropts & THR_DAEMON)
768 udp->ndaemons--;
769 else if (udp->nthreads == udp->ndaemons) {
770 /*
771 * We are the last non-daemon thread exiting.
772 * Exit the process. We retain our TSD and TLS so
773 * that atexit() application functions can use them.
774 */
775 lmutex_unlock(&udp->link_lock);
776 exit(0);
777 thr_panic("_thrp_exit(): exit(0) returned");
778 }
779 lmutex_unlock(&udp->link_lock);
780
781 tmem_exit(); /* deallocate tmem allocations */
782 tsd_exit(); /* deallocate thread-specific data */
783 tls_exit(); /* deallocate thread-local storage */
784 heldlock_exit(); /* deal with left-over held locks */
785
786 /* block all signals to finish exiting */
787 block_all_signals(self);
788 /* also prevent ourself from being suspended */
789 enter_critical(self);
790 rwl_free(self);
791 lmutex_lock(&udp->link_lock);
792 ulwp_free(self);
793 (void) ulwp_lock(self, udp);
794
795 if (self->ul_mapsiz && !self->ul_detached) {
796 /*
797 * We want to free the stack for reuse but must keep
798 * the ulwp_t struct for the benefit of thr_join().
799 * For this purpose we allocate a replacement ulwp_t.
800 */
801 if ((replace = udp->ulwp_replace_free) == NULL)
802 replace = lmalloc(REPLACEMENT_SIZE);
803 else if ((udp->ulwp_replace_free = replace->ul_next) == NULL)
804 udp->ulwp_replace_last = NULL;
805 }
806
807 if (udp->all_lwps == self)
808 udp->all_lwps = self->ul_forw;
809 if (udp->all_lwps == self)
810 udp->all_lwps = NULL;
811 else {
812 self->ul_forw->ul_back = self->ul_back;
813 self->ul_back->ul_forw = self->ul_forw;
814 }
815 self->ul_forw = self->ul_back = NULL;
816 #if defined(THREAD_DEBUG)
817 /* collect queue lock statistics before marking ourself dead */
818 record_spin_locks(self);
819 #endif
820 self->ul_dead = 1;
821 self->ul_pleasestop = 0;
822 if (replace != NULL) {
823 int ix = self->ul_ix; /* the hash index */
824 (void) memcpy(replace, self, REPLACEMENT_SIZE);
825 replace->ul_self = replace;
826 replace->ul_next = NULL; /* clone not on stack list */
827 replace->ul_mapsiz = 0; /* allows clone to be freed */
828 replace->ul_replace = 1; /* requires clone to be freed */
829 hash_out_unlocked(self, ix, udp);
830 hash_in_unlocked(replace, ix, udp);
831 ASSERT(!(self->ul_detached));
832 self->ul_detached = 1; /* this frees the stack */
833 self->ul_schedctl = NULL;
834 self->ul_schedctl_called = &udp->uberflags;
835 set_curthread(self = replace);
836 /*
837 * Having just changed the address of curthread, we
838 * must reset the ownership of the locks we hold so
839 * that assertions will not fire when we release them.
840 */
841 udp->link_lock.mutex_owner = (uintptr_t)self;
842 ulwp_mutex(self, udp)->mutex_owner = (uintptr_t)self;
843 /*
844 * NOTE:
845 * On i386, %gs still references the original, not the
846 * replacement, ulwp structure. Fetching the replacement
847 * curthread pointer via %gs:0 works correctly since the
848 * original ulwp structure will not be reallocated until
849 * this lwp has completed its lwp_exit() system call (see
850 * dead_and_buried()), but from here on out, we must make
851 * no references to %gs:<offset> other than %gs:0.
852 */
853 }
854 /*
855 * Put non-detached terminated threads in the all_zombies list.
856 */
857 if (!self->ul_detached) {
858 udp->nzombies++;
859 if (udp->all_zombies == NULL) {
860 ASSERT(udp->nzombies == 1);
861 udp->all_zombies = self->ul_forw = self->ul_back = self;
862 } else {
863 self->ul_forw = udp->all_zombies;
864 self->ul_back = udp->all_zombies->ul_back;
865 self->ul_back->ul_forw = self;
866 self->ul_forw->ul_back = self;
867 }
868 }
869 /*
870 * Notify everyone waiting for this thread.
871 */
872 ulwp_broadcast(self);
873 (void) ulwp_unlock(self, udp);
874 /*
875 * Prevent any more references to the schedctl data.
876 * We are exiting and continue_fork() may not find us.
877 * Do this just before dropping link_lock, since fork
878 * serializes on link_lock.
879 */
880 self->ul_schedctl = NULL;
881 self->ul_schedctl_called = &udp->uberflags;
882 lmutex_unlock(&udp->link_lock);
883
884 ASSERT(self->ul_critical == 1);
885 ASSERT(self->ul_preempt == 0);
886 _lwp_terminate(); /* never returns */
887 thr_panic("_thrp_exit(): _lwp_terminate() returned");
888 }
889
890 #if defined(THREAD_DEBUG)
891 void
892 collect_queue_statistics()
893 {
894 uberdata_t *udp = curthread->ul_uberdata;
895 ulwp_t *ulwp;
896
897 if (thread_queue_dump) {
898 lmutex_lock(&udp->link_lock);
899 if ((ulwp = udp->all_lwps) != NULL) {
900 do {
901 record_spin_locks(ulwp);
902 } while ((ulwp = ulwp->ul_forw) != udp->all_lwps);
903 }
904 lmutex_unlock(&udp->link_lock);
905 }
906 }
907 #endif
908
909 static void __NORETURN
910 _thrp_exit_common(void *status, int unwind)
911 {
912 ulwp_t *self = curthread;
913 int cancelled = (self->ul_cancel_pending && status == PTHREAD_CANCELED);
914
915 ASSERT(self->ul_critical == 0 && self->ul_preempt == 0);
916
917 /*
918 * Disable cancellation and call the special DCE cancellation
919 * cleanup hook if it is enabled. Do nothing else before calling
920 * the DCE cancellation cleanup hook; it may call longjmp() and
921 * never return here.
922 */
923 self->ul_cancel_disabled = 1;
924 self->ul_cancel_async = 0;
925 self->ul_save_async = 0;
926 self->ul_cancelable = 0;
927 self->ul_cancel_pending = 0;
928 set_cancel_pending_flag(self, 1);
929 if (cancelled && cleanuphndlr != NULL)
930 (*cleanuphndlr)();
931
932 /*
933 * Block application signals while we are exiting.
934 * We call out to C++, TSD, and TLS destructors while exiting
935 * and these are application-defined, so we cannot be assured
936 * that they won't reset the signal mask. We use sigoff() to
937 * defer any signals that may be received as a result of this
938 * bad behavior. Such signals will be lost to the process
939 * when the thread finishes exiting.
940 */
941 (void) thr_sigsetmask(SIG_SETMASK, &maskset, NULL);
942 sigoff(self);
943
944 self->ul_rval = status;
945
946 /*
947 * If thr_exit is being called from the places where
948 * C++ destructors are to be called such as cancellation
949 * points, then set this flag. It is checked in _t_cancel()
950 * to decide whether _ex_unwind() is to be called or not.
951 */
952 if (unwind)
953 self->ul_unwind = 1;
954
955 /*
956 * _thrp_unwind() will eventually call _thrp_exit().
957 * It never returns.
958 */
959 _thrp_unwind(NULL);
960 thr_panic("_thrp_exit_common(): _thrp_unwind() returned");
961
962 for (;;) /* to shut the compiler up about __NORETURN */
963 continue;
964 }
965
966 /*
967 * Called when a thread returns from its start function.
968 * We are at the top of the stack; no unwinding is necessary.
969 */
970 void
971 _thrp_terminate(void *status)
972 {
973 _thrp_exit_common(status, 0);
974 }
975
976 #pragma weak pthread_exit = thr_exit
977 #pragma weak _thr_exit = thr_exit
978 void
979 thr_exit(void *status)
980 {
981 _thrp_exit_common(status, 1);
982 }
983
984 int
985 _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
986 {
987 uberdata_t *udp = curthread->ul_uberdata;
988 mutex_t *mp;
989 void *rval;
990 thread_t found;
991 ulwp_t *ulwp;
992 ulwp_t **ulwpp;
993 int replace;
994 int error;
995
996 if (do_cancel)
997 error = lwp_wait(tid, &found);
998 else {
999 while ((error = __lwp_wait(tid, &found)) == EINTR)
1000 ;
1001 }
1002 if (error)
1003 return (error);
1004
1005 /*
1006 * We must hold link_lock to avoid a race condition with find_stack().
1007 */
1008 lmutex_lock(&udp->link_lock);
1009 if ((ulwpp = find_lwpp(found)) == NULL) {
1010 /*
1011 * lwp_wait() found an lwp that the library doesn't know
1012 * about. It must have been created with _lwp_create().
1013 * Just return its lwpid; we can't know its status.
1014 */
1015 lmutex_unlock(&udp->link_lock);
1016 rval = NULL;
1017 } else {
1018 /*
1019 * Remove ulwp from the hash table.
1020 */
1021 ulwp = *ulwpp;
1022 *ulwpp = ulwp->ul_hash;
1023 ulwp->ul_hash = NULL;
1024 /*
1025 * Remove ulwp from all_zombies list.
1026 */
1027 ASSERT(udp->nzombies >= 1);
1028 if (udp->all_zombies == ulwp)
1029 udp->all_zombies = ulwp->ul_forw;
1030 if (udp->all_zombies == ulwp)
1031 udp->all_zombies = NULL;
1032 else {
1033 ulwp->ul_forw->ul_back = ulwp->ul_back;
1034 ulwp->ul_back->ul_forw = ulwp->ul_forw;
1035 }
1036 ulwp->ul_forw = ulwp->ul_back = NULL;
1037 udp->nzombies--;
1038 ASSERT(ulwp->ul_dead && !ulwp->ul_detached &&
1039 !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
1040 /*
1041 * We can't call ulwp_unlock(ulwp) after we set
1042 * ulwp->ul_ix = -1 so we have to get a pointer to the
1043 * ulwp's hash table mutex now in order to unlock it below.
1044 */
1045 mp = ulwp_mutex(ulwp, udp);
1046 ulwp->ul_lwpid = (lwpid_t)(-1);
1047 ulwp->ul_ix = -1;
1048 rval = ulwp->ul_rval;
1049 replace = ulwp->ul_replace;
1050 lmutex_unlock(mp);
1051 if (replace) {
1052 ulwp->ul_next = NULL;
1053 if (udp->ulwp_replace_free == NULL)
1054 udp->ulwp_replace_free =
1055 udp->ulwp_replace_last = ulwp;
1056 else {
1057 udp->ulwp_replace_last->ul_next = ulwp;
1058 udp->ulwp_replace_last = ulwp;
1059 }
1060 }
1061 lmutex_unlock(&udp->link_lock);
1062 }
1063
1064 if (departed != NULL)
1065 *departed = found;
1066 if (status != NULL)
1067 *status = rval;
1068 return (0);
1069 }
1070
1071 int
1072 thr_join(thread_t tid, thread_t *departed, void **status)
1073 {
1074 int error = _thrp_join(tid, departed, status, 1);
1075 return ((error == EINVAL)? ESRCH : error);
1076 }
1077
1078 /*
1079 * pthread_join() differs from Solaris thr_join():
1080 * It does not return the departed thread's id
1081 * and hence does not have a "departed" argument.
1082 * It returns EINVAL if tid refers to a detached thread.
1083 */
1084 #pragma weak _pthread_join = pthread_join
1085 int
1086 pthread_join(pthread_t tid, void **status)
1087 {
1088 return ((tid == 0)? ESRCH : _thrp_join(tid, NULL, status, 1));
1089 }
1090
1091 int
1092 pthread_detach(pthread_t tid)
1093 {
1094 uberdata_t *udp = curthread->ul_uberdata;
1095 ulwp_t *ulwp;
1096 ulwp_t **ulwpp;
1097 int error = 0;
1098
1099 if ((ulwpp = find_lwpp(tid)) == NULL)
1100 return (ESRCH);
1101 ulwp = *ulwpp;
1102
1103 if (ulwp->ul_dead) {
1104 ulwp_unlock(ulwp, udp);
1105 error = _thrp_join(tid, NULL, NULL, 0);
1106 } else {
1107 error = __lwp_detach(tid);
1108 ulwp->ul_detached = 1;
1109 ulwp->ul_usropts |= THR_DETACHED;
1110 ulwp_unlock(ulwp, udp);
1111 }
1112 return (error);
1113 }
1114
1115 static const char *
1116 ematch(const char *ev, const char *match)
1117 {
1118 int c;
1119
1120 while ((c = *match++) != '\0') {
1121 if (*ev++ != c)
1122 return (NULL);
1123 }
1124 if (*ev++ != '=')
1125 return (NULL);
1126 return (ev);
1127 }
1128
1129 static int
1130 envvar(const char *ev, const char *match, int limit)
1131 {
1132 int val = -1;
1133 const char *ename;
1134
1135 if ((ename = ematch(ev, match)) != NULL) {
1136 int c;
1137 for (val = 0; (c = *ename) != '\0'; ename++) {
1138 if (!isdigit(c)) {
1139 val = -1;
1140 break;
1141 }
1142 val = val * 10 + (c - '0');
1143 if (val > limit) {
1144 val = limit;
1145 break;
1146 }
1147 }
1148 }
1149 return (val);
1150 }
1151
1152 static void
1153 etest(const char *ev)
1154 {
1155 int value;
1156
1157 if ((value = envvar(ev, "QUEUE_SPIN", 1000000)) >= 0)
1158 thread_queue_spin = value;
1159 if ((value = envvar(ev, "ADAPTIVE_SPIN", 1000000)) >= 0)
1160 thread_adaptive_spin = value;
1161 if ((value = envvar(ev, "MAX_SPINNERS", 255)) >= 0)
1162 thread_max_spinners = value;
1163 if ((value = envvar(ev, "QUEUE_FIFO", 8)) >= 0)
1164 thread_queue_fifo = value;
1165 #if defined(THREAD_DEBUG)
1166 if ((value = envvar(ev, "QUEUE_VERIFY", 1)) >= 0)
1167 thread_queue_verify = value;
1168 if ((value = envvar(ev, "QUEUE_DUMP", 1)) >= 0)
1169 thread_queue_dump = value;
1170 #endif
1171 if ((value = envvar(ev, "STACK_CACHE", 10000)) >= 0)
1172 thread_stack_cache = value;
1173 if ((value = envvar(ev, "COND_WAIT_DEFER", 1)) >= 0)
1174 thread_cond_wait_defer = value;
1175 if ((value = envvar(ev, "ERROR_DETECTION", 2)) >= 0)
1176 thread_error_detection = value;
1177 if ((value = envvar(ev, "ASYNC_SAFE", 1)) >= 0)
1178 thread_async_safe = value;
1179 if ((value = envvar(ev, "DOOR_NORESERVE", 1)) >= 0)
1180 thread_door_noreserve = value;
1181 if ((value = envvar(ev, "LOCKS_MISALIGNED", 1)) >= 0)
1182 thread_locks_misaligned = value;
1183 }
1184
1185 /*
1186 * Look for and evaluate environment variables of the form "_THREAD_*".
1187 * For compatibility with the past, we also look for environment
1188 * names of the form "LIBTHREAD_*".
1189 */
1190 static void
1191 set_thread_vars()
1192 {
1193 extern const char **_environ;
1194 const char **pev;
1195 const char *ev;
1196 char c;
1197
1198 if ((pev = _environ) == NULL)
1199 return;
1200 while ((ev = *pev++) != NULL) {
1201 c = *ev;
1202 if (c == '_' && strncmp(ev, "_THREAD_", 8) == 0)
1203 etest(ev + 8);
1204 if (c == 'L' && strncmp(ev, "LIBTHREAD_", 10) == 0)
1205 etest(ev + 10);
1206 }
1207 }
1208
1209 /* PROBE_SUPPORT begin */
1210 #pragma weak __tnf_probe_notify
1211 extern void __tnf_probe_notify(void);
1212 /* PROBE_SUPPORT end */
1213
1214 /* same as atexit() but private to the library */
1215 extern int _atexit(void (*)(void));
1216
1217 /* same as _cleanup() but private to the library */
1218 extern void __cleanup(void);
1219
1220 extern void atfork_init(void);
1221
1222 #ifdef __amd64
1223 extern void __proc64id(void);
1224 #endif
1225
1226 static void
1227 init_auxv_data(uberdata_t *udp)
1228 {
1229 Dl_argsinfo_t args;
1230
1231 udp->ub_broot = NULL;
1232 udp->ub_comm_page = NULL;
1233 if (dlinfo(RTLD_SELF, RTLD_DI_ARGSINFO, &args) < 0)
1234 return;
1235
1236 while (args.dla_auxv->a_type != AT_NULL) {
1237 switch (args.dla_auxv->a_type) {
1238 case AT_SUN_BRAND_NROOT:
1239 udp->ub_broot = args.dla_auxv->a_un.a_ptr;
1240 break;
1241 case AT_SUN_COMMPAGE:
1242 udp->ub_comm_page = args.dla_auxv->a_un.a_ptr;
1243 break;
1244 }
1245 args.dla_auxv++;
1246 }
1247 }
1248
1249 /*
1250 * libc_init() is called by ld.so.1 for library initialization.
1251 * We perform minimal initialization; enough to work with the main thread.
1252 */
1253 void
1254 libc_init(void)
1255 {
1256 uberdata_t *udp = &__uberdata;
1257 ulwp_t *oldself = __curthread();
1258 ucontext_t uc;
1259 ulwp_t *self;
1260 struct rlimit rl;
1261 caddr_t data;
1262 size_t tls_size;
1263 int setmask;
1264
1265 /*
1266 * For the initial stage of initialization, we must be careful
1267 * not to call any function that could possibly call _cerror().
1268 * For this purpose, we call only the raw system call wrappers.
1269 */
1270
1271 #ifdef __amd64
1272 /*
1273 * Gather information about cache layouts for optimized
1274 * AMD and Intel assembler strfoo() and memfoo() functions.
1275 */
1276 __proc64id();
1277 #endif
1278
1279 /*
1280 * Every libc, regardless of which link map, must register __cleanup().
1281 */
1282 (void) _atexit(__cleanup);
1283
1284 /*
1285 * Every libc, regardless of link map, needs to go through and check
1286 * its aux vectors. Doing so will indicate whether or not this has
1287 * been given a brand root (used to qualify various other data) or a
1288 * comm page (to optimize certain system actions).
1289 */
1290 init_auxv_data(udp);
1291
1292 /*
1293 * We keep our uberdata on one of (a) the first alternate link map
1294 * or (b) the primary link map. We switch to the primary link map
1295 * and stay there once we see it. All intermediate link maps are
1296 * subject to being unloaded at any time.
1297 */
1298 if (oldself != NULL && (oldself->ul_primarymap || !primary_link_map)) {
1299 __tdb_bootstrap = oldself->ul_uberdata->tdb_bootstrap;
1300 mutex_setup();
1301 atfork_init(); /* every link map needs atfork() processing */
1302 init_progname();
1303 return;
1304 }
1305
1306 /*
1307 * To establish the main stack information, we have to get our context.
1308 * This is also convenient to use for getting our signal mask.
1309 */
1310 uc.uc_flags = UC_ALL;
1311 (void) __getcontext(&uc);
1312 ASSERT(uc.uc_link == NULL);
1313
1314 tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
1315 ASSERT(primary_link_map || tls_size == 0);
1316 data = lmalloc(sizeof (ulwp_t) + tls_size);
1317 if (data == NULL)
1318 thr_panic("cannot allocate thread structure for main thread");
1319 /* LINTED pointer cast may result in improper alignment */
1320 self = (ulwp_t *)(data + tls_size);
1321 init_hash_table[0].hash_bucket = self;
1322
1323 self->ul_sigmask = uc.uc_sigmask;
1324 delete_reserved_signals(&self->ul_sigmask);
1325 /*
1326 * Are the old and new sets different?
1327 * (This can happen if we are currently blocking SIGCANCEL.)
1328 * If so, we must explicitly set our signal mask, below.
1329 */
1330 setmask =
1331 ((self->ul_sigmask.__sigbits[0] ^ uc.uc_sigmask.__sigbits[0]) |
1332 (self->ul_sigmask.__sigbits[1] ^ uc.uc_sigmask.__sigbits[1]) |
1333 (self->ul_sigmask.__sigbits[2] ^ uc.uc_sigmask.__sigbits[2]) |
1334 (self->ul_sigmask.__sigbits[3] ^ uc.uc_sigmask.__sigbits[3]));
1335
1336 #ifdef __sparc
1337 /*
1338 * We cache several instructions in the thread structure for use
1339 * by the fasttrap DTrace provider. When changing this, read the
1340 * comment in fasttrap.h for the all the other places that must
1341 * be changed.
1342 */
1343 self->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
1344 self->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
1345 self->ul_dftret = 0x91d0203a; /* ta 0x3a */
1346 self->ul_dreturn = 0x81ca0000; /* return %o0 */
1347 #endif
1348
1349 self->ul_stktop = (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
1350 (void) getrlimit(RLIMIT_STACK, &rl);
1351 self->ul_stksiz = rl.rlim_cur;
1352 self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz);
1353
1354 self->ul_forw = self->ul_back = self;
1355 self->ul_hash = NULL;
1356 self->ul_ix = 0;
1357 self->ul_lwpid = 1; /* _lwp_self() */
1358 self->ul_main = 1;
1359 self->ul_self = self;
1360 self->ul_policy = -1; /* initialize only when needed */
1361 self->ul_pri = 0;
1362 self->ul_cid = 0;
1363 self->ul_rtclassid = -1;
1364 self->ul_uberdata = udp;
1365 if (oldself != NULL) {
1366 int i;
1367
1368 ASSERT(primary_link_map);
1369 ASSERT(oldself->ul_main == 1);
1370 self->ul_stsd = oldself->ul_stsd;
1371 for (i = 0; i < TSD_NFAST; i++)
1372 self->ul_ftsd[i] = oldself->ul_ftsd[i];
1373 self->ul_tls = oldself->ul_tls;
1374 /*
1375 * Retrieve all pointers to uberdata allocated
1376 * while running on previous link maps.
1377 * We would like to do a structure assignment here, but
1378 * gcc turns structure assignments into calls to memcpy(),
1379 * a function exported from libc. We can't call any such
1380 * external functions until we establish curthread, below,
1381 * so we just call our private version of memcpy().
1382 */
1383 (void) memcpy(udp, oldself->ul_uberdata, sizeof (*udp));
1384 /*
1385 * These items point to global data on the primary link map.
1386 */
1387 udp->thr_hash_table = init_hash_table;
1388 udp->sigacthandler = sigacthandler;
1389 udp->tdb.tdb_events = tdb_events;
1390 ASSERT(udp->nthreads == 1 && !udp->uberflags.uf_mt);
1391 ASSERT(udp->lwp_stacks == NULL);
1392 ASSERT(udp->ulwp_freelist == NULL);
1393 ASSERT(udp->ulwp_replace_free == NULL);
1394 ASSERT(udp->hash_size == 1);
1395 }
1396 udp->all_lwps = self;
1397 udp->ulwp_one = self;
1398 udp->pid = getpid();
1399 udp->nthreads = 1;
1400 /*
1401 * In every link map, tdb_bootstrap points to the same piece of
1402 * allocated memory. When the primary link map is initialized,
1403 * the allocated memory is assigned a pointer to the one true
1404 * uberdata. This allows libc_db to initialize itself regardless
1405 * of which instance of libc it finds in the address space.
1406 */
1407 if (udp->tdb_bootstrap == NULL)
1408 udp->tdb_bootstrap = lmalloc(sizeof (uberdata_t *));
1409 __tdb_bootstrap = udp->tdb_bootstrap;
1410 if (primary_link_map) {
1411 self->ul_primarymap = 1;
1412 udp->primary_map = 1;
1413 *udp->tdb_bootstrap = udp;
1414 }
1415 /*
1416 * Cancellation can't happen until:
1417 * pthread_cancel() is called
1418 * or:
1419 * another thread is created
1420 * For now, as a single-threaded process, set the flag that tells
1421 * PROLOGUE/EPILOGUE (in scalls.c) that cancellation can't happen.
1422 */
1423 self->ul_nocancel = 1;
1424
1425 #if defined(__amd64)
1426 (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_FSBASE, self);
1427 #elif defined(__i386)
1428 (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_GSBASE, self);
1429 #endif /* __i386 || __amd64 */
1430 set_curthread(self); /* redundant on i386 */
1431 /*
1432 * Now curthread is established and it is safe to call any
1433 * function in libc except one that uses thread-local storage.
1434 */
1435 self->ul_errnop = &errno;
1436 if (oldself != NULL) {
1437 /* tls_size was zero when oldself was allocated */
1438 lfree(oldself, sizeof (ulwp_t));
1439 }
1440 mutex_setup();
1441 atfork_init();
1442 signal_init();
1443
1444 /*
1445 * If the stack is unlimited, we set the size to zero to disable
1446 * stack checking.
1447 * XXX: Work harder here. Get the stack size from /proc/self/rmap
1448 */
1449 if (self->ul_stksiz == RLIM_INFINITY) {
1450 self->ul_ustack.ss_sp = (void *)self->ul_stktop;
1451 self->ul_ustack.ss_size = 0;
1452 } else {
1453 self->ul_ustack.ss_sp = self->ul_stk;
1454 self->ul_ustack.ss_size = self->ul_stksiz;
1455 }
1456 self->ul_ustack.ss_flags = 0;
1457 (void) setustack(&self->ul_ustack);
1458
1459 /*
1460 * Get the variables that affect thread behavior from the environment.
1461 */
1462 set_thread_vars();
1463 udp->uberflags.uf_thread_error_detection = (char)thread_error_detection;
1464 udp->thread_stack_cache = thread_stack_cache;
1465
1466 /*
1467 * Make per-thread copies of global variables, for speed.
1468 */
1469 self->ul_queue_fifo = (char)thread_queue_fifo;
1470 self->ul_cond_wait_defer = (char)thread_cond_wait_defer;
1471 self->ul_error_detection = (char)thread_error_detection;
1472 self->ul_async_safe = (char)thread_async_safe;
1473 self->ul_door_noreserve = (char)thread_door_noreserve;
1474 self->ul_misaligned = (char)thread_locks_misaligned;
1475 self->ul_max_spinners = (uint8_t)thread_max_spinners;
1476 self->ul_adaptive_spin = thread_adaptive_spin;
1477 self->ul_queue_spin = thread_queue_spin;
1478
1479 #if defined(__sparc) && !defined(_LP64)
1480 if (self->ul_misaligned) {
1481 /*
1482 * Tell the kernel to fix up ldx/stx instructions that
1483 * refer to non-8-byte aligned data instead of giving
1484 * the process an alignment trap and generating SIGBUS.
1485 *
1486 * Programs compiled for 32-bit sparc with the Studio SS12
1487 * compiler get this done for them automatically (in _init()).
1488 * We do it here for the benefit of programs compiled with
1489 * other compilers, like gcc.
1490 *
1491 * This is necessary for the _THREAD_LOCKS_MISALIGNED=1
1492 * environment variable horrible hack to work.
1493 */
1494 extern void _do_fix_align(void);
1495 _do_fix_align();
1496 }
1497 #endif
1498
1499 /*
1500 * When we have initialized the primary link map, inform
1501 * the dynamic linker about our interface functions.
1502 * Set up our pointer to the program name.
1503 */
1504 if (self->ul_primarymap)
1505 _ld_libc((void *)rtld_funcs);
1506 init_progname();
1507
1508 /*
1509 * Defer signals until TLS constructors have been called.
1510 */
1511 sigoff(self);
1512 tls_setup();
1513 sigon(self);
1514 if (setmask)
1515 (void) restore_signals(self);
1516
1517 /*
1518 * Make private copies of __xpg4 and __xpg6 so libc can test
1519 * them after this point without invoking the dynamic linker.
1520 */
1521 libc__xpg4 = __xpg4;
1522 libc__xpg6 = __xpg6;
1523
1524 /* PROBE_SUPPORT begin */
1525 if (self->ul_primarymap && __tnf_probe_notify != NULL)
1526 __tnf_probe_notify();
1527 /* PROBE_SUPPORT end */
1528
1529 init_sigev_thread();
1530 init_aio();
1531
1532 /*
1533 * We need to reset __threaded dynamically at runtime, so that
1534 * __threaded can be bound to __threaded outside libc which may not
1535 * have initial value of 1 (without a copy relocation in a.out).
1536 */
1537 __threaded = 1;
1538 }
1539
1540 #pragma fini(libc_fini)
1541 void
1542 libc_fini()
1543 {
1544 /*
1545 * If we are doing fini processing for the instance of libc
1546 * on the first alternate link map (this happens only when
1547 * the dynamic linker rejects a bad audit library), then clear
1548 * __curthread(). We abandon whatever memory was allocated by
1549 * lmalloc() while running on this alternate link-map but we
1550 * don't care (and can't find the memory in any case); we just
1551 * want to protect the application from this bad audit library.
1552 * No fini processing is done by libc in the normal case.
1553 */
1554
1555 uberdata_t *udp = curthread->ul_uberdata;
1556
1557 if (udp->primary_map == 0 && udp == &__uberdata)
1558 set_curthread(NULL);
1559 }
1560
1561 /*
1562 * finish_init is called when we are about to become multi-threaded,
1563 * that is, on the first call to thr_create().
1564 */
1565 void
1566 finish_init()
1567 {
1568 ulwp_t *self = curthread;
1569 uberdata_t *udp = self->ul_uberdata;
1570 thr_hash_table_t *htp;
1571 void *data;
1572 int i;
1573
1574 /*
1575 * No locks needed here; we are single-threaded on the first call.
1576 * We can be called only after the primary link map has been set up.
1577 */
1578 ASSERT(self->ul_primarymap);
1579 ASSERT(self == udp->ulwp_one);
1580 ASSERT(!udp->uberflags.uf_mt);
1581 ASSERT(udp->hash_size == 1);
1582
1583 /*
1584 * Initialize self->ul_policy, self->ul_cid, and self->ul_pri.
1585 */
1586 update_sched(self);
1587
1588 /*
1589 * Allocate the queue_head array if not already allocated.
1590 */
1591 if (udp->queue_head == NULL)
1592 queue_alloc();
1593
1594 /*
1595 * Now allocate the thread hash table.
1596 */
1597 if ((data = mmap(NULL, HASHTBLSZ * sizeof (thr_hash_table_t),
1598 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0))
1599 == MAP_FAILED)
1600 thr_panic("cannot allocate thread hash table");
1601
1602 udp->thr_hash_table = htp = (thr_hash_table_t *)data;
1603 udp->hash_size = HASHTBLSZ;
1604 udp->hash_mask = HASHTBLSZ - 1;
1605
1606 for (i = 0; i < HASHTBLSZ; i++, htp++) {
1607 htp->hash_lock.mutex_flag = LOCK_INITED;
1608 htp->hash_lock.mutex_magic = MUTEX_MAGIC;
1609 htp->hash_cond.cond_magic = COND_MAGIC;
1610 }
1611 hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1612
1613 /*
1614 * Set up the SIGCANCEL handler for threads cancellation.
1615 */
1616 setup_cancelsig(SIGCANCEL);
1617
1618 /*
1619 * Arrange to do special things on exit --
1620 * - collect queue statistics from all remaining active threads.
1621 * - dump queue statistics to stderr if _THREAD_QUEUE_DUMP is set.
1622 * - grab assert_lock to ensure that assertion failures
1623 * and a core dump take precedence over _exit().
1624 * (Functions are called in the reverse order of their registration.)
1625 */
1626 (void) _atexit(grab_assert_lock);
1627 #if defined(THREAD_DEBUG)
1628 (void) _atexit(dump_queue_statistics);
1629 (void) _atexit(collect_queue_statistics);
1630 #endif
1631 }
1632
1633 /*
1634 * Used only by postfork1_child(), below.
1635 */
1636 static void
1637 mark_dead_and_buried(ulwp_t *ulwp)
1638 {
1639 ulwp->ul_dead = 1;
1640 ulwp->ul_lwpid = (lwpid_t)(-1);
1641 ulwp->ul_hash = NULL;
1642 ulwp->ul_ix = -1;
1643 ulwp->ul_schedctl = NULL;
1644 ulwp->ul_schedctl_called = NULL;
1645 }
1646
1647 /*
1648 * This is called from fork1() in the child.
1649 * Reset our data structures to reflect one lwp.
1650 */
1651 void
1652 postfork1_child()
1653 {
1654 ulwp_t *self = curthread;
1655 uberdata_t *udp = self->ul_uberdata;
1656 queue_head_t *qp;
1657 ulwp_t *next;
1658 ulwp_t *ulwp;
1659 int i;
1660
1661 /* daemon threads shouldn't call fork1(), but oh well... */
1662 self->ul_usropts &= ~THR_DAEMON;
1663 udp->nthreads = 1;
1664 udp->ndaemons = 0;
1665 udp->uberflags.uf_mt = 0;
1666 __libc_threaded = 0;
1667 for (i = 0; i < udp->hash_size; i++)
1668 udp->thr_hash_table[i].hash_bucket = NULL;
1669 self->ul_lwpid = _lwp_self();
1670 hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1671
1672 /*
1673 * Some thread in the parent might have been suspended
1674 * while holding udp->callout_lock or udp->ld_lock.
1675 * Reinitialize the child's copies.
1676 */
1677 (void) mutex_init(&udp->callout_lock,
1678 USYNC_THREAD | LOCK_RECURSIVE, NULL);
1679 (void) mutex_init(&udp->ld_lock,
1680 USYNC_THREAD | LOCK_RECURSIVE, NULL);
1681
1682 /* no one in the child is on a sleep queue; reinitialize */
1683 if ((qp = udp->queue_head) != NULL) {
1684 (void) memset(qp, 0, 2 * QHASHSIZE * sizeof (queue_head_t));
1685 for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
1686 qp->qh_type = (i < QHASHSIZE)? MX : CV;
1687 qp->qh_lock.mutex_flag = LOCK_INITED;
1688 qp->qh_lock.mutex_magic = MUTEX_MAGIC;
1689 qp->qh_hlist = &qp->qh_def_root;
1690 #if defined(THREAD_DEBUG)
1691 qp->qh_hlen = 1;
1692 qp->qh_hmax = 1;
1693 #endif
1694 }
1695 }
1696
1697 /*
1698 * Do post-fork1 processing for subsystems that need it.
1699 * We need to do this before unmapping all of the abandoned
1700 * threads' stacks, below(), because the post-fork1 actions
1701 * might require access to those stacks.
1702 */
1703 postfork1_child_sigev_aio();
1704 postfork1_child_sigev_mq();
1705 postfork1_child_sigev_timer();
1706 postfork1_child_aio();
1707 /*
1708 * The above subsystems use thread pools, so this action
1709 * must be performed after those actions.
1710 */
1711 postfork1_child_tpool();
1712
1713 /*
1714 * All lwps except ourself are gone. Mark them so.
1715 * First mark all of the lwps that have already been freed.
1716 * Then mark and free all of the active lwps except ourself.
1717 * Since we are single-threaded, no locks are required here.
1718 */
1719 for (ulwp = udp->lwp_stacks; ulwp != NULL; ulwp = ulwp->ul_next)
1720 mark_dead_and_buried(ulwp);
1721 for (ulwp = udp->ulwp_freelist; ulwp != NULL; ulwp = ulwp->ul_next)
1722 mark_dead_and_buried(ulwp);
1723 for (ulwp = self->ul_forw; ulwp != self; ulwp = next) {
1724 next = ulwp->ul_forw;
1725 ulwp->ul_forw = ulwp->ul_back = NULL;
1726 mark_dead_and_buried(ulwp);
1727 tsd_free(ulwp);
1728 tls_free(ulwp);
1729 rwl_free(ulwp);
1730 heldlock_free(ulwp);
1731 ulwp_free(ulwp);
1732 }
1733 self->ul_forw = self->ul_back = udp->all_lwps = self;
1734 if (self != udp->ulwp_one)
1735 mark_dead_and_buried(udp->ulwp_one);
1736 if ((ulwp = udp->all_zombies) != NULL) {
1737 ASSERT(udp->nzombies != 0);
1738 do {
1739 next = ulwp->ul_forw;
1740 ulwp->ul_forw = ulwp->ul_back = NULL;
1741 mark_dead_and_buried(ulwp);
1742 udp->nzombies--;
1743 if (ulwp->ul_replace) {
1744 ulwp->ul_next = NULL;
1745 if (udp->ulwp_replace_free == NULL) {
1746 udp->ulwp_replace_free =
1747 udp->ulwp_replace_last = ulwp;
1748 } else {
1749 udp->ulwp_replace_last->ul_next = ulwp;
1750 udp->ulwp_replace_last = ulwp;
1751 }
1752 }
1753 } while ((ulwp = next) != udp->all_zombies);
1754 ASSERT(udp->nzombies == 0);
1755 udp->all_zombies = NULL;
1756 udp->nzombies = 0;
1757 }
1758 trim_stack_cache(0);
1759 }
1760
1761 lwpid_t
1762 lwp_self(void)
1763 {
1764 return (curthread->ul_lwpid);
1765 }
1766
1767 #pragma weak _ti_thr_self = thr_self
1768 #pragma weak pthread_self = thr_self
1769 thread_t
1770 thr_self()
1771 {
1772 return (curthread->ul_lwpid);
1773 }
1774
1775 int
1776 thr_main()
1777 {
1778 ulwp_t *self = __curthread();
1779
1780 return ((self == NULL)? -1 : self->ul_main);
1781 }
1782
1783 int
1784 _thrp_cancelled(void)
1785 {
1786 return (curthread->ul_rval == PTHREAD_CANCELED);
1787 }
1788
1789 int
1790 _thrp_stksegment(ulwp_t *ulwp, stack_t *stk)
1791 {
1792 stk->ss_sp = (void *)ulwp->ul_stktop;
1793 stk->ss_size = ulwp->ul_stksiz;
1794 stk->ss_flags = 0;
1795 return (0);
1796 }
1797
1798 #pragma weak _thr_stksegment = thr_stksegment
1799 int
1800 thr_stksegment(stack_t *stk)
1801 {
1802 return (_thrp_stksegment(curthread, stk));
1803 }
1804
1805 void
1806 force_continue(ulwp_t *ulwp)
1807 {
1808 #if defined(THREAD_DEBUG)
1809 ulwp_t *self = curthread;
1810 uberdata_t *udp = self->ul_uberdata;
1811 #endif
1812 int error;
1813 timespec_t ts;
1814
1815 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1816 ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
1817
1818 for (;;) {
1819 error = _lwp_continue(ulwp->ul_lwpid);
1820 if (error != 0 && error != EINTR)
1821 break;
1822 error = 0;
1823 if (ulwp->ul_stopping) { /* he is stopping himself */
1824 ts.tv_sec = 0; /* give him a chance to run */
1825 ts.tv_nsec = 100000; /* 100 usecs or clock tick */
1826 (void) __nanosleep(&ts, NULL);
1827 }
1828 if (!ulwp->ul_stopping) /* he is running now */
1829 break; /* so we are done */
1830 /*
1831 * He is marked as being in the process of stopping
1832 * himself. Loop around and continue him again.
1833 * He may not have been stopped the first time.
1834 */
1835 }
1836 }
1837
1838 /*
1839 * Suspend an lwp with lwp_suspend(), then move it to a safe point,
1840 * that is, to a point where ul_critical and ul_rtld are both zero.
1841 * On return, the ulwp_lock() is dropped as with ulwp_unlock().
1842 * If 'link_dropped' is non-NULL, then 'link_lock' is held on entry.
1843 * If we have to drop link_lock, we store 1 through link_dropped.
1844 * If the lwp exits before it can be suspended, we return ESRCH.
1845 */
1846 int
1847 safe_suspend(ulwp_t *ulwp, uchar_t whystopped, int *link_dropped)
1848 {
1849 ulwp_t *self = curthread;
1850 uberdata_t *udp = self->ul_uberdata;
1851 cond_t *cvp = ulwp_condvar(ulwp, udp);
1852 mutex_t *mp = ulwp_mutex(ulwp, udp);
1853 thread_t tid = ulwp->ul_lwpid;
1854 int ix = ulwp->ul_ix;
1855 int error = 0;
1856
1857 ASSERT(whystopped == TSTP_REGULAR ||
1858 whystopped == TSTP_MUTATOR ||
1859 whystopped == TSTP_FORK);
1860 ASSERT(ulwp != self);
1861 ASSERT(!ulwp->ul_stop);
1862 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1863 ASSERT(MUTEX_OWNED(mp, self));
1864
1865 if (link_dropped != NULL)
1866 *link_dropped = 0;
1867
1868 /*
1869 * We must grab the target's spin lock before suspending it.
1870 * See the comments below and in _thrp_suspend() for why.
1871 */
1872 spin_lock_set(&ulwp->ul_spinlock);
1873 (void) ___lwp_suspend(tid);
1874 spin_lock_clear(&ulwp->ul_spinlock);
1875
1876 top:
1877 if ((ulwp->ul_critical == 0 && ulwp->ul_rtld == 0) ||
1878 ulwp->ul_stopping) {
1879 /* thread is already safe */
1880 ulwp->ul_stop |= whystopped;
1881 } else {
1882 /*
1883 * Setting ul_pleasestop causes the target thread to stop
1884 * itself in _thrp_suspend(), below, after we drop its lock.
1885 * We must continue the critical thread before dropping
1886 * link_lock because the critical thread may be holding
1887 * the queue lock for link_lock. This is delicate.
1888 */
1889 ulwp->ul_pleasestop |= whystopped;
1890 force_continue(ulwp);
1891 if (link_dropped != NULL) {
1892 *link_dropped = 1;
1893 lmutex_unlock(&udp->link_lock);
1894 /* be sure to drop link_lock only once */
1895 link_dropped = NULL;
1896 }
1897
1898 /*
1899 * The thread may disappear by calling thr_exit() so we
1900 * cannot rely on the ulwp pointer after dropping the lock.
1901 * Instead, we search the hash table to find it again.
1902 * When we return, we may find that the thread has been
1903 * continued by some other thread. The suspend/continue
1904 * interfaces are prone to such race conditions by design.
1905 */
1906 while (ulwp && !ulwp->ul_dead && !ulwp->ul_stop &&
1907 (ulwp->ul_pleasestop & whystopped)) {
1908 (void) __cond_wait(cvp, mp);
1909 for (ulwp = udp->thr_hash_table[ix].hash_bucket;
1910 ulwp != NULL; ulwp = ulwp->ul_hash) {
1911 if (ulwp->ul_lwpid == tid)
1912 break;
1913 }
1914 }
1915
1916 if (ulwp == NULL || ulwp->ul_dead)
1917 error = ESRCH;
1918 else {
1919 /*
1920 * Do another lwp_suspend() to make sure we don't
1921 * return until the target thread is fully stopped
1922 * in the kernel. Don't apply lwp_suspend() until
1923 * we know that the target is not holding any
1924 * queue locks, that is, that it has completed
1925 * ulwp_unlock(self) and has, or at least is
1926 * about to, call lwp_suspend() on itself. We do
1927 * this by grabbing the target's spin lock.
1928 */
1929 ASSERT(ulwp->ul_lwpid == tid);
1930 spin_lock_set(&ulwp->ul_spinlock);
1931 (void) ___lwp_suspend(tid);
1932 spin_lock_clear(&ulwp->ul_spinlock);
1933 /*
1934 * If some other thread did a thr_continue()
1935 * on the target thread we have to start over.
1936 */
1937 if (!ulwp->ul_stopping || !(ulwp->ul_stop & whystopped))
1938 goto top;
1939 }
1940 }
1941
1942 (void) cond_broadcast(cvp);
1943 lmutex_unlock(mp);
1944 return (error);
1945 }
1946
1947 int
1948 _thrp_suspend(thread_t tid, uchar_t whystopped)
1949 {
1950 ulwp_t *self = curthread;
1951 uberdata_t *udp = self->ul_uberdata;
1952 ulwp_t *ulwp;
1953 int error = 0;
1954
1955 ASSERT((whystopped & (TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) != 0);
1956 ASSERT((whystopped & ~(TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) == 0);
1957
1958 /*
1959 * We can't suspend anyone except ourself while
1960 * some other thread is performing a fork.
1961 * This also allows only one suspension at a time.
1962 */
1963 if (tid != self->ul_lwpid)
1964 fork_lock_enter();
1965
1966 if ((ulwp = find_lwp(tid)) == NULL)
1967 error = ESRCH;
1968 else if (whystopped == TSTP_MUTATOR && !ulwp->ul_mutator) {
1969 ulwp_unlock(ulwp, udp);
1970 error = EINVAL;
1971 } else if (ulwp->ul_stop) { /* already stopped */
1972 ulwp->ul_stop |= whystopped;
1973 ulwp_broadcast(ulwp);
1974 ulwp_unlock(ulwp, udp);
1975 } else if (ulwp != self) {
1976 /*
1977 * After suspending the other thread, move it out of a
1978 * critical section and deal with the schedctl mappings.
1979 * safe_suspend() suspends the other thread, calls
1980 * ulwp_broadcast(ulwp) and drops the ulwp lock.
1981 */
1982 error = safe_suspend(ulwp, whystopped, NULL);
1983 } else {
1984 int schedctl_after_fork = 0;
1985
1986 /*
1987 * We are suspending ourself. We must not take a signal
1988 * until we return from lwp_suspend() and clear ul_stopping.
1989 * This is to guard against siglongjmp().
1990 */
1991 enter_critical(self);
1992 self->ul_sp = stkptr();
1993 _flush_windows(); /* sparc */
1994 self->ul_pleasestop = 0;
1995 self->ul_stop |= whystopped;
1996 /*
1997 * Grab our spin lock before dropping ulwp_mutex(self).
1998 * This prevents the suspending thread from applying
1999 * lwp_suspend() to us before we emerge from
2000 * lmutex_unlock(mp) and have dropped mp's queue lock.
2001 */
2002 spin_lock_set(&self->ul_spinlock);
2003 self->ul_stopping = 1;
2004 ulwp_broadcast(self);
2005 ulwp_unlock(self, udp);
2006 /*
2007 * From this point until we return from lwp_suspend(),
2008 * we must not call any function that might invoke the
2009 * dynamic linker, that is, we can only call functions
2010 * private to the library.
2011 *
2012 * Also, this is a nasty race condition for a process
2013 * that is undergoing a forkall() operation:
2014 * Once we clear our spinlock (below), we are vulnerable
2015 * to being suspended by the forkall() thread before
2016 * we manage to suspend ourself in ___lwp_suspend().
2017 * See safe_suspend() and force_continue().
2018 *
2019 * To avoid a SIGSEGV due to the disappearance
2020 * of the schedctl mappings in the child process,
2021 * which can happen in spin_lock_clear() if we
2022 * are suspended while we are in the middle of
2023 * its call to preempt(), we preemptively clear
2024 * our own schedctl pointer before dropping our
2025 * spinlock. We reinstate it, in both the parent
2026 * and (if this really is a forkall()) the child.
2027 */
2028 if (whystopped & TSTP_FORK) {
2029 schedctl_after_fork = 1;
2030 self->ul_schedctl = NULL;
2031 self->ul_schedctl_called = &udp->uberflags;
2032 }
2033 spin_lock_clear(&self->ul_spinlock);
2034 (void) ___lwp_suspend(tid);
2035 /*
2036 * Somebody else continued us.
2037 * We can't grab ulwp_lock(self)
2038 * until after clearing ul_stopping.
2039 * force_continue() relies on this.
2040 */
2041 self->ul_stopping = 0;
2042 self->ul_sp = 0;
2043 if (schedctl_after_fork) {
2044 self->ul_schedctl_called = NULL;
2045 self->ul_schedctl = NULL;
2046 (void) setup_schedctl();
2047 }
2048 ulwp_lock(self, udp);
2049 ulwp_broadcast(self);
2050 ulwp_unlock(self, udp);
2051 exit_critical(self);
2052 }
2053
2054 if (tid != self->ul_lwpid)
2055 fork_lock_exit();
2056
2057 return (error);
2058 }
2059
2060 /*
2061 * Suspend all lwps other than ourself in preparation for fork.
2062 */
2063 void
2064 suspend_fork()
2065 {
2066 ulwp_t *self = curthread;
2067 uberdata_t *udp = self->ul_uberdata;
2068 ulwp_t *ulwp;
2069 int link_dropped;
2070
2071 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2072 top:
2073 lmutex_lock(&udp->link_lock);
2074
2075 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2076 ulwp_lock(ulwp, udp);
2077 if (ulwp->ul_stop) { /* already stopped */
2078 ulwp->ul_stop |= TSTP_FORK;
2079 ulwp_broadcast(ulwp);
2080 ulwp_unlock(ulwp, udp);
2081 } else {
2082 /*
2083 * Move the stopped lwp out of a critical section.
2084 */
2085 if (safe_suspend(ulwp, TSTP_FORK, &link_dropped) ||
2086 link_dropped)
2087 goto top;
2088 }
2089 }
2090
2091 lmutex_unlock(&udp->link_lock);
2092 }
2093
2094 void
2095 continue_fork(int child)
2096 {
2097 ulwp_t *self = curthread;
2098 uberdata_t *udp = self->ul_uberdata;
2099 ulwp_t *ulwp;
2100
2101 ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2102
2103 /*
2104 * Clear the schedctl pointers in the child of forkall().
2105 */
2106 if (child) {
2107 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2108 ulwp->ul_schedctl_called =
2109 ulwp->ul_dead? &udp->uberflags : NULL;
2110 ulwp->ul_schedctl = NULL;
2111 }
2112 }
2113
2114 /*
2115 * Set all lwps that were stopped for fork() running again.
2116 */
2117 lmutex_lock(&udp->link_lock);
2118 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2119 mutex_t *mp = ulwp_mutex(ulwp, udp);
2120 lmutex_lock(mp);
2121 ASSERT(ulwp->ul_stop & TSTP_FORK);
2122 ulwp->ul_stop &= ~TSTP_FORK;
2123 ulwp_broadcast(ulwp);
2124 if (!ulwp->ul_stop)
2125 force_continue(ulwp);
2126 lmutex_unlock(mp);
2127 }
2128 lmutex_unlock(&udp->link_lock);
2129 }
2130
2131 int
2132 _thrp_continue(thread_t tid, uchar_t whystopped)
2133 {
2134 uberdata_t *udp = curthread->ul_uberdata;
2135 ulwp_t *ulwp;
2136 mutex_t *mp;
2137 int error = 0;
2138
2139 ASSERT(whystopped == TSTP_REGULAR ||
2140 whystopped == TSTP_MUTATOR);
2141
2142 /*
2143 * We single-thread the entire thread suspend/continue mechanism.
2144 */
2145 fork_lock_enter();
2146
2147 if ((ulwp = find_lwp(tid)) == NULL) {
2148 fork_lock_exit();
2149 return (ESRCH);
2150 }
2151
2152 mp = ulwp_mutex(ulwp, udp);
2153 if ((whystopped == TSTP_MUTATOR && !ulwp->ul_mutator)) {
2154 error = EINVAL;
2155 } else if (ulwp->ul_stop & whystopped) {
2156 ulwp->ul_stop &= ~whystopped;
2157 ulwp_broadcast(ulwp);
2158 if (!ulwp->ul_stop) {
2159 if (whystopped == TSTP_REGULAR && ulwp->ul_created) {
2160 ulwp->ul_sp = 0;
2161 ulwp->ul_created = 0;
2162 }
2163 force_continue(ulwp);
2164 }
2165 }
2166 lmutex_unlock(mp);
2167
2168 fork_lock_exit();
2169 return (error);
2170 }
2171
2172 int
2173 thr_suspend(thread_t tid)
2174 {
2175 return (_thrp_suspend(tid, TSTP_REGULAR));
2176 }
2177
2178 int
2179 thr_continue(thread_t tid)
2180 {
2181 return (_thrp_continue(tid, TSTP_REGULAR));
2182 }
2183
2184 void
2185 thr_yield()
2186 {
2187 yield();
2188 }
2189
2190 #pragma weak pthread_kill = thr_kill
2191 #pragma weak _thr_kill = thr_kill
2192 int
2193 thr_kill(thread_t tid, int sig)
2194 {
2195 if (sig == SIGCANCEL)
2196 return (EINVAL);
2197 return (_lwp_kill(tid, sig));
2198 }
2199
2200 /*
2201 * Exit a critical section, take deferred actions if necessary.
2202 * Called from exit_critical() and from sigon().
2203 */
2204 void
2205 do_exit_critical()
2206 {
2207 ulwp_t *self = curthread;
2208 int sig;
2209
2210 ASSERT(self->ul_critical == 0);
2211
2212 /*
2213 * Don't suspend ourself or take a deferred signal while dying
2214 * or while executing inside the dynamic linker (ld.so.1).
2215 */
2216 if (self->ul_dead || self->ul_rtld)
2217 return;
2218
2219 while (self->ul_pleasestop ||
2220 (self->ul_cursig != 0 && self->ul_sigdefer == 0)) {
2221 /*
2222 * Avoid a recursive call to exit_critical() in _thrp_suspend()
2223 * by keeping self->ul_critical == 1 here.
2224 */
2225 self->ul_critical++;
2226 while (self->ul_pleasestop) {
2227 /*
2228 * Guard against suspending ourself while on a sleep
2229 * queue. See the comments in call_user_handler().
2230 */
2231 unsleep_self();
2232 set_parking_flag(self, 0);
2233 (void) _thrp_suspend(self->ul_lwpid,
2234 self->ul_pleasestop);
2235 }
2236 self->ul_critical--;
2237
2238 if ((sig = self->ul_cursig) != 0 && self->ul_sigdefer == 0) {
2239 /*
2240 * Clear ul_cursig before proceeding.
2241 * This protects us from the dynamic linker's
2242 * calls to bind_guard()/bind_clear() in the
2243 * event that it is invoked to resolve a symbol
2244 * like take_deferred_signal() below.
2245 */
2246 self->ul_cursig = 0;
2247 take_deferred_signal(sig);
2248 ASSERT(self->ul_cursig == 0);
2249 }
2250 }
2251 ASSERT(self->ul_critical == 0);
2252 }
2253
2254 /*
2255 * _ti_bind_guard() and _ti_bind_clear() are called by the dynamic linker
2256 * (ld.so.1) when it has do do something, like resolve a symbol to be called
2257 * by the application or one of its libraries. _ti_bind_guard() is called
2258 * on entry to ld.so.1, _ti_bind_clear() on exit from ld.so.1 back to the
2259 * application. The dynamic linker gets special dispensation from libc to
2260 * run in a critical region (all signals deferred and no thread suspension
2261 * or forking allowed), and to be immune from cancellation for the duration.
2262 */
2263 int
2264 _ti_bind_guard(int flags)
2265 {
2266 ulwp_t *self = curthread;
2267 uberdata_t *udp = self->ul_uberdata;
2268 int bindflag = (flags & THR_FLG_RTLD);
2269
2270 if ((self->ul_bindflags & bindflag) == bindflag)
2271 return (0);
2272 self->ul_bindflags |= bindflag;
2273 if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2274 sigoff(self); /* see no signals while holding ld_lock */
2275 self->ul_rtld++; /* don't suspend while in ld.so.1 */
2276 (void) mutex_lock(&udp->ld_lock);
2277 }
2278 enter_critical(self);
2279 self->ul_save_state = self->ul_cancel_disabled;
2280 self->ul_cancel_disabled = 1;
2281 set_cancel_pending_flag(self, 0);
2282 return (1);
2283 }
2284
2285 int
2286 _ti_bind_clear(int flags)
2287 {
2288 ulwp_t *self = curthread;
2289 uberdata_t *udp = self->ul_uberdata;
2290 int bindflag = (flags & THR_FLG_RTLD);
2291
2292 if ((self->ul_bindflags & bindflag) == 0)
2293 return (self->ul_bindflags);
2294 self->ul_bindflags &= ~bindflag;
2295 self->ul_cancel_disabled = self->ul_save_state;
2296 set_cancel_pending_flag(self, 0);
2297 exit_critical(self);
2298 if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2299 if (MUTEX_OWNED(&udp->ld_lock, self)) {
2300 (void) mutex_unlock(&udp->ld_lock);
2301 self->ul_rtld--;
2302 sigon(self); /* reenable signals */
2303 }
2304 }
2305 return (self->ul_bindflags);
2306 }
2307
2308 /*
2309 * Tell the dynamic linker (ld.so.1) whether or not it was entered from
2310 * a critical region in libc. Return zero if not, else return non-zero.
2311 */
2312 int
2313 _ti_critical(void)
2314 {
2315 ulwp_t *self = curthread;
2316 int level = self->ul_critical;
2317
2318 if ((self->ul_bindflags & THR_FLG_RTLD) == 0 || level == 0)
2319 return (level); /* ld.so.1 hasn't (yet) called enter() */
2320 return (level - 1);
2321 }
2322
2323 /*
2324 * sigoff() and sigon() enable cond_wait() to behave (optionally) like
2325 * it does in the old libthread (see the comments in cond_wait_queue()).
2326 * Also, signals are deferred at thread startup until TLS constructors
2327 * have all been called, at which time _thrp_setup() calls sigon().
2328 *
2329 * _sigoff() and _sigon() are external consolidation-private interfaces to
2330 * sigoff() and sigon(), respectively, in libc. These are used in libnsl.
2331 * Also, _sigoff() and _sigon() are called from dbx's run-time checking
2332 * (librtc.so) to defer signals during its critical sections (not to be
2333 * confused with libc critical sections [see exit_critical() above]).
2334 */
2335 void
2336 _sigoff(void)
2337 {
2338 ulwp_t *self = curthread;
2339
2340 sigoff(self);
2341 }
2342
2343 void
2344 _sigon(void)
2345 {
2346 ulwp_t *self = curthread;
2347
2348 ASSERT(self->ul_sigdefer > 0);
2349 sigon(self);
2350 }
2351
2352 int
2353 thr_getconcurrency()
2354 {
2355 return (thr_concurrency);
2356 }
2357
2358 int
2359 pthread_getconcurrency()
2360 {
2361 return (pthread_concurrency);
2362 }
2363
2364 int
2365 thr_setconcurrency(int new_level)
2366 {
2367 uberdata_t *udp = curthread->ul_uberdata;
2368
2369 if (new_level < 0)
2370 return (EINVAL);
2371 if (new_level > 65536) /* 65536 is totally arbitrary */
2372 return (EAGAIN);
2373 lmutex_lock(&udp->link_lock);
2374 if (new_level > thr_concurrency)
2375 thr_concurrency = new_level;
2376 lmutex_unlock(&udp->link_lock);
2377 return (0);
2378 }
2379
2380 int
2381 pthread_setconcurrency(int new_level)
2382 {
2383 if (new_level < 0)
2384 return (EINVAL);
2385 if (new_level > 65536) /* 65536 is totally arbitrary */
2386 return (EAGAIN);
2387 pthread_concurrency = new_level;
2388 return (0);
2389 }
2390
2391 size_t
2392 thr_min_stack(void)
2393 {
2394 return (MINSTACK);
2395 }
2396
2397 int
2398 __nthreads(void)
2399 {
2400 return (curthread->ul_uberdata->nthreads);
2401 }
2402
2403 /*
2404 * XXX
2405 * The remainder of this file implements the private interfaces to java for
2406 * garbage collection. It is no longer used, at least by java 1.2.
2407 * It can all go away once all old JVMs have disappeared.
2408 */
2409
2410 int suspendingallmutators; /* when non-zero, suspending all mutators. */
2411 int suspendedallmutators; /* when non-zero, all mutators suspended. */
2412 int mutatorsbarrier; /* when non-zero, mutators barrier imposed. */
2413 mutex_t mutatorslock = DEFAULTMUTEX; /* used to enforce mutators barrier. */
2414 cond_t mutatorscv = DEFAULTCV; /* where non-mutators sleep. */
2415
2416 /*
2417 * Get the available register state for the target thread.
2418 * Return non-volatile registers: TRS_NONVOLATILE
2419 */
2420 #pragma weak _thr_getstate = thr_getstate
2421 int
2422 thr_getstate(thread_t tid, int *flag, lwpid_t *lwp, stack_t *ss, gregset_t rs)
2423 {
2424 ulwp_t *self = curthread;
2425 uberdata_t *udp = self->ul_uberdata;
2426 ulwp_t **ulwpp;
2427 ulwp_t *ulwp;
2428 int error = 0;
2429 int trs_flag = TRS_LWPID;
2430
2431 if (tid == 0 || self->ul_lwpid == tid) {
2432 ulwp = self;
2433 ulwp_lock(ulwp, udp);
2434 } else if ((ulwpp = find_lwpp(tid)) != NULL) {
2435 ulwp = *ulwpp;
2436 } else {
2437 if (flag)
2438 *flag = TRS_INVALID;
2439 return (ESRCH);
2440 }
2441
2442 if (ulwp->ul_dead) {
2443 trs_flag = TRS_INVALID;
2444 } else if (!ulwp->ul_stop && !suspendedallmutators) {
2445 error = EINVAL;
2446 trs_flag = TRS_INVALID;
2447 } else if (ulwp->ul_stop) {
2448 trs_flag = TRS_NONVOLATILE;
2449 getgregs(ulwp, rs);
2450 }
2451
2452 if (flag)
2453 *flag = trs_flag;
2454 if (lwp)
2455 *lwp = tid;
2456 if (ss != NULL)
2457 (void) _thrp_stksegment(ulwp, ss);
2458
2459 ulwp_unlock(ulwp, udp);
2460 return (error);
2461 }
2462
2463 /*
2464 * Set the appropriate register state for the target thread.
2465 * This is not used by java. It exists solely for the MSTC test suite.
2466 */
2467 #pragma weak _thr_setstate = thr_setstate
2468 int
2469 thr_setstate(thread_t tid, int flag, gregset_t rs)
2470 {
2471 uberdata_t *udp = curthread->ul_uberdata;
2472 ulwp_t *ulwp;
2473 int error = 0;
2474
2475 if ((ulwp = find_lwp(tid)) == NULL)
2476 return (ESRCH);
2477
2478 if (!ulwp->ul_stop && !suspendedallmutators)
2479 error = EINVAL;
2480 else if (rs != NULL) {
2481 switch (flag) {
2482 case TRS_NONVOLATILE:
2483 /* do /proc stuff here? */
2484 if (ulwp->ul_stop)
2485 setgregs(ulwp, rs);
2486 else
2487 error = EINVAL;
2488 break;
2489 case TRS_LWPID: /* do /proc stuff here? */
2490 default:
2491 error = EINVAL;
2492 break;
2493 }
2494 }
2495
2496 ulwp_unlock(ulwp, udp);
2497 return (error);
2498 }
2499
2500 int
2501 getlwpstatus(thread_t tid, struct lwpstatus *sp)
2502 {
2503 extern ssize_t __pread(int, void *, size_t, off_t);
2504 char buf[100];
2505 int fd;
2506
2507 /* "/proc/self/lwp/%u/lwpstatus" w/o stdio */
2508 (void) strcpy(buf, "/proc/self/lwp/");
2509 ultos((uint64_t)tid, 10, buf + strlen(buf));
2510 (void) strcat(buf, "/lwpstatus");
2511 if ((fd = __open(buf, O_RDONLY, 0)) >= 0) {
2512 while (__pread(fd, sp, sizeof (*sp), 0) == sizeof (*sp)) {
2513 if (sp->pr_flags & PR_STOPPED) {
2514 (void) __close(fd);
2515 return (0);
2516 }
2517 yield(); /* give him a chance to stop */
2518 }
2519 (void) __close(fd);
2520 }
2521 return (-1);
2522 }
2523
2524 int
2525 putlwpregs(thread_t tid, prgregset_t prp)
2526 {
2527 extern ssize_t __writev(int, const struct iovec *, int);
2528 char buf[100];
2529 int fd;
2530 long dstop_sreg[2];
2531 long run_null[2];
2532 iovec_t iov[3];
2533
2534 /* "/proc/self/lwp/%u/lwpctl" w/o stdio */
2535 (void) strcpy(buf, "/proc/self/lwp/");
2536 ultos((uint64_t)tid, 10, buf + strlen(buf));
2537 (void) strcat(buf, "/lwpctl");
2538 if ((fd = __open(buf, O_WRONLY, 0)) >= 0) {
2539 dstop_sreg[0] = PCDSTOP; /* direct it to stop */
2540 dstop_sreg[1] = PCSREG; /* set the registers */
2541 iov[0].iov_base = (caddr_t)dstop_sreg;
2542 iov[0].iov_len = sizeof (dstop_sreg);
2543 iov[1].iov_base = (caddr_t)prp; /* from the register set */
2544 iov[1].iov_len = sizeof (prgregset_t);
2545 run_null[0] = PCRUN; /* make it runnable again */
2546 run_null[1] = 0;
2547 iov[2].iov_base = (caddr_t)run_null;
2548 iov[2].iov_len = sizeof (run_null);
2549 if (__writev(fd, iov, 3) >= 0) {
2550 (void) __close(fd);
2551 return (0);
2552 }
2553 (void) __close(fd);
2554 }
2555 return (-1);
2556 }
2557
2558 static ulong_t
2559 gettsp_slow(thread_t tid)
2560 {
2561 char buf[100];
2562 struct lwpstatus status;
2563
2564 if (getlwpstatus(tid, &status) != 0) {
2565 /* "__gettsp(%u): can't read lwpstatus" w/o stdio */
2566 (void) strcpy(buf, "__gettsp(");
2567 ultos((uint64_t)tid, 10, buf + strlen(buf));
2568 (void) strcat(buf, "): can't read lwpstatus");
2569 thr_panic(buf);
2570 }
2571 return (status.pr_reg[R_SP]);
2572 }
2573
2574 ulong_t
2575 __gettsp(thread_t tid)
2576 {
2577 uberdata_t *udp = curthread->ul_uberdata;
2578 ulwp_t *ulwp;
2579 ulong_t result;
2580
2581 if ((ulwp = find_lwp(tid)) == NULL)
2582 return (0);
2583
2584 if (ulwp->ul_stop && (result = ulwp->ul_sp) != 0) {
2585 ulwp_unlock(ulwp, udp);
2586 return (result);
2587 }
2588
2589 result = gettsp_slow(tid);
2590 ulwp_unlock(ulwp, udp);
2591 return (result);
2592 }
2593
2594 /*
2595 * This tells java stack walkers how to find the ucontext
2596 * structure passed to signal handlers.
2597 */
2598 #pragma weak _thr_sighndlrinfo = thr_sighndlrinfo
2599 void
2600 thr_sighndlrinfo(void (**func)(), int *funcsize)
2601 {
2602 *func = &__sighndlr;
2603 *funcsize = (char *)&__sighndlrend - (char *)&__sighndlr;
2604 }
2605
2606 /*
2607 * Mark a thread a mutator or reset a mutator to being a default,
2608 * non-mutator thread.
2609 */
2610 #pragma weak _thr_setmutator = thr_setmutator
2611 int
2612 thr_setmutator(thread_t tid, int enabled)
2613 {
2614 ulwp_t *self = curthread;
2615 uberdata_t *udp = self->ul_uberdata;
2616 ulwp_t *ulwp;
2617 int error;
2618 int cancel_state;
2619
2620 enabled = enabled? 1 : 0;
2621 top:
2622 if (tid == 0) {
2623 ulwp = self;
2624 ulwp_lock(ulwp, udp);
2625 } else if ((ulwp = find_lwp(tid)) == NULL) {
2626 return (ESRCH);
2627 }
2628
2629 /*
2630 * The target thread should be the caller itself or a suspended thread.
2631 * This prevents the target from also changing its ul_mutator field.
2632 */
2633 error = 0;
2634 if (ulwp != self && !ulwp->ul_stop && enabled)
2635 error = EINVAL;
2636 else if (ulwp->ul_mutator != enabled) {
2637 lmutex_lock(&mutatorslock);
2638 if (mutatorsbarrier) {
2639 ulwp_unlock(ulwp, udp);
2640 (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
2641 &cancel_state);
2642 while (mutatorsbarrier)
2643 (void) cond_wait(&mutatorscv, &mutatorslock);
2644 (void) pthread_setcancelstate(cancel_state, NULL);
2645 lmutex_unlock(&mutatorslock);
2646 goto top;
2647 }
2648 ulwp->ul_mutator = enabled;
2649 lmutex_unlock(&mutatorslock);
2650 }
2651
2652 ulwp_unlock(ulwp, udp);
2653 return (error);
2654 }
2655
2656 /*
2657 * Establish a barrier against new mutators. Any non-mutator trying
2658 * to become a mutator is suspended until the barrier is removed.
2659 */
2660 #pragma weak _thr_mutators_barrier = thr_mutators_barrier
2661 void
2662 thr_mutators_barrier(int enabled)
2663 {
2664 int oldvalue;
2665 int cancel_state;
2666
2667 lmutex_lock(&mutatorslock);
2668
2669 /*
2670 * Wait if trying to set the barrier while it is already set.
2671 */
2672 (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2673 while (mutatorsbarrier && enabled)
2674 (void) cond_wait(&mutatorscv, &mutatorslock);
2675 (void) pthread_setcancelstate(cancel_state, NULL);
2676
2677 oldvalue = mutatorsbarrier;
2678 mutatorsbarrier = enabled;
2679 /*
2680 * Wakeup any blocked non-mutators when barrier is removed.
2681 */
2682 if (oldvalue && !enabled)
2683 (void) cond_broadcast(&mutatorscv);
2684 lmutex_unlock(&mutatorslock);
2685 }
2686
2687 /*
2688 * Suspend the set of all mutators except for the caller. The list
2689 * of actively running threads is searched and only the mutators
2690 * in this list are suspended. Actively running non-mutators remain
2691 * running. Any other thread is suspended.
2692 */
2693 #pragma weak _thr_suspend_allmutators = thr_suspend_allmutators
2694 int
2695 thr_suspend_allmutators(void)
2696 {
2697 ulwp_t *self = curthread;
2698 uberdata_t *udp = self->ul_uberdata;
2699 ulwp_t *ulwp;
2700 int link_dropped;
2701
2702 /*
2703 * We single-thread the entire thread suspend/continue mechanism.
2704 */
2705 fork_lock_enter();
2706
2707 top:
2708 lmutex_lock(&udp->link_lock);
2709
2710 if (suspendingallmutators || suspendedallmutators) {
2711 lmutex_unlock(&udp->link_lock);
2712 fork_lock_exit();
2713 return (EINVAL);
2714 }
2715 suspendingallmutators = 1;
2716
2717 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2718 ulwp_lock(ulwp, udp);
2719 if (!ulwp->ul_mutator) {
2720 ulwp_unlock(ulwp, udp);
2721 } else if (ulwp->ul_stop) { /* already stopped */
2722 ulwp->ul_stop |= TSTP_MUTATOR;
2723 ulwp_broadcast(ulwp);
2724 ulwp_unlock(ulwp, udp);
2725 } else {
2726 /*
2727 * Move the stopped lwp out of a critical section.
2728 */
2729 if (safe_suspend(ulwp, TSTP_MUTATOR, &link_dropped) ||
2730 link_dropped) {
2731 suspendingallmutators = 0;
2732 goto top;
2733 }
2734 }
2735 }
2736
2737 suspendedallmutators = 1;
2738 suspendingallmutators = 0;
2739 lmutex_unlock(&udp->link_lock);
2740 fork_lock_exit();
2741 return (0);
2742 }
2743
2744 /*
2745 * Suspend the target mutator. The caller is permitted to suspend
2746 * itself. If a mutator barrier is enabled, the caller will suspend
2747 * itself as though it had been suspended by thr_suspend_allmutators().
2748 * When the barrier is removed, this thread will be resumed. Any
2749 * suspended mutator, whether suspended by thr_suspend_mutator(), or by
2750 * thr_suspend_allmutators(), can be resumed by thr_continue_mutator().
2751 */
2752 #pragma weak _thr_suspend_mutator = thr_suspend_mutator
2753 int
2754 thr_suspend_mutator(thread_t tid)
2755 {
2756 if (tid == 0)
2757 tid = curthread->ul_lwpid;
2758 return (_thrp_suspend(tid, TSTP_MUTATOR));
2759 }
2760
2761 /*
2762 * Resume the set of all suspended mutators.
2763 */
2764 #pragma weak _thr_continue_allmutators = thr_continue_allmutators
2765 int
2766 thr_continue_allmutators()
2767 {
2768 ulwp_t *self = curthread;
2769 uberdata_t *udp = self->ul_uberdata;
2770 ulwp_t *ulwp;
2771
2772 /*
2773 * We single-thread the entire thread suspend/continue mechanism.
2774 */
2775 fork_lock_enter();
2776
2777 lmutex_lock(&udp->link_lock);
2778 if (!suspendedallmutators) {
2779 lmutex_unlock(&udp->link_lock);
2780 fork_lock_exit();
2781 return (EINVAL);
2782 }
2783 suspendedallmutators = 0;
2784
2785 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2786 mutex_t *mp = ulwp_mutex(ulwp, udp);
2787 lmutex_lock(mp);
2788 if (ulwp->ul_stop & TSTP_MUTATOR) {
2789 ulwp->ul_stop &= ~TSTP_MUTATOR;
2790 ulwp_broadcast(ulwp);
2791 if (!ulwp->ul_stop)
2792 force_continue(ulwp);
2793 }
2794 lmutex_unlock(mp);
2795 }
2796
2797 lmutex_unlock(&udp->link_lock);
2798 fork_lock_exit();
2799 return (0);
2800 }
2801
2802 /*
2803 * Resume a suspended mutator.
2804 */
2805 #pragma weak _thr_continue_mutator = thr_continue_mutator
2806 int
2807 thr_continue_mutator(thread_t tid)
2808 {
2809 return (_thrp_continue(tid, TSTP_MUTATOR));
2810 }
2811
2812 #pragma weak _thr_wait_mutator = thr_wait_mutator
2813 int
2814 thr_wait_mutator(thread_t tid, int dontwait)
2815 {
2816 uberdata_t *udp = curthread->ul_uberdata;
2817 ulwp_t *ulwp;
2818 int cancel_state;
2819 int error = 0;
2820
2821 (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2822 top:
2823 if ((ulwp = find_lwp(tid)) == NULL) {
2824 (void) pthread_setcancelstate(cancel_state, NULL);
2825 return (ESRCH);
2826 }
2827
2828 if (!ulwp->ul_mutator)
2829 error = EINVAL;
2830 else if (dontwait) {
2831 if (!(ulwp->ul_stop & TSTP_MUTATOR))
2832 error = EWOULDBLOCK;
2833 } else if (!(ulwp->ul_stop & TSTP_MUTATOR)) {
2834 cond_t *cvp = ulwp_condvar(ulwp, udp);
2835 mutex_t *mp = ulwp_mutex(ulwp, udp);
2836
2837 (void) cond_wait(cvp, mp);
2838 (void) lmutex_unlock(mp);
2839 goto top;
2840 }
2841
2842 ulwp_unlock(ulwp, udp);
2843 (void) pthread_setcancelstate(cancel_state, NULL);
2844 return (error);
2845 }
2846
2847 /* PROBE_SUPPORT begin */
2848
2849 void
2850 thr_probe_setup(void *data)
2851 {
2852 curthread->ul_tpdp = data;
2853 }
2854
2855 static void *
2856 _thread_probe_getfunc()
2857 {
2858 return (curthread->ul_tpdp);
2859 }
2860
2861 void * (*thr_probe_getfunc_addr)(void) = _thread_probe_getfunc;
2862
2863 /* ARGSUSED */
2864 void
2865 _resume(ulwp_t *ulwp, caddr_t sp, int dontsave)
2866 {
2867 /* never called */
2868 }
2869
2870 /* ARGSUSED */
2871 void
2872 _resume_ret(ulwp_t *oldlwp)
2873 {
2874 /* never called */
2875 }
2876
2877 /* PROBE_SUPPORT end */