1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
29 */
30
31 #include <sys/param.h>
32 #include <sys/types.h>
33 #include <sys/sysmacros.h>
34 #include <sys/systm.h>
35 #include <sys/thread.h>
36 #include <sys/proc.h>
37 #include <sys/task.h>
38 #include <sys/project.h>
39 #include <sys/signal.h>
40 #include <sys/errno.h>
41 #include <sys/vmparam.h>
42 #include <sys/stack.h>
43 #include <sys/procfs.h>
44 #include <sys/prsystm.h>
45 #include <sys/cpuvar.h>
46 #include <sys/kmem.h>
47 #include <sys/vtrace.h>
48 #include <sys/door.h>
49 #include <vm/seg_kp.h>
50 #include <sys/debug.h>
51 #include <sys/tnf.h>
52 #include <sys/schedctl.h>
53 #include <sys/poll.h>
54 #include <sys/copyops.h>
55 #include <sys/lwp_upimutex_impl.h>
56 #include <sys/cpupart.h>
57 #include <sys/lgrp.h>
58 #include <sys/rctl.h>
59 #include <sys/contract_impl.h>
60 #include <sys/cpc_impl.h>
61 #include <sys/sdt.h>
62 #include <sys/cmn_err.h>
63 #include <sys/brand.h>
64 #include <sys/cyclic.h>
65 #include <sys/pool.h>
66
67 /* hash function for the lwpid hash table, p->p_tidhash[] */
68 #define TIDHASH(tid, hash_sz) ((tid) & ((hash_sz) - 1))
69
70 void *segkp_lwp; /* cookie for pool of segkp resources */
71 extern void reapq_move_lq_to_tq(kthread_t *);
72 extern void freectx_ctx(struct ctxop *);
73
74 /*
75 * Create a kernel thread associated with a particular system process. Give
76 * it an LWP so that microstate accounting will be available for it.
77 */
78 kthread_t *
79 lwp_kernel_create(proc_t *p, void (*proc)(), void *arg, int state, pri_t pri)
80 {
81 klwp_t *lwp;
82
83 VERIFY((p->p_flag & SSYS) != 0);
84
85 lwp = lwp_create(proc, arg, 0, p, state, pri, &t0.t_hold, syscid, 0);
86
87 VERIFY(lwp != NULL);
88
89 return (lwptot(lwp));
90 }
91
92 /*
93 * Create a thread that appears to be stopped at sys_rtt.
94 */
95 klwp_t *
96 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
97 int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
98 {
99 klwp_t *lwp = NULL;
100 kthread_t *t;
101 kthread_t *tx;
102 cpupart_t *oldpart = NULL;
103 size_t stksize;
104 caddr_t lwpdata = NULL;
105 processorid_t binding;
106 int err = 0;
107 kproject_t *oldkpj, *newkpj;
108 void *bufp = NULL;
109 klwp_t *curlwp;
110 lwpent_t *lep;
111 lwpdir_t *old_dir = NULL;
112 uint_t old_dirsz = 0;
113 tidhash_t *old_hash = NULL;
114 uint_t old_hashsz = 0;
115 ret_tidhash_t *ret_tidhash = NULL;
116 int i;
117 int rctlfail = 0;
118 boolean_t branded = 0;
119 struct ctxop *ctx = NULL;
120
121 ASSERT(cid != sysdccid); /* system threads must start in SYS */
122
123 ASSERT(p != &p0); /* No new LWPs in p0. */
124
125 mutex_enter(&p->p_lock);
126 mutex_enter(&p->p_zone->zone_nlwps_lock);
127 /*
128 * don't enforce rctl limits on system processes
129 */
130 if (!CLASS_KERNEL(cid)) {
131 if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
132 if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
133 1, 0) & RCT_DENY)
134 rctlfail = 1;
135 if (p->p_task->tk_proj->kpj_nlwps >=
136 p->p_task->tk_proj->kpj_nlwps_ctl)
137 if (rctl_test(rc_project_nlwps,
138 p->p_task->tk_proj->kpj_rctls, p, 1, 0)
139 & RCT_DENY)
140 rctlfail = 1;
141 if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
142 if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
143 1, 0) & RCT_DENY)
144 rctlfail = 1;
145 }
146 if (rctlfail) {
147 mutex_exit(&p->p_zone->zone_nlwps_lock);
148 mutex_exit(&p->p_lock);
149 atomic_inc_32(&p->p_zone->zone_ffcap);
150 return (NULL);
151 }
152 p->p_task->tk_nlwps++;
153 p->p_task->tk_proj->kpj_nlwps++;
154 p->p_zone->zone_nlwps++;
155 mutex_exit(&p->p_zone->zone_nlwps_lock);
156 mutex_exit(&p->p_lock);
157
158 curlwp = ttolwp(curthread);
159 if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
160 stksize = lwp_default_stksize;
161
162 if (CLASS_KERNEL(cid)) {
163 /*
164 * Since we are creating an LWP in an SSYS process, we do not
165 * inherit anything from the current thread's LWP. We set
166 * stksize and lwpdata to 0 in order to let thread_create()
167 * allocate a regular kernel thread stack for this thread.
168 */
169 curlwp = NULL;
170 stksize = 0;
171 lwpdata = NULL;
172
173 } else if (stksize == lwp_default_stksize) {
174 /*
175 * Try to reuse an <lwp,stack> from the LWP deathrow.
176 */
177 if (lwp_reapcnt > 0) {
178 mutex_enter(&reaplock);
179 if ((t = lwp_deathrow) != NULL) {
180 ASSERT(t->t_swap);
181 lwp_deathrow = t->t_forw;
182 lwp_reapcnt--;
183 lwpdata = t->t_swap;
184 lwp = t->t_lwp;
185 ctx = t->t_ctx;
186 t->t_swap = NULL;
187 t->t_lwp = NULL;
188 t->t_ctx = NULL;
189 reapq_move_lq_to_tq(t);
190 }
191 mutex_exit(&reaplock);
192 if (lwp != NULL) {
193 lwp_stk_fini(lwp);
194 }
195 if (ctx != NULL) {
196 freectx_ctx(ctx);
197 }
198 }
199 if (lwpdata == NULL &&
200 (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
201 mutex_enter(&p->p_lock);
202 mutex_enter(&p->p_zone->zone_nlwps_lock);
203 p->p_task->tk_nlwps--;
204 p->p_task->tk_proj->kpj_nlwps--;
205 p->p_zone->zone_nlwps--;
206 mutex_exit(&p->p_zone->zone_nlwps_lock);
207 mutex_exit(&p->p_lock);
208 atomic_inc_32(&p->p_zone->zone_ffnomem);
209 return (NULL);
210 }
211 } else {
212 stksize = roundup(stksize, PAGESIZE);
213 if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
214 (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
215 mutex_enter(&p->p_lock);
216 mutex_enter(&p->p_zone->zone_nlwps_lock);
217 p->p_task->tk_nlwps--;
218 p->p_task->tk_proj->kpj_nlwps--;
219 p->p_zone->zone_nlwps--;
220 mutex_exit(&p->p_zone->zone_nlwps_lock);
221 mutex_exit(&p->p_lock);
222 atomic_inc_32(&p->p_zone->zone_ffnomem);
223 return (NULL);
224 }
225 }
226
227 /*
228 * Create a thread, initializing the stack pointer
229 */
230 t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
231
232 /*
233 * If a non-NULL stack base is passed in, thread_create() assumes
234 * that the stack might be statically allocated (as opposed to being
235 * allocated from segkp), and so it does not set t_swap. Since
236 * the lwpdata was allocated from segkp, we must set t_swap to point
237 * to it ourselves.
238 *
239 * This would be less confusing if t_swap had a better name; it really
240 * indicates that the stack is allocated from segkp, regardless of
241 * whether or not it is swappable.
242 */
243 if (lwpdata != NULL) {
244 ASSERT(!CLASS_KERNEL(cid));
245 ASSERT(t->t_swap == NULL);
246 t->t_swap = lwpdata; /* Start of page-able data */
247 }
248
249 /*
250 * If the stack and lwp can be reused, mark the thread as such.
251 * When we get to reapq_add() from resume_from_zombie(), these
252 * threads will go onto lwp_deathrow instead of thread_deathrow.
253 */
254 if (!CLASS_KERNEL(cid) && stksize == lwp_default_stksize)
255 t->t_flag |= T_LWPREUSE;
256
257 if (lwp == NULL)
258 lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
259 bzero(lwp, sizeof (*lwp));
260 t->t_lwp = lwp;
261
262 t->t_hold = *smask;
263 lwp->lwp_thread = t;
264 lwp->lwp_procp = p;
265 lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
266 if (curlwp != NULL && curlwp->lwp_childstksz != 0)
267 lwp->lwp_childstksz = curlwp->lwp_childstksz;
268
269 t->t_stk = lwp_stk_init(lwp, t->t_stk);
270 thread_load(t, proc, arg, len);
271
272 /*
273 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
274 */
275 if (p->p_rprof_cyclic != CYCLIC_NONE)
276 t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
277
278 if (cid != NOCLASS)
279 (void) CL_ALLOC(&bufp, cid, KM_SLEEP);
280
281 /*
282 * Allocate an lwp directory entry for the new lwp.
283 */
284 lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
285
286 mutex_enter(&p->p_lock);
287 grow:
288 /*
289 * Grow the lwp (thread) directory and lwpid hash table if necessary.
290 * A note on the growth algorithm:
291 * The new lwp directory size is computed as:
292 * new = 2 * old + 2
293 * Starting with an initial size of 2 (see exec_common()),
294 * this yields numbers that are a power of two minus 2:
295 * 2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
296 * The size of the lwpid hash table must be a power of two
297 * and must be commensurate in size with the lwp directory
298 * so that hash bucket chains remain short. Therefore,
299 * the lwpid hash table size is computed as:
300 * hashsz = (dirsz + 2) / 2
301 * which leads to these hash table sizes corresponding to
302 * the above directory sizes:
303 * 2, 4, 8, 16, 32, 64, 128, 256, 512, ...
304 * A note on growing the hash table:
305 * For performance reasons, code in lwp_unpark() does not
306 * acquire curproc->p_lock when searching the hash table.
307 * Rather, it calls lwp_hash_lookup_and_lock() which
308 * acquires only the individual hash bucket lock, taking
309 * care to deal with reallocation of the hash table
310 * during the time it takes to acquire the lock.
311 *
312 * This is sufficient to protect the integrity of the
313 * hash table, but it requires us to acquire all of the
314 * old hash bucket locks before growing the hash table
315 * and to release them afterwards. It also requires us
316 * not to free the old hash table because some thread
317 * in lwp_hash_lookup_and_lock() might still be trying
318 * to acquire the old bucket lock.
319 *
320 * So we adopt the tactic of keeping all of the retired
321 * hash tables on a linked list, so they can be safely
322 * freed when the process exits or execs.
323 *
324 * Because the hash table grows in powers of two, the
325 * total size of all of the hash tables will be slightly
326 * less than twice the size of the largest hash table.
327 */
328 while (p->p_lwpfree == NULL) {
329 uint_t dirsz = p->p_lwpdir_sz;
330 lwpdir_t *new_dir;
331 uint_t new_dirsz;
332 lwpdir_t *ldp;
333 tidhash_t *new_hash;
334 uint_t new_hashsz;
335
336 mutex_exit(&p->p_lock);
337
338 /*
339 * Prepare to remember the old p_tidhash for later
340 * kmem_free()ing when the process exits or execs.
341 */
342 if (ret_tidhash == NULL)
343 ret_tidhash = kmem_zalloc(sizeof (ret_tidhash_t),
344 KM_SLEEP);
345 if (old_dir != NULL)
346 kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
347 if (old_hash != NULL)
348 kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
349
350 new_dirsz = 2 * dirsz + 2;
351 new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
352 for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
353 ldp->ld_next = ldp + 1;
354 new_hashsz = (new_dirsz + 2) / 2;
355 new_hash = kmem_zalloc(new_hashsz * sizeof (tidhash_t),
356 KM_SLEEP);
357
358 mutex_enter(&p->p_lock);
359 if (p == curproc)
360 prbarrier(p);
361
362 if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
363 /*
364 * Someone else beat us to it or some lwp exited.
365 * Set up to free our memory and take a lap.
366 */
367 old_dir = new_dir;
368 old_dirsz = new_dirsz;
369 old_hash = new_hash;
370 old_hashsz = new_hashsz;
371 } else {
372 /*
373 * For the benefit of lwp_hash_lookup_and_lock(),
374 * called from lwp_unpark(), which searches the
375 * tid hash table without acquiring p->p_lock,
376 * we must acquire all of the tid hash table
377 * locks before replacing p->p_tidhash.
378 */
379 old_hash = p->p_tidhash;
380 old_hashsz = p->p_tidhash_sz;
381 for (i = 0; i < old_hashsz; i++) {
382 mutex_enter(&old_hash[i].th_lock);
383 mutex_enter(&new_hash[i].th_lock);
384 }
385
386 /*
387 * We simply hash in all of the old directory entries.
388 * This works because the old directory has no empty
389 * slots and the new hash table starts out empty.
390 * This reproduces the original directory ordering
391 * (required for /proc directory semantics).
392 */
393 old_dir = p->p_lwpdir;
394 old_dirsz = p->p_lwpdir_sz;
395 p->p_lwpdir = new_dir;
396 p->p_lwpfree = new_dir;
397 p->p_lwpdir_sz = new_dirsz;
398 for (ldp = old_dir, i = 0; i < old_dirsz; i++, ldp++)
399 lwp_hash_in(p, ldp->ld_entry,
400 new_hash, new_hashsz, 0);
401
402 /*
403 * Remember the old hash table along with all
404 * of the previously-remembered hash tables.
405 * We will free them at process exit or exec.
406 */
407 ret_tidhash->rth_tidhash = old_hash;
408 ret_tidhash->rth_tidhash_sz = old_hashsz;
409 ret_tidhash->rth_next = p->p_ret_tidhash;
410 p->p_ret_tidhash = ret_tidhash;
411
412 /*
413 * Now establish the new tid hash table.
414 * As soon as we assign p->p_tidhash,
415 * code in lwp_unpark() can start using it.
416 */
417 membar_producer();
418 p->p_tidhash = new_hash;
419
420 /*
421 * It is necessary that p_tidhash reach global
422 * visibility before p_tidhash_sz. Otherwise,
423 * code in lwp_hash_lookup_and_lock() could
424 * index into the old p_tidhash using the new
425 * p_tidhash_sz and thereby access invalid data.
426 */
427 membar_producer();
428 p->p_tidhash_sz = new_hashsz;
429
430 /*
431 * Release the locks; allow lwp_unpark() to carry on.
432 */
433 for (i = 0; i < old_hashsz; i++) {
434 mutex_exit(&old_hash[i].th_lock);
435 mutex_exit(&new_hash[i].th_lock);
436 }
437
438 /*
439 * Avoid freeing these objects below.
440 */
441 ret_tidhash = NULL;
442 old_hash = NULL;
443 old_hashsz = 0;
444 }
445 }
446
447 /*
448 * Block the process against /proc while we manipulate p->p_tlist,
449 * unless lwp_create() was called by /proc for the PCAGENT operation.
450 * We want to do this early enough so that we don't drop p->p_lock
451 * until the thread is put on the p->p_tlist.
452 */
453 if (p == curproc) {
454 prbarrier(p);
455 /*
456 * If the current lwp has been requested to stop, do so now.
457 * Otherwise we have a race condition between /proc attempting
458 * to stop the process and this thread creating a new lwp
459 * that was not seen when the /proc PCSTOP request was issued.
460 * We rely on stop() to call prbarrier(p) before returning.
461 */
462 while ((curthread->t_proc_flag & TP_PRSTOP) &&
463 !ttolwp(curthread)->lwp_nostop) {
464 /*
465 * We called pool_barrier_enter() before calling
466 * here to lwp_create(). We have to call
467 * pool_barrier_exit() before stopping.
468 */
469 pool_barrier_exit();
470 prbarrier(p);
471 stop(PR_REQUESTED, 0);
472 /*
473 * And we have to repeat the call to
474 * pool_barrier_enter after stopping.
475 */
476 pool_barrier_enter();
477 prbarrier(p);
478 }
479
480 /*
481 * If process is exiting, there could be a race between
482 * the agent lwp creation and the new lwp currently being
483 * created. So to prevent this race lwp creation is failed
484 * if the process is exiting.
485 */
486 if (p->p_flag & (SEXITLWPS|SKILLED)) {
487 err = 1;
488 goto error;
489 }
490
491 /*
492 * Since we might have dropped p->p_lock, the
493 * lwp directory free list might have changed.
494 */
495 if (p->p_lwpfree == NULL)
496 goto grow;
497 }
498
499 kpreempt_disable(); /* can't grab cpu_lock here */
500
501 /*
502 * Inherit processor and processor set bindings from curthread.
503 *
504 * For kernel LWPs, we do not inherit processor set bindings at
505 * process creation time (i.e. when p != curproc). After the
506 * kernel process is created, any subsequent LWPs must be created
507 * by threads in the kernel process, at which point we *will*
508 * inherit processor set bindings.
509 */
510 if (CLASS_KERNEL(cid) && p != curproc) {
511 t->t_bind_cpu = binding = PBIND_NONE;
512 t->t_cpupart = oldpart = &cp_default;
513 t->t_bind_pset = PS_NONE;
514 t->t_bindflag = (uchar_t)default_binding_mode;
515 } else {
516 binding = curthread->t_bind_cpu;
517 t->t_bind_cpu = binding;
518 oldpart = t->t_cpupart;
519 t->t_cpupart = curthread->t_cpupart;
520 t->t_bind_pset = curthread->t_bind_pset;
521 t->t_bindflag = curthread->t_bindflag |
522 (uchar_t)default_binding_mode;
523 }
524
525 /*
526 * thread_create() initializes this thread's home lgroup to the root.
527 * Choose a more suitable lgroup, since this thread is associated
528 * with an lwp.
529 */
530 ASSERT(oldpart != NULL);
531 if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
532 t->t_bound_cpu = cpu[binding];
533 if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
534 lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
535 } else if (CLASS_KERNEL(cid)) {
536 /*
537 * Kernel threads are always in the root lgrp.
538 */
539 lgrp_move_thread(t,
540 &t->t_cpupart->cp_lgrploads[LGRP_ROOTID], 1);
541 } else {
542 lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
543 }
544
545 kpreempt_enable();
546
547 /*
548 * make sure lpl points to our own partition
549 */
550 ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
551 ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
552 t->t_cpupart->cp_nlgrploads);
553
554 /*
555 * It is safe to point the thread to the new project without holding it
556 * since we're holding the target process' p_lock here and therefore
557 * we're guaranteed that it will not move to another project.
558 */
559 newkpj = p->p_task->tk_proj;
560 oldkpj = ttoproj(t);
561 if (newkpj != oldkpj) {
562 t->t_proj = newkpj;
563 (void) project_hold(newkpj);
564 project_rele(oldkpj);
565 }
566
567 if (cid != NOCLASS) {
568 /*
569 * If the lwp is being created in the current process
570 * and matches the current thread's scheduling class,
571 * we should propagate the current thread's scheduling
572 * parameters by calling CL_FORK. Otherwise just use
573 * the defaults by calling CL_ENTERCLASS.
574 */
575 if (p != curproc || curthread->t_cid != cid) {
576 err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
577 t->t_pri = pri; /* CL_ENTERCLASS may have changed it */
578 /*
579 * We don't call schedctl_set_cidpri(t) here
580 * because the schedctl data is not yet set
581 * up for the newly-created lwp.
582 */
583 } else {
584 t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
585 err = CL_FORK(curthread, t, bufp);
586 t->t_cid = cid;
587 }
588 if (err) {
589 atomic_inc_32(&p->p_zone->zone_ffmisc);
590 goto error;
591 } else {
592 bufp = NULL;
593 }
594 }
595
596 /*
597 * If we were given an lwpid then use it, else allocate one.
598 */
599 if (lwpid != 0)
600 t->t_tid = lwpid;
601 else {
602 /*
603 * lwp/thread id 0 is never valid; reserved for special checks.
604 * lwp/thread id 1 is reserved for the main thread.
605 * Start again at 2 when INT_MAX has been reached
606 * (id_t is a signed 32-bit integer).
607 */
608 id_t prev_id = p->p_lwpid; /* last allocated tid */
609
610 do { /* avoid lwpid duplication */
611 if (p->p_lwpid == INT_MAX) {
612 p->p_flag |= SLWPWRAP;
613 p->p_lwpid = 1;
614 }
615 if ((t->t_tid = ++p->p_lwpid) == prev_id) {
616 /*
617 * All lwpids are allocated; fail the request.
618 */
619 err = 1;
620 atomic_inc_32(&p->p_zone->zone_ffnoproc);
621 goto error;
622 }
623 /*
624 * We only need to worry about colliding with an id
625 * that's already in use if this process has
626 * cycled through all available lwp ids.
627 */
628 if ((p->p_flag & SLWPWRAP) == 0)
629 break;
630 } while (lwp_hash_lookup(p, t->t_tid) != NULL);
631 }
632
633 /*
634 * If this is a branded process, let the brand do any necessary lwp
635 * initialization.
636 */
637 if (PROC_IS_BRANDED(p)) {
638 if (BROP(p)->b_initlwp(lwp)) {
639 err = 1;
640 atomic_inc_32(&p->p_zone->zone_ffmisc);
641 goto error;
642 }
643 branded = 1;
644 }
645
646 if (t->t_tid == 1) {
647 kpreempt_disable();
648 ASSERT(t->t_lpl != NULL);
649 p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
650 kpreempt_enable();
651 if (p->p_tr_lgrpid != LGRP_NONE &&
652 p->p_tr_lgrpid != p->p_t1_lgrpid) {
653 lgrp_update_trthr_migrations(1);
654 }
655 }
656
657 p->p_lwpcnt++;
658 t->t_waitfor = -1;
659
660 /*
661 * Turn microstate accounting on for thread if on for process.
662 */
663 if (p->p_flag & SMSACCT)
664 t->t_proc_flag |= TP_MSACCT;
665
666 /*
667 * If the process has watchpoints, mark the new thread as such.
668 */
669 if (pr_watch_active(p))
670 watch_enable(t);
671
672 /*
673 * The lwp is being created in the stopped state.
674 * We set all the necessary flags to indicate that fact here.
675 * We omit the TS_CREATE flag from t_schedflag so that the lwp
676 * cannot be set running until the caller is finished with it,
677 * even if lwp_continue() is called on it after we drop p->p_lock.
678 * When the caller is finished with the newly-created lwp,
679 * the caller must call lwp_create_done() to allow the lwp
680 * to be set running. If the TP_HOLDLWP is left set, the
681 * lwp will suspend itself after reaching system call exit.
682 */
683 init_mstate(t, LMS_STOPPED);
684 t->t_proc_flag |= TP_HOLDLWP;
685 t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
686 t->t_whystop = PR_SUSPENDED;
687 t->t_whatstop = SUSPEND_NORMAL;
688 t->t_sig_check = 1; /* ensure that TP_HOLDLWP is honored */
689
690 /*
691 * Set system call processing flags in case tracing or profiling
692 * is set. The first system call will evaluate these and turn
693 * them off if they aren't needed.
694 */
695 t->t_pre_sys = 1;
696 t->t_post_sys = 1;
697
698 /*
699 * Insert the new thread into the list of all threads.
700 */
701 if ((tx = p->p_tlist) == NULL) {
702 t->t_back = t;
703 t->t_forw = t;
704 p->p_tlist = t;
705 } else {
706 t->t_forw = tx;
707 t->t_back = tx->t_back;
708 tx->t_back->t_forw = t;
709 tx->t_back = t;
710 }
711
712 /*
713 * Insert the new lwp into an lwp directory slot position
714 * and into the lwpid hash table.
715 */
716 lep->le_thread = t;
717 lep->le_lwpid = t->t_tid;
718 lep->le_start = t->t_start;
719 lwp_hash_in(p, lep, p->p_tidhash, p->p_tidhash_sz, 1);
720
721 if (state == TS_RUN) {
722 /*
723 * We set the new lwp running immediately.
724 */
725 t->t_proc_flag &= ~TP_HOLDLWP;
726 lwp_create_done(t);
727 }
728
729 error:
730 if (err) {
731 if (CLASS_KERNEL(cid)) {
732 /*
733 * This should only happen if a system process runs
734 * out of lwpids, which shouldn't occur.
735 */
736 panic("Failed to create a system LWP");
737 }
738 /*
739 * We have failed to create an lwp, so decrement the number
740 * of lwps in the task and let the lgroup load averages know
741 * that this thread isn't going to show up.
742 */
743 kpreempt_disable();
744 lgrp_move_thread(t, NULL, 1);
745 kpreempt_enable();
746
747 ASSERT(MUTEX_HELD(&p->p_lock));
748 mutex_enter(&p->p_zone->zone_nlwps_lock);
749 p->p_task->tk_nlwps--;
750 p->p_task->tk_proj->kpj_nlwps--;
751 p->p_zone->zone_nlwps--;
752 mutex_exit(&p->p_zone->zone_nlwps_lock);
753 if (cid != NOCLASS && bufp != NULL)
754 CL_FREE(cid, bufp);
755
756 if (branded)
757 BROP(p)->b_freelwp(lwp);
758
759 mutex_exit(&p->p_lock);
760 t->t_state = TS_FREE;
761 thread_rele(t);
762
763 /*
764 * We need to remove t from the list of all threads
765 * because thread_exit()/lwp_exit() isn't called on t.
766 */
767 mutex_enter(&pidlock);
768 ASSERT(t != t->t_next); /* t0 never exits */
769 t->t_next->t_prev = t->t_prev;
770 t->t_prev->t_next = t->t_next;
771 mutex_exit(&pidlock);
772
773 thread_free(t);
774 kmem_free(lep, sizeof (*lep));
775 lwp = NULL;
776 } else {
777 mutex_exit(&p->p_lock);
778 }
779
780 if (old_dir != NULL)
781 kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
782 if (old_hash != NULL)
783 kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
784 if (ret_tidhash != NULL)
785 kmem_free(ret_tidhash, sizeof (ret_tidhash_t));
786
787 DTRACE_PROC1(lwp__create, kthread_t *, t);
788 return (lwp);
789 }
790
791 /*
792 * lwp_create_done() is called by the caller of lwp_create() to set the
793 * newly-created lwp running after the caller has finished manipulating it.
794 */
795 void
796 lwp_create_done(kthread_t *t)
797 {
798 proc_t *p = ttoproc(t);
799
800 ASSERT(MUTEX_HELD(&p->p_lock));
801
802 /*
803 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
804 * (The absence of the TS_CREATE flag prevents the lwp from running
805 * until we are finished with it, even if lwp_continue() is called on
806 * it by some other lwp in the process or elsewhere in the kernel.)
807 */
808 thread_lock(t);
809 ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
810 /*
811 * If TS_CSTART is set, lwp_continue(t) has been called and
812 * has already incremented p_lwprcnt; avoid doing this twice.
813 */
814 if (!(t->t_schedflag & TS_CSTART))
815 p->p_lwprcnt++;
816 t->t_schedflag |= (TS_CSTART | TS_CREATE);
817 setrun_locked(t);
818 thread_unlock(t);
819 }
820
821 /*
822 * Copy an LWP's active templates, and clear the latest contracts.
823 */
824 void
825 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
826 {
827 int i;
828
829 for (i = 0; i < ct_ntypes; i++) {
830 dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
831 dst->lwp_ct_latest[i] = NULL;
832 }
833 }
834
835 /*
836 * Clear an LWP's contract template state.
837 */
838 void
839 lwp_ctmpl_clear(klwp_t *lwp)
840 {
841 ct_template_t *tmpl;
842 int i;
843
844 for (i = 0; i < ct_ntypes; i++) {
845 if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
846 ctmpl_free(tmpl);
847 lwp->lwp_ct_active[i] = NULL;
848 }
849
850 if (lwp->lwp_ct_latest[i] != NULL) {
851 contract_rele(lwp->lwp_ct_latest[i]);
852 lwp->lwp_ct_latest[i] = NULL;
853 }
854 }
855 }
856
857 /*
858 * Individual lwp exit.
859 * If this is the last lwp, exit the whole process.
860 */
861 void
862 lwp_exit(void)
863 {
864 kthread_t *t = curthread;
865 klwp_t *lwp = ttolwp(t);
866 proc_t *p = ttoproc(t);
867
868 ASSERT(MUTEX_HELD(&p->p_lock));
869
870 mutex_exit(&p->p_lock);
871
872 #if defined(__sparc)
873 /*
874 * Ensure that the user stack is fully abandoned..
875 */
876 trash_user_windows();
877 #endif
878
879 tsd_exit(); /* free thread specific data */
880
881 kcpc_passivate(); /* Clean up performance counter state */
882
883 pollcleanup();
884
885 if (t->t_door)
886 door_slam();
887
888 if (t->t_schedctl != NULL)
889 schedctl_lwp_cleanup(t);
890
891 if (t->t_upimutex != NULL)
892 upimutex_cleanup();
893
894 /*
895 * Perform any brand specific exit processing, then release any
896 * brand data associated with the lwp
897 */
898 if (PROC_IS_BRANDED(p))
899 BROP(p)->b_lwpexit(lwp);
900
901 lwp_pcb_exit();
902
903 mutex_enter(&p->p_lock);
904 lwp_cleanup();
905
906 /*
907 * When this process is dumping core, its lwps are held here
908 * until the core dump is finished. Then exitlwps() is called
909 * again to release these lwps so that they can finish exiting.
910 */
911 if (p->p_flag & SCOREDUMP)
912 stop(PR_SUSPENDED, SUSPEND_NORMAL);
913
914 /*
915 * Block the process against /proc now that we have really acquired
916 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
917 */
918 prbarrier(p);
919
920 /*
921 * Call proc_exit() if this is the last non-daemon lwp in the process.
922 */
923 if (!(t->t_proc_flag & TP_DAEMON) &&
924 p->p_lwpcnt == p->p_lwpdaemon + 1) {
925 mutex_exit(&p->p_lock);
926 if (proc_exit(CLD_EXITED, 0) == 0) {
927 /* Restarting init. */
928 return;
929 }
930
931 /*
932 * proc_exit() returns a non-zero value when some other
933 * lwp got there first. We just have to continue in
934 * lwp_exit().
935 */
936 mutex_enter(&p->p_lock);
937 ASSERT(curproc->p_flag & SEXITLWPS);
938 prbarrier(p);
939 }
940
941 DTRACE_PROC(lwp__exit);
942
943 /*
944 * If the lwp is a detached lwp or if the process is exiting,
945 * remove (lwp_hash_out()) the lwp from the lwp directory.
946 * Otherwise null out the lwp's le_thread pointer in the lwp
947 * directory so that other threads will see it as a zombie lwp.
948 */
949 prlwpexit(t); /* notify /proc */
950 if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
951 lwp_hash_out(p, t->t_tid);
952 else {
953 ASSERT(!(t->t_proc_flag & TP_DAEMON));
954 p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
955 p->p_zombcnt++;
956 cv_broadcast(&p->p_lwpexit);
957 }
958 if (t->t_proc_flag & TP_DAEMON) {
959 p->p_lwpdaemon--;
960 t->t_proc_flag &= ~TP_DAEMON;
961 }
962 t->t_proc_flag &= ~TP_TWAIT;
963
964 /*
965 * Maintain accurate lwp count for task.max-lwps resource control.
966 */
967 mutex_enter(&p->p_zone->zone_nlwps_lock);
968 p->p_task->tk_nlwps--;
969 p->p_task->tk_proj->kpj_nlwps--;
970 p->p_zone->zone_nlwps--;
971 mutex_exit(&p->p_zone->zone_nlwps_lock);
972
973 CL_EXIT(t); /* tell the scheduler that t is exiting */
974 ASSERT(p->p_lwpcnt != 0);
975 p->p_lwpcnt--;
976
977 /*
978 * If all remaining non-daemon lwps are waiting in lwp_wait(),
979 * wake them up so someone can return EDEADLK.
980 * (See the block comment preceeding lwp_wait().)
981 */
982 if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
983 cv_broadcast(&p->p_lwpexit);
984
985 t->t_proc_flag |= TP_LWPEXIT;
986 term_mstate(t);
987
988 #ifndef NPROBE
989 /* Kernel probe */
990 if (t->t_tnf_tpdp)
991 tnf_thread_exit();
992 #endif /* NPROBE */
993
994 t->t_forw->t_back = t->t_back;
995 t->t_back->t_forw = t->t_forw;
996 if (t == p->p_tlist)
997 p->p_tlist = t->t_forw;
998
999 /*
1000 * Clean up the signal state.
1001 */
1002 if (t->t_sigqueue != NULL)
1003 sigdelq(p, t, 0);
1004 if (lwp->lwp_curinfo != NULL) {
1005 siginfofree(lwp->lwp_curinfo);
1006 lwp->lwp_curinfo = NULL;
1007 }
1008
1009 /*
1010 * If we have spymaster information (that is, if we're an agent LWP),
1011 * free that now.
1012 */
1013 if (lwp->lwp_spymaster != NULL) {
1014 kmem_free(lwp->lwp_spymaster, sizeof (psinfo_t));
1015 lwp->lwp_spymaster = NULL;
1016 }
1017
1018 thread_rele(t);
1019
1020 /*
1021 * Terminated lwps are associated with process zero and are put onto
1022 * death-row by resume(). Avoid preemption after resetting t->t_procp.
1023 */
1024 t->t_preempt++;
1025
1026 if (t->t_ctx != NULL)
1027 exitctx(t);
1028 if (p->p_pctx != NULL)
1029 exitpctx(p);
1030
1031 t->t_procp = &p0;
1032
1033 /*
1034 * Notify the HAT about the change of address space
1035 */
1036 hat_thread_exit(t);
1037 /*
1038 * When this is the last running lwp in this process and some lwp is
1039 * waiting for this condition to become true, or this thread was being
1040 * suspended, then the waiting lwp is awakened.
1041 *
1042 * Also, if the process is exiting, we may have a thread waiting in
1043 * exitlwps() that needs to be notified.
1044 */
1045 if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
1046 (p->p_flag & SEXITLWPS))
1047 cv_broadcast(&p->p_holdlwps);
1048
1049 /*
1050 * Need to drop p_lock so we can reacquire pidlock.
1051 */
1052 mutex_exit(&p->p_lock);
1053 mutex_enter(&pidlock);
1054
1055 ASSERT(t != t->t_next); /* t0 never exits */
1056 t->t_next->t_prev = t->t_prev;
1057 t->t_prev->t_next = t->t_next;
1058 cv_broadcast(&t->t_joincv); /* wake up anyone in thread_join */
1059 mutex_exit(&pidlock);
1060
1061 t->t_state = TS_ZOMB;
1062 swtch_from_zombie();
1063 /* never returns */
1064 }
1065
1066
1067 /*
1068 * Cleanup function for an exiting lwp.
1069 * Called both from lwp_exit() and from proc_exit().
1070 * p->p_lock is repeatedly released and grabbed in this function.
1071 */
1072 void
1073 lwp_cleanup(void)
1074 {
1075 kthread_t *t = curthread;
1076 proc_t *p = ttoproc(t);
1077
1078 ASSERT(MUTEX_HELD(&p->p_lock));
1079
1080 /* untimeout any lwp-bound realtime timers */
1081 if (p->p_itimer != NULL)
1082 timer_lwpexit();
1083
1084 /*
1085 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
1086 * so it appears that the agent never existed, and clear p_agenttp.
1087 */
1088 if (t == p->p_agenttp) {
1089 ASSERT(t->t_tid == p->p_lwpid);
1090 p->p_lwpid--;
1091 p->p_agenttp = NULL;
1092 }
1093
1094 /*
1095 * Do lgroup bookkeeping to account for thread exiting.
1096 */
1097 kpreempt_disable();
1098 lgrp_move_thread(t, NULL, 1);
1099 if (t->t_tid == 1) {
1100 p->p_t1_lgrpid = LGRP_NONE;
1101 }
1102 kpreempt_enable();
1103
1104 lwp_ctmpl_clear(ttolwp(t));
1105 }
1106
1107 int
1108 lwp_suspend(kthread_t *t)
1109 {
1110 int tid;
1111 proc_t *p = ttoproc(t);
1112
1113 ASSERT(MUTEX_HELD(&p->p_lock));
1114
1115 /*
1116 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
1117 * If an lwp is stopping itself, there is no need to wait.
1118 */
1119 top:
1120 t->t_proc_flag |= TP_HOLDLWP;
1121 if (t == curthread) {
1122 t->t_sig_check = 1;
1123 } else {
1124 /*
1125 * Make sure the lwp stops promptly.
1126 */
1127 thread_lock(t);
1128 t->t_sig_check = 1;
1129 /*
1130 * XXX Should use virtual stop like /proc does instead of
1131 * XXX waking the thread to get it to stop.
1132 */
1133 if (ISWAKEABLE(t) || ISWAITING(t)) {
1134 setrun_locked(t);
1135 } else if (t->t_state == TS_ONPROC && t->t_cpu != CPU) {
1136 poke_cpu(t->t_cpu->cpu_id);
1137 }
1138
1139 tid = t->t_tid; /* remember thread ID */
1140 /*
1141 * Wait for lwp to stop
1142 */
1143 while (!SUSPENDED(t)) {
1144 /*
1145 * Drop the thread lock before waiting and reacquire it
1146 * afterwards, so the thread can change its t_state
1147 * field.
1148 */
1149 thread_unlock(t);
1150
1151 /*
1152 * Check if aborted by exitlwps().
1153 */
1154 if (p->p_flag & SEXITLWPS)
1155 lwp_exit();
1156
1157 /*
1158 * Cooperate with jobcontrol signals and /proc stopping
1159 * by calling cv_wait_sig() to wait for the target
1160 * lwp to stop. Just using cv_wait() can lead to
1161 * deadlock because, if some other lwp has stopped
1162 * by either of these mechanisms, then p_lwprcnt will
1163 * never become zero if we do a cv_wait().
1164 */
1165 if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
1166 return (EINTR);
1167
1168 /*
1169 * Check to see if thread died while we were
1170 * waiting for it to suspend.
1171 */
1172 if (idtot(p, tid) == NULL)
1173 return (ESRCH);
1174
1175 thread_lock(t);
1176 /*
1177 * If the TP_HOLDLWP flag went away, lwp_continue()
1178 * or vfork() must have been called while we were
1179 * waiting, so start over again.
1180 */
1181 if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
1182 thread_unlock(t);
1183 goto top;
1184 }
1185 }
1186 thread_unlock(t);
1187 }
1188 return (0);
1189 }
1190
1191 /*
1192 * continue a lwp that's been stopped by lwp_suspend().
1193 */
1194 void
1195 lwp_continue(kthread_t *t)
1196 {
1197 proc_t *p = ttoproc(t);
1198 int was_suspended = t->t_proc_flag & TP_HOLDLWP;
1199
1200 ASSERT(MUTEX_HELD(&p->p_lock));
1201
1202 t->t_proc_flag &= ~TP_HOLDLWP;
1203 thread_lock(t);
1204 if (SUSPENDED(t) &&
1205 !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
1206 p->p_lwprcnt++;
1207 t->t_schedflag |= TS_CSTART;
1208 setrun_locked(t);
1209 }
1210 thread_unlock(t);
1211 /*
1212 * Wakeup anyone waiting for this thread to be suspended
1213 */
1214 if (was_suspended)
1215 cv_broadcast(&p->p_holdlwps);
1216 }
1217
1218 /*
1219 * ********************************
1220 * Miscellaneous lwp routines *
1221 * ********************************
1222 */
1223 /*
1224 * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
1225 * This will cause the process's lwps to stop at a hold point. A hold
1226 * point is where a kernel thread has a flat stack. This is at the
1227 * return from a system call and at the return from a user level trap.
1228 *
1229 * When a process is undergoing a fork1() or vfork(), its p_flag is set to
1230 * SHOLDFORK1. This will cause the process's lwps to stop at a modified
1231 * hold point. The lwps in the process are not being cloned, so they
1232 * are held at the usual hold points and also within issig_forreal().
1233 * This has the side-effect that their system calls do not return
1234 * showing EINTR.
1235 *
1236 * An lwp can also be held. This is identified by the TP_HOLDLWP flag on
1237 * the thread. The TP_HOLDLWP flag is set in lwp_suspend(), where the active
1238 * lwp is waiting for the target lwp to be stopped.
1239 */
1240 void
1241 holdlwp(void)
1242 {
1243 proc_t *p = curproc;
1244 kthread_t *t = curthread;
1245
1246 mutex_enter(&p->p_lock);
1247 /*
1248 * Don't terminate immediately if the process is dumping core.
1249 * Once the process has dumped core, all lwps are terminated.
1250 */
1251 if (!(p->p_flag & SCOREDUMP)) {
1252 if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
1253 lwp_exit();
1254 }
1255 if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
1256 mutex_exit(&p->p_lock);
1257 return;
1258 }
1259 /*
1260 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
1261 * when p->p_lwprcnt becomes zero.
1262 */
1263 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1264 if (p->p_flag & SEXITLWPS)
1265 lwp_exit();
1266 mutex_exit(&p->p_lock);
1267 }
1268
1269 /*
1270 * Have all lwps within the process hold at a point where they are
1271 * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
1272 */
1273 int
1274 holdlwps(int holdflag)
1275 {
1276 proc_t *p = curproc;
1277
1278 ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
1279 mutex_enter(&p->p_lock);
1280 schedctl_finish_sigblock(curthread);
1281 again:
1282 while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1283 /*
1284 * If another lwp is doing a forkall() or proc_exit(), bail out.
1285 */
1286 if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
1287 mutex_exit(&p->p_lock);
1288 return (0);
1289 }
1290 /*
1291 * Another lwp is doing a fork1() or is undergoing
1292 * watchpoint activity. We hold here for it to complete.
1293 */
1294 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1295 }
1296 p->p_flag |= holdflag;
1297 pokelwps(p);
1298 --p->p_lwprcnt;
1299 /*
1300 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
1301 */
1302 while (p->p_lwprcnt > 0) {
1303 /*
1304 * Check if aborted by exitlwps().
1305 * Also check if SHOLDWATCH is set; it takes precedence.
1306 */
1307 if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
1308 p->p_lwprcnt++;
1309 p->p_flag &= ~holdflag;
1310 cv_broadcast(&p->p_holdlwps);
1311 goto again;
1312 }
1313 /*
1314 * Cooperate with jobcontrol signals and /proc stopping.
1315 * If some other lwp has stopped by either of these
1316 * mechanisms, then p_lwprcnt will never become zero
1317 * and the process will appear deadlocked unless we
1318 * stop here in sympathy with the other lwp before
1319 * doing the cv_wait() below.
1320 *
1321 * If the other lwp stops after we do the cv_wait(), it
1322 * will wake us up to loop around and do the sympathy stop.
1323 *
1324 * Since stop() drops p->p_lock, we must start from
1325 * the top again on returning from stop().
1326 */
1327 if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
1328 int whystop = p->p_stopsig? PR_JOBCONTROL :
1329 PR_REQUESTED;
1330 p->p_lwprcnt++;
1331 p->p_flag &= ~holdflag;
1332 stop(whystop, p->p_stopsig);
1333 goto again;
1334 }
1335 cv_wait(&p->p_holdlwps, &p->p_lock);
1336 }
1337 p->p_lwprcnt++;
1338 p->p_flag &= ~holdflag;
1339 mutex_exit(&p->p_lock);
1340 return (1);
1341 }
1342
1343 /*
1344 * See comments for holdwatch(), below.
1345 */
1346 static int
1347 holdcheck(int clearflags)
1348 {
1349 proc_t *p = curproc;
1350
1351 /*
1352 * If we are trying to exit, that takes precedence over anything else.
1353 */
1354 if (p->p_flag & SEXITLWPS) {
1355 p->p_lwprcnt++;
1356 p->p_flag &= ~clearflags;
1357 lwp_exit();
1358 }
1359
1360 /*
1361 * If another thread is calling fork1(), stop the current thread so the
1362 * other can complete.
1363 */
1364 if (p->p_flag & SHOLDFORK1) {
1365 p->p_lwprcnt++;
1366 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1367 if (p->p_flag & SEXITLWPS) {
1368 p->p_flag &= ~clearflags;
1369 lwp_exit();
1370 }
1371 return (-1);
1372 }
1373
1374 /*
1375 * If another thread is calling fork(), then indicate we are doing
1376 * watchpoint activity. This will cause holdlwps() above to stop the
1377 * forking thread, at which point we can continue with watchpoint
1378 * activity.
1379 */
1380 if (p->p_flag & SHOLDFORK) {
1381 p->p_lwprcnt++;
1382 while (p->p_flag & SHOLDFORK) {
1383 p->p_flag |= SHOLDWATCH;
1384 cv_broadcast(&p->p_holdlwps);
1385 cv_wait(&p->p_holdlwps, &p->p_lock);
1386 p->p_flag &= ~SHOLDWATCH;
1387 }
1388 return (-1);
1389 }
1390
1391 return (0);
1392 }
1393
1394 /*
1395 * Stop all lwps within the process, holding themselves in the kernel while the
1396 * active lwp undergoes watchpoint activity. This is more complicated than
1397 * expected because stop() relies on calling holdwatch() in order to copyin data
1398 * from the user's address space. A double barrier is used to prevent an
1399 * infinite loop.
1400 *
1401 * o The first thread into holdwatch() is the 'master' thread and does
1402 * the following:
1403 *
1404 * - Sets SHOLDWATCH on the current process
1405 * - Sets TP_WATCHSTOP on the current thread
1406 * - Waits for all threads to be either stopped or have
1407 * TP_WATCHSTOP set.
1408 * - Sets the SWATCHOK flag on the process
1409 * - Unsets TP_WATCHSTOP
1410 * - Waits for the other threads to completely stop
1411 * - Unsets SWATCHOK
1412 *
1413 * o If SHOLDWATCH is already set when we enter this function, then another
1414 * thread is already trying to stop this thread. This 'slave' thread
1415 * does the following:
1416 *
1417 * - Sets TP_WATCHSTOP on the current thread
1418 * - Waits for SWATCHOK flag to be set
1419 * - Calls stop()
1420 *
1421 * o If SWATCHOK is set on the process, then this function immediately
1422 * returns, as we must have been called via stop().
1423 *
1424 * In addition, there are other flags that take precedence over SHOLDWATCH:
1425 *
1426 * o If SEXITLWPS is set, exit immediately.
1427 *
1428 * o If SHOLDFORK1 is set, wait for fork1() to complete.
1429 *
1430 * o If SHOLDFORK is set, then watchpoint activity takes precedence In this
1431 * case, set SHOLDWATCH, signalling the forking thread to stop first.
1432 *
1433 * o If the process is being stopped via /proc (TP_PRSTOP is set), then we
1434 * stop the current thread.
1435 *
1436 * Returns 0 if all threads have been quiesced. Returns non-zero if not all
1437 * threads were stopped, or the list of watched pages has changed.
1438 */
1439 int
1440 holdwatch(void)
1441 {
1442 proc_t *p = curproc;
1443 kthread_t *t = curthread;
1444 int ret = 0;
1445
1446 mutex_enter(&p->p_lock);
1447
1448 p->p_lwprcnt--;
1449
1450 /*
1451 * Check for bail-out conditions as outlined above.
1452 */
1453 if (holdcheck(0) != 0) {
1454 mutex_exit(&p->p_lock);
1455 return (-1);
1456 }
1457
1458 if (!(p->p_flag & SHOLDWATCH)) {
1459 /*
1460 * We are the master watchpoint thread. Set SHOLDWATCH and poke
1461 * the other threads.
1462 */
1463 p->p_flag |= SHOLDWATCH;
1464 pokelwps(p);
1465
1466 /*
1467 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
1468 */
1469 while (pr_allstopped(p, 1) > 0) {
1470 if (holdcheck(SHOLDWATCH) != 0) {
1471 p->p_flag &= ~SHOLDWATCH;
1472 mutex_exit(&p->p_lock);
1473 return (-1);
1474 }
1475
1476 cv_wait(&p->p_holdlwps, &p->p_lock);
1477 }
1478
1479 /*
1480 * All threads are now stopped or in the process of stopping.
1481 * Set SWATCHOK and let them stop completely.
1482 */
1483 p->p_flag |= SWATCHOK;
1484 t->t_proc_flag &= ~TP_WATCHSTOP;
1485 cv_broadcast(&p->p_holdlwps);
1486
1487 while (pr_allstopped(p, 0) > 0) {
1488 /*
1489 * At first glance, it may appear that we don't need a
1490 * call to holdcheck() here. But if the process gets a
1491 * SIGKILL signal, one of our stopped threads may have
1492 * been awakened and is waiting in exitlwps(), which
1493 * takes precedence over watchpoints.
1494 */
1495 if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
1496 p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
1497 mutex_exit(&p->p_lock);
1498 return (-1);
1499 }
1500
1501 cv_wait(&p->p_holdlwps, &p->p_lock);
1502 }
1503
1504 /*
1505 * All threads are now completely stopped.
1506 */
1507 p->p_flag &= ~SWATCHOK;
1508 p->p_flag &= ~SHOLDWATCH;
1509 p->p_lwprcnt++;
1510
1511 } else if (!(p->p_flag & SWATCHOK)) {
1512
1513 /*
1514 * SHOLDWATCH is set, so another thread is trying to do
1515 * watchpoint activity. Indicate this thread is stopping, and
1516 * wait for the OK from the master thread.
1517 */
1518 t->t_proc_flag |= TP_WATCHSTOP;
1519 cv_broadcast(&p->p_holdlwps);
1520
1521 while (!(p->p_flag & SWATCHOK)) {
1522 if (holdcheck(0) != 0) {
1523 t->t_proc_flag &= ~TP_WATCHSTOP;
1524 mutex_exit(&p->p_lock);
1525 return (-1);
1526 }
1527
1528 cv_wait(&p->p_holdlwps, &p->p_lock);
1529 }
1530
1531 /*
1532 * Once the master thread has given the OK, this thread can
1533 * actually call stop().
1534 */
1535 t->t_proc_flag &= ~TP_WATCHSTOP;
1536 p->p_lwprcnt++;
1537
1538 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1539
1540 /*
1541 * It's not OK to do watchpoint activity, notify caller to
1542 * retry.
1543 */
1544 ret = -1;
1545
1546 } else {
1547
1548 /*
1549 * The only way we can hit the case where SHOLDWATCH is set and
1550 * SWATCHOK is set is if we are triggering this from within a
1551 * stop() call. Assert that this is the case.
1552 */
1553
1554 ASSERT(t->t_proc_flag & TP_STOPPING);
1555 p->p_lwprcnt++;
1556 }
1557
1558 mutex_exit(&p->p_lock);
1559
1560 return (ret);
1561 }
1562
1563 /*
1564 * force all interruptible lwps to trap into the kernel.
1565 */
1566 void
1567 pokelwps(proc_t *p)
1568 {
1569 kthread_t *t;
1570
1571 ASSERT(MUTEX_HELD(&p->p_lock));
1572
1573 t = p->p_tlist;
1574 do {
1575 if (t == curthread)
1576 continue;
1577 thread_lock(t);
1578 aston(t); /* make thread trap or do post_syscall */
1579 if (ISWAKEABLE(t) || ISWAITING(t)) {
1580 setrun_locked(t);
1581 } else if (t->t_state == TS_STOPPED) {
1582 /*
1583 * Ensure that proc_exit() is not blocked by lwps
1584 * that were stopped via jobcontrol or /proc.
1585 */
1586 if (p->p_flag & SEXITLWPS) {
1587 p->p_stopsig = 0;
1588 t->t_schedflag |= (TS_XSTART | TS_PSTART);
1589 setrun_locked(t);
1590 }
1591 /*
1592 * If we are holding lwps for a forkall(),
1593 * force lwps that have been suspended via
1594 * lwp_suspend() and are suspended inside
1595 * of a system call to proceed to their
1596 * holdlwp() points where they are clonable.
1597 */
1598 if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
1599 if ((t->t_schedflag & TS_CSTART) == 0) {
1600 p->p_lwprcnt++;
1601 t->t_schedflag |= TS_CSTART;
1602 setrun_locked(t);
1603 }
1604 }
1605 } else if (t->t_state == TS_ONPROC) {
1606 if (t->t_cpu != CPU)
1607 poke_cpu(t->t_cpu->cpu_id);
1608 }
1609 thread_unlock(t);
1610 } while ((t = t->t_forw) != p->p_tlist);
1611 }
1612
1613 /*
1614 * undo the effects of holdlwps() or holdwatch().
1615 */
1616 void
1617 continuelwps(proc_t *p)
1618 {
1619 kthread_t *t;
1620
1621 /*
1622 * If this flag is set, then the original holdwatch() didn't actually
1623 * stop the process. See comments for holdwatch().
1624 */
1625 if (p->p_flag & SWATCHOK) {
1626 ASSERT(curthread->t_proc_flag & TP_STOPPING);
1627 return;
1628 }
1629
1630 ASSERT(MUTEX_HELD(&p->p_lock));
1631 ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
1632
1633 t = p->p_tlist;
1634 do {
1635 thread_lock(t); /* SUSPENDED looks at t_schedflag */
1636 if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
1637 p->p_lwprcnt++;
1638 t->t_schedflag |= TS_CSTART;
1639 setrun_locked(t);
1640 }
1641 thread_unlock(t);
1642 } while ((t = t->t_forw) != p->p_tlist);
1643 }
1644
1645 /*
1646 * Force all other LWPs in the current process other than the caller to exit,
1647 * and then cv_wait() on p_holdlwps for them to exit. The exitlwps() function
1648 * is typically used in these situations:
1649 *
1650 * (a) prior to an exec() system call
1651 * (b) prior to dumping a core file
1652 * (c) prior to a uadmin() shutdown
1653 *
1654 * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
1655 * Multiple threads in the process can call this function at one time by
1656 * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
1657 * to declare one particular thread the winner who gets to kill the others.
1658 * If a thread wins the exitlwps() dance, zero is returned; otherwise an
1659 * appropriate errno value is returned to caller for its system call to return.
1660 */
1661 int
1662 exitlwps(int coredump)
1663 {
1664 proc_t *p = curproc;
1665 int heldcnt;
1666
1667 if (curthread->t_door)
1668 door_slam();
1669 if (p->p_door_list)
1670 door_revoke_all();
1671 if (curthread->t_schedctl != NULL)
1672 schedctl_lwp_cleanup(curthread);
1673
1674 /*
1675 * Ensure that before starting to wait for other lwps to exit,
1676 * cleanup all upimutexes held by curthread. Otherwise, some other
1677 * lwp could be waiting (uninterruptibly) for a upimutex held by
1678 * curthread, and the call to pokelwps() below would deadlock.
1679 * Even if a blocked upimutex_lock is made interruptible,
1680 * curthread's upimutexes need to be unlocked: do it here.
1681 */
1682 if (curthread->t_upimutex != NULL)
1683 upimutex_cleanup();
1684
1685 /*
1686 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
1687 * We must also block any further /proc access from this point forward.
1688 */
1689 mutex_enter(&p->p_lock);
1690 prbarrier(p);
1691
1692 if (p->p_flag & SEXITLWPS) {
1693 mutex_exit(&p->p_lock);
1694 aston(curthread); /* force a trip through post_syscall */
1695 return (set_errno(EINTR));
1696 }
1697
1698 p->p_flag |= SEXITLWPS;
1699 if (coredump) /* tell other lwps to stop, not exit */
1700 p->p_flag |= SCOREDUMP;
1701
1702 /*
1703 * Give precedence to exitlwps() if a holdlwps() is
1704 * in progress. The lwp doing the holdlwps() operation
1705 * is aborted when it is awakened.
1706 */
1707 while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1708 cv_broadcast(&p->p_holdlwps);
1709 cv_wait(&p->p_holdlwps, &p->p_lock);
1710 prbarrier(p);
1711 }
1712 p->p_flag |= SHOLDFORK;
1713 pokelwps(p);
1714
1715 /*
1716 * Wait for process to become quiescent.
1717 */
1718 --p->p_lwprcnt;
1719 while (p->p_lwprcnt > 0) {
1720 cv_wait(&p->p_holdlwps, &p->p_lock);
1721 prbarrier(p);
1722 }
1723 p->p_lwprcnt++;
1724 ASSERT(p->p_lwprcnt == 1);
1725
1726 /*
1727 * The SCOREDUMP flag puts the process into a quiescent
1728 * state. The process's lwps remain attached to this
1729 * process until exitlwps() is called again without the
1730 * 'coredump' flag set, then the lwps are terminated
1731 * and the process can exit.
1732 */
1733 if (coredump) {
1734 p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
1735 goto out;
1736 }
1737
1738 /*
1739 * Determine if there are any lwps left dangling in
1740 * the stopped state. This happens when exitlwps()
1741 * aborts a holdlwps() operation.
1742 */
1743 p->p_flag &= ~SHOLDFORK;
1744 if ((heldcnt = p->p_lwpcnt) > 1) {
1745 kthread_t *t;
1746 for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
1747 t->t_proc_flag &= ~TP_TWAIT;
1748 lwp_continue(t);
1749 }
1750 }
1751
1752 /*
1753 * Wait for all other lwps to exit.
1754 */
1755 --p->p_lwprcnt;
1756 while (p->p_lwpcnt > 1) {
1757 cv_wait(&p->p_holdlwps, &p->p_lock);
1758 prbarrier(p);
1759 }
1760 ++p->p_lwprcnt;
1761 ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
1762
1763 p->p_flag &= ~SEXITLWPS;
1764 curthread->t_proc_flag &= ~TP_TWAIT;
1765
1766 out:
1767 if (!coredump && p->p_zombcnt) { /* cleanup the zombie lwps */
1768 lwpdir_t *ldp;
1769 lwpent_t *lep;
1770 int i;
1771
1772 for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
1773 lep = ldp->ld_entry;
1774 if (lep != NULL && lep->le_thread != curthread) {
1775 ASSERT(lep->le_thread == NULL);
1776 p->p_zombcnt--;
1777 lwp_hash_out(p, lep->le_lwpid);
1778 }
1779 }
1780 ASSERT(p->p_zombcnt == 0);
1781 }
1782
1783 /*
1784 * If some other LWP in the process wanted us to suspend ourself,
1785 * then we will not do it. The other LWP is now terminated and
1786 * no one will ever continue us again if we suspend ourself.
1787 */
1788 curthread->t_proc_flag &= ~TP_HOLDLWP;
1789 p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
1790 mutex_exit(&p->p_lock);
1791 return (0);
1792 }
1793
1794 /*
1795 * duplicate a lwp.
1796 */
1797 klwp_t *
1798 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
1799 {
1800 klwp_t *clwp;
1801 void *tregs, *tfpu;
1802 kthread_t *t = lwptot(lwp);
1803 kthread_t *ct;
1804 proc_t *p = lwptoproc(lwp);
1805 int cid;
1806 void *bufp;
1807 void *brand_data;
1808 int val;
1809
1810 ASSERT(p == curproc);
1811 ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
1812
1813 #if defined(__sparc)
1814 if (t == curthread)
1815 (void) flush_user_windows_to_stack(NULL);
1816 #endif
1817
1818 if (t == curthread)
1819 /* copy args out of registers first */
1820 (void) save_syscall_args();
1821
1822 clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
1823 NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
1824 if (clwp == NULL)
1825 return (NULL);
1826
1827 /*
1828 * most of the parent's lwp can be copied to its duplicate,
1829 * except for the fields that are unique to each lwp, like
1830 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
1831 */
1832 ct = clwp->lwp_thread;
1833 tregs = clwp->lwp_regs;
1834 tfpu = clwp->lwp_fpu;
1835 brand_data = clwp->lwp_brand;
1836
1837 /*
1838 * Copy parent lwp to child lwp. Hold child's p_lock to prevent
1839 * mstate_aggr_state() from reading stale mstate entries copied
1840 * from lwp to clwp.
1841 */
1842 mutex_enter(&cp->p_lock);
1843 *clwp = *lwp;
1844
1845 /* clear microstate and resource usage data in new lwp */
1846 init_mstate(ct, LMS_STOPPED);
1847 bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
1848 mutex_exit(&cp->p_lock);
1849
1850 /* fix up child's lwp */
1851
1852 clwp->lwp_pcb.pcb_flags = 0;
1853 #if defined(__sparc)
1854 clwp->lwp_pcb.pcb_step = STEP_NONE;
1855 #endif
1856 clwp->lwp_cursig = 0;
1857 clwp->lwp_extsig = 0;
1858 clwp->lwp_curinfo = (struct sigqueue *)0;
1859 clwp->lwp_thread = ct;
1860 ct->t_sysnum = t->t_sysnum;
1861 clwp->lwp_regs = tregs;
1862 clwp->lwp_fpu = tfpu;
1863 clwp->lwp_brand = brand_data;
1864 clwp->lwp_ap = clwp->lwp_arg;
1865 clwp->lwp_procp = cp;
1866 bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
1867 clwp->lwp_lastfault = 0;
1868 clwp->lwp_lastfaddr = 0;
1869
1870 /* copy parent's struct regs to child. */
1871 lwp_forkregs(lwp, clwp);
1872
1873 /*
1874 * Fork thread context ops, if any.
1875 */
1876 if (t->t_ctx)
1877 forkctx(t, ct);
1878
1879 /* fix door state in the child */
1880 if (t->t_door)
1881 door_fork(t, ct);
1882
1883 /* copy current contract templates, clear latest contracts */
1884 lwp_ctmpl_copy(clwp, lwp);
1885
1886 mutex_enter(&cp->p_lock);
1887 /* lwp_create() set the TP_HOLDLWP flag */
1888 if (!(t->t_proc_flag & TP_HOLDLWP))
1889 ct->t_proc_flag &= ~TP_HOLDLWP;
1890 if (cp->p_flag & SMSACCT)
1891 ct->t_proc_flag |= TP_MSACCT;
1892 mutex_exit(&cp->p_lock);
1893
1894 /* Allow brand to propagate brand-specific state */
1895 if (PROC_IS_BRANDED(p))
1896 BROP(p)->b_forklwp(lwp, clwp);
1897
1898 retry:
1899 cid = t->t_cid;
1900
1901 val = CL_ALLOC(&bufp, cid, KM_SLEEP);
1902 ASSERT(val == 0);
1903
1904 mutex_enter(&p->p_lock);
1905 if (cid != t->t_cid) {
1906 /*
1907 * Someone just changed this thread's scheduling class,
1908 * so try pre-allocating the buffer again. Hopefully we
1909 * don't hit this often.
1910 */
1911 mutex_exit(&p->p_lock);
1912 CL_FREE(cid, bufp);
1913 goto retry;
1914 }
1915
1916 ct->t_unpark = t->t_unpark;
1917 ct->t_clfuncs = t->t_clfuncs;
1918 CL_FORK(t, ct, bufp);
1919 ct->t_cid = t->t_cid; /* after data allocated so prgetpsinfo works */
1920 mutex_exit(&p->p_lock);
1921
1922 return (clwp);
1923 }
1924
1925 /*
1926 * Add a new lwp entry to the lwp directory and to the lwpid hash table.
1927 */
1928 void
1929 lwp_hash_in(proc_t *p, lwpent_t *lep, tidhash_t *tidhash, uint_t tidhash_sz,
1930 int do_lock)
1931 {
1932 tidhash_t *thp = &tidhash[TIDHASH(lep->le_lwpid, tidhash_sz)];
1933 lwpdir_t **ldpp;
1934 lwpdir_t *ldp;
1935 kthread_t *t;
1936
1937 /*
1938 * Allocate a directory element from the free list.
1939 * Code elsewhere guarantees a free slot.
1940 */
1941 ldp = p->p_lwpfree;
1942 p->p_lwpfree = ldp->ld_next;
1943 ASSERT(ldp->ld_entry == NULL);
1944 ldp->ld_entry = lep;
1945
1946 if (do_lock)
1947 mutex_enter(&thp->th_lock);
1948
1949 /*
1950 * Insert it into the lwpid hash table.
1951 */
1952 ldpp = &thp->th_list;
1953 ldp->ld_next = *ldpp;
1954 *ldpp = ldp;
1955
1956 /*
1957 * Set the active thread's directory slot entry.
1958 */
1959 if ((t = lep->le_thread) != NULL) {
1960 ASSERT(lep->le_lwpid == t->t_tid);
1961 t->t_dslot = (int)(ldp - p->p_lwpdir);
1962 }
1963
1964 if (do_lock)
1965 mutex_exit(&thp->th_lock);
1966 }
1967
1968 /*
1969 * Remove an lwp from the lwpid hash table and free its directory entry.
1970 * This is done when a detached lwp exits in lwp_exit() or
1971 * when a non-detached lwp is waited for in lwp_wait() or
1972 * when a zombie lwp is detached in lwp_detach().
1973 */
1974 void
1975 lwp_hash_out(proc_t *p, id_t lwpid)
1976 {
1977 tidhash_t *thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
1978 lwpdir_t **ldpp;
1979 lwpdir_t *ldp;
1980 lwpent_t *lep;
1981
1982 mutex_enter(&thp->th_lock);
1983 for (ldpp = &thp->th_list;
1984 (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
1985 lep = ldp->ld_entry;
1986 if (lep->le_lwpid == lwpid) {
1987 prlwpfree(p, lep); /* /proc deals with le_trace */
1988 *ldpp = ldp->ld_next;
1989 ldp->ld_entry = NULL;
1990 ldp->ld_next = p->p_lwpfree;
1991 p->p_lwpfree = ldp;
1992 kmem_free(lep, sizeof (*lep));
1993 break;
1994 }
1995 }
1996 mutex_exit(&thp->th_lock);
1997 }
1998
1999 /*
2000 * Lookup an lwp in the lwpid hash table by lwpid.
2001 */
2002 lwpdir_t *
2003 lwp_hash_lookup(proc_t *p, id_t lwpid)
2004 {
2005 tidhash_t *thp;
2006 lwpdir_t *ldp;
2007
2008 /*
2009 * The process may be exiting, after p_tidhash has been set to NULL in
2010 * proc_exit() but before prfee() has been called. Return failure in
2011 * this case.
2012 */
2013 if (p->p_tidhash == NULL)
2014 return (NULL);
2015
2016 thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
2017 for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
2018 if (ldp->ld_entry->le_lwpid == lwpid)
2019 return (ldp);
2020 }
2021
2022 return (NULL);
2023 }
2024
2025 /*
2026 * Same as lwp_hash_lookup(), but acquire and return
2027 * the tid hash table entry lock on success.
2028 */
2029 lwpdir_t *
2030 lwp_hash_lookup_and_lock(proc_t *p, id_t lwpid, kmutex_t **mpp)
2031 {
2032 tidhash_t *tidhash;
2033 uint_t tidhash_sz;
2034 tidhash_t *thp;
2035 lwpdir_t *ldp;
2036
2037 top:
2038 tidhash_sz = p->p_tidhash_sz;
2039 membar_consumer();
2040 if ((tidhash = p->p_tidhash) == NULL)
2041 return (NULL);
2042
2043 thp = &tidhash[TIDHASH(lwpid, tidhash_sz)];
2044 mutex_enter(&thp->th_lock);
2045
2046 /*
2047 * Since we are not holding p->p_lock, the tid hash table
2048 * may have changed. If so, start over. If not, then
2049 * it cannot change until after we drop &thp->th_lock;
2050 */
2051 if (tidhash != p->p_tidhash || tidhash_sz != p->p_tidhash_sz) {
2052 mutex_exit(&thp->th_lock);
2053 goto top;
2054 }
2055
2056 for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
2057 if (ldp->ld_entry->le_lwpid == lwpid) {
2058 *mpp = &thp->th_lock;
2059 return (ldp);
2060 }
2061 }
2062
2063 mutex_exit(&thp->th_lock);
2064 return (NULL);
2065 }
2066
2067 /*
2068 * Update the indicated LWP usage statistic for the current LWP.
2069 */
2070 void
2071 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
2072 {
2073 klwp_t *lwp = ttolwp(curthread);
2074
2075 if (lwp == NULL)
2076 return;
2077
2078 switch (lwp_stat_id) {
2079 case LWP_STAT_INBLK:
2080 lwp->lwp_ru.inblock += inc;
2081 break;
2082 case LWP_STAT_OUBLK:
2083 lwp->lwp_ru.oublock += inc;
2084 break;
2085 case LWP_STAT_MSGRCV:
2086 lwp->lwp_ru.msgrcv += inc;
2087 break;
2088 case LWP_STAT_MSGSND:
2089 lwp->lwp_ru.msgsnd += inc;
2090 break;
2091 default:
2092 panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
2093 }
2094 }