Print this page
13902 Fix for 13717 may break 8-disk raidz2
13915 installctx() blocking allocate causes problems
Portions contributed by: Jerry Jelinek <gjelinek@gmail.com>
Change-Id: I934d69946cec42630fc541fa8c7385b862b69ca2
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/disp/thread.c
+++ new/usr/src/uts/common/disp/thread.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
24 - * Copyright 2019 Joyent, Inc.
24 + * Copyright 2021 Joyent, Inc.
25 25 */
26 26
27 27 #include <sys/types.h>
28 28 #include <sys/param.h>
29 29 #include <sys/sysmacros.h>
30 30 #include <sys/signal.h>
31 31 #include <sys/stack.h>
32 32 #include <sys/pcb.h>
33 33 #include <sys/user.h>
34 34 #include <sys/systm.h>
35 35 #include <sys/sysinfo.h>
36 36 #include <sys/errno.h>
37 37 #include <sys/cmn_err.h>
38 38 #include <sys/cred.h>
39 39 #include <sys/resource.h>
40 40 #include <sys/task.h>
41 41 #include <sys/project.h>
42 42 #include <sys/proc.h>
43 43 #include <sys/debug.h>
44 44 #include <sys/disp.h>
45 45 #include <sys/class.h>
46 46 #include <vm/seg_kmem.h>
47 47 #include <vm/seg_kp.h>
48 48 #include <sys/machlock.h>
49 49 #include <sys/kmem.h>
50 50 #include <sys/varargs.h>
51 51 #include <sys/turnstile.h>
52 52 #include <sys/poll.h>
53 53 #include <sys/vtrace.h>
54 54 #include <sys/callb.h>
55 55 #include <c2/audit.h>
56 56 #include <sys/tnf.h>
57 57 #include <sys/sobject.h>
58 58 #include <sys/cpupart.h>
59 59 #include <sys/pset.h>
60 60 #include <sys/door.h>
61 61 #include <sys/spl.h>
62 62 #include <sys/copyops.h>
63 63 #include <sys/rctl.h>
64 64 #include <sys/brand.h>
65 65 #include <sys/pool.h>
66 66 #include <sys/zone.h>
67 67 #include <sys/tsol/label.h>
68 68 #include <sys/tsol/tndb.h>
69 69 #include <sys/cpc_impl.h>
70 70 #include <sys/sdt.h>
71 71 #include <sys/reboot.h>
72 72 #include <sys/kdi.h>
73 73 #include <sys/schedctl.h>
74 74 #include <sys/waitq.h>
75 75 #include <sys/cpucaps.h>
76 76 #include <sys/kiconv.h>
77 77 #include <sys/ctype.h>
78 78 #include <sys/smt.h>
79 79
80 80 struct kmem_cache *thread_cache; /* cache of free threads */
81 81 struct kmem_cache *lwp_cache; /* cache of free lwps */
82 82 struct kmem_cache *turnstile_cache; /* cache of free turnstiles */
83 83
84 84 /*
85 85 * allthreads is only for use by kmem_readers. All kernel loops can use
86 86 * the current thread as a start/end point.
87 87 */
88 88 kthread_t *allthreads = &t0; /* circular list of all threads */
89 89
90 90 static kcondvar_t reaper_cv; /* synchronization var */
91 91 kthread_t *thread_deathrow; /* circular list of reapable threads */
92 92 kthread_t *lwp_deathrow; /* circular list of reapable threads */
93 93 kmutex_t reaplock; /* protects lwp and thread deathrows */
94 94 int thread_reapcnt = 0; /* number of threads on deathrow */
95 95 int lwp_reapcnt = 0; /* number of lwps on deathrow */
96 96 int reaplimit = 16; /* delay reaping until reaplimit */
97 97
98 98 thread_free_lock_t *thread_free_lock;
99 99 /* protects tick thread from reaper */
100 100
101 101 extern int nthread;
102 102
103 103 /* System Scheduling classes. */
104 104 id_t syscid; /* system scheduling class ID */
105 105 id_t sysdccid = CLASS_UNUSED; /* reset when SDC loads */
106 106
107 107 void *segkp_thread; /* cookie for segkp pool */
108 108
109 109 int lwp_cache_sz = 32;
110 110 int t_cache_sz = 8;
111 111 static kt_did_t next_t_id = 1;
112 112
113 113 /* Default mode for thread binding to CPUs and processor sets */
114 114 int default_binding_mode = TB_ALLHARD;
115 115
116 116 /*
117 117 * Min/Max stack sizes for stack size parameters
118 118 */
119 119 #define MAX_STKSIZE (32 * DEFAULTSTKSZ)
120 120 #define MIN_STKSIZE DEFAULTSTKSZ
121 121
122 122 /*
123 123 * default_stksize overrides lwp_default_stksize if it is set.
124 124 */
125 125 int default_stksize;
126 126 int lwp_default_stksize;
127 127
128 128 static zone_key_t zone_thread_key;
129 129
130 130 unsigned int kmem_stackinfo; /* stackinfo feature on-off */
131 131 kmem_stkinfo_t *kmem_stkinfo_log; /* stackinfo circular log */
132 132 static kmutex_t kmem_stkinfo_lock; /* protects kmem_stkinfo_log */
133 133
134 134 /*
135 135 * forward declarations for internal thread specific data (tsd)
136 136 */
137 137 static void *tsd_realloc(void *, size_t, size_t);
138 138
139 139 void thread_reaper(void);
140 140
141 141 /* forward declarations for stackinfo feature */
142 142 static void stkinfo_begin(kthread_t *);
143 143 static void stkinfo_end(kthread_t *);
144 144 static size_t stkinfo_percent(caddr_t, caddr_t, caddr_t);
145 145
146 146 /*ARGSUSED*/
147 147 static int
148 148 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
149 149 {
150 150 bzero(buf, sizeof (turnstile_t));
151 151 return (0);
152 152 }
153 153
154 154 /*ARGSUSED*/
155 155 static void
156 156 turnstile_destructor(void *buf, void *cdrarg)
157 157 {
158 158 turnstile_t *ts = buf;
159 159
160 160 ASSERT(ts->ts_free == NULL);
161 161 ASSERT(ts->ts_waiters == 0);
162 162 ASSERT(ts->ts_inheritor == NULL);
163 163 ASSERT(ts->ts_sleepq[0].sq_first == NULL);
164 164 ASSERT(ts->ts_sleepq[1].sq_first == NULL);
165 165 }
166 166
167 167 void
168 168 thread_init(void)
169 169 {
170 170 kthread_t *tp;
171 171 extern char sys_name[];
172 172 extern void idle();
173 173 struct cpu *cpu = CPU;
174 174 int i;
175 175 kmutex_t *lp;
176 176
177 177 mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
178 178 thread_free_lock =
179 179 kmem_alloc(sizeof (thread_free_lock_t) * THREAD_FREE_NUM, KM_SLEEP);
180 180 for (i = 0; i < THREAD_FREE_NUM; i++) {
181 181 lp = &thread_free_lock[i].tf_lock;
182 182 mutex_init(lp, NULL, MUTEX_DEFAULT, NULL);
183 183 }
184 184
185 185 #if defined(__i386) || defined(__amd64)
186 186 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
187 187 PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
188 188
189 189 /*
190 190 * "struct _klwp" includes a "struct pcb", which includes a
191 191 * "struct fpu", which needs to be 64-byte aligned on amd64
192 192 * (and even on i386) for xsave/xrstor.
193 193 */
194 194 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
195 195 64, NULL, NULL, NULL, NULL, NULL, 0);
196 196 #else
197 197 /*
198 198 * Allocate thread structures from static_arena. This prevents
199 199 * issues where a thread tries to relocate its own thread
200 200 * structure and touches it after the mapping has been suspended.
201 201 */
202 202 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
203 203 PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
204 204
205 205 lwp_stk_cache_init();
206 206
207 207 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
208 208 0, NULL, NULL, NULL, NULL, NULL, 0);
209 209 #endif
210 210
211 211 turnstile_cache = kmem_cache_create("turnstile_cache",
212 212 sizeof (turnstile_t), 0,
213 213 turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
214 214
215 215 label_init();
216 216 cred_init();
217 217
218 218 /*
219 219 * Initialize various resource management facilities.
220 220 */
221 221 rctl_init();
222 222 cpucaps_init();
223 223 /*
224 224 * Zone_init() should be called before project_init() so that project ID
225 225 * for the first project is initialized correctly.
226 226 */
227 227 zone_init();
228 228 project_init();
229 229 brand_init();
230 230 kiconv_init();
231 231 task_init();
232 232 tcache_init();
233 233 pool_init();
234 234
235 235 curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
236 236
237 237 /*
238 238 * Originally, we had two parameters to set default stack
239 239 * size: one for lwp's (lwp_default_stksize), and one for
240 240 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
241 241 * Now we have a third parameter that overrides both if it is
242 242 * set to a legal stack size, called default_stksize.
243 243 */
244 244
245 245 if (default_stksize == 0) {
246 246 default_stksize = DEFAULTSTKSZ;
247 247 } else if (default_stksize % PAGESIZE != 0 ||
248 248 default_stksize > MAX_STKSIZE ||
249 249 default_stksize < MIN_STKSIZE) {
250 250 cmn_err(CE_WARN, "Illegal stack size. Using %d",
251 251 (int)DEFAULTSTKSZ);
252 252 default_stksize = DEFAULTSTKSZ;
253 253 } else {
254 254 lwp_default_stksize = default_stksize;
255 255 }
256 256
257 257 if (lwp_default_stksize == 0) {
258 258 lwp_default_stksize = default_stksize;
259 259 } else if (lwp_default_stksize % PAGESIZE != 0 ||
260 260 lwp_default_stksize > MAX_STKSIZE ||
261 261 lwp_default_stksize < MIN_STKSIZE) {
262 262 cmn_err(CE_WARN, "Illegal stack size. Using %d",
263 263 default_stksize);
264 264 lwp_default_stksize = default_stksize;
265 265 }
266 266
267 267 segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
268 268 lwp_default_stksize,
269 269 (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
270 270
271 271 segkp_thread = segkp_cache_init(segkp, t_cache_sz,
272 272 default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
273 273
274 274 (void) getcid(sys_name, &syscid);
275 275 curthread->t_cid = syscid; /* current thread is t0 */
276 276
277 277 /*
278 278 * Set up the first CPU's idle thread.
279 279 * It runs whenever the CPU has nothing worthwhile to do.
280 280 */
281 281 tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
282 282 cpu->cpu_idle_thread = tp;
283 283 tp->t_preempt = 1;
284 284 tp->t_disp_queue = cpu->cpu_disp;
285 285 ASSERT(tp->t_disp_queue != NULL);
286 286 tp->t_bound_cpu = cpu;
287 287 tp->t_affinitycnt = 1;
288 288
289 289 /*
290 290 * Registering a thread in the callback table is usually
291 291 * done in the initialization code of the thread. In this
292 292 * case, we do it right after thread creation to avoid
293 293 * blocking idle thread while registering itself. It also
294 294 * avoids the possibility of reregistration in case a CPU
295 295 * restarts its idle thread.
296 296 */
297 297 CALLB_CPR_INIT_SAFE(tp, "idle");
298 298
299 299 /*
300 300 * Create the thread_reaper daemon. From this point on, exited
301 301 * threads will get reaped.
302 302 */
303 303 (void) thread_create(NULL, 0, (void (*)())thread_reaper,
304 304 NULL, 0, &p0, TS_RUN, minclsyspri);
305 305
306 306 /*
307 307 * Finish initializing the kernel memory allocator now that
308 308 * thread_create() is available.
309 309 */
310 310 kmem_thread_init();
311 311
312 312 if (boothowto & RB_DEBUG)
313 313 kdi_dvec_thravail();
314 314 }
315 315
316 316 /*
317 317 * Create a thread.
318 318 *
319 319 * thread_create() blocks for memory if necessary. It never fails.
320 320 *
321 321 * If stk is NULL, the thread is created at the base of the stack
322 322 * and cannot be swapped.
323 323 */
324 324 kthread_t *
325 325 thread_create(
326 326 caddr_t stk,
327 327 size_t stksize,
328 328 void (*proc)(),
329 329 void *arg,
330 330 size_t len,
331 331 proc_t *pp,
332 332 int state,
333 333 pri_t pri)
334 334 {
335 335 kthread_t *t;
336 336 extern struct classfuncs sys_classfuncs;
337 337 turnstile_t *ts;
338 338
339 339 /*
340 340 * Every thread keeps a turnstile around in case it needs to block.
341 341 * The only reason the turnstile is not simply part of the thread
342 342 * structure is that we may have to break the association whenever
343 343 * more than one thread blocks on a given synchronization object.
344 344 * From a memory-management standpoint, turnstiles are like the
345 345 * "attached mblks" that hang off dblks in the streams allocator.
346 346 */
347 347 ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
348 348
349 349 if (stk == NULL) {
350 350 /*
351 351 * alloc both thread and stack in segkp chunk
352 352 */
353 353
354 354 if (stksize < default_stksize)
355 355 stksize = default_stksize;
356 356
357 357 if (stksize == default_stksize) {
358 358 stk = (caddr_t)segkp_cache_get(segkp_thread);
359 359 } else {
360 360 stksize = roundup(stksize, PAGESIZE);
361 361 stk = (caddr_t)segkp_get(segkp, stksize,
362 362 (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
363 363 }
364 364
365 365 ASSERT(stk != NULL);
366 366
367 367 /*
368 368 * The machine-dependent mutex code may require that
369 369 * thread pointers (since they may be used for mutex owner
370 370 * fields) have certain alignment requirements.
371 371 * PTR24_ALIGN is the size of the alignment quanta.
372 372 * XXX - assumes stack grows toward low addresses.
373 373 */
374 374 if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
375 375 cmn_err(CE_PANIC, "thread_create: proposed stack size"
376 376 " too small to hold thread.");
377 377 #ifdef STACK_GROWTH_DOWN
378 378 stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
379 379 stksize &= -PTR24_ALIGN; /* make thread aligned */
380 380 t = (kthread_t *)(stk + stksize);
381 381 bzero(t, sizeof (kthread_t));
382 382 if (audit_active)
383 383 audit_thread_create(t);
384 384 t->t_stk = stk + stksize;
385 385 t->t_stkbase = stk;
386 386 #else /* stack grows to larger addresses */
387 387 stksize -= SA(sizeof (kthread_t));
388 388 t = (kthread_t *)(stk);
389 389 bzero(t, sizeof (kthread_t));
390 390 t->t_stk = stk + sizeof (kthread_t);
391 391 t->t_stkbase = stk + stksize + sizeof (kthread_t);
392 392 #endif /* STACK_GROWTH_DOWN */
393 393 t->t_flag |= T_TALLOCSTK;
394 394 t->t_swap = stk;
395 395 } else {
396 396 t = kmem_cache_alloc(thread_cache, KM_SLEEP);
397 397 bzero(t, sizeof (kthread_t));
398 398 ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
399 399 if (audit_active)
400 400 audit_thread_create(t);
401 401 /*
402 402 * Initialize t_stk to the kernel stack pointer to use
403 403 * upon entry to the kernel
404 404 */
405 405 #ifdef STACK_GROWTH_DOWN
406 406 t->t_stk = stk + stksize;
407 407 t->t_stkbase = stk;
408 408 #else
409 409 t->t_stk = stk; /* 3b2-like */
410 410 t->t_stkbase = stk + stksize;
411 411 #endif /* STACK_GROWTH_DOWN */
412 412 }
413 413
414 414 if (kmem_stackinfo != 0) {
415 415 stkinfo_begin(t);
416 416 }
417 417
418 418 t->t_ts = ts;
419 419
420 420 /*
421 421 * p_cred could be NULL if it thread_create is called before cred_init
422 422 * is called in main.
423 423 */
424 424 mutex_enter(&pp->p_crlock);
425 425 if (pp->p_cred)
426 426 crhold(t->t_cred = pp->p_cred);
427 427 mutex_exit(&pp->p_crlock);
428 428 t->t_start = gethrestime_sec();
429 429 t->t_startpc = proc;
430 430 t->t_procp = pp;
431 431 t->t_clfuncs = &sys_classfuncs.thread;
432 432 t->t_cid = syscid;
433 433 t->t_pri = pri;
434 434 t->t_stime = ddi_get_lbolt();
435 435 t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
436 436 t->t_bind_cpu = PBIND_NONE;
437 437 t->t_bindflag = (uchar_t)default_binding_mode;
438 438 t->t_bind_pset = PS_NONE;
439 439 t->t_plockp = &pp->p_lock;
440 440 t->t_copyops = NULL;
441 441 t->t_taskq = NULL;
442 442 t->t_anttime = 0;
443 443 t->t_hatdepth = 0;
444 444
445 445 t->t_dtrace_vtime = 1; /* assure vtimestamp is always non-zero */
446 446
447 447 CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
448 448 #ifndef NPROBE
449 449 /* Kernel probe */
450 450 tnf_thread_create(t);
451 451 #endif /* NPROBE */
452 452 LOCK_INIT_CLEAR(&t->t_lock);
453 453
454 454 /*
455 455 * Callers who give us a NULL proc must do their own
456 456 * stack initialization. e.g. lwp_create()
457 457 */
458 458 if (proc != NULL) {
459 459 t->t_stk = thread_stk_init(t->t_stk);
460 460 thread_load(t, proc, arg, len);
461 461 }
462 462
463 463 /*
464 464 * Put a hold on project0. If this thread is actually in a
465 465 * different project, then t_proj will be changed later in
466 466 * lwp_create(). All kernel-only threads must be in project 0.
467 467 */
468 468 t->t_proj = project_hold(proj0p);
469 469
470 470 lgrp_affinity_init(&t->t_lgrp_affinity);
471 471
472 472 mutex_enter(&pidlock);
473 473 nthread++;
474 474 t->t_did = next_t_id++;
475 475 t->t_prev = curthread->t_prev;
476 476 t->t_next = curthread;
477 477
478 478 /*
479 479 * Add the thread to the list of all threads, and initialize
480 480 * its t_cpu pointer. We need to block preemption since
481 481 * cpu_offline walks the thread list looking for threads
482 482 * with t_cpu pointing to the CPU being offlined. We want
483 483 * to make sure that the list is consistent and that if t_cpu
484 484 * is set, the thread is on the list.
485 485 */
486 486 kpreempt_disable();
487 487 curthread->t_prev->t_next = t;
488 488 curthread->t_prev = t;
489 489
490 490 /*
491 491 * We'll always create in the default partition since that's where
492 492 * kernel threads go (we'll change this later if needed, in
493 493 * lwp_create()).
494 494 */
495 495 t->t_cpupart = &cp_default;
496 496
497 497 /*
498 498 * For now, affiliate this thread with the root lgroup.
499 499 * Since the kernel does not (presently) allocate its memory
500 500 * in a locality aware fashion, the root is an appropriate home.
501 501 * If this thread is later associated with an lwp, it will have
502 502 * its lgroup re-assigned at that time.
503 503 */
504 504 lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
505 505
506 506 /*
507 507 * If the current CPU is in the default cpupart, use it. Otherwise,
508 508 * pick one that is; before entering the dispatcher code, we'll
509 509 * make sure to keep the invariant that ->t_cpu is set. (In fact, we
510 510 * rely on this, in ht_should_run(), in the call tree of
511 511 * disp_lowpri_cpu().)
512 512 */
513 513 if (CPU->cpu_part == &cp_default) {
514 514 t->t_cpu = CPU;
515 515 } else {
516 516 t->t_cpu = cp_default.cp_cpulist;
517 517 t->t_cpu = disp_lowpri_cpu(t->t_cpu, t, t->t_pri);
518 518 }
519 519
520 520 t->t_disp_queue = t->t_cpu->cpu_disp;
521 521 kpreempt_enable();
522 522
523 523 /*
524 524 * Initialize thread state and the dispatcher lock pointer.
525 525 * Need to hold onto pidlock to block allthreads walkers until
526 526 * the state is set.
527 527 */
528 528 switch (state) {
529 529 case TS_RUN:
530 530 curthread->t_oldspl = splhigh(); /* get dispatcher spl */
531 531 THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
532 532 CL_SETRUN(t);
533 533 thread_unlock(t);
534 534 break;
535 535
536 536 case TS_ONPROC:
537 537 THREAD_ONPROC(t, t->t_cpu);
538 538 break;
539 539
540 540 case TS_FREE:
541 541 /*
542 542 * Free state will be used for intr threads.
543 543 * The interrupt routine must set the thread dispatcher
544 544 * lock pointer (t_lockp) if starting on a CPU
545 545 * other than the current one.
546 546 */
547 547 THREAD_FREEINTR(t, CPU);
548 548 break;
549 549
550 550 case TS_STOPPED:
551 551 THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
552 552 break;
553 553
554 554 default: /* TS_SLEEP, TS_ZOMB or TS_TRANS */
555 555 cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
556 556 }
557 557 mutex_exit(&pidlock);
558 558 return (t);
559 559 }
560 560
561 561 /*
562 562 * Move thread to project0 and take care of project reference counters.
563 563 */
564 564 void
565 565 thread_rele(kthread_t *t)
566 566 {
567 567 kproject_t *kpj;
568 568
569 569 thread_lock(t);
570 570
571 571 ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
572 572 kpj = ttoproj(t);
573 573 t->t_proj = proj0p;
574 574
575 575 thread_unlock(t);
576 576
577 577 if (kpj != proj0p) {
578 578 project_rele(kpj);
579 579 (void) project_hold(proj0p);
580 580 }
581 581 }
582 582
583 583 void
584 584 thread_exit(void)
585 585 {
586 586 kthread_t *t = curthread;
587 587
588 588 if ((t->t_proc_flag & TP_ZTHREAD) != 0)
589 589 cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
590 590
591 591 tsd_exit(); /* Clean up this thread's TSD */
592 592
593 593 kcpc_passivate(); /* clean up performance counter state */
594 594
595 595 /*
596 596 * No kernel thread should have called poll() without arranging
597 597 * calling pollcleanup() here.
598 598 */
599 599 ASSERT(t->t_pollstate == NULL);
600 600 ASSERT(t->t_schedctl == NULL);
601 601 if (t->t_door)
602 602 door_slam(); /* in case thread did an upcall */
603 603
604 604 #ifndef NPROBE
605 605 /* Kernel probe */
606 606 if (t->t_tnf_tpdp)
607 607 tnf_thread_exit();
608 608 #endif /* NPROBE */
609 609
610 610 thread_rele(t);
611 611 t->t_preempt++;
612 612
613 613 /*
614 614 * remove thread from the all threads list so that
615 615 * death-row can use the same pointers.
616 616 */
617 617 mutex_enter(&pidlock);
618 618 t->t_next->t_prev = t->t_prev;
619 619 t->t_prev->t_next = t->t_next;
620 620 ASSERT(allthreads != t); /* t0 never exits */
621 621 cv_broadcast(&t->t_joincv); /* wake up anyone in thread_join */
622 622 mutex_exit(&pidlock);
623 623
624 624 if (t->t_ctx != NULL)
625 625 exitctx(t);
626 626 if (t->t_procp->p_pctx != NULL)
627 627 exitpctx(t->t_procp);
628 628
629 629 if (kmem_stackinfo != 0) {
630 630 stkinfo_end(t);
631 631 }
632 632
633 633 t->t_state = TS_ZOMB; /* set zombie thread */
634 634
635 635 swtch_from_zombie(); /* give up the CPU */
636 636 /* NOTREACHED */
637 637 }
638 638
639 639 /*
640 640 * Check to see if the specified thread is active (defined as being on
641 641 * the thread list). This is certainly a slow way to do this; if there's
642 642 * ever a reason to speed it up, we could maintain a hash table of active
643 643 * threads indexed by their t_did.
644 644 */
645 645 static kthread_t *
646 646 did_to_thread(kt_did_t tid)
647 647 {
648 648 kthread_t *t;
649 649
650 650 ASSERT(MUTEX_HELD(&pidlock));
651 651 for (t = curthread->t_next; t != curthread; t = t->t_next) {
652 652 if (t->t_did == tid)
653 653 break;
654 654 }
655 655 if (t->t_did == tid)
656 656 return (t);
657 657 else
658 658 return (NULL);
659 659 }
660 660
661 661 /*
662 662 * Wait for specified thread to exit. Returns immediately if the thread
663 663 * could not be found, meaning that it has either already exited or never
664 664 * existed.
665 665 */
666 666 void
667 667 thread_join(kt_did_t tid)
668 668 {
669 669 kthread_t *t;
670 670
671 671 ASSERT(tid != curthread->t_did);
672 672 ASSERT(tid != t0.t_did);
673 673
674 674 mutex_enter(&pidlock);
675 675 /*
676 676 * Make sure we check that the thread is on the thread list
677 677 * before blocking on it; otherwise we could end up blocking on
678 678 * a cv that's already been freed. In other words, don't cache
679 679 * the thread pointer across calls to cv_wait.
680 680 *
681 681 * The choice of loop invariant means that whenever a thread
682 682 * is taken off the allthreads list, a cv_broadcast must be
683 683 * performed on that thread's t_joincv to wake up any waiters.
684 684 * The broadcast doesn't have to happen right away, but it
685 685 * shouldn't be postponed indefinitely (e.g., by doing it in
686 686 * thread_free which may only be executed when the deathrow
687 687 * queue is processed.
688 688 */
689 689 while (t = did_to_thread(tid))
690 690 cv_wait(&t->t_joincv, &pidlock);
691 691 mutex_exit(&pidlock);
692 692 }
693 693
694 694 void
695 695 thread_free_prevent(kthread_t *t)
696 696 {
697 697 kmutex_t *lp;
698 698
699 699 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
700 700 mutex_enter(lp);
701 701 }
702 702
703 703 void
704 704 thread_free_allow(kthread_t *t)
705 705 {
706 706 kmutex_t *lp;
707 707
708 708 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
709 709 mutex_exit(lp);
710 710 }
711 711
712 712 static void
713 713 thread_free_barrier(kthread_t *t)
714 714 {
715 715 kmutex_t *lp;
716 716
717 717 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
718 718 mutex_enter(lp);
719 719 mutex_exit(lp);
720 720 }
721 721
722 722 void
723 723 thread_free(kthread_t *t)
724 724 {
725 725 boolean_t allocstk = (t->t_flag & T_TALLOCSTK);
726 726 klwp_t *lwp = t->t_lwp;
727 727 caddr_t swap = t->t_swap;
728 728
729 729 ASSERT(t != &t0 && t->t_state == TS_FREE);
730 730 ASSERT(t->t_door == NULL);
731 731 ASSERT(t->t_schedctl == NULL);
732 732 ASSERT(t->t_pollstate == NULL);
733 733
734 734 t->t_pri = 0;
735 735 t->t_pc = 0;
736 736 t->t_sp = 0;
737 737 t->t_wchan0 = NULL;
738 738 t->t_wchan = NULL;
739 739 if (t->t_cred != NULL) {
740 740 crfree(t->t_cred);
741 741 t->t_cred = 0;
742 742 }
743 743 if (t->t_pdmsg) {
744 744 kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
745 745 t->t_pdmsg = NULL;
746 746 }
747 747 if (audit_active)
748 748 audit_thread_free(t);
749 749 #ifndef NPROBE
750 750 if (t->t_tnf_tpdp)
751 751 tnf_thread_free(t);
752 752 #endif /* NPROBE */
753 753 if (t->t_cldata) {
754 754 CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
755 755 }
756 756 if (t->t_rprof != NULL) {
757 757 kmem_free(t->t_rprof, sizeof (*t->t_rprof));
758 758 t->t_rprof = NULL;
759 759 }
760 760 t->t_lockp = NULL; /* nothing should try to lock this thread now */
761 761 if (lwp)
762 762 lwp_freeregs(lwp, 0);
763 763 if (t->t_ctx)
764 764 freectx(t, 0);
765 765 t->t_stk = NULL;
766 766 if (lwp)
767 767 lwp_stk_fini(lwp);
768 768 lock_clear(&t->t_lock);
769 769
770 770 if (t->t_ts->ts_waiters > 0)
771 771 panic("thread_free: turnstile still active");
772 772
773 773 kmem_cache_free(turnstile_cache, t->t_ts);
774 774
775 775 free_afd(&t->t_activefd);
776 776
777 777 /*
778 778 * Barrier for the tick accounting code. The tick accounting code
779 779 * holds this lock to keep the thread from going away while it's
780 780 * looking at it.
781 781 */
782 782 thread_free_barrier(t);
783 783
784 784 ASSERT(ttoproj(t) == proj0p);
785 785 project_rele(ttoproj(t));
786 786
787 787 lgrp_affinity_free(&t->t_lgrp_affinity);
788 788
789 789 mutex_enter(&pidlock);
790 790 nthread--;
791 791 mutex_exit(&pidlock);
792 792
793 793 if (t->t_name != NULL) {
794 794 kmem_free(t->t_name, THREAD_NAME_MAX);
795 795 t->t_name = NULL;
796 796 }
797 797
798 798 /*
799 799 * Free thread, lwp and stack. This needs to be done carefully, since
800 800 * if T_TALLOCSTK is set, the thread is part of the stack.
801 801 */
802 802 t->t_lwp = NULL;
803 803 t->t_swap = NULL;
804 804
805 805 if (swap) {
806 806 segkp_release(segkp, swap);
807 807 }
808 808 if (lwp) {
809 809 kmem_cache_free(lwp_cache, lwp);
810 810 }
811 811 if (!allocstk) {
812 812 kmem_cache_free(thread_cache, t);
813 813 }
814 814 }
815 815
816 816 /*
817 817 * Removes threads associated with the given zone from a deathrow queue.
818 818 * tp is a pointer to the head of the deathrow queue, and countp is a
819 819 * pointer to the current deathrow count. Returns a linked list of
820 820 * threads removed from the list.
821 821 */
822 822 static kthread_t *
823 823 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
824 824 {
825 825 kthread_t *tmp, *list = NULL;
826 826 cred_t *cr;
827 827
828 828 ASSERT(MUTEX_HELD(&reaplock));
829 829 while (*tp != NULL) {
830 830 if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
831 831 tmp = *tp;
832 832 *tp = tmp->t_forw;
833 833 tmp->t_forw = list;
834 834 list = tmp;
835 835 (*countp)--;
836 836 } else {
837 837 tp = &(*tp)->t_forw;
838 838 }
839 839 }
840 840 return (list);
841 841 }
842 842
843 843 static void
844 844 thread_reap_list(kthread_t *t)
845 845 {
846 846 kthread_t *next;
847 847
848 848 while (t != NULL) {
849 849 next = t->t_forw;
850 850 thread_free(t);
851 851 t = next;
852 852 }
853 853 }
854 854
855 855 /* ARGSUSED */
856 856 static void
857 857 thread_zone_destroy(zoneid_t zoneid, void *unused)
858 858 {
859 859 kthread_t *t, *l;
860 860
861 861 mutex_enter(&reaplock);
862 862 /*
863 863 * Pull threads and lwps associated with zone off deathrow lists.
864 864 */
865 865 t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
866 866 l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
867 867 mutex_exit(&reaplock);
868 868
869 869 /*
870 870 * Guard against race condition in mutex_owner_running:
871 871 * thread=owner(mutex)
872 872 * <interrupt>
873 873 * thread exits mutex
874 874 * thread exits
875 875 * thread reaped
876 876 * thread struct freed
877 877 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
878 878 * A cross call to all cpus will cause the interrupt handler
879 879 * to reset the PC if it is in mutex_owner_running, refreshing
880 880 * stale thread pointers.
881 881 */
882 882 mutex_sync(); /* sync with mutex code */
883 883
884 884 /*
885 885 * Reap threads
886 886 */
887 887 thread_reap_list(t);
888 888
889 889 /*
890 890 * Reap lwps
891 891 */
892 892 thread_reap_list(l);
893 893 }
894 894
895 895 /*
896 896 * cleanup zombie threads that are on deathrow.
897 897 */
898 898 void
899 899 thread_reaper()
900 900 {
901 901 kthread_t *t, *l;
902 902 callb_cpr_t cprinfo;
903 903
904 904 /*
905 905 * Register callback to clean up threads when zone is destroyed.
906 906 */
907 907 zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
908 908
909 909 CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
910 910 for (;;) {
911 911 mutex_enter(&reaplock);
912 912 while (thread_deathrow == NULL && lwp_deathrow == NULL) {
913 913 CALLB_CPR_SAFE_BEGIN(&cprinfo);
914 914 cv_wait(&reaper_cv, &reaplock);
915 915 CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
916 916 }
917 917 /*
918 918 * mutex_sync() needs to be called when reaping, but
919 919 * not too often. We limit reaping rate to once
920 920 * per second. Reaplimit is max rate at which threads can
921 921 * be freed. Does not impact thread destruction/creation.
922 922 */
923 923 t = thread_deathrow;
924 924 l = lwp_deathrow;
925 925 thread_deathrow = NULL;
926 926 lwp_deathrow = NULL;
927 927 thread_reapcnt = 0;
928 928 lwp_reapcnt = 0;
929 929 mutex_exit(&reaplock);
930 930
931 931 /*
932 932 * Guard against race condition in mutex_owner_running:
933 933 * thread=owner(mutex)
934 934 * <interrupt>
935 935 * thread exits mutex
936 936 * thread exits
937 937 * thread reaped
938 938 * thread struct freed
939 939 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
940 940 * A cross call to all cpus will cause the interrupt handler
941 941 * to reset the PC if it is in mutex_owner_running, refreshing
942 942 * stale thread pointers.
943 943 */
944 944 mutex_sync(); /* sync with mutex code */
945 945 /*
946 946 * Reap threads
947 947 */
948 948 thread_reap_list(t);
949 949
950 950 /*
951 951 * Reap lwps
952 952 */
953 953 thread_reap_list(l);
954 954 delay(hz);
955 955 }
956 956 }
957 957
958 958 /*
959 959 * This is called by lwpcreate, etc.() to put a lwp_deathrow thread onto
960 960 * thread_deathrow. The thread's state is changed already TS_FREE to indicate
961 961 * that is reapable. The thread already holds the reaplock, and was already
962 962 * freed.
963 963 */
964 964 void
965 965 reapq_move_lq_to_tq(kthread_t *t)
966 966 {
967 967 ASSERT(t->t_state == TS_FREE);
968 968 ASSERT(MUTEX_HELD(&reaplock));
969 969 t->t_forw = thread_deathrow;
970 970 thread_deathrow = t;
971 971 thread_reapcnt++;
972 972 if (lwp_reapcnt + thread_reapcnt > reaplimit)
973 973 cv_signal(&reaper_cv); /* wake the reaper */
974 974 }
975 975
976 976 /*
977 977 * This is called by resume() to put a zombie thread onto deathrow.
978 978 * The thread's state is changed to TS_FREE to indicate that is reapable.
979 979 * This is called from the idle thread so it must not block - just spin.
980 980 */
981 981 void
982 982 reapq_add(kthread_t *t)
983 983 {
984 984 mutex_enter(&reaplock);
985 985
986 986 /*
987 987 * lwp_deathrow contains threads with lwp linkage and
988 988 * swappable thread stacks which have the default stacksize.
989 989 * These threads' lwps and stacks may be reused by lwp_create().
990 990 *
991 991 * Anything else goes on thread_deathrow(), where it will eventually
992 992 * be thread_free()d.
993 993 */
994 994 if (t->t_flag & T_LWPREUSE) {
995 995 ASSERT(ttolwp(t) != NULL);
996 996 t->t_forw = lwp_deathrow;
997 997 lwp_deathrow = t;
998 998 lwp_reapcnt++;
999 999 } else {
1000 1000 t->t_forw = thread_deathrow;
1001 1001 thread_deathrow = t;
1002 1002 thread_reapcnt++;
1003 1003 }
1004 1004 if (lwp_reapcnt + thread_reapcnt > reaplimit)
1005 1005 cv_signal(&reaper_cv); /* wake the reaper */
1006 1006 t->t_state = TS_FREE;
1007 1007 lock_clear(&t->t_lock);
1008 1008
1009 1009 /*
1010 1010 * Before we return, we need to grab and drop the thread lock for
1011 1011 * the dead thread. At this point, the current thread is the idle
1012 1012 * thread, and the dead thread's CPU lock points to the current
1013 1013 * CPU -- and we must grab and drop the lock to synchronize with
1014 1014 * a racing thread walking a blocking chain that the zombie thread
1015 1015 * was recently in. By this point, that blocking chain is (by
1016 1016 * definition) stale: the dead thread is not holding any locks, and
1017 1017 * is therefore not in any blocking chains -- but if we do not regrab
1018 1018 * our lock before freeing the dead thread's data structures, the
1019 1019 * thread walking the (stale) blocking chain will die on memory
1020 1020 * corruption when it attempts to drop the dead thread's lock. We
1021 1021 * only need do this once because there is no way for the dead thread
1022 1022 * to ever again be on a blocking chain: once we have grabbed and
|
↓ open down ↓ |
988 lines elided |
↑ open up ↑ |
1023 1023 * dropped the thread lock, we are guaranteed that anyone that could
1024 1024 * have seen this thread in a blocking chain can no longer see it.
1025 1025 */
1026 1026 thread_lock(t);
1027 1027 thread_unlock(t);
1028 1028
1029 1029 mutex_exit(&reaplock);
1030 1030 }
1031 1031
1032 1032 /*
1033 + * Provide an allocation function for callers of installctx() that, for
1034 + * reasons of incomplete context-op initialization, must call installctx()
1035 + * in a kpreempt_disable() block. The caller, therefore, must call this
1036 + * without being in such a block.
1037 + */
1038 +struct ctxop *
1039 +installctx_preallocate(void)
1040 +{
1041 + /*
1042 + * NOTE: We could ASSERT/VERIFY that we are not in a place where
1043 + * a KM_SLEEP allocation could block indefinitely.
1044 + *
1045 + * ASSERT(curthread->t_preempt == 0);
1046 + */
1047 +
1048 + return (kmem_alloc(sizeof (struct ctxop), KM_SLEEP));
1049 +}
1050 +
1051 +/*
1033 1052 * Install thread context ops for the current thread.
1053 + * The caller can pass in a preallocated struct ctxop, eliminating the need
1054 + * for the requirement of entering with kernel preemption still enabled.
1034 1055 */
1035 1056 void
1036 1057 installctx(
1037 1058 kthread_t *t,
1038 1059 void *arg,
1039 1060 void (*save)(void *),
1040 1061 void (*restore)(void *),
1041 1062 void (*fork)(void *, void *),
1042 1063 void (*lwp_create)(void *, void *),
1043 1064 void (*exit)(void *),
1044 - void (*free)(void *, int))
1065 + void (*free)(void *, int),
1066 + struct ctxop *ctx)
1045 1067 {
1046 - struct ctxop *ctx;
1068 + if (ctx == NULL)
1069 + ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
1047 1070
1048 - ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
1049 1071 ctx->save_op = save;
1050 1072 ctx->restore_op = restore;
1051 1073 ctx->fork_op = fork;
1052 1074 ctx->lwp_create_op = lwp_create;
1053 1075 ctx->exit_op = exit;
1054 1076 ctx->free_op = free;
1055 1077 ctx->arg = arg;
1056 1078 ctx->save_ts = 0;
1057 1079 ctx->restore_ts = 0;
1058 1080
1059 1081 /*
1060 1082 * Keep ctxops in a doubly-linked list to allow traversal in both
1061 1083 * directions. Using only the newest-to-oldest ordering was adequate
1062 1084 * previously, but reversing the order for restore_op actions is
1063 1085 * necessary if later-added ctxops depends on earlier ones.
1064 1086 *
1065 1087 * One example of such a dependency: Hypervisor software handling the
1066 1088 * guest FPU expects that it save FPU state prior to host FPU handling
1067 1089 * and consequently handle the guest logic _after_ the host FPU has
1068 1090 * been restored.
1069 1091 *
1070 1092 * The t_ctx member points to the most recently added ctxop or is NULL
1071 1093 * if no ctxops are associated with the thread. The 'next' pointers
1072 1094 * form a loop of the ctxops in newest-to-oldest order. The 'prev'
1073 1095 * pointers form a loop in the reverse direction, where t_ctx->prev is
1074 1096 * the oldest entry associated with the thread.
1075 1097 *
1076 1098 * The protection of kpreempt_disable is required to safely perform the
1077 1099 * list insertion, since there are inconsistent states between some of
1078 1100 * the pointer assignments.
1079 1101 */
1080 1102 kpreempt_disable();
1081 1103 if (t->t_ctx == NULL) {
1082 1104 ctx->next = ctx;
1083 1105 ctx->prev = ctx;
1084 1106 } else {
1085 1107 struct ctxop *head = t->t_ctx, *tail = t->t_ctx->prev;
1086 1108
1087 1109 ctx->next = head;
1088 1110 ctx->prev = tail;
1089 1111 head->prev = ctx;
1090 1112 tail->next = ctx;
1091 1113 }
1092 1114 t->t_ctx = ctx;
1093 1115 kpreempt_enable();
1094 1116 }
1095 1117
1096 1118 /*
1097 1119 * Remove the thread context ops from a thread.
1098 1120 */
1099 1121 int
1100 1122 removectx(
1101 1123 kthread_t *t,
1102 1124 void *arg,
1103 1125 void (*save)(void *),
1104 1126 void (*restore)(void *),
1105 1127 void (*fork)(void *, void *),
1106 1128 void (*lwp_create)(void *, void *),
1107 1129 void (*exit)(void *),
1108 1130 void (*free)(void *, int))
1109 1131 {
1110 1132 struct ctxop *ctx, *head;
1111 1133
1112 1134 /*
1113 1135 * The incoming kthread_t (which is the thread for which the
1114 1136 * context ops will be removed) should be one of the following:
1115 1137 *
1116 1138 * a) the current thread,
1117 1139 *
1118 1140 * b) a thread of a process that's being forked (SIDL),
1119 1141 *
1120 1142 * c) a thread that belongs to the same process as the current
1121 1143 * thread and for which the current thread is the agent thread,
1122 1144 *
1123 1145 * d) a thread that is TS_STOPPED which is indicative of it
1124 1146 * being (if curthread is not an agent) a thread being created
1125 1147 * as part of an lwp creation.
1126 1148 */
1127 1149 ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
1128 1150 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1129 1151
1130 1152 /*
1131 1153 * Serialize modifications to t->t_ctx to prevent the agent thread
1132 1154 * and the target thread from racing with each other during lwp exit.
1133 1155 */
1134 1156 mutex_enter(&t->t_ctx_lock);
1135 1157 kpreempt_disable();
1136 1158
1137 1159 if (t->t_ctx == NULL) {
1138 1160 mutex_exit(&t->t_ctx_lock);
1139 1161 kpreempt_enable();
1140 1162 return (0);
1141 1163 }
1142 1164
1143 1165 ctx = head = t->t_ctx;
1144 1166 do {
1145 1167 if (ctx->save_op == save && ctx->restore_op == restore &&
1146 1168 ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
1147 1169 ctx->exit_op == exit && ctx->free_op == free &&
1148 1170 ctx->arg == arg) {
1149 1171 ctx->prev->next = ctx->next;
1150 1172 ctx->next->prev = ctx->prev;
1151 1173 if (ctx->next == ctx) {
1152 1174 /* last remaining item */
1153 1175 t->t_ctx = NULL;
1154 1176 } else if (ctx == t->t_ctx) {
1155 1177 /* fix up head of list */
1156 1178 t->t_ctx = ctx->next;
1157 1179 }
1158 1180 ctx->next = ctx->prev = NULL;
1159 1181
1160 1182 mutex_exit(&t->t_ctx_lock);
1161 1183 if (ctx->free_op != NULL)
1162 1184 (ctx->free_op)(ctx->arg, 0);
1163 1185 kmem_free(ctx, sizeof (struct ctxop));
1164 1186 kpreempt_enable();
1165 1187 return (1);
1166 1188 }
1167 1189
1168 1190 ctx = ctx->next;
1169 1191 } while (ctx != head);
1170 1192
1171 1193 mutex_exit(&t->t_ctx_lock);
1172 1194 kpreempt_enable();
1173 1195 return (0);
1174 1196 }
1175 1197
1176 1198 void
1177 1199 savectx(kthread_t *t)
1178 1200 {
1179 1201 ASSERT(t == curthread);
1180 1202
1181 1203 if (t->t_ctx != NULL) {
1182 1204 struct ctxop *ctx, *head;
1183 1205
1184 1206 /* Forward traversal */
1185 1207 ctx = head = t->t_ctx;
1186 1208 do {
1187 1209 if (ctx->save_op != NULL) {
1188 1210 ctx->save_ts = gethrtime_unscaled();
1189 1211 (ctx->save_op)(ctx->arg);
1190 1212 }
1191 1213 ctx = ctx->next;
1192 1214 } while (ctx != head);
1193 1215 }
1194 1216 }
1195 1217
1196 1218 void
1197 1219 restorectx(kthread_t *t)
1198 1220 {
1199 1221 ASSERT(t == curthread);
1200 1222
1201 1223 if (t->t_ctx != NULL) {
1202 1224 struct ctxop *ctx, *tail;
1203 1225
1204 1226 /* Backward traversal (starting at the tail) */
1205 1227 ctx = tail = t->t_ctx->prev;
1206 1228 do {
1207 1229 if (ctx->restore_op != NULL) {
1208 1230 ctx->restore_ts = gethrtime_unscaled();
1209 1231 (ctx->restore_op)(ctx->arg);
1210 1232 }
1211 1233 ctx = ctx->prev;
1212 1234 } while (ctx != tail);
1213 1235 }
1214 1236 }
1215 1237
1216 1238 void
1217 1239 forkctx(kthread_t *t, kthread_t *ct)
1218 1240 {
1219 1241 if (t->t_ctx != NULL) {
1220 1242 struct ctxop *ctx, *head;
1221 1243
1222 1244 /* Forward traversal */
1223 1245 ctx = head = t->t_ctx;
1224 1246 do {
1225 1247 if (ctx->fork_op != NULL) {
1226 1248 (ctx->fork_op)(t, ct);
1227 1249 }
1228 1250 ctx = ctx->next;
1229 1251 } while (ctx != head);
1230 1252 }
1231 1253 }
1232 1254
1233 1255 /*
1234 1256 * Note that this operator is only invoked via the _lwp_create
1235 1257 * system call. The system may have other reasons to create lwps
1236 1258 * e.g. the agent lwp or the doors unreferenced lwp.
1237 1259 */
1238 1260 void
1239 1261 lwp_createctx(kthread_t *t, kthread_t *ct)
1240 1262 {
1241 1263 if (t->t_ctx != NULL) {
1242 1264 struct ctxop *ctx, *head;
1243 1265
1244 1266 /* Forward traversal */
1245 1267 ctx = head = t->t_ctx;
1246 1268 do {
1247 1269 if (ctx->lwp_create_op != NULL) {
1248 1270 (ctx->lwp_create_op)(t, ct);
1249 1271 }
1250 1272 ctx = ctx->next;
1251 1273 } while (ctx != head);
1252 1274 }
1253 1275 }
1254 1276
1255 1277 /*
1256 1278 * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1257 1279 * needed when the thread/LWP leaves the processor for the last time. This
1258 1280 * routine is not intended to deal with freeing memory; freectx() is used for
1259 1281 * that purpose during thread_free(). This routine is provided to allow for
1260 1282 * clean-up that can't wait until thread_free().
1261 1283 */
1262 1284 void
1263 1285 exitctx(kthread_t *t)
1264 1286 {
1265 1287 if (t->t_ctx != NULL) {
1266 1288 struct ctxop *ctx, *head;
1267 1289
1268 1290 /* Forward traversal */
1269 1291 ctx = head = t->t_ctx;
1270 1292 do {
1271 1293 if (ctx->exit_op != NULL) {
1272 1294 (ctx->exit_op)(t);
1273 1295 }
1274 1296 ctx = ctx->next;
1275 1297 } while (ctx != head);
1276 1298 }
1277 1299 }
1278 1300
1279 1301 /*
1280 1302 * freectx is called from thread_free() and exec() to get
1281 1303 * rid of old thread context ops.
1282 1304 */
1283 1305 void
1284 1306 freectx(kthread_t *t, int isexec)
1285 1307 {
1286 1308 kpreempt_disable();
1287 1309 if (t->t_ctx != NULL) {
1288 1310 struct ctxop *ctx, *head;
1289 1311
1290 1312 ctx = head = t->t_ctx;
1291 1313 t->t_ctx = NULL;
1292 1314 do {
1293 1315 struct ctxop *next = ctx->next;
1294 1316
1295 1317 if (ctx->free_op != NULL) {
1296 1318 (ctx->free_op)(ctx->arg, isexec);
1297 1319 }
1298 1320 kmem_free(ctx, sizeof (struct ctxop));
1299 1321 ctx = next;
1300 1322 } while (ctx != head);
1301 1323 }
1302 1324 kpreempt_enable();
1303 1325 }
1304 1326
1305 1327 /*
1306 1328 * freectx_ctx is called from lwp_create() when lwp is reused from
1307 1329 * lwp_deathrow and its thread structure is added to thread_deathrow.
1308 1330 * The thread structure to which this ctx was attached may be already
1309 1331 * freed by the thread reaper so free_op implementations shouldn't rely
1310 1332 * on thread structure to which this ctx was attached still being around.
1311 1333 */
1312 1334 void
1313 1335 freectx_ctx(struct ctxop *ctx)
1314 1336 {
1315 1337 struct ctxop *head = ctx;
1316 1338
1317 1339 ASSERT(ctx != NULL);
1318 1340
1319 1341 kpreempt_disable();
1320 1342
1321 1343 head = ctx;
1322 1344 do {
1323 1345 struct ctxop *next = ctx->next;
1324 1346
1325 1347 if (ctx->free_op != NULL) {
1326 1348 (ctx->free_op)(ctx->arg, 0);
1327 1349 }
1328 1350 kmem_free(ctx, sizeof (struct ctxop));
1329 1351 ctx = next;
1330 1352 } while (ctx != head);
1331 1353 kpreempt_enable();
1332 1354 }
1333 1355
1334 1356 /*
1335 1357 * Set the thread running; arrange for it to be swapped in if necessary.
1336 1358 */
1337 1359 void
1338 1360 setrun_locked(kthread_t *t)
1339 1361 {
1340 1362 ASSERT(THREAD_LOCK_HELD(t));
1341 1363 if (t->t_state == TS_SLEEP) {
1342 1364 /*
1343 1365 * Take off sleep queue.
1344 1366 */
1345 1367 SOBJ_UNSLEEP(t->t_sobj_ops, t);
1346 1368 } else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1347 1369 /*
1348 1370 * Already on dispatcher queue.
1349 1371 */
1350 1372 return;
1351 1373 } else if (t->t_state == TS_WAIT) {
1352 1374 waitq_setrun(t);
1353 1375 } else if (t->t_state == TS_STOPPED) {
1354 1376 /*
1355 1377 * All of the sending of SIGCONT (TC_XSTART) and /proc
1356 1378 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1357 1379 * requested that the thread be run.
1358 1380 * Just calling setrun() is not sufficient to set a stopped
1359 1381 * thread running. TP_TXSTART is always set if the thread
1360 1382 * is not stopped by a jobcontrol stop signal.
1361 1383 * TP_TPSTART is always set if /proc is not controlling it.
1362 1384 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1363 1385 * The thread won't be stopped unless one of these
1364 1386 * three mechanisms did it.
1365 1387 *
1366 1388 * These flags must be set before calling setrun_locked(t).
1367 1389 * They can't be passed as arguments because the streams
1368 1390 * code calls setrun() indirectly and the mechanism for
1369 1391 * doing so admits only one argument. Note that the
1370 1392 * thread must be locked in order to change t_schedflags.
1371 1393 */
1372 1394 if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1373 1395 return;
1374 1396 /*
1375 1397 * Process is no longer stopped (a thread is running).
1376 1398 */
1377 1399 t->t_whystop = 0;
1378 1400 t->t_whatstop = 0;
1379 1401 /*
1380 1402 * Strictly speaking, we do not have to clear these
1381 1403 * flags here; they are cleared on entry to stop().
1382 1404 * However, they are confusing when doing kernel
1383 1405 * debugging or when they are revealed by ps(1).
1384 1406 */
1385 1407 t->t_schedflag &= ~TS_ALLSTART;
1386 1408 THREAD_TRANSITION(t); /* drop stopped-thread lock */
1387 1409 ASSERT(t->t_lockp == &transition_lock);
1388 1410 ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1389 1411 /*
1390 1412 * Let the class put the process on the dispatcher queue.
1391 1413 */
1392 1414 CL_SETRUN(t);
1393 1415 }
1394 1416 }
1395 1417
1396 1418 void
1397 1419 setrun(kthread_t *t)
1398 1420 {
1399 1421 thread_lock(t);
1400 1422 setrun_locked(t);
1401 1423 thread_unlock(t);
1402 1424 }
1403 1425
1404 1426 /*
1405 1427 * Unpin an interrupted thread.
1406 1428 * When an interrupt occurs, the interrupt is handled on the stack
1407 1429 * of an interrupt thread, taken from a pool linked to the CPU structure.
1408 1430 *
1409 1431 * When swtch() is switching away from an interrupt thread because it
1410 1432 * blocked or was preempted, this routine is called to complete the
1411 1433 * saving of the interrupted thread state, and returns the interrupted
1412 1434 * thread pointer so it may be resumed.
1413 1435 *
1414 1436 * Called by swtch() only at high spl.
1415 1437 */
1416 1438 kthread_t *
1417 1439 thread_unpin()
1418 1440 {
1419 1441 kthread_t *t = curthread; /* current thread */
1420 1442 kthread_t *itp; /* interrupted thread */
1421 1443 int i; /* interrupt level */
1422 1444 extern int intr_passivate();
1423 1445
1424 1446 ASSERT(t->t_intr != NULL);
1425 1447
1426 1448 itp = t->t_intr; /* interrupted thread */
1427 1449 t->t_intr = NULL; /* clear interrupt ptr */
1428 1450
1429 1451 smt_end_intr();
1430 1452
1431 1453 /*
1432 1454 * Get state from interrupt thread for the one
1433 1455 * it interrupted.
1434 1456 */
1435 1457
1436 1458 i = intr_passivate(t, itp);
1437 1459
1438 1460 TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1439 1461 "intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1440 1462 i, t, t, itp, itp);
1441 1463
1442 1464 /*
1443 1465 * Dissociate the current thread from the interrupted thread's LWP.
1444 1466 */
1445 1467 t->t_lwp = NULL;
1446 1468
1447 1469 /*
1448 1470 * Interrupt handlers above the level that spinlocks block must
1449 1471 * not block.
1450 1472 */
1451 1473 #if DEBUG
1452 1474 if (i < 0 || i > LOCK_LEVEL)
1453 1475 cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1454 1476 #endif
1455 1477
1456 1478 /*
1457 1479 * Compute the CPU's base interrupt level based on the active
1458 1480 * interrupts.
1459 1481 */
1460 1482 ASSERT(CPU->cpu_intr_actv & (1 << i));
1461 1483 set_base_spl();
1462 1484
1463 1485 return (itp);
1464 1486 }
1465 1487
1466 1488 /*
1467 1489 * Create and initialize an interrupt thread.
1468 1490 * Returns non-zero on error.
1469 1491 * Called at spl7() or better.
1470 1492 */
1471 1493 void
1472 1494 thread_create_intr(struct cpu *cp)
1473 1495 {
1474 1496 kthread_t *tp;
1475 1497
1476 1498 tp = thread_create(NULL, 0,
1477 1499 (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1478 1500
1479 1501 /*
1480 1502 * Set the thread in the TS_FREE state. The state will change
1481 1503 * to TS_ONPROC only while the interrupt is active. Think of these
1482 1504 * as being on a private free list for the CPU. Being TS_FREE keeps
1483 1505 * inactive interrupt threads out of debugger thread lists.
1484 1506 *
1485 1507 * We cannot call thread_create with TS_FREE because of the current
1486 1508 * checks there for ONPROC. Fix this when thread_create takes flags.
1487 1509 */
1488 1510 THREAD_FREEINTR(tp, cp);
1489 1511
1490 1512 /*
1491 1513 * Nobody should ever reference the credentials of an interrupt
1492 1514 * thread so make it NULL to catch any such references.
1493 1515 */
1494 1516 tp->t_cred = NULL;
1495 1517 tp->t_flag |= T_INTR_THREAD;
1496 1518 tp->t_cpu = cp;
1497 1519 tp->t_bound_cpu = cp;
1498 1520 tp->t_disp_queue = cp->cpu_disp;
1499 1521 tp->t_affinitycnt = 1;
1500 1522 tp->t_preempt = 1;
1501 1523
1502 1524 /*
1503 1525 * Don't make a user-requested binding on this thread so that
1504 1526 * the processor can be offlined.
1505 1527 */
1506 1528 tp->t_bind_cpu = PBIND_NONE; /* no USER-requested binding */
1507 1529 tp->t_bind_pset = PS_NONE;
1508 1530
1509 1531 #if defined(__i386) || defined(__amd64)
1510 1532 tp->t_stk -= STACK_ALIGN;
1511 1533 *(tp->t_stk) = 0; /* terminate intr thread stack */
1512 1534 #endif
1513 1535
1514 1536 /*
1515 1537 * Link onto CPU's interrupt pool.
1516 1538 */
1517 1539 tp->t_link = cp->cpu_intr_thread;
1518 1540 cp->cpu_intr_thread = tp;
1519 1541 }
1520 1542
1521 1543 /*
1522 1544 * TSD -- THREAD SPECIFIC DATA
1523 1545 */
1524 1546 static kmutex_t tsd_mutex; /* linked list spin lock */
1525 1547 static uint_t tsd_nkeys; /* size of destructor array */
1526 1548 /* per-key destructor funcs */
1527 1549 static void (**tsd_destructor)(void *);
1528 1550 /* list of tsd_thread's */
1529 1551 static struct tsd_thread *tsd_list;
1530 1552
1531 1553 /*
1532 1554 * Default destructor
1533 1555 * Needed because NULL destructor means that the key is unused
1534 1556 */
1535 1557 /* ARGSUSED */
1536 1558 void
1537 1559 tsd_defaultdestructor(void *value)
1538 1560 {}
1539 1561
1540 1562 /*
1541 1563 * Create a key (index into per thread array)
1542 1564 * Locks out tsd_create, tsd_destroy, and tsd_exit
1543 1565 * May allocate memory with lock held
1544 1566 */
1545 1567 void
1546 1568 tsd_create(uint_t *keyp, void (*destructor)(void *))
1547 1569 {
1548 1570 int i;
1549 1571 uint_t nkeys;
1550 1572
1551 1573 /*
1552 1574 * if key is allocated, do nothing
1553 1575 */
1554 1576 mutex_enter(&tsd_mutex);
1555 1577 if (*keyp) {
1556 1578 mutex_exit(&tsd_mutex);
1557 1579 return;
1558 1580 }
1559 1581 /*
1560 1582 * find an unused key
1561 1583 */
1562 1584 if (destructor == NULL)
1563 1585 destructor = tsd_defaultdestructor;
1564 1586
1565 1587 for (i = 0; i < tsd_nkeys; ++i)
1566 1588 if (tsd_destructor[i] == NULL)
1567 1589 break;
1568 1590
1569 1591 /*
1570 1592 * if no unused keys, increase the size of the destructor array
1571 1593 */
1572 1594 if (i == tsd_nkeys) {
1573 1595 if ((nkeys = (tsd_nkeys << 1)) == 0)
1574 1596 nkeys = 1;
1575 1597 tsd_destructor =
1576 1598 (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1577 1599 (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1578 1600 (size_t)(nkeys * sizeof (void (*)(void *))));
1579 1601 tsd_nkeys = nkeys;
1580 1602 }
1581 1603
1582 1604 /*
1583 1605 * allocate the next available unused key
1584 1606 */
1585 1607 tsd_destructor[i] = destructor;
1586 1608 *keyp = i + 1;
1587 1609 mutex_exit(&tsd_mutex);
1588 1610 }
1589 1611
1590 1612 /*
1591 1613 * Destroy a key -- this is for unloadable modules
1592 1614 *
1593 1615 * Assumes that the caller is preventing tsd_set and tsd_get
1594 1616 * Locks out tsd_create, tsd_destroy, and tsd_exit
1595 1617 * May free memory with lock held
1596 1618 */
1597 1619 void
1598 1620 tsd_destroy(uint_t *keyp)
1599 1621 {
1600 1622 uint_t key;
1601 1623 struct tsd_thread *tsd;
1602 1624
1603 1625 /*
1604 1626 * protect the key namespace and our destructor lists
1605 1627 */
1606 1628 mutex_enter(&tsd_mutex);
1607 1629 key = *keyp;
1608 1630 *keyp = 0;
1609 1631
1610 1632 ASSERT(key <= tsd_nkeys);
1611 1633
1612 1634 /*
1613 1635 * if the key is valid
1614 1636 */
1615 1637 if (key != 0) {
1616 1638 uint_t k = key - 1;
1617 1639 /*
1618 1640 * for every thread with TSD, call key's destructor
1619 1641 */
1620 1642 for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1621 1643 /*
1622 1644 * no TSD for key in this thread
1623 1645 */
1624 1646 if (key > tsd->ts_nkeys)
1625 1647 continue;
1626 1648 /*
1627 1649 * call destructor for key
1628 1650 */
1629 1651 if (tsd->ts_value[k] && tsd_destructor[k])
1630 1652 (*tsd_destructor[k])(tsd->ts_value[k]);
1631 1653 /*
1632 1654 * reset value for key
1633 1655 */
1634 1656 tsd->ts_value[k] = NULL;
1635 1657 }
1636 1658 /*
1637 1659 * actually free the key (NULL destructor == unused)
1638 1660 */
1639 1661 tsd_destructor[k] = NULL;
1640 1662 }
1641 1663
1642 1664 mutex_exit(&tsd_mutex);
1643 1665 }
1644 1666
1645 1667 /*
1646 1668 * Quickly return the per thread value that was stored with the specified key
1647 1669 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1648 1670 */
1649 1671 void *
1650 1672 tsd_get(uint_t key)
1651 1673 {
1652 1674 return (tsd_agent_get(curthread, key));
1653 1675 }
1654 1676
1655 1677 /*
1656 1678 * Set a per thread value indexed with the specified key
1657 1679 */
1658 1680 int
1659 1681 tsd_set(uint_t key, void *value)
1660 1682 {
1661 1683 return (tsd_agent_set(curthread, key, value));
1662 1684 }
1663 1685
1664 1686 /*
1665 1687 * Like tsd_get(), except that the agent lwp can get the tsd of
1666 1688 * another thread in the same process (the agent thread only runs when the
1667 1689 * process is completely stopped by /proc), or syslwp is creating a new lwp.
1668 1690 */
1669 1691 void *
1670 1692 tsd_agent_get(kthread_t *t, uint_t key)
1671 1693 {
1672 1694 struct tsd_thread *tsd = t->t_tsd;
1673 1695
1674 1696 ASSERT(t == curthread ||
1675 1697 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1676 1698
1677 1699 if (key && tsd != NULL && key <= tsd->ts_nkeys)
1678 1700 return (tsd->ts_value[key - 1]);
1679 1701 return (NULL);
1680 1702 }
1681 1703
1682 1704 /*
1683 1705 * Like tsd_set(), except that the agent lwp can set the tsd of
1684 1706 * another thread in the same process, or syslwp can set the tsd
1685 1707 * of a thread it's in the middle of creating.
1686 1708 *
1687 1709 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1688 1710 * May lock out tsd_destroy (and tsd_create), may allocate memory with
1689 1711 * lock held
1690 1712 */
1691 1713 int
1692 1714 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1693 1715 {
1694 1716 struct tsd_thread *tsd = t->t_tsd;
1695 1717
1696 1718 ASSERT(t == curthread ||
1697 1719 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1698 1720
1699 1721 if (key == 0)
1700 1722 return (EINVAL);
1701 1723 if (tsd == NULL)
1702 1724 tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1703 1725 if (key <= tsd->ts_nkeys) {
1704 1726 tsd->ts_value[key - 1] = value;
1705 1727 return (0);
1706 1728 }
1707 1729
1708 1730 ASSERT(key <= tsd_nkeys);
1709 1731
1710 1732 /*
1711 1733 * lock out tsd_destroy()
1712 1734 */
1713 1735 mutex_enter(&tsd_mutex);
1714 1736 if (tsd->ts_nkeys == 0) {
1715 1737 /*
1716 1738 * Link onto list of threads with TSD
1717 1739 */
1718 1740 if ((tsd->ts_next = tsd_list) != NULL)
1719 1741 tsd_list->ts_prev = tsd;
1720 1742 tsd_list = tsd;
1721 1743 }
1722 1744
1723 1745 /*
1724 1746 * Allocate thread local storage and set the value for key
1725 1747 */
1726 1748 tsd->ts_value = tsd_realloc(tsd->ts_value,
1727 1749 tsd->ts_nkeys * sizeof (void *),
1728 1750 key * sizeof (void *));
1729 1751 tsd->ts_nkeys = key;
1730 1752 tsd->ts_value[key - 1] = value;
1731 1753 mutex_exit(&tsd_mutex);
1732 1754
1733 1755 return (0);
1734 1756 }
1735 1757
1736 1758
1737 1759 /*
1738 1760 * Return the per thread value that was stored with the specified key
1739 1761 * If necessary, create the key and the value
1740 1762 * Assumes the caller is protecting *keyp from tsd_destroy
1741 1763 */
1742 1764 void *
1743 1765 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1744 1766 {
1745 1767 void *value;
1746 1768 uint_t key = *keyp;
1747 1769 struct tsd_thread *tsd = curthread->t_tsd;
1748 1770
1749 1771 if (tsd == NULL)
1750 1772 tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1751 1773 if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1752 1774 return (value);
1753 1775 if (key == 0)
1754 1776 tsd_create(keyp, destroy);
1755 1777 (void) tsd_set(*keyp, value = (*allocate)());
1756 1778
1757 1779 return (value);
1758 1780 }
1759 1781
1760 1782 /*
1761 1783 * Called from thread_exit() to run the destructor function for each tsd
1762 1784 * Locks out tsd_create and tsd_destroy
1763 1785 * Assumes that the destructor *DOES NOT* use tsd
1764 1786 */
1765 1787 void
1766 1788 tsd_exit(void)
1767 1789 {
1768 1790 int i;
1769 1791 struct tsd_thread *tsd = curthread->t_tsd;
1770 1792
1771 1793 if (tsd == NULL)
1772 1794 return;
1773 1795
1774 1796 if (tsd->ts_nkeys == 0) {
1775 1797 kmem_free(tsd, sizeof (*tsd));
1776 1798 curthread->t_tsd = NULL;
1777 1799 return;
1778 1800 }
1779 1801
1780 1802 /*
1781 1803 * lock out tsd_create and tsd_destroy, call
1782 1804 * the destructor, and mark the value as destroyed.
1783 1805 */
1784 1806 mutex_enter(&tsd_mutex);
1785 1807
1786 1808 for (i = 0; i < tsd->ts_nkeys; i++) {
1787 1809 if (tsd->ts_value[i] && tsd_destructor[i])
1788 1810 (*tsd_destructor[i])(tsd->ts_value[i]);
1789 1811 tsd->ts_value[i] = NULL;
1790 1812 }
1791 1813
1792 1814 /*
1793 1815 * remove from linked list of threads with TSD
1794 1816 */
1795 1817 if (tsd->ts_next)
1796 1818 tsd->ts_next->ts_prev = tsd->ts_prev;
1797 1819 if (tsd->ts_prev)
1798 1820 tsd->ts_prev->ts_next = tsd->ts_next;
1799 1821 if (tsd_list == tsd)
1800 1822 tsd_list = tsd->ts_next;
1801 1823
1802 1824 mutex_exit(&tsd_mutex);
1803 1825
1804 1826 /*
1805 1827 * free up the TSD
1806 1828 */
1807 1829 kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1808 1830 kmem_free(tsd, sizeof (struct tsd_thread));
1809 1831 curthread->t_tsd = NULL;
1810 1832 }
1811 1833
1812 1834 /*
1813 1835 * realloc
1814 1836 */
1815 1837 static void *
1816 1838 tsd_realloc(void *old, size_t osize, size_t nsize)
1817 1839 {
1818 1840 void *new;
1819 1841
1820 1842 new = kmem_zalloc(nsize, KM_SLEEP);
1821 1843 if (old) {
1822 1844 bcopy(old, new, osize);
1823 1845 kmem_free(old, osize);
1824 1846 }
1825 1847 return (new);
1826 1848 }
1827 1849
1828 1850 /*
1829 1851 * Return non-zero if an interrupt is being serviced.
1830 1852 */
1831 1853 int
1832 1854 servicing_interrupt()
1833 1855 {
1834 1856 int onintr = 0;
1835 1857
1836 1858 /* Are we an interrupt thread */
1837 1859 if (curthread->t_flag & T_INTR_THREAD)
1838 1860 return (1);
1839 1861 /* Are we servicing a high level interrupt? */
1840 1862 if (CPU_ON_INTR(CPU)) {
1841 1863 kpreempt_disable();
1842 1864 onintr = CPU_ON_INTR(CPU);
1843 1865 kpreempt_enable();
1844 1866 }
1845 1867 return (onintr);
1846 1868 }
1847 1869
1848 1870
1849 1871 /*
1850 1872 * Change the dispatch priority of a thread in the system.
1851 1873 * Used when raising or lowering a thread's priority.
1852 1874 * (E.g., priority inheritance)
1853 1875 *
1854 1876 * Since threads are queued according to their priority, we
1855 1877 * we must check the thread's state to determine whether it
1856 1878 * is on a queue somewhere. If it is, we've got to:
1857 1879 *
1858 1880 * o Dequeue the thread.
1859 1881 * o Change its effective priority.
1860 1882 * o Enqueue the thread.
1861 1883 *
1862 1884 * Assumptions: The thread whose priority we wish to change
1863 1885 * must be locked before we call thread_change_(e)pri().
1864 1886 * The thread_change(e)pri() function doesn't drop the thread
1865 1887 * lock--that must be done by its caller.
1866 1888 */
1867 1889 void
1868 1890 thread_change_epri(kthread_t *t, pri_t disp_pri)
1869 1891 {
1870 1892 uint_t state;
1871 1893
1872 1894 ASSERT(THREAD_LOCK_HELD(t));
1873 1895
1874 1896 /*
1875 1897 * If the inherited priority hasn't actually changed,
1876 1898 * just return.
1877 1899 */
1878 1900 if (t->t_epri == disp_pri)
1879 1901 return;
1880 1902
1881 1903 state = t->t_state;
1882 1904
1883 1905 /*
1884 1906 * If it's not on a queue, change the priority with impunity.
1885 1907 */
1886 1908 if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1887 1909 t->t_epri = disp_pri;
1888 1910 if (state == TS_ONPROC) {
1889 1911 cpu_t *cp = t->t_disp_queue->disp_cpu;
1890 1912
1891 1913 if (t == cp->cpu_dispthread)
1892 1914 cp->cpu_dispatch_pri = DISP_PRIO(t);
1893 1915 }
1894 1916 } else if (state == TS_SLEEP) {
1895 1917 /*
1896 1918 * Take the thread out of its sleep queue.
1897 1919 * Change the inherited priority.
1898 1920 * Re-enqueue the thread.
1899 1921 * Each synchronization object exports a function
1900 1922 * to do this in an appropriate manner.
1901 1923 */
1902 1924 SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1903 1925 } else if (state == TS_WAIT) {
1904 1926 /*
1905 1927 * Re-enqueue a thread on the wait queue if its
1906 1928 * effective priority needs to change.
1907 1929 */
1908 1930 if (disp_pri != t->t_epri)
1909 1931 waitq_change_pri(t, disp_pri);
1910 1932 } else {
1911 1933 /*
1912 1934 * The thread is on a run queue.
1913 1935 * Note: setbackdq() may not put the thread
1914 1936 * back on the same run queue where it originally
1915 1937 * resided.
1916 1938 */
1917 1939 (void) dispdeq(t);
1918 1940 t->t_epri = disp_pri;
1919 1941 setbackdq(t);
1920 1942 }
1921 1943 schedctl_set_cidpri(t);
1922 1944 }
1923 1945
1924 1946 /*
1925 1947 * Function: Change the t_pri field of a thread.
1926 1948 * Side Effects: Adjust the thread ordering on a run queue
1927 1949 * or sleep queue, if necessary.
1928 1950 * Returns: 1 if the thread was on a run queue, else 0.
1929 1951 */
1930 1952 int
1931 1953 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1932 1954 {
1933 1955 uint_t state;
1934 1956 int on_rq = 0;
1935 1957
1936 1958 ASSERT(THREAD_LOCK_HELD(t));
1937 1959
1938 1960 state = t->t_state;
1939 1961 THREAD_WILLCHANGE_PRI(t, disp_pri);
1940 1962
1941 1963 /*
1942 1964 * If it's not on a queue, change the priority with impunity.
1943 1965 */
1944 1966 if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1945 1967 t->t_pri = disp_pri;
1946 1968
1947 1969 if (state == TS_ONPROC) {
1948 1970 cpu_t *cp = t->t_disp_queue->disp_cpu;
1949 1971
1950 1972 if (t == cp->cpu_dispthread)
1951 1973 cp->cpu_dispatch_pri = DISP_PRIO(t);
1952 1974 }
1953 1975 } else if (state == TS_SLEEP) {
1954 1976 /*
1955 1977 * If the priority has changed, take the thread out of
1956 1978 * its sleep queue and change the priority.
1957 1979 * Re-enqueue the thread.
1958 1980 * Each synchronization object exports a function
1959 1981 * to do this in an appropriate manner.
1960 1982 */
1961 1983 if (disp_pri != t->t_pri)
1962 1984 SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1963 1985 } else if (state == TS_WAIT) {
1964 1986 /*
1965 1987 * Re-enqueue a thread on the wait queue if its
1966 1988 * priority needs to change.
1967 1989 */
1968 1990 if (disp_pri != t->t_pri)
1969 1991 waitq_change_pri(t, disp_pri);
1970 1992 } else {
1971 1993 /*
1972 1994 * The thread is on a run queue.
1973 1995 * Note: setbackdq() may not put the thread
1974 1996 * back on the same run queue where it originally
1975 1997 * resided.
1976 1998 *
1977 1999 * We still requeue the thread even if the priority
1978 2000 * is unchanged to preserve round-robin (and other)
1979 2001 * effects between threads of the same priority.
1980 2002 */
1981 2003 on_rq = dispdeq(t);
1982 2004 ASSERT(on_rq);
1983 2005 t->t_pri = disp_pri;
1984 2006 if (front) {
1985 2007 setfrontdq(t);
1986 2008 } else {
1987 2009 setbackdq(t);
1988 2010 }
1989 2011 }
1990 2012 schedctl_set_cidpri(t);
1991 2013 return (on_rq);
1992 2014 }
1993 2015
1994 2016 /*
1995 2017 * Tunable kmem_stackinfo is set, fill the kernel thread stack with a
1996 2018 * specific pattern.
1997 2019 */
1998 2020 static void
1999 2021 stkinfo_begin(kthread_t *t)
2000 2022 {
2001 2023 caddr_t start; /* stack start */
2002 2024 caddr_t end; /* stack end */
2003 2025 uint64_t *ptr; /* pattern pointer */
2004 2026
2005 2027 /*
2006 2028 * Stack grows up or down, see thread_create(),
2007 2029 * compute stack memory area start and end (start < end).
2008 2030 */
2009 2031 if (t->t_stk > t->t_stkbase) {
2010 2032 /* stack grows down */
2011 2033 start = t->t_stkbase;
2012 2034 end = t->t_stk;
2013 2035 } else {
2014 2036 /* stack grows up */
2015 2037 start = t->t_stk;
2016 2038 end = t->t_stkbase;
2017 2039 }
2018 2040
2019 2041 /*
2020 2042 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
2021 2043 * alignement for start and end in stack area boundaries
2022 2044 * (protection against corrupt t_stkbase/t_stk data).
2023 2045 */
2024 2046 if ((((uintptr_t)start) & 0x7) != 0) {
2025 2047 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
2026 2048 }
2027 2049 end = (caddr_t)(((uintptr_t)end) & (~0x7));
2028 2050
2029 2051 if ((end <= start) || (end - start) > (1024 * 1024)) {
2030 2052 /* negative or stack size > 1 meg, assume bogus */
2031 2053 return;
2032 2054 }
2033 2055
2034 2056 /* fill stack area with a pattern (instead of zeros) */
2035 2057 ptr = (uint64_t *)((void *)start);
2036 2058 while (ptr < (uint64_t *)((void *)end)) {
2037 2059 *ptr++ = KMEM_STKINFO_PATTERN;
2038 2060 }
2039 2061 }
2040 2062
2041 2063
2042 2064 /*
2043 2065 * Tunable kmem_stackinfo is set, create stackinfo log if doesn't already exist,
2044 2066 * compute the percentage of kernel stack really used, and set in the log
2045 2067 * if it's the latest highest percentage.
2046 2068 */
2047 2069 static void
2048 2070 stkinfo_end(kthread_t *t)
2049 2071 {
2050 2072 caddr_t start; /* stack start */
2051 2073 caddr_t end; /* stack end */
2052 2074 uint64_t *ptr; /* pattern pointer */
2053 2075 size_t stksz; /* stack size */
2054 2076 size_t smallest = 0;
2055 2077 size_t percent = 0;
2056 2078 uint_t index = 0;
2057 2079 uint_t i;
2058 2080 static size_t smallest_percent = (size_t)-1;
2059 2081 static uint_t full = 0;
2060 2082
2061 2083 /* create the stackinfo log, if doesn't already exist */
2062 2084 mutex_enter(&kmem_stkinfo_lock);
2063 2085 if (kmem_stkinfo_log == NULL) {
2064 2086 kmem_stkinfo_log = (kmem_stkinfo_t *)
2065 2087 kmem_zalloc(KMEM_STKINFO_LOG_SIZE *
2066 2088 (sizeof (kmem_stkinfo_t)), KM_NOSLEEP);
2067 2089 if (kmem_stkinfo_log == NULL) {
2068 2090 mutex_exit(&kmem_stkinfo_lock);
2069 2091 return;
2070 2092 }
2071 2093 }
2072 2094 mutex_exit(&kmem_stkinfo_lock);
2073 2095
2074 2096 /*
2075 2097 * Stack grows up or down, see thread_create(),
2076 2098 * compute stack memory area start and end (start < end).
2077 2099 */
2078 2100 if (t->t_stk > t->t_stkbase) {
2079 2101 /* stack grows down */
2080 2102 start = t->t_stkbase;
2081 2103 end = t->t_stk;
2082 2104 } else {
2083 2105 /* stack grows up */
2084 2106 start = t->t_stk;
2085 2107 end = t->t_stkbase;
2086 2108 }
2087 2109
2088 2110 /* stack size as found in kthread_t */
2089 2111 stksz = end - start;
2090 2112
2091 2113 /*
2092 2114 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
2093 2115 * alignement for start and end in stack area boundaries
2094 2116 * (protection against corrupt t_stkbase/t_stk data).
2095 2117 */
2096 2118 if ((((uintptr_t)start) & 0x7) != 0) {
2097 2119 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
2098 2120 }
2099 2121 end = (caddr_t)(((uintptr_t)end) & (~0x7));
2100 2122
2101 2123 if ((end <= start) || (end - start) > (1024 * 1024)) {
2102 2124 /* negative or stack size > 1 meg, assume bogus */
2103 2125 return;
2104 2126 }
2105 2127
2106 2128 /* search until no pattern in the stack */
2107 2129 if (t->t_stk > t->t_stkbase) {
2108 2130 /* stack grows down */
2109 2131 #if defined(__i386) || defined(__amd64)
2110 2132 /*
2111 2133 * 6 longs are pushed on stack, see thread_load(). Skip
2112 2134 * them, so if kthread has never run, percent is zero.
2113 2135 * 8 bytes alignement is preserved for a 32 bit kernel,
2114 2136 * 6 x 4 = 24, 24 is a multiple of 8.
2115 2137 *
2116 2138 */
2117 2139 end -= (6 * sizeof (long));
2118 2140 #endif
2119 2141 ptr = (uint64_t *)((void *)start);
2120 2142 while (ptr < (uint64_t *)((void *)end)) {
2121 2143 if (*ptr != KMEM_STKINFO_PATTERN) {
2122 2144 percent = stkinfo_percent(end,
2123 2145 start, (caddr_t)ptr);
2124 2146 break;
2125 2147 }
2126 2148 ptr++;
2127 2149 }
2128 2150 } else {
2129 2151 /* stack grows up */
2130 2152 ptr = (uint64_t *)((void *)end);
2131 2153 ptr--;
2132 2154 while (ptr >= (uint64_t *)((void *)start)) {
2133 2155 if (*ptr != KMEM_STKINFO_PATTERN) {
2134 2156 percent = stkinfo_percent(start,
2135 2157 end, (caddr_t)ptr);
2136 2158 break;
2137 2159 }
2138 2160 ptr--;
2139 2161 }
2140 2162 }
2141 2163
2142 2164 DTRACE_PROBE3(stack__usage, kthread_t *, t,
2143 2165 size_t, stksz, size_t, percent);
2144 2166
2145 2167 if (percent == 0) {
2146 2168 return;
2147 2169 }
2148 2170
2149 2171 mutex_enter(&kmem_stkinfo_lock);
2150 2172 if (full == KMEM_STKINFO_LOG_SIZE && percent < smallest_percent) {
2151 2173 /*
2152 2174 * The log is full and already contains the highest values
2153 2175 */
2154 2176 mutex_exit(&kmem_stkinfo_lock);
2155 2177 return;
2156 2178 }
2157 2179
2158 2180 /* keep a log of the highest used stack */
2159 2181 for (i = 0; i < KMEM_STKINFO_LOG_SIZE; i++) {
2160 2182 if (kmem_stkinfo_log[i].percent == 0) {
2161 2183 index = i;
2162 2184 full++;
2163 2185 break;
2164 2186 }
2165 2187 if (smallest == 0) {
2166 2188 smallest = kmem_stkinfo_log[i].percent;
2167 2189 index = i;
2168 2190 continue;
2169 2191 }
2170 2192 if (kmem_stkinfo_log[i].percent < smallest) {
2171 2193 smallest = kmem_stkinfo_log[i].percent;
2172 2194 index = i;
2173 2195 }
2174 2196 }
2175 2197
2176 2198 if (percent >= kmem_stkinfo_log[index].percent) {
2177 2199 kmem_stkinfo_log[index].kthread = (caddr_t)t;
2178 2200 kmem_stkinfo_log[index].t_startpc = (caddr_t)t->t_startpc;
2179 2201 kmem_stkinfo_log[index].start = start;
2180 2202 kmem_stkinfo_log[index].stksz = stksz;
2181 2203 kmem_stkinfo_log[index].percent = percent;
2182 2204 kmem_stkinfo_log[index].t_tid = t->t_tid;
2183 2205 kmem_stkinfo_log[index].cmd[0] = '\0';
2184 2206 if (t->t_tid != 0) {
2185 2207 stksz = strlen((t->t_procp)->p_user.u_comm);
2186 2208 if (stksz >= KMEM_STKINFO_STR_SIZE) {
2187 2209 stksz = KMEM_STKINFO_STR_SIZE - 1;
2188 2210 kmem_stkinfo_log[index].cmd[stksz] = '\0';
2189 2211 } else {
2190 2212 stksz += 1;
2191 2213 }
2192 2214 (void) memcpy(kmem_stkinfo_log[index].cmd,
2193 2215 (t->t_procp)->p_user.u_comm, stksz);
2194 2216 }
2195 2217 if (percent < smallest_percent) {
2196 2218 smallest_percent = percent;
2197 2219 }
2198 2220 }
2199 2221 mutex_exit(&kmem_stkinfo_lock);
2200 2222 }
2201 2223
2202 2224 /*
2203 2225 * Tunable kmem_stackinfo is set, compute stack utilization percentage.
2204 2226 */
2205 2227 static size_t
2206 2228 stkinfo_percent(caddr_t t_stk, caddr_t t_stkbase, caddr_t sp)
2207 2229 {
2208 2230 size_t percent;
2209 2231 size_t s;
2210 2232
2211 2233 if (t_stk > t_stkbase) {
2212 2234 /* stack grows down */
2213 2235 if (sp > t_stk) {
2214 2236 return (0);
2215 2237 }
2216 2238 if (sp < t_stkbase) {
2217 2239 return (100);
2218 2240 }
2219 2241 percent = t_stk - sp + 1;
2220 2242 s = t_stk - t_stkbase + 1;
2221 2243 } else {
2222 2244 /* stack grows up */
2223 2245 if (sp < t_stk) {
2224 2246 return (0);
2225 2247 }
2226 2248 if (sp > t_stkbase) {
2227 2249 return (100);
2228 2250 }
2229 2251 percent = sp - t_stk + 1;
2230 2252 s = t_stkbase - t_stk + 1;
2231 2253 }
2232 2254 percent = ((100 * percent) / s) + 1;
2233 2255 if (percent > 100) {
2234 2256 percent = 100;
2235 2257 }
2236 2258 return (percent);
2237 2259 }
2238 2260
2239 2261 /*
2240 2262 * NOTE: This will silently truncate a name > THREAD_NAME_MAX - 1 characters
2241 2263 * long. It is expected that callers (acting on behalf of userland clients)
2242 2264 * will perform any required checks to return the correct error semantics.
2243 2265 * It is also expected callers on behalf of userland clients have done
2244 2266 * any necessary permission checks.
2245 2267 */
2246 2268 int
2247 2269 thread_setname(kthread_t *t, const char *name)
2248 2270 {
2249 2271 char *buf = NULL;
2250 2272
2251 2273 /*
2252 2274 * We optimistically assume that a thread's name will only be set
2253 2275 * once and so allocate memory in preparation of setting t_name.
2254 2276 * If it turns out a name has already been set, we just discard (free)
2255 2277 * the buffer we just allocated and reuse the current buffer
2256 2278 * (as all should be THREAD_NAME_MAX large).
2257 2279 *
2258 2280 * Such an arrangement means over the lifetime of a kthread_t, t_name
2259 2281 * is either NULL or has one value (the address of the buffer holding
2260 2282 * the current thread name). The assumption is that most kthread_t
2261 2283 * instances will not have a name assigned, so dynamically allocating
2262 2284 * the memory should minimize the footprint of this feature, but by
2263 2285 * having the buffer persist for the life of the thread, it simplifies
2264 2286 * usage in highly constrained situations (e.g. dtrace).
2265 2287 */
2266 2288 if (name != NULL && name[0] != '\0') {
2267 2289 for (size_t i = 0; name[i] != '\0'; i++) {
2268 2290 if (!isprint(name[i]))
2269 2291 return (EINVAL);
2270 2292 }
2271 2293
2272 2294 buf = kmem_zalloc(THREAD_NAME_MAX, KM_SLEEP);
2273 2295 (void) strlcpy(buf, name, THREAD_NAME_MAX);
2274 2296 }
2275 2297
2276 2298 mutex_enter(&ttoproc(t)->p_lock);
2277 2299 if (t->t_name == NULL) {
2278 2300 t->t_name = buf;
2279 2301 } else {
2280 2302 if (buf != NULL) {
2281 2303 (void) strlcpy(t->t_name, name, THREAD_NAME_MAX);
2282 2304 kmem_free(buf, THREAD_NAME_MAX);
2283 2305 } else {
2284 2306 bzero(t->t_name, THREAD_NAME_MAX);
2285 2307 }
2286 2308 }
2287 2309 mutex_exit(&ttoproc(t)->p_lock);
2288 2310 return (0);
2289 2311 }
2290 2312
2291 2313 int
2292 2314 thread_vsetname(kthread_t *t, const char *fmt, ...)
2293 2315 {
2294 2316 char name[THREAD_NAME_MAX];
2295 2317 va_list va;
2296 2318 int rc;
2297 2319
2298 2320 va_start(va, fmt);
2299 2321 rc = vsnprintf(name, sizeof (name), fmt, va);
2300 2322 va_end(va);
2301 2323
2302 2324 if (rc < 0)
2303 2325 return (EINVAL);
2304 2326
2305 2327 if (rc >= sizeof (name))
2306 2328 return (ENAMETOOLONG);
2307 2329
2308 2330 return (thread_setname(t, name));
2309 2331 }
|
↓ open down ↓ |
1251 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX