Print this page
13902 Fix for 13717 may break 8-disk raidz2
13915 installctx() blocking allocate causes problems
Portions contributed by: Jerry Jelinek <gjelinek@gmail.com>
Change-Id: I934d69946cec42630fc541fa8c7385b862b69ca2
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/os/sundep.c
+++ new/usr/src/uts/intel/ia32/os/sundep.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 - * Copyright 2019 Joyent, Inc.
23 + * Copyright 2021 Joyent, Inc.
24 24 */
25 25
26 26 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
27 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
28 28 /* All Rights Reserved */
29 29
30 30 #include <sys/types.h>
31 31 #include <sys/param.h>
32 32 #include <sys/sysmacros.h>
33 33 #include <sys/signal.h>
34 34 #include <sys/systm.h>
35 35 #include <sys/user.h>
36 36 #include <sys/mman.h>
37 37 #include <sys/class.h>
38 38 #include <sys/proc.h>
39 39 #include <sys/procfs.h>
40 40 #include <sys/buf.h>
41 41 #include <sys/kmem.h>
42 42 #include <sys/cred.h>
43 43 #include <sys/archsystm.h>
44 44 #include <sys/vmparam.h>
45 45 #include <sys/prsystm.h>
46 46 #include <sys/reboot.h>
47 47 #include <sys/uadmin.h>
48 48 #include <sys/vfs.h>
49 49 #include <sys/vnode.h>
50 50 #include <sys/file.h>
51 51 #include <sys/session.h>
52 52 #include <sys/ucontext.h>
53 53 #include <sys/dnlc.h>
54 54 #include <sys/var.h>
55 55 #include <sys/cmn_err.h>
56 56 #include <sys/debugreg.h>
57 57 #include <sys/thread.h>
58 58 #include <sys/vtrace.h>
59 59 #include <sys/consdev.h>
60 60 #include <sys/psw.h>
61 61 #include <sys/regset.h>
62 62 #include <sys/privregs.h>
63 63 #include <sys/cpu.h>
64 64 #include <sys/stack.h>
65 65 #include <sys/swap.h>
66 66 #include <vm/hat.h>
67 67 #include <vm/anon.h>
68 68 #include <vm/as.h>
69 69 #include <vm/page.h>
70 70 #include <vm/seg.h>
71 71 #include <vm/seg_kmem.h>
72 72 #include <vm/seg_map.h>
73 73 #include <vm/seg_vn.h>
74 74 #include <sys/exec.h>
75 75 #include <sys/acct.h>
76 76 #include <sys/core.h>
77 77 #include <sys/corectl.h>
78 78 #include <sys/modctl.h>
79 79 #include <sys/tuneable.h>
80 80 #include <c2/audit.h>
81 81 #include <sys/bootconf.h>
82 82 #include <sys/brand.h>
83 83 #include <sys/dumphdr.h>
84 84 #include <sys/promif.h>
85 85 #include <sys/systeminfo.h>
86 86 #include <sys/kdi.h>
87 87 #include <sys/contract_impl.h>
88 88 #include <sys/x86_archext.h>
89 89 #include <sys/segments.h>
90 90 #include <sys/ontrap.h>
91 91 #include <sys/cpu.h>
92 92 #ifdef __xpv
93 93 #include <sys/hypervisor.h>
94 94 #endif
95 95
96 96 /*
97 97 * Compare the version of boot that boot says it is against
98 98 * the version of boot the kernel expects.
99 99 */
100 100 int
101 101 check_boot_version(int boots_version)
102 102 {
103 103 if (boots_version == BO_VERSION)
104 104 return (0);
105 105
106 106 prom_printf("Wrong boot interface - kernel needs v%d found v%d\n",
107 107 BO_VERSION, boots_version);
108 108 prom_panic("halting");
109 109 /*NOTREACHED*/
110 110 }
111 111
112 112 /*
113 113 * Process the physical installed list for boot.
114 114 * Finds:
115 115 * 1) the pfn of the highest installed physical page,
116 116 * 2) the number of pages installed
117 117 * 3) the number of distinct contiguous regions these pages fall into.
118 118 * 4) the number of contiguous memory ranges
119 119 */
120 120 void
121 121 installed_top_size_ex(
122 122 struct memlist *list, /* pointer to start of installed list */
123 123 pfn_t *high_pfn, /* return ptr for top value */
124 124 pgcnt_t *pgcnt, /* return ptr for sum of installed pages */
125 125 int *ranges) /* return ptr for the count of contig. ranges */
126 126 {
127 127 pfn_t top = 0;
128 128 pgcnt_t sumpages = 0;
129 129 pfn_t highp; /* high page in a chunk */
130 130 int cnt = 0;
131 131
132 132 for (; list; list = list->ml_next) {
133 133 ++cnt;
134 134 highp = (list->ml_address + list->ml_size - 1) >> PAGESHIFT;
135 135 if (top < highp)
136 136 top = highp;
137 137 sumpages += btop(list->ml_size);
138 138 }
139 139
140 140 *high_pfn = top;
141 141 *pgcnt = sumpages;
142 142 *ranges = cnt;
143 143 }
144 144
145 145 void
146 146 installed_top_size(
147 147 struct memlist *list, /* pointer to start of installed list */
148 148 pfn_t *high_pfn, /* return ptr for top value */
149 149 pgcnt_t *pgcnt) /* return ptr for sum of installed pages */
150 150 {
151 151 int ranges;
152 152
153 153 installed_top_size_ex(list, high_pfn, pgcnt, &ranges);
154 154 }
155 155
156 156 void
157 157 phys_install_has_changed(void)
158 158 {}
159 159
160 160 /*
161 161 * Copy in a memory list from boot to kernel, with a filter function
162 162 * to remove pages. The filter function can increase the address and/or
163 163 * decrease the size to filter out pages. It will also align addresses and
164 164 * sizes to PAGESIZE.
165 165 */
166 166 void
167 167 copy_memlist_filter(
168 168 struct memlist *src,
169 169 struct memlist **dstp,
170 170 void (*filter)(uint64_t *, uint64_t *))
171 171 {
172 172 struct memlist *dst, *prev;
173 173 uint64_t addr;
174 174 uint64_t size;
175 175 uint64_t eaddr;
176 176
177 177 dst = *dstp;
178 178 prev = dst;
179 179
180 180 /*
181 181 * Move through the memlist applying a filter against
182 182 * each range of memory. Note that we may apply the
183 183 * filter multiple times against each memlist entry.
184 184 */
185 185 for (; src; src = src->ml_next) {
186 186 addr = P2ROUNDUP(src->ml_address, PAGESIZE);
187 187 eaddr = P2ALIGN(src->ml_address + src->ml_size, PAGESIZE);
188 188 while (addr < eaddr) {
189 189 size = eaddr - addr;
190 190 if (filter != NULL)
191 191 filter(&addr, &size);
192 192 if (size == 0)
193 193 break;
194 194 dst->ml_address = addr;
195 195 dst->ml_size = size;
196 196 dst->ml_next = 0;
197 197 if (prev == dst) {
198 198 dst->ml_prev = 0;
199 199 dst++;
200 200 } else {
201 201 dst->ml_prev = prev;
202 202 prev->ml_next = dst;
203 203 dst++;
204 204 prev++;
205 205 }
206 206 addr += size;
207 207 }
208 208 }
209 209
210 210 *dstp = dst;
211 211 }
212 212
213 213 /*
214 214 * Kernel setup code, called from startup().
215 215 */
216 216 void
217 217 kern_setup1(void)
218 218 {
219 219 proc_t *pp;
220 220
221 221 pp = &p0;
222 222
223 223 proc_sched = pp;
224 224
225 225 /*
226 226 * Initialize process 0 data structures
227 227 */
228 228 pp->p_stat = SRUN;
229 229 pp->p_flag = SSYS;
230 230
231 231 pp->p_pidp = &pid0;
232 232 pp->p_pgidp = &pid0;
233 233 pp->p_sessp = &session0;
234 234 pp->p_tlist = &t0;
235 235 pid0.pid_pglink = pp;
236 236 pid0.pid_pgtail = pp;
237 237
238 238 /*
239 239 * XXX - we asssume that the u-area is zeroed out except for
240 240 * ttolwp(curthread)->lwp_regs.
241 241 */
242 242 PTOU(curproc)->u_cmask = (mode_t)CMASK;
243 243
244 244 thread_init(); /* init thread_free list */
245 245 pid_init(); /* initialize pid (proc) table */
246 246 contract_init(); /* initialize contracts */
247 247
248 248 init_pages_pp_maximum();
249 249 }
250 250
251 251 /*
252 252 * Load a procedure into a thread.
253 253 */
254 254 void
255 255 thread_load(kthread_t *t, void (*start)(), caddr_t arg, size_t len)
256 256 {
257 257 caddr_t sp;
258 258 size_t framesz;
259 259 caddr_t argp;
260 260 long *p;
261 261 extern void thread_start();
262 262
263 263 /*
264 264 * Push a "c" call frame onto the stack to represent
265 265 * the caller of "start".
266 266 */
267 267 sp = t->t_stk;
268 268 ASSERT(((uintptr_t)t->t_stk & (STACK_ENTRY_ALIGN - 1)) == 0);
269 269 if (len != 0) {
270 270 /*
271 271 * the object that arg points at is copied into the
272 272 * caller's frame.
273 273 */
274 274 framesz = SA(len);
275 275 sp -= framesz;
276 276 ASSERT(sp > t->t_stkbase);
277 277 argp = sp + SA(MINFRAME);
278 278 bcopy(arg, argp, len);
279 279 arg = argp;
280 280 }
281 281 /*
282 282 * Set up arguments (arg and len) on the caller's stack frame.
283 283 */
284 284 p = (long *)sp;
285 285
286 286 *--p = 0; /* fake call */
287 287 *--p = 0; /* null frame pointer terminates stack trace */
288 288 *--p = (long)len;
289 289 *--p = (intptr_t)arg;
290 290 *--p = (intptr_t)start;
291 291
292 292 /*
293 293 * initialize thread to resume at thread_start() which will
294 294 * turn around and invoke (*start)(arg, len).
295 295 */
296 296 t->t_pc = (uintptr_t)thread_start;
297 297 t->t_sp = (uintptr_t)p;
298 298
299 299 ASSERT((t->t_sp & (STACK_ENTRY_ALIGN - 1)) == 0);
300 300 }
301 301
302 302 /*
303 303 * load user registers into lwp.
304 304 */
305 305 /*ARGSUSED2*/
306 306 void
307 307 lwp_load(klwp_t *lwp, gregset_t grp, uintptr_t thrptr)
308 308 {
309 309 struct regs *rp = lwptoregs(lwp);
310 310
311 311 setgregs(lwp, grp);
312 312 rp->r_ps = PSL_USER;
313 313
314 314 /*
315 315 * For 64-bit lwps, we allow one magic %fs selector value, and one
316 316 * magic %gs selector to point anywhere in the address space using
317 317 * %fsbase and %gsbase behind the scenes. libc uses %fs to point
318 318 * at the ulwp_t structure.
319 319 *
320 320 * For 32-bit lwps, libc wedges its lwp thread pointer into the
321 321 * ucontext ESP slot (which is otherwise irrelevant to setting a
322 322 * ucontext) and LWPGS_SEL value into gregs[REG_GS]. This is so
323 323 * syslwp_create() can atomically setup %gs.
324 324 *
325 325 * See setup_context() in libc.
326 326 */
327 327 #ifdef _SYSCALL32_IMPL
328 328 if (lwp_getdatamodel(lwp) == DATAMODEL_ILP32) {
329 329 if (grp[REG_GS] == LWPGS_SEL)
330 330 (void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr);
331 331 } else {
332 332 /*
333 333 * See lwp_setprivate in kernel and setup_context in libc.
334 334 *
335 335 * Currently libc constructs a ucontext from whole cloth for
336 336 * every new (not main) lwp created. For 64 bit processes
337 337 * %fsbase is directly set to point to current thread pointer.
338 338 * In the past (solaris 10) %fs was also set LWPFS_SEL to
339 339 * indicate %fsbase. Now we use the null GDT selector for
340 340 * this purpose. LWP[FS|GS]_SEL are only intended for 32 bit
341 341 * processes. To ease transition we support older libcs in
342 342 * the newer kernel by forcing %fs or %gs selector to null
343 343 * by calling lwp_setprivate if LWP[FS|GS]_SEL is passed in
344 344 * the ucontext. This is should be ripped out at some future
345 345 * date. Another fix would be for libc to do a getcontext
346 346 * and inherit the null %fs/%gs from the current context but
347 347 * that means an extra system call and could hurt performance.
348 348 */
349 349 if (grp[REG_FS] == 0x1bb) /* hard code legacy LWPFS_SEL */
350 350 (void) lwp_setprivate(lwp, _LWP_FSBASE,
351 351 (uintptr_t)grp[REG_FSBASE]);
352 352
353 353 if (grp[REG_GS] == 0x1c3) /* hard code legacy LWPGS_SEL */
354 354 (void) lwp_setprivate(lwp, _LWP_GSBASE,
355 355 (uintptr_t)grp[REG_GSBASE]);
356 356 }
357 357 #else
358 358 if (grp[GS] == LWPGS_SEL)
359 359 (void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr);
360 360 #endif
361 361
362 362 lwp->lwp_eosys = JUSTRETURN;
363 363 lwptot(lwp)->t_post_sys = 1;
364 364 }
365 365
366 366 /*
367 367 * set syscall()'s return values for a lwp.
368 368 */
369 369 void
370 370 lwp_setrval(klwp_t *lwp, int v1, int v2)
371 371 {
372 372 lwptoregs(lwp)->r_ps &= ~PS_C;
373 373 lwptoregs(lwp)->r_r0 = v1;
374 374 lwptoregs(lwp)->r_r1 = v2;
375 375 }
376 376
377 377 /*
378 378 * set syscall()'s return values for a lwp.
379 379 */
380 380 void
381 381 lwp_setsp(klwp_t *lwp, caddr_t sp)
382 382 {
383 383 lwptoregs(lwp)->r_sp = (intptr_t)sp;
384 384 }
385 385
386 386 /*
387 387 * Copy regs from parent to child.
388 388 */
389 389 void
390 390 lwp_forkregs(klwp_t *lwp, klwp_t *clwp)
391 391 {
392 392 #if defined(__amd64)
393 393 struct pcb *pcb = &clwp->lwp_pcb;
394 394 struct regs *rp = lwptoregs(lwp);
395 395
396 396 if (!PCB_NEED_UPDATE_SEGS(pcb)) {
397 397 pcb->pcb_ds = rp->r_ds;
398 398 pcb->pcb_es = rp->r_es;
399 399 pcb->pcb_fs = rp->r_fs;
400 400 pcb->pcb_gs = rp->r_gs;
401 401 PCB_SET_UPDATE_SEGS(pcb);
402 402 lwptot(clwp)->t_post_sys = 1;
403 403 }
404 404 ASSERT(lwptot(clwp)->t_post_sys);
405 405 #endif
406 406
407 407 fp_lwp_dup(clwp);
408 408
409 409 bcopy(lwp->lwp_regs, clwp->lwp_regs, sizeof (struct regs));
410 410 }
411 411
412 412 /*
413 413 * This function is currently unused on x86.
414 414 */
415 415 /*ARGSUSED*/
416 416 void
417 417 lwp_freeregs(klwp_t *lwp, int isexec)
418 418 {}
419 419
420 420 /*
421 421 * This function is currently unused on x86.
422 422 */
423 423 void
424 424 lwp_pcb_exit(void)
425 425 {}
426 426
427 427 /*
428 428 * Lwp context ops for segment registers.
429 429 */
430 430
431 431 /*
432 432 * Every time we come into the kernel (syscall, interrupt or trap
433 433 * but not fast-traps) we capture the current values of the user's
434 434 * segment registers into the lwp's reg structure. This includes
435 435 * lcall for i386 generic system call support since it is handled
436 436 * as a segment-not-present trap.
437 437 *
438 438 * Here we save the current values from the lwp regs into the pcb
439 439 * and or PCB_UPDATE_SEGS (1) in pcb->pcb_rupdate to tell the rest
440 440 * of the kernel that the pcb copy of the segment registers is the
441 441 * current one. This ensures the lwp's next trip to user land via
442 442 * update_sregs. Finally we set t_post_sys to ensure that no
443 443 * system call fast-path's its way out of the kernel via sysret.
444 444 *
445 445 * (This means that we need to have interrupts disabled when we
446 446 * test t->t_post_sys in the syscall handlers; if the test fails,
447 447 * we need to keep interrupts disabled until we return to userland
448 448 * so we can't be switched away.)
449 449 *
450 450 * As a result of all this, we don't really have to do a whole lot
451 451 * if the thread is just mucking about in the kernel, switching on
452 452 * and off the cpu for whatever reason it feels like. And yet we
453 453 * still preserve fast syscalls, cause if we -don't- get
454 454 * descheduled, we never come here either.
455 455 */
456 456
457 457 #define VALID_LWP_DESC(udp) ((udp)->usd_type == SDT_MEMRWA && \
458 458 (udp)->usd_p == 1 && (udp)->usd_dpl == SEL_UPL)
459 459
460 460 /*ARGSUSED*/
461 461 void
462 462 lwp_segregs_save(klwp_t *lwp)
463 463 {
464 464 #if defined(__amd64)
465 465 pcb_t *pcb = &lwp->lwp_pcb;
466 466 struct regs *rp;
467 467
468 468 ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc));
469 469 ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc));
470 470
471 471 if (!PCB_NEED_UPDATE_SEGS(pcb)) {
472 472 rp = lwptoregs(lwp);
473 473
474 474 /*
475 475 * If there's no update already pending, capture the current
476 476 * %ds/%es/%fs/%gs values from lwp's regs in case the user
477 477 * changed them; %fsbase and %gsbase are privileged so the
478 478 * kernel versions of these registers in pcb_fsbase and
479 479 * pcb_gsbase are always up-to-date.
480 480 */
481 481 pcb->pcb_ds = rp->r_ds;
482 482 pcb->pcb_es = rp->r_es;
483 483 pcb->pcb_fs = rp->r_fs;
484 484 pcb->pcb_gs = rp->r_gs;
485 485 PCB_SET_UPDATE_SEGS(pcb);
486 486 lwp->lwp_thread->t_post_sys = 1;
487 487 }
488 488 #endif /* __amd64 */
489 489
490 490 #if !defined(__xpv) /* XXPV not sure if we can re-read gdt? */
491 491 ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPFS], &lwp->lwp_pcb.pcb_fsdesc,
492 492 sizeof (lwp->lwp_pcb.pcb_fsdesc)) == 0);
493 493 ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPGS], &lwp->lwp_pcb.pcb_gsdesc,
494 494 sizeof (lwp->lwp_pcb.pcb_gsdesc)) == 0);
495 495 #endif
496 496 }
497 497
498 498 #if defined(__amd64)
499 499
500 500 /*
501 501 * Update the segment registers with new values from the pcb.
502 502 *
503 503 * We have to do this carefully, and in the following order,
504 504 * in case any of the selectors points at a bogus descriptor.
505 505 * If they do, we'll catch trap with on_trap and return 1.
506 506 * returns 0 on success.
507 507 *
508 508 * This is particularly tricky for %gs.
509 509 * This routine must be executed under a cli.
510 510 */
511 511 int
512 512 update_sregs(struct regs *rp, klwp_t *lwp)
513 513 {
514 514 pcb_t *pcb = &lwp->lwp_pcb;
515 515 ulong_t kgsbase;
516 516 on_trap_data_t otd;
517 517 int rc = 0;
518 518
519 519 if (!on_trap(&otd, OT_SEGMENT_ACCESS)) {
520 520
521 521 #if defined(__xpv)
522 522 /*
523 523 * On the hyervisor this is easy. The hypercall below will
524 524 * swapgs and load %gs with the user selector. If the user
525 525 * selector is bad the hypervisor will catch the fault and
526 526 * load %gs with the null selector instead. Either way the
527 527 * kernel's gsbase is not damaged.
528 528 */
529 529 kgsbase = (ulong_t)CPU;
530 530 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL,
531 531 pcb->pcb_gs) != 0) {
532 532 no_trap();
533 533 return (1);
534 534 }
535 535
536 536 rp->r_gs = pcb->pcb_gs;
537 537 ASSERT((cpu_t *)kgsbase == CPU);
538 538
539 539 #else /* __xpv */
540 540
541 541 /*
542 542 * A little more complicated running native.
543 543 */
544 544 kgsbase = (ulong_t)CPU;
545 545 __set_gs(pcb->pcb_gs);
546 546
547 547 /*
548 548 * If __set_gs fails it's because the new %gs is a bad %gs,
549 549 * we'll be taking a trap but with the original %gs and %gsbase
550 550 * undamaged (i.e. pointing at curcpu).
551 551 *
552 552 * We've just mucked up the kernel's gsbase. Oops. In
553 553 * particular we can't take any traps at all. Make the newly
554 554 * computed gsbase be the hidden gs via swapgs, and fix
555 555 * the kernel's gsbase back again. Later, when we return to
556 556 * userland we'll swapgs again restoring gsbase just loaded
557 557 * above.
558 558 */
559 559 __asm__ __volatile__("mfence; swapgs");
560 560
561 561 rp->r_gs = pcb->pcb_gs;
562 562
563 563 /*
564 564 * Restore kernel's gsbase. Note that this also serializes any
565 565 * attempted speculation from loading the user-controlled
566 566 * %gsbase.
567 567 */
568 568 wrmsr(MSR_AMD_GSBASE, kgsbase);
569 569
570 570 #endif /* __xpv */
571 571
572 572 /*
573 573 * Only override the descriptor base address if
574 574 * r_gs == LWPGS_SEL or if r_gs == NULL. A note on
575 575 * NULL descriptors -- 32-bit programs take faults
576 576 * if they deference NULL descriptors; however,
577 577 * when 64-bit programs load them into %fs or %gs,
578 578 * they DONT fault -- only the base address remains
579 579 * whatever it was from the last load. Urk.
580 580 *
581 581 * XXX - note that lwp_setprivate now sets %fs/%gs to the
582 582 * null selector for 64 bit processes. Whereas before
583 583 * %fs/%gs were set to LWP(FS|GS)_SEL regardless of
584 584 * the process's data model. For now we check for both
585 585 * values so that the kernel can also support the older
586 586 * libc. This should be ripped out at some point in the
587 587 * future.
588 588 */
589 589 if (pcb->pcb_gs == LWPGS_SEL || pcb->pcb_gs == 0) {
590 590 #if defined(__xpv)
591 591 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER,
592 592 pcb->pcb_gsbase)) {
593 593 no_trap();
594 594 return (1);
595 595 }
596 596 #else
597 597 wrmsr(MSR_AMD_KGSBASE, pcb->pcb_gsbase);
598 598 #endif
599 599 }
600 600
601 601 __set_ds(pcb->pcb_ds);
602 602 rp->r_ds = pcb->pcb_ds;
603 603
604 604 __set_es(pcb->pcb_es);
605 605 rp->r_es = pcb->pcb_es;
606 606
607 607 __set_fs(pcb->pcb_fs);
608 608 rp->r_fs = pcb->pcb_fs;
609 609
610 610 /*
611 611 * Same as for %gs
612 612 */
613 613 if (pcb->pcb_fs == LWPFS_SEL || pcb->pcb_fs == 0) {
614 614 #if defined(__xpv)
615 615 if (HYPERVISOR_set_segment_base(SEGBASE_FS,
616 616 pcb->pcb_fsbase)) {
617 617 no_trap();
618 618 return (1);
619 619 }
620 620 #else
621 621 wrmsr(MSR_AMD_FSBASE, pcb->pcb_fsbase);
622 622 #endif
623 623 }
624 624
625 625 } else {
626 626 cli();
627 627 rc = 1;
628 628 }
629 629 no_trap();
630 630 return (rc);
631 631 }
632 632
633 633 /*
634 634 * Make sure any stale selectors are cleared from the segment registers
635 635 * by putting KDS_SEL (the kernel's default %ds gdt selector) into them.
636 636 * This is necessary because the kernel itself does not use %es, %fs, nor
637 637 * %ds. (%cs and %ss are necessary, and are set up by the kernel - along with
638 638 * %gs - to point to the current cpu struct.) If we enter kmdb while in the
639 639 * kernel and resume with a stale ldt or brandz selector sitting there in a
640 640 * segment register, kmdb will #gp fault if the stale selector points to,
641 641 * for example, an ldt in the context of another process.
642 642 *
643 643 * WARNING: Intel and AMD chips behave differently when storing
644 644 * the null selector into %fs and %gs while in long mode. On AMD
645 645 * chips fsbase and gsbase are not cleared. But on Intel chips, storing
646 646 * a null selector into %fs or %gs has the side effect of clearing
647 647 * fsbase or gsbase. For that reason we use KDS_SEL, which has
648 648 * consistent behavor between AMD and Intel.
649 649 *
650 650 * Caller responsible for preventing cpu migration.
651 651 */
652 652 void
653 653 reset_sregs(void)
654 654 {
655 655 ulong_t kgsbase = (ulong_t)CPU;
656 656
657 657 ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL);
658 658
659 659 cli();
660 660 __set_gs(KGS_SEL);
661 661
662 662 /*
663 663 * restore kernel gsbase
664 664 */
665 665 #if defined(__xpv)
666 666 xen_set_segment_base(SEGBASE_GS_KERNEL, kgsbase);
667 667 #else
668 668 wrmsr(MSR_AMD_GSBASE, kgsbase);
669 669 #endif
670 670
671 671 sti();
672 672
673 673 __set_ds(KDS_SEL);
674 674 __set_es(0 | SEL_KPL); /* selector RPL not ring 0 on hypervisor */
675 675 __set_fs(KFS_SEL);
676 676 }
677 677
678 678 #endif /* __amd64 */
679 679
680 680 #ifdef _SYSCALL32_IMPL
681 681
682 682 /*
683 683 * Make it impossible for a process to change its data model.
684 684 * We do this by toggling the present bits for the 32 and
685 685 * 64-bit user code descriptors. That way if a user lwp attempts
686 686 * to change its data model (by using the wrong code descriptor in
687 687 * %cs) it will fault immediately. This also allows us to simplify
688 688 * assertions and checks in the kernel.
689 689 */
690 690
691 691 static void
692 692 gdt_ucode_model(model_t model)
693 693 {
694 694 kpreempt_disable();
695 695 if (model == DATAMODEL_NATIVE) {
696 696 gdt_update_usegd(GDT_UCODE, &ucs_on);
697 697 gdt_update_usegd(GDT_U32CODE, &ucs32_off);
698 698 } else {
699 699 gdt_update_usegd(GDT_U32CODE, &ucs32_on);
700 700 gdt_update_usegd(GDT_UCODE, &ucs_off);
701 701 }
702 702 kpreempt_enable();
703 703 }
704 704
705 705 #endif /* _SYSCALL32_IMPL */
706 706
707 707 /*
708 708 * Restore lwp private fs and gs segment descriptors
709 709 * on current cpu's GDT.
710 710 */
711 711 static void
712 712 lwp_segregs_restore(klwp_t *lwp)
713 713 {
714 714 pcb_t *pcb = &lwp->lwp_pcb;
715 715
716 716 ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc));
717 717 ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc));
718 718
719 719 #ifdef _SYSCALL32_IMPL
720 720 gdt_ucode_model(DATAMODEL_NATIVE);
721 721 #endif
722 722
723 723 gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
724 724 gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);
725 725
726 726 }
727 727
728 728 #ifdef _SYSCALL32_IMPL
729 729
730 730 static void
731 731 lwp_segregs_restore32(klwp_t *lwp)
732 732 {
733 733 /*LINTED*/
734 734 cpu_t *cpu = CPU;
735 735 pcb_t *pcb = &lwp->lwp_pcb;
736 736
737 737 ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_fsdesc));
738 738 ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_gsdesc));
739 739
740 740 gdt_ucode_model(DATAMODEL_ILP32);
741 741 gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
742 742 gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);
743 743 }
744 744
745 745 #endif /* _SYSCALL32_IMPL */
746 746
747 747 /*
748 748 * If this is a process in a branded zone, then we want it to use the brand
749 749 * syscall entry points instead of the standard Solaris entry points. This
750 750 * routine must be called when a new lwp is created within a branded zone
751 751 * or when an existing lwp moves into a branded zone via a zone_enter()
752 752 * operation.
753 753 */
754 754 void
755 755 lwp_attach_brand_hdlrs(klwp_t *lwp)
|
↓ open down ↓ |
722 lines elided |
↑ open up ↑ |
756 756 {
757 757 kthread_t *t = lwptot(lwp);
758 758
759 759 ASSERT(PROC_IS_BRANDED(lwptoproc(lwp)));
760 760
761 761 ASSERT(removectx(t, NULL, brand_interpositioning_disable,
762 762 brand_interpositioning_enable, NULL, NULL,
763 763 brand_interpositioning_disable, NULL) == 0);
764 764 installctx(t, NULL, brand_interpositioning_disable,
765 765 brand_interpositioning_enable, NULL, NULL,
766 - brand_interpositioning_disable, NULL);
766 + brand_interpositioning_disable, NULL, NULL);
767 767
768 768 if (t == curthread) {
769 769 kpreempt_disable();
770 770 brand_interpositioning_enable();
771 771 kpreempt_enable();
772 772 }
773 773 }
774 774
775 775 /*
776 776 * If this is a process in a branded zone, then we want it to disable the
777 777 * brand syscall entry points. This routine must be called when the last
778 778 * lwp in a process is exiting in proc_exit().
779 779 */
780 780 void
781 781 lwp_detach_brand_hdlrs(klwp_t *lwp)
782 782 {
783 783 kthread_t *t = lwptot(lwp);
784 784
785 785 ASSERT(PROC_IS_BRANDED(lwptoproc(lwp)));
786 786 if (t == curthread)
787 787 kpreempt_disable();
788 788
789 789 /* Remove the original context handlers */
790 790 VERIFY(removectx(t, NULL, brand_interpositioning_disable,
791 791 brand_interpositioning_enable, NULL, NULL,
792 792 brand_interpositioning_disable, NULL) != 0);
793 793
794 794 if (t == curthread) {
795 795 /* Cleanup our MSR and IDT entries. */
796 796 brand_interpositioning_disable();
797 797 kpreempt_enable();
798 798 }
799 799 }
800 800
801 801 /*
802 802 * Add any lwp-associated context handlers to the lwp at the beginning
803 803 * of the lwp's useful life.
804 804 *
805 805 * All paths which create lwp's invoke lwp_create(); lwp_create()
806 806 * invokes lwp_stk_init() which initializes the stack, sets up
807 807 * lwp_regs, and invokes this routine.
808 808 *
809 809 * All paths which destroy lwp's invoke lwp_exit() to rip the lwp
810 810 * apart and put it on 'lwp_deathrow'; if the lwp is destroyed it
811 811 * ends up in thread_free() which invokes freectx(t, 0) before
812 812 * invoking lwp_stk_fini(). When the lwp is recycled from death
813 813 * row, lwp_stk_fini() is invoked, then thread_free(), and thus
814 814 * freectx(t, 0) as before.
815 815 *
816 816 * In the case of exec, the surviving lwp is thoroughly scrubbed
817 817 * clean; exec invokes freectx(t, 1) to destroy associated contexts.
818 818 * On the way back to the new image, it invokes setregs() which
819 819 * in turn invokes this routine.
820 820 */
821 821 void
|
↓ open down ↓ |
45 lines elided |
↑ open up ↑ |
822 822 lwp_installctx(klwp_t *lwp)
823 823 {
824 824 kthread_t *t = lwptot(lwp);
825 825 int thisthread = t == curthread;
826 826 #ifdef _SYSCALL32_IMPL
827 827 void (*restop)(klwp_t *) = lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ?
828 828 lwp_segregs_restore : lwp_segregs_restore32;
829 829 #else
830 830 void (*restop)(klwp_t *) = lwp_segregs_restore;
831 831 #endif
832 + struct ctxop *ctx;
832 833
833 834 /*
834 835 * Install the basic lwp context handlers on each lwp.
835 836 *
836 837 * On the amd64 kernel, the context handlers are responsible for
837 838 * virtualizing %ds, %es, %fs, and %gs to the lwp. The register
838 839 * values are only ever changed via sys_rtt when the
839 840 * PCB_UPDATE_SEGS bit (1) is set in pcb->pcb_rupdate. Only
840 841 * sys_rtt gets to clear the bit.
841 842 *
842 843 * On the i386 kernel, the context handlers are responsible for
843 844 * virtualizing %gs/%fs to the lwp by updating the per-cpu GDTs
844 845 */
845 846 ASSERT(removectx(t, lwp, lwp_segregs_save, restop,
846 847 NULL, NULL, NULL, NULL) == 0);
847 - if (thisthread)
848 + if (thisthread) {
849 + ctx = installctx_preallocate();
848 850 kpreempt_disable();
851 + } else {
852 + ctx = NULL;
853 + }
849 854 installctx(t, lwp, lwp_segregs_save, restop,
850 - NULL, NULL, NULL, NULL);
855 + NULL, NULL, NULL, NULL, ctx);
851 856 if (thisthread) {
852 857 /*
853 858 * Since we're the right thread, set the values in the GDT
854 859 */
855 860 restop(lwp);
856 861 kpreempt_enable();
857 862 }
858 863
859 864 /*
860 865 * If we have sysenter/sysexit instructions enabled, we need
861 866 * to ensure that the hardware mechanism is kept up-to-date with the
862 867 * lwp's kernel stack pointer across context switches.
863 868 *
864 869 * sep_save zeros the sysenter stack pointer msr; sep_restore sets
865 870 * it to the lwp's kernel stack pointer (kstktop).
866 871 */
|
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
867 872 if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
868 873 #if defined(__amd64)
869 874 caddr_t kstktop = (caddr_t)lwp->lwp_regs;
870 875 #elif defined(__i386)
871 876 caddr_t kstktop = ((caddr_t)lwp->lwp_regs - MINFRAME) +
872 877 SA(sizeof (struct regs) + MINFRAME);
873 878 #endif
874 879 ASSERT(removectx(t, kstktop,
875 880 sep_save, sep_restore, NULL, NULL, NULL, NULL) == 0);
876 881
877 - if (thisthread)
882 + if (thisthread) {
883 + ctx = installctx_preallocate();
878 884 kpreempt_disable();
885 + } else {
886 + ctx = NULL;
887 + }
879 888 installctx(t, kstktop,
880 - sep_save, sep_restore, NULL, NULL, NULL, NULL);
889 + sep_save, sep_restore, NULL, NULL, NULL, NULL, ctx);
881 890 if (thisthread) {
882 891 /*
883 892 * We're the right thread, so set the stack pointer
884 893 * for the first sysenter instruction to use
885 894 */
886 895 sep_restore(kstktop);
887 896 kpreempt_enable();
888 897 }
889 898 }
890 899
891 900 if (PROC_IS_BRANDED(ttoproc(t)))
892 901 lwp_attach_brand_hdlrs(lwp);
893 902 }
894 903
895 904 /*
896 905 * Clear registers on exec(2).
897 906 */
898 907 void
899 908 setregs(uarg_t *args)
900 909 {
901 910 struct regs *rp;
902 911 kthread_t *t = curthread;
903 912 klwp_t *lwp = ttolwp(t);
904 913 pcb_t *pcb = &lwp->lwp_pcb;
905 914 greg_t sp;
906 915
907 916 /*
908 917 * Initialize user registers
909 918 */
910 919 (void) save_syscall_args(); /* copy args from registers first */
911 920 rp = lwptoregs(lwp);
912 921 sp = rp->r_sp;
913 922 bzero(rp, sizeof (*rp));
914 923
915 924 rp->r_ss = UDS_SEL;
916 925 rp->r_sp = sp;
917 926 rp->r_pc = args->entry;
918 927 rp->r_ps = PSL_USER;
919 928
920 929 #if defined(__amd64)
921 930
922 931 pcb->pcb_fs = pcb->pcb_gs = 0;
923 932 pcb->pcb_fsbase = pcb->pcb_gsbase = 0;
924 933
925 934 if (ttoproc(t)->p_model == DATAMODEL_NATIVE) {
926 935
927 936 rp->r_cs = UCS_SEL;
928 937
929 938 /*
930 939 * Only allow 64-bit user code descriptor to be present.
931 940 */
932 941 gdt_ucode_model(DATAMODEL_NATIVE);
933 942
934 943 /*
935 944 * Arrange that the virtualized %fs and %gs GDT descriptors
936 945 * have a well-defined initial state (present, ring 3
937 946 * and of type data).
938 947 */
939 948 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
940 949
941 950 /*
942 951 * thrptr is either NULL or a value used by DTrace.
943 952 * 64-bit processes use %fs as their "thread" register.
944 953 */
945 954 if (args->thrptr)
946 955 (void) lwp_setprivate(lwp, _LWP_FSBASE, args->thrptr);
947 956
948 957 } else {
949 958
950 959 rp->r_cs = U32CS_SEL;
951 960 rp->r_ds = rp->r_es = UDS_SEL;
952 961
953 962 /*
954 963 * only allow 32-bit user code selector to be present.
955 964 */
956 965 gdt_ucode_model(DATAMODEL_ILP32);
957 966
958 967 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
959 968
960 969 /*
961 970 * thrptr is either NULL or a value used by DTrace.
962 971 * 32-bit processes use %gs as their "thread" register.
963 972 */
964 973 if (args->thrptr)
965 974 (void) lwp_setprivate(lwp, _LWP_GSBASE, args->thrptr);
966 975
967 976 }
968 977
969 978 pcb->pcb_ds = rp->r_ds;
970 979 pcb->pcb_es = rp->r_es;
971 980 PCB_SET_UPDATE_SEGS(pcb);
972 981
973 982 #elif defined(__i386)
974 983
975 984 rp->r_cs = UCS_SEL;
976 985 rp->r_ds = rp->r_es = UDS_SEL;
977 986
978 987 /*
979 988 * Arrange that the virtualized %fs and %gs GDT descriptors
980 989 * have a well-defined initial state (present, ring 3
981 990 * and of type data).
982 991 */
983 992 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
984 993
985 994 /*
986 995 * For %gs we need to reset LWP_GSBASE in pcb and the
987 996 * per-cpu GDT descriptor. thrptr is either NULL
988 997 * or a value used by DTrace.
989 998 */
990 999 if (args->thrptr)
991 1000 (void) lwp_setprivate(lwp, _LWP_GSBASE, args->thrptr);
992 1001 #endif
993 1002
994 1003 lwp->lwp_eosys = JUSTRETURN;
995 1004 t->t_post_sys = 1;
996 1005
997 1006 /*
998 1007 * Add the lwp context handlers that virtualize segment registers,
999 1008 * and/or system call stacks etc.
1000 1009 */
1001 1010 lwp_installctx(lwp);
1002 1011
1003 1012 /*
1004 1013 * Reset the FPU flags and then initialize the FPU for this lwp.
1005 1014 */
1006 1015 fp_exec();
1007 1016 }
1008 1017
1009 1018 user_desc_t *
1010 1019 cpu_get_gdt(void)
1011 1020 {
1012 1021 return (CPU->cpu_gdt);
1013 1022 }
1014 1023
1015 1024
1016 1025 #if !defined(lwp_getdatamodel)
1017 1026
1018 1027 /*
1019 1028 * Return the datamodel of the given lwp.
1020 1029 */
1021 1030 /*ARGSUSED*/
1022 1031 model_t
1023 1032 lwp_getdatamodel(klwp_t *lwp)
1024 1033 {
1025 1034 return (lwp->lwp_procp->p_model);
1026 1035 }
1027 1036
1028 1037 #endif /* !lwp_getdatamodel */
1029 1038
1030 1039 #if !defined(get_udatamodel)
1031 1040
1032 1041 model_t
1033 1042 get_udatamodel(void)
1034 1043 {
1035 1044 return (curproc->p_model);
1036 1045 }
1037 1046
1038 1047 #endif /* !get_udatamodel */
|
↓ open down ↓ |
148 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX