1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 /* All Rights Reserved */
27 /*
28 * Copyright (c) 2018, Joyent, Inc.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright 2023 Oxide Computer Company
31 */
32
33 #include <sys/param.h>
34 #include <sys/types.h>
35 #include <sys/vmparam.h>
36 #include <sys/systm.h>
37 #include <sys/signal.h>
38 #include <sys/stack.h>
39 #include <sys/regset.h>
40 #include <sys/privregs.h>
41 #include <sys/frame.h>
42 #include <sys/proc.h>
43 #include <sys/psw.h>
44 #include <sys/siginfo.h>
45 #include <sys/cpuvar.h>
46 #include <sys/asm_linkage.h>
47 #include <sys/kmem.h>
48 #include <sys/errno.h>
49 #include <sys/bootconf.h>
50 #include <sys/archsystm.h>
51 #include <sys/debug.h>
52 #include <sys/elf.h>
53 #include <sys/spl.h>
54 #include <sys/time.h>
55 #include <sys/atomic.h>
56 #include <sys/sysmacros.h>
57 #include <sys/cmn_err.h>
58 #include <sys/modctl.h>
59 #include <sys/kobj.h>
60 #include <sys/panic.h>
61 #include <sys/reboot.h>
62 #include <sys/time.h>
63 #include <sys/fp.h>
64 #include <sys/x86_archext.h>
65 #include <sys/auxv.h>
66 #include <sys/auxv_386.h>
67 #include <sys/dtrace.h>
68 #include <sys/brand.h>
69 #include <sys/machbrand.h>
70 #include <sys/cmn_err.h>
71
72 /*
73 * Map an fnsave-formatted save area into an fxsave-formatted save area.
74 *
75 * Most fields are the same width, content and semantics. However
76 * the tag word is compressed.
77 */
78 static void
79 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx)
80 {
81 uint_t i, tagbits;
82
83 fx->fx_fcw = fn->f_fcw;
84 fx->fx_fsw = fn->f_fsw;
85
86 /*
87 * copy element by element (because of holes)
88 */
89 for (i = 0; i < 8; i++)
90 bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0],
91 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
92
93 /*
94 * synthesize compressed tag bits
95 */
96 fx->fx_fctw = 0;
97 for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2)
98 if ((tagbits & 3) != 3)
99 fx->fx_fctw |= (1 << i);
100
101 fx->fx_fop = fn->f_fop;
102
103 fx->fx_rip = (uint64_t)fn->f_eip;
104 fx->fx_rdp = (uint64_t)fn->f_dp;
105 }
106
107 /*
108 * Map from an fxsave-format save area to an fnsave-format save area.
109 */
110 static void
111 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn)
112 {
113 uint_t i, top, tagbits;
114
115 fn->f_fcw = fx->fx_fcw;
116 fn->__f_ign0 = 0;
117 fn->f_fsw = fx->fx_fsw;
118 fn->__f_ign1 = 0;
119
120 top = (fx->fx_fsw & FPS_TOP) >> 11;
121
122 /*
123 * copy element by element (because of holes)
124 */
125 for (i = 0; i < 8; i++)
126 bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0],
127 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
128
129 /*
130 * synthesize uncompressed tag bits
131 */
132 fn->f_ftw = 0;
133 for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) {
134 uint_t ibit, expo;
135 const uint16_t *fpp;
136 static const uint16_t zero[5] = { 0, 0, 0, 0, 0 };
137
138 if ((tagbits & 1) == 0) {
139 fn->f_ftw |= 3 << (i << 1); /* empty */
140 continue;
141 }
142
143 /*
144 * (tags refer to *physical* registers)
145 */
146 fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0];
147 ibit = fpp[3] >> 15;
148 expo = fpp[4] & 0x7fff;
149
150 if (ibit && expo != 0 && expo != 0x7fff)
151 continue; /* valid fp number */
152
153 if (bcmp(fpp, &zero, sizeof (zero)))
154 fn->f_ftw |= 2 << (i << 1); /* NaN */
155 else
156 fn->f_ftw |= 1 << (i << 1); /* fp zero */
157 }
158
159 fn->f_fop = fx->fx_fop;
160
161 fn->__f_ign2 = 0;
162 fn->f_eip = (uint32_t)fx->fx_rip;
163 fn->f_cs = U32CS_SEL;
164 fn->f_dp = (uint32_t)fx->fx_rdp;
165 fn->f_ds = UDS_SEL;
166 fn->__f_ign3 = 0;
167 }
168
169 /*
170 * Map from an fpregset_t into an fxsave-format save area
171 */
172 static void
173 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx)
174 {
175 bcopy(fp, fx, sizeof (*fx));
176 /*
177 * avoid useless #gp exceptions - mask reserved bits
178 */
179 fx->fx_mxcsr &= sse_mxcsr_mask;
180 }
181
182 /*
183 * Map from an fxsave-format save area into a fpregset_t
184 */
185 static void
186 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp)
187 {
188 bcopy(fx, fp, sizeof (*fx));
189 }
190
191 #if defined(_SYSCALL32_IMPL)
192 static void
193 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx)
194 {
195 const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
196
197 fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
198 /*
199 * avoid useless #gp exceptions - mask reserved bits
200 */
201 fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr;
202 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
203 }
204
205 static void
206 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp)
207 {
208 struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
209
210 fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
211 fc->mxcsr = fx->fx_mxcsr;
212 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
213 }
214
215 static void
216 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst)
217 {
218 fxsave_to_fpregset32((struct fxsave_state *)src, dst);
219 dst->fp_reg_set.fpchip_state.status =
220 src->fp_reg_set.fpchip_state.status;
221 dst->fp_reg_set.fpchip_state.xstatus =
222 src->fp_reg_set.fpchip_state.xstatus;
223 }
224
225 static void
226 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst)
227 {
228 fpregset32_to_fxsave(src, (struct fxsave_state *)dst);
229 dst->fp_reg_set.fpchip_state.status =
230 src->fp_reg_set.fpchip_state.status;
231 dst->fp_reg_set.fpchip_state.xstatus =
232 src->fp_reg_set.fpchip_state.xstatus;
233 }
234 #endif
235
236 /*
237 * Set floating-point registers from a native fpregset_t.
238 */
239 void
240 setfpregs(klwp_t *lwp, fpregset_t *fp)
241 {
242 fpu_set_fpregset(lwp, fp);
243 }
244
245 /*
246 * Get floating-point registers into a native fpregset_t.
247 */
248 void
249 getfpregs(klwp_t *lwp, fpregset_t *fp)
250 {
251 bzero(fp, sizeof (*fp));
252 fpu_get_fpregset(lwp, fp);
253 }
254
255 #if defined(_SYSCALL32_IMPL)
256
257 /*
258 * Set floating-point registers from an fpregset32_t.
259 */
260 void
261 setfpregs32(klwp_t *lwp, fpregset32_t *fp)
262 {
263 fpregset_t fpregs;
264
265 fpregset_32ton(fp, &fpregs);
266 setfpregs(lwp, &fpregs);
267 }
268
269 /*
270 * Get floating-point registers into an fpregset32_t.
271 */
272 void
273 getfpregs32(klwp_t *lwp, fpregset32_t *fp)
274 {
275 fpregset_t fpregs;
276
277 getfpregs(lwp, &fpregs);
278 fpregset_nto32(&fpregs, fp);
279 }
280
281 #endif /* _SYSCALL32_IMPL */
282
283 /*
284 * Return the general registers
285 */
286 void
287 getgregs(klwp_t *lwp, gregset_t grp)
288 {
289 struct regs *rp = lwptoregs(lwp);
290 struct pcb *pcb = &lwp->lwp_pcb;
291 int thisthread = lwptot(lwp) == curthread;
292
293 grp[REG_RDI] = rp->r_rdi;
294 grp[REG_RSI] = rp->r_rsi;
295 grp[REG_RDX] = rp->r_rdx;
296 grp[REG_RCX] = rp->r_rcx;
297 grp[REG_R8] = rp->r_r8;
298 grp[REG_R9] = rp->r_r9;
299 grp[REG_RAX] = rp->r_rax;
300 grp[REG_RBX] = rp->r_rbx;
301 grp[REG_RBP] = rp->r_rbp;
302 grp[REG_R10] = rp->r_r10;
303 grp[REG_R11] = rp->r_r11;
304 grp[REG_R12] = rp->r_r12;
305 grp[REG_R13] = rp->r_r13;
306 grp[REG_R14] = rp->r_r14;
307 grp[REG_R15] = rp->r_r15;
308 grp[REG_FSBASE] = pcb->pcb_fsbase;
309 grp[REG_GSBASE] = pcb->pcb_gsbase;
310 if (thisthread)
311 kpreempt_disable();
312 if (PCB_NEED_UPDATE_SEGS(pcb)) {
313 grp[REG_DS] = pcb->pcb_ds;
314 grp[REG_ES] = pcb->pcb_es;
315 grp[REG_FS] = pcb->pcb_fs;
316 grp[REG_GS] = pcb->pcb_gs;
317 } else {
318 grp[REG_DS] = rp->r_ds;
319 grp[REG_ES] = rp->r_es;
320 grp[REG_FS] = rp->r_fs;
321 grp[REG_GS] = rp->r_gs;
322 }
323 if (thisthread)
324 kpreempt_enable();
325 grp[REG_TRAPNO] = rp->r_trapno;
326 grp[REG_ERR] = rp->r_err;
327 grp[REG_RIP] = rp->r_rip;
328 grp[REG_CS] = rp->r_cs;
329 grp[REG_SS] = rp->r_ss;
330 grp[REG_RFL] = rp->r_rfl;
331 grp[REG_RSP] = rp->r_rsp;
332 }
333
334 #if defined(_SYSCALL32_IMPL)
335
336 void
337 getgregs32(klwp_t *lwp, gregset32_t grp)
338 {
339 struct regs *rp = lwptoregs(lwp);
340 struct pcb *pcb = &lwp->lwp_pcb;
341 int thisthread = lwptot(lwp) == curthread;
342
343 if (thisthread)
344 kpreempt_disable();
345 if (PCB_NEED_UPDATE_SEGS(pcb)) {
346 grp[GS] = (uint16_t)pcb->pcb_gs;
347 grp[FS] = (uint16_t)pcb->pcb_fs;
348 grp[DS] = (uint16_t)pcb->pcb_ds;
349 grp[ES] = (uint16_t)pcb->pcb_es;
350 } else {
351 grp[GS] = (uint16_t)rp->r_gs;
352 grp[FS] = (uint16_t)rp->r_fs;
353 grp[DS] = (uint16_t)rp->r_ds;
354 grp[ES] = (uint16_t)rp->r_es;
355 }
356 if (thisthread)
357 kpreempt_enable();
358 grp[EDI] = (greg32_t)rp->r_rdi;
359 grp[ESI] = (greg32_t)rp->r_rsi;
360 grp[EBP] = (greg32_t)rp->r_rbp;
361 grp[ESP] = 0;
362 grp[EBX] = (greg32_t)rp->r_rbx;
363 grp[EDX] = (greg32_t)rp->r_rdx;
364 grp[ECX] = (greg32_t)rp->r_rcx;
365 grp[EAX] = (greg32_t)rp->r_rax;
366 grp[TRAPNO] = (greg32_t)rp->r_trapno;
367 grp[ERR] = (greg32_t)rp->r_err;
368 grp[EIP] = (greg32_t)rp->r_rip;
369 grp[CS] = (uint16_t)rp->r_cs;
370 grp[EFL] = (greg32_t)rp->r_rfl;
371 grp[UESP] = (greg32_t)rp->r_rsp;
372 grp[SS] = (uint16_t)rp->r_ss;
373 }
374
375 void
376 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst)
377 {
378 mcontext_t *dmc = &dst->uc_mcontext;
379 const mcontext32_t *smc = &src->uc_mcontext;
380
381 bzero(dst, sizeof (*dst));
382 dst->uc_flags = src->uc_flags;
383 dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link;
384
385 bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask));
386
387 dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp;
388 dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size;
389 dst->uc_stack.ss_flags = src->uc_stack.ss_flags;
390
391 dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS];
392 dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS];
393 dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES];
394 dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS];
395 dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI];
396 dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI];
397 dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP];
398 dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX];
399 dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX];
400 dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX];
401 dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX];
402 dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO];
403 dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR];
404 dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP];
405 dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS];
406 dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL];
407 dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP];
408 dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS];
409
410 /*
411 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set
412 * otherwise there is no guarantee that anything in fpregs is valid.
413 */
414 if (src->uc_flags & UC_FPU)
415 fpregset_32ton(&src->uc_mcontext.fpregs,
416 &dst->uc_mcontext.fpregs);
417
418 /*
419 * Copy the brand-private data:
420 */
421 dst->uc_brand_data[0] = (void *)(uintptr_t)src->uc_brand_data[0];
422 dst->uc_brand_data[1] = (void *)(uintptr_t)src->uc_brand_data[1];
423 dst->uc_brand_data[2] = (void *)(uintptr_t)src->uc_brand_data[2];
424
425 if (src->uc_flags & UC_XSAVE) {
426 dst->uc_xsave = (long)(uint32_t)src->uc_xsave;
427 } else {
428 dst->uc_xsave = 0;
429 }
430 }
431
432 #endif /* _SYSCALL32_IMPL */
433
434 /*
435 * Return the user-level PC.
436 * If in a system call, return the address of the syscall trap.
437 */
438 greg_t
439 getuserpc(void)
440 {
441 greg_t upc = lwptoregs(ttolwp(curthread))->r_pc;
442 uint32_t insn;
443
444 if (curthread->t_sysnum == 0)
445 return (upc);
446
447 /*
448 * We might've gotten here from sysenter (0xf 0x34),
449 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0).
450 *
451 * Go peek at the binary to figure it out..
452 */
453 if (fuword32((void *)(upc - 2), &insn) != -1 &&
454 (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f)
455 return (upc - 2);
456 return (upc - 7);
457 }
458
459 /*
460 * Protect segment registers from non-user privilege levels and GDT selectors
461 * other than USER_CS, USER_DS and lwp FS and GS values. If the segment
462 * selector is non-null and not USER_CS/USER_DS, we make sure that the
463 * TI bit is set to point into the LDT and that the RPL is set to 3.
464 *
465 * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we
466 * also explicitly zero the top 16 bits since they may be coming from the
467 * user's address space via setcontext(2) or /proc.
468 *
469 * Note about null selector. When running on the hypervisor if we allow a
470 * process to set its %cs to null selector with RPL of 0 the hypervisor will
471 * crash the domain. If running on bare metal we would get a #gp fault and
472 * be able to kill the process and continue on. Therefore we make sure to
473 * force RPL to SEL_UPL even for null selector when setting %cs.
474 */
475
476 #if defined(IS_CS) || defined(IS_NOT_CS)
477 #error "IS_CS and IS_NOT_CS already defined"
478 #endif
479
480 #define IS_CS 1
481 #define IS_NOT_CS 0
482
483 /*ARGSUSED*/
484 greg_t
485 fix_segreg(greg_t sr, int iscs, model_t datamodel)
486 {
487 kthread_t *t = curthread;
488
489 switch (sr &= 0xffff) {
490
491 case 0:
492 if (iscs == IS_CS)
493 return (0 | SEL_UPL);
494 else
495 return (0);
496
497 /*
498 * If lwp attempts to switch data model then force their
499 * code selector to be null selector.
500 */
501 case U32CS_SEL:
502 if (datamodel == DATAMODEL_NATIVE)
503 return (0 | SEL_UPL);
504 else
505 return (sr);
506
507 case UCS_SEL:
508 if (datamodel == DATAMODEL_ILP32)
509 return (0 | SEL_UPL);
510 /*FALLTHROUGH*/
511 case UDS_SEL:
512 case LWPFS_SEL:
513 case LWPGS_SEL:
514 case SEL_UPL:
515 return (sr);
516 default:
517 break;
518 }
519
520 /*
521 * Allow this process's brand to do any necessary segment register
522 * manipulation.
523 */
524 if (PROC_IS_BRANDED(t->t_procp) && BRMOP(t->t_procp)->b_fixsegreg) {
525 greg_t bsr = BRMOP(t->t_procp)->b_fixsegreg(sr, datamodel);
526
527 if (bsr == 0 && iscs == IS_CS)
528 return (0 | SEL_UPL);
529 else
530 return (bsr);
531 }
532
533 /*
534 * Force it into the LDT in ring 3 for 32-bit processes, which by
535 * default do not have an LDT, so that any attempt to use an invalid
536 * selector will reference the (non-existant) LDT, and cause a #gp
537 * fault for the process.
538 *
539 * 64-bit processes get the null gdt selector since they
540 * are not allowed to have a private LDT.
541 */
542 if (datamodel == DATAMODEL_ILP32) {
543 return (sr | SEL_TI_LDT | SEL_UPL);
544 } else {
545 if (iscs == IS_CS)
546 return (0 | SEL_UPL);
547 else
548 return (0);
549 }
550
551 }
552
553 /*
554 * Set general registers.
555 */
556 void
557 setgregs(klwp_t *lwp, gregset_t grp)
558 {
559 struct regs *rp = lwptoregs(lwp);
560 model_t datamodel = lwp_getdatamodel(lwp);
561
562 struct pcb *pcb = &lwp->lwp_pcb;
563 int thisthread = lwptot(lwp) == curthread;
564
565 if (datamodel == DATAMODEL_NATIVE) {
566 if (thisthread)
567 (void) save_syscall_args(); /* copy the args */
568
569 rp->r_rdi = grp[REG_RDI];
570 rp->r_rsi = grp[REG_RSI];
571 rp->r_rdx = grp[REG_RDX];
572 rp->r_rcx = grp[REG_RCX];
573 rp->r_r8 = grp[REG_R8];
574 rp->r_r9 = grp[REG_R9];
575 rp->r_rax = grp[REG_RAX];
576 rp->r_rbx = grp[REG_RBX];
577 rp->r_rbp = grp[REG_RBP];
578 rp->r_r10 = grp[REG_R10];
579 rp->r_r11 = grp[REG_R11];
580 rp->r_r12 = grp[REG_R12];
581 rp->r_r13 = grp[REG_R13];
582 rp->r_r14 = grp[REG_R14];
583 rp->r_r15 = grp[REG_R15];
584 rp->r_trapno = grp[REG_TRAPNO];
585 rp->r_err = grp[REG_ERR];
586 rp->r_rip = grp[REG_RIP];
587 /*
588 * Setting %cs or %ss to anything else is quietly but
589 * quite definitely forbidden!
590 */
591 rp->r_cs = UCS_SEL;
592 rp->r_ss = UDS_SEL;
593 rp->r_rsp = grp[REG_RSP];
594
595 if (thisthread)
596 kpreempt_disable();
597
598 pcb->pcb_ds = UDS_SEL;
599 pcb->pcb_es = UDS_SEL;
600
601 /*
602 * 64-bit processes -are- allowed to set their fsbase/gsbase
603 * values directly, but only if they're using the segment
604 * selectors that allow that semantic.
605 *
606 * (32-bit processes must use lwp_set_private().)
607 */
608 pcb->pcb_fsbase = grp[REG_FSBASE];
609 pcb->pcb_gsbase = grp[REG_GSBASE];
610 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
611 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
612
613 /*
614 * Ensure that we go out via update_sregs
615 */
616 PCB_SET_UPDATE_SEGS(pcb);
617 lwptot(lwp)->t_post_sys = 1;
618 if (thisthread)
619 kpreempt_enable();
620 #if defined(_SYSCALL32_IMPL)
621 } else {
622 rp->r_rdi = (uint32_t)grp[REG_RDI];
623 rp->r_rsi = (uint32_t)grp[REG_RSI];
624 rp->r_rdx = (uint32_t)grp[REG_RDX];
625 rp->r_rcx = (uint32_t)grp[REG_RCX];
626 rp->r_rax = (uint32_t)grp[REG_RAX];
627 rp->r_rbx = (uint32_t)grp[REG_RBX];
628 rp->r_rbp = (uint32_t)grp[REG_RBP];
629 rp->r_trapno = (uint32_t)grp[REG_TRAPNO];
630 rp->r_err = (uint32_t)grp[REG_ERR];
631 rp->r_rip = (uint32_t)grp[REG_RIP];
632
633 rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel);
634 rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
635
636 rp->r_rsp = (uint32_t)grp[REG_RSP];
637
638 if (thisthread)
639 kpreempt_disable();
640
641 pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
642 pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel);
643
644 /*
645 * (See fsbase/gsbase commentary above)
646 */
647 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
648 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
649
650 /*
651 * Ensure that we go out via update_sregs
652 */
653 PCB_SET_UPDATE_SEGS(pcb);
654 lwptot(lwp)->t_post_sys = 1;
655 if (thisthread)
656 kpreempt_enable();
657 #endif
658 }
659
660 /*
661 * Only certain bits of the flags register can be modified.
662 */
663 rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) |
664 (grp[REG_RFL] & PSL_USERMASK);
665 }
666
667 /*
668 * Determine whether eip is likely to have an interrupt frame
669 * on the stack. We do this by comparing the address to the
670 * range of addresses spanned by several well-known routines.
671 */
672 extern void _interrupt();
673 extern void _allsyscalls();
674 extern void _cmntrap();
675 extern void fakesoftint();
676
677 extern size_t _interrupt_size;
678 extern size_t _allsyscalls_size;
679 extern size_t _cmntrap_size;
680 extern size_t _fakesoftint_size;
681
682 /*
683 * Get a pc-only stacktrace. Used for kmem_alloc() buffer ownership tracking.
684 * Returns MIN(current stack depth, pcstack_limit).
685 */
686 int
687 getpcstack(pc_t *pcstack, int pcstack_limit)
688 {
689 struct frame *fp = (struct frame *)getfp();
690 struct frame *nextfp, *minfp, *stacktop;
691 int depth = 0;
692 int on_intr;
693 uintptr_t pc;
694
695 if ((on_intr = CPU_ON_INTR(CPU)) != 0)
696 stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
697 else
698 stacktop = (struct frame *)curthread->t_stk;
699 minfp = fp;
700
701 pc = ((struct regs *)fp)->r_pc;
702
703 while (depth < pcstack_limit) {
704 nextfp = (struct frame *)fp->fr_savfp;
705 pc = fp->fr_savpc;
706 if (nextfp <= minfp || nextfp >= stacktop) {
707 if (on_intr) {
708 /*
709 * Hop from interrupt stack to thread stack.
710 */
711 stacktop = (struct frame *)curthread->t_stk;
712 minfp = (struct frame *)curthread->t_stkbase;
713 on_intr = 0;
714 continue;
715 }
716 break;
717 }
718 pcstack[depth++] = (pc_t)pc;
719 fp = nextfp;
720 minfp = fp;
721 }
722 return (depth);
723 }
724
725 /*
726 * The following ELF header fields are defined as processor-specific
727 * in the V8 ABI:
728 *
729 * e_ident[EI_DATA] encoding of the processor-specific
730 * data in the object file
731 * e_machine processor identification
732 * e_flags processor-specific flags associated
733 * with the file
734 */
735
736 /*
737 * The value of at_flags reflects a platform's cpu module support.
738 * at_flags is used to check for allowing a binary to execute and
739 * is passed as the value of the AT_FLAGS auxiliary vector.
740 */
741 int at_flags = 0;
742
743 /*
744 * Check the processor-specific fields of an ELF header.
745 *
746 * returns 1 if the fields are valid, 0 otherwise
747 */
748 /*ARGSUSED2*/
749 int
750 elfheadcheck(
751 unsigned char e_data,
752 Elf32_Half e_machine,
753 Elf32_Word e_flags)
754 {
755 if (e_data != ELFDATA2LSB)
756 return (0);
757 if (e_machine == EM_AMD64)
758 return (1);
759 return (e_machine == EM_386);
760 }
761
762 uint_t auxv_hwcap_include = 0; /* patch to enable unrecognized features */
763 uint_t auxv_hwcap_include_2 = 0; /* second word */
764 uint_t auxv_hwcap_exclude = 0; /* patch for broken cpus, debugging */
765 uint_t auxv_hwcap_exclude_2 = 0; /* second word */
766 #if defined(_SYSCALL32_IMPL)
767 uint_t auxv_hwcap32_include = 0; /* ditto for 32-bit apps */
768 uint_t auxv_hwcap32_include_2 = 0; /* ditto for 32-bit apps */
769 uint_t auxv_hwcap32_exclude = 0; /* ditto for 32-bit apps */
770 uint_t auxv_hwcap32_exclude_2 = 0; /* ditto for 32-bit apps */
771 #endif
772
773 /*
774 * Gather information about the processor and place it into auxv_hwcap
775 * so that it can be exported to the linker via the aux vector.
776 *
777 * We use this seemingly complicated mechanism so that we can ensure
778 * that /etc/system can be used to override what the system can or
779 * cannot discover for itself. Due to a lack of use, this has not
780 * been extended to the 3rd word.
781 */
782 void
783 bind_hwcap(void)
784 {
785 uint_t cpu_hwcap_flags[3];
786 cpuid_execpass(NULL, CPUID_PASS_RESOLVE, cpu_hwcap_flags);
787
788 auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags[0]) &
789 ~auxv_hwcap_exclude;
790 auxv_hwcap_2 = (auxv_hwcap_include_2 | cpu_hwcap_flags[1]) &
791 ~auxv_hwcap_exclude_2;
792 auxv_hwcap_3 = cpu_hwcap_flags[2];
793
794 /*
795 * On AMD processors, sysenter just doesn't work at all
796 * when the kernel is in long mode. On IA-32e processors
797 * it does, but there's no real point in all the alternate
798 * mechanism when syscall works on both.
799 *
800 * Besides, the kernel's sysenter handler is expecting a
801 * 32-bit lwp ...
802 */
803 auxv_hwcap &= ~AV_386_SEP;
804
805 if (auxv_hwcap_include || auxv_hwcap_exclude || auxv_hwcap_include_2 ||
806 auxv_hwcap_exclude_2) {
807 /*
808 * The below assignment is regrettably required to get lint
809 * to accept the validity of our format string. The format
810 * string is in fact valid, but whatever intelligence in lint
811 * understands the cmn_err()-specific %b appears to have an
812 * off-by-one error: it (mistakenly) complains about bit
813 * number 32 (even though this is explicitly permitted).
814 * Normally, one would will away such warnings with a "LINTED"
815 * directive, but for reasons unclear and unknown, lint
816 * refuses to be assuaged in this case. Fortunately, lint
817 * doesn't pretend to have solved the Halting Problem --
818 * and as soon as the format string is programmatic, it
819 * knows enough to shut up.
820 */
821 char *fmt = "?user ABI extensions: %b\n";
822 cmn_err(CE_CONT, fmt, auxv_hwcap, FMT_AV_386);
823 fmt = "?user ABI extensions (word 2): %b\n";
824 cmn_err(CE_CONT, fmt, auxv_hwcap_2, FMT_AV_386_2);
825 fmt = "?user ABI extensions (word 2): %b\n";
826 cmn_err(CE_CONT, fmt, auxv_hwcap_3, FMT_AV_386_3);
827 }
828
829 #if defined(_SYSCALL32_IMPL)
830 auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags[0]) &
831 ~auxv_hwcap32_exclude;
832 auxv_hwcap32_2 = (auxv_hwcap32_include_2 | cpu_hwcap_flags[1]) &
833 ~auxv_hwcap32_exclude_2;
834 auxv_hwcap32_3 = auxv_hwcap_3;
835
836 /*
837 * If this is an amd64 architecture machine from Intel, then
838 * syscall -doesn't- work in compatibility mode, only sysenter does.
839 *
840 * Sigh.
841 */
842 if (!cpuid_syscall32_insn(NULL))
843 auxv_hwcap32 &= ~AV_386_AMD_SYSC;
844
845 /*
846 * 32-bit processes can -always- use the lahf/sahf instructions
847 */
848 auxv_hwcap32 |= AV_386_AHF;
849
850 /*
851 * 32-bit processes can -never- use fsgsbase instructions.
852 */
853 auxv_hwcap32_2 &= ~AV_386_2_FSGSBASE;
854
855 if (auxv_hwcap32_include || auxv_hwcap32_exclude ||
856 auxv_hwcap32_include_2 || auxv_hwcap32_exclude_2) {
857 /*
858 * See the block comment in the cmn_err() of auxv_hwcap, above.
859 */
860 char *fmt = "?32-bit user ABI extensions: %b\n";
861 cmn_err(CE_CONT, fmt, auxv_hwcap32, FMT_AV_386);
862 fmt = "?32-bit user ABI extensions (word 2): %b\n";
863 cmn_err(CE_CONT, fmt, auxv_hwcap32_2, FMT_AV_386_2);
864 fmt = "?32-bit user ABI extensions (word 3): %b\n";
865 cmn_err(CE_CONT, fmt, auxv_hwcap32_3, FMT_AV_386_3);
866 }
867 #endif
868 }
869
870 /*
871 * sync_icache() - this is called
872 * in proc/fs/prusrio.c. x86 has an unified cache and therefore
873 * this is a nop.
874 */
875 /* ARGSUSED */
876 void
877 sync_icache(caddr_t addr, uint_t len)
878 {
879 /* Do nothing for now */
880 }
881
882 /*ARGSUSED*/
883 void
884 sync_data_memory(caddr_t va, size_t len)
885 {
886 /* Not implemented for this platform */
887 }
888
889 int
890 __ipltospl(int ipl)
891 {
892 return (ipltospl(ipl));
893 }
894
895 /*
896 * The panic code invokes panic_saveregs() to record the contents of a
897 * regs structure into the specified panic_data structure for debuggers.
898 */
899 void
900 panic_saveregs(panic_data_t *pdp, struct regs *rp)
901 {
902 panic_nv_t *pnv = PANICNVGET(pdp);
903
904 struct cregs creg;
905
906 getcregs(&creg);
907
908 PANICNVADD(pnv, "rdi", rp->r_rdi);
909 PANICNVADD(pnv, "rsi", rp->r_rsi);
910 PANICNVADD(pnv, "rdx", rp->r_rdx);
911 PANICNVADD(pnv, "rcx", rp->r_rcx);
912 PANICNVADD(pnv, "r8", rp->r_r8);
913 PANICNVADD(pnv, "r9", rp->r_r9);
914 PANICNVADD(pnv, "rax", rp->r_rax);
915 PANICNVADD(pnv, "rbx", rp->r_rbx);
916 PANICNVADD(pnv, "rbp", rp->r_rbp);
917 PANICNVADD(pnv, "r10", rp->r_r10);
918 PANICNVADD(pnv, "r11", rp->r_r11);
919 PANICNVADD(pnv, "r12", rp->r_r12);
920 PANICNVADD(pnv, "r13", rp->r_r13);
921 PANICNVADD(pnv, "r14", rp->r_r14);
922 PANICNVADD(pnv, "r15", rp->r_r15);
923 PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE));
924 PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE));
925 PANICNVADD(pnv, "ds", rp->r_ds);
926 PANICNVADD(pnv, "es", rp->r_es);
927 PANICNVADD(pnv, "fs", rp->r_fs);
928 PANICNVADD(pnv, "gs", rp->r_gs);
929 PANICNVADD(pnv, "trapno", rp->r_trapno);
930 PANICNVADD(pnv, "err", rp->r_err);
931 PANICNVADD(pnv, "rip", rp->r_rip);
932 PANICNVADD(pnv, "cs", rp->r_cs);
933 PANICNVADD(pnv, "rflags", rp->r_rfl);
934 PANICNVADD(pnv, "rsp", rp->r_rsp);
935 PANICNVADD(pnv, "ss", rp->r_ss);
936 PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3]));
937 PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0]));
938 PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3]));
939 PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0]));
940
941 PANICNVADD(pnv, "ldt", creg.cr_ldt);
942 PANICNVADD(pnv, "task", creg.cr_task);
943 PANICNVADD(pnv, "cr0", creg.cr_cr0);
944 PANICNVADD(pnv, "cr2", creg.cr_cr2);
945 PANICNVADD(pnv, "cr3", creg.cr_cr3);
946 if (creg.cr_cr4)
947 PANICNVADD(pnv, "cr4", creg.cr_cr4);
948
949 PANICNVSET(pdp, pnv);
950 }
951
952 #define TR_ARG_MAX 6 /* Max args to print, same as SPARC */
953
954
955 /*
956 * Print a stack backtrace using the specified frame pointer. We delay two
957 * seconds before continuing, unless this is the panic traceback.
958 * If we are in the process of panicking, we also attempt to write the
959 * stack backtrace to a staticly assigned buffer, to allow the panic
960 * code to find it and write it in to uncompressed pages within the
961 * system crash dump.
962 * Note that the frame for the starting stack pointer value is omitted because
963 * the corresponding %eip is not known.
964 */
965
966 extern char *dump_stack_scratch;
967
968
969 void
970 traceback(caddr_t fpreg)
971 {
972 struct frame *fp = (struct frame *)fpreg;
973 struct frame *nextfp;
974 uintptr_t pc, nextpc;
975 ulong_t off;
976 char args[TR_ARG_MAX * 2 + 16], *sym;
977 uint_t offset = 0;
978 uint_t next_offset = 0;
979 char stack_buffer[1024];
980
981 if (!panicstr)
982 printf("traceback: %%fp = %p\n", (void *)fp);
983
984 if (panicstr && !dump_stack_scratch) {
985 printf("Warning - stack not written to the dump buffer\n");
986 }
987
988 fp = (struct frame *)plat_traceback(fpreg);
989 if ((uintptr_t)fp < KERNELBASE)
990 goto out;
991
992 pc = fp->fr_savpc;
993 fp = (struct frame *)fp->fr_savfp;
994
995 while ((uintptr_t)fp >= KERNELBASE) {
996 /*
997 * XX64 Until port is complete tolerate 8-byte aligned
998 * frame pointers but flag with a warning so they can
999 * be fixed.
1000 */
1001 if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) {
1002 if (((uintptr_t)fp & (8 - 1)) == 0) {
1003 printf(" >> warning! 8-byte"
1004 " aligned %%fp = %p\n", (void *)fp);
1005 } else {
1006 printf(
1007 " >> mis-aligned %%fp = %p\n", (void *)fp);
1008 break;
1009 }
1010 }
1011
1012 args[0] = '\0';
1013 nextpc = (uintptr_t)fp->fr_savpc;
1014 nextfp = (struct frame *)fp->fr_savfp;
1015 if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1016 printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1017 mod_containing_pc((caddr_t)pc), sym, off, args);
1018 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1019 "%s:%s+%lx (%s) | ",
1020 mod_containing_pc((caddr_t)pc), sym, off, args);
1021 } else {
1022 printf("%016lx %lx (%s)\n",
1023 (uintptr_t)fp, pc, args);
1024 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1025 "%lx (%s) | ", pc, args);
1026 }
1027
1028 if (panicstr && dump_stack_scratch) {
1029 next_offset = offset + strlen(stack_buffer);
1030 if (next_offset < STACK_BUF_SIZE) {
1031 bcopy(stack_buffer, dump_stack_scratch + offset,
1032 strlen(stack_buffer));
1033 offset = next_offset;
1034 } else {
1035 /*
1036 * In attempting to save the panic stack
1037 * to the dumpbuf we have overflowed that area.
1038 * Print a warning and continue to printf the
1039 * stack to the msgbuf
1040 */
1041 printf("Warning: stack in the dump buffer"
1042 " may be incomplete\n");
1043 offset = next_offset;
1044 }
1045 }
1046
1047 pc = nextpc;
1048 fp = nextfp;
1049 }
1050 out:
1051 if (!panicstr) {
1052 printf("end of traceback\n");
1053 DELAY(2 * MICROSEC);
1054 } else if (dump_stack_scratch) {
1055 dump_stack_scratch[offset] = '\0';
1056 }
1057 }
1058
1059
1060 /*
1061 * Generate a stack backtrace from a saved register set.
1062 */
1063 void
1064 traceregs(struct regs *rp)
1065 {
1066 traceback((caddr_t)rp->r_fp);
1067 }
1068
1069 void
1070 exec_set_sp(size_t stksize)
1071 {
1072 klwp_t *lwp = ttolwp(curthread);
1073
1074 lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize;
1075 }
1076
1077 hrtime_t
1078 gethrtime_waitfree(void)
1079 {
1080 return (dtrace_gethrtime());
1081 }
1082
1083 hrtime_t
1084 gethrtime(void)
1085 {
1086 return (gethrtimef());
1087 }
1088
1089 hrtime_t
1090 gethrtime_unscaled(void)
1091 {
1092 return (gethrtimeunscaledf());
1093 }
1094
1095 void
1096 scalehrtime(hrtime_t *hrt)
1097 {
1098 scalehrtimef(hrt);
1099 }
1100
1101 uint64_t
1102 unscalehrtime(hrtime_t nsecs)
1103 {
1104 return (unscalehrtimef(nsecs));
1105 }
1106
1107 void
1108 gethrestime(timespec_t *tp)
1109 {
1110 gethrestimef(tp);
1111 }
1112
1113 /*
1114 * Part of the implementation of hres_tick(); this routine is
1115 * easier in C than assembler .. called with the hres_lock held.
1116 *
1117 * XX64 Many of these timekeeping variables need to be extern'ed in a header
1118 */
1119
1120 #include <sys/time.h>
1121 #include <sys/machlock.h>
1122
1123 extern int one_sec;
1124 extern int max_hres_adj;
1125
1126 void
1127 __adj_hrestime(void)
1128 {
1129 long long adj;
1130
1131 if (hrestime_adj == 0)
1132 adj = 0;
1133 else if (hrestime_adj > 0) {
1134 if (hrestime_adj < max_hres_adj)
1135 adj = hrestime_adj;
1136 else
1137 adj = max_hres_adj;
1138 } else {
1139 if (hrestime_adj < -max_hres_adj)
1140 adj = -max_hres_adj;
1141 else
1142 adj = hrestime_adj;
1143 }
1144
1145 timedelta -= adj;
1146 hrestime_adj = timedelta;
1147 hrestime.tv_nsec += adj;
1148
1149 while (hrestime.tv_nsec >= NANOSEC) {
1150 one_sec++;
1151 hrestime.tv_sec++;
1152 hrestime.tv_nsec -= NANOSEC;
1153 }
1154 }
1155
1156 /*
1157 * Wrapper functions to maintain backwards compability
1158 */
1159 int
1160 xcopyin(const void *uaddr, void *kaddr, size_t count)
1161 {
1162 return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED));
1163 }
1164
1165 int
1166 xcopyout(const void *kaddr, void *uaddr, size_t count)
1167 {
1168 return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED));
1169 }