1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2016 Joyent, Inc.
24 */
25
26 #include <sys/asm_linkage.h>
27 #include <sys/asm_misc.h>
28 #include <sys/regset.h>
29 #include <sys/privregs.h>
30 #include <sys/psw.h>
31 #include <sys/machbrand.h>
32
33 #if defined(__lint)
34
35 #include <sys/types.h>
36 #include <sys/thread.h>
37 #include <sys/systm.h>
38
39 #else /* __lint */
40
41 #include <sys/segments.h>
42 #include <sys/pcb.h>
43 #include <sys/trap.h>
44 #include <sys/ftrace.h>
45 #include <sys/traptrace.h>
46 #include <sys/clock.h>
47 #include <sys/model.h>
48 #include <sys/panic.h>
49
50 #if defined(__xpv)
51 #include <sys/hypervisor.h>
52 #endif
53
54 #include "assym.h"
55
56 #endif /* __lint */
57
58 /*
59 * We implement five flavours of system call entry points
60 *
61 * - syscall/sysretq (amd64 generic)
62 * - syscall/sysretl (i386 plus SYSC bit)
63 * - sysenter/sysexit (i386 plus SEP bit)
64 * - int/iret (i386 generic)
65 * - lcall/iret (i386 generic)
66 *
67 * The current libc included in Solaris uses int/iret as the base unoptimized
68 * kernel entry method. Older libc implementations and legacy binaries may use
69 * the lcall call gate, so it must continue to be supported.
70 *
71 * System calls that use an lcall call gate are processed in trap() via a
72 * segment-not-present trap, i.e. lcalls are extremely slow(!).
73 *
74 * The basic pattern used in the 32-bit SYSC handler at this point in time is
75 * to have the bare minimum of assembler, and get to the C handlers as
76 * quickly as possible.
77 *
78 * The 64-bit handler is much closer to the sparcv9 handler; that's
79 * because of passing arguments in registers. The 32-bit world still
80 * passes arguments on the stack -- that makes that handler substantially
81 * more complex.
82 *
83 * The two handlers share a few code fragments which are broken
84 * out into preprocessor macros below.
85 *
86 * XX64 come back and speed all this up later. The 32-bit stuff looks
87 * especially easy to speed up the argument copying part ..
88 *
89 *
90 * Notes about segment register usage (c.f. the 32-bit kernel)
91 *
92 * In the 32-bit kernel, segment registers are dutifully saved and
93 * restored on all mode transitions because the kernel uses them directly.
94 * When the processor is running in 64-bit mode, segment registers are
95 * largely ignored.
96 *
97 * %cs and %ss
98 * controlled by the hardware mechanisms that make mode transitions
99 *
100 * The remaining segment registers have to either be pointing at a valid
101 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
102 *
103 * %ds and %es
104 * always ignored
105 *
106 * %fs and %gs
107 * fsbase and gsbase are used to control the place they really point at.
108 * The kernel only depends on %gs, and controls its own gsbase via swapgs
109 *
110 * Note that loading segment registers is still costly because the GDT
111 * lookup still happens (this is because the hardware can't know that we're
112 * not setting up these segment registers for a 32-bit program). Thus we
113 * avoid doing this in the syscall path, and defer them to lwp context switch
114 * handlers, so the register values remain virtualized to the lwp.
115 */
116
117 #if defined(SYSCALLTRACE)
118 #define ORL_SYSCALLTRACE(r32) \
119 orl syscalltrace(%rip), r32
120 #else
121 #define ORL_SYSCALLTRACE(r32)
122 #endif
123
124 /*
125 * In the 32-bit kernel, we do absolutely nothing before getting into the
126 * brand callback checks. In 64-bit land, we do swapgs and then come here.
127 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
128 * are still unused.
129 *
130 * Check if a brand_mach_ops callback is defined for the specified callback_id
131 * type. If so invoke it with the kernel's %gs value loaded and the following
132 * data on the stack:
133 *
134 * stack: --------------------------------------
135 * 32 | callback pointer |
136 * | 24 | user (or interrupt) stack pointer |
137 * | 16 | lwp pointer |
138 * v 8 | userland return address |
139 * 0 | callback wrapper return addr |
140 * --------------------------------------
141 *
142 * Since we're pushing the userland return address onto the kernel stack
143 * we need to get that address without accessing the user's stack (since we
144 * can't trust that data). There are different ways to get the userland
145 * return address depending on how the syscall trap was made:
146 *
147 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
148 * b) For sys_sysenter the return address is in %rdx.
149 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
150 * the stack pointer points at the state saved when we took the interrupt:
151 * ------------------------
152 * | | user's %ss |
153 * | | user's %esp |
154 * | | EFLAGS register |
155 * v | user's %cs |
156 * | user's %eip |
157 * ------------------------
158 *
159 * The 2nd parameter to the BRAND_CALLBACK macro is either the
160 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are
161 * used to generate the proper code to get the userland return address for
162 * each syscall entry point.
163 *
164 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
165 * is available as a scratch register within the callback. If the callback
166 * returns within the kernel then this macro will restore %r15. If the
167 * callback is going to return directly to userland then it should restore
168 * %r15 before returning to userland.
169 */
170 #define BRAND_URET_FROM_REG(rip_reg) \
171 pushq rip_reg /* push the return address */
172
173 /*
174 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
175 * is currently pointing at the user return address (%eip).
176 */
177 #define BRAND_URET_FROM_INTR_STACK() \
178 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\
179 pushq (%r15) /* push the return address */
180
181 #define BRAND_CALLBACK(callback_id, push_userland_ret) \
182 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\
183 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\
184 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\
185 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\
186 subq $16, %rsp /* save space for 2 pointers */ ;\
187 pushq %r14 /* save %r14 */ ;\
188 movq %gs:CPU_RTMP_RSP, %r14 ;\
189 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\
190 popq %r14 /* restore %r14 */ ;\
191 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\
192 pushq %r15 /* push the lwp pointer */ ;\
193 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\
194 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\
195 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\
196 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\
197 cmpq $0, %r15 ;\
198 je 1f ;\
199 movq %r15, 16(%rsp) /* save the callback pointer */ ;\
200 push_userland_ret /* push the return address */ ;\
201 call *24(%rsp) /* call callback */ ;\
202 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\
203 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */
204
205 #define MSTATE_TRANSITION(from, to) \
206 movl $from, %edi; \
207 movl $to, %esi; \
208 call syscall_mstate
209
210 /*
211 * Check to see if a simple (direct) return is possible i.e.
212 *
213 * if (t->t_post_sys_ast | syscalltrace |
214 * lwp->lwp_pcb.pcb_rupdate == 1)
215 * do full version ;
216 *
217 * Preconditions:
218 * - t is curthread
219 * Postconditions:
220 * - condition code NE is set if post-sys is too complex
221 * - rtmp is zeroed if it isn't (we rely on this!)
222 * - ltmp is smashed
223 */
224 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \
225 movq T_LWP(t), ltmp; \
226 movzbl PCB_RUPDATE(ltmp), rtmp; \
227 ORL_SYSCALLTRACE(rtmp); \
228 orl T_POST_SYS_AST(t), rtmp; \
229 cmpl $0, rtmp
230
231 /*
232 * Fix up the lwp, thread, and eflags for a successful return
233 *
234 * Preconditions:
235 * - zwreg contains zero
236 */
237 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \
238 movb $LWP_USER, LWP_STATE(lwp); \
239 movw zwreg, T_SYSNUM(t); \
240 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
241
242 /*
243 * ASSERT(lwptoregs(lwp) == rp);
244 *
245 * This may seem obvious, but very odd things happen if this
246 * assertion is false
247 *
248 * Preconditions:
249 * (%rsp is ready for normal call sequence)
250 * Postconditions (if assertion is true):
251 * %r11 is smashed
252 *
253 * ASSERT(rp->r_cs == descnum)
254 *
255 * The code selector is written into the regs structure when the
256 * lwp stack is created. We use this ASSERT to validate that
257 * the regs structure really matches how we came in.
258 *
259 * Preconditions:
260 * (%rsp is ready for normal call sequence)
261 * Postconditions (if assertion is true):
262 * -none-
263 *
264 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
265 *
266 * If this is false, it meant that we returned to userland without
267 * updating the segment registers as we were supposed to.
268 *
269 * Note that we must ensure no interrupts or other traps intervene
270 * between entering privileged mode and performing the assertion,
271 * otherwise we may perform a context switch on the thread, which
272 * will end up setting pcb_rupdate to 1 again.
273 */
274 #if defined(DEBUG)
275
276 #if !defined(__lint)
277
278 __lwptoregs_msg:
279 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
280
281 __codesel_msg:
282 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
283
284 __no_rupdate_msg:
285 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
286
287 #endif /* !__lint */
288
289 #define ASSERT_LWPTOREGS(lwp, rp) \
290 movq LWP_REGS(lwp), %r11; \
291 cmpq rp, %r11; \
292 je 7f; \
293 leaq __lwptoregs_msg(%rip), %rdi; \
294 movl $__LINE__, %esi; \
295 movq lwp, %rdx; \
296 movq %r11, %rcx; \
297 movq rp, %r8; \
298 xorl %eax, %eax; \
299 call panic; \
300 7:
301
302 #define ASSERT_NO_RUPDATE_PENDING(lwp) \
303 testb $0x1, PCB_RUPDATE(lwp); \
304 je 8f; \
305 movq lwp, %rdx; \
306 leaq __no_rupdate_msg(%rip), %rdi; \
307 movl $__LINE__, %esi; \
308 xorl %eax, %eax; \
309 call panic; \
310 8:
311
312 #else
313 #define ASSERT_LWPTOREGS(lwp, rp)
314 #define ASSERT_NO_RUPDATE_PENDING(lwp)
315 #endif
316
317 /*
318 * Do the traptrace thing and restore any registers we used
319 * in situ. Assumes that %rsp is pointing at the base of
320 * the struct regs, obviously ..
321 */
322 #ifdef TRAPTRACE
323 #define SYSCALL_TRAPTRACE(ttype) \
324 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \
325 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \
326 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \
327 movq REGOFF_RAX(%rsp), %rax; \
328 movq REGOFF_RBX(%rsp), %rbx; \
329 movq REGOFF_RCX(%rsp), %rcx; \
330 movq REGOFF_RDX(%rsp), %rdx; \
331 movl %eax, TTR_SYSNUM(%rdi); \
332 movq REGOFF_RDI(%rsp), %rdi
333
334 #define SYSCALL_TRAPTRACE32(ttype) \
335 SYSCALL_TRAPTRACE(ttype); \
336 /* paranoia: clean the top 32-bits of the registers */ \
337 orl %eax, %eax; \
338 orl %ebx, %ebx; \
339 orl %ecx, %ecx; \
340 orl %edx, %edx; \
341 orl %edi, %edi
342 #else /* TRAPTRACE */
343 #define SYSCALL_TRAPTRACE(ttype)
344 #define SYSCALL_TRAPTRACE32(ttype)
345 #endif /* TRAPTRACE */
346
347 /*
348 * The 64-bit libc syscall wrapper does this:
349 *
350 * fn(<args>)
351 * {
352 * movq %rcx, %r10 -- because syscall smashes %rcx
353 * movl $CODE, %eax
354 * syscall
355 * <error processing>
356 * }
357 *
358 * Thus when we come into the kernel:
359 *
360 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
361 * %rax is the syscall number
362 * %r12-%r15 contain caller state
363 *
364 * The syscall instruction arranges that:
365 *
366 * %rcx contains the return %rip
367 * %r11d contains bottom 32-bits of %rflags
368 * %rflags is masked (as determined by the SFMASK msr)
369 * %cs is set to UCS_SEL (as determined by the STAR msr)
370 * %ss is set to UDS_SEL (as determined by the STAR msr)
371 * %rip is set to sys_syscall (as determined by the LSTAR msr)
372 *
373 * Or in other words, we have no registers available at all.
374 * Only swapgs can save us!
375 *
376 * Under the hypervisor, the swapgs has happened already. However, the
377 * state of the world is very different from that we're familiar with.
378 *
379 * In particular, we have a stack structure like that for interrupt
380 * gates, except that the %cs and %ss registers are modified for reasons
381 * that are not entirely clear. Critically, the %rcx/%r11 values do
382 * *not* reflect the usage of those registers under a 'real' syscall[1];
383 * the stack, therefore, looks like this:
384 *
385 * 0x0(rsp) potentially junk %rcx
386 * 0x8(rsp) potentially junk %r11
387 * 0x10(rsp) user %rip
388 * 0x18(rsp) modified %cs
389 * 0x20(rsp) user %rflags
390 * 0x28(rsp) user %rsp
391 * 0x30(rsp) modified %ss
392 *
393 *
394 * and before continuing on, we must load the %rip into %rcx and the
395 * %rflags into %r11.
396 *
397 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
398 * Sigh.
399 */
400 #if defined(__xpv)
401 #define XPV_SYSCALL_PROD \
402 movq 0x10(%rsp), %rcx; \
403 movq 0x20(%rsp), %r11; \
404 movq 0x28(%rsp), %rsp
405 #else
406 #define XPV_SYSCALL_PROD /* nothing */
407 #endif
408
409 #if defined(__lint)
410
411 /*ARGSUSED*/
412 void
413 sys_syscall()
414 {}
415
416 void
417 _allsyscalls()
418 {}
419
420 size_t _allsyscalls_size;
421
422 #else /* __lint */
423
424 ENTRY_NP2(brand_sys_syscall,_allsyscalls)
425 SWAPGS /* kernel gsbase */
426 XPV_SYSCALL_PROD
427 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
428 jmp noprod_sys_syscall
429
430 ALTENTRY(sys_syscall)
431 SWAPGS /* kernel gsbase */
432 XPV_SYSCALL_PROD
433
434 noprod_sys_syscall:
435 movq %r15, %gs:CPU_RTMP_R15
436 movq %rsp, %gs:CPU_RTMP_RSP
437
438 movq %gs:CPU_THREAD, %r15
439 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */
440
441 ASSERT_UPCALL_MASK_IS_SET
442
443 movl $UCS_SEL, REGOFF_CS(%rsp)
444 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
445 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
446 movl $UDS_SEL, REGOFF_SS(%rsp)
447
448 movl %eax, %eax /* wrapper: sysc# -> %eax */
449 movq %rdi, REGOFF_RDI(%rsp)
450 movq %rsi, REGOFF_RSI(%rsp)
451 movq %rdx, REGOFF_RDX(%rsp)
452 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */
453 movq %r10, %rcx /* arg[3] for direct calls */
454
455 movq %r8, REGOFF_R8(%rsp)
456 movq %r9, REGOFF_R9(%rsp)
457 movq %rax, REGOFF_RAX(%rsp)
458 movq %rbx, REGOFF_RBX(%rsp)
459
460 movq %rbp, REGOFF_RBP(%rsp)
461 movq %r10, REGOFF_R10(%rsp)
462 movq %gs:CPU_RTMP_RSP, %r11
463 movq %r11, REGOFF_RSP(%rsp)
464 movq %r12, REGOFF_R12(%rsp)
465
466 movq %r13, REGOFF_R13(%rsp)
467 movq %r14, REGOFF_R14(%rsp)
468 movq %gs:CPU_RTMP_R15, %r10
469 movq %r10, REGOFF_R15(%rsp)
470 movq $0, REGOFF_SAVFP(%rsp)
471 movq $0, REGOFF_SAVPC(%rsp)
472
473 /*
474 * Copy these registers here in case we end up stopped with
475 * someone (like, say, /proc) messing with our register state.
476 * We don't -restore- them unless we have to in update_sregs.
477 *
478 * Since userland -can't- change fsbase or gsbase directly,
479 * and capturing them involves two serializing instructions,
480 * we don't bother to capture them here.
481 */
482 xorl %ebx, %ebx
483 movw %ds, %bx
484 movq %rbx, REGOFF_DS(%rsp)
485 movw %es, %bx
486 movq %rbx, REGOFF_ES(%rsp)
487 movw %fs, %bx
488 movq %rbx, REGOFF_FS(%rsp)
489 movw %gs, %bx
490 movq %rbx, REGOFF_GS(%rsp)
491
492 /*
493 * Machine state saved in the regs structure on the stack
494 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
495 * %eax is the syscall number
496 * %rsp is the thread's stack, %r15 is curthread
497 * REG_RSP(%rsp) is the user's stack
498 */
499
500 SYSCALL_TRAPTRACE($TT_SYSC64)
501
502 movq %rsp, %rbp
503
504 movq T_LWP(%r15), %r14
505 ASSERT_NO_RUPDATE_PENDING(%r14)
506
507 ENABLE_INTR_FLAGS
508
509 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
510 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
511
512 ASSERT_LWPTOREGS(%r14, %rsp)
513
514 movb $LWP_SYS, LWP_STATE(%r14)
515 incq LWP_RU_SYSC(%r14)
516 movb $NORMALRETURN, LWP_EOSYS(%r14)
517
518 incq %gs:CPU_STATS_SYS_SYSCALL
519
520 /*
521 * If our LWP has an alternate system call handler, run that instead of
522 * the regular system call path.
523 */
524 movq LWP_BRAND_SYSCALL(%r14), %rdi
525 testq %rdi, %rdi
526 jz _syscall_no_brand
527
528 pushq %rax
529 subq $8, %rsp /* align stack for call to C */
530 call *%rdi
531 addq $8, %rsp
532
533 /*
534 * If the alternate handler returns non-zero, the normal system call
535 * processing is resumed.
536 */
537 testl %eax, %eax
538 popq %rax
539 jnz _syscall_no_brand
540
541 /*
542 * For branded syscalls which were handled in-kernel, shuffle the
543 * register state as would be done by the native handler before jumping
544 * to the post-syscall logic.
545 */
546 movq REGOFF_RAX(%rsp), %r12
547 movq REGOFF_RDX(%rsp), %r13
548 jmp _syscall_after_brand
549
550 _syscall_no_brand:
551 movw %ax, T_SYSNUM(%r15)
552 movzbl T_PRE_SYS(%r15), %ebx
553 ORL_SYSCALLTRACE(%ebx)
554 testl %ebx, %ebx
555 jne _syscall_pre
556
557 _syscall_invoke:
558 movq REGOFF_RDI(%rbp), %rdi
559 movq REGOFF_RSI(%rbp), %rsi
560 movq REGOFF_RDX(%rbp), %rdx
561 movq REGOFF_RCX(%rbp), %rcx
562 movq REGOFF_R8(%rbp), %r8
563 movq REGOFF_R9(%rbp), %r9
564
565 cmpl $NSYSCALL, %eax
566 jae _syscall_ill
567 shll $SYSENT_SIZE_SHIFT, %eax
568 leaq sysent(%rax), %rbx
569
570 call *SY_CALLC(%rbx)
571
572 movq %rax, %r12
573 movq %rdx, %r13
574
575 /*
576 * If the handler returns two ints, then we need to split the
577 * 64-bit return value into two 32-bit values.
578 */
579 testw $SE_32RVAL2, SY_FLAGS(%rbx)
580 je 5f
581 movq %r12, %r13
582 shrq $32, %r13 /* upper 32-bits into %edx */
583 movl %r12d, %r12d /* lower 32-bits into %eax */
584 5:
585
586 _syscall_after_brand:
587 /*
588 * Optimistically assume that there's no post-syscall
589 * work to do. (This is to avoid having to call syscall_mstate()
590 * with interrupts disabled)
591 */
592 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
593
594 /*
595 * We must protect ourselves from being descheduled here;
596 * If we were, and we ended up on another cpu, or another
597 * lwp got in ahead of us, it could change the segment
598 * registers without us noticing before we return to userland.
599 */
600 CLI(%r14)
601 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
602 jne _syscall_post
603
604 /*
605 * We need to protect ourselves against non-canonical return values
606 * because Intel doesn't check for them on sysret (AMD does). Canonical
607 * addresses on current amd64 processors only use 48-bits for VAs; an
608 * address is canonical if all upper bits (47-63) are identical. If we
609 * find a non-canonical %rip, we opt to go through the full
610 * _syscall_post path which takes us into an iretq which is not
611 * susceptible to the same problems sysret is.
612 *
613 * We're checking for a canonical address by first doing an arithmetic
614 * shift. This will fill in the remaining bits with the value of bit 63.
615 * If the address were canonical, the register would now have either all
616 * zeroes or all ones in it. Therefore we add one (inducing overflow)
617 * and compare against 1. A canonical address will either be zero or one
618 * at this point, hence the use of ja.
619 *
620 * At this point, r12 and r13 have the return value so we can't use
621 * those registers.
622 */
623 movq REGOFF_RIP(%rsp), %rcx
624 sarq $47, %rcx
625 incq %rcx
626 cmpq $1, %rcx
627 ja _syscall_post
628
629
630 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
631
632 movq %r12, REGOFF_RAX(%rsp)
633 movq %r13, REGOFF_RDX(%rsp)
634
635 /*
636 * To get back to userland, we need the return %rip in %rcx and
637 * the return %rfl in %r11d. The sysretq instruction also arranges
638 * to fix up %cs and %ss; everything else is our responsibility.
639 */
640 movq REGOFF_RDI(%rsp), %rdi
641 movq REGOFF_RSI(%rsp), %rsi
642 movq REGOFF_RDX(%rsp), %rdx
643 /* %rcx used to restore %rip value */
644
645 movq REGOFF_R8(%rsp), %r8
646 movq REGOFF_R9(%rsp), %r9
647 movq REGOFF_RAX(%rsp), %rax
648 movq REGOFF_RBX(%rsp), %rbx
649
650 movq REGOFF_RBP(%rsp), %rbp
651 movq REGOFF_R10(%rsp), %r10
652 /* %r11 used to restore %rfl value */
653 movq REGOFF_R12(%rsp), %r12
654
655 movq REGOFF_R13(%rsp), %r13
656 movq REGOFF_R14(%rsp), %r14
657 movq REGOFF_R15(%rsp), %r15
658
659 movq REGOFF_RIP(%rsp), %rcx
660 movl REGOFF_RFL(%rsp), %r11d
661
662 #if defined(__xpv)
663 addq $REGOFF_RIP, %rsp
664 #else
665 movq REGOFF_RSP(%rsp), %rsp
666 #endif
667
668 /*
669 * There can be no instructions between the ALTENTRY below and
670 * SYSRET or we could end up breaking brand support. See label usage
671 * in sn1_brand_syscall_callback for an example.
672 */
673 ASSERT_UPCALL_MASK_IS_SET
674 #if defined(__xpv)
675 SYSRETQ
676 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
677
678 /*
679 * We can only get here after executing a brand syscall
680 * interposition callback handler and simply need to
681 * "sysretq" back to userland. On the hypervisor this
682 * involves the iret hypercall which requires us to construct
683 * just enough of the stack needed for the hypercall.
684 * (rip, cs, rflags, rsp, ss).
685 */
686 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */
687 movq %gs:CPU_THREAD, %r11
688 movq T_STACK(%r11), %rsp
689
690 movq %rcx, REGOFF_RIP(%rsp)
691 movl $UCS_SEL, REGOFF_CS(%rsp)
692 movq %gs:CPU_RTMP_RSP, %r11
693 movq %r11, REGOFF_RSP(%rsp)
694 pushfq
695 popq %r11 /* hypercall enables ints */
696 movq %r11, REGOFF_RFL(%rsp)
697 movl $UDS_SEL, REGOFF_SS(%rsp)
698 addq $REGOFF_RIP, %rsp
699 /*
700 * XXPV: see comment in SYSRETQ definition for future optimization
701 * we could take.
702 */
703 ASSERT_UPCALL_MASK_IS_SET
704 SYSRETQ
705 #else
706 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
707 SWAPGS /* user gsbase */
708 SYSRETQ
709 #endif
710 /*NOTREACHED*/
711 SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
712
713 _syscall_pre:
714 call pre_syscall
715 movl %eax, %r12d
716 testl %eax, %eax
717 jne _syscall_post_call
718 /*
719 * Didn't abort, so reload the syscall args and invoke the handler.
720 */
721 movzwl T_SYSNUM(%r15), %eax
722 jmp _syscall_invoke
723
724 _syscall_ill:
725 call nosys
726 movq %rax, %r12
727 movq %rdx, %r13
728 jmp _syscall_post_call
729
730 _syscall_post:
731 STI
732 /*
733 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
734 * so that we can account for the extra work it takes us to finish.
735 */
736 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
737 _syscall_post_call:
738 movq %r12, %rdi
739 movq %r13, %rsi
740 call post_syscall
741 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
742 jmp _sys_rtt
743 SET_SIZE(sys_syscall)
744 SET_SIZE(brand_sys_syscall)
745
746 #endif /* __lint */
747
748 #if defined(__lint)
749
750 /*ARGSUSED*/
751 void
752 sys_syscall32()
753 {}
754
755 #else /* __lint */
756
757 ENTRY_NP(brand_sys_syscall32)
758 SWAPGS /* kernel gsbase */
759 XPV_TRAP_POP
760 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
761 jmp nopop_sys_syscall32
762
763 ALTENTRY(sys_syscall32)
764 SWAPGS /* kernel gsbase */
765 XPV_TRAP_POP
766
767 nopop_sys_syscall32:
768 movl %esp, %r10d
769 movq %gs:CPU_THREAD, %r15
770 movq T_STACK(%r15), %rsp
771 movl %eax, %eax
772
773 movl $U32CS_SEL, REGOFF_CS(%rsp)
774 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
775 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
776 movq %r10, REGOFF_RSP(%rsp)
777 movl $UDS_SEL, REGOFF_SS(%rsp)
778
779 _syscall32_save:
780 movl %edi, REGOFF_RDI(%rsp)
781 movl %esi, REGOFF_RSI(%rsp)
782 movl %ebp, REGOFF_RBP(%rsp)
783 movl %ebx, REGOFF_RBX(%rsp)
784 movl %edx, REGOFF_RDX(%rsp)
785 movl %ecx, REGOFF_RCX(%rsp)
786 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
787 movq $0, REGOFF_SAVFP(%rsp)
788 movq $0, REGOFF_SAVPC(%rsp)
789
790 /*
791 * Copy these registers here in case we end up stopped with
792 * someone (like, say, /proc) messing with our register state.
793 * We don't -restore- them unless we have to in update_sregs.
794 *
795 * Since userland -can't- change fsbase or gsbase directly,
796 * we don't bother to capture them here.
797 */
798 xorl %ebx, %ebx
799 movw %ds, %bx
800 movq %rbx, REGOFF_DS(%rsp)
801 movw %es, %bx
802 movq %rbx, REGOFF_ES(%rsp)
803 movw %fs, %bx
804 movq %rbx, REGOFF_FS(%rsp)
805 movw %gs, %bx
806 movq %rbx, REGOFF_GS(%rsp)
807
808 /*
809 * Application state saved in the regs structure on the stack
810 * %eax is the syscall number
811 * %rsp is the thread's stack, %r15 is curthread
812 * REG_RSP(%rsp) is the user's stack
813 */
814
815 SYSCALL_TRAPTRACE32($TT_SYSC)
816
817 movq %rsp, %rbp
818
819 movq T_LWP(%r15), %r14
820 ASSERT_NO_RUPDATE_PENDING(%r14)
821
822 ENABLE_INTR_FLAGS
823
824 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
825 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
826
827 ASSERT_LWPTOREGS(%r14, %rsp)
828
829 incq %gs:CPU_STATS_SYS_SYSCALL
830
831 /*
832 * If our lwp has an alternate system call handler, run that instead
833 * of the regular system call path.
834 */
835 movq LWP_BRAND_SYSCALL(%r14), %rax
836 testq %rax, %rax
837 jz _syscall32_no_brand
838
839 movb $LWP_SYS, LWP_STATE(%r14)
840 call *%rax
841
842 /*
843 * If the alternate handler returns non-zero, the normal system call
844 * processing is resumed.
845 */
846 testl %eax, %eax
847 jnz _syscall32_no_brand
848
849 /*
850 * For branded syscalls which were handled in-kernel, shuffle the
851 * register state as would be done by the native handler before jumping
852 * to the post-syscall logic.
853 */
854 movl REGOFF_RAX(%rsp), %r12d
855 movl REGOFF_RDX(%rsp), %r13d
856 jmp _syscall32_after_brand
857
858 _syscall32_no_brand:
859 /*
860 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
861 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or
862 * more succinctly:
863 *
864 * SA(MAXSYSARGS * sizeof (long)) == 64
865 *
866 * Note, this space is used both to copy in the arguments from user
867 * land, but also to as part of the old UNIX style syscall_ap() method.
868 * syscall_entry expects that we do not change the values of this space
869 * that we give it. However, this means that when we end up in the more
870 * recent model of passing the arguments based on the calling
871 * conventions, we'll need to save an additional 16 bytes of stack.
872 */
873 #define SYS_DROP 64 /* drop for args */
874 subq $SYS_DROP, %rsp
875 movb $LWP_SYS, LWP_STATE(%r14)
876 movq %r15, %rdi
877 movq %rsp, %rsi
878 call syscall_entry
879
880 /*
881 * Fetch the arguments copied onto the kernel stack and put
882 * them in the right registers to invoke a C-style syscall handler.
883 * %rax contains the handler address.
884 *
885 * Ideas for making all this go faster of course include simply
886 * forcibly fetching 6 arguments from the user stack under lofault
887 * protection, reverting to copyin_args only when watchpoints
888 * are in effect.
889 *
890 * (If we do this, make sure that exec and libthread leave
891 * enough space at the top of the stack to ensure that we'll
892 * never do a fetch from an invalid page.)
893 *
894 * Lots of ideas here, but they won't really help with bringup B-)
895 * Correctness can't wait, performance can wait a little longer ..
896 */
897
898 movq %rax, %rbx
899 movl 0x0(%rsp), %edi /* arg0 */
900 movl 0x8(%rsp), %esi /* arg1 */
901 movl 0x10(%rsp), %edx /* arg2 */
902 movl 0x38(%rsp), %eax /* arg7 load */
903 movl 0x18(%rsp), %ecx /* arg3 */
904 pushq %rax /* arg7 saved to stack */
905 movl 0x28(%rsp), %r8d /* arg4 */
906 movl 0x38(%rsp), %eax /* arg6 load */
907 movl 0x30(%rsp), %r9d /* arg5 */
908 pushq %rax /* arg6 saved to stack */
909
910 call *SY_CALLC(%rbx)
911
912 movq %rbp, %rsp /* pop the args */
913
914 /*
915 * amd64 syscall handlers -always- return a 64-bit value in %rax.
916 * On the 32-bit kernel, they always return that value in %eax:%edx
917 * as required by the 32-bit ABI.
918 *
919 * Simulate the same behaviour by unconditionally splitting the
920 * return value in the same way.
921 */
922 movq %rax, %r13
923 shrq $32, %r13 /* upper 32-bits into %edx */
924 movl %eax, %r12d /* lower 32-bits into %eax */
925
926 _syscall32_after_brand:
927
928 /*
929 * Optimistically assume that there's no post-syscall
930 * work to do. (This is to avoid having to call syscall_mstate()
931 * with interrupts disabled)
932 */
933 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
934
935 /*
936 * We must protect ourselves from being descheduled here;
937 * If we were, and we ended up on another cpu, or another
938 * lwp got in ahead of us, it could change the segment
939 * registers without us noticing before we return to userland.
940 */
941 CLI(%r14)
942 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
943 jne _full_syscall_postsys32
944 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
945
946 /*
947 * To get back to userland, we need to put the return %rip in %rcx and
948 * the return %rfl in %r11d. The sysret instruction also arranges
949 * to fix up %cs and %ss; everything else is our responsibility.
950 */
951
952 movl %r12d, %eax /* %eax: rval1 */
953 movl REGOFF_RBX(%rsp), %ebx
954 /* %ecx used for return pointer */
955 movl %r13d, %edx /* %edx: rval2 */
956 movl REGOFF_RBP(%rsp), %ebp
957 movl REGOFF_RSI(%rsp), %esi
958 movl REGOFF_RDI(%rsp), %edi
959
960 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */
961 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */
962 movl REGOFF_RSP(%rsp), %esp
963
964 ASSERT_UPCALL_MASK_IS_SET
965 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
966 SWAPGS /* user gsbase */
967 SYSRETL
968 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
969 /*NOTREACHED*/
970
971 _full_syscall_postsys32:
972 STI
973 /*
974 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
975 * so that we can account for the extra work it takes us to finish.
976 */
977 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
978 movq %r15, %rdi
979 movq %r12, %rsi /* rval1 - %eax */
980 movq %r13, %rdx /* rval2 - %edx */
981 call syscall_exit
982 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
983 jmp _sys_rtt
984 SET_SIZE(sys_syscall32)
985 SET_SIZE(brand_sys_syscall32)
986
987 #endif /* __lint */
988
989 /*
990 * System call handler via the sysenter instruction
991 * Used only for 32-bit system calls on the 64-bit kernel.
992 *
993 * The caller in userland has arranged that:
994 *
995 * - %eax contains the syscall number
996 * - %ecx contains the user %esp
997 * - %edx contains the return %eip
998 * - the user stack contains the args to the syscall
999 *
1000 * Hardware and (privileged) initialization code have arranged that by
1001 * the time the sysenter instructions completes:
1002 *
1003 * - %rip is pointing to sys_sysenter (below).
1004 * - %cs and %ss are set to kernel text and stack (data) selectors.
1005 * - %rsp is pointing at the lwp's stack
1006 * - interrupts have been disabled.
1007 *
1008 * Note that we are unable to return both "rvals" to userland with
1009 * this call, as %edx is used by the sysexit instruction.
1010 *
1011 * One final complication in this routine is its interaction with
1012 * single-stepping in a debugger. For most of the system call mechanisms,
1013 * the CPU automatically clears the single-step flag before we enter the
1014 * kernel. The sysenter mechanism does not clear the flag, so a user
1015 * single-stepping through a libc routine may suddenly find him/herself
1016 * single-stepping through the kernel. To detect this, kmdb compares the
1017 * trap %pc to the [brand_]sys_enter addresses on each single-step trap.
1018 * If it finds that we have single-stepped to a sysenter entry point, it
1019 * explicitly clears the flag and executes the sys_sysenter routine.
1020 *
1021 * One final complication in this final complication is the fact that we
1022 * have two different entry points for sysenter: brand_sys_sysenter and
1023 * sys_sysenter. If we enter at brand_sys_sysenter and start single-stepping
1024 * through the kernel with kmdb, we will eventually hit the instruction at
1025 * sys_sysenter. kmdb cannot distinguish between that valid single-step
1026 * and the undesirable one mentioned above. To avoid this situation, we
1027 * simply add a jump over the instruction at sys_sysenter to make it
1028 * impossible to single-step to it.
1029 */
1030 #if defined(__lint)
1031
1032 void
1033 sys_sysenter()
1034 {}
1035
1036 #else /* __lint */
1037
1038 ENTRY_NP(brand_sys_sysenter)
1039 SWAPGS /* kernel gsbase */
1040 ALTENTRY(_brand_sys_sysenter_post_swapgs)
1041 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
1042 /*
1043 * Jump over sys_sysenter to allow single-stepping as described
1044 * above.
1045 */
1046 jmp _sys_sysenter_post_swapgs
1047
1048 ALTENTRY(sys_sysenter)
1049 SWAPGS /* kernel gsbase */
1050
1051 ALTENTRY(_sys_sysenter_post_swapgs)
1052 movq %gs:CPU_THREAD, %r15
1053
1054 movl $U32CS_SEL, REGOFF_CS(%rsp)
1055 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */
1056 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */
1057 pushfq
1058 popq %r10
1059 movl $UDS_SEL, REGOFF_SS(%rsp)
1060
1061 /*
1062 * Set the interrupt flag before storing the flags to the
1063 * flags image on the stack so we can return to user with
1064 * interrupts enabled if we return via sys_rtt_syscall32
1065 */
1066 orq $PS_IE, %r10
1067 movq %r10, REGOFF_RFL(%rsp)
1068
1069 movl %edi, REGOFF_RDI(%rsp)
1070 movl %esi, REGOFF_RSI(%rsp)
1071 movl %ebp, REGOFF_RBP(%rsp)
1072 movl %ebx, REGOFF_RBX(%rsp)
1073 movl %edx, REGOFF_RDX(%rsp)
1074 movl %ecx, REGOFF_RCX(%rsp)
1075 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
1076 movq $0, REGOFF_SAVFP(%rsp)
1077 movq $0, REGOFF_SAVPC(%rsp)
1078
1079 /*
1080 * Copy these registers here in case we end up stopped with
1081 * someone (like, say, /proc) messing with our register state.
1082 * We don't -restore- them unless we have to in update_sregs.
1083 *
1084 * Since userland -can't- change fsbase or gsbase directly,
1085 * we don't bother to capture them here.
1086 */
1087 xorl %ebx, %ebx
1088 movw %ds, %bx
1089 movq %rbx, REGOFF_DS(%rsp)
1090 movw %es, %bx
1091 movq %rbx, REGOFF_ES(%rsp)
1092 movw %fs, %bx
1093 movq %rbx, REGOFF_FS(%rsp)
1094 movw %gs, %bx
1095 movq %rbx, REGOFF_GS(%rsp)
1096
1097 /*
1098 * Application state saved in the regs structure on the stack
1099 * %eax is the syscall number
1100 * %rsp is the thread's stack, %r15 is curthread
1101 * REG_RSP(%rsp) is the user's stack
1102 */
1103
1104 SYSCALL_TRAPTRACE($TT_SYSENTER)
1105
1106 movq %rsp, %rbp
1107
1108 movq T_LWP(%r15), %r14
1109 ASSERT_NO_RUPDATE_PENDING(%r14)
1110
1111 ENABLE_INTR_FLAGS
1112
1113 /*
1114 * Catch 64-bit process trying to issue sysenter instruction
1115 * on Nocona based systems.
1116 */
1117 movq LWP_PROCP(%r14), %rax
1118 cmpq $DATAMODEL_ILP32, P_MODEL(%rax)
1119 je 7f
1120
1121 /*
1122 * For a non-32-bit process, simulate a #ud, since that's what
1123 * native hardware does. The traptrace entry (above) will
1124 * let you know what really happened.
1125 */
1126 movq $T_ILLINST, REGOFF_TRAPNO(%rsp)
1127 movq REGOFF_CS(%rsp), %rdi
1128 movq %rdi, REGOFF_ERR(%rsp)
1129 movq %rsp, %rdi
1130 movq REGOFF_RIP(%rsp), %rsi
1131 movl %gs:CPU_ID, %edx
1132 call trap
1133 jmp _sys_rtt
1134 7:
1135
1136 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1137 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */
1138
1139 ASSERT_LWPTOREGS(%r14, %rsp)
1140
1141 incq %gs:CPU_STATS_SYS_SYSCALL
1142
1143 /*
1144 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1145 * placed into 64-bit (long) arg slots, plus one 64-bit
1146 * (long) arg count, maintaining 16 byte alignment.
1147 */
1148 subq $SYS_DROP, %rsp
1149 movb $LWP_SYS, LWP_STATE(%r14)
1150 movq %r15, %rdi
1151 movq %rsp, %rsi
1152 call syscall_entry
1153
1154 /*
1155 * Fetch the arguments copied onto the kernel stack and put
1156 * them in the right registers to invoke a C-style syscall handler.
1157 * %rax contains the handler address. For the last two arguments, we
1158 * push them onto the stack -- we can't clobber the old arguments.
1159 */
1160 movq %rax, %rbx
1161 movl 0x0(%rsp), %edi /* arg0 */
1162 movl 0x8(%rsp), %esi /* arg1 */
1163 movl 0x10(%rsp), %edx /* arg2 */
1164 movl 0x38(%rsp), %eax /* arg7 load */
1165 movl 0x18(%rsp), %ecx /* arg3 */
1166 pushq %rax /* arg7 saved to stack */
1167 movl 0x28(%rsp), %r8d /* arg4 */
1168 movl 0x38(%rsp), %eax /* arg6 load */
1169 movl 0x30(%rsp), %r9d /* arg5 */
1170 pushq %rax /* arg6 saved to stack */
1171
1172 call *SY_CALLC(%rbx)
1173
1174 movq %rbp, %rsp /* pop the args */
1175
1176 /*
1177 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1178 * On the 32-bit kernel, the always return that value in %eax:%edx
1179 * as required by the 32-bit ABI.
1180 *
1181 * Simulate the same behaviour by unconditionally splitting the
1182 * return value in the same way.
1183 */
1184 movq %rax, %r13
1185 shrq $32, %r13 /* upper 32-bits into %edx */
1186 movl %eax, %r12d /* lower 32-bits into %eax */
1187
1188 /*
1189 * Optimistically assume that there's no post-syscall
1190 * work to do. (This is to avoid having to call syscall_mstate()
1191 * with interrupts disabled)
1192 */
1193 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1194
1195 /*
1196 * We must protect ourselves from being descheduled here;
1197 * If we were, and we ended up on another cpu, or another
1198 * lwp got int ahead of us, it could change the segment
1199 * registers without us noticing before we return to userland.
1200 */
1201 cli
1202 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1203 jne _full_syscall_postsys32
1204 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1205
1206 /*
1207 * To get back to userland, load up the 32-bit registers and
1208 * sysexit back where we came from.
1209 */
1210
1211 /*
1212 * Interrupts will be turned on by the 'sti' executed just before
1213 * sysexit. The following ensures that restoring the user's rflags
1214 * doesn't enable interrupts too soon.
1215 */
1216 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1217
1218 /*
1219 * (There's no point in loading up %edx because the sysexit
1220 * mechanism smashes it.)
1221 */
1222 movl %r12d, %eax
1223 movl REGOFF_RBX(%rsp), %ebx
1224 movl REGOFF_RBP(%rsp), %ebp
1225 movl REGOFF_RSI(%rsp), %esi
1226 movl REGOFF_RDI(%rsp), %edi
1227
1228 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */
1229 pushq REGOFF_RFL(%rsp)
1230 popfq
1231 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */
1232 ALTENTRY(sys_sysenter_swapgs_sysexit)
1233 swapgs
1234 sti
1235 sysexit
1236 SET_SIZE(sys_sysenter_swapgs_sysexit)
1237 SET_SIZE(sys_sysenter)
1238 SET_SIZE(_sys_sysenter_post_swapgs)
1239 SET_SIZE(brand_sys_sysenter)
1240
1241 #endif /* __lint */
1242
1243 #if defined(__lint)
1244 /*
1245 * System call via an int80. This entry point is only used by the Linux
1246 * application environment. Unlike the other entry points, there is no
1247 * default action to take if no callback is registered for this process.
1248 */
1249 void
1250 sys_int80()
1251 {}
1252
1253 #else /* __lint */
1254
1255 ENTRY_NP(brand_sys_int80)
1256 SWAPGS /* kernel gsbase */
1257 XPV_TRAP_POP
1258 call smap_enable
1259
1260 /*
1261 * We first attempt to call the "b_int80" handler from the "struct
1262 * brand_mach_ops" for this brand. If no handler function is installed
1263 * for this brand, the BRAND_CALLBACK() macro returns here and we
1264 * check the lwp for a "lwp_brand_syscall" handler.
1265 */
1266 BRAND_CALLBACK(BRAND_CB_INT80, BRAND_URET_FROM_INTR_STACK())
1267
1268 /*
1269 * Check to see if this lwp provides "lwp_brand_syscall". If so, we
1270 * will route this int80 through the regular system call handling path.
1271 */
1272 movq %r15, %gs:CPU_RTMP_R15
1273 movq %gs:CPU_THREAD, %r15
1274 movq T_LWP(%r15), %r15
1275 movq LWP_BRAND_SYSCALL(%r15), %r15
1276 testq %r15, %r15
1277 movq %gs:CPU_RTMP_R15, %r15
1278 jnz nopop_syscall_int
1279
1280 /*
1281 * The brand provided neither a "b_int80", nor a "lwp_brand_syscall"
1282 * function, and has thus opted out of handling this trap.
1283 */
1284 SWAPGS /* user gsbase */
1285 jmp nopop_int80
1286
1287 ENTRY_NP(sys_int80)
1288 /*
1289 * We hit an int80, but this process isn't of a brand with an int80
1290 * handler. Bad process! Make it look as if the INT failed.
1291 * Modify %rip to point before the INT, push the expected error
1292 * code and fake a GP fault. Note on 64-bit hypervisor we need
1293 * to undo the XPV_TRAP_POP and push rcx and r11 back on the stack
1294 * because gptrap will pop them again with its own XPV_TRAP_POP.
1295 */
1296 XPV_TRAP_POP
1297 call smap_enable
1298 nopop_int80:
1299 subq $2, (%rsp) /* int insn 2-bytes */
1300 pushq $_CONST(_MUL(T_INT80, GATE_DESC_SIZE) + 2)
1301 #if defined(__xpv)
1302 push %r11
1303 push %rcx
1304 #endif
1305 jmp gptrap / GP fault
1306 SET_SIZE(sys_int80)
1307 SET_SIZE(brand_sys_int80)
1308 #endif /* __lint */
1309
1310
1311 /*
1312 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1313 * the generic i386 libc to do system calls. We do a small amount of setup
1314 * before jumping into the existing sys_syscall32 path.
1315 */
1316 #if defined(__lint)
1317
1318 /*ARGSUSED*/
1319 void
1320 sys_syscall_int()
1321 {}
1322
1323 #else /* __lint */
1324
1325 ENTRY_NP(brand_sys_syscall_int)
1326 SWAPGS /* kernel gsbase */
1327 XPV_TRAP_POP
1328 call smap_enable
1329 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1330 jmp nopop_syscall_int
1331
1332 ALTENTRY(sys_syscall_int)
1333 SWAPGS /* kernel gsbase */
1334 XPV_TRAP_POP
1335 call smap_enable
1336
1337 nopop_syscall_int:
1338 movq %gs:CPU_THREAD, %r15
1339 movq T_STACK(%r15), %rsp
1340 movl %eax, %eax
1341 /*
1342 * Set t_post_sys on this thread to force ourselves out via the slow
1343 * path. It might be possible at some later date to optimize this out
1344 * and use a faster return mechanism.
1345 */
1346 movb $1, T_POST_SYS(%r15)
1347 CLEAN_CS
1348 jmp _syscall32_save
1349 /*
1350 * There should be no instructions between this label and SWAPGS/IRET
1351 * or we could end up breaking branded zone support. See the usage of
1352 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1353 * for examples.
1354 */
1355 ALTENTRY(sys_sysint_swapgs_iret)
1356 SWAPGS /* user gsbase */
1357 IRET
1358 /*NOTREACHED*/
1359 SET_SIZE(sys_sysint_swapgs_iret)
1360 SET_SIZE(sys_syscall_int)
1361 SET_SIZE(brand_sys_syscall_int)
1362
1363 #endif /* __lint */
1364
1365 /*
1366 * Legacy 32-bit applications and old libc implementations do lcalls;
1367 * we should never get here because the LDT entry containing the syscall
1368 * segment descriptor has the "segment present" bit cleared, which means
1369 * we end up processing those system calls in trap() via a not-present trap.
1370 *
1371 * We do it this way because a call gate unhelpfully does -nothing- to the
1372 * interrupt flag bit, so an interrupt can run us just after the lcall
1373 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and
1374 * INTR_POP paths would have to be slightly more complex to dance around
1375 * this problem, and end up depending explicitly on the first
1376 * instruction of this handler being either swapgs or cli.
1377 */
1378
1379 #if defined(__lint)
1380
1381 /*ARGSUSED*/
1382 void
1383 sys_lcall32()
1384 {}
1385
1386 #else /* __lint */
1387
1388 ENTRY_NP(sys_lcall32)
1389 SWAPGS /* kernel gsbase */
1390 pushq $0
1391 pushq %rbp
1392 movq %rsp, %rbp
1393 leaq __lcall_panic_str(%rip), %rdi
1394 xorl %eax, %eax
1395 call panic
1396 SET_SIZE(sys_lcall32)
1397
1398 __lcall_panic_str:
1399 .string "sys_lcall32: shouldn't be here!"
1400
1401 /*
1402 * Declare a uintptr_t which covers the entire pc range of syscall
1403 * handlers for the stack walkers that need this.
1404 */
1405 .align CPTRSIZE
1406 .globl _allsyscalls_size
1407 .type _allsyscalls_size, @object
1408 _allsyscalls_size:
1409 .NWORD . - _allsyscalls
1410 SET_SIZE(_allsyscalls_size)
1411
1412 #endif /* __lint */
1413
1414 /*
1415 * These are the thread context handlers for lwps using sysenter/sysexit.
1416 */
1417
1418 #if defined(__lint)
1419
1420 /*ARGSUSED*/
1421 void
1422 sep_save(void *ksp)
1423 {}
1424
1425 /*ARGSUSED*/
1426 void
1427 sep_restore(void *ksp)
1428 {}
1429
1430 #else /* __lint */
1431
1432 /*
1433 * setting this value to zero as we switch away causes the
1434 * stack-pointer-on-sysenter to be NULL, ensuring that we
1435 * don't silently corrupt another (preempted) thread stack
1436 * when running an lwp that (somehow) didn't get sep_restore'd
1437 */
1438 ENTRY_NP(sep_save)
1439 xorl %edx, %edx
1440 xorl %eax, %eax
1441 movl $MSR_INTC_SEP_ESP, %ecx
1442 wrmsr
1443 ret
1444 SET_SIZE(sep_save)
1445
1446 /*
1447 * Update the kernel stack pointer as we resume onto this cpu.
1448 */
1449 ENTRY_NP(sep_restore)
1450 movq %rdi, %rdx
1451 shrq $32, %rdx
1452 movl %edi, %eax
1453 movl $MSR_INTC_SEP_ESP, %ecx
1454 wrmsr
1455 ret
1456 SET_SIZE(sep_restore)
1457
1458 #endif /* __lint */