Print this page
OS-5510 remove lwp_brand_syscall_fast handler
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Lint-clean syscall_asm_amd64.s
OS-4961 lxbrand want fasttrap-like brand hook
Reviewed by: Joshua M. Clulow <jmc@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
OS-3937 lxbrand incorrect stack alignment for lx_syscall_enter()
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
OS-3561 lxbrand emulation library should execute on alternate stack
OS-3558 lxbrand add support for full in-kernel syscall handling
OS-3545 lx_syscall_regs should not walk stack
OS-3868 many LTP testcases now hang
OS-3901 lxbrand lx_recvmsg fails to translate control messages when 64-bit
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Bryan Cantrill <bryan@joyent.com>
OS-3215 32bit syscalls with more than 6 parameters re-use arg0/arg1 as arg6/arg7
OS-3223 Passing arg6 and arg7 can't clobber the stack for ap-style calls
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Keith M Wesolowski <wesolows@foobazco.org>
back out OS-3215: causes OS-3223
OS-3215 32bit syscalls with more than 6 parameters re-use arg0/arg1 as arg6/arg7
OS-2834 ship lx brand
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/ml/syscall_asm_amd64.s
+++ new/usr/src/uts/i86pc/ml/syscall_asm_amd64.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 - * Copyright 2015 Joyent, Inc.
23 + * Copyright 2016 Joyent, Inc.
24 24 */
25 25
26 26 #include <sys/asm_linkage.h>
27 27 #include <sys/asm_misc.h>
28 28 #include <sys/regset.h>
29 29 #include <sys/privregs.h>
30 30 #include <sys/psw.h>
31 -#include <sys/machbrand.h>
32 31
33 32 #if defined(__lint)
34 33
35 34 #include <sys/types.h>
36 35 #include <sys/thread.h>
37 36 #include <sys/systm.h>
38 37
39 38 #else /* __lint */
40 39
40 +#include <sys/machbrand.h>
41 41 #include <sys/segments.h>
42 42 #include <sys/pcb.h>
43 43 #include <sys/trap.h>
44 44 #include <sys/ftrace.h>
45 45 #include <sys/traptrace.h>
46 46 #include <sys/clock.h>
47 47 #include <sys/model.h>
48 48 #include <sys/panic.h>
49 49
50 50 #if defined(__xpv)
51 51 #include <sys/hypervisor.h>
52 52 #endif
53 53
54 54 #include "assym.h"
55 55
56 56 #endif /* __lint */
57 57
58 58 /*
59 59 * We implement five flavours of system call entry points
60 60 *
61 61 * - syscall/sysretq (amd64 generic)
62 62 * - syscall/sysretl (i386 plus SYSC bit)
63 63 * - sysenter/sysexit (i386 plus SEP bit)
64 64 * - int/iret (i386 generic)
65 65 * - lcall/iret (i386 generic)
66 66 *
67 67 * The current libc included in Solaris uses int/iret as the base unoptimized
68 68 * kernel entry method. Older libc implementations and legacy binaries may use
69 69 * the lcall call gate, so it must continue to be supported.
70 70 *
71 71 * System calls that use an lcall call gate are processed in trap() via a
72 72 * segment-not-present trap, i.e. lcalls are extremely slow(!).
73 73 *
74 74 * The basic pattern used in the 32-bit SYSC handler at this point in time is
75 75 * to have the bare minimum of assembler, and get to the C handlers as
76 76 * quickly as possible.
77 77 *
78 78 * The 64-bit handler is much closer to the sparcv9 handler; that's
79 79 * because of passing arguments in registers. The 32-bit world still
80 80 * passes arguments on the stack -- that makes that handler substantially
81 81 * more complex.
82 82 *
83 83 * The two handlers share a few code fragments which are broken
84 84 * out into preprocessor macros below.
85 85 *
86 86 * XX64 come back and speed all this up later. The 32-bit stuff looks
87 87 * especially easy to speed up the argument copying part ..
88 88 *
89 89 *
90 90 * Notes about segment register usage (c.f. the 32-bit kernel)
91 91 *
92 92 * In the 32-bit kernel, segment registers are dutifully saved and
93 93 * restored on all mode transitions because the kernel uses them directly.
94 94 * When the processor is running in 64-bit mode, segment registers are
95 95 * largely ignored.
96 96 *
97 97 * %cs and %ss
98 98 * controlled by the hardware mechanisms that make mode transitions
99 99 *
100 100 * The remaining segment registers have to either be pointing at a valid
101 101 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
102 102 *
103 103 * %ds and %es
104 104 * always ignored
105 105 *
106 106 * %fs and %gs
107 107 * fsbase and gsbase are used to control the place they really point at.
108 108 * The kernel only depends on %gs, and controls its own gsbase via swapgs
109 109 *
110 110 * Note that loading segment registers is still costly because the GDT
111 111 * lookup still happens (this is because the hardware can't know that we're
112 112 * not setting up these segment registers for a 32-bit program). Thus we
113 113 * avoid doing this in the syscall path, and defer them to lwp context switch
114 114 * handlers, so the register values remain virtualized to the lwp.
115 115 */
116 116
117 117 #if defined(SYSCALLTRACE)
118 118 #define ORL_SYSCALLTRACE(r32) \
119 119 orl syscalltrace(%rip), r32
120 120 #else
121 121 #define ORL_SYSCALLTRACE(r32)
122 122 #endif
123 123
124 124 /*
125 125 * In the 32-bit kernel, we do absolutely nothing before getting into the
126 126 * brand callback checks. In 64-bit land, we do swapgs and then come here.
127 127 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
128 128 * are still unused.
129 129 *
130 130 * Check if a brand_mach_ops callback is defined for the specified callback_id
131 131 * type. If so invoke it with the kernel's %gs value loaded and the following
132 132 * data on the stack:
133 133 *
134 134 * stack: --------------------------------------
135 135 * 32 | callback pointer |
136 136 * | 24 | user (or interrupt) stack pointer |
137 137 * | 16 | lwp pointer |
138 138 * v 8 | userland return address |
139 139 * 0 | callback wrapper return addr |
140 140 * --------------------------------------
141 141 *
142 142 * Since we're pushing the userland return address onto the kernel stack
143 143 * we need to get that address without accessing the user's stack (since we
144 144 * can't trust that data). There are different ways to get the userland
145 145 * return address depending on how the syscall trap was made:
146 146 *
147 147 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
148 148 * b) For sys_sysenter the return address is in %rdx.
149 149 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
150 150 * the stack pointer points at the state saved when we took the interrupt:
151 151 * ------------------------
152 152 * | | user's %ss |
153 153 * | | user's %esp |
154 154 * | | EFLAGS register |
155 155 * v | user's %cs |
156 156 * | user's %eip |
157 157 * ------------------------
158 158 *
159 159 * The 2nd parameter to the BRAND_CALLBACK macro is either the
160 160 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are
161 161 * used to generate the proper code to get the userland return address for
162 162 * each syscall entry point.
163 163 *
164 164 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
165 165 * is available as a scratch register within the callback. If the callback
166 166 * returns within the kernel then this macro will restore %r15. If the
167 167 * callback is going to return directly to userland then it should restore
168 168 * %r15 before returning to userland.
169 169 */
170 170 #define BRAND_URET_FROM_REG(rip_reg) \
171 171 pushq rip_reg /* push the return address */
172 172
173 173 /*
174 174 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
175 175 * is currently pointing at the user return address (%eip).
176 176 */
177 177 #define BRAND_URET_FROM_INTR_STACK() \
178 178 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\
179 179 pushq (%r15) /* push the return address */
180 180
181 181 #define BRAND_CALLBACK(callback_id, push_userland_ret) \
182 182 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\
183 183 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\
184 184 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\
185 185 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\
186 186 subq $16, %rsp /* save space for 2 pointers */ ;\
187 187 pushq %r14 /* save %r14 */ ;\
188 188 movq %gs:CPU_RTMP_RSP, %r14 ;\
189 189 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\
190 190 popq %r14 /* restore %r14 */ ;\
191 191 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\
192 192 pushq %r15 /* push the lwp pointer */ ;\
193 193 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\
194 194 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\
195 195 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\
196 196 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\
197 197 cmpq $0, %r15 ;\
198 198 je 1f ;\
199 199 movq %r15, 16(%rsp) /* save the callback pointer */ ;\
200 200 push_userland_ret /* push the return address */ ;\
201 201 call *24(%rsp) /* call callback */ ;\
202 202 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\
203 203 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */
204 204
205 205 #define MSTATE_TRANSITION(from, to) \
206 206 movl $from, %edi; \
207 207 movl $to, %esi; \
208 208 call syscall_mstate
209 209
210 210 /*
211 211 * Check to see if a simple (direct) return is possible i.e.
212 212 *
213 213 * if (t->t_post_sys_ast | syscalltrace |
214 214 * lwp->lwp_pcb.pcb_rupdate == 1)
215 215 * do full version ;
216 216 *
217 217 * Preconditions:
218 218 * - t is curthread
219 219 * Postconditions:
220 220 * - condition code NE is set if post-sys is too complex
221 221 * - rtmp is zeroed if it isn't (we rely on this!)
222 222 * - ltmp is smashed
223 223 */
224 224 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \
225 225 movq T_LWP(t), ltmp; \
226 226 movzbl PCB_RUPDATE(ltmp), rtmp; \
227 227 ORL_SYSCALLTRACE(rtmp); \
228 228 orl T_POST_SYS_AST(t), rtmp; \
229 229 cmpl $0, rtmp
230 230
231 231 /*
232 232 * Fix up the lwp, thread, and eflags for a successful return
233 233 *
234 234 * Preconditions:
235 235 * - zwreg contains zero
236 236 */
237 237 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \
238 238 movb $LWP_USER, LWP_STATE(lwp); \
239 239 movw zwreg, T_SYSNUM(t); \
240 240 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
241 241
242 242 /*
243 243 * ASSERT(lwptoregs(lwp) == rp);
244 244 *
245 245 * This may seem obvious, but very odd things happen if this
246 246 * assertion is false
247 247 *
248 248 * Preconditions:
249 249 * (%rsp is ready for normal call sequence)
250 250 * Postconditions (if assertion is true):
251 251 * %r11 is smashed
252 252 *
253 253 * ASSERT(rp->r_cs == descnum)
254 254 *
255 255 * The code selector is written into the regs structure when the
256 256 * lwp stack is created. We use this ASSERT to validate that
257 257 * the regs structure really matches how we came in.
258 258 *
259 259 * Preconditions:
260 260 * (%rsp is ready for normal call sequence)
261 261 * Postconditions (if assertion is true):
262 262 * -none-
263 263 *
264 264 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
265 265 *
266 266 * If this is false, it meant that we returned to userland without
267 267 * updating the segment registers as we were supposed to.
268 268 *
269 269 * Note that we must ensure no interrupts or other traps intervene
270 270 * between entering privileged mode and performing the assertion,
271 271 * otherwise we may perform a context switch on the thread, which
272 272 * will end up setting pcb_rupdate to 1 again.
273 273 */
274 274 #if defined(DEBUG)
275 275
276 276 #if !defined(__lint)
277 277
278 278 __lwptoregs_msg:
279 279 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
280 280
281 281 __codesel_msg:
282 282 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
283 283
284 284 __no_rupdate_msg:
285 285 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
286 286
287 287 #endif /* !__lint */
288 288
289 289 #define ASSERT_LWPTOREGS(lwp, rp) \
290 290 movq LWP_REGS(lwp), %r11; \
291 291 cmpq rp, %r11; \
292 292 je 7f; \
293 293 leaq __lwptoregs_msg(%rip), %rdi; \
294 294 movl $__LINE__, %esi; \
295 295 movq lwp, %rdx; \
296 296 movq %r11, %rcx; \
297 297 movq rp, %r8; \
298 298 xorl %eax, %eax; \
299 299 call panic; \
300 300 7:
301 301
302 302 #define ASSERT_NO_RUPDATE_PENDING(lwp) \
303 303 testb $0x1, PCB_RUPDATE(lwp); \
304 304 je 8f; \
305 305 movq lwp, %rdx; \
306 306 leaq __no_rupdate_msg(%rip), %rdi; \
307 307 movl $__LINE__, %esi; \
308 308 xorl %eax, %eax; \
309 309 call panic; \
310 310 8:
311 311
312 312 #else
313 313 #define ASSERT_LWPTOREGS(lwp, rp)
314 314 #define ASSERT_NO_RUPDATE_PENDING(lwp)
315 315 #endif
316 316
317 317 /*
318 318 * Do the traptrace thing and restore any registers we used
319 319 * in situ. Assumes that %rsp is pointing at the base of
320 320 * the struct regs, obviously ..
321 321 */
322 322 #ifdef TRAPTRACE
323 323 #define SYSCALL_TRAPTRACE(ttype) \
324 324 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \
325 325 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \
326 326 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \
327 327 movq REGOFF_RAX(%rsp), %rax; \
328 328 movq REGOFF_RBX(%rsp), %rbx; \
329 329 movq REGOFF_RCX(%rsp), %rcx; \
330 330 movq REGOFF_RDX(%rsp), %rdx; \
331 331 movl %eax, TTR_SYSNUM(%rdi); \
332 332 movq REGOFF_RDI(%rsp), %rdi
333 333
334 334 #define SYSCALL_TRAPTRACE32(ttype) \
335 335 SYSCALL_TRAPTRACE(ttype); \
336 336 /* paranoia: clean the top 32-bits of the registers */ \
337 337 orl %eax, %eax; \
338 338 orl %ebx, %ebx; \
339 339 orl %ecx, %ecx; \
340 340 orl %edx, %edx; \
341 341 orl %edi, %edi
342 342 #else /* TRAPTRACE */
343 343 #define SYSCALL_TRAPTRACE(ttype)
344 344 #define SYSCALL_TRAPTRACE32(ttype)
345 345 #endif /* TRAPTRACE */
346 346
347 347 /*
348 348 * The 64-bit libc syscall wrapper does this:
349 349 *
350 350 * fn(<args>)
351 351 * {
352 352 * movq %rcx, %r10 -- because syscall smashes %rcx
353 353 * movl $CODE, %eax
354 354 * syscall
355 355 * <error processing>
356 356 * }
357 357 *
358 358 * Thus when we come into the kernel:
359 359 *
360 360 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
361 361 * %rax is the syscall number
362 362 * %r12-%r15 contain caller state
363 363 *
364 364 * The syscall instruction arranges that:
365 365 *
366 366 * %rcx contains the return %rip
367 367 * %r11d contains bottom 32-bits of %rflags
368 368 * %rflags is masked (as determined by the SFMASK msr)
369 369 * %cs is set to UCS_SEL (as determined by the STAR msr)
370 370 * %ss is set to UDS_SEL (as determined by the STAR msr)
371 371 * %rip is set to sys_syscall (as determined by the LSTAR msr)
372 372 *
373 373 * Or in other words, we have no registers available at all.
374 374 * Only swapgs can save us!
375 375 *
376 376 * Under the hypervisor, the swapgs has happened already. However, the
377 377 * state of the world is very different from that we're familiar with.
378 378 *
379 379 * In particular, we have a stack structure like that for interrupt
380 380 * gates, except that the %cs and %ss registers are modified for reasons
381 381 * that are not entirely clear. Critically, the %rcx/%r11 values do
382 382 * *not* reflect the usage of those registers under a 'real' syscall[1];
383 383 * the stack, therefore, looks like this:
384 384 *
385 385 * 0x0(rsp) potentially junk %rcx
386 386 * 0x8(rsp) potentially junk %r11
387 387 * 0x10(rsp) user %rip
388 388 * 0x18(rsp) modified %cs
389 389 * 0x20(rsp) user %rflags
390 390 * 0x28(rsp) user %rsp
391 391 * 0x30(rsp) modified %ss
392 392 *
393 393 *
394 394 * and before continuing on, we must load the %rip into %rcx and the
395 395 * %rflags into %r11.
396 396 *
397 397 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
398 398 * Sigh.
399 399 */
400 400 #if defined(__xpv)
401 401 #define XPV_SYSCALL_PROD \
402 402 movq 0x10(%rsp), %rcx; \
403 403 movq 0x20(%rsp), %r11; \
404 404 movq 0x28(%rsp), %rsp
405 405 #else
406 406 #define XPV_SYSCALL_PROD /* nothing */
407 407 #endif
408 408
409 409 #if defined(__lint)
410 410
411 411 /*ARGSUSED*/
412 412 void
413 413 sys_syscall()
414 414 {}
415 415
416 416 void
417 417 _allsyscalls()
418 418 {}
419 419
420 420 size_t _allsyscalls_size;
421 421
422 422 #else /* __lint */
423 423
424 424 ENTRY_NP2(brand_sys_syscall,_allsyscalls)
425 425 SWAPGS /* kernel gsbase */
426 426 XPV_SYSCALL_PROD
427 427 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
428 428 jmp noprod_sys_syscall
429 429
430 430 ALTENTRY(sys_syscall)
431 431 SWAPGS /* kernel gsbase */
432 432 XPV_SYSCALL_PROD
433 433
434 434 noprod_sys_syscall:
435 435 movq %r15, %gs:CPU_RTMP_R15
436 436 movq %rsp, %gs:CPU_RTMP_RSP
437 437
438 438 movq %gs:CPU_THREAD, %r15
439 439 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */
440 440
441 441 ASSERT_UPCALL_MASK_IS_SET
442 442
443 443 movl $UCS_SEL, REGOFF_CS(%rsp)
444 444 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
445 445 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
446 446 movl $UDS_SEL, REGOFF_SS(%rsp)
447 447
448 448 movl %eax, %eax /* wrapper: sysc# -> %eax */
449 449 movq %rdi, REGOFF_RDI(%rsp)
450 450 movq %rsi, REGOFF_RSI(%rsp)
451 451 movq %rdx, REGOFF_RDX(%rsp)
452 452 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */
453 453 movq %r10, %rcx /* arg[3] for direct calls */
454 454
455 455 movq %r8, REGOFF_R8(%rsp)
456 456 movq %r9, REGOFF_R9(%rsp)
457 457 movq %rax, REGOFF_RAX(%rsp)
458 458 movq %rbx, REGOFF_RBX(%rsp)
459 459
460 460 movq %rbp, REGOFF_RBP(%rsp)
461 461 movq %r10, REGOFF_R10(%rsp)
462 462 movq %gs:CPU_RTMP_RSP, %r11
463 463 movq %r11, REGOFF_RSP(%rsp)
464 464 movq %r12, REGOFF_R12(%rsp)
465 465
466 466 movq %r13, REGOFF_R13(%rsp)
467 467 movq %r14, REGOFF_R14(%rsp)
468 468 movq %gs:CPU_RTMP_R15, %r10
469 469 movq %r10, REGOFF_R15(%rsp)
470 470 movq $0, REGOFF_SAVFP(%rsp)
471 471 movq $0, REGOFF_SAVPC(%rsp)
472 472
473 473 /*
474 474 * Copy these registers here in case we end up stopped with
475 475 * someone (like, say, /proc) messing with our register state.
476 476 * We don't -restore- them unless we have to in update_sregs.
477 477 *
478 478 * Since userland -can't- change fsbase or gsbase directly,
479 479 * and capturing them involves two serializing instructions,
480 480 * we don't bother to capture them here.
481 481 */
482 482 xorl %ebx, %ebx
483 483 movw %ds, %bx
484 484 movq %rbx, REGOFF_DS(%rsp)
485 485 movw %es, %bx
486 486 movq %rbx, REGOFF_ES(%rsp)
487 487 movw %fs, %bx
488 488 movq %rbx, REGOFF_FS(%rsp)
489 489 movw %gs, %bx
490 490 movq %rbx, REGOFF_GS(%rsp)
491 491
492 492 /*
493 493 * Machine state saved in the regs structure on the stack
494 494 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
495 495 * %eax is the syscall number
|
↓ open down ↓ |
445 lines elided |
↑ open up ↑ |
496 496 * %rsp is the thread's stack, %r15 is curthread
497 497 * REG_RSP(%rsp) is the user's stack
498 498 */
499 499
500 500 SYSCALL_TRAPTRACE($TT_SYSC64)
501 501
502 502 movq %rsp, %rbp
503 503
504 504 movq T_LWP(%r15), %r14
505 505 ASSERT_NO_RUPDATE_PENDING(%r14)
506 +
506 507 ENABLE_INTR_FLAGS
507 508
508 509 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
509 510 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
510 511
511 512 ASSERT_LWPTOREGS(%r14, %rsp)
512 513
513 514 movb $LWP_SYS, LWP_STATE(%r14)
514 515 incq LWP_RU_SYSC(%r14)
515 516 movb $NORMALRETURN, LWP_EOSYS(%r14)
516 517
517 518 incq %gs:CPU_STATS_SYS_SYSCALL
518 519
520 + /*
521 + * If our LWP has an alternate system call handler, run that instead of
522 + * the regular system call path.
523 + */
524 + movq LWP_BRAND_SYSCALL(%r14), %rdi
525 + testq %rdi, %rdi
526 + jz _syscall_no_brand
527 +
528 + pushq %rax
529 + subq $8, %rsp /* align stack for call to C */
530 + call *%rdi
531 + addq $8, %rsp
532 +
533 + /*
534 + * If the alternate handler returns 0, we skip straight to the return to
535 + * usermode. Otherwise, we resume regular system call processing.
536 + */
537 + testl %eax, %eax
538 + popq %rax
539 + jz _syscall_after_brand
540 +
541 +_syscall_no_brand:
519 542 movw %ax, T_SYSNUM(%r15)
520 543 movzbl T_PRE_SYS(%r15), %ebx
521 544 ORL_SYSCALLTRACE(%ebx)
522 545 testl %ebx, %ebx
523 546 jne _syscall_pre
524 547
525 548 _syscall_invoke:
526 549 movq REGOFF_RDI(%rbp), %rdi
527 550 movq REGOFF_RSI(%rbp), %rsi
528 551 movq REGOFF_RDX(%rbp), %rdx
529 552 movq REGOFF_RCX(%rbp), %rcx
530 553 movq REGOFF_R8(%rbp), %r8
531 554 movq REGOFF_R9(%rbp), %r9
532 555
533 556 cmpl $NSYSCALL, %eax
534 557 jae _syscall_ill
535 558 shll $SYSENT_SIZE_SHIFT, %eax
536 559 leaq sysent(%rax), %rbx
537 560
538 561 call *SY_CALLC(%rbx)
539 562
540 563 movq %rax, %r12
541 564 movq %rdx, %r13
542 565
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
543 566 /*
544 567 * If the handler returns two ints, then we need to split the
545 568 * 64-bit return value into two 32-bit values.
546 569 */
547 570 testw $SE_32RVAL2, SY_FLAGS(%rbx)
548 571 je 5f
549 572 movq %r12, %r13
550 573 shrq $32, %r13 /* upper 32-bits into %edx */
551 574 movl %r12d, %r12d /* lower 32-bits into %eax */
552 575 5:
576 +
577 +_syscall_after_brand:
553 578 /*
554 579 * Optimistically assume that there's no post-syscall
555 580 * work to do. (This is to avoid having to call syscall_mstate()
556 581 * with interrupts disabled)
557 582 */
558 583 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
559 584
560 585 /*
561 586 * We must protect ourselves from being descheduled here;
562 587 * If we were, and we ended up on another cpu, or another
563 588 * lwp got in ahead of us, it could change the segment
564 589 * registers without us noticing before we return to userland.
565 590 */
566 591 CLI(%r14)
567 592 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
568 593 jne _syscall_post
569 594
570 595 /*
571 596 * We need to protect ourselves against non-canonical return values
572 597 * because Intel doesn't check for them on sysret (AMD does). Canonical
573 598 * addresses on current amd64 processors only use 48-bits for VAs; an
574 599 * address is canonical if all upper bits (47-63) are identical. If we
575 600 * find a non-canonical %rip, we opt to go through the full
576 601 * _syscall_post path which takes us into an iretq which is not
577 602 * susceptible to the same problems sysret is.
578 603 *
579 604 * We're checking for a canonical address by first doing an arithmetic
580 605 * shift. This will fill in the remaining bits with the value of bit 63.
581 606 * If the address were canonical, the register would now have either all
582 607 * zeroes or all ones in it. Therefore we add one (inducing overflow)
583 608 * and compare against 1. A canonical address will either be zero or one
584 609 * at this point, hence the use of ja.
585 610 *
586 611 * At this point, r12 and r13 have the return value so we can't use
587 612 * those registers.
588 613 */
589 614 movq REGOFF_RIP(%rsp), %rcx
590 615 sarq $47, %rcx
591 616 incq %rcx
592 617 cmpq $1, %rcx
593 618 ja _syscall_post
594 619
595 620
596 621 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
597 622
598 623 movq %r12, REGOFF_RAX(%rsp)
599 624 movq %r13, REGOFF_RDX(%rsp)
600 625
601 626 /*
602 627 * To get back to userland, we need the return %rip in %rcx and
603 628 * the return %rfl in %r11d. The sysretq instruction also arranges
604 629 * to fix up %cs and %ss; everything else is our responsibility.
605 630 */
606 631 movq REGOFF_RDI(%rsp), %rdi
607 632 movq REGOFF_RSI(%rsp), %rsi
608 633 movq REGOFF_RDX(%rsp), %rdx
609 634 /* %rcx used to restore %rip value */
610 635
611 636 movq REGOFF_R8(%rsp), %r8
612 637 movq REGOFF_R9(%rsp), %r9
613 638 movq REGOFF_RAX(%rsp), %rax
614 639 movq REGOFF_RBX(%rsp), %rbx
615 640
616 641 movq REGOFF_RBP(%rsp), %rbp
617 642 movq REGOFF_R10(%rsp), %r10
618 643 /* %r11 used to restore %rfl value */
619 644 movq REGOFF_R12(%rsp), %r12
620 645
621 646 movq REGOFF_R13(%rsp), %r13
622 647 movq REGOFF_R14(%rsp), %r14
623 648 movq REGOFF_R15(%rsp), %r15
624 649
625 650 movq REGOFF_RIP(%rsp), %rcx
626 651 movl REGOFF_RFL(%rsp), %r11d
627 652
628 653 #if defined(__xpv)
629 654 addq $REGOFF_RIP, %rsp
630 655 #else
631 656 movq REGOFF_RSP(%rsp), %rsp
632 657 #endif
633 658
634 659 /*
635 660 * There can be no instructions between the ALTENTRY below and
636 661 * SYSRET or we could end up breaking brand support. See label usage
637 662 * in sn1_brand_syscall_callback for an example.
638 663 */
639 664 ASSERT_UPCALL_MASK_IS_SET
640 665 #if defined(__xpv)
641 666 SYSRETQ
642 667 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
643 668
644 669 /*
645 670 * We can only get here after executing a brand syscall
646 671 * interposition callback handler and simply need to
647 672 * "sysretq" back to userland. On the hypervisor this
648 673 * involves the iret hypercall which requires us to construct
649 674 * just enough of the stack needed for the hypercall.
650 675 * (rip, cs, rflags, rsp, ss).
651 676 */
652 677 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */
653 678 movq %gs:CPU_THREAD, %r11
654 679 movq T_STACK(%r11), %rsp
655 680
656 681 movq %rcx, REGOFF_RIP(%rsp)
657 682 movl $UCS_SEL, REGOFF_CS(%rsp)
658 683 movq %gs:CPU_RTMP_RSP, %r11
659 684 movq %r11, REGOFF_RSP(%rsp)
660 685 pushfq
661 686 popq %r11 /* hypercall enables ints */
662 687 movq %r11, REGOFF_RFL(%rsp)
663 688 movl $UDS_SEL, REGOFF_SS(%rsp)
664 689 addq $REGOFF_RIP, %rsp
665 690 /*
666 691 * XXPV: see comment in SYSRETQ definition for future optimization
667 692 * we could take.
668 693 */
669 694 ASSERT_UPCALL_MASK_IS_SET
670 695 SYSRETQ
671 696 #else
672 697 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
673 698 SWAPGS /* user gsbase */
674 699 SYSRETQ
675 700 #endif
676 701 /*NOTREACHED*/
677 702 SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
678 703
679 704 _syscall_pre:
680 705 call pre_syscall
681 706 movl %eax, %r12d
682 707 testl %eax, %eax
683 708 jne _syscall_post_call
684 709 /*
685 710 * Didn't abort, so reload the syscall args and invoke the handler.
686 711 */
687 712 movzwl T_SYSNUM(%r15), %eax
688 713 jmp _syscall_invoke
689 714
690 715 _syscall_ill:
691 716 call nosys
692 717 movq %rax, %r12
693 718 movq %rdx, %r13
694 719 jmp _syscall_post_call
695 720
696 721 _syscall_post:
697 722 STI
698 723 /*
699 724 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
700 725 * so that we can account for the extra work it takes us to finish.
701 726 */
702 727 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
703 728 _syscall_post_call:
704 729 movq %r12, %rdi
705 730 movq %r13, %rsi
706 731 call post_syscall
707 732 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
708 733 jmp _sys_rtt
709 734 SET_SIZE(sys_syscall)
710 735 SET_SIZE(brand_sys_syscall)
711 736
712 737 #endif /* __lint */
713 738
714 739 #if defined(__lint)
715 740
716 741 /*ARGSUSED*/
717 742 void
718 743 sys_syscall32()
719 744 {}
720 745
721 746 #else /* __lint */
722 747
723 748 ENTRY_NP(brand_sys_syscall32)
724 749 SWAPGS /* kernel gsbase */
725 750 XPV_TRAP_POP
726 751 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
727 752 jmp nopop_sys_syscall32
728 753
729 754 ALTENTRY(sys_syscall32)
730 755 SWAPGS /* kernel gsbase */
731 756 XPV_TRAP_POP
732 757
733 758 nopop_sys_syscall32:
734 759 movl %esp, %r10d
735 760 movq %gs:CPU_THREAD, %r15
736 761 movq T_STACK(%r15), %rsp
737 762 movl %eax, %eax
738 763
739 764 movl $U32CS_SEL, REGOFF_CS(%rsp)
740 765 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
741 766 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
742 767 movq %r10, REGOFF_RSP(%rsp)
743 768 movl $UDS_SEL, REGOFF_SS(%rsp)
744 769
745 770 _syscall32_save:
746 771 movl %edi, REGOFF_RDI(%rsp)
747 772 movl %esi, REGOFF_RSI(%rsp)
748 773 movl %ebp, REGOFF_RBP(%rsp)
749 774 movl %ebx, REGOFF_RBX(%rsp)
750 775 movl %edx, REGOFF_RDX(%rsp)
751 776 movl %ecx, REGOFF_RCX(%rsp)
752 777 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
753 778 movq $0, REGOFF_SAVFP(%rsp)
754 779 movq $0, REGOFF_SAVPC(%rsp)
755 780
756 781 /*
757 782 * Copy these registers here in case we end up stopped with
758 783 * someone (like, say, /proc) messing with our register state.
759 784 * We don't -restore- them unless we have to in update_sregs.
760 785 *
761 786 * Since userland -can't- change fsbase or gsbase directly,
762 787 * we don't bother to capture them here.
763 788 */
764 789 xorl %ebx, %ebx
765 790 movw %ds, %bx
766 791 movq %rbx, REGOFF_DS(%rsp)
767 792 movw %es, %bx
768 793 movq %rbx, REGOFF_ES(%rsp)
769 794 movw %fs, %bx
770 795 movq %rbx, REGOFF_FS(%rsp)
771 796 movw %gs, %bx
772 797 movq %rbx, REGOFF_GS(%rsp)
773 798
774 799 /*
775 800 * Application state saved in the regs structure on the stack
776 801 * %eax is the syscall number
777 802 * %rsp is the thread's stack, %r15 is curthread
778 803 * REG_RSP(%rsp) is the user's stack
779 804 */
780 805
781 806 SYSCALL_TRAPTRACE32($TT_SYSC)
782 807
783 808 movq %rsp, %rbp
784 809
785 810 movq T_LWP(%r15), %r14
786 811 ASSERT_NO_RUPDATE_PENDING(%r14)
787 812
|
↓ open down ↓ |
225 lines elided |
↑ open up ↑ |
788 813 ENABLE_INTR_FLAGS
789 814
790 815 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
791 816 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
792 817
793 818 ASSERT_LWPTOREGS(%r14, %rsp)
794 819
795 820 incq %gs:CPU_STATS_SYS_SYSCALL
796 821
797 822 /*
823 + * If our lwp has an alternate system call handler, run that instead
824 + * of the regular system call path.
825 + */
826 + movq LWP_BRAND_SYSCALL(%r14), %rax
827 + testq %rax, %rax
828 + jz _syscall32_no_brand
829 +
830 + movb $LWP_SYS, LWP_STATE(%r14)
831 + call *%rax
832 +
833 + /*
834 + * If the alternate handler returns 0, we skip straight to the return
835 + * to usermode. Otherwise, we resume regular system call processing.
836 + */
837 + testl %eax, %eax
838 + jz _syscall32_after_brand
839 +
840 +_syscall32_no_brand:
841 + /*
798 842 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
799 843 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or
800 844 * more succinctly:
801 845 *
802 846 * SA(MAXSYSARGS * sizeof (long)) == 64
847 + *
848 + * Note, this space is used both to copy in the arguments from user
849 + * land, but also to as part of the old UNIX style syscall_ap() method.
850 + * syscall_entry expects that we do not change the values of this space
851 + * that we give it. However, this means that when we end up in the more
852 + * recent model of passing the arguments based on the calling
853 + * conventions, we'll need to save an additional 16 bytes of stack.
803 854 */
804 855 #define SYS_DROP 64 /* drop for args */
805 856 subq $SYS_DROP, %rsp
806 857 movb $LWP_SYS, LWP_STATE(%r14)
807 858 movq %r15, %rdi
808 859 movq %rsp, %rsi
809 860 call syscall_entry
810 861
811 862 /*
812 863 * Fetch the arguments copied onto the kernel stack and put
813 864 * them in the right registers to invoke a C-style syscall handler.
814 865 * %rax contains the handler address.
815 866 *
816 867 * Ideas for making all this go faster of course include simply
817 868 * forcibly fetching 6 arguments from the user stack under lofault
818 869 * protection, reverting to copyin_args only when watchpoints
819 870 * are in effect.
|
↓ open down ↓ |
7 lines elided |
↑ open up ↑ |
820 871 *
821 872 * (If we do this, make sure that exec and libthread leave
822 873 * enough space at the top of the stack to ensure that we'll
823 874 * never do a fetch from an invalid page.)
824 875 *
825 876 * Lots of ideas here, but they won't really help with bringup B-)
826 877 * Correctness can't wait, performance can wait a little longer ..
827 878 */
828 879
829 880 movq %rax, %rbx
830 - movl 0(%rsp), %edi
831 - movl 8(%rsp), %esi
832 - movl 0x10(%rsp), %edx
833 - movl 0x18(%rsp), %ecx
834 - movl 0x20(%rsp), %r8d
835 - movl 0x28(%rsp), %r9d
881 + movl 0x0(%rsp), %edi /* arg0 */
882 + movl 0x8(%rsp), %esi /* arg1 */
883 + movl 0x10(%rsp), %edx /* arg2 */
884 + movl 0x38(%rsp), %eax /* arg7 load */
885 + movl 0x18(%rsp), %ecx /* arg3 */
886 + pushq %rax /* arg7 saved to stack */
887 + movl 0x28(%rsp), %r8d /* arg4 */
888 + movl 0x38(%rsp), %eax /* arg6 load */
889 + movl 0x30(%rsp), %r9d /* arg5 */
890 + pushq %rax /* arg6 saved to stack */
836 891
837 892 call *SY_CALLC(%rbx)
838 893
839 894 movq %rbp, %rsp /* pop the args */
840 895
841 896 /*
842 897 * amd64 syscall handlers -always- return a 64-bit value in %rax.
843 898 * On the 32-bit kernel, they always return that value in %eax:%edx
844 899 * as required by the 32-bit ABI.
845 900 *
846 901 * Simulate the same behaviour by unconditionally splitting the
847 902 * return value in the same way.
848 903 */
849 904 movq %rax, %r13
850 905 shrq $32, %r13 /* upper 32-bits into %edx */
851 906 movl %eax, %r12d /* lower 32-bits into %eax */
852 907
908 +_syscall32_after_brand:
909 +
853 910 /*
854 911 * Optimistically assume that there's no post-syscall
855 912 * work to do. (This is to avoid having to call syscall_mstate()
856 913 * with interrupts disabled)
857 914 */
858 915 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
859 916
860 917 /*
861 918 * We must protect ourselves from being descheduled here;
862 919 * If we were, and we ended up on another cpu, or another
863 920 * lwp got in ahead of us, it could change the segment
864 921 * registers without us noticing before we return to userland.
865 922 */
866 923 CLI(%r14)
867 924 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
868 925 jne _full_syscall_postsys32
869 926 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
870 927
871 928 /*
872 929 * To get back to userland, we need to put the return %rip in %rcx and
873 930 * the return %rfl in %r11d. The sysret instruction also arranges
874 931 * to fix up %cs and %ss; everything else is our responsibility.
875 932 */
876 933
877 934 movl %r12d, %eax /* %eax: rval1 */
878 935 movl REGOFF_RBX(%rsp), %ebx
879 936 /* %ecx used for return pointer */
880 937 movl %r13d, %edx /* %edx: rval2 */
881 938 movl REGOFF_RBP(%rsp), %ebp
882 939 movl REGOFF_RSI(%rsp), %esi
883 940 movl REGOFF_RDI(%rsp), %edi
884 941
885 942 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */
886 943 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */
887 944 movl REGOFF_RSP(%rsp), %esp
888 945
889 946 ASSERT_UPCALL_MASK_IS_SET
890 947 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
891 948 SWAPGS /* user gsbase */
892 949 SYSRETL
893 950 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
894 951 /*NOTREACHED*/
895 952
896 953 _full_syscall_postsys32:
897 954 STI
898 955 /*
899 956 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
900 957 * so that we can account for the extra work it takes us to finish.
901 958 */
902 959 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
903 960 movq %r15, %rdi
904 961 movq %r12, %rsi /* rval1 - %eax */
905 962 movq %r13, %rdx /* rval2 - %edx */
906 963 call syscall_exit
907 964 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
908 965 jmp _sys_rtt
909 966 SET_SIZE(sys_syscall32)
910 967 SET_SIZE(brand_sys_syscall32)
911 968
912 969 #endif /* __lint */
913 970
914 971 /*
915 972 * System call handler via the sysenter instruction
916 973 * Used only for 32-bit system calls on the 64-bit kernel.
917 974 *
918 975 * The caller in userland has arranged that:
919 976 *
920 977 * - %eax contains the syscall number
921 978 * - %ecx contains the user %esp
922 979 * - %edx contains the return %eip
923 980 * - the user stack contains the args to the syscall
924 981 *
925 982 * Hardware and (privileged) initialization code have arranged that by
926 983 * the time the sysenter instructions completes:
927 984 *
928 985 * - %rip is pointing to sys_sysenter (below).
929 986 * - %cs and %ss are set to kernel text and stack (data) selectors.
930 987 * - %rsp is pointing at the lwp's stack
931 988 * - interrupts have been disabled.
932 989 *
933 990 * Note that we are unable to return both "rvals" to userland with
934 991 * this call, as %edx is used by the sysexit instruction.
935 992 *
936 993 * One final complication in this routine is its interaction with
937 994 * single-stepping in a debugger. For most of the system call mechanisms,
938 995 * the CPU automatically clears the single-step flag before we enter the
939 996 * kernel. The sysenter mechanism does not clear the flag, so a user
940 997 * single-stepping through a libc routine may suddenly find him/herself
941 998 * single-stepping through the kernel. To detect this, kmdb compares the
942 999 * trap %pc to the [brand_]sys_enter addresses on each single-step trap.
943 1000 * If it finds that we have single-stepped to a sysenter entry point, it
944 1001 * explicitly clears the flag and executes the sys_sysenter routine.
945 1002 *
946 1003 * One final complication in this final complication is the fact that we
947 1004 * have two different entry points for sysenter: brand_sys_sysenter and
948 1005 * sys_sysenter. If we enter at brand_sys_sysenter and start single-stepping
949 1006 * through the kernel with kmdb, we will eventually hit the instruction at
950 1007 * sys_sysenter. kmdb cannot distinguish between that valid single-step
951 1008 * and the undesirable one mentioned above. To avoid this situation, we
952 1009 * simply add a jump over the instruction at sys_sysenter to make it
953 1010 * impossible to single-step to it.
954 1011 */
955 1012 #if defined(__lint)
956 1013
957 1014 void
958 1015 sys_sysenter()
959 1016 {}
960 1017
961 1018 #else /* __lint */
962 1019
963 1020 ENTRY_NP(brand_sys_sysenter)
964 1021 SWAPGS /* kernel gsbase */
965 1022 ALTENTRY(_brand_sys_sysenter_post_swapgs)
966 1023 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
967 1024 /*
968 1025 * Jump over sys_sysenter to allow single-stepping as described
969 1026 * above.
970 1027 */
971 1028 jmp _sys_sysenter_post_swapgs
972 1029
973 1030 ALTENTRY(sys_sysenter)
974 1031 SWAPGS /* kernel gsbase */
975 1032
976 1033 ALTENTRY(_sys_sysenter_post_swapgs)
977 1034 movq %gs:CPU_THREAD, %r15
978 1035
979 1036 movl $U32CS_SEL, REGOFF_CS(%rsp)
980 1037 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */
981 1038 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */
982 1039 pushfq
983 1040 popq %r10
984 1041 movl $UDS_SEL, REGOFF_SS(%rsp)
985 1042
986 1043 /*
987 1044 * Set the interrupt flag before storing the flags to the
988 1045 * flags image on the stack so we can return to user with
989 1046 * interrupts enabled if we return via sys_rtt_syscall32
990 1047 */
991 1048 orq $PS_IE, %r10
992 1049 movq %r10, REGOFF_RFL(%rsp)
993 1050
994 1051 movl %edi, REGOFF_RDI(%rsp)
995 1052 movl %esi, REGOFF_RSI(%rsp)
996 1053 movl %ebp, REGOFF_RBP(%rsp)
997 1054 movl %ebx, REGOFF_RBX(%rsp)
998 1055 movl %edx, REGOFF_RDX(%rsp)
999 1056 movl %ecx, REGOFF_RCX(%rsp)
1000 1057 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
1001 1058 movq $0, REGOFF_SAVFP(%rsp)
1002 1059 movq $0, REGOFF_SAVPC(%rsp)
1003 1060
1004 1061 /*
1005 1062 * Copy these registers here in case we end up stopped with
1006 1063 * someone (like, say, /proc) messing with our register state.
1007 1064 * We don't -restore- them unless we have to in update_sregs.
1008 1065 *
1009 1066 * Since userland -can't- change fsbase or gsbase directly,
1010 1067 * we don't bother to capture them here.
1011 1068 */
1012 1069 xorl %ebx, %ebx
1013 1070 movw %ds, %bx
1014 1071 movq %rbx, REGOFF_DS(%rsp)
1015 1072 movw %es, %bx
1016 1073 movq %rbx, REGOFF_ES(%rsp)
1017 1074 movw %fs, %bx
1018 1075 movq %rbx, REGOFF_FS(%rsp)
1019 1076 movw %gs, %bx
1020 1077 movq %rbx, REGOFF_GS(%rsp)
1021 1078
1022 1079 /*
1023 1080 * Application state saved in the regs structure on the stack
1024 1081 * %eax is the syscall number
1025 1082 * %rsp is the thread's stack, %r15 is curthread
1026 1083 * REG_RSP(%rsp) is the user's stack
1027 1084 */
1028 1085
1029 1086 SYSCALL_TRAPTRACE($TT_SYSENTER)
1030 1087
1031 1088 movq %rsp, %rbp
1032 1089
1033 1090 movq T_LWP(%r15), %r14
1034 1091 ASSERT_NO_RUPDATE_PENDING(%r14)
1035 1092
1036 1093 ENABLE_INTR_FLAGS
1037 1094
1038 1095 /*
1039 1096 * Catch 64-bit process trying to issue sysenter instruction
1040 1097 * on Nocona based systems.
1041 1098 */
1042 1099 movq LWP_PROCP(%r14), %rax
1043 1100 cmpq $DATAMODEL_ILP32, P_MODEL(%rax)
1044 1101 je 7f
1045 1102
1046 1103 /*
1047 1104 * For a non-32-bit process, simulate a #ud, since that's what
1048 1105 * native hardware does. The traptrace entry (above) will
1049 1106 * let you know what really happened.
1050 1107 */
1051 1108 movq $T_ILLINST, REGOFF_TRAPNO(%rsp)
1052 1109 movq REGOFF_CS(%rsp), %rdi
1053 1110 movq %rdi, REGOFF_ERR(%rsp)
1054 1111 movq %rsp, %rdi
1055 1112 movq REGOFF_RIP(%rsp), %rsi
1056 1113 movl %gs:CPU_ID, %edx
1057 1114 call trap
1058 1115 jmp _sys_rtt
1059 1116 7:
1060 1117
1061 1118 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1062 1119 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */
1063 1120
1064 1121 ASSERT_LWPTOREGS(%r14, %rsp)
1065 1122
1066 1123 incq %gs:CPU_STATS_SYS_SYSCALL
1067 1124
1068 1125 /*
1069 1126 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1070 1127 * placed into 64-bit (long) arg slots, plus one 64-bit
1071 1128 * (long) arg count, maintaining 16 byte alignment.
|
↓ open down ↓ |
209 lines elided |
↑ open up ↑ |
1072 1129 */
1073 1130 subq $SYS_DROP, %rsp
1074 1131 movb $LWP_SYS, LWP_STATE(%r14)
1075 1132 movq %r15, %rdi
1076 1133 movq %rsp, %rsi
1077 1134 call syscall_entry
1078 1135
1079 1136 /*
1080 1137 * Fetch the arguments copied onto the kernel stack and put
1081 1138 * them in the right registers to invoke a C-style syscall handler.
1082 - * %rax contains the handler address.
1139 + * %rax contains the handler address. For the last two arguments, we
1140 + * push them onto the stack -- we can't clobber the old arguments.
1083 1141 */
1084 1142 movq %rax, %rbx
1085 - movl 0(%rsp), %edi
1086 - movl 8(%rsp), %esi
1087 - movl 0x10(%rsp), %edx
1088 - movl 0x18(%rsp), %ecx
1089 - movl 0x20(%rsp), %r8d
1090 - movl 0x28(%rsp), %r9d
1143 + movl 0x0(%rsp), %edi /* arg0 */
1144 + movl 0x8(%rsp), %esi /* arg1 */
1145 + movl 0x10(%rsp), %edx /* arg2 */
1146 + movl 0x38(%rsp), %eax /* arg7 load */
1147 + movl 0x18(%rsp), %ecx /* arg3 */
1148 + pushq %rax /* arg7 saved to stack */
1149 + movl 0x28(%rsp), %r8d /* arg4 */
1150 + movl 0x38(%rsp), %eax /* arg6 load */
1151 + movl 0x30(%rsp), %r9d /* arg5 */
1152 + pushq %rax /* arg6 saved to stack */
1091 1153
1092 1154 call *SY_CALLC(%rbx)
1093 1155
1094 1156 movq %rbp, %rsp /* pop the args */
1095 1157
1096 1158 /*
1097 1159 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1098 1160 * On the 32-bit kernel, the always return that value in %eax:%edx
1099 1161 * as required by the 32-bit ABI.
1100 1162 *
1101 1163 * Simulate the same behaviour by unconditionally splitting the
1102 1164 * return value in the same way.
1103 1165 */
1104 1166 movq %rax, %r13
1105 1167 shrq $32, %r13 /* upper 32-bits into %edx */
1106 1168 movl %eax, %r12d /* lower 32-bits into %eax */
1107 1169
1108 1170 /*
1109 1171 * Optimistically assume that there's no post-syscall
1110 1172 * work to do. (This is to avoid having to call syscall_mstate()
1111 1173 * with interrupts disabled)
1112 1174 */
1113 1175 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1114 1176
1115 1177 /*
1116 1178 * We must protect ourselves from being descheduled here;
1117 1179 * If we were, and we ended up on another cpu, or another
1118 1180 * lwp got int ahead of us, it could change the segment
1119 1181 * registers without us noticing before we return to userland.
1120 1182 */
1121 1183 cli
1122 1184 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1123 1185 jne _full_syscall_postsys32
1124 1186 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1125 1187
1126 1188 /*
1127 1189 * To get back to userland, load up the 32-bit registers and
1128 1190 * sysexit back where we came from.
1129 1191 */
1130 1192
1131 1193 /*
1132 1194 * Interrupts will be turned on by the 'sti' executed just before
1133 1195 * sysexit. The following ensures that restoring the user's rflags
1134 1196 * doesn't enable interrupts too soon.
1135 1197 */
1136 1198 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1137 1199
1138 1200 /*
1139 1201 * (There's no point in loading up %edx because the sysexit
1140 1202 * mechanism smashes it.)
1141 1203 */
1142 1204 movl %r12d, %eax
1143 1205 movl REGOFF_RBX(%rsp), %ebx
1144 1206 movl REGOFF_RBP(%rsp), %ebp
1145 1207 movl REGOFF_RSI(%rsp), %esi
1146 1208 movl REGOFF_RDI(%rsp), %edi
1147 1209
1148 1210 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */
1149 1211 pushq REGOFF_RFL(%rsp)
1150 1212 popfq
1151 1213 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */
|
↓ open down ↓ |
51 lines elided |
↑ open up ↑ |
1152 1214 ALTENTRY(sys_sysenter_swapgs_sysexit)
1153 1215 swapgs
1154 1216 sti
1155 1217 sysexit
1156 1218 SET_SIZE(sys_sysenter_swapgs_sysexit)
1157 1219 SET_SIZE(sys_sysenter)
1158 1220 SET_SIZE(_sys_sysenter_post_swapgs)
1159 1221 SET_SIZE(brand_sys_sysenter)
1160 1222
1161 1223 #endif /* __lint */
1224 +
1225 +#if defined(__lint)
1226 +/*
1227 + * System call via an int80. This entry point is only used by the Linux
1228 + * application environment. Unlike the other entry points, there is no
1229 + * default action to take if no callback is registered for this process.
1230 + */
1231 +void
1232 +sys_int80()
1233 +{}
1162 1234
1235 +#else /* __lint */
1236 +
1237 + ENTRY_NP(brand_sys_int80)
1238 + SWAPGS /* kernel gsbase */
1239 + XPV_TRAP_POP
1240 +
1241 + /*
1242 + * We first attempt to call the "b_int80" handler from the "struct
1243 + * brand_mach_ops" for this brand. If no handler function is installed
1244 + * for this brand, the BRAND_CALLBACK() macro returns here and we
1245 + * check the lwp for a "lwp_brand_syscall" handler.
1246 + */
1247 + BRAND_CALLBACK(BRAND_CB_INT80, BRAND_URET_FROM_INTR_STACK())
1248 +
1249 + /*
1250 + * Check to see if this lwp provides "lwp_brand_syscall". If so, we
1251 + * will route this int80 through the regular system call handling path.
1252 + */
1253 + movq %r15, %gs:CPU_RTMP_R15
1254 + movq %gs:CPU_THREAD, %r15
1255 + movq T_LWP(%r15), %r15
1256 + movq LWP_BRAND_SYSCALL(%r15), %r15
1257 + testq %r15, %r15
1258 + movq %gs:CPU_RTMP_R15, %r15
1259 + jnz nopop_syscall_int
1260 +
1261 + /*
1262 + * The brand provided neither a "b_int80", nor a "lwp_brand_syscall"
1263 + * function, and has thus opted out of handling this trap.
1264 + */
1265 + SWAPGS /* user gsbase */
1266 + jmp nopop_int80
1267 +
1268 + ENTRY_NP(sys_int80)
1269 + /*
1270 + * We hit an int80, but this process isn't of a brand with an int80
1271 + * handler. Bad process! Make it look as if the INT failed.
1272 + * Modify %rip to point before the INT, push the expected error
1273 + * code and fake a GP fault. Note on 64-bit hypervisor we need
1274 + * to undo the XPV_TRAP_POP and push rcx and r11 back on the stack
1275 + * because gptrap will pop them again with its own XPV_TRAP_POP.
1276 + */
1277 + XPV_TRAP_POP
1278 +nopop_int80:
1279 + subq $2, (%rsp) /* int insn 2-bytes */
1280 + pushq $_CONST(_MUL(T_INT80, GATE_DESC_SIZE) + 2)
1281 +#if defined(__xpv)
1282 + push %r11
1283 + push %rcx
1284 +#endif
1285 + jmp gptrap / GP fault
1286 + SET_SIZE(sys_int80)
1287 + SET_SIZE(brand_sys_int80)
1288 +#endif /* __lint */
1289 +
1290 +
1163 1291 /*
1164 1292 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1165 1293 * the generic i386 libc to do system calls. We do a small amount of setup
1166 1294 * before jumping into the existing sys_syscall32 path.
1167 1295 */
1168 1296 #if defined(__lint)
1169 1297
1170 1298 /*ARGSUSED*/
1171 1299 void
1172 1300 sys_syscall_int()
1173 1301 {}
1174 1302
1175 1303 #else /* __lint */
1176 1304
1177 1305 ENTRY_NP(brand_sys_syscall_int)
1178 1306 SWAPGS /* kernel gsbase */
1179 1307 XPV_TRAP_POP
1180 1308 call smap_enable
1181 1309 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1182 1310 jmp nopop_syscall_int
1183 1311
1184 1312 ALTENTRY(sys_syscall_int)
1185 1313 SWAPGS /* kernel gsbase */
1186 1314 XPV_TRAP_POP
1187 1315 call smap_enable
1188 1316
1189 1317 nopop_syscall_int:
1190 1318 movq %gs:CPU_THREAD, %r15
1191 1319 movq T_STACK(%r15), %rsp
1192 1320 movl %eax, %eax
1193 1321 /*
1194 1322 * Set t_post_sys on this thread to force ourselves out via the slow
1195 1323 * path. It might be possible at some later date to optimize this out
1196 1324 * and use a faster return mechanism.
1197 1325 */
1198 1326 movb $1, T_POST_SYS(%r15)
1199 1327 CLEAN_CS
1200 1328 jmp _syscall32_save
1201 1329 /*
1202 1330 * There should be no instructions between this label and SWAPGS/IRET
1203 1331 * or we could end up breaking branded zone support. See the usage of
1204 1332 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1205 1333 * for examples.
1206 1334 */
1207 1335 ALTENTRY(sys_sysint_swapgs_iret)
1208 1336 SWAPGS /* user gsbase */
1209 1337 IRET
1210 1338 /*NOTREACHED*/
1211 1339 SET_SIZE(sys_sysint_swapgs_iret)
1212 1340 SET_SIZE(sys_syscall_int)
1213 1341 SET_SIZE(brand_sys_syscall_int)
1214 1342
1215 1343 #endif /* __lint */
1216 1344
1217 1345 /*
1218 1346 * Legacy 32-bit applications and old libc implementations do lcalls;
1219 1347 * we should never get here because the LDT entry containing the syscall
1220 1348 * segment descriptor has the "segment present" bit cleared, which means
1221 1349 * we end up processing those system calls in trap() via a not-present trap.
1222 1350 *
1223 1351 * We do it this way because a call gate unhelpfully does -nothing- to the
1224 1352 * interrupt flag bit, so an interrupt can run us just after the lcall
1225 1353 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and
1226 1354 * INTR_POP paths would have to be slightly more complex to dance around
1227 1355 * this problem, and end up depending explicitly on the first
1228 1356 * instruction of this handler being either swapgs or cli.
1229 1357 */
1230 1358
1231 1359 #if defined(__lint)
1232 1360
1233 1361 /*ARGSUSED*/
1234 1362 void
1235 1363 sys_lcall32()
1236 1364 {}
1237 1365
1238 1366 #else /* __lint */
1239 1367
1240 1368 ENTRY_NP(sys_lcall32)
1241 1369 SWAPGS /* kernel gsbase */
1242 1370 pushq $0
1243 1371 pushq %rbp
1244 1372 movq %rsp, %rbp
1245 1373 leaq __lcall_panic_str(%rip), %rdi
1246 1374 xorl %eax, %eax
1247 1375 call panic
1248 1376 SET_SIZE(sys_lcall32)
1249 1377
1250 1378 __lcall_panic_str:
1251 1379 .string "sys_lcall32: shouldn't be here!"
1252 1380
1253 1381 /*
1254 1382 * Declare a uintptr_t which covers the entire pc range of syscall
1255 1383 * handlers for the stack walkers that need this.
1256 1384 */
1257 1385 .align CPTRSIZE
1258 1386 .globl _allsyscalls_size
1259 1387 .type _allsyscalls_size, @object
1260 1388 _allsyscalls_size:
1261 1389 .NWORD . - _allsyscalls
1262 1390 SET_SIZE(_allsyscalls_size)
1263 1391
1264 1392 #endif /* __lint */
1265 1393
1266 1394 /*
1267 1395 * These are the thread context handlers for lwps using sysenter/sysexit.
1268 1396 */
1269 1397
1270 1398 #if defined(__lint)
1271 1399
1272 1400 /*ARGSUSED*/
1273 1401 void
1274 1402 sep_save(void *ksp)
1275 1403 {}
1276 1404
1277 1405 /*ARGSUSED*/
1278 1406 void
1279 1407 sep_restore(void *ksp)
1280 1408 {}
1281 1409
1282 1410 #else /* __lint */
1283 1411
1284 1412 /*
1285 1413 * setting this value to zero as we switch away causes the
1286 1414 * stack-pointer-on-sysenter to be NULL, ensuring that we
1287 1415 * don't silently corrupt another (preempted) thread stack
1288 1416 * when running an lwp that (somehow) didn't get sep_restore'd
1289 1417 */
1290 1418 ENTRY_NP(sep_save)
1291 1419 xorl %edx, %edx
1292 1420 xorl %eax, %eax
1293 1421 movl $MSR_INTC_SEP_ESP, %ecx
1294 1422 wrmsr
1295 1423 ret
1296 1424 SET_SIZE(sep_save)
1297 1425
1298 1426 /*
1299 1427 * Update the kernel stack pointer as we resume onto this cpu.
1300 1428 */
1301 1429 ENTRY_NP(sep_restore)
1302 1430 movq %rdi, %rdx
1303 1431 shrq $32, %rdx
1304 1432 movl %edi, %eax
1305 1433 movl $MSR_INTC_SEP_ESP, %ecx
1306 1434 wrmsr
1307 1435 ret
1308 1436 SET_SIZE(sep_restore)
1309 1437
1310 1438 #endif /* __lint */
|
↓ open down ↓ |
138 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX