1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 */
27
28 /*
29 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
30 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
31 * All Rights Reserved
32 */
33
34 /*
35 * Copyright (c) 2009, Intel Corporation.
36 * All rights reserved.
37 */
38
39 /*
40 * General assembly language routines.
41 * It is the intent of this file to contain routines that are
42 * independent of the specific kernel architecture, and those that are
43 * common across kernel architectures.
44 * As architectures diverge, and implementations of specific
45 * architecture-dependent routines change, the routines should be moved
46 * from this file into the respective ../`arch -k`/subr.s file.
47 */
48
49 #include <sys/asm_linkage.h>
50 #include <sys/asm_misc.h>
51 #include <sys/panic.h>
52 #include <sys/ontrap.h>
53 #include <sys/regset.h>
54 #include <sys/privregs.h>
55 #include <sys/reboot.h>
56 #include <sys/psw.h>
57 #include <sys/x86_archext.h>
58
59 #if defined(__lint)
60 #include <sys/types.h>
61 #include <sys/systm.h>
62 #include <sys/thread.h>
63 #include <sys/archsystm.h>
64 #include <sys/byteorder.h>
65 #include <sys/dtrace.h>
66 #include <sys/ftrace.h>
67 #else /* __lint */
68 #include "assym.h"
69 #endif /* __lint */
70 #include <sys/dditypes.h>
71
72 /*
73 * on_fault()
74 *
75 * Catch lofault faults. Like setjmp except it returns one
76 * if code following causes uncorrectable fault. Turned off
77 * by calling no_fault(). Note that while under on_fault(),
78 * SMAP is disabled. For more information see
79 * uts/intel/ia32/ml/copy.s.
80 */
81
82 #if defined(__lint)
83
84 /* ARGSUSED */
85 int
86 on_fault(label_t *ljb)
87 { return (0); }
88
89 void
90 no_fault(void)
91 {}
92
93 #else /* __lint */
94
95 #if defined(__amd64)
96
97 ENTRY(on_fault)
98 movq %gs:CPU_THREAD, %rsi
99 leaq catch_fault(%rip), %rdx
100 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
101 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
102 call smap_disable /* allow user accesses */
103 jmp setjmp /* let setjmp do the rest */
104
105 catch_fault:
106 movq %gs:CPU_THREAD, %rsi
107 movq T_ONFAULT(%rsi), %rdi /* address of save area */
108 xorl %eax, %eax
109 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
110 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
111 call smap_enable /* disallow user accesses */
112 jmp longjmp /* let longjmp do the rest */
113 SET_SIZE(on_fault)
114
115 ENTRY(no_fault)
116 movq %gs:CPU_THREAD, %rsi
117 xorl %eax, %eax
118 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
119 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
120 call smap_enable /* disallow user accesses */
121 ret
122 SET_SIZE(no_fault)
123
124 #elif defined(__i386)
125
126 ENTRY(on_fault)
127 movl %gs:CPU_THREAD, %edx
128 movl 4(%esp), %eax /* jumpbuf address */
129 leal catch_fault, %ecx
130 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
131 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
132 jmp setjmp /* let setjmp do the rest */
133
134 catch_fault:
135 movl %gs:CPU_THREAD, %edx
136 xorl %eax, %eax
137 movl T_ONFAULT(%edx), %ecx /* address of save area */
138 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
139 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
140 pushl %ecx
141 call longjmp /* let longjmp do the rest */
142 SET_SIZE(on_fault)
143
144 ENTRY(no_fault)
145 movl %gs:CPU_THREAD, %edx
146 xorl %eax, %eax
147 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
148 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
149 ret
150 SET_SIZE(no_fault)
151
152 #endif /* __i386 */
153 #endif /* __lint */
154
155 /*
156 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
157 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
158 */
159
160 #if defined(lint)
161
162 void
163 on_trap_trampoline(void)
164 {}
165
166 #else /* __lint */
167
168 #if defined(__amd64)
169
170 ENTRY(on_trap_trampoline)
171 movq %gs:CPU_THREAD, %rsi
172 movq T_ONTRAP(%rsi), %rdi
173 addq $OT_JMPBUF, %rdi
174 jmp longjmp
175 SET_SIZE(on_trap_trampoline)
176
177 #elif defined(__i386)
178
179 ENTRY(on_trap_trampoline)
180 movl %gs:CPU_THREAD, %eax
181 movl T_ONTRAP(%eax), %eax
182 addl $OT_JMPBUF, %eax
183 pushl %eax
184 call longjmp
185 SET_SIZE(on_trap_trampoline)
186
187 #endif /* __i386 */
188 #endif /* __lint */
189
190 /*
191 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
192 * more information about the on_trap() mechanism. If the on_trap_data is the
193 * same as the topmost stack element, we just modify that element.
194 */
195 #if defined(lint)
196
197 /*ARGSUSED*/
198 int
199 on_trap(on_trap_data_t *otp, uint_t prot)
200 { return (0); }
201
202 #else /* __lint */
203
204 #if defined(__amd64)
205
206 ENTRY(on_trap)
207 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
208 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
209 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
210 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
211 xorl %ecx, %ecx
212 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
213 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
214 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
215 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
216 cmpq %rdi, %rcx /* if (otp == %rcx) */
217 je 0f /* don't modify t_ontrap */
218
219 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
220 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
221
222 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
223 jmp setjmp
224 SET_SIZE(on_trap)
225
226 #elif defined(__i386)
227
228 ENTRY(on_trap)
229 movl 4(%esp), %eax /* %eax = otp */
230 movl 8(%esp), %edx /* %edx = prot */
231
232 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
233 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
234 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
235 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
236 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
237 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
238 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
239 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
240 cmpl %eax, %ecx /* if (otp == %ecx) */
241 je 0f /* don't modify t_ontrap */
242
243 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
244 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
245
246 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
247 movl %eax, 4(%esp) /* put %eax back on the stack */
248 jmp setjmp /* let setjmp do the rest */
249 SET_SIZE(on_trap)
250
251 #endif /* __i386 */
252 #endif /* __lint */
253
254 /*
255 * Setjmp and longjmp implement non-local gotos using state vectors
256 * type label_t.
257 */
258
259 #if defined(__lint)
260
261 /* ARGSUSED */
262 int
263 setjmp(label_t *lp)
264 { return (0); }
265
266 /* ARGSUSED */
267 void
268 longjmp(label_t *lp)
269 {}
270
271 #else /* __lint */
272
273 #if LABEL_PC != 0
274 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
275 #endif /* LABEL_PC != 0 */
276
277 #if defined(__amd64)
278
279 ENTRY(setjmp)
280 movq %rsp, LABEL_SP(%rdi)
281 movq %rbp, LABEL_RBP(%rdi)
282 movq %rbx, LABEL_RBX(%rdi)
283 movq %r12, LABEL_R12(%rdi)
284 movq %r13, LABEL_R13(%rdi)
285 movq %r14, LABEL_R14(%rdi)
286 movq %r15, LABEL_R15(%rdi)
287 movq (%rsp), %rdx /* return address */
288 movq %rdx, (%rdi) /* LABEL_PC is 0 */
289 xorl %eax, %eax /* return 0 */
290 ret
291 SET_SIZE(setjmp)
292
293 ENTRY(longjmp)
294 movq LABEL_SP(%rdi), %rsp
295 movq LABEL_RBP(%rdi), %rbp
296 movq LABEL_RBX(%rdi), %rbx
297 movq LABEL_R12(%rdi), %r12
298 movq LABEL_R13(%rdi), %r13
299 movq LABEL_R14(%rdi), %r14
300 movq LABEL_R15(%rdi), %r15
301 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
302 movq %rdx, (%rsp)
303 xorl %eax, %eax
304 incl %eax /* return 1 */
305 ret
306 SET_SIZE(longjmp)
307
308 #elif defined(__i386)
309
310 ENTRY(setjmp)
311 movl 4(%esp), %edx /* address of save area */
312 movl %ebp, LABEL_EBP(%edx)
313 movl %ebx, LABEL_EBX(%edx)
314 movl %esi, LABEL_ESI(%edx)
315 movl %edi, LABEL_EDI(%edx)
316 movl %esp, 4(%edx)
317 movl (%esp), %ecx /* %eip (return address) */
318 movl %ecx, (%edx) /* LABEL_PC is 0 */
319 subl %eax, %eax /* return 0 */
320 ret
321 SET_SIZE(setjmp)
322
323 ENTRY(longjmp)
324 movl 4(%esp), %edx /* address of save area */
325 movl LABEL_EBP(%edx), %ebp
326 movl LABEL_EBX(%edx), %ebx
327 movl LABEL_ESI(%edx), %esi
328 movl LABEL_EDI(%edx), %edi
329 movl 4(%edx), %esp
330 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
331 movl $1, %eax
332 addl $4, %esp /* pop ret adr */
333 jmp *%ecx /* indirect */
334 SET_SIZE(longjmp)
335
336 #endif /* __i386 */
337 #endif /* __lint */
338
339 /*
340 * if a() calls b() calls caller(),
341 * caller() returns return address in a().
342 * (Note: We assume a() and b() are C routines which do the normal entry/exit
343 * sequence.)
344 */
345
346 #if defined(__lint)
347
348 caddr_t
349 caller(void)
350 { return (0); }
351
352 #else /* __lint */
353
354 #if defined(__amd64)
355
356 ENTRY(caller)
357 movq 8(%rbp), %rax /* b()'s return pc, in a() */
358 ret
359 SET_SIZE(caller)
360
361 #elif defined(__i386)
362
363 ENTRY(caller)
364 movl 4(%ebp), %eax /* b()'s return pc, in a() */
365 ret
366 SET_SIZE(caller)
367
368 #endif /* __i386 */
369 #endif /* __lint */
370
371 /*
372 * if a() calls callee(), callee() returns the
373 * return address in a();
374 */
375
376 #if defined(__lint)
377
378 caddr_t
379 callee(void)
380 { return (0); }
381
382 #else /* __lint */
383
384 #if defined(__amd64)
385
386 ENTRY(callee)
387 movq (%rsp), %rax /* callee()'s return pc, in a() */
388 ret
389 SET_SIZE(callee)
390
391 #elif defined(__i386)
392
393 ENTRY(callee)
394 movl (%esp), %eax /* callee()'s return pc, in a() */
395 ret
396 SET_SIZE(callee)
397
398 #endif /* __i386 */
399 #endif /* __lint */
400
401 /*
402 * return the current frame pointer
403 */
404
405 #if defined(__lint)
406
407 greg_t
408 getfp(void)
409 { return (0); }
410
411 #else /* __lint */
412
413 #if defined(__amd64)
414
415 ENTRY(getfp)
416 movq %rbp, %rax
417 ret
418 SET_SIZE(getfp)
419
420 #elif defined(__i386)
421
422 ENTRY(getfp)
423 movl %ebp, %eax
424 ret
425 SET_SIZE(getfp)
426
427 #endif /* __i386 */
428 #endif /* __lint */
429
430 /*
431 * Invalidate a single page table entry in the TLB
432 */
433
434 #if defined(__lint)
435
436 /* ARGSUSED */
437 void
438 mmu_tlbflush_entry(caddr_t m)
439 {}
440
441 #else /* __lint */
442
443 #if defined(__amd64)
444
445 ENTRY(mmu_tlbflush_entry)
446 invlpg (%rdi)
447 ret
448 SET_SIZE(mmu_tlbflush_entry)
449
450 #elif defined(__i386)
451
452 ENTRY(mmu_tlbflush_entry)
453 movl 4(%esp), %eax
454 invlpg (%eax)
455 ret
456 SET_SIZE(mmu_tlbflush_entry)
457
458 #endif /* __i386 */
459 #endif /* __lint */
460
461
462 /*
463 * Get/Set the value of various control registers
464 */
465
466 #if defined(__lint)
467
468 ulong_t
469 getcr0(void)
470 { return (0); }
471
472 /* ARGSUSED */
473 void
474 setcr0(ulong_t value)
475 {}
476
477 ulong_t
478 getcr2(void)
479 { return (0); }
480
481 ulong_t
482 getcr3(void)
483 { return (0); }
484
485 #if !defined(__xpv)
486 /* ARGSUSED */
487 void
488 setcr3(ulong_t val)
489 {}
490
491 void
492 reload_cr3(void)
493 {}
494 #endif
495
496 ulong_t
497 getcr4(void)
498 { return (0); }
499
500 /* ARGSUSED */
501 void
502 setcr4(ulong_t val)
503 {}
504
505 #if defined(__amd64)
506
507 ulong_t
508 getcr8(void)
509 { return (0); }
510
511 /* ARGSUSED */
512 void
513 setcr8(ulong_t val)
514 {}
515
516 #endif /* __amd64 */
517
518 #else /* __lint */
519
520 #if defined(__amd64)
521
522 ENTRY(getcr0)
523 movq %cr0, %rax
524 ret
525 SET_SIZE(getcr0)
526
527 ENTRY(setcr0)
528 movq %rdi, %cr0
529 ret
530 SET_SIZE(setcr0)
531
532 ENTRY(getcr2)
533 #if defined(__xpv)
534 movq %gs:CPU_VCPU_INFO, %rax
535 movq VCPU_INFO_ARCH_CR2(%rax), %rax
536 #else
537 movq %cr2, %rax
538 #endif
539 ret
540 SET_SIZE(getcr2)
541
542 ENTRY(getcr3)
543 movq %cr3, %rax
544 ret
545 SET_SIZE(getcr3)
546
547 #if !defined(__xpv)
548
549 ENTRY(setcr3)
550 movq %rdi, %cr3
551 ret
552 SET_SIZE(setcr3)
553
554 ENTRY(reload_cr3)
555 movq %cr3, %rdi
556 movq %rdi, %cr3
557 ret
558 SET_SIZE(reload_cr3)
559
560 #endif /* __xpv */
561
562 ENTRY(getcr4)
563 movq %cr4, %rax
564 ret
565 SET_SIZE(getcr4)
566
567 ENTRY(setcr4)
568 movq %rdi, %cr4
569 ret
570 SET_SIZE(setcr4)
571
572 ENTRY(getcr8)
573 movq %cr8, %rax
574 ret
575 SET_SIZE(getcr8)
576
577 ENTRY(setcr8)
578 movq %rdi, %cr8
579 ret
580 SET_SIZE(setcr8)
581
582 #elif defined(__i386)
583
584 ENTRY(getcr0)
585 movl %cr0, %eax
586 ret
587 SET_SIZE(getcr0)
588
589 ENTRY(setcr0)
590 movl 4(%esp), %eax
591 movl %eax, %cr0
592 ret
593 SET_SIZE(setcr0)
594
595 /*
596 * "lock mov %cr0" is used on processors which indicate it is
597 * supported via CPUID. Normally the 32 bit TPR is accessed via
598 * the local APIC.
599 */
600 ENTRY(getcr8)
601 lock
602 movl %cr0, %eax
603 ret
604 SET_SIZE(getcr8)
605
606 ENTRY(setcr8)
607 movl 4(%esp), %eax
608 lock
609 movl %eax, %cr0
610 ret
611 SET_SIZE(setcr8)
612
613 ENTRY(getcr2)
614 #if defined(__xpv)
615 movl %gs:CPU_VCPU_INFO, %eax
616 movl VCPU_INFO_ARCH_CR2(%eax), %eax
617 #else
618 movl %cr2, %eax
619 #endif
620 ret
621 SET_SIZE(getcr2)
622
623 ENTRY(getcr3)
624 movl %cr3, %eax
625 ret
626 SET_SIZE(getcr3)
627
628 #if !defined(__xpv)
629
630 ENTRY(setcr3)
631 movl 4(%esp), %eax
632 movl %eax, %cr3
633 ret
634 SET_SIZE(setcr3)
635
636 ENTRY(reload_cr3)
637 movl %cr3, %eax
638 movl %eax, %cr3
639 ret
640 SET_SIZE(reload_cr3)
641
642 #endif /* __xpv */
643
644 ENTRY(getcr4)
645 movl %cr4, %eax
646 ret
647 SET_SIZE(getcr4)
648
649 ENTRY(setcr4)
650 movl 4(%esp), %eax
651 movl %eax, %cr4
652 ret
653 SET_SIZE(setcr4)
654
655 #endif /* __i386 */
656 #endif /* __lint */
657
658 #if defined(__lint)
659
660 /*ARGSUSED*/
661 uint32_t
662 __cpuid_insn(struct cpuid_regs *regs)
663 { return (0); }
664
665 #else /* __lint */
666
667 #if defined(__amd64)
668
669 ENTRY(__cpuid_insn)
670 movq %rbx, %r8
671 movq %rcx, %r9
672 movq %rdx, %r11
673 movl (%rdi), %eax /* %eax = regs->cp_eax */
674 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
675 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
676 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
677 cpuid
678 movl %eax, (%rdi) /* regs->cp_eax = %eax */
679 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
680 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
681 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
682 movq %r8, %rbx
683 movq %r9, %rcx
684 movq %r11, %rdx
685 ret
686 SET_SIZE(__cpuid_insn)
687
688 #elif defined(__i386)
689
690 ENTRY(__cpuid_insn)
691 pushl %ebp
692 movl 0x8(%esp), %ebp /* %ebp = regs */
693 pushl %ebx
694 pushl %ecx
695 pushl %edx
696 movl (%ebp), %eax /* %eax = regs->cp_eax */
697 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
698 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
699 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
700 cpuid
701 movl %eax, (%ebp) /* regs->cp_eax = %eax */
702 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
703 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
704 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
705 popl %edx
706 popl %ecx
707 popl %ebx
708 popl %ebp
709 ret
710 SET_SIZE(__cpuid_insn)
711
712 #endif /* __i386 */
713 #endif /* __lint */
714
715 #if defined(__lint)
716
717 /*ARGSUSED*/
718 void
719 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
720 {}
721
722 #else /* __lint */
723
724 #if defined(__amd64)
725
726 ENTRY_NP(i86_monitor)
727 pushq %rbp
728 movq %rsp, %rbp
729 movq %rdi, %rax /* addr */
730 movq %rsi, %rcx /* extensions */
731 /* rdx contains input arg3: hints */
732 clflush (%rax)
733 .byte 0x0f, 0x01, 0xc8 /* monitor */
734 leave
735 ret
736 SET_SIZE(i86_monitor)
737
738 #elif defined(__i386)
739
740 ENTRY_NP(i86_monitor)
741 pushl %ebp
742 movl %esp, %ebp
743 movl 0x8(%ebp),%eax /* addr */
744 movl 0xc(%ebp),%ecx /* extensions */
745 movl 0x10(%ebp),%edx /* hints */
746 clflush (%eax)
747 .byte 0x0f, 0x01, 0xc8 /* monitor */
748 leave
749 ret
750 SET_SIZE(i86_monitor)
751
752 #endif /* __i386 */
753 #endif /* __lint */
754
755 #if defined(__lint)
756
757 /*ARGSUSED*/
758 void
759 i86_mwait(uint32_t data, uint32_t extensions)
760 {}
761
762 #else /* __lint */
763
764 #if defined(__amd64)
765
766 ENTRY_NP(i86_mwait)
767 pushq %rbp
768 movq %rsp, %rbp
769 movq %rdi, %rax /* data */
770 movq %rsi, %rcx /* extensions */
771 .byte 0x0f, 0x01, 0xc9 /* mwait */
772 leave
773 ret
774 SET_SIZE(i86_mwait)
775
776 #elif defined(__i386)
777
778 ENTRY_NP(i86_mwait)
779 pushl %ebp
780 movl %esp, %ebp
781 movl 0x8(%ebp),%eax /* data */
782 movl 0xc(%ebp),%ecx /* extensions */
783 .byte 0x0f, 0x01, 0xc9 /* mwait */
784 leave
785 ret
786 SET_SIZE(i86_mwait)
787
788 #endif /* __i386 */
789 #endif /* __lint */
790
791 #if defined(__xpv)
792 /*
793 * Defined in C
794 */
795 #else
796
797 #if defined(__lint)
798
799 hrtime_t
800 tsc_read(void)
801 {
802 return (0);
803 }
804
805 #else /* __lint */
806
807 #if defined(__amd64)
808
809 ENTRY_NP(tsc_read)
810 movq %rbx, %r11
811 movl $0, %eax
812 cpuid
813 rdtsc
814 movq %r11, %rbx
815 shlq $32, %rdx
816 orq %rdx, %rax
817 ret
818 .globl _tsc_mfence_start
819 _tsc_mfence_start:
820 mfence
821 rdtsc
822 shlq $32, %rdx
823 orq %rdx, %rax
824 ret
825 .globl _tsc_mfence_end
826 _tsc_mfence_end:
827 .globl _tscp_start
828 _tscp_start:
829 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
830 shlq $32, %rdx
831 orq %rdx, %rax
832 ret
833 .globl _tscp_end
834 _tscp_end:
835 .globl _no_rdtsc_start
836 _no_rdtsc_start:
837 xorl %edx, %edx
838 xorl %eax, %eax
839 ret
840 .globl _no_rdtsc_end
841 _no_rdtsc_end:
842 .globl _tsc_lfence_start
843 _tsc_lfence_start:
844 lfence
845 rdtsc
846 shlq $32, %rdx
847 orq %rdx, %rax
848 ret
849 .globl _tsc_lfence_end
850 _tsc_lfence_end:
851 SET_SIZE(tsc_read)
852
853 #else /* __i386 */
854
855 ENTRY_NP(tsc_read)
856 pushl %ebx
857 movl $0, %eax
858 cpuid
859 rdtsc
860 popl %ebx
861 ret
862 .globl _tsc_mfence_start
863 _tsc_mfence_start:
864 mfence
865 rdtsc
866 ret
867 .globl _tsc_mfence_end
868 _tsc_mfence_end:
869 .globl _tscp_start
870 _tscp_start:
871 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
872 ret
873 .globl _tscp_end
874 _tscp_end:
875 .globl _no_rdtsc_start
876 _no_rdtsc_start:
877 xorl %edx, %edx
878 xorl %eax, %eax
879 ret
880 .globl _no_rdtsc_end
881 _no_rdtsc_end:
882 .globl _tsc_lfence_start
883 _tsc_lfence_start:
884 lfence
885 rdtsc
886 ret
887 .globl _tsc_lfence_end
888 _tsc_lfence_end:
889 SET_SIZE(tsc_read)
890
891 #endif /* __i386 */
892
893 #endif /* __lint */
894
895
896 #endif /* __xpv */
897
898 #ifdef __lint
899 /*
900 * Do not use this function for obtaining clock tick. This
901 * is called by callers who do not need to have a guarenteed
902 * correct tick value. The proper routine to use is tsc_read().
903 */
904 u_longlong_t
905 randtick(void)
906 {
907 return (0);
908 }
909 #else
910 #if defined(__amd64)
911 ENTRY_NP(randtick)
912 rdtsc
913 shlq $32, %rdx
914 orq %rdx, %rax
915 ret
916 SET_SIZE(randtick)
917 #else
918 ENTRY_NP(randtick)
919 rdtsc
920 ret
921 SET_SIZE(randtick)
922 #endif /* __i386 */
923 #endif /* __lint */
924 /*
925 * Insert entryp after predp in a doubly linked list.
926 */
927
928 #if defined(__lint)
929
930 /*ARGSUSED*/
931 void
932 _insque(caddr_t entryp, caddr_t predp)
933 {}
934
935 #else /* __lint */
936
937 #if defined(__amd64)
938
939 ENTRY(_insque)
940 movq (%rsi), %rax /* predp->forw */
941 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
942 movq %rax, (%rdi) /* entryp->forw = predp->forw */
943 movq %rdi, (%rsi) /* predp->forw = entryp */
944 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
945 ret
946 SET_SIZE(_insque)
947
948 #elif defined(__i386)
949
950 ENTRY(_insque)
951 movl 8(%esp), %edx
952 movl 4(%esp), %ecx
953 movl (%edx), %eax /* predp->forw */
954 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
955 movl %eax, (%ecx) /* entryp->forw = predp->forw */
956 movl %ecx, (%edx) /* predp->forw = entryp */
957 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
958 ret
959 SET_SIZE(_insque)
960
961 #endif /* __i386 */
962 #endif /* __lint */
963
964 /*
965 * Remove entryp from a doubly linked list
966 */
967
968 #if defined(__lint)
969
970 /*ARGSUSED*/
971 void
972 _remque(caddr_t entryp)
973 {}
974
975 #else /* __lint */
976
977 #if defined(__amd64)
978
979 ENTRY(_remque)
980 movq (%rdi), %rax /* entry->forw */
981 movq CPTRSIZE(%rdi), %rdx /* entry->back */
982 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
983 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
984 ret
985 SET_SIZE(_remque)
986
987 #elif defined(__i386)
988
989 ENTRY(_remque)
990 movl 4(%esp), %ecx
991 movl (%ecx), %eax /* entry->forw */
992 movl CPTRSIZE(%ecx), %edx /* entry->back */
993 movl %eax, (%edx) /* entry->back->forw = entry->forw */
994 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
995 ret
996 SET_SIZE(_remque)
997
998 #endif /* __i386 */
999 #endif /* __lint */
1000
1001 /*
1002 * Returns the number of
1003 * non-NULL bytes in string argument.
1004 */
1005
1006 #if defined(__lint)
1007
1008 /* ARGSUSED */
1009 size_t
1010 strlen(const char *str)
1011 { return (0); }
1012
1013 #else /* __lint */
1014
1015 #if defined(__amd64)
1016
1017 /*
1018 * This is close to a simple transliteration of a C version of this
1019 * routine. We should either just -make- this be a C version, or
1020 * justify having it in assembler by making it significantly faster.
1021 *
1022 * size_t
1023 * strlen(const char *s)
1024 * {
1025 * const char *s0;
1026 * #if defined(DEBUG)
1027 * if ((uintptr_t)s < KERNELBASE)
1028 * panic(.str_panic_msg);
1029 * #endif
1030 * for (s0 = s; *s; s++)
1031 * ;
1032 * return (s - s0);
1033 * }
1034 */
1035
1036 ENTRY(strlen)
1037 #ifdef DEBUG
1038 movq postbootkernelbase(%rip), %rax
1039 cmpq %rax, %rdi
1040 jae str_valid
1041 pushq %rbp
1042 movq %rsp, %rbp
1043 leaq .str_panic_msg(%rip), %rdi
1044 xorl %eax, %eax
1045 call panic
1046 #endif /* DEBUG */
1047 str_valid:
1048 cmpb $0, (%rdi)
1049 movq %rdi, %rax
1050 je .null_found
1051 .align 4
1052 .strlen_loop:
1053 incq %rdi
1054 cmpb $0, (%rdi)
1055 jne .strlen_loop
1056 .null_found:
1057 subq %rax, %rdi
1058 movq %rdi, %rax
1059 ret
1060 SET_SIZE(strlen)
1061
1062 #elif defined(__i386)
1063
1064 ENTRY(strlen)
1065 #ifdef DEBUG
1066 movl postbootkernelbase, %eax
1067 cmpl %eax, 4(%esp)
1068 jae str_valid
1069 pushl %ebp
1070 movl %esp, %ebp
1071 pushl $.str_panic_msg
1072 call panic
1073 #endif /* DEBUG */
1074
1075 str_valid:
1076 movl 4(%esp), %eax /* %eax = string address */
1077 testl $3, %eax /* if %eax not word aligned */
1078 jnz .not_word_aligned /* goto .not_word_aligned */
1079 .align 4
1080 .word_aligned:
1081 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1082 movl $0x7f7f7f7f, %ecx
1083 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1084 addl $4, %eax /* next word */
1085 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1086 orl %edx, %ecx /* %ecx |= %edx */
1087 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1088 cmpl $0x80808080, %ecx /* if no null byte in this word */
1089 je .word_aligned /* goto .word_aligned */
1090 subl $4, %eax /* post-incremented */
1091 .not_word_aligned:
1092 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1093 je .null_found /* goto .null_found */
1094 incl %eax /* next byte */
1095 testl $3, %eax /* if %eax not word aligned */
1096 jnz .not_word_aligned /* goto .not_word_aligned */
1097 jmp .word_aligned /* goto .word_aligned */
1098 .align 4
1099 .null_found:
1100 subl 4(%esp), %eax /* %eax -= string address */
1101 ret
1102 SET_SIZE(strlen)
1103
1104 #endif /* __i386 */
1105
1106 #ifdef DEBUG
1107 .text
1108 .str_panic_msg:
1109 .string "strlen: argument below kernelbase"
1110 #endif /* DEBUG */
1111
1112 #endif /* __lint */
1113
1114 /*
1115 * Berkeley 4.3 introduced symbolically named interrupt levels
1116 * as a way deal with priority in a machine independent fashion.
1117 * Numbered priorities are machine specific, and should be
1118 * discouraged where possible.
1119 *
1120 * Note, for the machine specific priorities there are
1121 * examples listed for devices that use a particular priority.
1122 * It should not be construed that all devices of that
1123 * type should be at that priority. It is currently were
1124 * the current devices fit into the priority scheme based
1125 * upon time criticalness.
1126 *
1127 * The underlying assumption of these assignments is that
1128 * IPL 10 is the highest level from which a device
1129 * routine can call wakeup. Devices that interrupt from higher
1130 * levels are restricted in what they can do. If they need
1131 * kernels services they should schedule a routine at a lower
1132 * level (via software interrupt) to do the required
1133 * processing.
1134 *
1135 * Examples of this higher usage:
1136 * Level Usage
1137 * 14 Profiling clock (and PROM uart polling clock)
1138 * 12 Serial ports
1139 *
1140 * The serial ports request lower level processing on level 6.
1141 *
1142 * Also, almost all splN routines (where N is a number or a
1143 * mnemonic) will do a RAISE(), on the assumption that they are
1144 * never used to lower our priority.
1145 * The exceptions are:
1146 * spl8() Because you can't be above 15 to begin with!
1147 * splzs() Because this is used at boot time to lower our
1148 * priority, to allow the PROM to poll the uart.
1149 * spl0() Used to lower priority to 0.
1150 */
1151
1152 #if defined(__lint)
1153
1154 int spl0(void) { return (0); }
1155 int spl6(void) { return (0); }
1156 int spl7(void) { return (0); }
1157 int spl8(void) { return (0); }
1158 int splhigh(void) { return (0); }
1159 int splhi(void) { return (0); }
1160 int splzs(void) { return (0); }
1161
1162 /* ARGSUSED */
1163 void
1164 splx(int level)
1165 {}
1166
1167 #else /* __lint */
1168
1169 #if defined(__amd64)
1170
1171 #define SETPRI(level) \
1172 movl $/**/level, %edi; /* new priority */ \
1173 jmp do_splx /* redirect to do_splx */
1174
1175 #define RAISE(level) \
1176 movl $/**/level, %edi; /* new priority */ \
1177 jmp splr /* redirect to splr */
1178
1179 #elif defined(__i386)
1180
1181 #define SETPRI(level) \
1182 pushl $/**/level; /* new priority */ \
1183 call do_splx; /* invoke common splx code */ \
1184 addl $4, %esp; /* unstack arg */ \
1185 ret
1186
1187 #define RAISE(level) \
1188 pushl $/**/level; /* new priority */ \
1189 call splr; /* invoke common splr code */ \
1190 addl $4, %esp; /* unstack args */ \
1191 ret
1192
1193 #endif /* __i386 */
1194
1195 /* locks out all interrupts, including memory errors */
1196 ENTRY(spl8)
1197 SETPRI(15)
1198 SET_SIZE(spl8)
1199
1200 /* just below the level that profiling runs */
1201 ENTRY(spl7)
1202 RAISE(13)
1203 SET_SIZE(spl7)
1204
1205 /* sun specific - highest priority onboard serial i/o asy ports */
1206 ENTRY(splzs)
1207 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1208 SET_SIZE(splzs)
1209
1210 ENTRY(splhi)
1211 ALTENTRY(splhigh)
1212 ALTENTRY(spl6)
1213 ALTENTRY(i_ddi_splhigh)
1214
1215 RAISE(DISP_LEVEL)
1216
1217 SET_SIZE(i_ddi_splhigh)
1218 SET_SIZE(spl6)
1219 SET_SIZE(splhigh)
1220 SET_SIZE(splhi)
1221
1222 /* allow all interrupts */
1223 ENTRY(spl0)
1224 SETPRI(0)
1225 SET_SIZE(spl0)
1226
1227
1228 /* splx implementation */
1229 ENTRY(splx)
1230 jmp do_splx /* redirect to common splx code */
1231 SET_SIZE(splx)
1232
1233 #endif /* __lint */
1234
1235 #if defined(__i386)
1236
1237 /*
1238 * Read and write the %gs register
1239 */
1240
1241 #if defined(__lint)
1242
1243 /*ARGSUSED*/
1244 uint16_t
1245 getgs(void)
1246 { return (0); }
1247
1248 /*ARGSUSED*/
1249 void
1250 setgs(uint16_t sel)
1251 {}
1252
1253 #else /* __lint */
1254
1255 ENTRY(getgs)
1256 clr %eax
1257 movw %gs, %ax
1258 ret
1259 SET_SIZE(getgs)
1260
1261 ENTRY(setgs)
1262 movw 4(%esp), %gs
1263 ret
1264 SET_SIZE(setgs)
1265
1266 #endif /* __lint */
1267 #endif /* __i386 */
1268
1269 #if defined(__lint)
1270
1271 void
1272 pc_reset(void)
1273 {}
1274
1275 void
1276 efi_reset(void)
1277 {}
1278
1279 #else /* __lint */
1280
1281 ENTRY(wait_500ms)
1282 #if defined(__amd64)
1283 pushq %rbx
1284 #elif defined(__i386)
1285 push %ebx
1286 #endif
1287 movl $50000, %ebx
1288 1:
1289 call tenmicrosec
1290 decl %ebx
1291 jnz 1b
1292 #if defined(__amd64)
1293 popq %rbx
1294 #elif defined(__i386)
1295 pop %ebx
1296 #endif
1297 ret
1298 SET_SIZE(wait_500ms)
1299
1300 #define RESET_METHOD_KBC 1
1301 #define RESET_METHOD_PORT92 2
1302 #define RESET_METHOD_PCI 4
1303
1304 DGDEF3(pc_reset_methods, 4, 8)
1305 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1306
1307 ENTRY(pc_reset)
1308
1309 #if defined(__i386)
1310 testl $RESET_METHOD_KBC, pc_reset_methods
1311 #elif defined(__amd64)
1312 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1313 #endif
1314 jz 1f
1315
1316 /
1317 / Try the classic keyboard controller-triggered reset.
1318 /
1319 movw $0x64, %dx
1320 movb $0xfe, %al
1321 outb (%dx)
1322
1323 / Wait up to 500 milliseconds here for the keyboard controller
1324 / to pull the reset line. On some systems where the keyboard
1325 / controller is slow to pull the reset line, the next reset method
1326 / may be executed (which may be bad if those systems hang when the
1327 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1328 / and Ferrari 4000 (doesn't like the cf9 reset method))
1329
1330 call wait_500ms
1331
1332 1:
1333 #if defined(__i386)
1334 testl $RESET_METHOD_PORT92, pc_reset_methods
1335 #elif defined(__amd64)
1336 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1337 #endif
1338 jz 3f
1339
1340 /
1341 / Try port 0x92 fast reset
1342 /
1343 movw $0x92, %dx
1344 inb (%dx)
1345 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1346 je 1f
1347 testb $1, %al / If bit 0
1348 jz 2f / is clear, jump to perform the reset
1349 andb $0xfe, %al / otherwise,
1350 outb (%dx) / clear bit 0 first, then
1351 2:
1352 orb $1, %al / Set bit 0
1353 outb (%dx) / and reset the system
1354 1:
1355
1356 call wait_500ms
1357
1358 3:
1359 #if defined(__i386)
1360 testl $RESET_METHOD_PCI, pc_reset_methods
1361 #elif defined(__amd64)
1362 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1363 #endif
1364 jz 4f
1365
1366 / Try the PCI (soft) reset vector (should work on all modern systems,
1367 / but has been shown to cause problems on 450NX systems, and some newer
1368 / systems (e.g. ATI IXP400-equipped systems))
1369 / When resetting via this method, 2 writes are required. The first
1370 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1371 / power cycle).
1372 / The reset occurs on the second write, during bit 2's transition from
1373 / 0->1.
1374 movw $0xcf9, %dx
1375 movb $0x2, %al / Reset mode = hard, no power cycle
1376 outb (%dx)
1377 movb $0x6, %al
1378 outb (%dx)
1379
1380 call wait_500ms
1381
1382 4:
1383 /
1384 / port 0xcf9 failed also. Last-ditch effort is to
1385 / triple-fault the CPU.
1386 / Also, use triple fault for EFI firmware
1387 /
1388 ENTRY(efi_reset)
1389 #if defined(__amd64)
1390 pushq $0x0
1391 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1392 lidt (%rsp)
1393 #elif defined(__i386)
1394 pushl $0x0
1395 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1396 lidt (%esp)
1397 #endif
1398 int $0x0 / Trigger interrupt, generate triple-fault
1399
1400 cli
1401 hlt / Wait forever
1402 /*NOTREACHED*/
1403 SET_SIZE(efi_reset)
1404 SET_SIZE(pc_reset)
1405
1406 #endif /* __lint */
1407
1408 /*
1409 * C callable in and out routines
1410 */
1411
1412 #if defined(__lint)
1413
1414 /* ARGSUSED */
1415 void
1416 outl(int port_address, uint32_t val)
1417 {}
1418
1419 #else /* __lint */
1420
1421 #if defined(__amd64)
1422
1423 ENTRY(outl)
1424 movw %di, %dx
1425 movl %esi, %eax
1426 outl (%dx)
1427 ret
1428 SET_SIZE(outl)
1429
1430 #elif defined(__i386)
1431
1432 .set PORT, 4
1433 .set VAL, 8
1434
1435 ENTRY(outl)
1436 movw PORT(%esp), %dx
1437 movl VAL(%esp), %eax
1438 outl (%dx)
1439 ret
1440 SET_SIZE(outl)
1441
1442 #endif /* __i386 */
1443 #endif /* __lint */
1444
1445 #if defined(__lint)
1446
1447 /* ARGSUSED */
1448 void
1449 outw(int port_address, uint16_t val)
1450 {}
1451
1452 #else /* __lint */
1453
1454 #if defined(__amd64)
1455
1456 ENTRY(outw)
1457 movw %di, %dx
1458 movw %si, %ax
1459 D16 outl (%dx) /* XX64 why not outw? */
1460 ret
1461 SET_SIZE(outw)
1462
1463 #elif defined(__i386)
1464
1465 ENTRY(outw)
1466 movw PORT(%esp), %dx
1467 movw VAL(%esp), %ax
1468 D16 outl (%dx)
1469 ret
1470 SET_SIZE(outw)
1471
1472 #endif /* __i386 */
1473 #endif /* __lint */
1474
1475 #if defined(__lint)
1476
1477 /* ARGSUSED */
1478 void
1479 outb(int port_address, uint8_t val)
1480 {}
1481
1482 #else /* __lint */
1483
1484 #if defined(__amd64)
1485
1486 ENTRY(outb)
1487 movw %di, %dx
1488 movb %sil, %al
1489 outb (%dx)
1490 ret
1491 SET_SIZE(outb)
1492
1493 #elif defined(__i386)
1494
1495 ENTRY(outb)
1496 movw PORT(%esp), %dx
1497 movb VAL(%esp), %al
1498 outb (%dx)
1499 ret
1500 SET_SIZE(outb)
1501
1502 #endif /* __i386 */
1503 #endif /* __lint */
1504
1505 #if defined(__lint)
1506
1507 /* ARGSUSED */
1508 uint32_t
1509 inl(int port_address)
1510 { return (0); }
1511
1512 #else /* __lint */
1513
1514 #if defined(__amd64)
1515
1516 ENTRY(inl)
1517 xorl %eax, %eax
1518 movw %di, %dx
1519 inl (%dx)
1520 ret
1521 SET_SIZE(inl)
1522
1523 #elif defined(__i386)
1524
1525 ENTRY(inl)
1526 movw PORT(%esp), %dx
1527 inl (%dx)
1528 ret
1529 SET_SIZE(inl)
1530
1531 #endif /* __i386 */
1532 #endif /* __lint */
1533
1534 #if defined(__lint)
1535
1536 /* ARGSUSED */
1537 uint16_t
1538 inw(int port_address)
1539 { return (0); }
1540
1541 #else /* __lint */
1542
1543 #if defined(__amd64)
1544
1545 ENTRY(inw)
1546 xorl %eax, %eax
1547 movw %di, %dx
1548 D16 inl (%dx)
1549 ret
1550 SET_SIZE(inw)
1551
1552 #elif defined(__i386)
1553
1554 ENTRY(inw)
1555 subl %eax, %eax
1556 movw PORT(%esp), %dx
1557 D16 inl (%dx)
1558 ret
1559 SET_SIZE(inw)
1560
1561 #endif /* __i386 */
1562 #endif /* __lint */
1563
1564
1565 #if defined(__lint)
1566
1567 /* ARGSUSED */
1568 uint8_t
1569 inb(int port_address)
1570 { return (0); }
1571
1572 #else /* __lint */
1573
1574 #if defined(__amd64)
1575
1576 ENTRY(inb)
1577 xorl %eax, %eax
1578 movw %di, %dx
1579 inb (%dx)
1580 ret
1581 SET_SIZE(inb)
1582
1583 #elif defined(__i386)
1584
1585 ENTRY(inb)
1586 subl %eax, %eax
1587 movw PORT(%esp), %dx
1588 inb (%dx)
1589 ret
1590 SET_SIZE(inb)
1591
1592 #endif /* __i386 */
1593 #endif /* __lint */
1594
1595
1596 #if defined(__lint)
1597
1598 /* ARGSUSED */
1599 void
1600 repoutsw(int port, uint16_t *addr, int cnt)
1601 {}
1602
1603 #else /* __lint */
1604
1605 #if defined(__amd64)
1606
1607 ENTRY(repoutsw)
1608 movl %edx, %ecx
1609 movw %di, %dx
1610 rep
1611 D16 outsl
1612 ret
1613 SET_SIZE(repoutsw)
1614
1615 #elif defined(__i386)
1616
1617 /*
1618 * The arguments and saved registers are on the stack in the
1619 * following order:
1620 * | cnt | +16
1621 * | *addr | +12
1622 * | port | +8
1623 * | eip | +4
1624 * | esi | <-- %esp
1625 * If additional values are pushed onto the stack, make sure
1626 * to adjust the following constants accordingly.
1627 */
1628 .set PORT, 8
1629 .set ADDR, 12
1630 .set COUNT, 16
1631
1632 ENTRY(repoutsw)
1633 pushl %esi
1634 movl PORT(%esp), %edx
1635 movl ADDR(%esp), %esi
1636 movl COUNT(%esp), %ecx
1637 rep
1638 D16 outsl
1639 popl %esi
1640 ret
1641 SET_SIZE(repoutsw)
1642
1643 #endif /* __i386 */
1644 #endif /* __lint */
1645
1646
1647 #if defined(__lint)
1648
1649 /* ARGSUSED */
1650 void
1651 repinsw(int port_addr, uint16_t *addr, int cnt)
1652 {}
1653
1654 #else /* __lint */
1655
1656 #if defined(__amd64)
1657
1658 ENTRY(repinsw)
1659 movl %edx, %ecx
1660 movw %di, %dx
1661 rep
1662 D16 insl
1663 ret
1664 SET_SIZE(repinsw)
1665
1666 #elif defined(__i386)
1667
1668 ENTRY(repinsw)
1669 pushl %edi
1670 movl PORT(%esp), %edx
1671 movl ADDR(%esp), %edi
1672 movl COUNT(%esp), %ecx
1673 rep
1674 D16 insl
1675 popl %edi
1676 ret
1677 SET_SIZE(repinsw)
1678
1679 #endif /* __i386 */
1680 #endif /* __lint */
1681
1682
1683 #if defined(__lint)
1684
1685 /* ARGSUSED */
1686 void
1687 repinsb(int port, uint8_t *addr, int count)
1688 {}
1689
1690 #else /* __lint */
1691
1692 #if defined(__amd64)
1693
1694 ENTRY(repinsb)
1695 movl %edx, %ecx
1696 movw %di, %dx
1697 movq %rsi, %rdi
1698 rep
1699 insb
1700 ret
1701 SET_SIZE(repinsb)
1702
1703 #elif defined(__i386)
1704
1705 /*
1706 * The arguments and saved registers are on the stack in the
1707 * following order:
1708 * | cnt | +16
1709 * | *addr | +12
1710 * | port | +8
1711 * | eip | +4
1712 * | esi | <-- %esp
1713 * If additional values are pushed onto the stack, make sure
1714 * to adjust the following constants accordingly.
1715 */
1716 .set IO_PORT, 8
1717 .set IO_ADDR, 12
1718 .set IO_COUNT, 16
1719
1720 ENTRY(repinsb)
1721 pushl %edi
1722 movl IO_ADDR(%esp), %edi
1723 movl IO_COUNT(%esp), %ecx
1724 movl IO_PORT(%esp), %edx
1725 rep
1726 insb
1727 popl %edi
1728 ret
1729 SET_SIZE(repinsb)
1730
1731 #endif /* __i386 */
1732 #endif /* __lint */
1733
1734
1735 /*
1736 * Input a stream of 32-bit words.
1737 * NOTE: count is a DWORD count.
1738 */
1739 #if defined(__lint)
1740
1741 /* ARGSUSED */
1742 void
1743 repinsd(int port, uint32_t *addr, int count)
1744 {}
1745
1746 #else /* __lint */
1747
1748 #if defined(__amd64)
1749
1750 ENTRY(repinsd)
1751 movl %edx, %ecx
1752 movw %di, %dx
1753 movq %rsi, %rdi
1754 rep
1755 insl
1756 ret
1757 SET_SIZE(repinsd)
1758
1759 #elif defined(__i386)
1760
1761 ENTRY(repinsd)
1762 pushl %edi
1763 movl IO_ADDR(%esp), %edi
1764 movl IO_COUNT(%esp), %ecx
1765 movl IO_PORT(%esp), %edx
1766 rep
1767 insl
1768 popl %edi
1769 ret
1770 SET_SIZE(repinsd)
1771
1772 #endif /* __i386 */
1773 #endif /* __lint */
1774
1775 /*
1776 * Output a stream of bytes
1777 * NOTE: count is a byte count
1778 */
1779 #if defined(__lint)
1780
1781 /* ARGSUSED */
1782 void
1783 repoutsb(int port, uint8_t *addr, int count)
1784 {}
1785
1786 #else /* __lint */
1787
1788 #if defined(__amd64)
1789
1790 ENTRY(repoutsb)
1791 movl %edx, %ecx
1792 movw %di, %dx
1793 rep
1794 outsb
1795 ret
1796 SET_SIZE(repoutsb)
1797
1798 #elif defined(__i386)
1799
1800 ENTRY(repoutsb)
1801 pushl %esi
1802 movl IO_ADDR(%esp), %esi
1803 movl IO_COUNT(%esp), %ecx
1804 movl IO_PORT(%esp), %edx
1805 rep
1806 outsb
1807 popl %esi
1808 ret
1809 SET_SIZE(repoutsb)
1810
1811 #endif /* __i386 */
1812 #endif /* __lint */
1813
1814 /*
1815 * Output a stream of 32-bit words
1816 * NOTE: count is a DWORD count
1817 */
1818 #if defined(__lint)
1819
1820 /* ARGSUSED */
1821 void
1822 repoutsd(int port, uint32_t *addr, int count)
1823 {}
1824
1825 #else /* __lint */
1826
1827 #if defined(__amd64)
1828
1829 ENTRY(repoutsd)
1830 movl %edx, %ecx
1831 movw %di, %dx
1832 rep
1833 outsl
1834 ret
1835 SET_SIZE(repoutsd)
1836
1837 #elif defined(__i386)
1838
1839 ENTRY(repoutsd)
1840 pushl %esi
1841 movl IO_ADDR(%esp), %esi
1842 movl IO_COUNT(%esp), %ecx
1843 movl IO_PORT(%esp), %edx
1844 rep
1845 outsl
1846 popl %esi
1847 ret
1848 SET_SIZE(repoutsd)
1849
1850 #endif /* __i386 */
1851 #endif /* __lint */
1852
1853 /*
1854 * void int3(void)
1855 * void int18(void)
1856 * void int20(void)
1857 * void int_cmci(void)
1858 */
1859
1860 #if defined(__lint)
1861
1862 void
1863 int3(void)
1864 {}
1865
1866 void
1867 int18(void)
1868 {}
1869
1870 void
1871 int20(void)
1872 {}
1873
1874 void
1875 int_cmci(void)
1876 {}
1877
1878 #else /* __lint */
1879
1880 ENTRY(int3)
1881 int $T_BPTFLT
1882 ret
1883 SET_SIZE(int3)
1884
1885 ENTRY(int18)
1886 int $T_MCE
1887 ret
1888 SET_SIZE(int18)
1889
1890 ENTRY(int20)
1891 movl boothowto, %eax
1892 andl $RB_DEBUG, %eax
1893 jz 1f
1894
1895 int $T_DBGENTR
1896 1:
1897 rep; ret /* use 2 byte return instruction when branch target */
1898 /* AMD Software Optimization Guide - Section 6.2 */
1899 SET_SIZE(int20)
1900
1901 ENTRY(int_cmci)
1902 int $T_ENOEXTFLT
1903 ret
1904 SET_SIZE(int_cmci)
1905
1906 #endif /* __lint */
1907
1908 #if defined(__lint)
1909
1910 /* ARGSUSED */
1911 int
1912 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1913 { return (0); }
1914
1915 #else /* __lint */
1916
1917 #if defined(__amd64)
1918
1919 ENTRY(scanc)
1920 /* rdi == size */
1921 /* rsi == cp */
1922 /* rdx == table */
1923 /* rcx == mask */
1924 addq %rsi, %rdi /* end = &cp[size] */
1925 .scanloop:
1926 cmpq %rdi, %rsi /* while (cp < end */
1927 jnb .scandone
1928 movzbq (%rsi), %r8 /* %r8 = *cp */
1929 incq %rsi /* cp++ */
1930 testb %cl, (%r8, %rdx)
1931 jz .scanloop /* && (table[*cp] & mask) == 0) */
1932 decq %rsi /* (fix post-increment) */
1933 .scandone:
1934 movl %edi, %eax
1935 subl %esi, %eax /* return (end - cp) */
1936 ret
1937 SET_SIZE(scanc)
1938
1939 #elif defined(__i386)
1940
1941 ENTRY(scanc)
1942 pushl %edi
1943 pushl %esi
1944 movb 24(%esp), %cl /* mask = %cl */
1945 movl 16(%esp), %esi /* cp = %esi */
1946 movl 20(%esp), %edx /* table = %edx */
1947 movl %esi, %edi
1948 addl 12(%esp), %edi /* end = &cp[size]; */
1949 .scanloop:
1950 cmpl %edi, %esi /* while (cp < end */
1951 jnb .scandone
1952 movzbl (%esi), %eax /* %al = *cp */
1953 incl %esi /* cp++ */
1954 movb (%edx, %eax), %al /* %al = table[*cp] */
1955 testb %al, %cl
1956 jz .scanloop /* && (table[*cp] & mask) == 0) */
1957 dec %esi /* post-incremented */
1958 .scandone:
1959 movl %edi, %eax
1960 subl %esi, %eax /* return (end - cp) */
1961 popl %esi
1962 popl %edi
1963 ret
1964 SET_SIZE(scanc)
1965
1966 #endif /* __i386 */
1967 #endif /* __lint */
1968
1969 /*
1970 * Replacement functions for ones that are normally inlined.
1971 * In addition to the copy in i86.il, they are defined here just in case.
1972 */
1973
1974 #if defined(__lint)
1975
1976 ulong_t
1977 intr_clear(void)
1978 { return (0); }
1979
1980 ulong_t
1981 clear_int_flag(void)
1982 { return (0); }
1983
1984 #else /* __lint */
1985
1986 #if defined(__amd64)
1987
1988 ENTRY(intr_clear)
1989 ENTRY(clear_int_flag)
1990 pushfq
1991 popq %rax
1992 #if defined(__xpv)
1993 leaq xpv_panicking, %rdi
1994 movl (%rdi), %edi
1995 cmpl $0, %edi
1996 jne 2f
1997 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1998 /*
1999 * Synthesize the PS_IE bit from the event mask bit
2000 */
2001 andq $_BITNOT(PS_IE), %rax
2002 testb $1, %dl
2003 jnz 1f
2004 orq $PS_IE, %rax
2005 1:
2006 ret
2007 2:
2008 #endif
2009 CLI(%rdi)
2010 ret
2011 SET_SIZE(clear_int_flag)
2012 SET_SIZE(intr_clear)
2013
2014 #elif defined(__i386)
2015
2016 ENTRY(intr_clear)
2017 ENTRY(clear_int_flag)
2018 pushfl
2019 popl %eax
2020 #if defined(__xpv)
2021 leal xpv_panicking, %edx
2022 movl (%edx), %edx
2023 cmpl $0, %edx
2024 jne 2f
2025 CLIRET(%edx, %cl) /* returns event mask in %cl */
2026 /*
2027 * Synthesize the PS_IE bit from the event mask bit
2028 */
2029 andl $_BITNOT(PS_IE), %eax
2030 testb $1, %cl
2031 jnz 1f
2032 orl $PS_IE, %eax
2033 1:
2034 ret
2035 2:
2036 #endif
2037 CLI(%edx)
2038 ret
2039 SET_SIZE(clear_int_flag)
2040 SET_SIZE(intr_clear)
2041
2042 #endif /* __i386 */
2043 #endif /* __lint */
2044
2045 #if defined(__lint)
2046
2047 struct cpu *
2048 curcpup(void)
2049 { return 0; }
2050
2051 #else /* __lint */
2052
2053 #if defined(__amd64)
2054
2055 ENTRY(curcpup)
2056 movq %gs:CPU_SELF, %rax
2057 ret
2058 SET_SIZE(curcpup)
2059
2060 #elif defined(__i386)
2061
2062 ENTRY(curcpup)
2063 movl %gs:CPU_SELF, %eax
2064 ret
2065 SET_SIZE(curcpup)
2066
2067 #endif /* __i386 */
2068 #endif /* __lint */
2069
2070 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2071 * These functions reverse the byte order of the input parameter and returns
2072 * the result. This is to convert the byte order from host byte order
2073 * (little endian) to network byte order (big endian), or vice versa.
2074 */
2075
2076 #if defined(__lint)
2077
2078 uint64_t
2079 htonll(uint64_t i)
2080 { return (i); }
2081
2082 uint64_t
2083 ntohll(uint64_t i)
2084 { return (i); }
2085
2086 uint32_t
2087 htonl(uint32_t i)
2088 { return (i); }
2089
2090 uint32_t
2091 ntohl(uint32_t i)
2092 { return (i); }
2093
2094 uint16_t
2095 htons(uint16_t i)
2096 { return (i); }
2097
2098 uint16_t
2099 ntohs(uint16_t i)
2100 { return (i); }
2101
2102 #else /* __lint */
2103
2104 #if defined(__amd64)
2105
2106 ENTRY(htonll)
2107 ALTENTRY(ntohll)
2108 movq %rdi, %rax
2109 bswapq %rax
2110 ret
2111 SET_SIZE(ntohll)
2112 SET_SIZE(htonll)
2113
2114 /* XX64 there must be shorter sequences for this */
2115 ENTRY(htonl)
2116 ALTENTRY(ntohl)
2117 movl %edi, %eax
2118 bswap %eax
2119 ret
2120 SET_SIZE(ntohl)
2121 SET_SIZE(htonl)
2122
2123 /* XX64 there must be better sequences for this */
2124 ENTRY(htons)
2125 ALTENTRY(ntohs)
2126 movl %edi, %eax
2127 bswap %eax
2128 shrl $16, %eax
2129 ret
2130 SET_SIZE(ntohs)
2131 SET_SIZE(htons)
2132
2133 #elif defined(__i386)
2134
2135 ENTRY(htonll)
2136 ALTENTRY(ntohll)
2137 movl 4(%esp), %edx
2138 movl 8(%esp), %eax
2139 bswap %edx
2140 bswap %eax
2141 ret
2142 SET_SIZE(ntohll)
2143 SET_SIZE(htonll)
2144
2145 ENTRY(htonl)
2146 ALTENTRY(ntohl)
2147 movl 4(%esp), %eax
2148 bswap %eax
2149 ret
2150 SET_SIZE(ntohl)
2151 SET_SIZE(htonl)
2152
2153 ENTRY(htons)
2154 ALTENTRY(ntohs)
2155 movl 4(%esp), %eax
2156 bswap %eax
2157 shrl $16, %eax
2158 ret
2159 SET_SIZE(ntohs)
2160 SET_SIZE(htons)
2161
2162 #endif /* __i386 */
2163 #endif /* __lint */
2164
2165
2166 #if defined(__lint)
2167
2168 /* ARGSUSED */
2169 void
2170 intr_restore(ulong_t i)
2171 { return; }
2172
2173 /* ARGSUSED */
2174 void
2175 restore_int_flag(ulong_t i)
2176 { return; }
2177
2178 #else /* __lint */
2179
2180 #if defined(__amd64)
2181
2182 ENTRY(intr_restore)
2183 ENTRY(restore_int_flag)
2184 testq $PS_IE, %rdi
2185 jz 1f
2186 #if defined(__xpv)
2187 leaq xpv_panicking, %rsi
2188 movl (%rsi), %esi
2189 cmpl $0, %esi
2190 jne 1f
2191 /*
2192 * Since we're -really- running unprivileged, our attempt
2193 * to change the state of the IF bit will be ignored.
2194 * The virtual IF bit is tweaked by CLI and STI.
2195 */
2196 IE_TO_EVENT_MASK(%rsi, %rdi)
2197 #else
2198 sti
2199 #endif
2200 1:
2201 ret
2202 SET_SIZE(restore_int_flag)
2203 SET_SIZE(intr_restore)
2204
2205 #elif defined(__i386)
2206
2207 ENTRY(intr_restore)
2208 ENTRY(restore_int_flag)
2209 testl $PS_IE, 4(%esp)
2210 jz 1f
2211 #if defined(__xpv)
2212 leal xpv_panicking, %edx
2213 movl (%edx), %edx
2214 cmpl $0, %edx
2215 jne 1f
2216 /*
2217 * Since we're -really- running unprivileged, our attempt
2218 * to change the state of the IF bit will be ignored.
2219 * The virtual IF bit is tweaked by CLI and STI.
2220 */
2221 IE_TO_EVENT_MASK(%edx, 4(%esp))
2222 #else
2223 sti
2224 #endif
2225 1:
2226 ret
2227 SET_SIZE(restore_int_flag)
2228 SET_SIZE(intr_restore)
2229
2230 #endif /* __i386 */
2231 #endif /* __lint */
2232
2233 #if defined(__lint)
2234
2235 void
2236 sti(void)
2237 {}
2238
2239 void
2240 cli(void)
2241 {}
2242
2243 #else /* __lint */
2244
2245 ENTRY(sti)
2246 STI
2247 ret
2248 SET_SIZE(sti)
2249
2250 ENTRY(cli)
2251 #if defined(__amd64)
2252 CLI(%rax)
2253 #elif defined(__i386)
2254 CLI(%eax)
2255 #endif /* __i386 */
2256 ret
2257 SET_SIZE(cli)
2258
2259 #endif /* __lint */
2260
2261 #if defined(__lint)
2262
2263 dtrace_icookie_t
2264 dtrace_interrupt_disable(void)
2265 { return (0); }
2266
2267 #else /* __lint */
2268
2269 #if defined(__amd64)
2270
2271 ENTRY(dtrace_interrupt_disable)
2272 pushfq
2273 popq %rax
2274 #if defined(__xpv)
2275 leaq xpv_panicking, %rdi
2276 movl (%rdi), %edi
2277 cmpl $0, %edi
2278 jne .dtrace_interrupt_disable_done
2279 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2280 /*
2281 * Synthesize the PS_IE bit from the event mask bit
2282 */
2283 andq $_BITNOT(PS_IE), %rax
2284 testb $1, %dl
2285 jnz .dtrace_interrupt_disable_done
2286 orq $PS_IE, %rax
2287 #else
2288 CLI(%rdx)
2289 #endif
2290 .dtrace_interrupt_disable_done:
2291 ret
2292 SET_SIZE(dtrace_interrupt_disable)
2293
2294 #elif defined(__i386)
2295
2296 ENTRY(dtrace_interrupt_disable)
2297 pushfl
2298 popl %eax
2299 #if defined(__xpv)
2300 leal xpv_panicking, %edx
2301 movl (%edx), %edx
2302 cmpl $0, %edx
2303 jne .dtrace_interrupt_disable_done
2304 CLIRET(%edx, %cl) /* returns event mask in %cl */
2305 /*
2306 * Synthesize the PS_IE bit from the event mask bit
2307 */
2308 andl $_BITNOT(PS_IE), %eax
2309 testb $1, %cl
2310 jnz .dtrace_interrupt_disable_done
2311 orl $PS_IE, %eax
2312 #else
2313 CLI(%edx)
2314 #endif
2315 .dtrace_interrupt_disable_done:
2316 ret
2317 SET_SIZE(dtrace_interrupt_disable)
2318
2319 #endif /* __i386 */
2320 #endif /* __lint */
2321
2322 #if defined(__lint)
2323
2324 /*ARGSUSED*/
2325 void
2326 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2327 {}
2328
2329 #else /* __lint */
2330
2331 #if defined(__amd64)
2332
2333 ENTRY(dtrace_interrupt_enable)
2334 pushq %rdi
2335 popfq
2336 #if defined(__xpv)
2337 leaq xpv_panicking, %rdx
2338 movl (%rdx), %edx
2339 cmpl $0, %edx
2340 jne .dtrace_interrupt_enable_done
2341 /*
2342 * Since we're -really- running unprivileged, our attempt
2343 * to change the state of the IF bit will be ignored. The
2344 * virtual IF bit is tweaked by CLI and STI.
2345 */
2346 IE_TO_EVENT_MASK(%rdx, %rdi)
2347 #endif
2348 .dtrace_interrupt_enable_done:
2349 ret
2350 SET_SIZE(dtrace_interrupt_enable)
2351
2352 #elif defined(__i386)
2353
2354 ENTRY(dtrace_interrupt_enable)
2355 movl 4(%esp), %eax
2356 pushl %eax
2357 popfl
2358 #if defined(__xpv)
2359 leal xpv_panicking, %edx
2360 movl (%edx), %edx
2361 cmpl $0, %edx
2362 jne .dtrace_interrupt_enable_done
2363 /*
2364 * Since we're -really- running unprivileged, our attempt
2365 * to change the state of the IF bit will be ignored. The
2366 * virtual IF bit is tweaked by CLI and STI.
2367 */
2368 IE_TO_EVENT_MASK(%edx, %eax)
2369 #endif
2370 .dtrace_interrupt_enable_done:
2371 ret
2372 SET_SIZE(dtrace_interrupt_enable)
2373
2374 #endif /* __i386 */
2375 #endif /* __lint */
2376
2377
2378 #if defined(lint)
2379
2380 void
2381 dtrace_membar_producer(void)
2382 {}
2383
2384 void
2385 dtrace_membar_consumer(void)
2386 {}
2387
2388 #else /* __lint */
2389
2390 ENTRY(dtrace_membar_producer)
2391 rep; ret /* use 2 byte return instruction when branch target */
2392 /* AMD Software Optimization Guide - Section 6.2 */
2393 SET_SIZE(dtrace_membar_producer)
2394
2395 ENTRY(dtrace_membar_consumer)
2396 rep; ret /* use 2 byte return instruction when branch target */
2397 /* AMD Software Optimization Guide - Section 6.2 */
2398 SET_SIZE(dtrace_membar_consumer)
2399
2400 #endif /* __lint */
2401
2402 #if defined(__lint)
2403
2404 kthread_id_t
2405 threadp(void)
2406 { return ((kthread_id_t)0); }
2407
2408 #else /* __lint */
2409
2410 #if defined(__amd64)
2411
2412 ENTRY(threadp)
2413 movq %gs:CPU_THREAD, %rax
2414 ret
2415 SET_SIZE(threadp)
2416
2417 #elif defined(__i386)
2418
2419 ENTRY(threadp)
2420 movl %gs:CPU_THREAD, %eax
2421 ret
2422 SET_SIZE(threadp)
2423
2424 #endif /* __i386 */
2425 #endif /* __lint */
2426
2427 /*
2428 * Checksum routine for Internet Protocol Headers
2429 */
2430
2431 #if defined(__lint)
2432
2433 /* ARGSUSED */
2434 unsigned int
2435 ip_ocsum(
2436 ushort_t *address, /* ptr to 1st message buffer */
2437 int halfword_count, /* length of data */
2438 unsigned int sum) /* partial checksum */
2439 {
2440 int i;
2441 unsigned int psum = 0; /* partial sum */
2442
2443 for (i = 0; i < halfword_count; i++, address++) {
2444 psum += *address;
2445 }
2446
2447 while ((psum >> 16) != 0) {
2448 psum = (psum & 0xffff) + (psum >> 16);
2449 }
2450
2451 psum += sum;
2452
2453 while ((psum >> 16) != 0) {
2454 psum = (psum & 0xffff) + (psum >> 16);
2455 }
2456
2457 return (psum);
2458 }
2459
2460 #else /* __lint */
2461
2462 #if defined(__amd64)
2463
2464 ENTRY(ip_ocsum)
2465 pushq %rbp
2466 movq %rsp, %rbp
2467 #ifdef DEBUG
2468 movq postbootkernelbase(%rip), %rax
2469 cmpq %rax, %rdi
2470 jnb 1f
2471 xorl %eax, %eax
2472 movq %rdi, %rsi
2473 leaq .ip_ocsum_panic_msg(%rip), %rdi
2474 call panic
2475 /*NOTREACHED*/
2476 .ip_ocsum_panic_msg:
2477 .string "ip_ocsum: address 0x%p below kernelbase\n"
2478 1:
2479 #endif
2480 movl %esi, %ecx /* halfword_count */
2481 movq %rdi, %rsi /* address */
2482 /* partial sum in %edx */
2483 xorl %eax, %eax
2484 testl %ecx, %ecx
2485 jz .ip_ocsum_done
2486 testq $3, %rsi
2487 jnz .ip_csum_notaligned
2488 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2489 .next_iter:
2490 /* XX64 opportunities for prefetch? */
2491 /* XX64 compute csum with 64 bit quantities? */
2492 subl $32, %ecx
2493 jl .less_than_32
2494
2495 addl 0(%rsi), %edx
2496 .only60:
2497 adcl 4(%rsi), %eax
2498 .only56:
2499 adcl 8(%rsi), %edx
2500 .only52:
2501 adcl 12(%rsi), %eax
2502 .only48:
2503 adcl 16(%rsi), %edx
2504 .only44:
2505 adcl 20(%rsi), %eax
2506 .only40:
2507 adcl 24(%rsi), %edx
2508 .only36:
2509 adcl 28(%rsi), %eax
2510 .only32:
2511 adcl 32(%rsi), %edx
2512 .only28:
2513 adcl 36(%rsi), %eax
2514 .only24:
2515 adcl 40(%rsi), %edx
2516 .only20:
2517 adcl 44(%rsi), %eax
2518 .only16:
2519 adcl 48(%rsi), %edx
2520 .only12:
2521 adcl 52(%rsi), %eax
2522 .only8:
2523 adcl 56(%rsi), %edx
2524 .only4:
2525 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2526 .only0:
2527 adcl $0, %eax /* could be adding -1 in eax with a carry */
2528 adcl $0, %eax
2529
2530 addq $64, %rsi
2531 testl %ecx, %ecx
2532 jnz .next_iter
2533
2534 .ip_ocsum_done:
2535 addl %eax, %edx
2536 adcl $0, %edx
2537 movl %edx, %eax /* form a 16 bit checksum by */
2538 shrl $16, %eax /* adding two halves of 32 bit checksum */
2539 addw %dx, %ax
2540 adcw $0, %ax
2541 andl $0xffff, %eax
2542 leave
2543 ret
2544
2545 .ip_csum_notaligned:
2546 xorl %edi, %edi
2547 movw (%rsi), %di
2548 addl %edi, %edx
2549 adcl $0, %edx
2550 addq $2, %rsi
2551 decl %ecx
2552 jmp .ip_csum_aligned
2553
2554 .less_than_32:
2555 addl $32, %ecx
2556 testl $1, %ecx
2557 jz .size_aligned
2558 andl $0xfe, %ecx
2559 movzwl (%rsi, %rcx, 2), %edi
2560 addl %edi, %edx
2561 adcl $0, %edx
2562 .size_aligned:
2563 movl %ecx, %edi
2564 shrl $1, %ecx
2565 shl $1, %edi
2566 subq $64, %rdi
2567 addq %rdi, %rsi
2568 leaq .ip_ocsum_jmptbl(%rip), %rdi
2569 leaq (%rdi, %rcx, 8), %rdi
2570 xorl %ecx, %ecx
2571 clc
2572 jmp *(%rdi)
2573
2574 .align 8
2575 .ip_ocsum_jmptbl:
2576 .quad .only0, .only4, .only8, .only12, .only16, .only20
2577 .quad .only24, .only28, .only32, .only36, .only40, .only44
2578 .quad .only48, .only52, .only56, .only60
2579 SET_SIZE(ip_ocsum)
2580
2581 #elif defined(__i386)
2582
2583 ENTRY(ip_ocsum)
2584 pushl %ebp
2585 movl %esp, %ebp
2586 pushl %ebx
2587 pushl %esi
2588 pushl %edi
2589 movl 12(%ebp), %ecx /* count of half words */
2590 movl 16(%ebp), %edx /* partial checksum */
2591 movl 8(%ebp), %esi
2592 xorl %eax, %eax
2593 testl %ecx, %ecx
2594 jz .ip_ocsum_done
2595
2596 testl $3, %esi
2597 jnz .ip_csum_notaligned
2598 .ip_csum_aligned:
2599 .next_iter:
2600 subl $32, %ecx
2601 jl .less_than_32
2602
2603 addl 0(%esi), %edx
2604 .only60:
2605 adcl 4(%esi), %eax
2606 .only56:
2607 adcl 8(%esi), %edx
2608 .only52:
2609 adcl 12(%esi), %eax
2610 .only48:
2611 adcl 16(%esi), %edx
2612 .only44:
2613 adcl 20(%esi), %eax
2614 .only40:
2615 adcl 24(%esi), %edx
2616 .only36:
2617 adcl 28(%esi), %eax
2618 .only32:
2619 adcl 32(%esi), %edx
2620 .only28:
2621 adcl 36(%esi), %eax
2622 .only24:
2623 adcl 40(%esi), %edx
2624 .only20:
2625 adcl 44(%esi), %eax
2626 .only16:
2627 adcl 48(%esi), %edx
2628 .only12:
2629 adcl 52(%esi), %eax
2630 .only8:
2631 adcl 56(%esi), %edx
2632 .only4:
2633 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2634 .only0:
2635 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2636 adcl $0, %eax
2637
2638 addl $64, %esi
2639 andl %ecx, %ecx
2640 jnz .next_iter
2641
2642 .ip_ocsum_done:
2643 addl %eax, %edx
2644 adcl $0, %edx
2645 movl %edx, %eax /* form a 16 bit checksum by */
2646 shrl $16, %eax /* adding two halves of 32 bit checksum */
2647 addw %dx, %ax
2648 adcw $0, %ax
2649 andl $0xffff, %eax
2650 popl %edi /* restore registers */
2651 popl %esi
2652 popl %ebx
2653 leave
2654 ret
2655
2656 .ip_csum_notaligned:
2657 xorl %edi, %edi
2658 movw (%esi), %di
2659 addl %edi, %edx
2660 adcl $0, %edx
2661 addl $2, %esi
2662 decl %ecx
2663 jmp .ip_csum_aligned
2664
2665 .less_than_32:
2666 addl $32, %ecx
2667 testl $1, %ecx
2668 jz .size_aligned
2669 andl $0xfe, %ecx
2670 movzwl (%esi, %ecx, 2), %edi
2671 addl %edi, %edx
2672 adcl $0, %edx
2673 .size_aligned:
2674 movl %ecx, %edi
2675 shrl $1, %ecx
2676 shl $1, %edi
2677 subl $64, %edi
2678 addl %edi, %esi
2679 movl $.ip_ocsum_jmptbl, %edi
2680 lea (%edi, %ecx, 4), %edi
2681 xorl %ecx, %ecx
2682 clc
2683 jmp *(%edi)
2684 SET_SIZE(ip_ocsum)
2685
2686 .data
2687 .align 4
2688
2689 .ip_ocsum_jmptbl:
2690 .long .only0, .only4, .only8, .only12, .only16, .only20
2691 .long .only24, .only28, .only32, .only36, .only40, .only44
2692 .long .only48, .only52, .only56, .only60
2693
2694
2695 #endif /* __i386 */
2696 #endif /* __lint */
2697
2698 /*
2699 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2700 * Provided to manipulate hrtime_t values.
2701 */
2702 #if defined(__lint)
2703
2704 /* result = a * b; */
2705
2706 /* ARGSUSED */
2707 unsigned long long
2708 mul32(uint_t a, uint_t b)
2709 { return (0); }
2710
2711 #else /* __lint */
2712
2713 #if defined(__amd64)
2714
2715 ENTRY(mul32)
2716 xorl %edx, %edx /* XX64 joe, paranoia? */
2717 movl %edi, %eax
2718 mull %esi
2719 shlq $32, %rdx
2720 orq %rdx, %rax
2721 ret
2722 SET_SIZE(mul32)
2723
2724 #elif defined(__i386)
2725
2726 ENTRY(mul32)
2727 movl 8(%esp), %eax
2728 movl 4(%esp), %ecx
2729 mull %ecx
2730 ret
2731 SET_SIZE(mul32)
2732
2733 #endif /* __i386 */
2734 #endif /* __lint */
2735
2736 #if defined(notused)
2737 #if defined(__lint)
2738 /* ARGSUSED */
2739 void
2740 load_pte64(uint64_t *pte, uint64_t pte_value)
2741 {}
2742 #else /* __lint */
2743 .globl load_pte64
2744 load_pte64:
2745 movl 4(%esp), %eax
2746 movl 8(%esp), %ecx
2747 movl 12(%esp), %edx
2748 movl %edx, 4(%eax)
2749 movl %ecx, (%eax)
2750 ret
2751 #endif /* __lint */
2752 #endif /* notused */
2753
2754 #if defined(__lint)
2755
2756 /*ARGSUSED*/
2757 void
2758 scan_memory(caddr_t addr, size_t size)
2759 {}
2760
2761 #else /* __lint */
2762
2763 #if defined(__amd64)
2764
2765 ENTRY(scan_memory)
2766 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2767 jz .scanm_done
2768 movq %rsi, %rcx /* move count into rep control register */
2769 movq %rdi, %rsi /* move addr into lodsq control reg. */
2770 rep lodsq /* scan the memory range */
2771 .scanm_done:
2772 rep; ret /* use 2 byte return instruction when branch target */
2773 /* AMD Software Optimization Guide - Section 6.2 */
2774 SET_SIZE(scan_memory)
2775
2776 #elif defined(__i386)
2777
2778 ENTRY(scan_memory)
2779 pushl %ecx
2780 pushl %esi
2781 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2782 shrl $2, %ecx /* convert from byte count to word count */
2783 jz .scanm_done
2784 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2785 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2786 lodsl
2787 .scanm_done:
2788 popl %esi
2789 popl %ecx
2790 ret
2791 SET_SIZE(scan_memory)
2792
2793 #endif /* __i386 */
2794 #endif /* __lint */
2795
2796
2797 #if defined(__lint)
2798
2799 /*ARGSUSED */
2800 int
2801 lowbit(ulong_t i)
2802 { return (0); }
2803
2804 #else /* __lint */
2805
2806 #if defined(__amd64)
2807
2808 ENTRY(lowbit)
2809 movl $-1, %eax
2810 bsfq %rdi, %rdi
2811 cmovnz %edi, %eax
2812 incl %eax
2813 ret
2814 SET_SIZE(lowbit)
2815
2816 #elif defined(__i386)
2817
2818 ENTRY(lowbit)
2819 bsfl 4(%esp), %eax
2820 jz 0f
2821 incl %eax
2822 ret
2823 0:
2824 xorl %eax, %eax
2825 ret
2826 SET_SIZE(lowbit)
2827
2828 #endif /* __i386 */
2829 #endif /* __lint */
2830
2831 #if defined(__lint)
2832
2833 /*ARGSUSED*/
2834 int
2835 highbit(ulong_t i)
2836 { return (0); }
2837
2838 /*ARGSUSED*/
2839 int
2840 highbit64(uint64_t i)
2841 { return (0); }
2842
2843 #else /* __lint */
2844
2845 #if defined(__amd64)
2846
2847 ENTRY(highbit)
2848 ALTENTRY(highbit64)
2849 movl $-1, %eax
2850 bsrq %rdi, %rdi
2851 cmovnz %edi, %eax
2852 incl %eax
2853 ret
2854 SET_SIZE(highbit64)
2855 SET_SIZE(highbit)
2856
2857 #elif defined(__i386)
2858
2859 ENTRY(highbit)
2860 bsrl 4(%esp), %eax
2861 jz 0f
2862 incl %eax
2863 ret
2864 0:
2865 xorl %eax, %eax
2866 ret
2867 SET_SIZE(highbit)
2868
2869 ENTRY(highbit64)
2870 bsrl 8(%esp), %eax
2871 jz highbit
2872 addl $33, %eax
2873 ret
2874 SET_SIZE(highbit64)
2875
2876 #endif /* __i386 */
2877 #endif /* __lint */
2878
2879 #if defined(__lint)
2880
2881 /*ARGSUSED*/
2882 uint64_t
2883 rdmsr(uint_t r)
2884 { return (0); }
2885
2886 /*ARGSUSED*/
2887 void
2888 wrmsr(uint_t r, const uint64_t val)
2889 {}
2890
2891 /*ARGSUSED*/
2892 uint64_t
2893 xrdmsr(uint_t r)
2894 { return (0); }
2895
2896 /*ARGSUSED*/
2897 void
2898 xwrmsr(uint_t r, const uint64_t val)
2899 {}
2900
2901 void
2902 invalidate_cache(void)
2903 {}
2904
2905 /*ARGSUSED*/
2906 uint64_t
2907 get_xcr(uint_t r)
2908 { return (0); }
2909
2910 /*ARGSUSED*/
2911 void
2912 set_xcr(uint_t r, const uint64_t val)
2913 {}
2914
2915 #else /* __lint */
2916
2917 #define XMSR_ACCESS_VAL $0x9c5a203a
2918
2919 #if defined(__amd64)
2920
2921 ENTRY(rdmsr)
2922 movl %edi, %ecx
2923 rdmsr
2924 shlq $32, %rdx
2925 orq %rdx, %rax
2926 ret
2927 SET_SIZE(rdmsr)
2928
2929 ENTRY(wrmsr)
2930 movq %rsi, %rdx
2931 shrq $32, %rdx
2932 movl %esi, %eax
2933 movl %edi, %ecx
2934 wrmsr
2935 ret
2936 SET_SIZE(wrmsr)
2937
2938 ENTRY(xrdmsr)
2939 pushq %rbp
2940 movq %rsp, %rbp
2941 movl %edi, %ecx
2942 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2943 rdmsr
2944 shlq $32, %rdx
2945 orq %rdx, %rax
2946 leave
2947 ret
2948 SET_SIZE(xrdmsr)
2949
2950 ENTRY(xwrmsr)
2951 pushq %rbp
2952 movq %rsp, %rbp
2953 movl %edi, %ecx
2954 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2955 movq %rsi, %rdx
2956 shrq $32, %rdx
2957 movl %esi, %eax
2958 wrmsr
2959 leave
2960 ret
2961 SET_SIZE(xwrmsr)
2962
2963 ENTRY(get_xcr)
2964 movl %edi, %ecx
2965 #xgetbv
2966 .byte 0x0f,0x01,0xd0
2967 shlq $32, %rdx
2968 orq %rdx, %rax
2969 ret
2970 SET_SIZE(get_xcr)
2971
2972 ENTRY(set_xcr)
2973 movq %rsi, %rdx
2974 shrq $32, %rdx
2975 movl %esi, %eax
2976 movl %edi, %ecx
2977 #xsetbv
2978 .byte 0x0f,0x01,0xd1
2979 ret
2980 SET_SIZE(set_xcr)
2981
2982 #elif defined(__i386)
2983
2984 ENTRY(rdmsr)
2985 movl 4(%esp), %ecx
2986 rdmsr
2987 ret
2988 SET_SIZE(rdmsr)
2989
2990 ENTRY(wrmsr)
2991 movl 4(%esp), %ecx
2992 movl 8(%esp), %eax
2993 movl 12(%esp), %edx
2994 wrmsr
2995 ret
2996 SET_SIZE(wrmsr)
2997
2998 ENTRY(xrdmsr)
2999 pushl %ebp
3000 movl %esp, %ebp
3001 movl 8(%esp), %ecx
3002 pushl %edi
3003 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3004 rdmsr
3005 popl %edi
3006 leave
3007 ret
3008 SET_SIZE(xrdmsr)
3009
3010 ENTRY(xwrmsr)
3011 pushl %ebp
3012 movl %esp, %ebp
3013 movl 8(%esp), %ecx
3014 movl 12(%esp), %eax
3015 movl 16(%esp), %edx
3016 pushl %edi
3017 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3018 wrmsr
3019 popl %edi
3020 leave
3021 ret
3022 SET_SIZE(xwrmsr)
3023
3024 ENTRY(get_xcr)
3025 movl 4(%esp), %ecx
3026 #xgetbv
3027 .byte 0x0f,0x01,0xd0
3028 ret
3029 SET_SIZE(get_xcr)
3030
3031 ENTRY(set_xcr)
3032 movl 4(%esp), %ecx
3033 movl 8(%esp), %eax
3034 movl 12(%esp), %edx
3035 #xsetbv
3036 .byte 0x0f,0x01,0xd1
3037 ret
3038 SET_SIZE(set_xcr)
3039
3040 #endif /* __i386 */
3041
3042 ENTRY(invalidate_cache)
3043 wbinvd
3044 ret
3045 SET_SIZE(invalidate_cache)
3046
3047 #endif /* __lint */
3048
3049 #if defined(__lint)
3050
3051 /*ARGSUSED*/
3052 void
3053 getcregs(struct cregs *crp)
3054 {}
3055
3056 #else /* __lint */
3057
3058 #if defined(__amd64)
3059
3060 ENTRY_NP(getcregs)
3061 #if defined(__xpv)
3062 /*
3063 * Only a few of the hardware control registers or descriptor tables
3064 * are directly accessible to us, so just zero the structure.
3065 *
3066 * XXPV Perhaps it would be helpful for the hypervisor to return
3067 * virtualized versions of these for post-mortem use.
3068 * (Need to reevaluate - perhaps it already does!)
3069 */
3070 pushq %rdi /* save *crp */
3071 movq $CREGSZ, %rsi
3072 call bzero
3073 popq %rdi
3074
3075 /*
3076 * Dump what limited information we can
3077 */
3078 movq %cr0, %rax
3079 movq %rax, CREG_CR0(%rdi) /* cr0 */
3080 movq %cr2, %rax
3081 movq %rax, CREG_CR2(%rdi) /* cr2 */
3082 movq %cr3, %rax
3083 movq %rax, CREG_CR3(%rdi) /* cr3 */
3084 movq %cr4, %rax
3085 movq %rax, CREG_CR4(%rdi) /* cr4 */
3086
3087 #else /* __xpv */
3088
3089 #define GETMSR(r, off, d) \
3090 movl $r, %ecx; \
3091 rdmsr; \
3092 movl %eax, off(d); \
3093 movl %edx, off+4(d)
3094
3095 xorl %eax, %eax
3096 movq %rax, CREG_GDT+8(%rdi)
3097 sgdt CREG_GDT(%rdi) /* 10 bytes */
3098 movq %rax, CREG_IDT+8(%rdi)
3099 sidt CREG_IDT(%rdi) /* 10 bytes */
3100 movq %rax, CREG_LDT(%rdi)
3101 sldt CREG_LDT(%rdi) /* 2 bytes */
3102 movq %rax, CREG_TASKR(%rdi)
3103 str CREG_TASKR(%rdi) /* 2 bytes */
3104 movq %cr0, %rax
3105 movq %rax, CREG_CR0(%rdi) /* cr0 */
3106 movq %cr2, %rax
3107 movq %rax, CREG_CR2(%rdi) /* cr2 */
3108 movq %cr3, %rax
3109 movq %rax, CREG_CR3(%rdi) /* cr3 */
3110 movq %cr4, %rax
3111 movq %rax, CREG_CR4(%rdi) /* cr4 */
3112 movq %cr8, %rax
3113 movq %rax, CREG_CR8(%rdi) /* cr8 */
3114 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3115 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3116 #endif /* __xpv */
3117 ret
3118 SET_SIZE(getcregs)
3119
3120 #undef GETMSR
3121
3122 #elif defined(__i386)
3123
3124 ENTRY_NP(getcregs)
3125 #if defined(__xpv)
3126 /*
3127 * Only a few of the hardware control registers or descriptor tables
3128 * are directly accessible to us, so just zero the structure.
3129 *
3130 * XXPV Perhaps it would be helpful for the hypervisor to return
3131 * virtualized versions of these for post-mortem use.
3132 * (Need to reevaluate - perhaps it already does!)
3133 */
3134 movl 4(%esp), %edx
3135 pushl $CREGSZ
3136 pushl %edx
3137 call bzero
3138 addl $8, %esp
3139 movl 4(%esp), %edx
3140
3141 /*
3142 * Dump what limited information we can
3143 */
3144 movl %cr0, %eax
3145 movl %eax, CREG_CR0(%edx) /* cr0 */
3146 movl %cr2, %eax
3147 movl %eax, CREG_CR2(%edx) /* cr2 */
3148 movl %cr3, %eax
3149 movl %eax, CREG_CR3(%edx) /* cr3 */
3150 movl %cr4, %eax
3151 movl %eax, CREG_CR4(%edx) /* cr4 */
3152
3153 #else /* __xpv */
3154
3155 movl 4(%esp), %edx
3156 movw $0, CREG_GDT+6(%edx)
3157 movw $0, CREG_IDT+6(%edx)
3158 sgdt CREG_GDT(%edx) /* gdt */
3159 sidt CREG_IDT(%edx) /* idt */
3160 sldt CREG_LDT(%edx) /* ldt */
3161 str CREG_TASKR(%edx) /* task */
3162 movl %cr0, %eax
3163 movl %eax, CREG_CR0(%edx) /* cr0 */
3164 movl %cr2, %eax
3165 movl %eax, CREG_CR2(%edx) /* cr2 */
3166 movl %cr3, %eax
3167 movl %eax, CREG_CR3(%edx) /* cr3 */
3168 bt $X86FSET_LARGEPAGE, x86_featureset
3169 jnc .nocr4
3170 movl %cr4, %eax
3171 movl %eax, CREG_CR4(%edx) /* cr4 */
3172 jmp .skip
3173 .nocr4:
3174 movl $0, CREG_CR4(%edx)
3175 .skip:
3176 #endif
3177 ret
3178 SET_SIZE(getcregs)
3179
3180 #endif /* __i386 */
3181 #endif /* __lint */
3182
3183
3184 /*
3185 * A panic trigger is a word which is updated atomically and can only be set
3186 * once. We atomically store 0xDEFACEDD and load the old value. If the
3187 * previous value was 0, we succeed and return 1; otherwise return 0.
3188 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3189 * has its own version of this function to allow it to panic correctly from
3190 * probe context.
3191 */
3192 #if defined(__lint)
3193
3194 /*ARGSUSED*/
3195 int
3196 panic_trigger(int *tp)
3197 { return (0); }
3198
3199 /*ARGSUSED*/
3200 int
3201 dtrace_panic_trigger(int *tp)
3202 { return (0); }
3203
3204 #else /* __lint */
3205
3206 #if defined(__amd64)
3207
3208 ENTRY_NP(panic_trigger)
3209 xorl %eax, %eax
3210 movl $0xdefacedd, %edx
3211 lock
3212 xchgl %edx, (%rdi)
3213 cmpl $0, %edx
3214 je 0f
3215 movl $0, %eax
3216 ret
3217 0: movl $1, %eax
3218 ret
3219 SET_SIZE(panic_trigger)
3220
3221 ENTRY_NP(dtrace_panic_trigger)
3222 xorl %eax, %eax
3223 movl $0xdefacedd, %edx
3224 lock
3225 xchgl %edx, (%rdi)
3226 cmpl $0, %edx
3227 je 0f
3228 movl $0, %eax
3229 ret
3230 0: movl $1, %eax
3231 ret
3232 SET_SIZE(dtrace_panic_trigger)
3233
3234 #elif defined(__i386)
3235
3236 ENTRY_NP(panic_trigger)
3237 movl 4(%esp), %edx / %edx = address of trigger
3238 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3239 lock / assert lock
3240 xchgl %eax, (%edx) / exchange %eax and the trigger
3241 cmpl $0, %eax / if (%eax == 0x0)
3242 je 0f / return (1);
3243 movl $0, %eax / else
3244 ret / return (0);
3245 0: movl $1, %eax
3246 ret
3247 SET_SIZE(panic_trigger)
3248
3249 ENTRY_NP(dtrace_panic_trigger)
3250 movl 4(%esp), %edx / %edx = address of trigger
3251 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3252 lock / assert lock
3253 xchgl %eax, (%edx) / exchange %eax and the trigger
3254 cmpl $0, %eax / if (%eax == 0x0)
3255 je 0f / return (1);
3256 movl $0, %eax / else
3257 ret / return (0);
3258 0: movl $1, %eax
3259 ret
3260 SET_SIZE(dtrace_panic_trigger)
3261
3262 #endif /* __i386 */
3263 #endif /* __lint */
3264
3265 /*
3266 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3267 * into the panic code implemented in panicsys(). vpanic() is responsible
3268 * for passing through the format string and arguments, and constructing a
3269 * regs structure on the stack into which it saves the current register
3270 * values. If we are not dying due to a fatal trap, these registers will
3271 * then be preserved in panicbuf as the current processor state. Before
3272 * invoking panicsys(), vpanic() activates the first panic trigger (see
3273 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3274 * DTrace takes a slightly different panic path if it must panic from probe
3275 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3276 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3277 * branches back into vpanic().
3278 */
3279 #if defined(__lint)
3280
3281 /*ARGSUSED*/
3282 void
3283 vpanic(const char *format, va_list alist)
3284 {}
3285
3286 /*ARGSUSED*/
3287 void
3288 dtrace_vpanic(const char *format, va_list alist)
3289 {}
3290
3291 #else /* __lint */
3292
3293 #if defined(__amd64)
3294
3295 ENTRY_NP(vpanic) /* Initial stack layout: */
3296
3297 pushq %rbp /* | %rip | 0x60 */
3298 movq %rsp, %rbp /* | %rbp | 0x58 */
3299 pushfq /* | rfl | 0x50 */
3300 pushq %r11 /* | %r11 | 0x48 */
3301 pushq %r10 /* | %r10 | 0x40 */
3302 pushq %rbx /* | %rbx | 0x38 */
3303 pushq %rax /* | %rax | 0x30 */
3304 pushq %r9 /* | %r9 | 0x28 */
3305 pushq %r8 /* | %r8 | 0x20 */
3306 pushq %rcx /* | %rcx | 0x18 */
3307 pushq %rdx /* | %rdx | 0x10 */
3308 pushq %rsi /* | %rsi | 0x8 alist */
3309 pushq %rdi /* | %rdi | 0x0 format */
3310
3311 movq %rsp, %rbx /* %rbx = current %rsp */
3312
3313 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3314 call panic_trigger /* %eax = panic_trigger() */
3315
3316 vpanic_common:
3317 /*
3318 * The panic_trigger result is in %eax from the call above, and
3319 * dtrace_panic places it in %eax before branching here.
3320 * The rdmsr instructions that follow below will clobber %eax so
3321 * we stash the panic_trigger result in %r11d.
3322 */
3323 movl %eax, %r11d
3324 cmpl $0, %r11d
3325 je 0f
3326
3327 /*
3328 * If panic_trigger() was successful, we are the first to initiate a
3329 * panic: we now switch to the reserved panic_stack before continuing.
3330 */
3331 leaq panic_stack(%rip), %rsp
3332 addq $PANICSTKSIZE, %rsp
3333 0: subq $REGSIZE, %rsp
3334 /*
3335 * Now that we've got everything set up, store the register values as
3336 * they were when we entered vpanic() to the designated location in
3337 * the regs structure we allocated on the stack.
3338 */
3339 movq 0x0(%rbx), %rcx
3340 movq %rcx, REGOFF_RDI(%rsp)
3341 movq 0x8(%rbx), %rcx
3342 movq %rcx, REGOFF_RSI(%rsp)
3343 movq 0x10(%rbx), %rcx
3344 movq %rcx, REGOFF_RDX(%rsp)
3345 movq 0x18(%rbx), %rcx
3346 movq %rcx, REGOFF_RCX(%rsp)
3347 movq 0x20(%rbx), %rcx
3348
3349 movq %rcx, REGOFF_R8(%rsp)
3350 movq 0x28(%rbx), %rcx
3351 movq %rcx, REGOFF_R9(%rsp)
3352 movq 0x30(%rbx), %rcx
3353 movq %rcx, REGOFF_RAX(%rsp)
3354 movq 0x38(%rbx), %rcx
3355 movq %rcx, REGOFF_RBX(%rsp)
3356 movq 0x58(%rbx), %rcx
3357
3358 movq %rcx, REGOFF_RBP(%rsp)
3359 movq 0x40(%rbx), %rcx
3360 movq %rcx, REGOFF_R10(%rsp)
3361 movq 0x48(%rbx), %rcx
3362 movq %rcx, REGOFF_R11(%rsp)
3363 movq %r12, REGOFF_R12(%rsp)
3364
3365 movq %r13, REGOFF_R13(%rsp)
3366 movq %r14, REGOFF_R14(%rsp)
3367 movq %r15, REGOFF_R15(%rsp)
3368
3369 xorl %ecx, %ecx
3370 movw %ds, %cx
3371 movq %rcx, REGOFF_DS(%rsp)
3372 movw %es, %cx
3373 movq %rcx, REGOFF_ES(%rsp)
3374 movw %fs, %cx
3375 movq %rcx, REGOFF_FS(%rsp)
3376 movw %gs, %cx
3377 movq %rcx, REGOFF_GS(%rsp)
3378
3379 movq $0, REGOFF_TRAPNO(%rsp)
3380
3381 movq $0, REGOFF_ERR(%rsp)
3382 leaq vpanic(%rip), %rcx
3383 movq %rcx, REGOFF_RIP(%rsp)
3384 movw %cs, %cx
3385 movzwq %cx, %rcx
3386 movq %rcx, REGOFF_CS(%rsp)
3387 movq 0x50(%rbx), %rcx
3388 movq %rcx, REGOFF_RFL(%rsp)
3389 movq %rbx, %rcx
3390 addq $0x60, %rcx
3391 movq %rcx, REGOFF_RSP(%rsp)
3392 movw %ss, %cx
3393 movzwq %cx, %rcx
3394 movq %rcx, REGOFF_SS(%rsp)
3395
3396 /*
3397 * panicsys(format, alist, rp, on_panic_stack)
3398 */
3399 movq REGOFF_RDI(%rsp), %rdi /* format */
3400 movq REGOFF_RSI(%rsp), %rsi /* alist */
3401 movq %rsp, %rdx /* struct regs */
3402 movl %r11d, %ecx /* on_panic_stack */
3403 call panicsys
3404 addq $REGSIZE, %rsp
3405 popq %rdi
3406 popq %rsi
3407 popq %rdx
3408 popq %rcx
3409 popq %r8
3410 popq %r9
3411 popq %rax
3412 popq %rbx
3413 popq %r10
3414 popq %r11
3415 popfq
3416 leave
3417 ret
3418 SET_SIZE(vpanic)
3419
3420 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3421
3422 pushq %rbp /* | %rip | 0x60 */
3423 movq %rsp, %rbp /* | %rbp | 0x58 */
3424 pushfq /* | rfl | 0x50 */
3425 pushq %r11 /* | %r11 | 0x48 */
3426 pushq %r10 /* | %r10 | 0x40 */
3427 pushq %rbx /* | %rbx | 0x38 */
3428 pushq %rax /* | %rax | 0x30 */
3429 pushq %r9 /* | %r9 | 0x28 */
3430 pushq %r8 /* | %r8 | 0x20 */
3431 pushq %rcx /* | %rcx | 0x18 */
3432 pushq %rdx /* | %rdx | 0x10 */
3433 pushq %rsi /* | %rsi | 0x8 alist */
3434 pushq %rdi /* | %rdi | 0x0 format */
3435
3436 movq %rsp, %rbx /* %rbx = current %rsp */
3437
3438 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3439 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3440 jmp vpanic_common
3441
3442 SET_SIZE(dtrace_vpanic)
3443
3444 #elif defined(__i386)
3445
3446 ENTRY_NP(vpanic) / Initial stack layout:
3447
3448 pushl %ebp / | %eip | 20
3449 movl %esp, %ebp / | %ebp | 16
3450 pushl %eax / | %eax | 12
3451 pushl %ebx / | %ebx | 8
3452 pushl %ecx / | %ecx | 4
3453 pushl %edx / | %edx | 0
3454
3455 movl %esp, %ebx / %ebx = current stack pointer
3456
3457 lea panic_quiesce, %eax / %eax = &panic_quiesce
3458 pushl %eax / push &panic_quiesce
3459 call panic_trigger / %eax = panic_trigger()
3460 addl $4, %esp / reset stack pointer
3461
3462 vpanic_common:
3463 cmpl $0, %eax / if (%eax == 0)
3464 je 0f / goto 0f;
3465
3466 /*
3467 * If panic_trigger() was successful, we are the first to initiate a
3468 * panic: we now switch to the reserved panic_stack before continuing.
3469 */
3470 lea panic_stack, %esp / %esp = panic_stack
3471 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3472
3473 0: subl $REGSIZE, %esp / allocate struct regs
3474
3475 /*
3476 * Now that we've got everything set up, store the register values as
3477 * they were when we entered vpanic() to the designated location in
3478 * the regs structure we allocated on the stack.
3479 */
3480 #if !defined(__GNUC_AS__)
3481 movw %gs, %edx
3482 movl %edx, REGOFF_GS(%esp)
3483 movw %fs, %edx
3484 movl %edx, REGOFF_FS(%esp)
3485 movw %es, %edx
3486 movl %edx, REGOFF_ES(%esp)
3487 movw %ds, %edx
3488 movl %edx, REGOFF_DS(%esp)
3489 #else /* __GNUC_AS__ */
3490 mov %gs, %edx
3491 mov %edx, REGOFF_GS(%esp)
3492 mov %fs, %edx
3493 mov %edx, REGOFF_FS(%esp)
3494 mov %es, %edx
3495 mov %edx, REGOFF_ES(%esp)
3496 mov %ds, %edx
3497 mov %edx, REGOFF_DS(%esp)
3498 #endif /* __GNUC_AS__ */
3499 movl %edi, REGOFF_EDI(%esp)
3500 movl %esi, REGOFF_ESI(%esp)
3501 movl 16(%ebx), %ecx
3502 movl %ecx, REGOFF_EBP(%esp)
3503 movl %ebx, %ecx
3504 addl $20, %ecx
3505 movl %ecx, REGOFF_ESP(%esp)
3506 movl 8(%ebx), %ecx
3507 movl %ecx, REGOFF_EBX(%esp)
3508 movl 0(%ebx), %ecx
3509 movl %ecx, REGOFF_EDX(%esp)
3510 movl 4(%ebx), %ecx
3511 movl %ecx, REGOFF_ECX(%esp)
3512 movl 12(%ebx), %ecx
3513 movl %ecx, REGOFF_EAX(%esp)
3514 movl $0, REGOFF_TRAPNO(%esp)
3515 movl $0, REGOFF_ERR(%esp)
3516 lea vpanic, %ecx
3517 movl %ecx, REGOFF_EIP(%esp)
3518 #if !defined(__GNUC_AS__)
3519 movw %cs, %edx
3520 #else /* __GNUC_AS__ */
3521 mov %cs, %edx
3522 #endif /* __GNUC_AS__ */
3523 movl %edx, REGOFF_CS(%esp)
3524 pushfl
3525 popl %ecx
3526 #if defined(__xpv)
3527 /*
3528 * Synthesize the PS_IE bit from the event mask bit
3529 */
3530 CURTHREAD(%edx)
3531 KPREEMPT_DISABLE(%edx)
3532 EVENT_MASK_TO_IE(%edx, %ecx)
3533 CURTHREAD(%edx)
3534 KPREEMPT_ENABLE_NOKP(%edx)
3535 #endif
3536 movl %ecx, REGOFF_EFL(%esp)
3537 movl $0, REGOFF_UESP(%esp)
3538 #if !defined(__GNUC_AS__)
3539 movw %ss, %edx
3540 #else /* __GNUC_AS__ */
3541 mov %ss, %edx
3542 #endif /* __GNUC_AS__ */
3543 movl %edx, REGOFF_SS(%esp)
3544
3545 movl %esp, %ecx / %ecx = ®s
3546 pushl %eax / push on_panic_stack
3547 pushl %ecx / push ®s
3548 movl 12(%ebp), %ecx / %ecx = alist
3549 pushl %ecx / push alist
3550 movl 8(%ebp), %ecx / %ecx = format
3551 pushl %ecx / push format
3552 call panicsys / panicsys();
3553 addl $16, %esp / pop arguments
3554
3555 addl $REGSIZE, %esp
3556 popl %edx
3557 popl %ecx
3558 popl %ebx
3559 popl %eax
3560 leave
3561 ret
3562 SET_SIZE(vpanic)
3563
3564 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3565
3566 pushl %ebp / | %eip | 20
3567 movl %esp, %ebp / | %ebp | 16
3568 pushl %eax / | %eax | 12
3569 pushl %ebx / | %ebx | 8
3570 pushl %ecx / | %ecx | 4
3571 pushl %edx / | %edx | 0
3572
3573 movl %esp, %ebx / %ebx = current stack pointer
3574
3575 lea panic_quiesce, %eax / %eax = &panic_quiesce
3576 pushl %eax / push &panic_quiesce
3577 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3578 addl $4, %esp / reset stack pointer
3579 jmp vpanic_common / jump back to common code
3580
3581 SET_SIZE(dtrace_vpanic)
3582
3583 #endif /* __i386 */
3584 #endif /* __lint */
3585
3586 #if defined(__lint)
3587
3588 void
3589 hres_tick(void)
3590 {}
3591
3592 int64_t timedelta;
3593 hrtime_t hres_last_tick;
3594 volatile timestruc_t hrestime;
3595 int64_t hrestime_adj;
3596 volatile int hres_lock;
3597 hrtime_t hrtime_base;
3598
3599 #else /* __lint */
3600
3601 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8)
3602 .NWORD 0, 0
3603
3604 DGDEF3(hrestime_adj, 8, 8)
3605 .long 0, 0
3606
3607 DGDEF3(hres_last_tick, 8, 8)
3608 .long 0, 0
3609
3610 DGDEF3(timedelta, 8, 8)
3611 .long 0, 0
3612
3613 DGDEF3(hres_lock, 4, 8)
3614 .long 0
3615
3616 /*
3617 * initialized to a non zero value to make pc_gethrtime()
3618 * work correctly even before clock is initialized
3619 */
3620 DGDEF3(hrtime_base, 8, 8)
3621 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3622
3623 DGDEF3(adj_shift, 4, 4)
3624 .long ADJ_SHIFT
3625
3626 #if defined(__amd64)
3627
3628 ENTRY_NP(hres_tick)
3629 pushq %rbp
3630 movq %rsp, %rbp
3631
3632 /*
3633 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3634 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3635 * At worst, performing this now instead of under CLOCK_LOCK may
3636 * introduce some jitter in pc_gethrestime().
3637 */
3638 call *gethrtimef(%rip)
3639 movq %rax, %r8
3640
3641 leaq hres_lock(%rip), %rax
3642 movb $-1, %dl
3643 .CL1:
3644 xchgb %dl, (%rax)
3645 testb %dl, %dl
3646 jz .CL3 /* got it */
3647 .CL2:
3648 cmpb $0, (%rax) /* possible to get lock? */
3649 pause
3650 jne .CL2
3651 jmp .CL1 /* yes, try again */
3652 .CL3:
3653 /*
3654 * compute the interval since last time hres_tick was called
3655 * and adjust hrtime_base and hrestime accordingly
3656 * hrtime_base is an 8 byte value (in nsec), hrestime is
3657 * a timestruc_t (sec, nsec)
3658 */
3659 leaq hres_last_tick(%rip), %rax
3660 movq %r8, %r11
3661 subq (%rax), %r8
3662 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3663 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3664 /*
3665 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3666 */
3667 movq %r11, (%rax)
3668
3669 call __adj_hrestime
3670
3671 /*
3672 * release the hres_lock
3673 */
3674 incl hres_lock(%rip)
3675 leave
3676 ret
3677 SET_SIZE(hres_tick)
3678
3679 #elif defined(__i386)
3680
3681 ENTRY_NP(hres_tick)
3682 pushl %ebp
3683 movl %esp, %ebp
3684 pushl %esi
3685 pushl %ebx
3686
3687 /*
3688 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3689 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3690 * At worst, performing this now instead of under CLOCK_LOCK may
3691 * introduce some jitter in pc_gethrestime().
3692 */
3693 call *gethrtimef
3694 movl %eax, %ebx
3695 movl %edx, %esi
3696
3697 movl $hres_lock, %eax
3698 movl $-1, %edx
3699 .CL1:
3700 xchgb %dl, (%eax)
3701 testb %dl, %dl
3702 jz .CL3 / got it
3703 .CL2:
3704 cmpb $0, (%eax) / possible to get lock?
3705 pause
3706 jne .CL2
3707 jmp .CL1 / yes, try again
3708 .CL3:
3709 /*
3710 * compute the interval since last time hres_tick was called
3711 * and adjust hrtime_base and hrestime accordingly
3712 * hrtime_base is an 8 byte value (in nsec), hrestime is
3713 * timestruc_t (sec, nsec)
3714 */
3715
3716 lea hres_last_tick, %eax
3717
3718 movl %ebx, %edx
3719 movl %esi, %ecx
3720
3721 subl (%eax), %edx
3722 sbbl 4(%eax), %ecx
3723
3724 addl %edx, hrtime_base / add interval to hrtime_base
3725 adcl %ecx, hrtime_base+4
3726
3727 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3728
3729 /
3730 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3731 /
3732 movl %ebx, (%eax)
3733 movl %esi, 4(%eax)
3734
3735 / get hrestime at this moment. used as base for pc_gethrestime
3736 /
3737 / Apply adjustment, if any
3738 /
3739 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3740 / (max_hres_adj)
3741 /
3742 / void
3743 / adj_hrestime()
3744 / {
3745 / long long adj;
3746 /
3747 / if (hrestime_adj == 0)
3748 / adj = 0;
3749 / else if (hrestime_adj > 0) {
3750 / if (hrestime_adj < HRES_ADJ)
3751 / adj = hrestime_adj;
3752 / else
3753 / adj = HRES_ADJ;
3754 / }
3755 / else {
3756 / if (hrestime_adj < -(HRES_ADJ))
3757 / adj = -(HRES_ADJ);
3758 / else
3759 / adj = hrestime_adj;
3760 / }
3761 /
3762 / timedelta -= adj;
3763 / hrestime_adj = timedelta;
3764 / hrestime.tv_nsec += adj;
3765 /
3766 / while (hrestime.tv_nsec >= NANOSEC) {
3767 / one_sec++;
3768 / hrestime.tv_sec++;
3769 / hrestime.tv_nsec -= NANOSEC;
3770 / }
3771 / }
3772 __adj_hrestime:
3773 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3774 movl hrestime_adj+4, %edx
3775 andl %esi, %esi
3776 jne .CL4 / no
3777 andl %edx, %edx
3778 jne .CL4 / no
3779 subl %ecx, %ecx / yes, adj = 0;
3780 subl %edx, %edx
3781 jmp .CL5
3782 .CL4:
3783 subl %ecx, %ecx
3784 subl %eax, %eax
3785 subl %esi, %ecx
3786 sbbl %edx, %eax
3787 andl %eax, %eax / if (hrestime_adj > 0)
3788 jge .CL6
3789
3790 / In the following comments, HRES_ADJ is used, while in the code
3791 / max_hres_adj is used.
3792 /
3793 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3794 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3795 / on the logical equivalence of:
3796 /
3797 / !(hrestime_adj < HRES_ADJ)
3798 /
3799 / and the two step sequence:
3800 /
3801 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3802 /
3803 / which computes whether or not the least significant 32-bits
3804 / of hrestime_adj is greater than HRES_ADJ, followed by:
3805 /
3806 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3807 /
3808 / which generates a carry whenever step 1 is true or the most
3809 / significant long of the longlong hrestime_adj is non-zero.
3810
3811 movl max_hres_adj, %ecx / hrestime_adj is positive
3812 subl %esi, %ecx
3813 movl %edx, %eax
3814 adcl $-1, %eax
3815 jnc .CL7
3816 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3817 subl %edx, %edx
3818 jmp .CL5
3819
3820 / The following computation is similar to the one above.
3821 /
3822 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3823 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3824 / on the logical equivalence of:
3825 /
3826 / (hrestime_adj > -HRES_ADJ)
3827 /
3828 / and the two step sequence:
3829 /
3830 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3831 /
3832 / which means the least significant 32-bits of hrestime_adj is
3833 / greater than -HRES_ADJ, followed by:
3834 /
3835 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3836 /
3837 / which generates a carry only when step 1 is true and the most
3838 / significant long of the longlong hrestime_adj is -1.
3839
3840 .CL6: / hrestime_adj is negative
3841 movl %esi, %ecx
3842 addl max_hres_adj, %ecx
3843 movl %edx, %eax
3844 adcl $0, %eax
3845 jc .CL7
3846 xor %ecx, %ecx
3847 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3848 movl $-1, %edx
3849 jmp .CL5
3850 .CL7:
3851 movl %esi, %ecx / adj = hrestime_adj;
3852 .CL5:
3853 movl timedelta, %esi
3854 subl %ecx, %esi
3855 movl timedelta+4, %eax
3856 sbbl %edx, %eax
3857 movl %esi, timedelta
3858 movl %eax, timedelta+4 / timedelta -= adj;
3859 movl %esi, hrestime_adj
3860 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3861 addl hrestime+4, %ecx
3862
3863 movl %ecx, %eax / eax = tv_nsec
3864 1:
3865 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3866 jb .CL8 / no
3867 incl one_sec / yes, one_sec++;
3868 incl hrestime / hrestime.tv_sec++;
3869 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3870 jmp 1b / check for more seconds
3871
3872 .CL8:
3873 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3874 incl hres_lock / release the hres_lock
3875
3876 popl %ebx
3877 popl %esi
3878 leave
3879 ret
3880 SET_SIZE(hres_tick)
3881
3882 #endif /* __i386 */
3883 #endif /* __lint */
3884
3885 /*
3886 * void prefetch_smap_w(void *)
3887 *
3888 * Prefetch ahead within a linear list of smap structures.
3889 * Not implemented for ia32. Stub for compatibility.
3890 */
3891
3892 #if defined(__lint)
3893
3894 /*ARGSUSED*/
3895 void prefetch_smap_w(void *smp)
3896 {}
3897
3898 #else /* __lint */
3899
3900 ENTRY(prefetch_smap_w)
3901 rep; ret /* use 2 byte return instruction when branch target */
3902 /* AMD Software Optimization Guide - Section 6.2 */
3903 SET_SIZE(prefetch_smap_w)
3904
3905 #endif /* __lint */
3906
3907 /*
3908 * prefetch_page_r(page_t *)
3909 * issue prefetch instructions for a page_t
3910 */
3911 #if defined(__lint)
3912
3913 /*ARGSUSED*/
3914 void
3915 prefetch_page_r(void *pp)
3916 {}
3917
3918 #else /* __lint */
3919
3920 ENTRY(prefetch_page_r)
3921 rep; ret /* use 2 byte return instruction when branch target */
3922 /* AMD Software Optimization Guide - Section 6.2 */
3923 SET_SIZE(prefetch_page_r)
3924
3925 #endif /* __lint */
3926
3927 #if defined(__lint)
3928
3929 /*ARGSUSED*/
3930 int
3931 bcmp(const void *s1, const void *s2, size_t count)
3932 { return (0); }
3933
3934 #else /* __lint */
3935
3936 #if defined(__amd64)
3937
3938 ENTRY(bcmp)
3939 pushq %rbp
3940 movq %rsp, %rbp
3941 #ifdef DEBUG
3942 testq %rdx,%rdx
3943 je 1f
3944 movq postbootkernelbase(%rip), %r11
3945 cmpq %r11, %rdi
3946 jb 0f
3947 cmpq %r11, %rsi
3948 jnb 1f
3949 0: leaq .bcmp_panic_msg(%rip), %rdi
3950 xorl %eax, %eax
3951 call panic
3952 1:
3953 #endif /* DEBUG */
3954 call memcmp
3955 testl %eax, %eax
3956 setne %dl
3957 leave
3958 movzbl %dl, %eax
3959 ret
3960 SET_SIZE(bcmp)
3961
3962 #elif defined(__i386)
3963
3964 #define ARG_S1 8
3965 #define ARG_S2 12
3966 #define ARG_LENGTH 16
3967
3968 ENTRY(bcmp)
3969 pushl %ebp
3970 movl %esp, %ebp / create new stack frame
3971 #ifdef DEBUG
3972 cmpl $0, ARG_LENGTH(%ebp)
3973 je 1f
3974 movl postbootkernelbase, %eax
3975 cmpl %eax, ARG_S1(%ebp)
3976 jb 0f
3977 cmpl %eax, ARG_S2(%ebp)
3978 jnb 1f
3979 0: pushl $.bcmp_panic_msg
3980 call panic
3981 1:
3982 #endif /* DEBUG */
3983
3984 pushl %edi / save register variable
3985 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3986 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3987 cmpl %eax, %ecx / if the same string
3988 je .equal / goto .equal
3989 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3990 cmpl $4, %edi / if %edi < 4
3991 jb .byte_check / goto .byte_check
3992 .align 4
3993 .word_loop:
3994 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3995 leal -4(%edi), %edi / %edi -= 4
3996 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3997 jne .word_not_equal / if not equal, goto .word_not_equal
3998 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3999 leal 4(%eax), %eax / %eax += 4 (next word)
4000 cmpl $4, %edi / if %edi >= 4
4001 jae .word_loop / goto .word_loop
4002 .byte_check:
4003 cmpl $0, %edi / if %edi == 0
4004 je .equal / goto .equal
4005 jmp .byte_loop / goto .byte_loop (checks in bytes)
4006 .word_not_equal:
4007 leal 4(%edi), %edi / %edi += 4 (post-decremented)
4008 .align 4
4009 .byte_loop:
4010 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
4011 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
4012 jne .not_equal / if not equal, goto .not_equal
4013 incl %ecx / %ecx++ (next byte)
4014 incl %eax / %eax++ (next byte)
4015 decl %edi / %edi--
4016 jnz .byte_loop / if not zero, goto .byte_loop
4017 .equal:
4018 xorl %eax, %eax / %eax = 0
4019 popl %edi / restore register variable
4020 leave / restore old stack frame
4021 ret / return (NULL)
4022 .align 4
4023 .not_equal:
4024 movl $1, %eax / return 1
4025 popl %edi / restore register variable
4026 leave / restore old stack frame
4027 ret / return (NULL)
4028 SET_SIZE(bcmp)
4029
4030 #endif /* __i386 */
4031
4032 #ifdef DEBUG
4033 .text
4034 .bcmp_panic_msg:
4035 .string "bcmp: arguments below kernelbase"
4036 #endif /* DEBUG */
4037
4038 #endif /* __lint */
4039
4040 #if defined(__lint)
4041
4042 uint_t
4043 bsrw_insn(uint16_t mask)
4044 {
4045 uint_t index = sizeof (mask) * NBBY - 1;
4046
4047 while ((mask & (1 << index)) == 0)
4048 index--;
4049 return (index);
4050 }
4051
4052 #else /* __lint */
4053
4054 #if defined(__amd64)
4055
4056 ENTRY_NP(bsrw_insn)
4057 xorl %eax, %eax
4058 bsrw %di, %ax
4059 ret
4060 SET_SIZE(bsrw_insn)
4061
4062 #elif defined(__i386)
4063
4064 ENTRY_NP(bsrw_insn)
4065 movw 4(%esp), %cx
4066 xorl %eax, %eax
4067 bsrw %cx, %ax
4068 ret
4069 SET_SIZE(bsrw_insn)
4070
4071 #endif /* __i386 */
4072 #endif /* __lint */
4073
4074 #if defined(__lint)
4075
4076 uint_t
4077 atomic_btr32(uint32_t *pending, uint_t pil)
4078 {
4079 return (*pending &= ~(1 << pil));
4080 }
4081
4082 #else /* __lint */
4083
4084 #if defined(__i386)
4085
4086 ENTRY_NP(atomic_btr32)
4087 movl 4(%esp), %ecx
4088 movl 8(%esp), %edx
4089 xorl %eax, %eax
4090 lock
4091 btrl %edx, (%ecx)
4092 setc %al
4093 ret
4094 SET_SIZE(atomic_btr32)
4095
4096 #endif /* __i386 */
4097 #endif /* __lint */
4098
4099 #if defined(__lint)
4100
4101 /*ARGSUSED*/
4102 void
4103 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4104 uint_t arg2)
4105 {}
4106
4107 #else /* __lint */
4108
4109 #if defined(__amd64)
4110
4111 ENTRY_NP(switch_sp_and_call)
4112 pushq %rbp
4113 movq %rsp, %rbp /* set up stack frame */
4114 movq %rdi, %rsp /* switch stack pointer */
4115 movq %rdx, %rdi /* pass func arg 1 */
4116 movq %rsi, %r11 /* save function to call */
4117 movq %rcx, %rsi /* pass func arg 2 */
4118 call *%r11 /* call function */
4119 leave /* restore stack */
4120 ret
4121 SET_SIZE(switch_sp_and_call)
4122
4123 #elif defined(__i386)
4124
4125 ENTRY_NP(switch_sp_and_call)
4126 pushl %ebp
4127 mov %esp, %ebp /* set up stack frame */
4128 movl 8(%ebp), %esp /* switch stack pointer */
4129 pushl 20(%ebp) /* push func arg 2 */
4130 pushl 16(%ebp) /* push func arg 1 */
4131 call *12(%ebp) /* call function */
4132 addl $8, %esp /* pop arguments */
4133 leave /* restore stack */
4134 ret
4135 SET_SIZE(switch_sp_and_call)
4136
4137 #endif /* __i386 */
4138 #endif /* __lint */
4139
4140 #if defined(__lint)
4141
4142 void
4143 kmdb_enter(void)
4144 {}
4145
4146 #else /* __lint */
4147
4148 #if defined(__amd64)
4149
4150 ENTRY_NP(kmdb_enter)
4151 pushq %rbp
4152 movq %rsp, %rbp
4153
4154 /*
4155 * Save flags, do a 'cli' then return the saved flags
4156 */
4157 call intr_clear
4158
4159 int $T_DBGENTR
4160
4161 /*
4162 * Restore the saved flags
4163 */
4164 movq %rax, %rdi
4165 call intr_restore
4166
4167 leave
4168 ret
4169 SET_SIZE(kmdb_enter)
4170
4171 #elif defined(__i386)
4172
4173 ENTRY_NP(kmdb_enter)
4174 pushl %ebp
4175 movl %esp, %ebp
4176
4177 /*
4178 * Save flags, do a 'cli' then return the saved flags
4179 */
4180 call intr_clear
4181
4182 int $T_DBGENTR
4183
4184 /*
4185 * Restore the saved flags
4186 */
4187 pushl %eax
4188 call intr_restore
4189 addl $4, %esp
4190
4191 leave
4192 ret
4193 SET_SIZE(kmdb_enter)
4194
4195 #endif /* __i386 */
4196 #endif /* __lint */
4197
4198 #if defined(__lint)
4199
4200 void
4201 return_instr(void)
4202 {}
4203
4204 #else /* __lint */
4205
4206 ENTRY_NP(return_instr)
4207 rep; ret /* use 2 byte instruction when branch target */
4208 /* AMD Software Optimization Guide - Section 6.2 */
4209 SET_SIZE(return_instr)
4210
4211 #endif /* __lint */
4212
4213 #if defined(__lint)
4214
4215 ulong_t
4216 getflags(void)
4217 {
4218 return (0);
4219 }
4220
4221 #else /* __lint */
4222
4223 #if defined(__amd64)
4224
4225 ENTRY(getflags)
4226 pushfq
4227 popq %rax
4228 #if defined(__xpv)
4229 CURTHREAD(%rdi)
4230 KPREEMPT_DISABLE(%rdi)
4231 /*
4232 * Synthesize the PS_IE bit from the event mask bit
4233 */
4234 CURVCPU(%r11)
4235 andq $_BITNOT(PS_IE), %rax
4236 XEN_TEST_UPCALL_MASK(%r11)
4237 jnz 1f
4238 orq $PS_IE, %rax
4239 1:
4240 KPREEMPT_ENABLE_NOKP(%rdi)
4241 #endif
4242 ret
4243 SET_SIZE(getflags)
4244
4245 #elif defined(__i386)
4246
4247 ENTRY(getflags)
4248 pushfl
4249 popl %eax
4250 #if defined(__xpv)
4251 CURTHREAD(%ecx)
4252 KPREEMPT_DISABLE(%ecx)
4253 /*
4254 * Synthesize the PS_IE bit from the event mask bit
4255 */
4256 CURVCPU(%edx)
4257 andl $_BITNOT(PS_IE), %eax
4258 XEN_TEST_UPCALL_MASK(%edx)
4259 jnz 1f
4260 orl $PS_IE, %eax
4261 1:
4262 KPREEMPT_ENABLE_NOKP(%ecx)
4263 #endif
4264 ret
4265 SET_SIZE(getflags)
4266
4267 #endif /* __i386 */
4268
4269 #endif /* __lint */
4270
4271 #if defined(__lint)
4272
4273 ftrace_icookie_t
4274 ftrace_interrupt_disable(void)
4275 { return (0); }
4276
4277 #else /* __lint */
4278
4279 #if defined(__amd64)
4280
4281 ENTRY(ftrace_interrupt_disable)
4282 pushfq
4283 popq %rax
4284 CLI(%rdx)
4285 ret
4286 SET_SIZE(ftrace_interrupt_disable)
4287
4288 #elif defined(__i386)
4289
4290 ENTRY(ftrace_interrupt_disable)
4291 pushfl
4292 popl %eax
4293 CLI(%edx)
4294 ret
4295 SET_SIZE(ftrace_interrupt_disable)
4296
4297 #endif /* __i386 */
4298 #endif /* __lint */
4299
4300 #if defined(__lint)
4301
4302 /*ARGSUSED*/
4303 void
4304 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4305 {}
4306
4307 #else /* __lint */
4308
4309 #if defined(__amd64)
4310
4311 ENTRY(ftrace_interrupt_enable)
4312 pushq %rdi
4313 popfq
4314 ret
4315 SET_SIZE(ftrace_interrupt_enable)
4316
4317 #elif defined(__i386)
4318
4319 ENTRY(ftrace_interrupt_enable)
4320 movl 4(%esp), %eax
4321 pushl %eax
4322 popfl
4323 ret
4324 SET_SIZE(ftrace_interrupt_enable)
4325
4326 #endif /* __i386 */
4327 #endif /* __lint */
4328
4329 #if defined (__lint)
4330
4331 /*ARGSUSED*/
4332 void
4333 clflush_insn(caddr_t addr)
4334 {}
4335
4336 #else /* __lint */
4337
4338 #if defined (__amd64)
4339 ENTRY(clflush_insn)
4340 clflush (%rdi)
4341 ret
4342 SET_SIZE(clflush_insn)
4343 #elif defined (__i386)
4344 ENTRY(clflush_insn)
4345 movl 4(%esp), %eax
4346 clflush (%eax)
4347 ret
4348 SET_SIZE(clflush_insn)
4349
4350 #endif /* __i386 */
4351 #endif /* __lint */
4352
4353 #if defined (__lint)
4354 /*ARGSUSED*/
4355 void
4356 mfence_insn(void)
4357 {}
4358
4359 #else /* __lint */
4360
4361 #if defined (__amd64)
4362 ENTRY(mfence_insn)
4363 mfence
4364 ret
4365 SET_SIZE(mfence_insn)
4366 #elif defined (__i386)
4367 ENTRY(mfence_insn)
4368 mfence
4369 ret
4370 SET_SIZE(mfence_insn)
4371
4372 #endif /* __i386 */
4373 #endif /* __lint */
4374
4375 /*
4376 * VMware implements an I/O port that programs can query to detect if software
4377 * is running in a VMware hypervisor. This hypervisor port behaves differently
4378 * depending on magic values in certain registers and modifies some registers
4379 * as a side effect.
4380 *
4381 * References: http://kb.vmware.com/kb/1009458
4382 */
4383
4384 #if defined(__lint)
4385
4386 /* ARGSUSED */
4387 void
4388 vmware_port(int cmd, uint32_t *regs) { return; }
4389
4390 #else
4391
4392 #if defined(__amd64)
4393
4394 ENTRY(vmware_port)
4395 pushq %rbx
4396 movl $VMWARE_HVMAGIC, %eax
4397 movl $0xffffffff, %ebx
4398 movl %edi, %ecx
4399 movl $VMWARE_HVPORT, %edx
4400 inl (%dx)
4401 movl %eax, (%rsi)
4402 movl %ebx, 4(%rsi)
4403 movl %ecx, 8(%rsi)
4404 movl %edx, 12(%rsi)
4405 popq %rbx
4406 ret
4407 SET_SIZE(vmware_port)
4408
4409 #elif defined(__i386)
4410
4411 ENTRY(vmware_port)
4412 pushl %ebx
4413 pushl %esi
4414 movl $VMWARE_HVMAGIC, %eax
4415 movl $0xffffffff, %ebx
4416 movl 12(%esp), %ecx
4417 movl $VMWARE_HVPORT, %edx
4418 inl (%dx)
4419 movl 16(%esp), %esi
4420 movl %eax, (%esi)
4421 movl %ebx, 4(%esi)
4422 movl %ecx, 8(%esi)
4423 movl %edx, 12(%esi)
4424 popl %esi
4425 popl %ebx
4426 ret
4427 SET_SIZE(vmware_port)
4428
4429 #endif /* __i386 */
4430 #endif /* __lint */