Print this page
4374 dn_free_ranges should use range_tree_t
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Max Grossman <max.grossman@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com
Reviewed by: Garrett D'Amore <garrett@damore.org>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Dan McDonald <danmcd@omniti.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/i86_subr.s
+++ new/usr/src/uts/intel/ia32/ml/i86_subr.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 + * Copyright (c) 2014 by Delphix. All rights reserved.
25 26 */
26 27
27 28 /*
28 29 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
29 30 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
30 31 * All Rights Reserved
31 32 */
32 33
33 34 /*
34 35 * Copyright (c) 2009, Intel Corporation.
35 36 * All rights reserved.
36 37 */
37 38
38 39 /*
39 40 * General assembly language routines.
40 41 * It is the intent of this file to contain routines that are
41 42 * independent of the specific kernel architecture, and those that are
42 43 * common across kernel architectures.
43 44 * As architectures diverge, and implementations of specific
44 45 * architecture-dependent routines change, the routines should be moved
45 46 * from this file into the respective ../`arch -k`/subr.s file.
46 47 */
47 48
48 49 #include <sys/asm_linkage.h>
49 50 #include <sys/asm_misc.h>
50 51 #include <sys/panic.h>
51 52 #include <sys/ontrap.h>
52 53 #include <sys/regset.h>
53 54 #include <sys/privregs.h>
54 55 #include <sys/reboot.h>
55 56 #include <sys/psw.h>
56 57 #include <sys/x86_archext.h>
57 58
58 59 #if defined(__lint)
59 60 #include <sys/types.h>
60 61 #include <sys/systm.h>
61 62 #include <sys/thread.h>
62 63 #include <sys/archsystm.h>
63 64 #include <sys/byteorder.h>
64 65 #include <sys/dtrace.h>
65 66 #include <sys/ftrace.h>
66 67 #else /* __lint */
67 68 #include "assym.h"
68 69 #endif /* __lint */
69 70 #include <sys/dditypes.h>
70 71
71 72 /*
72 73 * on_fault()
73 74 * Catch lofault faults. Like setjmp except it returns one
74 75 * if code following causes uncorrectable fault. Turned off
75 76 * by calling no_fault().
76 77 */
77 78
78 79 #if defined(__lint)
79 80
80 81 /* ARGSUSED */
81 82 int
82 83 on_fault(label_t *ljb)
83 84 { return (0); }
84 85
85 86 void
86 87 no_fault(void)
87 88 {}
88 89
89 90 #else /* __lint */
90 91
91 92 #if defined(__amd64)
92 93
93 94 ENTRY(on_fault)
94 95 movq %gs:CPU_THREAD, %rsi
95 96 leaq catch_fault(%rip), %rdx
96 97 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
97 98 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
98 99 jmp setjmp /* let setjmp do the rest */
99 100
100 101 catch_fault:
101 102 movq %gs:CPU_THREAD, %rsi
102 103 movq T_ONFAULT(%rsi), %rdi /* address of save area */
103 104 xorl %eax, %eax
104 105 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
105 106 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
106 107 jmp longjmp /* let longjmp do the rest */
107 108 SET_SIZE(on_fault)
108 109
109 110 ENTRY(no_fault)
110 111 movq %gs:CPU_THREAD, %rsi
111 112 xorl %eax, %eax
112 113 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
113 114 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
114 115 ret
115 116 SET_SIZE(no_fault)
116 117
117 118 #elif defined(__i386)
118 119
119 120 ENTRY(on_fault)
120 121 movl %gs:CPU_THREAD, %edx
121 122 movl 4(%esp), %eax /* jumpbuf address */
122 123 leal catch_fault, %ecx
123 124 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
124 125 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
125 126 jmp setjmp /* let setjmp do the rest */
126 127
127 128 catch_fault:
128 129 movl %gs:CPU_THREAD, %edx
129 130 xorl %eax, %eax
130 131 movl T_ONFAULT(%edx), %ecx /* address of save area */
131 132 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
132 133 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
133 134 pushl %ecx
134 135 call longjmp /* let longjmp do the rest */
135 136 SET_SIZE(on_fault)
136 137
137 138 ENTRY(no_fault)
138 139 movl %gs:CPU_THREAD, %edx
139 140 xorl %eax, %eax
140 141 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
141 142 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
142 143 ret
143 144 SET_SIZE(no_fault)
144 145
145 146 #endif /* __i386 */
146 147 #endif /* __lint */
147 148
148 149 /*
149 150 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
150 151 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
151 152 */
152 153
153 154 #if defined(lint)
154 155
155 156 void
156 157 on_trap_trampoline(void)
157 158 {}
158 159
159 160 #else /* __lint */
160 161
161 162 #if defined(__amd64)
162 163
163 164 ENTRY(on_trap_trampoline)
164 165 movq %gs:CPU_THREAD, %rsi
165 166 movq T_ONTRAP(%rsi), %rdi
166 167 addq $OT_JMPBUF, %rdi
167 168 jmp longjmp
168 169 SET_SIZE(on_trap_trampoline)
169 170
170 171 #elif defined(__i386)
171 172
172 173 ENTRY(on_trap_trampoline)
173 174 movl %gs:CPU_THREAD, %eax
174 175 movl T_ONTRAP(%eax), %eax
175 176 addl $OT_JMPBUF, %eax
176 177 pushl %eax
177 178 call longjmp
178 179 SET_SIZE(on_trap_trampoline)
179 180
180 181 #endif /* __i386 */
181 182 #endif /* __lint */
182 183
183 184 /*
184 185 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
185 186 * more information about the on_trap() mechanism. If the on_trap_data is the
186 187 * same as the topmost stack element, we just modify that element.
187 188 */
188 189 #if defined(lint)
189 190
190 191 /*ARGSUSED*/
191 192 int
192 193 on_trap(on_trap_data_t *otp, uint_t prot)
193 194 { return (0); }
194 195
195 196 #else /* __lint */
196 197
197 198 #if defined(__amd64)
198 199
199 200 ENTRY(on_trap)
200 201 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
201 202 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
202 203 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
203 204 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
204 205 xorl %ecx, %ecx
205 206 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
206 207 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
207 208 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
208 209 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
209 210 cmpq %rdi, %rcx /* if (otp == %rcx) */
210 211 je 0f /* don't modify t_ontrap */
211 212
212 213 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
213 214 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
214 215
215 216 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
216 217 jmp setjmp
217 218 SET_SIZE(on_trap)
218 219
219 220 #elif defined(__i386)
220 221
221 222 ENTRY(on_trap)
222 223 movl 4(%esp), %eax /* %eax = otp */
223 224 movl 8(%esp), %edx /* %edx = prot */
224 225
225 226 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
226 227 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
227 228 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
228 229 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
229 230 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
230 231 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
231 232 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
232 233 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
233 234 cmpl %eax, %ecx /* if (otp == %ecx) */
234 235 je 0f /* don't modify t_ontrap */
235 236
236 237 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
237 238 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
238 239
239 240 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
240 241 movl %eax, 4(%esp) /* put %eax back on the stack */
241 242 jmp setjmp /* let setjmp do the rest */
242 243 SET_SIZE(on_trap)
243 244
244 245 #endif /* __i386 */
245 246 #endif /* __lint */
246 247
247 248 /*
248 249 * Setjmp and longjmp implement non-local gotos using state vectors
249 250 * type label_t.
250 251 */
251 252
252 253 #if defined(__lint)
253 254
254 255 /* ARGSUSED */
255 256 int
256 257 setjmp(label_t *lp)
257 258 { return (0); }
258 259
259 260 /* ARGSUSED */
260 261 void
261 262 longjmp(label_t *lp)
262 263 {}
263 264
264 265 #else /* __lint */
265 266
266 267 #if LABEL_PC != 0
267 268 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
268 269 #endif /* LABEL_PC != 0 */
269 270
270 271 #if defined(__amd64)
271 272
272 273 ENTRY(setjmp)
273 274 movq %rsp, LABEL_SP(%rdi)
274 275 movq %rbp, LABEL_RBP(%rdi)
275 276 movq %rbx, LABEL_RBX(%rdi)
276 277 movq %r12, LABEL_R12(%rdi)
277 278 movq %r13, LABEL_R13(%rdi)
278 279 movq %r14, LABEL_R14(%rdi)
279 280 movq %r15, LABEL_R15(%rdi)
280 281 movq (%rsp), %rdx /* return address */
281 282 movq %rdx, (%rdi) /* LABEL_PC is 0 */
282 283 xorl %eax, %eax /* return 0 */
283 284 ret
284 285 SET_SIZE(setjmp)
285 286
286 287 ENTRY(longjmp)
287 288 movq LABEL_SP(%rdi), %rsp
288 289 movq LABEL_RBP(%rdi), %rbp
289 290 movq LABEL_RBX(%rdi), %rbx
290 291 movq LABEL_R12(%rdi), %r12
291 292 movq LABEL_R13(%rdi), %r13
292 293 movq LABEL_R14(%rdi), %r14
293 294 movq LABEL_R15(%rdi), %r15
294 295 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
295 296 movq %rdx, (%rsp)
296 297 xorl %eax, %eax
297 298 incl %eax /* return 1 */
298 299 ret
299 300 SET_SIZE(longjmp)
300 301
301 302 #elif defined(__i386)
302 303
303 304 ENTRY(setjmp)
304 305 movl 4(%esp), %edx /* address of save area */
305 306 movl %ebp, LABEL_EBP(%edx)
306 307 movl %ebx, LABEL_EBX(%edx)
307 308 movl %esi, LABEL_ESI(%edx)
308 309 movl %edi, LABEL_EDI(%edx)
309 310 movl %esp, 4(%edx)
310 311 movl (%esp), %ecx /* %eip (return address) */
311 312 movl %ecx, (%edx) /* LABEL_PC is 0 */
312 313 subl %eax, %eax /* return 0 */
313 314 ret
314 315 SET_SIZE(setjmp)
315 316
316 317 ENTRY(longjmp)
317 318 movl 4(%esp), %edx /* address of save area */
318 319 movl LABEL_EBP(%edx), %ebp
319 320 movl LABEL_EBX(%edx), %ebx
320 321 movl LABEL_ESI(%edx), %esi
321 322 movl LABEL_EDI(%edx), %edi
322 323 movl 4(%edx), %esp
323 324 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
324 325 movl $1, %eax
325 326 addl $4, %esp /* pop ret adr */
326 327 jmp *%ecx /* indirect */
327 328 SET_SIZE(longjmp)
328 329
329 330 #endif /* __i386 */
330 331 #endif /* __lint */
331 332
332 333 /*
333 334 * if a() calls b() calls caller(),
334 335 * caller() returns return address in a().
335 336 * (Note: We assume a() and b() are C routines which do the normal entry/exit
336 337 * sequence.)
337 338 */
338 339
339 340 #if defined(__lint)
340 341
341 342 caddr_t
342 343 caller(void)
343 344 { return (0); }
344 345
345 346 #else /* __lint */
346 347
347 348 #if defined(__amd64)
348 349
349 350 ENTRY(caller)
350 351 movq 8(%rbp), %rax /* b()'s return pc, in a() */
351 352 ret
352 353 SET_SIZE(caller)
353 354
354 355 #elif defined(__i386)
355 356
356 357 ENTRY(caller)
357 358 movl 4(%ebp), %eax /* b()'s return pc, in a() */
358 359 ret
359 360 SET_SIZE(caller)
360 361
361 362 #endif /* __i386 */
362 363 #endif /* __lint */
363 364
364 365 /*
365 366 * if a() calls callee(), callee() returns the
366 367 * return address in a();
367 368 */
368 369
369 370 #if defined(__lint)
370 371
371 372 caddr_t
372 373 callee(void)
373 374 { return (0); }
374 375
375 376 #else /* __lint */
376 377
377 378 #if defined(__amd64)
378 379
379 380 ENTRY(callee)
380 381 movq (%rsp), %rax /* callee()'s return pc, in a() */
381 382 ret
382 383 SET_SIZE(callee)
383 384
384 385 #elif defined(__i386)
385 386
386 387 ENTRY(callee)
387 388 movl (%esp), %eax /* callee()'s return pc, in a() */
388 389 ret
389 390 SET_SIZE(callee)
390 391
391 392 #endif /* __i386 */
392 393 #endif /* __lint */
393 394
394 395 /*
395 396 * return the current frame pointer
396 397 */
397 398
398 399 #if defined(__lint)
399 400
400 401 greg_t
401 402 getfp(void)
402 403 { return (0); }
403 404
404 405 #else /* __lint */
405 406
406 407 #if defined(__amd64)
407 408
408 409 ENTRY(getfp)
409 410 movq %rbp, %rax
410 411 ret
411 412 SET_SIZE(getfp)
412 413
413 414 #elif defined(__i386)
414 415
415 416 ENTRY(getfp)
416 417 movl %ebp, %eax
417 418 ret
418 419 SET_SIZE(getfp)
419 420
420 421 #endif /* __i386 */
421 422 #endif /* __lint */
422 423
423 424 /*
424 425 * Invalidate a single page table entry in the TLB
425 426 */
426 427
427 428 #if defined(__lint)
428 429
429 430 /* ARGSUSED */
430 431 void
431 432 mmu_tlbflush_entry(caddr_t m)
432 433 {}
433 434
434 435 #else /* __lint */
435 436
436 437 #if defined(__amd64)
437 438
438 439 ENTRY(mmu_tlbflush_entry)
439 440 invlpg (%rdi)
440 441 ret
441 442 SET_SIZE(mmu_tlbflush_entry)
442 443
443 444 #elif defined(__i386)
444 445
445 446 ENTRY(mmu_tlbflush_entry)
446 447 movl 4(%esp), %eax
447 448 invlpg (%eax)
448 449 ret
449 450 SET_SIZE(mmu_tlbflush_entry)
450 451
451 452 #endif /* __i386 */
452 453 #endif /* __lint */
453 454
454 455
455 456 /*
456 457 * Get/Set the value of various control registers
457 458 */
458 459
459 460 #if defined(__lint)
460 461
461 462 ulong_t
462 463 getcr0(void)
463 464 { return (0); }
464 465
465 466 /* ARGSUSED */
466 467 void
467 468 setcr0(ulong_t value)
468 469 {}
469 470
470 471 ulong_t
471 472 getcr2(void)
472 473 { return (0); }
473 474
474 475 ulong_t
475 476 getcr3(void)
476 477 { return (0); }
477 478
478 479 #if !defined(__xpv)
479 480 /* ARGSUSED */
480 481 void
481 482 setcr3(ulong_t val)
482 483 {}
483 484
484 485 void
485 486 reload_cr3(void)
486 487 {}
487 488 #endif
488 489
489 490 ulong_t
490 491 getcr4(void)
491 492 { return (0); }
492 493
493 494 /* ARGSUSED */
494 495 void
495 496 setcr4(ulong_t val)
496 497 {}
497 498
498 499 #if defined(__amd64)
499 500
500 501 ulong_t
501 502 getcr8(void)
502 503 { return (0); }
503 504
504 505 /* ARGSUSED */
505 506 void
506 507 setcr8(ulong_t val)
507 508 {}
508 509
509 510 #endif /* __amd64 */
510 511
511 512 #else /* __lint */
512 513
513 514 #if defined(__amd64)
514 515
515 516 ENTRY(getcr0)
516 517 movq %cr0, %rax
517 518 ret
518 519 SET_SIZE(getcr0)
519 520
520 521 ENTRY(setcr0)
521 522 movq %rdi, %cr0
522 523 ret
523 524 SET_SIZE(setcr0)
524 525
525 526 ENTRY(getcr2)
526 527 #if defined(__xpv)
527 528 movq %gs:CPU_VCPU_INFO, %rax
528 529 movq VCPU_INFO_ARCH_CR2(%rax), %rax
529 530 #else
530 531 movq %cr2, %rax
531 532 #endif
532 533 ret
533 534 SET_SIZE(getcr2)
534 535
535 536 ENTRY(getcr3)
536 537 movq %cr3, %rax
537 538 ret
538 539 SET_SIZE(getcr3)
539 540
540 541 #if !defined(__xpv)
541 542
542 543 ENTRY(setcr3)
543 544 movq %rdi, %cr3
544 545 ret
545 546 SET_SIZE(setcr3)
546 547
547 548 ENTRY(reload_cr3)
548 549 movq %cr3, %rdi
549 550 movq %rdi, %cr3
550 551 ret
551 552 SET_SIZE(reload_cr3)
552 553
553 554 #endif /* __xpv */
554 555
555 556 ENTRY(getcr4)
556 557 movq %cr4, %rax
557 558 ret
558 559 SET_SIZE(getcr4)
559 560
560 561 ENTRY(setcr4)
561 562 movq %rdi, %cr4
562 563 ret
563 564 SET_SIZE(setcr4)
564 565
565 566 ENTRY(getcr8)
566 567 movq %cr8, %rax
567 568 ret
568 569 SET_SIZE(getcr8)
569 570
570 571 ENTRY(setcr8)
571 572 movq %rdi, %cr8
572 573 ret
573 574 SET_SIZE(setcr8)
574 575
575 576 #elif defined(__i386)
576 577
577 578 ENTRY(getcr0)
578 579 movl %cr0, %eax
579 580 ret
580 581 SET_SIZE(getcr0)
581 582
582 583 ENTRY(setcr0)
583 584 movl 4(%esp), %eax
584 585 movl %eax, %cr0
585 586 ret
586 587 SET_SIZE(setcr0)
587 588
588 589 /*
589 590 * "lock mov %cr0" is used on processors which indicate it is
590 591 * supported via CPUID. Normally the 32 bit TPR is accessed via
591 592 * the local APIC.
592 593 */
593 594 ENTRY(getcr8)
594 595 lock
595 596 movl %cr0, %eax
596 597 ret
597 598 SET_SIZE(getcr8)
598 599
599 600 ENTRY(setcr8)
600 601 movl 4(%esp), %eax
601 602 lock
602 603 movl %eax, %cr0
603 604 ret
604 605 SET_SIZE(setcr8)
605 606
606 607 ENTRY(getcr2)
607 608 #if defined(__xpv)
608 609 movl %gs:CPU_VCPU_INFO, %eax
609 610 movl VCPU_INFO_ARCH_CR2(%eax), %eax
610 611 #else
611 612 movl %cr2, %eax
612 613 #endif
613 614 ret
614 615 SET_SIZE(getcr2)
615 616
616 617 ENTRY(getcr3)
617 618 movl %cr3, %eax
618 619 ret
619 620 SET_SIZE(getcr3)
620 621
621 622 #if !defined(__xpv)
622 623
623 624 ENTRY(setcr3)
624 625 movl 4(%esp), %eax
625 626 movl %eax, %cr3
626 627 ret
627 628 SET_SIZE(setcr3)
628 629
629 630 ENTRY(reload_cr3)
630 631 movl %cr3, %eax
631 632 movl %eax, %cr3
632 633 ret
633 634 SET_SIZE(reload_cr3)
634 635
635 636 #endif /* __xpv */
636 637
637 638 ENTRY(getcr4)
638 639 movl %cr4, %eax
639 640 ret
640 641 SET_SIZE(getcr4)
641 642
642 643 ENTRY(setcr4)
643 644 movl 4(%esp), %eax
644 645 movl %eax, %cr4
645 646 ret
646 647 SET_SIZE(setcr4)
647 648
648 649 #endif /* __i386 */
649 650 #endif /* __lint */
650 651
651 652 #if defined(__lint)
652 653
653 654 /*ARGSUSED*/
654 655 uint32_t
655 656 __cpuid_insn(struct cpuid_regs *regs)
656 657 { return (0); }
657 658
658 659 #else /* __lint */
659 660
660 661 #if defined(__amd64)
661 662
662 663 ENTRY(__cpuid_insn)
663 664 movq %rbx, %r8
664 665 movq %rcx, %r9
665 666 movq %rdx, %r11
666 667 movl (%rdi), %eax /* %eax = regs->cp_eax */
667 668 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
668 669 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
669 670 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
670 671 cpuid
671 672 movl %eax, (%rdi) /* regs->cp_eax = %eax */
672 673 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
673 674 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
674 675 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
675 676 movq %r8, %rbx
676 677 movq %r9, %rcx
677 678 movq %r11, %rdx
678 679 ret
679 680 SET_SIZE(__cpuid_insn)
680 681
681 682 #elif defined(__i386)
682 683
683 684 ENTRY(__cpuid_insn)
684 685 pushl %ebp
685 686 movl 0x8(%esp), %ebp /* %ebp = regs */
686 687 pushl %ebx
687 688 pushl %ecx
688 689 pushl %edx
689 690 movl (%ebp), %eax /* %eax = regs->cp_eax */
690 691 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
691 692 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
692 693 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
693 694 cpuid
694 695 movl %eax, (%ebp) /* regs->cp_eax = %eax */
695 696 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
696 697 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
697 698 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
698 699 popl %edx
699 700 popl %ecx
700 701 popl %ebx
701 702 popl %ebp
702 703 ret
703 704 SET_SIZE(__cpuid_insn)
704 705
705 706 #endif /* __i386 */
706 707 #endif /* __lint */
707 708
708 709 #if defined(__lint)
709 710
710 711 /*ARGSUSED*/
711 712 void
712 713 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
713 714 {}
714 715
715 716 #else /* __lint */
716 717
717 718 #if defined(__amd64)
718 719
719 720 ENTRY_NP(i86_monitor)
720 721 pushq %rbp
721 722 movq %rsp, %rbp
722 723 movq %rdi, %rax /* addr */
723 724 movq %rsi, %rcx /* extensions */
724 725 /* rdx contains input arg3: hints */
725 726 clflush (%rax)
726 727 .byte 0x0f, 0x01, 0xc8 /* monitor */
727 728 leave
728 729 ret
729 730 SET_SIZE(i86_monitor)
730 731
731 732 #elif defined(__i386)
732 733
733 734 ENTRY_NP(i86_monitor)
734 735 pushl %ebp
735 736 movl %esp, %ebp
736 737 movl 0x8(%ebp),%eax /* addr */
737 738 movl 0xc(%ebp),%ecx /* extensions */
738 739 movl 0x10(%ebp),%edx /* hints */
739 740 clflush (%eax)
740 741 .byte 0x0f, 0x01, 0xc8 /* monitor */
741 742 leave
742 743 ret
743 744 SET_SIZE(i86_monitor)
744 745
745 746 #endif /* __i386 */
746 747 #endif /* __lint */
747 748
748 749 #if defined(__lint)
749 750
750 751 /*ARGSUSED*/
751 752 void
752 753 i86_mwait(uint32_t data, uint32_t extensions)
753 754 {}
754 755
755 756 #else /* __lint */
756 757
757 758 #if defined(__amd64)
758 759
759 760 ENTRY_NP(i86_mwait)
760 761 pushq %rbp
761 762 movq %rsp, %rbp
762 763 movq %rdi, %rax /* data */
763 764 movq %rsi, %rcx /* extensions */
764 765 .byte 0x0f, 0x01, 0xc9 /* mwait */
765 766 leave
766 767 ret
767 768 SET_SIZE(i86_mwait)
768 769
769 770 #elif defined(__i386)
770 771
771 772 ENTRY_NP(i86_mwait)
772 773 pushl %ebp
773 774 movl %esp, %ebp
774 775 movl 0x8(%ebp),%eax /* data */
775 776 movl 0xc(%ebp),%ecx /* extensions */
776 777 .byte 0x0f, 0x01, 0xc9 /* mwait */
777 778 leave
778 779 ret
779 780 SET_SIZE(i86_mwait)
780 781
781 782 #endif /* __i386 */
782 783 #endif /* __lint */
783 784
784 785 #if defined(__xpv)
785 786 /*
786 787 * Defined in C
787 788 */
788 789 #else
789 790
790 791 #if defined(__lint)
791 792
792 793 hrtime_t
793 794 tsc_read(void)
794 795 {
795 796 return (0);
796 797 }
797 798
798 799 #else /* __lint */
799 800
800 801 #if defined(__amd64)
801 802
802 803 ENTRY_NP(tsc_read)
803 804 movq %rbx, %r11
804 805 movl $0, %eax
805 806 cpuid
806 807 rdtsc
807 808 movq %r11, %rbx
808 809 shlq $32, %rdx
809 810 orq %rdx, %rax
810 811 ret
811 812 .globl _tsc_mfence_start
812 813 _tsc_mfence_start:
813 814 mfence
814 815 rdtsc
815 816 shlq $32, %rdx
816 817 orq %rdx, %rax
817 818 ret
818 819 .globl _tsc_mfence_end
819 820 _tsc_mfence_end:
820 821 .globl _tscp_start
821 822 _tscp_start:
822 823 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
823 824 shlq $32, %rdx
824 825 orq %rdx, %rax
825 826 ret
826 827 .globl _tscp_end
827 828 _tscp_end:
828 829 .globl _no_rdtsc_start
829 830 _no_rdtsc_start:
830 831 xorl %edx, %edx
831 832 xorl %eax, %eax
832 833 ret
833 834 .globl _no_rdtsc_end
834 835 _no_rdtsc_end:
835 836 .globl _tsc_lfence_start
836 837 _tsc_lfence_start:
837 838 lfence
838 839 rdtsc
839 840 shlq $32, %rdx
840 841 orq %rdx, %rax
841 842 ret
842 843 .globl _tsc_lfence_end
843 844 _tsc_lfence_end:
844 845 SET_SIZE(tsc_read)
845 846
846 847 #else /* __i386 */
847 848
848 849 ENTRY_NP(tsc_read)
849 850 pushl %ebx
850 851 movl $0, %eax
851 852 cpuid
852 853 rdtsc
853 854 popl %ebx
854 855 ret
855 856 .globl _tsc_mfence_start
856 857 _tsc_mfence_start:
857 858 mfence
858 859 rdtsc
859 860 ret
860 861 .globl _tsc_mfence_end
861 862 _tsc_mfence_end:
862 863 .globl _tscp_start
863 864 _tscp_start:
864 865 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
865 866 ret
866 867 .globl _tscp_end
867 868 _tscp_end:
868 869 .globl _no_rdtsc_start
869 870 _no_rdtsc_start:
870 871 xorl %edx, %edx
871 872 xorl %eax, %eax
872 873 ret
873 874 .globl _no_rdtsc_end
874 875 _no_rdtsc_end:
875 876 .globl _tsc_lfence_start
876 877 _tsc_lfence_start:
877 878 lfence
878 879 rdtsc
879 880 ret
880 881 .globl _tsc_lfence_end
881 882 _tsc_lfence_end:
882 883 SET_SIZE(tsc_read)
883 884
884 885 #endif /* __i386 */
885 886
886 887 #endif /* __lint */
887 888
888 889
889 890 #endif /* __xpv */
890 891
891 892 #ifdef __lint
892 893 /*
893 894 * Do not use this function for obtaining clock tick. This
894 895 * is called by callers who do not need to have a guarenteed
895 896 * correct tick value. The proper routine to use is tsc_read().
896 897 */
897 898 u_longlong_t
898 899 randtick(void)
899 900 {
900 901 return (0);
901 902 }
902 903 #else
903 904 #if defined(__amd64)
904 905 ENTRY_NP(randtick)
905 906 rdtsc
906 907 shlq $32, %rdx
907 908 orq %rdx, %rax
908 909 ret
909 910 SET_SIZE(randtick)
910 911 #else
911 912 ENTRY_NP(randtick)
912 913 rdtsc
913 914 ret
914 915 SET_SIZE(randtick)
915 916 #endif /* __i386 */
916 917 #endif /* __lint */
917 918 /*
918 919 * Insert entryp after predp in a doubly linked list.
919 920 */
920 921
921 922 #if defined(__lint)
922 923
923 924 /*ARGSUSED*/
924 925 void
925 926 _insque(caddr_t entryp, caddr_t predp)
926 927 {}
927 928
928 929 #else /* __lint */
929 930
930 931 #if defined(__amd64)
931 932
932 933 ENTRY(_insque)
933 934 movq (%rsi), %rax /* predp->forw */
934 935 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
935 936 movq %rax, (%rdi) /* entryp->forw = predp->forw */
936 937 movq %rdi, (%rsi) /* predp->forw = entryp */
937 938 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
938 939 ret
939 940 SET_SIZE(_insque)
940 941
941 942 #elif defined(__i386)
942 943
943 944 ENTRY(_insque)
944 945 movl 8(%esp), %edx
945 946 movl 4(%esp), %ecx
946 947 movl (%edx), %eax /* predp->forw */
947 948 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
948 949 movl %eax, (%ecx) /* entryp->forw = predp->forw */
949 950 movl %ecx, (%edx) /* predp->forw = entryp */
950 951 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
951 952 ret
952 953 SET_SIZE(_insque)
953 954
954 955 #endif /* __i386 */
955 956 #endif /* __lint */
956 957
957 958 /*
958 959 * Remove entryp from a doubly linked list
959 960 */
960 961
961 962 #if defined(__lint)
962 963
963 964 /*ARGSUSED*/
964 965 void
965 966 _remque(caddr_t entryp)
966 967 {}
967 968
968 969 #else /* __lint */
969 970
970 971 #if defined(__amd64)
971 972
972 973 ENTRY(_remque)
973 974 movq (%rdi), %rax /* entry->forw */
974 975 movq CPTRSIZE(%rdi), %rdx /* entry->back */
975 976 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
976 977 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
977 978 ret
978 979 SET_SIZE(_remque)
979 980
980 981 #elif defined(__i386)
981 982
982 983 ENTRY(_remque)
983 984 movl 4(%esp), %ecx
984 985 movl (%ecx), %eax /* entry->forw */
985 986 movl CPTRSIZE(%ecx), %edx /* entry->back */
986 987 movl %eax, (%edx) /* entry->back->forw = entry->forw */
987 988 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
988 989 ret
989 990 SET_SIZE(_remque)
990 991
991 992 #endif /* __i386 */
992 993 #endif /* __lint */
993 994
994 995 /*
995 996 * Returns the number of
996 997 * non-NULL bytes in string argument.
997 998 */
998 999
999 1000 #if defined(__lint)
1000 1001
1001 1002 /* ARGSUSED */
1002 1003 size_t
1003 1004 strlen(const char *str)
1004 1005 { return (0); }
1005 1006
1006 1007 #else /* __lint */
1007 1008
1008 1009 #if defined(__amd64)
1009 1010
1010 1011 /*
1011 1012 * This is close to a simple transliteration of a C version of this
1012 1013 * routine. We should either just -make- this be a C version, or
1013 1014 * justify having it in assembler by making it significantly faster.
1014 1015 *
1015 1016 * size_t
1016 1017 * strlen(const char *s)
1017 1018 * {
1018 1019 * const char *s0;
1019 1020 * #if defined(DEBUG)
1020 1021 * if ((uintptr_t)s < KERNELBASE)
1021 1022 * panic(.str_panic_msg);
1022 1023 * #endif
1023 1024 * for (s0 = s; *s; s++)
1024 1025 * ;
1025 1026 * return (s - s0);
1026 1027 * }
1027 1028 */
1028 1029
1029 1030 ENTRY(strlen)
1030 1031 #ifdef DEBUG
1031 1032 movq postbootkernelbase(%rip), %rax
1032 1033 cmpq %rax, %rdi
1033 1034 jae str_valid
1034 1035 pushq %rbp
1035 1036 movq %rsp, %rbp
1036 1037 leaq .str_panic_msg(%rip), %rdi
1037 1038 xorl %eax, %eax
1038 1039 call panic
1039 1040 #endif /* DEBUG */
1040 1041 str_valid:
1041 1042 cmpb $0, (%rdi)
1042 1043 movq %rdi, %rax
1043 1044 je .null_found
1044 1045 .align 4
1045 1046 .strlen_loop:
1046 1047 incq %rdi
1047 1048 cmpb $0, (%rdi)
1048 1049 jne .strlen_loop
1049 1050 .null_found:
1050 1051 subq %rax, %rdi
1051 1052 movq %rdi, %rax
1052 1053 ret
1053 1054 SET_SIZE(strlen)
1054 1055
1055 1056 #elif defined(__i386)
1056 1057
1057 1058 ENTRY(strlen)
1058 1059 #ifdef DEBUG
1059 1060 movl postbootkernelbase, %eax
1060 1061 cmpl %eax, 4(%esp)
1061 1062 jae str_valid
1062 1063 pushl %ebp
1063 1064 movl %esp, %ebp
1064 1065 pushl $.str_panic_msg
1065 1066 call panic
1066 1067 #endif /* DEBUG */
1067 1068
1068 1069 str_valid:
1069 1070 movl 4(%esp), %eax /* %eax = string address */
1070 1071 testl $3, %eax /* if %eax not word aligned */
1071 1072 jnz .not_word_aligned /* goto .not_word_aligned */
1072 1073 .align 4
1073 1074 .word_aligned:
1074 1075 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1075 1076 movl $0x7f7f7f7f, %ecx
1076 1077 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1077 1078 addl $4, %eax /* next word */
1078 1079 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1079 1080 orl %edx, %ecx /* %ecx |= %edx */
1080 1081 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1081 1082 cmpl $0x80808080, %ecx /* if no null byte in this word */
1082 1083 je .word_aligned /* goto .word_aligned */
1083 1084 subl $4, %eax /* post-incremented */
1084 1085 .not_word_aligned:
1085 1086 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1086 1087 je .null_found /* goto .null_found */
1087 1088 incl %eax /* next byte */
1088 1089 testl $3, %eax /* if %eax not word aligned */
1089 1090 jnz .not_word_aligned /* goto .not_word_aligned */
1090 1091 jmp .word_aligned /* goto .word_aligned */
1091 1092 .align 4
1092 1093 .null_found:
1093 1094 subl 4(%esp), %eax /* %eax -= string address */
1094 1095 ret
1095 1096 SET_SIZE(strlen)
1096 1097
1097 1098 #endif /* __i386 */
1098 1099
1099 1100 #ifdef DEBUG
1100 1101 .text
1101 1102 .str_panic_msg:
1102 1103 .string "strlen: argument below kernelbase"
1103 1104 #endif /* DEBUG */
1104 1105
1105 1106 #endif /* __lint */
1106 1107
1107 1108 /*
1108 1109 * Berkeley 4.3 introduced symbolically named interrupt levels
1109 1110 * as a way deal with priority in a machine independent fashion.
1110 1111 * Numbered priorities are machine specific, and should be
1111 1112 * discouraged where possible.
1112 1113 *
1113 1114 * Note, for the machine specific priorities there are
1114 1115 * examples listed for devices that use a particular priority.
1115 1116 * It should not be construed that all devices of that
1116 1117 * type should be at that priority. It is currently were
1117 1118 * the current devices fit into the priority scheme based
1118 1119 * upon time criticalness.
1119 1120 *
1120 1121 * The underlying assumption of these assignments is that
1121 1122 * IPL 10 is the highest level from which a device
1122 1123 * routine can call wakeup. Devices that interrupt from higher
1123 1124 * levels are restricted in what they can do. If they need
1124 1125 * kernels services they should schedule a routine at a lower
1125 1126 * level (via software interrupt) to do the required
1126 1127 * processing.
1127 1128 *
1128 1129 * Examples of this higher usage:
1129 1130 * Level Usage
1130 1131 * 14 Profiling clock (and PROM uart polling clock)
1131 1132 * 12 Serial ports
1132 1133 *
1133 1134 * The serial ports request lower level processing on level 6.
1134 1135 *
1135 1136 * Also, almost all splN routines (where N is a number or a
1136 1137 * mnemonic) will do a RAISE(), on the assumption that they are
1137 1138 * never used to lower our priority.
1138 1139 * The exceptions are:
1139 1140 * spl8() Because you can't be above 15 to begin with!
1140 1141 * splzs() Because this is used at boot time to lower our
1141 1142 * priority, to allow the PROM to poll the uart.
1142 1143 * spl0() Used to lower priority to 0.
1143 1144 */
1144 1145
1145 1146 #if defined(__lint)
1146 1147
1147 1148 int spl0(void) { return (0); }
1148 1149 int spl6(void) { return (0); }
1149 1150 int spl7(void) { return (0); }
1150 1151 int spl8(void) { return (0); }
1151 1152 int splhigh(void) { return (0); }
1152 1153 int splhi(void) { return (0); }
1153 1154 int splzs(void) { return (0); }
1154 1155
1155 1156 /* ARGSUSED */
1156 1157 void
1157 1158 splx(int level)
1158 1159 {}
1159 1160
1160 1161 #else /* __lint */
1161 1162
1162 1163 #if defined(__amd64)
1163 1164
1164 1165 #define SETPRI(level) \
1165 1166 movl $/**/level, %edi; /* new priority */ \
1166 1167 jmp do_splx /* redirect to do_splx */
1167 1168
1168 1169 #define RAISE(level) \
1169 1170 movl $/**/level, %edi; /* new priority */ \
1170 1171 jmp splr /* redirect to splr */
1171 1172
1172 1173 #elif defined(__i386)
1173 1174
1174 1175 #define SETPRI(level) \
1175 1176 pushl $/**/level; /* new priority */ \
1176 1177 call do_splx; /* invoke common splx code */ \
1177 1178 addl $4, %esp; /* unstack arg */ \
1178 1179 ret
1179 1180
1180 1181 #define RAISE(level) \
1181 1182 pushl $/**/level; /* new priority */ \
1182 1183 call splr; /* invoke common splr code */ \
1183 1184 addl $4, %esp; /* unstack args */ \
1184 1185 ret
1185 1186
1186 1187 #endif /* __i386 */
1187 1188
1188 1189 /* locks out all interrupts, including memory errors */
1189 1190 ENTRY(spl8)
1190 1191 SETPRI(15)
1191 1192 SET_SIZE(spl8)
1192 1193
1193 1194 /* just below the level that profiling runs */
1194 1195 ENTRY(spl7)
1195 1196 RAISE(13)
1196 1197 SET_SIZE(spl7)
1197 1198
1198 1199 /* sun specific - highest priority onboard serial i/o asy ports */
1199 1200 ENTRY(splzs)
1200 1201 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1201 1202 SET_SIZE(splzs)
1202 1203
1203 1204 ENTRY(splhi)
1204 1205 ALTENTRY(splhigh)
1205 1206 ALTENTRY(spl6)
1206 1207 ALTENTRY(i_ddi_splhigh)
1207 1208
1208 1209 RAISE(DISP_LEVEL)
1209 1210
1210 1211 SET_SIZE(i_ddi_splhigh)
1211 1212 SET_SIZE(spl6)
1212 1213 SET_SIZE(splhigh)
1213 1214 SET_SIZE(splhi)
1214 1215
1215 1216 /* allow all interrupts */
1216 1217 ENTRY(spl0)
1217 1218 SETPRI(0)
1218 1219 SET_SIZE(spl0)
1219 1220
1220 1221
1221 1222 /* splx implementation */
1222 1223 ENTRY(splx)
1223 1224 jmp do_splx /* redirect to common splx code */
1224 1225 SET_SIZE(splx)
1225 1226
1226 1227 #endif /* __lint */
1227 1228
1228 1229 #if defined(__i386)
1229 1230
1230 1231 /*
1231 1232 * Read and write the %gs register
1232 1233 */
1233 1234
1234 1235 #if defined(__lint)
1235 1236
1236 1237 /*ARGSUSED*/
1237 1238 uint16_t
1238 1239 getgs(void)
1239 1240 { return (0); }
1240 1241
1241 1242 /*ARGSUSED*/
1242 1243 void
1243 1244 setgs(uint16_t sel)
1244 1245 {}
1245 1246
1246 1247 #else /* __lint */
1247 1248
1248 1249 ENTRY(getgs)
1249 1250 clr %eax
1250 1251 movw %gs, %ax
1251 1252 ret
1252 1253 SET_SIZE(getgs)
1253 1254
1254 1255 ENTRY(setgs)
1255 1256 movw 4(%esp), %gs
1256 1257 ret
1257 1258 SET_SIZE(setgs)
1258 1259
1259 1260 #endif /* __lint */
1260 1261 #endif /* __i386 */
1261 1262
1262 1263 #if defined(__lint)
1263 1264
1264 1265 void
1265 1266 pc_reset(void)
1266 1267 {}
1267 1268
1268 1269 void
1269 1270 efi_reset(void)
1270 1271 {}
1271 1272
1272 1273 #else /* __lint */
1273 1274
1274 1275 ENTRY(wait_500ms)
1275 1276 #if defined(__amd64)
1276 1277 pushq %rbx
1277 1278 #elif defined(__i386)
1278 1279 push %ebx
1279 1280 #endif
1280 1281 movl $50000, %ebx
1281 1282 1:
1282 1283 call tenmicrosec
1283 1284 decl %ebx
1284 1285 jnz 1b
1285 1286 #if defined(__amd64)
1286 1287 popq %rbx
1287 1288 #elif defined(__i386)
1288 1289 pop %ebx
1289 1290 #endif
1290 1291 ret
1291 1292 SET_SIZE(wait_500ms)
1292 1293
1293 1294 #define RESET_METHOD_KBC 1
1294 1295 #define RESET_METHOD_PORT92 2
1295 1296 #define RESET_METHOD_PCI 4
1296 1297
1297 1298 DGDEF3(pc_reset_methods, 4, 8)
1298 1299 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1299 1300
1300 1301 ENTRY(pc_reset)
1301 1302
1302 1303 #if defined(__i386)
1303 1304 testl $RESET_METHOD_KBC, pc_reset_methods
1304 1305 #elif defined(__amd64)
1305 1306 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1306 1307 #endif
1307 1308 jz 1f
1308 1309
1309 1310 /
1310 1311 / Try the classic keyboard controller-triggered reset.
1311 1312 /
1312 1313 movw $0x64, %dx
1313 1314 movb $0xfe, %al
1314 1315 outb (%dx)
1315 1316
1316 1317 / Wait up to 500 milliseconds here for the keyboard controller
1317 1318 / to pull the reset line. On some systems where the keyboard
1318 1319 / controller is slow to pull the reset line, the next reset method
1319 1320 / may be executed (which may be bad if those systems hang when the
1320 1321 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1321 1322 / and Ferrari 4000 (doesn't like the cf9 reset method))
1322 1323
1323 1324 call wait_500ms
1324 1325
1325 1326 1:
1326 1327 #if defined(__i386)
1327 1328 testl $RESET_METHOD_PORT92, pc_reset_methods
1328 1329 #elif defined(__amd64)
1329 1330 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1330 1331 #endif
1331 1332 jz 3f
1332 1333
1333 1334 /
1334 1335 / Try port 0x92 fast reset
1335 1336 /
1336 1337 movw $0x92, %dx
1337 1338 inb (%dx)
1338 1339 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1339 1340 je 1f
1340 1341 testb $1, %al / If bit 0
1341 1342 jz 2f / is clear, jump to perform the reset
1342 1343 andb $0xfe, %al / otherwise,
1343 1344 outb (%dx) / clear bit 0 first, then
1344 1345 2:
1345 1346 orb $1, %al / Set bit 0
1346 1347 outb (%dx) / and reset the system
1347 1348 1:
1348 1349
1349 1350 call wait_500ms
1350 1351
1351 1352 3:
1352 1353 #if defined(__i386)
1353 1354 testl $RESET_METHOD_PCI, pc_reset_methods
1354 1355 #elif defined(__amd64)
1355 1356 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1356 1357 #endif
1357 1358 jz 4f
1358 1359
1359 1360 / Try the PCI (soft) reset vector (should work on all modern systems,
1360 1361 / but has been shown to cause problems on 450NX systems, and some newer
1361 1362 / systems (e.g. ATI IXP400-equipped systems))
1362 1363 / When resetting via this method, 2 writes are required. The first
1363 1364 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1364 1365 / power cycle).
1365 1366 / The reset occurs on the second write, during bit 2's transition from
1366 1367 / 0->1.
1367 1368 movw $0xcf9, %dx
1368 1369 movb $0x2, %al / Reset mode = hard, no power cycle
1369 1370 outb (%dx)
1370 1371 movb $0x6, %al
1371 1372 outb (%dx)
1372 1373
1373 1374 call wait_500ms
1374 1375
1375 1376 4:
1376 1377 /
1377 1378 / port 0xcf9 failed also. Last-ditch effort is to
1378 1379 / triple-fault the CPU.
1379 1380 / Also, use triple fault for EFI firmware
1380 1381 /
1381 1382 ENTRY(efi_reset)
1382 1383 #if defined(__amd64)
1383 1384 pushq $0x0
1384 1385 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1385 1386 lidt (%rsp)
1386 1387 #elif defined(__i386)
1387 1388 pushl $0x0
1388 1389 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1389 1390 lidt (%esp)
1390 1391 #endif
1391 1392 int $0x0 / Trigger interrupt, generate triple-fault
1392 1393
1393 1394 cli
1394 1395 hlt / Wait forever
1395 1396 /*NOTREACHED*/
1396 1397 SET_SIZE(efi_reset)
1397 1398 SET_SIZE(pc_reset)
1398 1399
1399 1400 #endif /* __lint */
1400 1401
1401 1402 /*
1402 1403 * C callable in and out routines
1403 1404 */
1404 1405
1405 1406 #if defined(__lint)
1406 1407
1407 1408 /* ARGSUSED */
1408 1409 void
1409 1410 outl(int port_address, uint32_t val)
1410 1411 {}
1411 1412
1412 1413 #else /* __lint */
1413 1414
1414 1415 #if defined(__amd64)
1415 1416
1416 1417 ENTRY(outl)
1417 1418 movw %di, %dx
1418 1419 movl %esi, %eax
1419 1420 outl (%dx)
1420 1421 ret
1421 1422 SET_SIZE(outl)
1422 1423
1423 1424 #elif defined(__i386)
1424 1425
1425 1426 .set PORT, 4
1426 1427 .set VAL, 8
1427 1428
1428 1429 ENTRY(outl)
1429 1430 movw PORT(%esp), %dx
1430 1431 movl VAL(%esp), %eax
1431 1432 outl (%dx)
1432 1433 ret
1433 1434 SET_SIZE(outl)
1434 1435
1435 1436 #endif /* __i386 */
1436 1437 #endif /* __lint */
1437 1438
1438 1439 #if defined(__lint)
1439 1440
1440 1441 /* ARGSUSED */
1441 1442 void
1442 1443 outw(int port_address, uint16_t val)
1443 1444 {}
1444 1445
1445 1446 #else /* __lint */
1446 1447
1447 1448 #if defined(__amd64)
1448 1449
1449 1450 ENTRY(outw)
1450 1451 movw %di, %dx
1451 1452 movw %si, %ax
1452 1453 D16 outl (%dx) /* XX64 why not outw? */
1453 1454 ret
1454 1455 SET_SIZE(outw)
1455 1456
1456 1457 #elif defined(__i386)
1457 1458
1458 1459 ENTRY(outw)
1459 1460 movw PORT(%esp), %dx
1460 1461 movw VAL(%esp), %ax
1461 1462 D16 outl (%dx)
1462 1463 ret
1463 1464 SET_SIZE(outw)
1464 1465
1465 1466 #endif /* __i386 */
1466 1467 #endif /* __lint */
1467 1468
1468 1469 #if defined(__lint)
1469 1470
1470 1471 /* ARGSUSED */
1471 1472 void
1472 1473 outb(int port_address, uint8_t val)
1473 1474 {}
1474 1475
1475 1476 #else /* __lint */
1476 1477
1477 1478 #if defined(__amd64)
1478 1479
1479 1480 ENTRY(outb)
1480 1481 movw %di, %dx
1481 1482 movb %sil, %al
1482 1483 outb (%dx)
1483 1484 ret
1484 1485 SET_SIZE(outb)
1485 1486
1486 1487 #elif defined(__i386)
1487 1488
1488 1489 ENTRY(outb)
1489 1490 movw PORT(%esp), %dx
1490 1491 movb VAL(%esp), %al
1491 1492 outb (%dx)
1492 1493 ret
1493 1494 SET_SIZE(outb)
1494 1495
1495 1496 #endif /* __i386 */
1496 1497 #endif /* __lint */
1497 1498
1498 1499 #if defined(__lint)
1499 1500
1500 1501 /* ARGSUSED */
1501 1502 uint32_t
1502 1503 inl(int port_address)
1503 1504 { return (0); }
1504 1505
1505 1506 #else /* __lint */
1506 1507
1507 1508 #if defined(__amd64)
1508 1509
1509 1510 ENTRY(inl)
1510 1511 xorl %eax, %eax
1511 1512 movw %di, %dx
1512 1513 inl (%dx)
1513 1514 ret
1514 1515 SET_SIZE(inl)
1515 1516
1516 1517 #elif defined(__i386)
1517 1518
1518 1519 ENTRY(inl)
1519 1520 movw PORT(%esp), %dx
1520 1521 inl (%dx)
1521 1522 ret
1522 1523 SET_SIZE(inl)
1523 1524
1524 1525 #endif /* __i386 */
1525 1526 #endif /* __lint */
1526 1527
1527 1528 #if defined(__lint)
1528 1529
1529 1530 /* ARGSUSED */
1530 1531 uint16_t
1531 1532 inw(int port_address)
1532 1533 { return (0); }
1533 1534
1534 1535 #else /* __lint */
1535 1536
1536 1537 #if defined(__amd64)
1537 1538
1538 1539 ENTRY(inw)
1539 1540 xorl %eax, %eax
1540 1541 movw %di, %dx
1541 1542 D16 inl (%dx)
1542 1543 ret
1543 1544 SET_SIZE(inw)
1544 1545
1545 1546 #elif defined(__i386)
1546 1547
1547 1548 ENTRY(inw)
1548 1549 subl %eax, %eax
1549 1550 movw PORT(%esp), %dx
1550 1551 D16 inl (%dx)
1551 1552 ret
1552 1553 SET_SIZE(inw)
1553 1554
1554 1555 #endif /* __i386 */
1555 1556 #endif /* __lint */
1556 1557
1557 1558
1558 1559 #if defined(__lint)
1559 1560
1560 1561 /* ARGSUSED */
1561 1562 uint8_t
1562 1563 inb(int port_address)
1563 1564 { return (0); }
1564 1565
1565 1566 #else /* __lint */
1566 1567
1567 1568 #if defined(__amd64)
1568 1569
1569 1570 ENTRY(inb)
1570 1571 xorl %eax, %eax
1571 1572 movw %di, %dx
1572 1573 inb (%dx)
1573 1574 ret
1574 1575 SET_SIZE(inb)
1575 1576
1576 1577 #elif defined(__i386)
1577 1578
1578 1579 ENTRY(inb)
1579 1580 subl %eax, %eax
1580 1581 movw PORT(%esp), %dx
1581 1582 inb (%dx)
1582 1583 ret
1583 1584 SET_SIZE(inb)
1584 1585
1585 1586 #endif /* __i386 */
1586 1587 #endif /* __lint */
1587 1588
1588 1589
1589 1590 #if defined(__lint)
1590 1591
1591 1592 /* ARGSUSED */
1592 1593 void
1593 1594 repoutsw(int port, uint16_t *addr, int cnt)
1594 1595 {}
1595 1596
1596 1597 #else /* __lint */
1597 1598
1598 1599 #if defined(__amd64)
1599 1600
1600 1601 ENTRY(repoutsw)
1601 1602 movl %edx, %ecx
1602 1603 movw %di, %dx
1603 1604 rep
1604 1605 D16 outsl
1605 1606 ret
1606 1607 SET_SIZE(repoutsw)
1607 1608
1608 1609 #elif defined(__i386)
1609 1610
1610 1611 /*
1611 1612 * The arguments and saved registers are on the stack in the
1612 1613 * following order:
1613 1614 * | cnt | +16
1614 1615 * | *addr | +12
1615 1616 * | port | +8
1616 1617 * | eip | +4
1617 1618 * | esi | <-- %esp
1618 1619 * If additional values are pushed onto the stack, make sure
1619 1620 * to adjust the following constants accordingly.
1620 1621 */
1621 1622 .set PORT, 8
1622 1623 .set ADDR, 12
1623 1624 .set COUNT, 16
1624 1625
1625 1626 ENTRY(repoutsw)
1626 1627 pushl %esi
1627 1628 movl PORT(%esp), %edx
1628 1629 movl ADDR(%esp), %esi
1629 1630 movl COUNT(%esp), %ecx
1630 1631 rep
1631 1632 D16 outsl
1632 1633 popl %esi
1633 1634 ret
1634 1635 SET_SIZE(repoutsw)
1635 1636
1636 1637 #endif /* __i386 */
1637 1638 #endif /* __lint */
1638 1639
1639 1640
1640 1641 #if defined(__lint)
1641 1642
1642 1643 /* ARGSUSED */
1643 1644 void
1644 1645 repinsw(int port_addr, uint16_t *addr, int cnt)
1645 1646 {}
1646 1647
1647 1648 #else /* __lint */
1648 1649
1649 1650 #if defined(__amd64)
1650 1651
1651 1652 ENTRY(repinsw)
1652 1653 movl %edx, %ecx
1653 1654 movw %di, %dx
1654 1655 rep
1655 1656 D16 insl
1656 1657 ret
1657 1658 SET_SIZE(repinsw)
1658 1659
1659 1660 #elif defined(__i386)
1660 1661
1661 1662 ENTRY(repinsw)
1662 1663 pushl %edi
1663 1664 movl PORT(%esp), %edx
1664 1665 movl ADDR(%esp), %edi
1665 1666 movl COUNT(%esp), %ecx
1666 1667 rep
1667 1668 D16 insl
1668 1669 popl %edi
1669 1670 ret
1670 1671 SET_SIZE(repinsw)
1671 1672
1672 1673 #endif /* __i386 */
1673 1674 #endif /* __lint */
1674 1675
1675 1676
1676 1677 #if defined(__lint)
1677 1678
1678 1679 /* ARGSUSED */
1679 1680 void
1680 1681 repinsb(int port, uint8_t *addr, int count)
1681 1682 {}
1682 1683
1683 1684 #else /* __lint */
1684 1685
1685 1686 #if defined(__amd64)
1686 1687
1687 1688 ENTRY(repinsb)
1688 1689 movl %edx, %ecx
1689 1690 movw %di, %dx
1690 1691 movq %rsi, %rdi
1691 1692 rep
1692 1693 insb
1693 1694 ret
1694 1695 SET_SIZE(repinsb)
1695 1696
1696 1697 #elif defined(__i386)
1697 1698
1698 1699 /*
1699 1700 * The arguments and saved registers are on the stack in the
1700 1701 * following order:
1701 1702 * | cnt | +16
1702 1703 * | *addr | +12
1703 1704 * | port | +8
1704 1705 * | eip | +4
1705 1706 * | esi | <-- %esp
1706 1707 * If additional values are pushed onto the stack, make sure
1707 1708 * to adjust the following constants accordingly.
1708 1709 */
1709 1710 .set IO_PORT, 8
1710 1711 .set IO_ADDR, 12
1711 1712 .set IO_COUNT, 16
1712 1713
1713 1714 ENTRY(repinsb)
1714 1715 pushl %edi
1715 1716 movl IO_ADDR(%esp), %edi
1716 1717 movl IO_COUNT(%esp), %ecx
1717 1718 movl IO_PORT(%esp), %edx
1718 1719 rep
1719 1720 insb
1720 1721 popl %edi
1721 1722 ret
1722 1723 SET_SIZE(repinsb)
1723 1724
1724 1725 #endif /* __i386 */
1725 1726 #endif /* __lint */
1726 1727
1727 1728
1728 1729 /*
1729 1730 * Input a stream of 32-bit words.
1730 1731 * NOTE: count is a DWORD count.
1731 1732 */
1732 1733 #if defined(__lint)
1733 1734
1734 1735 /* ARGSUSED */
1735 1736 void
1736 1737 repinsd(int port, uint32_t *addr, int count)
1737 1738 {}
1738 1739
1739 1740 #else /* __lint */
1740 1741
1741 1742 #if defined(__amd64)
1742 1743
1743 1744 ENTRY(repinsd)
1744 1745 movl %edx, %ecx
1745 1746 movw %di, %dx
1746 1747 movq %rsi, %rdi
1747 1748 rep
1748 1749 insl
1749 1750 ret
1750 1751 SET_SIZE(repinsd)
1751 1752
1752 1753 #elif defined(__i386)
1753 1754
1754 1755 ENTRY(repinsd)
1755 1756 pushl %edi
1756 1757 movl IO_ADDR(%esp), %edi
1757 1758 movl IO_COUNT(%esp), %ecx
1758 1759 movl IO_PORT(%esp), %edx
1759 1760 rep
1760 1761 insl
1761 1762 popl %edi
1762 1763 ret
1763 1764 SET_SIZE(repinsd)
1764 1765
1765 1766 #endif /* __i386 */
1766 1767 #endif /* __lint */
1767 1768
1768 1769 /*
1769 1770 * Output a stream of bytes
1770 1771 * NOTE: count is a byte count
1771 1772 */
1772 1773 #if defined(__lint)
1773 1774
1774 1775 /* ARGSUSED */
1775 1776 void
1776 1777 repoutsb(int port, uint8_t *addr, int count)
1777 1778 {}
1778 1779
1779 1780 #else /* __lint */
1780 1781
1781 1782 #if defined(__amd64)
1782 1783
1783 1784 ENTRY(repoutsb)
1784 1785 movl %edx, %ecx
1785 1786 movw %di, %dx
1786 1787 rep
1787 1788 outsb
1788 1789 ret
1789 1790 SET_SIZE(repoutsb)
1790 1791
1791 1792 #elif defined(__i386)
1792 1793
1793 1794 ENTRY(repoutsb)
1794 1795 pushl %esi
1795 1796 movl IO_ADDR(%esp), %esi
1796 1797 movl IO_COUNT(%esp), %ecx
1797 1798 movl IO_PORT(%esp), %edx
1798 1799 rep
1799 1800 outsb
1800 1801 popl %esi
1801 1802 ret
1802 1803 SET_SIZE(repoutsb)
1803 1804
1804 1805 #endif /* __i386 */
1805 1806 #endif /* __lint */
1806 1807
1807 1808 /*
1808 1809 * Output a stream of 32-bit words
1809 1810 * NOTE: count is a DWORD count
1810 1811 */
1811 1812 #if defined(__lint)
1812 1813
1813 1814 /* ARGSUSED */
1814 1815 void
1815 1816 repoutsd(int port, uint32_t *addr, int count)
1816 1817 {}
1817 1818
1818 1819 #else /* __lint */
1819 1820
1820 1821 #if defined(__amd64)
1821 1822
1822 1823 ENTRY(repoutsd)
1823 1824 movl %edx, %ecx
1824 1825 movw %di, %dx
1825 1826 rep
1826 1827 outsl
1827 1828 ret
1828 1829 SET_SIZE(repoutsd)
1829 1830
1830 1831 #elif defined(__i386)
1831 1832
1832 1833 ENTRY(repoutsd)
1833 1834 pushl %esi
1834 1835 movl IO_ADDR(%esp), %esi
1835 1836 movl IO_COUNT(%esp), %ecx
1836 1837 movl IO_PORT(%esp), %edx
1837 1838 rep
1838 1839 outsl
1839 1840 popl %esi
1840 1841 ret
1841 1842 SET_SIZE(repoutsd)
1842 1843
1843 1844 #endif /* __i386 */
1844 1845 #endif /* __lint */
1845 1846
1846 1847 /*
1847 1848 * void int3(void)
1848 1849 * void int18(void)
1849 1850 * void int20(void)
1850 1851 * void int_cmci(void)
1851 1852 */
1852 1853
1853 1854 #if defined(__lint)
1854 1855
1855 1856 void
1856 1857 int3(void)
1857 1858 {}
1858 1859
1859 1860 void
1860 1861 int18(void)
1861 1862 {}
1862 1863
1863 1864 void
1864 1865 int20(void)
1865 1866 {}
1866 1867
1867 1868 void
1868 1869 int_cmci(void)
1869 1870 {}
1870 1871
1871 1872 #else /* __lint */
1872 1873
1873 1874 ENTRY(int3)
1874 1875 int $T_BPTFLT
1875 1876 ret
1876 1877 SET_SIZE(int3)
1877 1878
1878 1879 ENTRY(int18)
1879 1880 int $T_MCE
1880 1881 ret
1881 1882 SET_SIZE(int18)
1882 1883
1883 1884 ENTRY(int20)
1884 1885 movl boothowto, %eax
1885 1886 andl $RB_DEBUG, %eax
1886 1887 jz 1f
1887 1888
1888 1889 int $T_DBGENTR
1889 1890 1:
1890 1891 rep; ret /* use 2 byte return instruction when branch target */
1891 1892 /* AMD Software Optimization Guide - Section 6.2 */
1892 1893 SET_SIZE(int20)
1893 1894
1894 1895 ENTRY(int_cmci)
1895 1896 int $T_ENOEXTFLT
1896 1897 ret
1897 1898 SET_SIZE(int_cmci)
1898 1899
1899 1900 #endif /* __lint */
1900 1901
1901 1902 #if defined(__lint)
1902 1903
1903 1904 /* ARGSUSED */
1904 1905 int
1905 1906 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1906 1907 { return (0); }
1907 1908
1908 1909 #else /* __lint */
1909 1910
1910 1911 #if defined(__amd64)
1911 1912
1912 1913 ENTRY(scanc)
1913 1914 /* rdi == size */
1914 1915 /* rsi == cp */
1915 1916 /* rdx == table */
1916 1917 /* rcx == mask */
1917 1918 addq %rsi, %rdi /* end = &cp[size] */
1918 1919 .scanloop:
1919 1920 cmpq %rdi, %rsi /* while (cp < end */
1920 1921 jnb .scandone
1921 1922 movzbq (%rsi), %r8 /* %r8 = *cp */
1922 1923 incq %rsi /* cp++ */
1923 1924 testb %cl, (%r8, %rdx)
1924 1925 jz .scanloop /* && (table[*cp] & mask) == 0) */
1925 1926 decq %rsi /* (fix post-increment) */
1926 1927 .scandone:
1927 1928 movl %edi, %eax
1928 1929 subl %esi, %eax /* return (end - cp) */
1929 1930 ret
1930 1931 SET_SIZE(scanc)
1931 1932
1932 1933 #elif defined(__i386)
1933 1934
1934 1935 ENTRY(scanc)
1935 1936 pushl %edi
1936 1937 pushl %esi
1937 1938 movb 24(%esp), %cl /* mask = %cl */
1938 1939 movl 16(%esp), %esi /* cp = %esi */
1939 1940 movl 20(%esp), %edx /* table = %edx */
1940 1941 movl %esi, %edi
1941 1942 addl 12(%esp), %edi /* end = &cp[size]; */
1942 1943 .scanloop:
1943 1944 cmpl %edi, %esi /* while (cp < end */
1944 1945 jnb .scandone
1945 1946 movzbl (%esi), %eax /* %al = *cp */
1946 1947 incl %esi /* cp++ */
1947 1948 movb (%edx, %eax), %al /* %al = table[*cp] */
1948 1949 testb %al, %cl
1949 1950 jz .scanloop /* && (table[*cp] & mask) == 0) */
1950 1951 dec %esi /* post-incremented */
1951 1952 .scandone:
1952 1953 movl %edi, %eax
1953 1954 subl %esi, %eax /* return (end - cp) */
1954 1955 popl %esi
1955 1956 popl %edi
1956 1957 ret
1957 1958 SET_SIZE(scanc)
1958 1959
1959 1960 #endif /* __i386 */
1960 1961 #endif /* __lint */
1961 1962
1962 1963 /*
1963 1964 * Replacement functions for ones that are normally inlined.
1964 1965 * In addition to the copy in i86.il, they are defined here just in case.
1965 1966 */
1966 1967
1967 1968 #if defined(__lint)
1968 1969
1969 1970 ulong_t
1970 1971 intr_clear(void)
1971 1972 { return (0); }
1972 1973
1973 1974 ulong_t
1974 1975 clear_int_flag(void)
1975 1976 { return (0); }
1976 1977
1977 1978 #else /* __lint */
1978 1979
1979 1980 #if defined(__amd64)
1980 1981
1981 1982 ENTRY(intr_clear)
1982 1983 ENTRY(clear_int_flag)
1983 1984 pushfq
1984 1985 popq %rax
1985 1986 #if defined(__xpv)
1986 1987 leaq xpv_panicking, %rdi
1987 1988 movl (%rdi), %edi
1988 1989 cmpl $0, %edi
1989 1990 jne 2f
1990 1991 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1991 1992 /*
1992 1993 * Synthesize the PS_IE bit from the event mask bit
1993 1994 */
1994 1995 andq $_BITNOT(PS_IE), %rax
1995 1996 testb $1, %dl
1996 1997 jnz 1f
1997 1998 orq $PS_IE, %rax
1998 1999 1:
1999 2000 ret
2000 2001 2:
2001 2002 #endif
2002 2003 CLI(%rdi)
2003 2004 ret
2004 2005 SET_SIZE(clear_int_flag)
2005 2006 SET_SIZE(intr_clear)
2006 2007
2007 2008 #elif defined(__i386)
2008 2009
2009 2010 ENTRY(intr_clear)
2010 2011 ENTRY(clear_int_flag)
2011 2012 pushfl
2012 2013 popl %eax
2013 2014 #if defined(__xpv)
2014 2015 leal xpv_panicking, %edx
2015 2016 movl (%edx), %edx
2016 2017 cmpl $0, %edx
2017 2018 jne 2f
2018 2019 CLIRET(%edx, %cl) /* returns event mask in %cl */
2019 2020 /*
2020 2021 * Synthesize the PS_IE bit from the event mask bit
2021 2022 */
2022 2023 andl $_BITNOT(PS_IE), %eax
2023 2024 testb $1, %cl
2024 2025 jnz 1f
2025 2026 orl $PS_IE, %eax
2026 2027 1:
2027 2028 ret
2028 2029 2:
2029 2030 #endif
2030 2031 CLI(%edx)
2031 2032 ret
2032 2033 SET_SIZE(clear_int_flag)
2033 2034 SET_SIZE(intr_clear)
2034 2035
2035 2036 #endif /* __i386 */
2036 2037 #endif /* __lint */
2037 2038
2038 2039 #if defined(__lint)
2039 2040
2040 2041 struct cpu *
2041 2042 curcpup(void)
2042 2043 { return 0; }
2043 2044
2044 2045 #else /* __lint */
2045 2046
2046 2047 #if defined(__amd64)
2047 2048
2048 2049 ENTRY(curcpup)
2049 2050 movq %gs:CPU_SELF, %rax
2050 2051 ret
2051 2052 SET_SIZE(curcpup)
2052 2053
2053 2054 #elif defined(__i386)
2054 2055
2055 2056 ENTRY(curcpup)
2056 2057 movl %gs:CPU_SELF, %eax
2057 2058 ret
2058 2059 SET_SIZE(curcpup)
2059 2060
2060 2061 #endif /* __i386 */
2061 2062 #endif /* __lint */
2062 2063
2063 2064 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2064 2065 * These functions reverse the byte order of the input parameter and returns
2065 2066 * the result. This is to convert the byte order from host byte order
2066 2067 * (little endian) to network byte order (big endian), or vice versa.
2067 2068 */
2068 2069
2069 2070 #if defined(__lint)
2070 2071
2071 2072 uint64_t
2072 2073 htonll(uint64_t i)
2073 2074 { return (i); }
2074 2075
2075 2076 uint64_t
2076 2077 ntohll(uint64_t i)
2077 2078 { return (i); }
2078 2079
2079 2080 uint32_t
2080 2081 htonl(uint32_t i)
2081 2082 { return (i); }
2082 2083
2083 2084 uint32_t
2084 2085 ntohl(uint32_t i)
2085 2086 { return (i); }
2086 2087
2087 2088 uint16_t
2088 2089 htons(uint16_t i)
2089 2090 { return (i); }
2090 2091
2091 2092 uint16_t
2092 2093 ntohs(uint16_t i)
2093 2094 { return (i); }
2094 2095
2095 2096 #else /* __lint */
2096 2097
2097 2098 #if defined(__amd64)
2098 2099
2099 2100 ENTRY(htonll)
2100 2101 ALTENTRY(ntohll)
2101 2102 movq %rdi, %rax
2102 2103 bswapq %rax
2103 2104 ret
2104 2105 SET_SIZE(ntohll)
2105 2106 SET_SIZE(htonll)
2106 2107
2107 2108 /* XX64 there must be shorter sequences for this */
2108 2109 ENTRY(htonl)
2109 2110 ALTENTRY(ntohl)
2110 2111 movl %edi, %eax
2111 2112 bswap %eax
2112 2113 ret
2113 2114 SET_SIZE(ntohl)
2114 2115 SET_SIZE(htonl)
2115 2116
2116 2117 /* XX64 there must be better sequences for this */
2117 2118 ENTRY(htons)
2118 2119 ALTENTRY(ntohs)
2119 2120 movl %edi, %eax
2120 2121 bswap %eax
2121 2122 shrl $16, %eax
2122 2123 ret
2123 2124 SET_SIZE(ntohs)
2124 2125 SET_SIZE(htons)
2125 2126
2126 2127 #elif defined(__i386)
2127 2128
2128 2129 ENTRY(htonll)
2129 2130 ALTENTRY(ntohll)
2130 2131 movl 4(%esp), %edx
2131 2132 movl 8(%esp), %eax
2132 2133 bswap %edx
2133 2134 bswap %eax
2134 2135 ret
2135 2136 SET_SIZE(ntohll)
2136 2137 SET_SIZE(htonll)
2137 2138
2138 2139 ENTRY(htonl)
2139 2140 ALTENTRY(ntohl)
2140 2141 movl 4(%esp), %eax
2141 2142 bswap %eax
2142 2143 ret
2143 2144 SET_SIZE(ntohl)
2144 2145 SET_SIZE(htonl)
2145 2146
2146 2147 ENTRY(htons)
2147 2148 ALTENTRY(ntohs)
2148 2149 movl 4(%esp), %eax
2149 2150 bswap %eax
2150 2151 shrl $16, %eax
2151 2152 ret
2152 2153 SET_SIZE(ntohs)
2153 2154 SET_SIZE(htons)
2154 2155
2155 2156 #endif /* __i386 */
2156 2157 #endif /* __lint */
2157 2158
2158 2159
2159 2160 #if defined(__lint)
2160 2161
2161 2162 /* ARGSUSED */
2162 2163 void
2163 2164 intr_restore(ulong_t i)
2164 2165 { return; }
2165 2166
2166 2167 /* ARGSUSED */
2167 2168 void
2168 2169 restore_int_flag(ulong_t i)
2169 2170 { return; }
2170 2171
2171 2172 #else /* __lint */
2172 2173
2173 2174 #if defined(__amd64)
2174 2175
2175 2176 ENTRY(intr_restore)
2176 2177 ENTRY(restore_int_flag)
2177 2178 testq $PS_IE, %rdi
2178 2179 jz 1f
2179 2180 #if defined(__xpv)
2180 2181 leaq xpv_panicking, %rsi
2181 2182 movl (%rsi), %esi
2182 2183 cmpl $0, %esi
2183 2184 jne 1f
2184 2185 /*
2185 2186 * Since we're -really- running unprivileged, our attempt
2186 2187 * to change the state of the IF bit will be ignored.
2187 2188 * The virtual IF bit is tweaked by CLI and STI.
2188 2189 */
2189 2190 IE_TO_EVENT_MASK(%rsi, %rdi)
2190 2191 #else
2191 2192 sti
2192 2193 #endif
2193 2194 1:
2194 2195 ret
2195 2196 SET_SIZE(restore_int_flag)
2196 2197 SET_SIZE(intr_restore)
2197 2198
2198 2199 #elif defined(__i386)
2199 2200
2200 2201 ENTRY(intr_restore)
2201 2202 ENTRY(restore_int_flag)
2202 2203 testl $PS_IE, 4(%esp)
2203 2204 jz 1f
2204 2205 #if defined(__xpv)
2205 2206 leal xpv_panicking, %edx
2206 2207 movl (%edx), %edx
2207 2208 cmpl $0, %edx
2208 2209 jne 1f
2209 2210 /*
2210 2211 * Since we're -really- running unprivileged, our attempt
2211 2212 * to change the state of the IF bit will be ignored.
2212 2213 * The virtual IF bit is tweaked by CLI and STI.
2213 2214 */
2214 2215 IE_TO_EVENT_MASK(%edx, 4(%esp))
2215 2216 #else
2216 2217 sti
2217 2218 #endif
2218 2219 1:
2219 2220 ret
2220 2221 SET_SIZE(restore_int_flag)
2221 2222 SET_SIZE(intr_restore)
2222 2223
2223 2224 #endif /* __i386 */
2224 2225 #endif /* __lint */
2225 2226
2226 2227 #if defined(__lint)
2227 2228
2228 2229 void
2229 2230 sti(void)
2230 2231 {}
2231 2232
2232 2233 void
2233 2234 cli(void)
2234 2235 {}
2235 2236
2236 2237 #else /* __lint */
2237 2238
2238 2239 ENTRY(sti)
2239 2240 STI
2240 2241 ret
2241 2242 SET_SIZE(sti)
2242 2243
2243 2244 ENTRY(cli)
2244 2245 #if defined(__amd64)
2245 2246 CLI(%rax)
2246 2247 #elif defined(__i386)
2247 2248 CLI(%eax)
2248 2249 #endif /* __i386 */
2249 2250 ret
2250 2251 SET_SIZE(cli)
2251 2252
2252 2253 #endif /* __lint */
2253 2254
2254 2255 #if defined(__lint)
2255 2256
2256 2257 dtrace_icookie_t
2257 2258 dtrace_interrupt_disable(void)
2258 2259 { return (0); }
2259 2260
2260 2261 #else /* __lint */
2261 2262
2262 2263 #if defined(__amd64)
2263 2264
2264 2265 ENTRY(dtrace_interrupt_disable)
2265 2266 pushfq
2266 2267 popq %rax
2267 2268 #if defined(__xpv)
2268 2269 leaq xpv_panicking, %rdi
2269 2270 movl (%rdi), %edi
2270 2271 cmpl $0, %edi
2271 2272 jne .dtrace_interrupt_disable_done
2272 2273 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2273 2274 /*
2274 2275 * Synthesize the PS_IE bit from the event mask bit
2275 2276 */
2276 2277 andq $_BITNOT(PS_IE), %rax
2277 2278 testb $1, %dl
2278 2279 jnz .dtrace_interrupt_disable_done
2279 2280 orq $PS_IE, %rax
2280 2281 #else
2281 2282 CLI(%rdx)
2282 2283 #endif
2283 2284 .dtrace_interrupt_disable_done:
2284 2285 ret
2285 2286 SET_SIZE(dtrace_interrupt_disable)
2286 2287
2287 2288 #elif defined(__i386)
2288 2289
2289 2290 ENTRY(dtrace_interrupt_disable)
2290 2291 pushfl
2291 2292 popl %eax
2292 2293 #if defined(__xpv)
2293 2294 leal xpv_panicking, %edx
2294 2295 movl (%edx), %edx
2295 2296 cmpl $0, %edx
2296 2297 jne .dtrace_interrupt_disable_done
2297 2298 CLIRET(%edx, %cl) /* returns event mask in %cl */
2298 2299 /*
2299 2300 * Synthesize the PS_IE bit from the event mask bit
2300 2301 */
2301 2302 andl $_BITNOT(PS_IE), %eax
2302 2303 testb $1, %cl
2303 2304 jnz .dtrace_interrupt_disable_done
2304 2305 orl $PS_IE, %eax
2305 2306 #else
2306 2307 CLI(%edx)
2307 2308 #endif
2308 2309 .dtrace_interrupt_disable_done:
2309 2310 ret
2310 2311 SET_SIZE(dtrace_interrupt_disable)
2311 2312
2312 2313 #endif /* __i386 */
2313 2314 #endif /* __lint */
2314 2315
2315 2316 #if defined(__lint)
2316 2317
2317 2318 /*ARGSUSED*/
2318 2319 void
2319 2320 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2320 2321 {}
2321 2322
2322 2323 #else /* __lint */
2323 2324
2324 2325 #if defined(__amd64)
2325 2326
2326 2327 ENTRY(dtrace_interrupt_enable)
2327 2328 pushq %rdi
2328 2329 popfq
2329 2330 #if defined(__xpv)
2330 2331 leaq xpv_panicking, %rdx
2331 2332 movl (%rdx), %edx
2332 2333 cmpl $0, %edx
2333 2334 jne .dtrace_interrupt_enable_done
2334 2335 /*
2335 2336 * Since we're -really- running unprivileged, our attempt
2336 2337 * to change the state of the IF bit will be ignored. The
2337 2338 * virtual IF bit is tweaked by CLI and STI.
2338 2339 */
2339 2340 IE_TO_EVENT_MASK(%rdx, %rdi)
2340 2341 #endif
2341 2342 .dtrace_interrupt_enable_done:
2342 2343 ret
2343 2344 SET_SIZE(dtrace_interrupt_enable)
2344 2345
2345 2346 #elif defined(__i386)
2346 2347
2347 2348 ENTRY(dtrace_interrupt_enable)
2348 2349 movl 4(%esp), %eax
2349 2350 pushl %eax
2350 2351 popfl
2351 2352 #if defined(__xpv)
2352 2353 leal xpv_panicking, %edx
2353 2354 movl (%edx), %edx
2354 2355 cmpl $0, %edx
2355 2356 jne .dtrace_interrupt_enable_done
2356 2357 /*
2357 2358 * Since we're -really- running unprivileged, our attempt
2358 2359 * to change the state of the IF bit will be ignored. The
2359 2360 * virtual IF bit is tweaked by CLI and STI.
2360 2361 */
2361 2362 IE_TO_EVENT_MASK(%edx, %eax)
2362 2363 #endif
2363 2364 .dtrace_interrupt_enable_done:
2364 2365 ret
2365 2366 SET_SIZE(dtrace_interrupt_enable)
2366 2367
2367 2368 #endif /* __i386 */
2368 2369 #endif /* __lint */
2369 2370
2370 2371
2371 2372 #if defined(lint)
2372 2373
2373 2374 void
2374 2375 dtrace_membar_producer(void)
2375 2376 {}
2376 2377
2377 2378 void
2378 2379 dtrace_membar_consumer(void)
2379 2380 {}
2380 2381
2381 2382 #else /* __lint */
2382 2383
2383 2384 ENTRY(dtrace_membar_producer)
2384 2385 rep; ret /* use 2 byte return instruction when branch target */
2385 2386 /* AMD Software Optimization Guide - Section 6.2 */
2386 2387 SET_SIZE(dtrace_membar_producer)
2387 2388
2388 2389 ENTRY(dtrace_membar_consumer)
2389 2390 rep; ret /* use 2 byte return instruction when branch target */
2390 2391 /* AMD Software Optimization Guide - Section 6.2 */
2391 2392 SET_SIZE(dtrace_membar_consumer)
2392 2393
2393 2394 #endif /* __lint */
2394 2395
2395 2396 #if defined(__lint)
2396 2397
2397 2398 kthread_id_t
2398 2399 threadp(void)
2399 2400 { return ((kthread_id_t)0); }
2400 2401
2401 2402 #else /* __lint */
2402 2403
2403 2404 #if defined(__amd64)
2404 2405
2405 2406 ENTRY(threadp)
2406 2407 movq %gs:CPU_THREAD, %rax
2407 2408 ret
2408 2409 SET_SIZE(threadp)
2409 2410
2410 2411 #elif defined(__i386)
2411 2412
2412 2413 ENTRY(threadp)
2413 2414 movl %gs:CPU_THREAD, %eax
2414 2415 ret
2415 2416 SET_SIZE(threadp)
2416 2417
2417 2418 #endif /* __i386 */
2418 2419 #endif /* __lint */
2419 2420
2420 2421 /*
2421 2422 * Checksum routine for Internet Protocol Headers
2422 2423 */
2423 2424
2424 2425 #if defined(__lint)
2425 2426
2426 2427 /* ARGSUSED */
2427 2428 unsigned int
2428 2429 ip_ocsum(
2429 2430 ushort_t *address, /* ptr to 1st message buffer */
2430 2431 int halfword_count, /* length of data */
2431 2432 unsigned int sum) /* partial checksum */
2432 2433 {
2433 2434 int i;
2434 2435 unsigned int psum = 0; /* partial sum */
2435 2436
2436 2437 for (i = 0; i < halfword_count; i++, address++) {
2437 2438 psum += *address;
2438 2439 }
2439 2440
2440 2441 while ((psum >> 16) != 0) {
2441 2442 psum = (psum & 0xffff) + (psum >> 16);
2442 2443 }
2443 2444
2444 2445 psum += sum;
2445 2446
2446 2447 while ((psum >> 16) != 0) {
2447 2448 psum = (psum & 0xffff) + (psum >> 16);
2448 2449 }
2449 2450
2450 2451 return (psum);
2451 2452 }
2452 2453
2453 2454 #else /* __lint */
2454 2455
2455 2456 #if defined(__amd64)
2456 2457
2457 2458 ENTRY(ip_ocsum)
2458 2459 pushq %rbp
2459 2460 movq %rsp, %rbp
2460 2461 #ifdef DEBUG
2461 2462 movq postbootkernelbase(%rip), %rax
2462 2463 cmpq %rax, %rdi
2463 2464 jnb 1f
2464 2465 xorl %eax, %eax
2465 2466 movq %rdi, %rsi
2466 2467 leaq .ip_ocsum_panic_msg(%rip), %rdi
2467 2468 call panic
2468 2469 /*NOTREACHED*/
2469 2470 .ip_ocsum_panic_msg:
2470 2471 .string "ip_ocsum: address 0x%p below kernelbase\n"
2471 2472 1:
2472 2473 #endif
2473 2474 movl %esi, %ecx /* halfword_count */
2474 2475 movq %rdi, %rsi /* address */
2475 2476 /* partial sum in %edx */
2476 2477 xorl %eax, %eax
2477 2478 testl %ecx, %ecx
2478 2479 jz .ip_ocsum_done
2479 2480 testq $3, %rsi
2480 2481 jnz .ip_csum_notaligned
2481 2482 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2482 2483 .next_iter:
2483 2484 /* XX64 opportunities for prefetch? */
2484 2485 /* XX64 compute csum with 64 bit quantities? */
2485 2486 subl $32, %ecx
2486 2487 jl .less_than_32
2487 2488
2488 2489 addl 0(%rsi), %edx
2489 2490 .only60:
2490 2491 adcl 4(%rsi), %eax
2491 2492 .only56:
2492 2493 adcl 8(%rsi), %edx
2493 2494 .only52:
2494 2495 adcl 12(%rsi), %eax
2495 2496 .only48:
2496 2497 adcl 16(%rsi), %edx
2497 2498 .only44:
2498 2499 adcl 20(%rsi), %eax
2499 2500 .only40:
2500 2501 adcl 24(%rsi), %edx
2501 2502 .only36:
2502 2503 adcl 28(%rsi), %eax
2503 2504 .only32:
2504 2505 adcl 32(%rsi), %edx
2505 2506 .only28:
2506 2507 adcl 36(%rsi), %eax
2507 2508 .only24:
2508 2509 adcl 40(%rsi), %edx
2509 2510 .only20:
2510 2511 adcl 44(%rsi), %eax
2511 2512 .only16:
2512 2513 adcl 48(%rsi), %edx
2513 2514 .only12:
2514 2515 adcl 52(%rsi), %eax
2515 2516 .only8:
2516 2517 adcl 56(%rsi), %edx
2517 2518 .only4:
2518 2519 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2519 2520 .only0:
2520 2521 adcl $0, %eax /* could be adding -1 in eax with a carry */
2521 2522 adcl $0, %eax
2522 2523
2523 2524 addq $64, %rsi
2524 2525 testl %ecx, %ecx
2525 2526 jnz .next_iter
2526 2527
2527 2528 .ip_ocsum_done:
2528 2529 addl %eax, %edx
2529 2530 adcl $0, %edx
2530 2531 movl %edx, %eax /* form a 16 bit checksum by */
2531 2532 shrl $16, %eax /* adding two halves of 32 bit checksum */
2532 2533 addw %dx, %ax
2533 2534 adcw $0, %ax
2534 2535 andl $0xffff, %eax
2535 2536 leave
2536 2537 ret
2537 2538
2538 2539 .ip_csum_notaligned:
2539 2540 xorl %edi, %edi
2540 2541 movw (%rsi), %di
2541 2542 addl %edi, %edx
2542 2543 adcl $0, %edx
2543 2544 addq $2, %rsi
2544 2545 decl %ecx
2545 2546 jmp .ip_csum_aligned
2546 2547
2547 2548 .less_than_32:
2548 2549 addl $32, %ecx
2549 2550 testl $1, %ecx
2550 2551 jz .size_aligned
2551 2552 andl $0xfe, %ecx
2552 2553 movzwl (%rsi, %rcx, 2), %edi
2553 2554 addl %edi, %edx
2554 2555 adcl $0, %edx
2555 2556 .size_aligned:
2556 2557 movl %ecx, %edi
2557 2558 shrl $1, %ecx
2558 2559 shl $1, %edi
2559 2560 subq $64, %rdi
2560 2561 addq %rdi, %rsi
2561 2562 leaq .ip_ocsum_jmptbl(%rip), %rdi
2562 2563 leaq (%rdi, %rcx, 8), %rdi
2563 2564 xorl %ecx, %ecx
2564 2565 clc
2565 2566 jmp *(%rdi)
2566 2567
2567 2568 .align 8
2568 2569 .ip_ocsum_jmptbl:
2569 2570 .quad .only0, .only4, .only8, .only12, .only16, .only20
2570 2571 .quad .only24, .only28, .only32, .only36, .only40, .only44
2571 2572 .quad .only48, .only52, .only56, .only60
2572 2573 SET_SIZE(ip_ocsum)
2573 2574
2574 2575 #elif defined(__i386)
2575 2576
2576 2577 ENTRY(ip_ocsum)
2577 2578 pushl %ebp
2578 2579 movl %esp, %ebp
2579 2580 pushl %ebx
2580 2581 pushl %esi
2581 2582 pushl %edi
2582 2583 movl 12(%ebp), %ecx /* count of half words */
2583 2584 movl 16(%ebp), %edx /* partial checksum */
2584 2585 movl 8(%ebp), %esi
2585 2586 xorl %eax, %eax
2586 2587 testl %ecx, %ecx
2587 2588 jz .ip_ocsum_done
2588 2589
2589 2590 testl $3, %esi
2590 2591 jnz .ip_csum_notaligned
2591 2592 .ip_csum_aligned:
2592 2593 .next_iter:
2593 2594 subl $32, %ecx
2594 2595 jl .less_than_32
2595 2596
2596 2597 addl 0(%esi), %edx
2597 2598 .only60:
2598 2599 adcl 4(%esi), %eax
2599 2600 .only56:
2600 2601 adcl 8(%esi), %edx
2601 2602 .only52:
2602 2603 adcl 12(%esi), %eax
2603 2604 .only48:
2604 2605 adcl 16(%esi), %edx
2605 2606 .only44:
2606 2607 adcl 20(%esi), %eax
2607 2608 .only40:
2608 2609 adcl 24(%esi), %edx
2609 2610 .only36:
2610 2611 adcl 28(%esi), %eax
2611 2612 .only32:
2612 2613 adcl 32(%esi), %edx
2613 2614 .only28:
2614 2615 adcl 36(%esi), %eax
2615 2616 .only24:
2616 2617 adcl 40(%esi), %edx
2617 2618 .only20:
2618 2619 adcl 44(%esi), %eax
2619 2620 .only16:
2620 2621 adcl 48(%esi), %edx
2621 2622 .only12:
2622 2623 adcl 52(%esi), %eax
2623 2624 .only8:
2624 2625 adcl 56(%esi), %edx
2625 2626 .only4:
2626 2627 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2627 2628 .only0:
2628 2629 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2629 2630 adcl $0, %eax
2630 2631
2631 2632 addl $64, %esi
2632 2633 andl %ecx, %ecx
2633 2634 jnz .next_iter
2634 2635
2635 2636 .ip_ocsum_done:
2636 2637 addl %eax, %edx
2637 2638 adcl $0, %edx
2638 2639 movl %edx, %eax /* form a 16 bit checksum by */
2639 2640 shrl $16, %eax /* adding two halves of 32 bit checksum */
2640 2641 addw %dx, %ax
2641 2642 adcw $0, %ax
2642 2643 andl $0xffff, %eax
2643 2644 popl %edi /* restore registers */
2644 2645 popl %esi
2645 2646 popl %ebx
2646 2647 leave
2647 2648 ret
2648 2649
2649 2650 .ip_csum_notaligned:
2650 2651 xorl %edi, %edi
2651 2652 movw (%esi), %di
2652 2653 addl %edi, %edx
2653 2654 adcl $0, %edx
2654 2655 addl $2, %esi
2655 2656 decl %ecx
2656 2657 jmp .ip_csum_aligned
2657 2658
2658 2659 .less_than_32:
2659 2660 addl $32, %ecx
2660 2661 testl $1, %ecx
2661 2662 jz .size_aligned
2662 2663 andl $0xfe, %ecx
2663 2664 movzwl (%esi, %ecx, 2), %edi
2664 2665 addl %edi, %edx
2665 2666 adcl $0, %edx
2666 2667 .size_aligned:
2667 2668 movl %ecx, %edi
2668 2669 shrl $1, %ecx
2669 2670 shl $1, %edi
2670 2671 subl $64, %edi
2671 2672 addl %edi, %esi
2672 2673 movl $.ip_ocsum_jmptbl, %edi
2673 2674 lea (%edi, %ecx, 4), %edi
2674 2675 xorl %ecx, %ecx
2675 2676 clc
2676 2677 jmp *(%edi)
2677 2678 SET_SIZE(ip_ocsum)
2678 2679
2679 2680 .data
2680 2681 .align 4
2681 2682
2682 2683 .ip_ocsum_jmptbl:
2683 2684 .long .only0, .only4, .only8, .only12, .only16, .only20
2684 2685 .long .only24, .only28, .only32, .only36, .only40, .only44
2685 2686 .long .only48, .only52, .only56, .only60
2686 2687
2687 2688
2688 2689 #endif /* __i386 */
2689 2690 #endif /* __lint */
2690 2691
2691 2692 /*
2692 2693 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2693 2694 * Provided to manipulate hrtime_t values.
2694 2695 */
2695 2696 #if defined(__lint)
2696 2697
2697 2698 /* result = a * b; */
2698 2699
2699 2700 /* ARGSUSED */
2700 2701 unsigned long long
2701 2702 mul32(uint_t a, uint_t b)
2702 2703 { return (0); }
2703 2704
2704 2705 #else /* __lint */
2705 2706
2706 2707 #if defined(__amd64)
2707 2708
2708 2709 ENTRY(mul32)
2709 2710 xorl %edx, %edx /* XX64 joe, paranoia? */
2710 2711 movl %edi, %eax
2711 2712 mull %esi
2712 2713 shlq $32, %rdx
2713 2714 orq %rdx, %rax
2714 2715 ret
2715 2716 SET_SIZE(mul32)
2716 2717
2717 2718 #elif defined(__i386)
2718 2719
2719 2720 ENTRY(mul32)
2720 2721 movl 8(%esp), %eax
2721 2722 movl 4(%esp), %ecx
2722 2723 mull %ecx
2723 2724 ret
2724 2725 SET_SIZE(mul32)
2725 2726
2726 2727 #endif /* __i386 */
2727 2728 #endif /* __lint */
2728 2729
2729 2730 #if defined(notused)
2730 2731 #if defined(__lint)
2731 2732 /* ARGSUSED */
2732 2733 void
2733 2734 load_pte64(uint64_t *pte, uint64_t pte_value)
2734 2735 {}
2735 2736 #else /* __lint */
2736 2737 .globl load_pte64
2737 2738 load_pte64:
2738 2739 movl 4(%esp), %eax
2739 2740 movl 8(%esp), %ecx
2740 2741 movl 12(%esp), %edx
2741 2742 movl %edx, 4(%eax)
2742 2743 movl %ecx, (%eax)
2743 2744 ret
2744 2745 #endif /* __lint */
2745 2746 #endif /* notused */
2746 2747
2747 2748 #if defined(__lint)
2748 2749
2749 2750 /*ARGSUSED*/
2750 2751 void
2751 2752 scan_memory(caddr_t addr, size_t size)
2752 2753 {}
2753 2754
2754 2755 #else /* __lint */
2755 2756
2756 2757 #if defined(__amd64)
2757 2758
2758 2759 ENTRY(scan_memory)
2759 2760 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2760 2761 jz .scanm_done
2761 2762 movq %rsi, %rcx /* move count into rep control register */
2762 2763 movq %rdi, %rsi /* move addr into lodsq control reg. */
2763 2764 rep lodsq /* scan the memory range */
2764 2765 .scanm_done:
2765 2766 rep; ret /* use 2 byte return instruction when branch target */
2766 2767 /* AMD Software Optimization Guide - Section 6.2 */
2767 2768 SET_SIZE(scan_memory)
2768 2769
2769 2770 #elif defined(__i386)
2770 2771
2771 2772 ENTRY(scan_memory)
2772 2773 pushl %ecx
2773 2774 pushl %esi
2774 2775 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2775 2776 shrl $2, %ecx /* convert from byte count to word count */
2776 2777 jz .scanm_done
2777 2778 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2778 2779 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2779 2780 lodsl
2780 2781 .scanm_done:
2781 2782 popl %esi
2782 2783 popl %ecx
2783 2784 ret
2784 2785 SET_SIZE(scan_memory)
2785 2786
2786 2787 #endif /* __i386 */
2787 2788 #endif /* __lint */
2788 2789
2789 2790
2790 2791 #if defined(__lint)
2791 2792
2792 2793 /*ARGSUSED */
2793 2794 int
2794 2795 lowbit(ulong_t i)
2795 2796 { return (0); }
2796 2797
2797 2798 #else /* __lint */
2798 2799
2799 2800 #if defined(__amd64)
2800 2801
2801 2802 ENTRY(lowbit)
2802 2803 movl $-1, %eax
2803 2804 bsfq %rdi, %rax
2804 2805 incl %eax
2805 2806 ret
2806 2807 SET_SIZE(lowbit)
2807 2808
2808 2809 #elif defined(__i386)
2809 2810
2810 2811 ENTRY(lowbit)
2811 2812 movl $-1, %eax
2812 2813 bsfl 4(%esp), %eax
2813 2814 incl %eax
2814 2815 ret
2815 2816 SET_SIZE(lowbit)
2816 2817
2817 2818 #endif /* __i386 */
2818 2819 #endif /* __lint */
2819 2820
2820 2821 #if defined(__lint)
2821 2822
2822 2823 /*ARGSUSED*/
2823 2824 int
2824 2825 highbit(ulong_t i)
2825 2826 { return (0); }
2826 2827
2827 2828 #else /* __lint */
2828 2829
2829 2830 #if defined(__amd64)
2830 2831
2831 2832 ENTRY(highbit)
2832 2833 movl $-1, %eax
2833 2834 bsrq %rdi, %rax
2834 2835 incl %eax
2835 2836 ret
2836 2837 SET_SIZE(highbit)
2837 2838
|
↓ open down ↓ |
2803 lines elided |
↑ open up ↑ |
2838 2839 #elif defined(__i386)
2839 2840
2840 2841 ENTRY(highbit)
2841 2842 movl $-1, %eax
2842 2843 bsrl 4(%esp), %eax
2843 2844 incl %eax
2844 2845 ret
2845 2846 SET_SIZE(highbit)
2846 2847
2847 2848 #endif /* __i386 */
2849 +#endif /* __lint */
2850 +
2851 +#if defined(__lint)
2852 +
2853 +/*ARGSUSED*/
2854 +int
2855 +highbit64(uint64_t i)
2856 +{ return (0); }
2857 +
2858 +#else /* __lint */
2859 +
2860 +#if defined(__amd64)
2861 +
2862 + ENTRY(highbit64)
2863 + movl $-1, %eax
2864 + bsrq %rdi, %rax
2865 + incl %eax
2866 + ret
2867 + SET_SIZE(highbit64)
2868 +
2869 +#elif defined(__i386)
2870 +
2871 + ENTRY(highbit64)
2872 + bsrl 8(%esp), %eax
2873 + jz .lowbit
2874 + addl $32, %eax
2875 + jmp .done
2876 +
2877 +.lowbit:
2878 + movl $-1, %eax
2879 + bsrl 4(%esp), %eax
2880 +.done:
2881 + incl %eax
2882 + ret
2883 + SET_SIZE(highbit64)
2884 +
2885 +#endif /* __i386 */
2848 2886 #endif /* __lint */
2849 2887
2850 2888 #if defined(__lint)
2851 2889
2852 2890 /*ARGSUSED*/
2853 2891 uint64_t
2854 2892 rdmsr(uint_t r)
2855 2893 { return (0); }
2856 2894
2857 2895 /*ARGSUSED*/
2858 2896 void
2859 2897 wrmsr(uint_t r, const uint64_t val)
2860 2898 {}
2861 2899
2862 2900 /*ARGSUSED*/
2863 2901 uint64_t
2864 2902 xrdmsr(uint_t r)
2865 2903 { return (0); }
2866 2904
2867 2905 /*ARGSUSED*/
2868 2906 void
2869 2907 xwrmsr(uint_t r, const uint64_t val)
2870 2908 {}
2871 2909
2872 2910 void
2873 2911 invalidate_cache(void)
2874 2912 {}
2875 2913
2876 2914 /*ARGSUSED*/
2877 2915 uint64_t
2878 2916 get_xcr(uint_t r)
2879 2917 { return (0); }
2880 2918
2881 2919 /*ARGSUSED*/
2882 2920 void
2883 2921 set_xcr(uint_t r, const uint64_t val)
2884 2922 {}
2885 2923
2886 2924 #else /* __lint */
2887 2925
2888 2926 #define XMSR_ACCESS_VAL $0x9c5a203a
2889 2927
2890 2928 #if defined(__amd64)
2891 2929
2892 2930 ENTRY(rdmsr)
2893 2931 movl %edi, %ecx
2894 2932 rdmsr
2895 2933 shlq $32, %rdx
2896 2934 orq %rdx, %rax
2897 2935 ret
2898 2936 SET_SIZE(rdmsr)
2899 2937
2900 2938 ENTRY(wrmsr)
2901 2939 movq %rsi, %rdx
2902 2940 shrq $32, %rdx
2903 2941 movl %esi, %eax
2904 2942 movl %edi, %ecx
2905 2943 wrmsr
2906 2944 ret
2907 2945 SET_SIZE(wrmsr)
2908 2946
2909 2947 ENTRY(xrdmsr)
2910 2948 pushq %rbp
2911 2949 movq %rsp, %rbp
2912 2950 movl %edi, %ecx
2913 2951 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2914 2952 rdmsr
2915 2953 shlq $32, %rdx
2916 2954 orq %rdx, %rax
2917 2955 leave
2918 2956 ret
2919 2957 SET_SIZE(xrdmsr)
2920 2958
2921 2959 ENTRY(xwrmsr)
2922 2960 pushq %rbp
2923 2961 movq %rsp, %rbp
2924 2962 movl %edi, %ecx
2925 2963 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2926 2964 movq %rsi, %rdx
2927 2965 shrq $32, %rdx
2928 2966 movl %esi, %eax
2929 2967 wrmsr
2930 2968 leave
2931 2969 ret
2932 2970 SET_SIZE(xwrmsr)
2933 2971
2934 2972 ENTRY(get_xcr)
2935 2973 movl %edi, %ecx
2936 2974 #xgetbv
2937 2975 .byte 0x0f,0x01,0xd0
2938 2976 shlq $32, %rdx
2939 2977 orq %rdx, %rax
2940 2978 ret
2941 2979 SET_SIZE(get_xcr)
2942 2980
2943 2981 ENTRY(set_xcr)
2944 2982 movq %rsi, %rdx
2945 2983 shrq $32, %rdx
2946 2984 movl %esi, %eax
2947 2985 movl %edi, %ecx
2948 2986 #xsetbv
2949 2987 .byte 0x0f,0x01,0xd1
2950 2988 ret
2951 2989 SET_SIZE(set_xcr)
2952 2990
2953 2991 #elif defined(__i386)
2954 2992
2955 2993 ENTRY(rdmsr)
2956 2994 movl 4(%esp), %ecx
2957 2995 rdmsr
2958 2996 ret
2959 2997 SET_SIZE(rdmsr)
2960 2998
2961 2999 ENTRY(wrmsr)
2962 3000 movl 4(%esp), %ecx
2963 3001 movl 8(%esp), %eax
2964 3002 movl 12(%esp), %edx
2965 3003 wrmsr
2966 3004 ret
2967 3005 SET_SIZE(wrmsr)
2968 3006
2969 3007 ENTRY(xrdmsr)
2970 3008 pushl %ebp
2971 3009 movl %esp, %ebp
2972 3010 movl 8(%esp), %ecx
2973 3011 pushl %edi
2974 3012 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2975 3013 rdmsr
2976 3014 popl %edi
2977 3015 leave
2978 3016 ret
2979 3017 SET_SIZE(xrdmsr)
2980 3018
2981 3019 ENTRY(xwrmsr)
2982 3020 pushl %ebp
2983 3021 movl %esp, %ebp
2984 3022 movl 8(%esp), %ecx
2985 3023 movl 12(%esp), %eax
2986 3024 movl 16(%esp), %edx
2987 3025 pushl %edi
2988 3026 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2989 3027 wrmsr
2990 3028 popl %edi
2991 3029 leave
2992 3030 ret
2993 3031 SET_SIZE(xwrmsr)
2994 3032
2995 3033 ENTRY(get_xcr)
2996 3034 movl 4(%esp), %ecx
2997 3035 #xgetbv
2998 3036 .byte 0x0f,0x01,0xd0
2999 3037 ret
3000 3038 SET_SIZE(get_xcr)
3001 3039
3002 3040 ENTRY(set_xcr)
3003 3041 movl 4(%esp), %ecx
3004 3042 movl 8(%esp), %eax
3005 3043 movl 12(%esp), %edx
3006 3044 #xsetbv
3007 3045 .byte 0x0f,0x01,0xd1
3008 3046 ret
3009 3047 SET_SIZE(set_xcr)
3010 3048
3011 3049 #endif /* __i386 */
3012 3050
3013 3051 ENTRY(invalidate_cache)
3014 3052 wbinvd
3015 3053 ret
3016 3054 SET_SIZE(invalidate_cache)
3017 3055
3018 3056 #endif /* __lint */
3019 3057
3020 3058 #if defined(__lint)
3021 3059
3022 3060 /*ARGSUSED*/
3023 3061 void
3024 3062 getcregs(struct cregs *crp)
3025 3063 {}
3026 3064
3027 3065 #else /* __lint */
3028 3066
3029 3067 #if defined(__amd64)
3030 3068
3031 3069 ENTRY_NP(getcregs)
3032 3070 #if defined(__xpv)
3033 3071 /*
3034 3072 * Only a few of the hardware control registers or descriptor tables
3035 3073 * are directly accessible to us, so just zero the structure.
3036 3074 *
3037 3075 * XXPV Perhaps it would be helpful for the hypervisor to return
3038 3076 * virtualized versions of these for post-mortem use.
3039 3077 * (Need to reevaluate - perhaps it already does!)
3040 3078 */
3041 3079 pushq %rdi /* save *crp */
3042 3080 movq $CREGSZ, %rsi
3043 3081 call bzero
3044 3082 popq %rdi
3045 3083
3046 3084 /*
3047 3085 * Dump what limited information we can
3048 3086 */
3049 3087 movq %cr0, %rax
3050 3088 movq %rax, CREG_CR0(%rdi) /* cr0 */
3051 3089 movq %cr2, %rax
3052 3090 movq %rax, CREG_CR2(%rdi) /* cr2 */
3053 3091 movq %cr3, %rax
3054 3092 movq %rax, CREG_CR3(%rdi) /* cr3 */
3055 3093 movq %cr4, %rax
3056 3094 movq %rax, CREG_CR4(%rdi) /* cr4 */
3057 3095
3058 3096 #else /* __xpv */
3059 3097
3060 3098 #define GETMSR(r, off, d) \
3061 3099 movl $r, %ecx; \
3062 3100 rdmsr; \
3063 3101 movl %eax, off(d); \
3064 3102 movl %edx, off+4(d)
3065 3103
3066 3104 xorl %eax, %eax
3067 3105 movq %rax, CREG_GDT+8(%rdi)
3068 3106 sgdt CREG_GDT(%rdi) /* 10 bytes */
3069 3107 movq %rax, CREG_IDT+8(%rdi)
3070 3108 sidt CREG_IDT(%rdi) /* 10 bytes */
3071 3109 movq %rax, CREG_LDT(%rdi)
3072 3110 sldt CREG_LDT(%rdi) /* 2 bytes */
3073 3111 movq %rax, CREG_TASKR(%rdi)
3074 3112 str CREG_TASKR(%rdi) /* 2 bytes */
3075 3113 movq %cr0, %rax
3076 3114 movq %rax, CREG_CR0(%rdi) /* cr0 */
3077 3115 movq %cr2, %rax
3078 3116 movq %rax, CREG_CR2(%rdi) /* cr2 */
3079 3117 movq %cr3, %rax
3080 3118 movq %rax, CREG_CR3(%rdi) /* cr3 */
3081 3119 movq %cr4, %rax
3082 3120 movq %rax, CREG_CR4(%rdi) /* cr4 */
3083 3121 movq %cr8, %rax
3084 3122 movq %rax, CREG_CR8(%rdi) /* cr8 */
3085 3123 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3086 3124 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3087 3125 #endif /* __xpv */
3088 3126 ret
3089 3127 SET_SIZE(getcregs)
3090 3128
3091 3129 #undef GETMSR
3092 3130
3093 3131 #elif defined(__i386)
3094 3132
3095 3133 ENTRY_NP(getcregs)
3096 3134 #if defined(__xpv)
3097 3135 /*
3098 3136 * Only a few of the hardware control registers or descriptor tables
3099 3137 * are directly accessible to us, so just zero the structure.
3100 3138 *
3101 3139 * XXPV Perhaps it would be helpful for the hypervisor to return
3102 3140 * virtualized versions of these for post-mortem use.
3103 3141 * (Need to reevaluate - perhaps it already does!)
3104 3142 */
3105 3143 movl 4(%esp), %edx
3106 3144 pushl $CREGSZ
3107 3145 pushl %edx
3108 3146 call bzero
3109 3147 addl $8, %esp
3110 3148 movl 4(%esp), %edx
3111 3149
3112 3150 /*
3113 3151 * Dump what limited information we can
3114 3152 */
3115 3153 movl %cr0, %eax
3116 3154 movl %eax, CREG_CR0(%edx) /* cr0 */
3117 3155 movl %cr2, %eax
3118 3156 movl %eax, CREG_CR2(%edx) /* cr2 */
3119 3157 movl %cr3, %eax
3120 3158 movl %eax, CREG_CR3(%edx) /* cr3 */
3121 3159 movl %cr4, %eax
3122 3160 movl %eax, CREG_CR4(%edx) /* cr4 */
3123 3161
3124 3162 #else /* __xpv */
3125 3163
3126 3164 movl 4(%esp), %edx
3127 3165 movw $0, CREG_GDT+6(%edx)
3128 3166 movw $0, CREG_IDT+6(%edx)
3129 3167 sgdt CREG_GDT(%edx) /* gdt */
3130 3168 sidt CREG_IDT(%edx) /* idt */
3131 3169 sldt CREG_LDT(%edx) /* ldt */
3132 3170 str CREG_TASKR(%edx) /* task */
3133 3171 movl %cr0, %eax
3134 3172 movl %eax, CREG_CR0(%edx) /* cr0 */
3135 3173 movl %cr2, %eax
3136 3174 movl %eax, CREG_CR2(%edx) /* cr2 */
3137 3175 movl %cr3, %eax
3138 3176 movl %eax, CREG_CR3(%edx) /* cr3 */
3139 3177 bt $X86FSET_LARGEPAGE, x86_featureset
3140 3178 jnc .nocr4
3141 3179 movl %cr4, %eax
3142 3180 movl %eax, CREG_CR4(%edx) /* cr4 */
3143 3181 jmp .skip
3144 3182 .nocr4:
3145 3183 movl $0, CREG_CR4(%edx)
3146 3184 .skip:
3147 3185 #endif
3148 3186 ret
3149 3187 SET_SIZE(getcregs)
3150 3188
3151 3189 #endif /* __i386 */
3152 3190 #endif /* __lint */
3153 3191
3154 3192
3155 3193 /*
3156 3194 * A panic trigger is a word which is updated atomically and can only be set
3157 3195 * once. We atomically store 0xDEFACEDD and load the old value. If the
3158 3196 * previous value was 0, we succeed and return 1; otherwise return 0.
3159 3197 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3160 3198 * has its own version of this function to allow it to panic correctly from
3161 3199 * probe context.
3162 3200 */
3163 3201 #if defined(__lint)
3164 3202
3165 3203 /*ARGSUSED*/
3166 3204 int
3167 3205 panic_trigger(int *tp)
3168 3206 { return (0); }
3169 3207
3170 3208 /*ARGSUSED*/
3171 3209 int
3172 3210 dtrace_panic_trigger(int *tp)
3173 3211 { return (0); }
3174 3212
3175 3213 #else /* __lint */
3176 3214
3177 3215 #if defined(__amd64)
3178 3216
3179 3217 ENTRY_NP(panic_trigger)
3180 3218 xorl %eax, %eax
3181 3219 movl $0xdefacedd, %edx
3182 3220 lock
3183 3221 xchgl %edx, (%rdi)
3184 3222 cmpl $0, %edx
3185 3223 je 0f
3186 3224 movl $0, %eax
3187 3225 ret
3188 3226 0: movl $1, %eax
3189 3227 ret
3190 3228 SET_SIZE(panic_trigger)
3191 3229
3192 3230 ENTRY_NP(dtrace_panic_trigger)
3193 3231 xorl %eax, %eax
3194 3232 movl $0xdefacedd, %edx
3195 3233 lock
3196 3234 xchgl %edx, (%rdi)
3197 3235 cmpl $0, %edx
3198 3236 je 0f
3199 3237 movl $0, %eax
3200 3238 ret
3201 3239 0: movl $1, %eax
3202 3240 ret
3203 3241 SET_SIZE(dtrace_panic_trigger)
3204 3242
3205 3243 #elif defined(__i386)
3206 3244
3207 3245 ENTRY_NP(panic_trigger)
3208 3246 movl 4(%esp), %edx / %edx = address of trigger
3209 3247 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3210 3248 lock / assert lock
3211 3249 xchgl %eax, (%edx) / exchange %eax and the trigger
3212 3250 cmpl $0, %eax / if (%eax == 0x0)
3213 3251 je 0f / return (1);
3214 3252 movl $0, %eax / else
3215 3253 ret / return (0);
3216 3254 0: movl $1, %eax
3217 3255 ret
3218 3256 SET_SIZE(panic_trigger)
3219 3257
3220 3258 ENTRY_NP(dtrace_panic_trigger)
3221 3259 movl 4(%esp), %edx / %edx = address of trigger
3222 3260 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3223 3261 lock / assert lock
3224 3262 xchgl %eax, (%edx) / exchange %eax and the trigger
3225 3263 cmpl $0, %eax / if (%eax == 0x0)
3226 3264 je 0f / return (1);
3227 3265 movl $0, %eax / else
3228 3266 ret / return (0);
3229 3267 0: movl $1, %eax
3230 3268 ret
3231 3269 SET_SIZE(dtrace_panic_trigger)
3232 3270
3233 3271 #endif /* __i386 */
3234 3272 #endif /* __lint */
3235 3273
3236 3274 /*
3237 3275 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3238 3276 * into the panic code implemented in panicsys(). vpanic() is responsible
3239 3277 * for passing through the format string and arguments, and constructing a
3240 3278 * regs structure on the stack into which it saves the current register
3241 3279 * values. If we are not dying due to a fatal trap, these registers will
3242 3280 * then be preserved in panicbuf as the current processor state. Before
3243 3281 * invoking panicsys(), vpanic() activates the first panic trigger (see
3244 3282 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3245 3283 * DTrace takes a slightly different panic path if it must panic from probe
3246 3284 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3247 3285 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3248 3286 * branches back into vpanic().
3249 3287 */
3250 3288 #if defined(__lint)
3251 3289
3252 3290 /*ARGSUSED*/
3253 3291 void
3254 3292 vpanic(const char *format, va_list alist)
3255 3293 {}
3256 3294
3257 3295 /*ARGSUSED*/
3258 3296 void
3259 3297 dtrace_vpanic(const char *format, va_list alist)
3260 3298 {}
3261 3299
3262 3300 #else /* __lint */
3263 3301
3264 3302 #if defined(__amd64)
3265 3303
3266 3304 ENTRY_NP(vpanic) /* Initial stack layout: */
3267 3305
3268 3306 pushq %rbp /* | %rip | 0x60 */
3269 3307 movq %rsp, %rbp /* | %rbp | 0x58 */
3270 3308 pushfq /* | rfl | 0x50 */
3271 3309 pushq %r11 /* | %r11 | 0x48 */
3272 3310 pushq %r10 /* | %r10 | 0x40 */
3273 3311 pushq %rbx /* | %rbx | 0x38 */
3274 3312 pushq %rax /* | %rax | 0x30 */
3275 3313 pushq %r9 /* | %r9 | 0x28 */
3276 3314 pushq %r8 /* | %r8 | 0x20 */
3277 3315 pushq %rcx /* | %rcx | 0x18 */
3278 3316 pushq %rdx /* | %rdx | 0x10 */
3279 3317 pushq %rsi /* | %rsi | 0x8 alist */
3280 3318 pushq %rdi /* | %rdi | 0x0 format */
3281 3319
3282 3320 movq %rsp, %rbx /* %rbx = current %rsp */
3283 3321
3284 3322 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3285 3323 call panic_trigger /* %eax = panic_trigger() */
3286 3324
3287 3325 vpanic_common:
3288 3326 /*
3289 3327 * The panic_trigger result is in %eax from the call above, and
3290 3328 * dtrace_panic places it in %eax before branching here.
3291 3329 * The rdmsr instructions that follow below will clobber %eax so
3292 3330 * we stash the panic_trigger result in %r11d.
3293 3331 */
3294 3332 movl %eax, %r11d
3295 3333 cmpl $0, %r11d
3296 3334 je 0f
3297 3335
3298 3336 /*
3299 3337 * If panic_trigger() was successful, we are the first to initiate a
3300 3338 * panic: we now switch to the reserved panic_stack before continuing.
3301 3339 */
3302 3340 leaq panic_stack(%rip), %rsp
3303 3341 addq $PANICSTKSIZE, %rsp
3304 3342 0: subq $REGSIZE, %rsp
3305 3343 /*
3306 3344 * Now that we've got everything set up, store the register values as
3307 3345 * they were when we entered vpanic() to the designated location in
3308 3346 * the regs structure we allocated on the stack.
3309 3347 */
3310 3348 movq 0x0(%rbx), %rcx
3311 3349 movq %rcx, REGOFF_RDI(%rsp)
3312 3350 movq 0x8(%rbx), %rcx
3313 3351 movq %rcx, REGOFF_RSI(%rsp)
3314 3352 movq 0x10(%rbx), %rcx
3315 3353 movq %rcx, REGOFF_RDX(%rsp)
3316 3354 movq 0x18(%rbx), %rcx
3317 3355 movq %rcx, REGOFF_RCX(%rsp)
3318 3356 movq 0x20(%rbx), %rcx
3319 3357
3320 3358 movq %rcx, REGOFF_R8(%rsp)
3321 3359 movq 0x28(%rbx), %rcx
3322 3360 movq %rcx, REGOFF_R9(%rsp)
3323 3361 movq 0x30(%rbx), %rcx
3324 3362 movq %rcx, REGOFF_RAX(%rsp)
3325 3363 movq 0x38(%rbx), %rcx
3326 3364 movq %rcx, REGOFF_RBX(%rsp)
3327 3365 movq 0x58(%rbx), %rcx
3328 3366
3329 3367 movq %rcx, REGOFF_RBP(%rsp)
3330 3368 movq 0x40(%rbx), %rcx
3331 3369 movq %rcx, REGOFF_R10(%rsp)
3332 3370 movq 0x48(%rbx), %rcx
3333 3371 movq %rcx, REGOFF_R11(%rsp)
3334 3372 movq %r12, REGOFF_R12(%rsp)
3335 3373
3336 3374 movq %r13, REGOFF_R13(%rsp)
3337 3375 movq %r14, REGOFF_R14(%rsp)
3338 3376 movq %r15, REGOFF_R15(%rsp)
3339 3377
3340 3378 xorl %ecx, %ecx
3341 3379 movw %ds, %cx
3342 3380 movq %rcx, REGOFF_DS(%rsp)
3343 3381 movw %es, %cx
3344 3382 movq %rcx, REGOFF_ES(%rsp)
3345 3383 movw %fs, %cx
3346 3384 movq %rcx, REGOFF_FS(%rsp)
3347 3385 movw %gs, %cx
3348 3386 movq %rcx, REGOFF_GS(%rsp)
3349 3387
3350 3388 movq $0, REGOFF_TRAPNO(%rsp)
3351 3389
3352 3390 movq $0, REGOFF_ERR(%rsp)
3353 3391 leaq vpanic(%rip), %rcx
3354 3392 movq %rcx, REGOFF_RIP(%rsp)
3355 3393 movw %cs, %cx
3356 3394 movzwq %cx, %rcx
3357 3395 movq %rcx, REGOFF_CS(%rsp)
3358 3396 movq 0x50(%rbx), %rcx
3359 3397 movq %rcx, REGOFF_RFL(%rsp)
3360 3398 movq %rbx, %rcx
3361 3399 addq $0x60, %rcx
3362 3400 movq %rcx, REGOFF_RSP(%rsp)
3363 3401 movw %ss, %cx
3364 3402 movzwq %cx, %rcx
3365 3403 movq %rcx, REGOFF_SS(%rsp)
3366 3404
3367 3405 /*
3368 3406 * panicsys(format, alist, rp, on_panic_stack)
3369 3407 */
3370 3408 movq REGOFF_RDI(%rsp), %rdi /* format */
3371 3409 movq REGOFF_RSI(%rsp), %rsi /* alist */
3372 3410 movq %rsp, %rdx /* struct regs */
3373 3411 movl %r11d, %ecx /* on_panic_stack */
3374 3412 call panicsys
3375 3413 addq $REGSIZE, %rsp
3376 3414 popq %rdi
3377 3415 popq %rsi
3378 3416 popq %rdx
3379 3417 popq %rcx
3380 3418 popq %r8
3381 3419 popq %r9
3382 3420 popq %rax
3383 3421 popq %rbx
3384 3422 popq %r10
3385 3423 popq %r11
3386 3424 popfq
3387 3425 leave
3388 3426 ret
3389 3427 SET_SIZE(vpanic)
3390 3428
3391 3429 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3392 3430
3393 3431 pushq %rbp /* | %rip | 0x60 */
3394 3432 movq %rsp, %rbp /* | %rbp | 0x58 */
3395 3433 pushfq /* | rfl | 0x50 */
3396 3434 pushq %r11 /* | %r11 | 0x48 */
3397 3435 pushq %r10 /* | %r10 | 0x40 */
3398 3436 pushq %rbx /* | %rbx | 0x38 */
3399 3437 pushq %rax /* | %rax | 0x30 */
3400 3438 pushq %r9 /* | %r9 | 0x28 */
3401 3439 pushq %r8 /* | %r8 | 0x20 */
3402 3440 pushq %rcx /* | %rcx | 0x18 */
3403 3441 pushq %rdx /* | %rdx | 0x10 */
3404 3442 pushq %rsi /* | %rsi | 0x8 alist */
3405 3443 pushq %rdi /* | %rdi | 0x0 format */
3406 3444
3407 3445 movq %rsp, %rbx /* %rbx = current %rsp */
3408 3446
3409 3447 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3410 3448 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3411 3449 jmp vpanic_common
3412 3450
3413 3451 SET_SIZE(dtrace_vpanic)
3414 3452
3415 3453 #elif defined(__i386)
3416 3454
3417 3455 ENTRY_NP(vpanic) / Initial stack layout:
3418 3456
3419 3457 pushl %ebp / | %eip | 20
3420 3458 movl %esp, %ebp / | %ebp | 16
3421 3459 pushl %eax / | %eax | 12
3422 3460 pushl %ebx / | %ebx | 8
3423 3461 pushl %ecx / | %ecx | 4
3424 3462 pushl %edx / | %edx | 0
3425 3463
3426 3464 movl %esp, %ebx / %ebx = current stack pointer
3427 3465
3428 3466 lea panic_quiesce, %eax / %eax = &panic_quiesce
3429 3467 pushl %eax / push &panic_quiesce
3430 3468 call panic_trigger / %eax = panic_trigger()
3431 3469 addl $4, %esp / reset stack pointer
3432 3470
3433 3471 vpanic_common:
3434 3472 cmpl $0, %eax / if (%eax == 0)
3435 3473 je 0f / goto 0f;
3436 3474
3437 3475 /*
3438 3476 * If panic_trigger() was successful, we are the first to initiate a
3439 3477 * panic: we now switch to the reserved panic_stack before continuing.
3440 3478 */
3441 3479 lea panic_stack, %esp / %esp = panic_stack
3442 3480 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3443 3481
3444 3482 0: subl $REGSIZE, %esp / allocate struct regs
3445 3483
3446 3484 /*
3447 3485 * Now that we've got everything set up, store the register values as
3448 3486 * they were when we entered vpanic() to the designated location in
3449 3487 * the regs structure we allocated on the stack.
3450 3488 */
3451 3489 #if !defined(__GNUC_AS__)
3452 3490 movw %gs, %edx
3453 3491 movl %edx, REGOFF_GS(%esp)
3454 3492 movw %fs, %edx
3455 3493 movl %edx, REGOFF_FS(%esp)
3456 3494 movw %es, %edx
3457 3495 movl %edx, REGOFF_ES(%esp)
3458 3496 movw %ds, %edx
3459 3497 movl %edx, REGOFF_DS(%esp)
3460 3498 #else /* __GNUC_AS__ */
3461 3499 mov %gs, %edx
3462 3500 mov %edx, REGOFF_GS(%esp)
3463 3501 mov %fs, %edx
3464 3502 mov %edx, REGOFF_FS(%esp)
3465 3503 mov %es, %edx
3466 3504 mov %edx, REGOFF_ES(%esp)
3467 3505 mov %ds, %edx
3468 3506 mov %edx, REGOFF_DS(%esp)
3469 3507 #endif /* __GNUC_AS__ */
3470 3508 movl %edi, REGOFF_EDI(%esp)
3471 3509 movl %esi, REGOFF_ESI(%esp)
3472 3510 movl 16(%ebx), %ecx
3473 3511 movl %ecx, REGOFF_EBP(%esp)
3474 3512 movl %ebx, %ecx
3475 3513 addl $20, %ecx
3476 3514 movl %ecx, REGOFF_ESP(%esp)
3477 3515 movl 8(%ebx), %ecx
3478 3516 movl %ecx, REGOFF_EBX(%esp)
3479 3517 movl 0(%ebx), %ecx
3480 3518 movl %ecx, REGOFF_EDX(%esp)
3481 3519 movl 4(%ebx), %ecx
3482 3520 movl %ecx, REGOFF_ECX(%esp)
3483 3521 movl 12(%ebx), %ecx
3484 3522 movl %ecx, REGOFF_EAX(%esp)
3485 3523 movl $0, REGOFF_TRAPNO(%esp)
3486 3524 movl $0, REGOFF_ERR(%esp)
3487 3525 lea vpanic, %ecx
3488 3526 movl %ecx, REGOFF_EIP(%esp)
3489 3527 #if !defined(__GNUC_AS__)
3490 3528 movw %cs, %edx
3491 3529 #else /* __GNUC_AS__ */
3492 3530 mov %cs, %edx
3493 3531 #endif /* __GNUC_AS__ */
3494 3532 movl %edx, REGOFF_CS(%esp)
3495 3533 pushfl
3496 3534 popl %ecx
3497 3535 #if defined(__xpv)
3498 3536 /*
3499 3537 * Synthesize the PS_IE bit from the event mask bit
3500 3538 */
3501 3539 CURTHREAD(%edx)
3502 3540 KPREEMPT_DISABLE(%edx)
3503 3541 EVENT_MASK_TO_IE(%edx, %ecx)
3504 3542 CURTHREAD(%edx)
3505 3543 KPREEMPT_ENABLE_NOKP(%edx)
3506 3544 #endif
3507 3545 movl %ecx, REGOFF_EFL(%esp)
3508 3546 movl $0, REGOFF_UESP(%esp)
3509 3547 #if !defined(__GNUC_AS__)
3510 3548 movw %ss, %edx
3511 3549 #else /* __GNUC_AS__ */
3512 3550 mov %ss, %edx
3513 3551 #endif /* __GNUC_AS__ */
3514 3552 movl %edx, REGOFF_SS(%esp)
3515 3553
3516 3554 movl %esp, %ecx / %ecx = ®s
3517 3555 pushl %eax / push on_panic_stack
3518 3556 pushl %ecx / push ®s
3519 3557 movl 12(%ebp), %ecx / %ecx = alist
3520 3558 pushl %ecx / push alist
3521 3559 movl 8(%ebp), %ecx / %ecx = format
3522 3560 pushl %ecx / push format
3523 3561 call panicsys / panicsys();
3524 3562 addl $16, %esp / pop arguments
3525 3563
3526 3564 addl $REGSIZE, %esp
3527 3565 popl %edx
3528 3566 popl %ecx
3529 3567 popl %ebx
3530 3568 popl %eax
3531 3569 leave
3532 3570 ret
3533 3571 SET_SIZE(vpanic)
3534 3572
3535 3573 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3536 3574
3537 3575 pushl %ebp / | %eip | 20
3538 3576 movl %esp, %ebp / | %ebp | 16
3539 3577 pushl %eax / | %eax | 12
3540 3578 pushl %ebx / | %ebx | 8
3541 3579 pushl %ecx / | %ecx | 4
3542 3580 pushl %edx / | %edx | 0
3543 3581
3544 3582 movl %esp, %ebx / %ebx = current stack pointer
3545 3583
3546 3584 lea panic_quiesce, %eax / %eax = &panic_quiesce
3547 3585 pushl %eax / push &panic_quiesce
3548 3586 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3549 3587 addl $4, %esp / reset stack pointer
3550 3588 jmp vpanic_common / jump back to common code
3551 3589
3552 3590 SET_SIZE(dtrace_vpanic)
3553 3591
3554 3592 #endif /* __i386 */
3555 3593 #endif /* __lint */
3556 3594
3557 3595 #if defined(__lint)
3558 3596
3559 3597 void
3560 3598 hres_tick(void)
3561 3599 {}
3562 3600
3563 3601 int64_t timedelta;
3564 3602 hrtime_t hres_last_tick;
3565 3603 volatile timestruc_t hrestime;
3566 3604 int64_t hrestime_adj;
3567 3605 volatile int hres_lock;
3568 3606 hrtime_t hrtime_base;
3569 3607
3570 3608 #else /* __lint */
3571 3609
3572 3610 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8)
3573 3611 .NWORD 0, 0
3574 3612
3575 3613 DGDEF3(hrestime_adj, 8, 8)
3576 3614 .long 0, 0
3577 3615
3578 3616 DGDEF3(hres_last_tick, 8, 8)
3579 3617 .long 0, 0
3580 3618
3581 3619 DGDEF3(timedelta, 8, 8)
3582 3620 .long 0, 0
3583 3621
3584 3622 DGDEF3(hres_lock, 4, 8)
3585 3623 .long 0
3586 3624
3587 3625 /*
3588 3626 * initialized to a non zero value to make pc_gethrtime()
3589 3627 * work correctly even before clock is initialized
3590 3628 */
3591 3629 DGDEF3(hrtime_base, 8, 8)
3592 3630 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3593 3631
3594 3632 DGDEF3(adj_shift, 4, 4)
3595 3633 .long ADJ_SHIFT
3596 3634
3597 3635 #if defined(__amd64)
3598 3636
3599 3637 ENTRY_NP(hres_tick)
3600 3638 pushq %rbp
3601 3639 movq %rsp, %rbp
3602 3640
3603 3641 /*
3604 3642 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3605 3643 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3606 3644 * At worst, performing this now instead of under CLOCK_LOCK may
3607 3645 * introduce some jitter in pc_gethrestime().
3608 3646 */
3609 3647 call *gethrtimef(%rip)
3610 3648 movq %rax, %r8
3611 3649
3612 3650 leaq hres_lock(%rip), %rax
3613 3651 movb $-1, %dl
3614 3652 .CL1:
3615 3653 xchgb %dl, (%rax)
3616 3654 testb %dl, %dl
3617 3655 jz .CL3 /* got it */
3618 3656 .CL2:
3619 3657 cmpb $0, (%rax) /* possible to get lock? */
3620 3658 pause
3621 3659 jne .CL2
3622 3660 jmp .CL1 /* yes, try again */
3623 3661 .CL3:
3624 3662 /*
3625 3663 * compute the interval since last time hres_tick was called
3626 3664 * and adjust hrtime_base and hrestime accordingly
3627 3665 * hrtime_base is an 8 byte value (in nsec), hrestime is
3628 3666 * a timestruc_t (sec, nsec)
3629 3667 */
3630 3668 leaq hres_last_tick(%rip), %rax
3631 3669 movq %r8, %r11
3632 3670 subq (%rax), %r8
3633 3671 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3634 3672 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3635 3673 /*
3636 3674 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3637 3675 */
3638 3676 movq %r11, (%rax)
3639 3677
3640 3678 call __adj_hrestime
3641 3679
3642 3680 /*
3643 3681 * release the hres_lock
3644 3682 */
3645 3683 incl hres_lock(%rip)
3646 3684 leave
3647 3685 ret
3648 3686 SET_SIZE(hres_tick)
3649 3687
3650 3688 #elif defined(__i386)
3651 3689
3652 3690 ENTRY_NP(hres_tick)
3653 3691 pushl %ebp
3654 3692 movl %esp, %ebp
3655 3693 pushl %esi
3656 3694 pushl %ebx
3657 3695
3658 3696 /*
3659 3697 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3660 3698 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3661 3699 * At worst, performing this now instead of under CLOCK_LOCK may
3662 3700 * introduce some jitter in pc_gethrestime().
3663 3701 */
3664 3702 call *gethrtimef
3665 3703 movl %eax, %ebx
3666 3704 movl %edx, %esi
3667 3705
3668 3706 movl $hres_lock, %eax
3669 3707 movl $-1, %edx
3670 3708 .CL1:
3671 3709 xchgb %dl, (%eax)
3672 3710 testb %dl, %dl
3673 3711 jz .CL3 / got it
3674 3712 .CL2:
3675 3713 cmpb $0, (%eax) / possible to get lock?
3676 3714 pause
3677 3715 jne .CL2
3678 3716 jmp .CL1 / yes, try again
3679 3717 .CL3:
3680 3718 /*
3681 3719 * compute the interval since last time hres_tick was called
3682 3720 * and adjust hrtime_base and hrestime accordingly
3683 3721 * hrtime_base is an 8 byte value (in nsec), hrestime is
3684 3722 * timestruc_t (sec, nsec)
3685 3723 */
3686 3724
3687 3725 lea hres_last_tick, %eax
3688 3726
3689 3727 movl %ebx, %edx
3690 3728 movl %esi, %ecx
3691 3729
3692 3730 subl (%eax), %edx
3693 3731 sbbl 4(%eax), %ecx
3694 3732
3695 3733 addl %edx, hrtime_base / add interval to hrtime_base
3696 3734 adcl %ecx, hrtime_base+4
3697 3735
3698 3736 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3699 3737
3700 3738 /
3701 3739 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3702 3740 /
3703 3741 movl %ebx, (%eax)
3704 3742 movl %esi, 4(%eax)
3705 3743
3706 3744 / get hrestime at this moment. used as base for pc_gethrestime
3707 3745 /
3708 3746 / Apply adjustment, if any
3709 3747 /
3710 3748 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3711 3749 / (max_hres_adj)
3712 3750 /
3713 3751 / void
3714 3752 / adj_hrestime()
3715 3753 / {
3716 3754 / long long adj;
3717 3755 /
3718 3756 / if (hrestime_adj == 0)
3719 3757 / adj = 0;
3720 3758 / else if (hrestime_adj > 0) {
3721 3759 / if (hrestime_adj < HRES_ADJ)
3722 3760 / adj = hrestime_adj;
3723 3761 / else
3724 3762 / adj = HRES_ADJ;
3725 3763 / }
3726 3764 / else {
3727 3765 / if (hrestime_adj < -(HRES_ADJ))
3728 3766 / adj = -(HRES_ADJ);
3729 3767 / else
3730 3768 / adj = hrestime_adj;
3731 3769 / }
3732 3770 /
3733 3771 / timedelta -= adj;
3734 3772 / hrestime_adj = timedelta;
3735 3773 / hrestime.tv_nsec += adj;
3736 3774 /
3737 3775 / while (hrestime.tv_nsec >= NANOSEC) {
3738 3776 / one_sec++;
3739 3777 / hrestime.tv_sec++;
3740 3778 / hrestime.tv_nsec -= NANOSEC;
3741 3779 / }
3742 3780 / }
3743 3781 __adj_hrestime:
3744 3782 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3745 3783 movl hrestime_adj+4, %edx
3746 3784 andl %esi, %esi
3747 3785 jne .CL4 / no
3748 3786 andl %edx, %edx
3749 3787 jne .CL4 / no
3750 3788 subl %ecx, %ecx / yes, adj = 0;
3751 3789 subl %edx, %edx
3752 3790 jmp .CL5
3753 3791 .CL4:
3754 3792 subl %ecx, %ecx
3755 3793 subl %eax, %eax
3756 3794 subl %esi, %ecx
3757 3795 sbbl %edx, %eax
3758 3796 andl %eax, %eax / if (hrestime_adj > 0)
3759 3797 jge .CL6
3760 3798
3761 3799 / In the following comments, HRES_ADJ is used, while in the code
3762 3800 / max_hres_adj is used.
3763 3801 /
3764 3802 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3765 3803 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3766 3804 / on the logical equivalence of:
3767 3805 /
3768 3806 / !(hrestime_adj < HRES_ADJ)
3769 3807 /
3770 3808 / and the two step sequence:
3771 3809 /
3772 3810 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3773 3811 /
3774 3812 / which computes whether or not the least significant 32-bits
3775 3813 / of hrestime_adj is greater than HRES_ADJ, followed by:
3776 3814 /
3777 3815 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3778 3816 /
3779 3817 / which generates a carry whenever step 1 is true or the most
3780 3818 / significant long of the longlong hrestime_adj is non-zero.
3781 3819
3782 3820 movl max_hres_adj, %ecx / hrestime_adj is positive
3783 3821 subl %esi, %ecx
3784 3822 movl %edx, %eax
3785 3823 adcl $-1, %eax
3786 3824 jnc .CL7
3787 3825 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3788 3826 subl %edx, %edx
3789 3827 jmp .CL5
3790 3828
3791 3829 / The following computation is similar to the one above.
3792 3830 /
3793 3831 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3794 3832 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3795 3833 / on the logical equivalence of:
3796 3834 /
3797 3835 / (hrestime_adj > -HRES_ADJ)
3798 3836 /
3799 3837 / and the two step sequence:
3800 3838 /
3801 3839 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3802 3840 /
3803 3841 / which means the least significant 32-bits of hrestime_adj is
3804 3842 / greater than -HRES_ADJ, followed by:
3805 3843 /
3806 3844 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3807 3845 /
3808 3846 / which generates a carry only when step 1 is true and the most
3809 3847 / significant long of the longlong hrestime_adj is -1.
3810 3848
3811 3849 .CL6: / hrestime_adj is negative
3812 3850 movl %esi, %ecx
3813 3851 addl max_hres_adj, %ecx
3814 3852 movl %edx, %eax
3815 3853 adcl $0, %eax
3816 3854 jc .CL7
3817 3855 xor %ecx, %ecx
3818 3856 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3819 3857 movl $-1, %edx
3820 3858 jmp .CL5
3821 3859 .CL7:
3822 3860 movl %esi, %ecx / adj = hrestime_adj;
3823 3861 .CL5:
3824 3862 movl timedelta, %esi
3825 3863 subl %ecx, %esi
3826 3864 movl timedelta+4, %eax
3827 3865 sbbl %edx, %eax
3828 3866 movl %esi, timedelta
3829 3867 movl %eax, timedelta+4 / timedelta -= adj;
3830 3868 movl %esi, hrestime_adj
3831 3869 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3832 3870 addl hrestime+4, %ecx
3833 3871
3834 3872 movl %ecx, %eax / eax = tv_nsec
3835 3873 1:
3836 3874 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3837 3875 jb .CL8 / no
3838 3876 incl one_sec / yes, one_sec++;
3839 3877 incl hrestime / hrestime.tv_sec++;
3840 3878 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3841 3879 jmp 1b / check for more seconds
3842 3880
3843 3881 .CL8:
3844 3882 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3845 3883 incl hres_lock / release the hres_lock
3846 3884
3847 3885 popl %ebx
3848 3886 popl %esi
3849 3887 leave
3850 3888 ret
3851 3889 SET_SIZE(hres_tick)
3852 3890
3853 3891 #endif /* __i386 */
3854 3892 #endif /* __lint */
3855 3893
3856 3894 /*
3857 3895 * void prefetch_smap_w(void *)
3858 3896 *
3859 3897 * Prefetch ahead within a linear list of smap structures.
3860 3898 * Not implemented for ia32. Stub for compatibility.
3861 3899 */
3862 3900
3863 3901 #if defined(__lint)
3864 3902
3865 3903 /*ARGSUSED*/
3866 3904 void prefetch_smap_w(void *smp)
3867 3905 {}
3868 3906
3869 3907 #else /* __lint */
3870 3908
3871 3909 ENTRY(prefetch_smap_w)
3872 3910 rep; ret /* use 2 byte return instruction when branch target */
3873 3911 /* AMD Software Optimization Guide - Section 6.2 */
3874 3912 SET_SIZE(prefetch_smap_w)
3875 3913
3876 3914 #endif /* __lint */
3877 3915
3878 3916 /*
3879 3917 * prefetch_page_r(page_t *)
3880 3918 * issue prefetch instructions for a page_t
3881 3919 */
3882 3920 #if defined(__lint)
3883 3921
3884 3922 /*ARGSUSED*/
3885 3923 void
3886 3924 prefetch_page_r(void *pp)
3887 3925 {}
3888 3926
3889 3927 #else /* __lint */
3890 3928
3891 3929 ENTRY(prefetch_page_r)
3892 3930 rep; ret /* use 2 byte return instruction when branch target */
3893 3931 /* AMD Software Optimization Guide - Section 6.2 */
3894 3932 SET_SIZE(prefetch_page_r)
3895 3933
3896 3934 #endif /* __lint */
3897 3935
3898 3936 #if defined(__lint)
3899 3937
3900 3938 /*ARGSUSED*/
3901 3939 int
3902 3940 bcmp(const void *s1, const void *s2, size_t count)
3903 3941 { return (0); }
3904 3942
3905 3943 #else /* __lint */
3906 3944
3907 3945 #if defined(__amd64)
3908 3946
3909 3947 ENTRY(bcmp)
3910 3948 pushq %rbp
3911 3949 movq %rsp, %rbp
3912 3950 #ifdef DEBUG
3913 3951 testq %rdx,%rdx
3914 3952 je 1f
3915 3953 movq postbootkernelbase(%rip), %r11
3916 3954 cmpq %r11, %rdi
3917 3955 jb 0f
3918 3956 cmpq %r11, %rsi
3919 3957 jnb 1f
3920 3958 0: leaq .bcmp_panic_msg(%rip), %rdi
3921 3959 xorl %eax, %eax
3922 3960 call panic
3923 3961 1:
3924 3962 #endif /* DEBUG */
3925 3963 call memcmp
3926 3964 testl %eax, %eax
3927 3965 setne %dl
3928 3966 leave
3929 3967 movzbl %dl, %eax
3930 3968 ret
3931 3969 SET_SIZE(bcmp)
3932 3970
3933 3971 #elif defined(__i386)
3934 3972
3935 3973 #define ARG_S1 8
3936 3974 #define ARG_S2 12
3937 3975 #define ARG_LENGTH 16
3938 3976
3939 3977 ENTRY(bcmp)
3940 3978 pushl %ebp
3941 3979 movl %esp, %ebp / create new stack frame
3942 3980 #ifdef DEBUG
3943 3981 cmpl $0, ARG_LENGTH(%ebp)
3944 3982 je 1f
3945 3983 movl postbootkernelbase, %eax
3946 3984 cmpl %eax, ARG_S1(%ebp)
3947 3985 jb 0f
3948 3986 cmpl %eax, ARG_S2(%ebp)
3949 3987 jnb 1f
3950 3988 0: pushl $.bcmp_panic_msg
3951 3989 call panic
3952 3990 1:
3953 3991 #endif /* DEBUG */
3954 3992
3955 3993 pushl %edi / save register variable
3956 3994 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3957 3995 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3958 3996 cmpl %eax, %ecx / if the same string
3959 3997 je .equal / goto .equal
3960 3998 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3961 3999 cmpl $4, %edi / if %edi < 4
3962 4000 jb .byte_check / goto .byte_check
3963 4001 .align 4
3964 4002 .word_loop:
3965 4003 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3966 4004 leal -4(%edi), %edi / %edi -= 4
3967 4005 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3968 4006 jne .word_not_equal / if not equal, goto .word_not_equal
3969 4007 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3970 4008 leal 4(%eax), %eax / %eax += 4 (next word)
3971 4009 cmpl $4, %edi / if %edi >= 4
3972 4010 jae .word_loop / goto .word_loop
3973 4011 .byte_check:
3974 4012 cmpl $0, %edi / if %edi == 0
3975 4013 je .equal / goto .equal
3976 4014 jmp .byte_loop / goto .byte_loop (checks in bytes)
3977 4015 .word_not_equal:
3978 4016 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3979 4017 .align 4
3980 4018 .byte_loop:
3981 4019 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3982 4020 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3983 4021 jne .not_equal / if not equal, goto .not_equal
3984 4022 incl %ecx / %ecx++ (next byte)
3985 4023 incl %eax / %eax++ (next byte)
3986 4024 decl %edi / %edi--
3987 4025 jnz .byte_loop / if not zero, goto .byte_loop
3988 4026 .equal:
3989 4027 xorl %eax, %eax / %eax = 0
3990 4028 popl %edi / restore register variable
3991 4029 leave / restore old stack frame
3992 4030 ret / return (NULL)
3993 4031 .align 4
3994 4032 .not_equal:
3995 4033 movl $1, %eax / return 1
3996 4034 popl %edi / restore register variable
3997 4035 leave / restore old stack frame
3998 4036 ret / return (NULL)
3999 4037 SET_SIZE(bcmp)
4000 4038
4001 4039 #endif /* __i386 */
4002 4040
4003 4041 #ifdef DEBUG
4004 4042 .text
4005 4043 .bcmp_panic_msg:
4006 4044 .string "bcmp: arguments below kernelbase"
4007 4045 #endif /* DEBUG */
4008 4046
4009 4047 #endif /* __lint */
4010 4048
4011 4049 #if defined(__lint)
4012 4050
4013 4051 uint_t
4014 4052 bsrw_insn(uint16_t mask)
4015 4053 {
4016 4054 uint_t index = sizeof (mask) * NBBY - 1;
4017 4055
4018 4056 while ((mask & (1 << index)) == 0)
4019 4057 index--;
4020 4058 return (index);
4021 4059 }
4022 4060
4023 4061 #else /* __lint */
4024 4062
4025 4063 #if defined(__amd64)
4026 4064
4027 4065 ENTRY_NP(bsrw_insn)
4028 4066 xorl %eax, %eax
4029 4067 bsrw %di, %ax
4030 4068 ret
4031 4069 SET_SIZE(bsrw_insn)
4032 4070
4033 4071 #elif defined(__i386)
4034 4072
4035 4073 ENTRY_NP(bsrw_insn)
4036 4074 movw 4(%esp), %cx
4037 4075 xorl %eax, %eax
4038 4076 bsrw %cx, %ax
4039 4077 ret
4040 4078 SET_SIZE(bsrw_insn)
4041 4079
4042 4080 #endif /* __i386 */
4043 4081 #endif /* __lint */
4044 4082
4045 4083 #if defined(__lint)
4046 4084
4047 4085 uint_t
4048 4086 atomic_btr32(uint32_t *pending, uint_t pil)
4049 4087 {
4050 4088 return (*pending &= ~(1 << pil));
4051 4089 }
4052 4090
4053 4091 #else /* __lint */
4054 4092
4055 4093 #if defined(__i386)
4056 4094
4057 4095 ENTRY_NP(atomic_btr32)
4058 4096 movl 4(%esp), %ecx
4059 4097 movl 8(%esp), %edx
4060 4098 xorl %eax, %eax
4061 4099 lock
4062 4100 btrl %edx, (%ecx)
4063 4101 setc %al
4064 4102 ret
4065 4103 SET_SIZE(atomic_btr32)
4066 4104
4067 4105 #endif /* __i386 */
4068 4106 #endif /* __lint */
4069 4107
4070 4108 #if defined(__lint)
4071 4109
4072 4110 /*ARGSUSED*/
4073 4111 void
4074 4112 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4075 4113 uint_t arg2)
4076 4114 {}
4077 4115
4078 4116 #else /* __lint */
4079 4117
4080 4118 #if defined(__amd64)
4081 4119
4082 4120 ENTRY_NP(switch_sp_and_call)
4083 4121 pushq %rbp
4084 4122 movq %rsp, %rbp /* set up stack frame */
4085 4123 movq %rdi, %rsp /* switch stack pointer */
4086 4124 movq %rdx, %rdi /* pass func arg 1 */
4087 4125 movq %rsi, %r11 /* save function to call */
4088 4126 movq %rcx, %rsi /* pass func arg 2 */
4089 4127 call *%r11 /* call function */
4090 4128 leave /* restore stack */
4091 4129 ret
4092 4130 SET_SIZE(switch_sp_and_call)
4093 4131
4094 4132 #elif defined(__i386)
4095 4133
4096 4134 ENTRY_NP(switch_sp_and_call)
4097 4135 pushl %ebp
4098 4136 mov %esp, %ebp /* set up stack frame */
4099 4137 movl 8(%ebp), %esp /* switch stack pointer */
4100 4138 pushl 20(%ebp) /* push func arg 2 */
4101 4139 pushl 16(%ebp) /* push func arg 1 */
4102 4140 call *12(%ebp) /* call function */
4103 4141 addl $8, %esp /* pop arguments */
4104 4142 leave /* restore stack */
4105 4143 ret
4106 4144 SET_SIZE(switch_sp_and_call)
4107 4145
4108 4146 #endif /* __i386 */
4109 4147 #endif /* __lint */
4110 4148
4111 4149 #if defined(__lint)
4112 4150
4113 4151 void
4114 4152 kmdb_enter(void)
4115 4153 {}
4116 4154
4117 4155 #else /* __lint */
4118 4156
4119 4157 #if defined(__amd64)
4120 4158
4121 4159 ENTRY_NP(kmdb_enter)
4122 4160 pushq %rbp
4123 4161 movq %rsp, %rbp
4124 4162
4125 4163 /*
4126 4164 * Save flags, do a 'cli' then return the saved flags
4127 4165 */
4128 4166 call intr_clear
4129 4167
4130 4168 int $T_DBGENTR
4131 4169
4132 4170 /*
4133 4171 * Restore the saved flags
4134 4172 */
4135 4173 movq %rax, %rdi
4136 4174 call intr_restore
4137 4175
4138 4176 leave
4139 4177 ret
4140 4178 SET_SIZE(kmdb_enter)
4141 4179
4142 4180 #elif defined(__i386)
4143 4181
4144 4182 ENTRY_NP(kmdb_enter)
4145 4183 pushl %ebp
4146 4184 movl %esp, %ebp
4147 4185
4148 4186 /*
4149 4187 * Save flags, do a 'cli' then return the saved flags
4150 4188 */
4151 4189 call intr_clear
4152 4190
4153 4191 int $T_DBGENTR
4154 4192
4155 4193 /*
4156 4194 * Restore the saved flags
4157 4195 */
4158 4196 pushl %eax
4159 4197 call intr_restore
4160 4198 addl $4, %esp
4161 4199
4162 4200 leave
4163 4201 ret
4164 4202 SET_SIZE(kmdb_enter)
4165 4203
4166 4204 #endif /* __i386 */
4167 4205 #endif /* __lint */
4168 4206
4169 4207 #if defined(__lint)
4170 4208
4171 4209 void
4172 4210 return_instr(void)
4173 4211 {}
4174 4212
4175 4213 #else /* __lint */
4176 4214
4177 4215 ENTRY_NP(return_instr)
4178 4216 rep; ret /* use 2 byte instruction when branch target */
4179 4217 /* AMD Software Optimization Guide - Section 6.2 */
4180 4218 SET_SIZE(return_instr)
4181 4219
4182 4220 #endif /* __lint */
4183 4221
4184 4222 #if defined(__lint)
4185 4223
4186 4224 ulong_t
4187 4225 getflags(void)
4188 4226 {
4189 4227 return (0);
4190 4228 }
4191 4229
4192 4230 #else /* __lint */
4193 4231
4194 4232 #if defined(__amd64)
4195 4233
4196 4234 ENTRY(getflags)
4197 4235 pushfq
4198 4236 popq %rax
4199 4237 #if defined(__xpv)
4200 4238 CURTHREAD(%rdi)
4201 4239 KPREEMPT_DISABLE(%rdi)
4202 4240 /*
4203 4241 * Synthesize the PS_IE bit from the event mask bit
4204 4242 */
4205 4243 CURVCPU(%r11)
4206 4244 andq $_BITNOT(PS_IE), %rax
4207 4245 XEN_TEST_UPCALL_MASK(%r11)
4208 4246 jnz 1f
4209 4247 orq $PS_IE, %rax
4210 4248 1:
4211 4249 KPREEMPT_ENABLE_NOKP(%rdi)
4212 4250 #endif
4213 4251 ret
4214 4252 SET_SIZE(getflags)
4215 4253
4216 4254 #elif defined(__i386)
4217 4255
4218 4256 ENTRY(getflags)
4219 4257 pushfl
4220 4258 popl %eax
4221 4259 #if defined(__xpv)
4222 4260 CURTHREAD(%ecx)
4223 4261 KPREEMPT_DISABLE(%ecx)
4224 4262 /*
4225 4263 * Synthesize the PS_IE bit from the event mask bit
4226 4264 */
4227 4265 CURVCPU(%edx)
4228 4266 andl $_BITNOT(PS_IE), %eax
4229 4267 XEN_TEST_UPCALL_MASK(%edx)
4230 4268 jnz 1f
4231 4269 orl $PS_IE, %eax
4232 4270 1:
4233 4271 KPREEMPT_ENABLE_NOKP(%ecx)
4234 4272 #endif
4235 4273 ret
4236 4274 SET_SIZE(getflags)
4237 4275
4238 4276 #endif /* __i386 */
4239 4277
4240 4278 #endif /* __lint */
4241 4279
4242 4280 #if defined(__lint)
4243 4281
4244 4282 ftrace_icookie_t
4245 4283 ftrace_interrupt_disable(void)
4246 4284 { return (0); }
4247 4285
4248 4286 #else /* __lint */
4249 4287
4250 4288 #if defined(__amd64)
4251 4289
4252 4290 ENTRY(ftrace_interrupt_disable)
4253 4291 pushfq
4254 4292 popq %rax
4255 4293 CLI(%rdx)
4256 4294 ret
4257 4295 SET_SIZE(ftrace_interrupt_disable)
4258 4296
4259 4297 #elif defined(__i386)
4260 4298
4261 4299 ENTRY(ftrace_interrupt_disable)
4262 4300 pushfl
4263 4301 popl %eax
4264 4302 CLI(%edx)
4265 4303 ret
4266 4304 SET_SIZE(ftrace_interrupt_disable)
4267 4305
4268 4306 #endif /* __i386 */
4269 4307 #endif /* __lint */
4270 4308
4271 4309 #if defined(__lint)
4272 4310
4273 4311 /*ARGSUSED*/
4274 4312 void
4275 4313 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4276 4314 {}
4277 4315
4278 4316 #else /* __lint */
4279 4317
4280 4318 #if defined(__amd64)
4281 4319
4282 4320 ENTRY(ftrace_interrupt_enable)
4283 4321 pushq %rdi
4284 4322 popfq
4285 4323 ret
4286 4324 SET_SIZE(ftrace_interrupt_enable)
4287 4325
4288 4326 #elif defined(__i386)
4289 4327
4290 4328 ENTRY(ftrace_interrupt_enable)
4291 4329 movl 4(%esp), %eax
4292 4330 pushl %eax
4293 4331 popfl
4294 4332 ret
4295 4333 SET_SIZE(ftrace_interrupt_enable)
4296 4334
4297 4335 #endif /* __i386 */
4298 4336 #endif /* __lint */
4299 4337
4300 4338 #if defined (__lint)
4301 4339
4302 4340 /*ARGSUSED*/
4303 4341 void
4304 4342 clflush_insn(caddr_t addr)
4305 4343 {}
4306 4344
4307 4345 #else /* __lint */
4308 4346
4309 4347 #if defined (__amd64)
4310 4348 ENTRY(clflush_insn)
4311 4349 clflush (%rdi)
4312 4350 ret
4313 4351 SET_SIZE(clflush_insn)
4314 4352 #elif defined (__i386)
4315 4353 ENTRY(clflush_insn)
4316 4354 movl 4(%esp), %eax
4317 4355 clflush (%eax)
4318 4356 ret
4319 4357 SET_SIZE(clflush_insn)
4320 4358
4321 4359 #endif /* __i386 */
4322 4360 #endif /* __lint */
4323 4361
4324 4362 #if defined (__lint)
4325 4363 /*ARGSUSED*/
4326 4364 void
4327 4365 mfence_insn(void)
4328 4366 {}
4329 4367
4330 4368 #else /* __lint */
4331 4369
4332 4370 #if defined (__amd64)
4333 4371 ENTRY(mfence_insn)
4334 4372 mfence
4335 4373 ret
4336 4374 SET_SIZE(mfence_insn)
4337 4375 #elif defined (__i386)
4338 4376 ENTRY(mfence_insn)
4339 4377 mfence
4340 4378 ret
4341 4379 SET_SIZE(mfence_insn)
4342 4380
4343 4381 #endif /* __i386 */
4344 4382 #endif /* __lint */
4345 4383
4346 4384 /*
4347 4385 * VMware implements an I/O port that programs can query to detect if software
4348 4386 * is running in a VMware hypervisor. This hypervisor port behaves differently
4349 4387 * depending on magic values in certain registers and modifies some registers
4350 4388 * as a side effect.
4351 4389 *
4352 4390 * References: http://kb.vmware.com/kb/1009458
4353 4391 */
4354 4392
4355 4393 #if defined(__lint)
4356 4394
4357 4395 /* ARGSUSED */
4358 4396 void
4359 4397 vmware_port(int cmd, uint32_t *regs) { return; }
4360 4398
4361 4399 #else
4362 4400
4363 4401 #if defined(__amd64)
4364 4402
4365 4403 ENTRY(vmware_port)
4366 4404 pushq %rbx
4367 4405 movl $VMWARE_HVMAGIC, %eax
4368 4406 movl $0xffffffff, %ebx
4369 4407 movl %edi, %ecx
4370 4408 movl $VMWARE_HVPORT, %edx
4371 4409 inl (%dx)
4372 4410 movl %eax, (%rsi)
4373 4411 movl %ebx, 4(%rsi)
4374 4412 movl %ecx, 8(%rsi)
4375 4413 movl %edx, 12(%rsi)
4376 4414 popq %rbx
4377 4415 ret
4378 4416 SET_SIZE(vmware_port)
4379 4417
4380 4418 #elif defined(__i386)
4381 4419
4382 4420 ENTRY(vmware_port)
4383 4421 pushl %ebx
4384 4422 pushl %esi
4385 4423 movl $VMWARE_HVMAGIC, %eax
4386 4424 movl $0xffffffff, %ebx
4387 4425 movl 12(%esp), %ecx
4388 4426 movl $VMWARE_HVPORT, %edx
4389 4427 inl (%dx)
4390 4428 movl 16(%esp), %esi
4391 4429 movl %eax, (%esi)
4392 4430 movl %ebx, 4(%esi)
4393 4431 movl %ecx, 8(%esi)
4394 4432 movl %edx, 12(%esi)
4395 4433 popl %esi
4396 4434 popl %ebx
4397 4435 ret
4398 4436 SET_SIZE(vmware_port)
4399 4437
4400 4438 #endif /* __i386 */
4401 4439 #endif /* __lint */
|
↓ open down ↓ |
1544 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX