1 /*
   2  * This file and its contents are supplied under the terms of the
   3  * Common Development and Distribution License ("CDDL"), version 1.0.
   4  * You may only use this file in accordance with the terms of version
   5  * 1.0 of the CDDL.
   6  *
   7  * A full copy of the text of the CDDL should have accompanied this
   8  * source.  A copy of the CDDL is also available via the Internet at
   9  * http://www.illumos.org/license/CDDL.
  10  */
  11 
  12 /*
  13  * Copyright 2016 Joyent, Inc.
  14  */
  15 
  16 #include <sys/asm_linkage.h>
  17 #include <sys/segments.h>
  18 #include <sys/time_impl.h>
  19 #include <sys/tsc.h>
  20 #include <cp_offsets.h>
  21 
  22 #define GETCPU_GDT_OFFSET       SEL_GDT(GDT_CPUID, SEL_UPL)
  23 
  24         .file   "cp_subr.s"
  25 
  26 /*
  27  * hrtime_t
  28  * __cp_tsc_read(uint_t cp_tsc_type)
  29  *
  30  * Stack usage: 0x18 bytes
  31  */
  32         ENTRY_NP(__cp_tsc_read)
  33         pushl   %ebp
  34         movl    %esp, %ebp
  35         pushl   %edi
  36         pushl   %esi
  37         subl    $0x4, %esp
  38 
  39         movl    0x8(%ebp), %edi
  40         movl    CP_TSC_TYPE(%edi), %eax
  41         movl    CP_TSC_NCPU(%edi), %esi
  42         cmpl    $TSC_TSCP, %eax
  43         jne     3f
  44         rdtscp
  45         cmpl    $0, %esi
  46         jne     2f
  47 1:
  48         addl    $0x4, %esp
  49         popl    %esi
  50         popl    %edi
  51         leave
  52         ret
  53 2:
  54         /*
  55          * When cp_tsc_ncpu is non-zero, it indicates the length of the
  56          * cp_tsc_sync_tick_delta array, which contains per-CPU offsets for the
  57          * TSC.  The CPU ID furnished by the IA32_TSC_AUX register via rdtscp
  58          * is used to look up an offset value in that array and apply it to the
  59          * TSC reading.
  60          */
  61         leal    CP_TSC_SYNC_TICK_DELTA(%edi), %esi
  62         leal    (%esi, %ecx, 8), %ecx
  63         addl    (%ecx), %eax
  64         adcl    0x4(%ecx), %edx
  65         jmp     1b
  66 
  67 3:
  68         cmpl    $0, %esi
  69         je      4f
  70         mov     $GETCPU_GDT_OFFSET, %eax
  71         lsl     %ax, %eax
  72         movl    %eax, (%esp)
  73         movl    CP_TSC_TYPE(%edi), %eax
  74 
  75 4:
  76         cmpl    $TSC_RDTSC_MFENCE, %eax
  77         jne     5f
  78         mfence
  79         rdtsc
  80         jmp     8f
  81 
  82 5:
  83         cmpl    $TSC_RDTSC_LFENCE, %eax
  84         jne     6f
  85         lfence
  86         rdtsc
  87         jmp     8f
  88 
  89 6:
  90         cmpl    $TSC_RDTSC_CPUID, %eax
  91         jne     7f
  92         pushl   %ebx
  93         xorl    %eax, %eax
  94         cpuid
  95         rdtsc
  96         popl    %ebx
  97         jmp     8f
  98 
  99 7:
 100         /*
 101          * Other protections should have prevented this function from being
 102          * called in the first place.  The only sane action is to abort.
 103          * The easiest means in this context is via SIGILL.
 104          */
 105         ud2a
 106 
 107 8:
 108 
 109         cmpl    $0, %esi
 110         je      1b
 111         /*
 112          * With a TSC reading in-hand, confirm that the thread has not migrated
 113          * since the cpu_id was first checked.
 114          */
 115         movl    $GETCPU_GDT_OFFSET, %ecx
 116         lsl     %cx, %ecx
 117         movl    (%esp), %esi
 118         cmpl    %ecx, %esi
 119         je      9f
 120         /*
 121          * There was a CPU migration, perform another reading.
 122          */
 123         movl    %eax, (%esp)
 124         movl    CP_TSC_NCPU(%edi), %esi
 125         movl    CP_TSC_TYPE(%edi), %eax
 126         jmp     4b
 127 
 128 9:
 129         /* Grab the per-cpu offset and add it to the TSC result */
 130         leal    CP_TSC_SYNC_TICK_DELTA(%edi), %esi
 131         leal    (%esi, %ecx, 8), %ecx
 132         addl    (%ecx), %eax
 133         adcl    0x4(%ecx), %edx
 134         jmp     1b
 135         SET_SIZE(__cp_tsc_read)
 136 
 137 /*
 138  * uint_t
 139  * __cp_getcpu(uint_t cp_tsc_type)
 140  */
 141         ENTRY_NP(__cp_getcpu)
 142         /*
 143          * If RDTSCP is available, it is a quick way to grab the cpu_id which
 144          * is stored in the TSC_AUX MSR by the kernel.
 145          */
 146         movl    4(%esp), %eax
 147         movl    CP_TSC_TYPE(%eax), %eax
 148         cmpl    $TSC_TSCP, %eax
 149         jne     1f
 150         rdtscp
 151         movl    %ecx, %eax
 152         ret
 153 1:
 154         mov     $GETCPU_GDT_OFFSET, %eax
 155         lsl     %ax, %eax
 156         ret
 157         SET_SIZE(__cp_getcpu)