--- old/usr/src/lib/commpage/i386/cp_subr.s
          +++ new/usr/src/lib/commpage/i386/cp_subr.s

   1    1  /*
   2    2   * This file and its contents are supplied under the terms of the
   3    3   * Common Development and Distribution License ("CDDL"), version 1.0.
   4    4   * You may only use this file in accordance with the terms of version
   5    5   * 1.0 of the CDDL.
   6    6   *
   7    7   * A full copy of the text of the CDDL should have accompanied this
   8    8   * source.  A copy of the CDDL is also available via the Internet at
   9    9   * http://www.illumos.org/license/CDDL.
  10   10   */
  11   11  
  12   12  /*
  13   13   * Copyright 2016 Joyent, Inc.
  14   14   */
  15   15  
  16   16  #include <sys/asm_linkage.h>
  17   17  #include <sys/segments.h>
  18   18  #include <sys/time_impl.h>
  19   19  #include <sys/tsc.h>
  20   20  #include <cp_offsets.h>
  21   21  
  22   22  #define GETCPU_GDT_OFFSET       SEL_GDT(GDT_CPUID, SEL_UPL)
  23   23  
  24   24          .file   "cp_subr.s"
  25   25  
  26   26  /*
  27   27   * hrtime_t
  28   28   * __cp_tsc_read(uint_t cp_tsc_type)
  29   29   *
  30   30   * Stack usage: 0x18 bytes
  31   31   */
  32   32          ENTRY_NP(__cp_tsc_read)
  33   33          pushl   %ebp
  34   34          movl    %esp, %ebp
  35   35          pushl   %edi
  36   36          pushl   %esi
  37   37          subl    $0x4, %esp
  38   38  
  39   39          movl    0x8(%ebp), %edi
  40   40          movl    CP_TSC_TYPE(%edi), %eax
  41   41          movl    CP_TSC_NCPU(%edi), %esi
  42   42          cmpl    $TSC_TSCP, %eax
  43   43          jne     3f
  44   44          rdtscp
  45   45          cmpl    $0, %esi
  46   46          jne     2f
  47   47  1:
  48   48          addl    $0x4, %esp
  49   49          popl    %esi
  50   50          popl    %edi
  51   51          leave
  52   52          ret
  53   53  2:
  54   54          /*
  55   55           * When cp_tsc_ncpu is non-zero, it indicates the length of the
  56   56           * cp_tsc_sync_tick_delta array, which contains per-CPU offsets for the
  57   57           * TSC.  The CPU ID furnished by the IA32_TSC_AUX register via rdtscp
  58   58           * is used to look up an offset value in that array and apply it to the
  59   59           * TSC reading.
  60   60           */
  61   61          leal    CP_TSC_SYNC_TICK_DELTA(%edi), %esi
  62   62          leal    (%esi, %ecx, 8), %ecx
  63   63          addl    (%ecx), %eax
  64   64          adcl    0x4(%ecx), %edx
  65   65          jmp     1b
  66   66  
  67   67  3:
  68   68          cmpl    $0, %esi
  69   69          je      4f
  70   70          mov     $GETCPU_GDT_OFFSET, %eax
  71   71          lsl     %ax, %eax
  72   72          movl    %eax, (%esp)
  73   73          movl    CP_TSC_TYPE(%edi), %eax
  74   74  
  75   75  4:
  76   76          cmpl    $TSC_RDTSC_MFENCE, %eax
  77   77          jne     5f
  78   78          mfence
  79   79          rdtsc
  80   80          jmp     8f
  81   81  
  82   82  5:
  83   83          cmpl    $TSC_RDTSC_LFENCE, %eax
  84   84          jne     6f
  85   85          lfence
  86   86          rdtsc
  87   87          jmp     8f
  88   88  
  89   89  6:
  90   90          cmpl    $TSC_RDTSC_CPUID, %eax
  91   91          jne     7f
  92   92          pushl   %ebx
  93   93          xorl    %eax, %eax
  94   94          cpuid
  95   95          rdtsc
  96   96          popl    %ebx
  97   97          jmp     8f
  98   98  
  99   99  7:
 100  100          /*
 101  101           * Other protections should have prevented this function from being
 102  102           * called in the first place.  The only sane action is to abort.
 103  103           * The easiest means in this context is via SIGILL.
 104  104           */
 105  105          ud2a
 106  106  
 107  107  8:
 108  108  
 109  109          cmpl    $0, %esi
 110  110          je      1b
 111  111          /*
 112  112           * With a TSC reading in-hand, confirm that the thread has not migrated
 113  113           * since the cpu_id was first checked.
 114  114           */
 115  115          movl    $GETCPU_GDT_OFFSET, %ecx
 116  116          lsl     %cx, %ecx
 117  117          movl    (%esp), %esi
 118  118          cmpl    %ecx, %esi
 119  119          je      9f
 120  120          /*
 121  121           * There was a CPU migration, perform another reading.
 122  122           */
 123  123          movl    %eax, (%esp)
 124  124          movl    CP_TSC_NCPU(%edi), %esi
 125  125          movl    CP_TSC_TYPE(%edi), %eax
 126  126          jmp     4b
 127  127  
 128  128  9:
 129  129          /* Grab the per-cpu offset and add it to the TSC result */
 130  130          leal    CP_TSC_SYNC_TICK_DELTA(%edi), %esi
 131  131          leal    (%esi, %ecx, 8), %ecx
 132  132          addl    (%ecx), %eax
 133  133          adcl    0x4(%ecx), %edx
 134  134          jmp     1b
 135  135          SET_SIZE(__cp_tsc_read)
 136  136  
 137  137  /*
 138  138   * uint_t
 139  139   * __cp_getcpu(uint_t cp_tsc_type)
 140  140   */
 141  141          ENTRY_NP(__cp_getcpu)
 142  142          /*
 143  143           * If RDTSCP is available, it is a quick way to grab the cpu_id which
 144  144           * is stored in the TSC_AUX MSR by the kernel.
 145  145           */
 146  146          movl    4(%esp), %eax
 147  147          movl    CP_TSC_TYPE(%eax), %eax
 148  148          cmpl    $TSC_TSCP, %eax
 149  149          jne     1f
 150  150          rdtscp
 151  151          movl    %ecx, %eax
 152  152          ret
 153  153  1:
 154  154          mov     $GETCPU_GDT_OFFSET, %eax
 155  155          lsl     %ax, %eax
 156  156          ret
 157  157          SET_SIZE(__cp_getcpu)

XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX