6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2012 Gary Mills
23 *
24 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2011 by Delphix. All rights reserved.
26 */
27 /*
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
30 */
31
32 #include <sys/types.h>
33 #include <sys/sysmacros.h>
34 #include <sys/disp.h>
35 #include <sys/promif.h>
36 #include <sys/clock.h>
37 #include <sys/cpuvar.h>
38 #include <sys/stack.h>
39 #include <vm/as.h>
40 #include <vm/hat.h>
41 #include <sys/reboot.h>
42 #include <sys/avintr.h>
43 #include <sys/vtrace.h>
44 #include <sys/proc.h>
45 #include <sys/thread.h>
46 #include <sys/cpupart.h>
47 #include <sys/pset.h>
48 #include <sys/copyops.h>
49 #include <sys/pg.h>
50 #include <sys/disp.h>
51 #include <sys/debug.h>
52 #include <sys/sunddi.h>
53 #include <sys/x86_archext.h>
54 #include <sys/privregs.h>
55 #include <sys/machsystm.h>
56 #include <sys/ontrap.h>
57 #include <sys/bootconf.h>
58 #include <sys/boot_console.h>
59 #include <sys/kdi_machimpl.h>
60 #include <sys/archsystm.h>
61 #include <sys/promif.h>
62 #include <sys/pci_cfgspace.h>
63 #include <sys/bootvfs.h>
64 #ifdef __xpv
65 #include <sys/hypervisor.h>
66 #else
67 #include <sys/xpv_support.h>
68 #endif
69
70 /*
71 * some globals for patching the result of cpuid
72 * to solve problems w/ creative cpu vendors
73 */
74
75 extern uint32_t cpuid_feature_ecx_include;
76 extern uint32_t cpuid_feature_ecx_exclude;
77 extern uint32_t cpuid_feature_edx_include;
78 extern uint32_t cpuid_feature_edx_exclude;
79
80 /*
81 * Set console mode
82 */
83 static void
210 * Patch the tsc_read routine with appropriate set of instructions,
211 * depending on the processor family and architecure, to read the
212 * time-stamp counter while ensuring no out-of-order execution.
213 * Patch it while the kernel text is still writable.
214 *
215 * Note: tsc_read is not patched for intel processors whose family
216 * is >6 and for amd whose family >f (in case they don't support rdtscp
217 * instruction, unlikely). By default tsc_read will use cpuid for
218 * serialization in such cases. The following code needs to be
219 * revisited if intel processors of family >= f retains the
220 * instruction serialization nature of mfence instruction.
221 * Note: tsc_read is not patched for x86 processors which do
222 * not support "mfence". By default tsc_read will use cpuid for
223 * serialization in such cases.
224 *
225 * The Xen hypervisor does not correctly report whether rdtscp is
226 * supported or not, so we must assume that it is not.
227 */
228 if ((get_hwenv() & HW_XEN_HVM) == 0 &&
229 is_x86_feature(x86_featureset, X86FSET_TSCP))
230 patch_tsc_read(X86_HAVE_TSCP);
231 else if (cpuid_getvendor(CPU) == X86_VENDOR_AMD &&
232 cpuid_getfamily(CPU) <= 0xf &&
233 is_x86_feature(x86_featureset, X86FSET_SSE2))
234 patch_tsc_read(X86_TSC_MFENCE);
235 else if (cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
236 cpuid_getfamily(CPU) <= 6 &&
237 is_x86_feature(x86_featureset, X86FSET_SSE2))
238 patch_tsc_read(X86_TSC_LFENCE);
239
240 #endif /* !__xpv */
241
242 #if defined(__i386) && !defined(__xpv)
243 /*
244 * Some i386 processors do not implement the rdtsc instruction,
245 * or at least they do not implement it correctly. Patch them to
246 * return 0.
247 */
248 if (!is_x86_feature(x86_featureset, X86FSET_TSC))
249 patch_tsc_read(X86_NO_TSC);
250 #endif /* __i386 && !__xpv */
251
252 #if defined(__amd64) && !defined(__xpv)
253 patch_memops(cpuid_getvendor(CPU));
254 #endif /* __amd64 && !__xpv */
255
256 #if !defined(__xpv)
257 /* XXPV what, if anything, should be dorked with here under xen? */
258
259 /*
260 * While we're thinking about the TSC, let's set up %cr4 so that
261 * userland can issue rdtsc, and initialize the TSC_AUX value
262 * (the cpuid) for the rdtscp instruction on appropriately
263 * capable hardware.
264 */
265 if (is_x86_feature(x86_featureset, X86FSET_TSC))
266 setcr4(getcr4() & ~CR4_TSD);
267
268 if (is_x86_feature(x86_featureset, X86FSET_TSCP))
269 (void) wrmsr(MSR_AMD_TSCAUX, 0);
|
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2012 Gary Mills
23 *
24 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2011 by Delphix. All rights reserved.
26 * Copyright 2016 Joyent, Inc.
27 */
28 /*
29 * Copyright (c) 2010, Intel Corporation.
30 * All rights reserved.
31 */
32
33 #include <sys/types.h>
34 #include <sys/sysmacros.h>
35 #include <sys/disp.h>
36 #include <sys/promif.h>
37 #include <sys/clock.h>
38 #include <sys/cpuvar.h>
39 #include <sys/stack.h>
40 #include <vm/as.h>
41 #include <vm/hat.h>
42 #include <sys/reboot.h>
43 #include <sys/avintr.h>
44 #include <sys/vtrace.h>
45 #include <sys/proc.h>
46 #include <sys/thread.h>
47 #include <sys/cpupart.h>
48 #include <sys/pset.h>
49 #include <sys/copyops.h>
50 #include <sys/pg.h>
51 #include <sys/disp.h>
52 #include <sys/debug.h>
53 #include <sys/sunddi.h>
54 #include <sys/x86_archext.h>
55 #include <sys/privregs.h>
56 #include <sys/machsystm.h>
57 #include <sys/ontrap.h>
58 #include <sys/bootconf.h>
59 #include <sys/boot_console.h>
60 #include <sys/kdi_machimpl.h>
61 #include <sys/archsystm.h>
62 #include <sys/promif.h>
63 #include <sys/pci_cfgspace.h>
64 #include <sys/bootvfs.h>
65 #include <sys/tsc.h>
66 #ifdef __xpv
67 #include <sys/hypervisor.h>
68 #else
69 #include <sys/xpv_support.h>
70 #endif
71
72 /*
73 * some globals for patching the result of cpuid
74 * to solve problems w/ creative cpu vendors
75 */
76
77 extern uint32_t cpuid_feature_ecx_include;
78 extern uint32_t cpuid_feature_ecx_exclude;
79 extern uint32_t cpuid_feature_edx_include;
80 extern uint32_t cpuid_feature_edx_exclude;
81
82 /*
83 * Set console mode
84 */
85 static void
212 * Patch the tsc_read routine with appropriate set of instructions,
213 * depending on the processor family and architecure, to read the
214 * time-stamp counter while ensuring no out-of-order execution.
215 * Patch it while the kernel text is still writable.
216 *
217 * Note: tsc_read is not patched for intel processors whose family
218 * is >6 and for amd whose family >f (in case they don't support rdtscp
219 * instruction, unlikely). By default tsc_read will use cpuid for
220 * serialization in such cases. The following code needs to be
221 * revisited if intel processors of family >= f retains the
222 * instruction serialization nature of mfence instruction.
223 * Note: tsc_read is not patched for x86 processors which do
224 * not support "mfence". By default tsc_read will use cpuid for
225 * serialization in such cases.
226 *
227 * The Xen hypervisor does not correctly report whether rdtscp is
228 * supported or not, so we must assume that it is not.
229 */
230 if ((get_hwenv() & HW_XEN_HVM) == 0 &&
231 is_x86_feature(x86_featureset, X86FSET_TSCP))
232 patch_tsc_read(TSC_TSCP);
233 else if (cpuid_getvendor(CPU) == X86_VENDOR_AMD &&
234 cpuid_getfamily(CPU) <= 0xf &&
235 is_x86_feature(x86_featureset, X86FSET_SSE2))
236 patch_tsc_read(TSC_RDTSC_MFENCE);
237 else if (cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
238 cpuid_getfamily(CPU) <= 6 &&
239 is_x86_feature(x86_featureset, X86FSET_SSE2))
240 patch_tsc_read(TSC_RDTSC_LFENCE);
241
242 #endif /* !__xpv */
243
244 #if defined(__i386) && !defined(__xpv)
245 /*
246 * Some i386 processors do not implement the rdtsc instruction,
247 * or at least they do not implement it correctly. Patch them to
248 * return 0.
249 */
250 if (!is_x86_feature(x86_featureset, X86FSET_TSC))
251 patch_tsc_read(TSC_NONE);
252 #endif /* __i386 && !__xpv */
253
254 #if defined(__amd64) && !defined(__xpv)
255 patch_memops(cpuid_getvendor(CPU));
256 #endif /* __amd64 && !__xpv */
257
258 #if !defined(__xpv)
259 /* XXPV what, if anything, should be dorked with here under xen? */
260
261 /*
262 * While we're thinking about the TSC, let's set up %cr4 so that
263 * userland can issue rdtsc, and initialize the TSC_AUX value
264 * (the cpuid) for the rdtscp instruction on appropriately
265 * capable hardware.
266 */
267 if (is_x86_feature(x86_featureset, X86FSET_TSC))
268 setcr4(getcr4() & ~CR4_TSD);
269
270 if (is_x86_feature(x86_featureset, X86FSET_TSCP))
271 (void) wrmsr(MSR_AMD_TSCAUX, 0);
|