1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2019 Joyent, Inc.
14 * Copyright 2024 MNX Cloud, Inc.
15 */
16
17 .file "retpoline.s"
18
19 /*
20 * This file implements the various hooks that are needed for retpolines and
21 * return stack buffer (RSB) stuffing. For more information, please see the
22 * 'Speculative Execution CPU Side Channel Security' section of the
23 * uts/i86pc/os/cpuid.c big theory statement.
24 */
25
26 #include <sys/asm_linkage.h>
27 #include <sys/x86_archext.h>
28
29 #if defined(__amd64)
30
31 /*
32 * This macro generates the default retpoline entry point that the compiler
33 * expects. It implements the expected retpoline form.
34 */
35 #define RETPOLINE_MKTHUNK(reg) \
36 ENTRY(__x86_indirect_thunk_##reg) \
37 call 2f; \
38 1: \
39 pause; \
40 lfence; \
41 jmp 1b; \
42 2: \
43 movq %##reg, (%rsp); \
44 ret; \
45 SET_SIZE(__x86_indirect_thunk_##reg)
46
47 /*
48 * This macro generates the default retpoline form. It exists in addition to the
49 * thunk so if we need to restore the default retpoline behavior to the thunk
50 * we can.
51 */
52 #define RETPOLINE_MKGENERIC(reg) \
53 ENTRY(__x86_indirect_thunk_gen_##reg) \
54 call 2f; \
55 1: \
56 pause; \
57 lfence; \
58 jmp 1b; \
59 2: \
60 movq %##reg, (%rsp); \
61 ret; \
62 SET_SIZE(__x86_indirect_thunk_gen_##reg)
63
64 /*
65 * This macro generates the no-op form of the retpoline which will be used if we
66 * either need to disable retpolines because we have enhanced IBRS or because we
67 * have been asked to disable mitigations.
68 */
69 #define RETPOLINE_MKJUMP(reg) \
70 ENTRY(__x86_indirect_thunk_jmp_##reg) \
71 jmp *%##reg; \
72 SET_SIZE(__x86_indirect_thunk_jmp_##reg)
73
74 RETPOLINE_MKTHUNK(rax)
75 RETPOLINE_MKTHUNK(rbx)
76 RETPOLINE_MKTHUNK(rcx)
77 RETPOLINE_MKTHUNK(rdx)
78 RETPOLINE_MKTHUNK(rdi)
79 RETPOLINE_MKTHUNK(rsi)
80 RETPOLINE_MKTHUNK(rbp)
81 RETPOLINE_MKTHUNK(r8)
82 RETPOLINE_MKTHUNK(r9)
83 RETPOLINE_MKTHUNK(r10)
84 RETPOLINE_MKTHUNK(r11)
85 RETPOLINE_MKTHUNK(r12)
86 RETPOLINE_MKTHUNK(r13)
87 RETPOLINE_MKTHUNK(r14)
88 RETPOLINE_MKTHUNK(r15)
89
90 RETPOLINE_MKGENERIC(rax)
91 RETPOLINE_MKGENERIC(rbx)
92 RETPOLINE_MKGENERIC(rcx)
93 RETPOLINE_MKGENERIC(rdx)
94 RETPOLINE_MKGENERIC(rdi)
95 RETPOLINE_MKGENERIC(rsi)
96 RETPOLINE_MKGENERIC(rbp)
97 RETPOLINE_MKGENERIC(r8)
98 RETPOLINE_MKGENERIC(r9)
99 RETPOLINE_MKGENERIC(r10)
100 RETPOLINE_MKGENERIC(r11)
101 RETPOLINE_MKGENERIC(r12)
102 RETPOLINE_MKGENERIC(r13)
103 RETPOLINE_MKGENERIC(r14)
104 RETPOLINE_MKGENERIC(r15)
105
106 RETPOLINE_MKJUMP(rax)
107 RETPOLINE_MKJUMP(rbx)
108 RETPOLINE_MKJUMP(rcx)
109 RETPOLINE_MKJUMP(rdx)
110 RETPOLINE_MKJUMP(rdi)
111 RETPOLINE_MKJUMP(rsi)
112 RETPOLINE_MKJUMP(rbp)
113 RETPOLINE_MKJUMP(r8)
114 RETPOLINE_MKJUMP(r9)
115 RETPOLINE_MKJUMP(r10)
116 RETPOLINE_MKJUMP(r11)
117 RETPOLINE_MKJUMP(r12)
118 RETPOLINE_MKJUMP(r13)
119 RETPOLINE_MKJUMP(r14)
120 RETPOLINE_MKJUMP(r15)
121
122 /*
123 * The x86_rsb_stuff{,_vmexit} functions can be called from pretty
124 * arbitrary contexts. It's much easier for us to save and restore all
125 * the registers we touch rather than clobber them for callers. You
126 * must preserve this property or the system will panic at best.
127 *
128 * The two entry points are because the need to RSB stuff on Intel
129 * depends greatly on factors that are different in the VMEXIT case,
130 * vs. the other switching cases. See cpuid.c's cpuid_patch_rsb()
131 * for details.
132 */
133 ENTRY(x86_rsb_stuff_vmexit)
134 nop
135 ALTENTRY(x86_rsb_stuff)
136 nop
137 pushq %rdi
138 pushq %rax
139 movl $16, %edi
140 movq %rsp, %rax
141 rsb_loop:
142 call 2f
143 1:
144 pause
145 call 1b
146 2:
147 call 2f
148 1:
149 pause
150 call 1b
151 2:
152 subl $1, %edi
153 jnz rsb_loop
154 movq %rax, %rsp
155 popq %rax
156 popq %rdi
157 ret
158 SET_SIZE(x86_rsb_stuff)
159 SET_SIZE(x86_rsb_stuff_vmexit)
160
161 #elif defined(__i386)
162
163 /*
164 * While the kernel is 64-bit only, dboot is still 32-bit, so there are a
165 * limited number of variants that are used for 32-bit. However as dboot is
166 * short lived and uses them sparingly, we only do the full variant and do not
167 * have an AMD specific version.
168 */
169
170 #define RETPOLINE_MKTHUNK(reg) \
171 ENTRY(__x86_indirect_thunk_##reg) \
172 call 2f; \
173 1: \
174 pause; \
175 lfence; \
176 jmp 1b; \
177 2: \
178 movl %##reg, (%esp); \
179 ret; \
180 SET_SIZE(__x86_indirect_thunk_##reg)
181
182 RETPOLINE_MKTHUNK(edi)
183 RETPOLINE_MKTHUNK(eax)
184
185 #else
186 #error "Your architecture is in another castle."
187 #endif