Print this page
13275 bhyve needs richer INIT/SIPI support
Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Approved by: Gordon Ross <gordon.w.ross@gmail.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/vmm/io/vlapic.c
+++ new/usr/src/uts/i86pc/io/vmm/io/vlapic.c
1 1 /*-
2 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 3 *
4 4 * Copyright (c) 2011 NetApp, Inc.
5 5 * All rights reserved.
6 6 * Copyright (c) 2019 Joyent, Inc.
7 7 *
8 8 * Redistribution and use in source and binary forms, with or without
9 9 * modification, are permitted provided that the following conditions
10 10 * are met:
11 11 * 1. Redistributions of source code must retain the above copyright
12 12 * notice, this list of conditions and the following disclaimer.
13 13 * 2. Redistributions in binary form must reproduce the above copyright
14 14 * notice, this list of conditions and the following disclaimer in the
15 15 * documentation and/or other materials provided with the distribution.
16 16 *
17 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 27 * SUCH DAMAGE.
28 28 *
29 29 * $FreeBSD$
30 30 */
31 31 /*
32 32 * This file and its contents are supplied under the terms of the
33 33 * Common Development and Distribution License ("CDDL"), version 1.0.
34 34 * You may only use this file in accordance with the terms of version
35 35 * 1.0 of the CDDL.
36 36 *
37 37 * A full copy of the text of the CDDL should have accompanied this
38 38 * source. A copy of the CDDL is also available via the Internet at
39 39 * http://www.illumos.org/license/CDDL.
40 40 *
41 41 * Copyright 2014 Pluribus Networks Inc.
42 42 * Copyright 2018 Joyent, Inc.
43 43 * Copyright 2020 Oxide Computer Company
44 44 */
45 45
46 46 #include <sys/cdefs.h>
47 47 __FBSDID("$FreeBSD$");
48 48
49 49 #include <sys/param.h>
50 50 #include <sys/lock.h>
51 51 #include <sys/kernel.h>
52 52 #include <sys/malloc.h>
53 53 #include <sys/mutex.h>
54 54 #include <sys/systm.h>
55 55 #include <sys/smp.h>
56 56
57 57 #include <x86/specialreg.h>
58 58 #include <x86/apicreg.h>
59 59
60 60 #include <machine/clock.h>
61 61 #include <machine/smp.h>
62 62
63 63 #include <machine/vmm.h>
64 64
65 65 #include "vmm_lapic.h"
66 66 #include "vmm_ktr.h"
67 67 #include "vmm_stat.h"
68 68
69 69 #include "vlapic.h"
70 70 #include "vlapic_priv.h"
71 71 #include "vioapic.h"
72 72
73 73
74 74 /*
75 75 * The 4 high bits of a given interrupt vector represent its priority. The same
76 76 * is true for the contents of the TPR when it is used to calculate the ultimate
77 77 * PPR of an APIC - the 4 high bits hold the priority.
78 78 */
79 79 #define PRIO(x) ((x) & 0xf0)
80 80
81 81 #define VLAPIC_VERSION (16)
82 82
83 83 #define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
84 84
85 85 /*
86 86 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
87 87 * vlapic_callout_handler() and vcpu accesses to:
88 88 * - timer_freq_bt, timer_period_bt, timer_fire_bt
89 89 * - timer LVT register
90 90 */
91 91 #define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
92 92 #define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
93 93 #define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
94 94
95 95 /*
96 96 * APIC timer frequency:
97 97 * - arbitrary but chosen to be in the ballpark of contemporary hardware.
98 98 * - power-of-two to avoid loss of precision when converted to a bintime.
99 99 */
100 100 #define VLAPIC_BUS_FREQ (128 * 1024 * 1024)
101 101
102 102 static void vlapic_set_error(struct vlapic *, uint32_t, bool);
103 103
104 104 #ifdef __ISRVEC_DEBUG
105 105 static void vlapic_isrstk_accept(struct vlapic *, int);
106 106 static void vlapic_isrstk_eoi(struct vlapic *, int);
107 107 static void vlapic_isrstk_verify(const struct vlapic *);
108 108 #endif /* __ISRVEC_DEBUG */
109 109
110 110 static __inline uint32_t
111 111 vlapic_get_id(struct vlapic *vlapic)
112 112 {
113 113
114 114 if (x2apic(vlapic))
115 115 return (vlapic->vcpuid);
116 116 else
117 117 return (vlapic->vcpuid << 24);
118 118 }
119 119
120 120 static uint32_t
121 121 x2apic_ldr(struct vlapic *vlapic)
122 122 {
123 123 int apicid;
124 124 uint32_t ldr;
125 125
126 126 apicid = vlapic_get_id(vlapic);
127 127 ldr = 1 << (apicid & 0xf);
128 128 ldr |= (apicid & 0xffff0) << 12;
129 129 return (ldr);
130 130 }
131 131
132 132 void
133 133 vlapic_dfr_write_handler(struct vlapic *vlapic)
134 134 {
135 135 struct LAPIC *lapic;
136 136
137 137 lapic = vlapic->apic_page;
138 138 if (x2apic(vlapic)) {
139 139 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
140 140 lapic->dfr);
141 141 lapic->dfr = 0;
142 142 return;
143 143 }
144 144
145 145 lapic->dfr &= APIC_DFR_MODEL_MASK;
146 146 lapic->dfr |= APIC_DFR_RESERVED;
147 147
148 148 #ifdef __FreeBSD__
149 149 if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
150 150 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
151 151 else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
152 152 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
153 153 else
154 154 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
155 155 #endif
156 156 }
157 157
158 158 void
159 159 vlapic_ldr_write_handler(struct vlapic *vlapic)
160 160 {
161 161 struct LAPIC *lapic;
162 162
163 163 lapic = vlapic->apic_page;
164 164
165 165 /* LDR is read-only in x2apic mode */
166 166 if (x2apic(vlapic)) {
167 167 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
168 168 lapic->ldr);
169 169 lapic->ldr = x2apic_ldr(vlapic);
170 170 } else {
171 171 lapic->ldr &= ~APIC_LDR_RESERVED;
172 172 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
173 173 }
174 174 }
175 175
176 176 void
177 177 vlapic_id_write_handler(struct vlapic *vlapic)
178 178 {
179 179 struct LAPIC *lapic;
180 180
181 181 /*
182 182 * We don't allow the ID register to be modified so reset it back to
183 183 * its default value.
184 184 */
185 185 lapic = vlapic->apic_page;
186 186 lapic->id = vlapic_get_id(vlapic);
187 187 }
188 188
189 189 static int
190 190 vlapic_timer_divisor(uint32_t dcr)
191 191 {
192 192 switch (dcr & 0xB) {
193 193 case APIC_TDCR_1:
194 194 return (1);
195 195 case APIC_TDCR_2:
196 196 return (2);
197 197 case APIC_TDCR_4:
198 198 return (4);
199 199 case APIC_TDCR_8:
200 200 return (8);
201 201 case APIC_TDCR_16:
202 202 return (16);
203 203 case APIC_TDCR_32:
204 204 return (32);
205 205 case APIC_TDCR_64:
206 206 return (64);
207 207 case APIC_TDCR_128:
208 208 return (128);
209 209 default:
210 210 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
211 211 }
212 212 }
213 213
214 214 #if 0
215 215 static inline void
216 216 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
217 217 {
218 218 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
219 219 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
220 220 *lvt & APIC_LVTT_M);
221 221 }
222 222 #endif
223 223
224 224 static uint32_t
225 225 vlapic_get_ccr(struct vlapic *vlapic)
226 226 {
227 227 struct bintime bt_now, bt_rem;
228 228 struct LAPIC *lapic;
229 229 uint32_t ccr;
230 230
231 231 ccr = 0;
232 232 lapic = vlapic->apic_page;
233 233
234 234 VLAPIC_TIMER_LOCK(vlapic);
235 235 if (callout_active(&vlapic->callout)) {
236 236 /*
237 237 * If the timer is scheduled to expire in the future then
238 238 * compute the value of 'ccr' based on the remaining time.
239 239 */
240 240 binuptime(&bt_now);
241 241 if (BINTIME_CMP(&vlapic->timer_fire_bt, >, &bt_now)) {
242 242 bt_rem = vlapic->timer_fire_bt;
243 243 bintime_sub(&bt_rem, &bt_now);
244 244 ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
245 245 ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
246 246 }
247 247 }
248 248 #ifdef __FreeBSD__
249 249 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
250 250 "icr_timer is %#x", ccr, lapic->icr_timer));
251 251 #else
252 252 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %x, "
253 253 "icr_timer is %x", ccr, lapic->icr_timer));
254 254 #endif
255 255 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
256 256 ccr, lapic->icr_timer);
257 257 VLAPIC_TIMER_UNLOCK(vlapic);
258 258 return (ccr);
259 259 }
260 260
261 261 void
262 262 vlapic_dcr_write_handler(struct vlapic *vlapic)
263 263 {
264 264 struct LAPIC *lapic;
265 265 int divisor;
266 266
267 267 lapic = vlapic->apic_page;
268 268 VLAPIC_TIMER_LOCK(vlapic);
269 269
270 270 divisor = vlapic_timer_divisor(lapic->dcr_timer);
271 271 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
272 272 lapic->dcr_timer, divisor);
273 273
274 274 /*
275 275 * Update the timer frequency and the timer period.
276 276 *
277 277 * XXX changes to the frequency divider will not take effect until
278 278 * the timer is reloaded.
279 279 */
280 280 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
281 281 vlapic->timer_period_bt = vlapic->timer_freq_bt;
282 282 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
283 283
284 284 VLAPIC_TIMER_UNLOCK(vlapic);
285 285 }
286 286
287 287 void
288 288 vlapic_esr_write_handler(struct vlapic *vlapic)
289 289 {
290 290 struct LAPIC *lapic;
291 291
292 292 lapic = vlapic->apic_page;
293 293 lapic->esr = vlapic->esr_pending;
294 294 vlapic->esr_pending = 0;
295 295 }
296 296
297 297 vcpu_notify_t
298 298 vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
299 299 {
300 300 struct LAPIC *lapic;
301 301 uint32_t *irrptr, *tmrptr, mask, tmr;
302 302 int idx;
303 303
304 304 KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
305 305
306 306 lapic = vlapic->apic_page;
307 307 if (!(lapic->svr & APIC_SVR_ENABLE)) {
308 308 /* ignore interrupt on software-disabled APIC */
309 309 return (VCPU_NOTIFY_NONE);
310 310 }
311 311
312 312 if (vector < 16) {
313 313 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR,
314 314 false);
315 315
316 316 /*
317 317 * If the error LVT is configured to interrupt the vCPU, it will
318 318 * have delivered a notification through that mechanism.
319 319 */
320 320 return (VCPU_NOTIFY_NONE);
321 321 }
322 322
323 323 if (vlapic->ops.set_intr_ready) {
324 324 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
325 325 }
326 326
327 327 idx = (vector / 32) * 4;
328 328 mask = 1 << (vector % 32);
329 329 tmrptr = &lapic->tmr0;
330 330 irrptr = &lapic->irr0;
331 331
332 332 /*
333 333 * Update TMR for requested vector, if necessary.
334 334 * This must be done prior to asserting the bit in IRR so that the
335 335 * proper TMR state is always visible before the to-be-queued interrupt
336 336 * can be injected.
337 337 */
338 338 tmr = atomic_load_acq_32(&tmrptr[idx]);
339 339 if ((tmr & mask) != (level ? mask : 0)) {
340 340 if (level) {
341 341 atomic_set_int(&tmrptr[idx], mask);
342 342 } else {
343 343 atomic_clear_int(&tmrptr[idx], mask);
344 344 }
345 345 }
346 346
347 347 /* Now set the bit in IRR */
348 348 atomic_set_int(&irrptr[idx], mask);
349 349
350 350 return (VCPU_NOTIFY_EXIT);
351 351 }
352 352
353 353 static __inline uint32_t *
354 354 vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
355 355 {
356 356 struct LAPIC *lapic = vlapic->apic_page;
357 357 int i;
358 358
359 359 switch (offset) {
360 360 case APIC_OFFSET_CMCI_LVT:
361 361 return (&lapic->lvt_cmci);
362 362 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
363 363 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
364 364 return ((&lapic->lvt_timer) + i);
365 365 default:
366 366 panic("vlapic_get_lvt: invalid LVT\n");
367 367 }
368 368 }
369 369
370 370 static __inline int
371 371 lvt_off_to_idx(uint32_t offset)
372 372 {
373 373 int index;
374 374
375 375 switch (offset) {
376 376 case APIC_OFFSET_CMCI_LVT:
377 377 index = APIC_LVT_CMCI;
378 378 break;
379 379 case APIC_OFFSET_TIMER_LVT:
380 380 index = APIC_LVT_TIMER;
381 381 break;
382 382 case APIC_OFFSET_THERM_LVT:
383 383 index = APIC_LVT_THERMAL;
384 384 break;
385 385 case APIC_OFFSET_PERF_LVT:
386 386 index = APIC_LVT_PMC;
387 387 break;
388 388 case APIC_OFFSET_LINT0_LVT:
389 389 index = APIC_LVT_LINT0;
390 390 break;
391 391 case APIC_OFFSET_LINT1_LVT:
392 392 index = APIC_LVT_LINT1;
393 393 break;
394 394 case APIC_OFFSET_ERROR_LVT:
395 395 index = APIC_LVT_ERROR;
396 396 break;
397 397 default:
398 398 index = -1;
399 399 break;
400 400 }
401 401 #ifdef __FreeBSD__
402 402 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
403 403 "invalid lvt index %d for offset %#x", index, offset));
404 404 #else
405 405 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
406 406 "invalid lvt index %d for offset %x", index, offset));
407 407 #endif
408 408
409 409 return (index);
410 410 }
411 411
412 412 static __inline uint32_t
413 413 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
414 414 {
415 415 int idx;
416 416 uint32_t val;
417 417
418 418 idx = lvt_off_to_idx(offset);
419 419 val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
420 420 return (val);
421 421 }
422 422
423 423 void
424 424 vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
425 425 {
426 426 uint32_t *lvtptr, mask, val;
427 427 struct LAPIC *lapic;
428 428 int idx;
429 429
430 430 lapic = vlapic->apic_page;
431 431 lvtptr = vlapic_get_lvtptr(vlapic, offset);
432 432 val = *lvtptr;
433 433 idx = lvt_off_to_idx(offset);
434 434
435 435 if (!(lapic->svr & APIC_SVR_ENABLE))
436 436 val |= APIC_LVT_M;
437 437 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
438 438 switch (offset) {
439 439 case APIC_OFFSET_TIMER_LVT:
440 440 mask |= APIC_LVTT_TM;
441 441 break;
442 442 case APIC_OFFSET_ERROR_LVT:
443 443 break;
444 444 case APIC_OFFSET_LINT0_LVT:
445 445 case APIC_OFFSET_LINT1_LVT:
446 446 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
447 447 /* FALLTHROUGH */
448 448 default:
449 449 mask |= APIC_LVT_DM;
450 450 break;
451 451 }
452 452 val &= mask;
453 453 *lvtptr = val;
454 454 atomic_store_rel_32(&vlapic->lvt_last[idx], val);
455 455 }
456 456
457 457 static void
458 458 vlapic_mask_lvts(struct vlapic *vlapic)
459 459 {
460 460 struct LAPIC *lapic = vlapic->apic_page;
461 461
462 462 lapic->lvt_cmci |= APIC_LVT_M;
463 463 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
464 464
465 465 lapic->lvt_timer |= APIC_LVT_M;
466 466 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
467 467
468 468 lapic->lvt_thermal |= APIC_LVT_M;
469 469 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
470 470
471 471 lapic->lvt_pcint |= APIC_LVT_M;
472 472 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
473 473
474 474 lapic->lvt_lint0 |= APIC_LVT_M;
475 475 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
476 476
477 477 lapic->lvt_lint1 |= APIC_LVT_M;
478 478 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
479 479
480 480 lapic->lvt_error |= APIC_LVT_M;
481 481 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
482 482 }
483 483
484 484 static int
485 485 vlapic_fire_lvt(struct vlapic *vlapic, uint_t lvt)
486 486 {
487 487 uint32_t mode, reg, vec;
488 488 vcpu_notify_t notify;
489 489
490 490 reg = atomic_load_acq_32(&vlapic->lvt_last[lvt]);
491 491
492 492 if (reg & APIC_LVT_M)
493 493 return (0);
494 494 vec = reg & APIC_LVT_VECTOR;
495 495 mode = reg & APIC_LVT_DM;
496 496
497 497 switch (mode) {
498 498 case APIC_LVT_DM_FIXED:
499 499 if (vec < 16) {
500 500 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR,
501 501 lvt == APIC_LVT_ERROR);
502 502 return (0);
503 503 }
504 504 notify = vlapic_set_intr_ready(vlapic, vec, false);
505 505 vcpu_notify_event_type(vlapic->vm, vlapic->vcpuid, notify);
506 506 break;
507 507 case APIC_LVT_DM_NMI:
508 508 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
509 509 break;
510 510 case APIC_LVT_DM_EXTINT:
511 511 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
512 512 break;
513 513 default:
514 514 // Other modes ignored
515 515 return (0);
516 516 }
517 517 return (1);
518 518 }
519 519
520 520 static uint_t
521 521 vlapic_active_isr(struct vlapic *vlapic)
522 522 {
523 523 int i;
524 524 uint32_t *isrp;
525 525
526 526 isrp = &vlapic->apic_page->isr7;
527 527
528 528 for (i = 7; i >= 0; i--, isrp -= 4) {
529 529 uint32_t reg = *isrp;
530 530
531 531 if (reg != 0) {
532 532 uint_t vec = (i * 32) + bsrl(reg);
533 533
534 534 if (vec < 16) {
535 535 /*
536 536 * Truncate the illegal low vectors to value of
537 537 * 0, indicating that no active ISR was found.
538 538 */
539 539 return (0);
540 540 }
541 541 return (vec);
542 542 }
543 543 }
544 544
545 545 return (0);
546 546 }
547 547
548 548 /*
549 549 * After events which might arbitrarily change the value of PPR, such as a TPR
550 550 * write or an EOI, calculate that new PPR value and store it in the APIC page.
551 551 */
552 552 static void
553 553 vlapic_update_ppr(struct vlapic *vlapic)
554 554 {
555 555 int isrvec, tpr, ppr;
556 556
557 557 isrvec = vlapic_active_isr(vlapic);
558 558 tpr = vlapic->apic_page->tpr;
559 559
560 560 /*
561 561 * Algorithm adopted from section "Interrupt, Task and Processor
562 562 * Priority" in Intel Architecture Manual Vol 3a.
563 563 */
564 564 if (PRIO(tpr) >= PRIO(isrvec)) {
565 565 ppr = tpr;
566 566 } else {
567 567 ppr = PRIO(isrvec);
568 568 }
569 569
570 570 vlapic->apic_page->ppr = ppr;
571 571 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
572 572 }
573 573
574 574 /*
575 575 * When a vector is asserted in ISR as in-service, the PPR must be raised to the
576 576 * priority of that vector, as the vCPU would have been at a lower priority in
577 577 * order for the vector to be accepted.
578 578 */
579 579 static void
580 580 vlapic_raise_ppr(struct vlapic *vlapic, int vec)
581 581 {
582 582 struct LAPIC *lapic = vlapic->apic_page;
583 583 int ppr;
584 584
585 585 ppr = PRIO(vec);
586 586
587 587 #ifdef __ISRVEC_DEBUG
588 588 KASSERT(vec >= 16 && vec < 256, ("invalid vector %d", vec));
589 589 KASSERT(ppr > lapic->tpr, ("ppr %x <= tpr %x", ppr, lapic->tpr));
590 590 KASSERT(ppr > lapic->ppr, ("ppr %x <= old ppr %x", ppr, lapic->ppr));
591 591 KASSERT(vec == (int)vlapic_active_isr(vlapic), ("ISR missing for ppr"));
592 592 #endif /* __ISRVEC_DEBUG */
593 593
594 594 lapic->ppr = ppr;
595 595 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
596 596 }
597 597
598 598 void
599 599 vlapic_sync_tpr(struct vlapic *vlapic)
600 600 {
601 601 vlapic_update_ppr(vlapic);
602 602 }
603 603
604 604 static VMM_STAT(VLAPIC_GRATUITOUS_EOI, "EOI without any in-service interrupt");
605 605
606 606 static void
607 607 vlapic_process_eoi(struct vlapic *vlapic)
608 608 {
609 609 struct LAPIC *lapic = vlapic->apic_page;
610 610 uint32_t *isrptr, *tmrptr;
611 611 int i;
612 612 uint_t idx, bitpos, vector;
613 613
614 614 isrptr = &lapic->isr0;
615 615 tmrptr = &lapic->tmr0;
616 616
617 617 for (i = 7; i >= 0; i--) {
618 618 idx = i * 4;
619 619 if (isrptr[idx] != 0) {
620 620 bitpos = bsrl(isrptr[idx]);
621 621 vector = i * 32 + bitpos;
622 622
623 623 isrptr[idx] &= ~(1 << bitpos);
624 624 VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "EOI vector %d",
625 625 vector);
626 626 VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
627 627 #ifdef __ISRVEC_DEBUG
628 628 vlapic_isrstk_eoi(vlapic, vector);
629 629 #endif
630 630 vlapic_update_ppr(vlapic);
631 631 if ((tmrptr[idx] & (1 << bitpos)) != 0) {
632 632 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
633 633 vector);
634 634 }
635 635 return;
636 636 }
637 637 }
638 638 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "Gratuitous EOI");
639 639 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_GRATUITOUS_EOI, 1);
640 640 }
641 641
642 642 static __inline int
643 643 vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
644 644 {
645 645
646 646 return (lvt & mask);
647 647 }
648 648
649 649 static __inline int
650 650 vlapic_periodic_timer(struct vlapic *vlapic)
651 651 {
652 652 uint32_t lvt;
653 653
654 654 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
655 655
656 656 return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
657 657 }
658 658
659 659 static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
660 660
661 661 static void
662 662 vlapic_set_error(struct vlapic *vlapic, uint32_t mask, bool lvt_error)
663 663 {
664 664
665 665 vlapic->esr_pending |= mask;
666 666
667 667 /*
668 668 * Avoid infinite recursion if the error LVT itself is configured with
669 669 * an illegal vector.
670 670 */
671 671 if (lvt_error)
672 672 return;
673 673
674 674 if (vlapic_fire_lvt(vlapic, APIC_LVT_ERROR)) {
675 675 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
676 676 }
677 677 }
678 678
679 679 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
680 680
681 681 static void
682 682 vlapic_fire_timer(struct vlapic *vlapic)
683 683 {
684 684
685 685 KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
686 686
687 687 if (vlapic_fire_lvt(vlapic, APIC_LVT_TIMER)) {
688 688 VLAPIC_CTR0(vlapic, "vlapic timer fired");
689 689 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
690 690 }
691 691 }
692 692
693 693 static VMM_STAT(VLAPIC_INTR_CMC,
694 694 "corrected machine check interrupts generated by vlapic");
695 695
696 696 void
697 697 vlapic_fire_cmci(struct vlapic *vlapic)
698 698 {
699 699
700 700 if (vlapic_fire_lvt(vlapic, APIC_LVT_CMCI)) {
701 701 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
702 702 }
703 703 }
704 704
705 705 static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
706 706 "lvts triggered");
707 707
708 708 int
709 709 vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
710 710 {
711 711
712 712 if (vlapic_enabled(vlapic) == false) {
713 713 /*
714 714 * When the local APIC is global/hardware disabled,
715 715 * LINT[1:0] pins are configured as INTR and NMI pins,
716 716 * respectively.
717 717 */
718 718 switch (vector) {
719 719 case APIC_LVT_LINT0:
720 720 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
721 721 break;
722 722 case APIC_LVT_LINT1:
723 723 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
724 724 break;
725 725 default:
726 726 break;
727 727 }
728 728 return (0);
729 729 }
730 730
731 731 switch (vector) {
732 732 case APIC_LVT_LINT0:
733 733 case APIC_LVT_LINT1:
734 734 case APIC_LVT_TIMER:
735 735 case APIC_LVT_ERROR:
736 736 case APIC_LVT_PMC:
737 737 case APIC_LVT_THERMAL:
738 738 case APIC_LVT_CMCI:
739 739 if (vlapic_fire_lvt(vlapic, vector)) {
740 740 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
741 741 LVTS_TRIGGERRED, vector, 1);
742 742 }
743 743 break;
744 744 default:
745 745 return (EINVAL);
746 746 }
747 747 return (0);
748 748 }
749 749
750 750 static void
751 751 vlapic_callout_handler(void *arg)
752 752 {
753 753 struct vlapic *vlapic;
754 754 struct bintime bt, btnow;
755 755 sbintime_t rem_sbt;
756 756
757 757 vlapic = arg;
758 758
759 759 VLAPIC_TIMER_LOCK(vlapic);
760 760 if (callout_pending(&vlapic->callout)) /* callout was reset */
761 761 goto done;
762 762
763 763 if (!callout_active(&vlapic->callout)) /* callout was stopped */
764 764 goto done;
765 765
766 766 callout_deactivate(&vlapic->callout);
767 767
768 768 vlapic_fire_timer(vlapic);
769 769
770 770 if (vlapic_periodic_timer(vlapic)) {
771 771 binuptime(&btnow);
772 772
773 773 KASSERT(BINTIME_CMP(&btnow, >=, &vlapic->timer_fire_bt),
774 774 ("vlapic callout at %lx.%lx, expected at %lx.%lx",
775 775 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
776 776 vlapic->timer_fire_bt.frac));
777 777
778 778 /*
779 779 * Compute the delta between when the timer was supposed to
780 780 * fire and the present time.
781 781 */
782 782 bt = btnow;
783 783 bintime_sub(&bt, &vlapic->timer_fire_bt);
784 784
785 785 rem_sbt = bttosbt(vlapic->timer_period_bt);
786 786 if (BINTIME_CMP(&bt, <, &vlapic->timer_period_bt)) {
787 787 /*
788 788 * Adjust the time until the next countdown downward
789 789 * to account for the lost time.
790 790 */
791 791 rem_sbt -= bttosbt(bt);
792 792 } else {
793 793 /*
794 794 * If the delta is greater than the timer period then
795 795 * just reset our time base instead of trying to catch
796 796 * up.
797 797 */
798 798 vlapic->timer_fire_bt = btnow;
799 799 VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
800 800 "usecs, period is %lu usecs - resetting time base",
801 801 bttosbt(bt) / SBT_1US,
802 802 bttosbt(vlapic->timer_period_bt) / SBT_1US);
803 803 }
804 804
805 805 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
806 806 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
807 807 vlapic_callout_handler, vlapic, 0);
808 808 }
809 809 done:
810 810 VLAPIC_TIMER_UNLOCK(vlapic);
811 811 }
812 812
813 813 void
814 814 vlapic_icrtmr_write_handler(struct vlapic *vlapic)
815 815 {
816 816 struct LAPIC *lapic;
817 817 sbintime_t sbt;
818 818 uint32_t icr_timer;
819 819
820 820 VLAPIC_TIMER_LOCK(vlapic);
821 821
822 822 lapic = vlapic->apic_page;
823 823 icr_timer = lapic->icr_timer;
824 824
825 825 vlapic->timer_period_bt = vlapic->timer_freq_bt;
826 826 bintime_mul(&vlapic->timer_period_bt, icr_timer);
827 827
828 828 if (icr_timer != 0) {
829 829 binuptime(&vlapic->timer_fire_bt);
830 830 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
831 831
832 832 sbt = bttosbt(vlapic->timer_period_bt);
833 833 callout_reset_sbt(&vlapic->callout, sbt, 0,
834 834 vlapic_callout_handler, vlapic, 0);
835 835 } else
836 836 callout_stop(&vlapic->callout);
837 837
838 838 VLAPIC_TIMER_UNLOCK(vlapic);
839 839 }
840 840
841 841 /*
842 842 * This function populates 'dmask' with the set of vcpus that match the
843 843 * addressing specified by the (dest, phys, lowprio) tuple.
844 844 *
845 845 * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
846 846 * or xAPIC (8-bit) destination field.
847 847 */
848 848 void
849 849 vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
850 850 bool lowprio, bool x2apic_dest)
851 851 {
852 852 struct vlapic *vlapic;
853 853 uint32_t dfr, ldr, ldest, cluster;
854 854 uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
855 855 cpuset_t amask;
856 856 int vcpuid;
857 857
858 858 if ((x2apic_dest && dest == 0xffffffff) ||
859 859 (!x2apic_dest && dest == 0xff)) {
860 860 /*
861 861 * Broadcast in both logical and physical modes.
862 862 */
863 863 *dmask = vm_active_cpus(vm);
864 864 return;
865 865 }
866 866
867 867 if (phys) {
868 868 /*
869 869 * Physical mode: destination is APIC ID.
870 870 */
871 871 CPU_ZERO(dmask);
872 872 vcpuid = vm_apicid2vcpuid(vm, dest);
873 873 amask = vm_active_cpus(vm);
874 874 if (vcpuid < vm_get_maxcpus(vm) && CPU_ISSET(vcpuid, &amask))
875 875 CPU_SET(vcpuid, dmask);
876 876 } else {
877 877 /*
878 878 * In the "Flat Model" the MDA is interpreted as an 8-bit wide
879 879 * bitmask. This model is only available in the xAPIC mode.
880 880 */
881 881 mda_flat_ldest = dest & 0xff;
882 882
883 883 /*
884 884 * In the "Cluster Model" the MDA is used to identify a
885 885 * specific cluster and a set of APICs in that cluster.
886 886 */
887 887 if (x2apic_dest) {
888 888 mda_cluster_id = dest >> 16;
889 889 mda_cluster_ldest = dest & 0xffff;
890 890 } else {
891 891 mda_cluster_id = (dest >> 4) & 0xf;
892 892 mda_cluster_ldest = dest & 0xf;
893 893 }
894 894
895 895 /*
896 896 * Logical mode: match each APIC that has a bit set
897 897 * in its LDR that matches a bit in the ldest.
898 898 */
899 899 CPU_ZERO(dmask);
900 900 amask = vm_active_cpus(vm);
901 901 while ((vcpuid = CPU_FFS(&amask)) != 0) {
902 902 vcpuid--;
903 903 CPU_CLR(vcpuid, &amask);
904 904
905 905 vlapic = vm_lapic(vm, vcpuid);
906 906 dfr = vlapic->apic_page->dfr;
907 907 ldr = vlapic->apic_page->ldr;
908 908
909 909 if ((dfr & APIC_DFR_MODEL_MASK) ==
910 910 APIC_DFR_MODEL_FLAT) {
911 911 ldest = ldr >> 24;
912 912 mda_ldest = mda_flat_ldest;
913 913 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
914 914 APIC_DFR_MODEL_CLUSTER) {
915 915 if (x2apic(vlapic)) {
916 916 cluster = ldr >> 16;
917 917 ldest = ldr & 0xffff;
918 918 } else {
919 919 cluster = ldr >> 28;
920 920 ldest = (ldr >> 24) & 0xf;
921 921 }
922 922 if (cluster != mda_cluster_id)
923 923 continue;
924 924 mda_ldest = mda_cluster_ldest;
925 925 } else {
926 926 /*
927 927 * Guest has configured a bad logical
928 928 * model for this vcpu - skip it.
929 929 */
930 930 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
931 931 "model %x - cannot deliver interrupt", dfr);
932 932 continue;
933 933 }
934 934
935 935 if ((mda_ldest & ldest) != 0) {
936 936 CPU_SET(vcpuid, dmask);
937 937 if (lowprio)
938 938 break;
939 939 }
940 940 }
941 941 }
942 942 }
943 943
944 944 static VMM_STAT(VLAPIC_IPI_SEND, "ipis sent from vcpu");
945 945 static VMM_STAT(VLAPIC_IPI_RECV, "ipis received by vcpu");
946 946
947 947 static void
948 948 vlapic_set_tpr(struct vlapic *vlapic, uint8_t val)
949 949 {
950 950 struct LAPIC *lapic = vlapic->apic_page;
951 951
952 952 if (lapic->tpr != val) {
953 953 VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vlapic TPR changed "
954 954 "from %#x to %#x", lapic->tpr, val);
955 955 lapic->tpr = val;
956 956 vlapic_update_ppr(vlapic);
957 957 }
958 958 }
959 959
960 960 static uint8_t
961 961 vlapic_get_tpr(struct vlapic *vlapic)
962 962 {
963 963 struct LAPIC *lapic = vlapic->apic_page;
964 964
965 965 return (lapic->tpr);
966 966 }
967 967
968 968 void
969 969 vlapic_set_cr8(struct vlapic *vlapic, uint64_t val)
970 970 {
971 971 uint8_t tpr;
972 972
973 973 if (val & ~0xf) {
974 974 vm_inject_gp(vlapic->vm, vlapic->vcpuid);
975 975 return;
976 976 }
977 977
978 978 tpr = val << 4;
979 979 vlapic_set_tpr(vlapic, tpr);
980 980 }
981 981
982 982 uint64_t
983 983 vlapic_get_cr8(struct vlapic *vlapic)
984 984 {
|
↓ open down ↓ |
984 lines elided |
↑ open up ↑ |
985 985 uint8_t tpr;
986 986
987 987 tpr = vlapic_get_tpr(vlapic);
988 988 return (tpr >> 4);
989 989 }
990 990
991 991 int
992 992 vlapic_icrlo_write_handler(struct vlapic *vlapic)
993 993 {
994 994 int i;
995 - bool phys;
996 995 cpuset_t dmask;
997 996 uint64_t icrval;
998 - uint32_t dest, vec, mode;
999 - struct vlapic *vlapic2;
997 + uint32_t dest, vec, mode, dsh;
1000 998 struct LAPIC *lapic;
1001 - uint16_t maxcpus;
1002 999
1003 1000 lapic = vlapic->apic_page;
1004 1001 lapic->icr_lo &= ~APIC_DELSTAT_PEND;
1005 1002 icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
1006 1003
1007 1004 if (x2apic(vlapic))
1008 1005 dest = icrval >> 32;
1009 1006 else
1010 1007 dest = icrval >> (32 + 24);
1011 1008 vec = icrval & APIC_VECTOR_MASK;
1012 1009 mode = icrval & APIC_DELMODE_MASK;
1010 + dsh = icrval & APIC_DEST_MASK;
1013 1011
1014 1012 if (mode == APIC_DELMODE_FIXED && vec < 16) {
1015 1013 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR, false);
1016 - VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
1017 1014 return (0);
1018 1015 }
1016 + if (mode == APIC_DELMODE_INIT &&
1017 + (icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT) {
1018 + /* No work required to deassert INIT */
1019 + return (0);
1020 + }
1021 + if ((mode == APIC_DELMODE_STARTUP || mode == APIC_DELMODE_INIT) &&
1022 + !(dsh == APIC_DEST_DESTFLD || dsh == APIC_DEST_ALLESELF)) {
1023 + /*
1024 + * While Intel makes no mention of restrictions for destination
1025 + * shorthand when sending INIT or SIPI, AMD requires either a
1026 + * specific destination or all-excluding self. Common use seems
1027 + * to be restricted to those two cases.
1028 + */
1029 + return (-1);
1030 + }
1019 1031
1020 - VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
1032 + switch (dsh) {
1033 + case APIC_DEST_DESTFLD:
1034 + vlapic_calcdest(vlapic->vm, &dmask, dest,
1035 + (icrval & APIC_DESTMODE_LOG) == 0, false, x2apic(vlapic));
1036 + break;
1037 + case APIC_DEST_SELF:
1038 + CPU_SETOF(vlapic->vcpuid, &dmask);
1039 + break;
1040 + case APIC_DEST_ALLISELF:
1041 + dmask = vm_active_cpus(vlapic->vm);
1042 + break;
1043 + case APIC_DEST_ALLESELF:
1044 + dmask = vm_active_cpus(vlapic->vm);
1045 + CPU_CLR(vlapic->vcpuid, &dmask);
1046 + break;
1047 + default:
1048 + /*
1049 + * All possible delivery notations are covered above.
1050 + * We should never end up here.
1051 + */
1052 + panic("unknown delivery shorthand: %x", dsh);
1053 + }
1021 1054
1022 - if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
1023 - switch (icrval & APIC_DEST_MASK) {
1024 - case APIC_DEST_DESTFLD:
1025 - phys = ((icrval & APIC_DESTMODE_LOG) == 0);
1026 - vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
1027 - x2apic(vlapic));
1055 + while ((i = CPU_FFS(&dmask)) != 0) {
1056 + i--;
1057 + CPU_CLR(i, &dmask);
1058 + switch (mode) {
1059 + case APIC_DELMODE_FIXED:
1060 + lapic_intr_edge(vlapic->vm, i, vec);
1061 + vmm_stat_incr(vlapic->vm, vlapic->vcpuid,
1062 + VLAPIC_IPI_SEND, 1);
1063 + vmm_stat_incr(vlapic->vm, i,
1064 + VLAPIC_IPI_RECV, 1);
1028 1065 break;
1029 - case APIC_DEST_SELF:
1030 - CPU_SETOF(vlapic->vcpuid, &dmask);
1066 + case APIC_DELMODE_NMI:
1067 + vm_inject_nmi(vlapic->vm, i);
1031 1068 break;
1032 - case APIC_DEST_ALLISELF:
1033 - dmask = vm_active_cpus(vlapic->vm);
1069 + case APIC_DELMODE_INIT:
1070 + (void) vm_inject_init(vlapic->vm, i);
1034 1071 break;
1035 - case APIC_DEST_ALLESELF:
1036 - dmask = vm_active_cpus(vlapic->vm);
1037 - CPU_CLR(vlapic->vcpuid, &dmask);
1072 + case APIC_DELMODE_STARTUP:
1073 + (void) vm_inject_sipi(vlapic->vm, i, vec);
1038 1074 break;
1075 + case APIC_DELMODE_LOWPRIO:
1076 + case APIC_DELMODE_SMI:
1039 1077 default:
1040 - CPU_ZERO(&dmask); /* satisfy gcc */
1078 + /* Unhandled IPI modes (for now) */
1041 1079 break;
1042 1080 }
1043 -
1044 - while ((i = CPU_FFS(&dmask)) != 0) {
1045 - i--;
1046 - CPU_CLR(i, &dmask);
1047 - if (mode == APIC_DELMODE_FIXED) {
1048 - lapic_intr_edge(vlapic->vm, i, vec);
1049 - vmm_stat_incr(vlapic->vm, vlapic->vcpuid,
1050 - VLAPIC_IPI_SEND, 1);
1051 - vmm_stat_incr(vlapic->vm, i,
1052 - VLAPIC_IPI_RECV, 1);
1053 - VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
1054 - "to vcpuid %d", vec, i);
1055 - } else {
1056 - vm_inject_nmi(vlapic->vm, i);
1057 - VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
1058 - "to vcpuid %d", i);
1059 - }
1060 - }
1061 -
1062 - return (0); /* handled completely in the kernel */
1063 1081 }
1064 -
1065 - maxcpus = vm_get_maxcpus(vlapic->vm);
1066 - if (mode == APIC_DELMODE_INIT) {
1067 - if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
1068 - return (0);
1069 -
1070 - if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) {
1071 - vlapic2 = vm_lapic(vlapic->vm, dest);
1072 -
1073 - /* move from INIT to waiting-for-SIPI state */
1074 - if (vlapic2->boot_state == BS_INIT) {
1075 - vlapic2->boot_state = BS_SIPI;
1076 - }
1077 -
1078 - return (0);
1079 - }
1080 - }
1081 -
1082 - if (mode == APIC_DELMODE_STARTUP) {
1083 - if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) {
1084 - vlapic2 = vm_lapic(vlapic->vm, dest);
1085 -
1086 - /*
1087 - * Ignore SIPIs in any state other than wait-for-SIPI
1088 - */
1089 - if (vlapic2->boot_state != BS_SIPI)
1090 - return (0);
1091 -
1092 - vlapic2->boot_state = BS_RUNNING;
1093 - vm_req_spinup_ap(vlapic->vm, dest, vec << PAGE_SHIFT);
1094 - return (0);
1095 - }
1096 - }
1097 -
1098 - /* Return to userland. */
1099 - return (-1);
1082 + return (0);
1100 1083 }
1101 1084
1102 1085 void
1103 1086 vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
1104 1087 {
1105 1088 int vec;
1106 1089
1107 1090 KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
1108 1091
1109 1092 vec = val & 0xff;
1110 1093 lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
1111 1094 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_IPI_SEND, 1);
1112 1095 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_IPI_RECV, 1);
1113 1096 VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
1114 1097 }
1115 1098
1116 1099 int
1117 1100 vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1118 1101 {
1119 1102 struct LAPIC *lapic = vlapic->apic_page;
1120 1103 int idx, i, bitpos, vector;
1121 1104 uint32_t *irrptr, val;
1122 1105
1123 1106 if (vlapic->ops.sync_state) {
1124 1107 (*vlapic->ops.sync_state)(vlapic);
1125 1108 }
1126 1109
1127 1110 irrptr = &lapic->irr0;
1128 1111
1129 1112 for (i = 7; i >= 0; i--) {
1130 1113 idx = i * 4;
1131 1114 val = atomic_load_acq_int(&irrptr[idx]);
1132 1115 bitpos = fls(val);
1133 1116 if (bitpos != 0) {
1134 1117 vector = i * 32 + (bitpos - 1);
1135 1118 if (PRIO(vector) > PRIO(lapic->ppr)) {
1136 1119 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1137 1120 if (vecptr != NULL)
1138 1121 *vecptr = vector;
1139 1122 return (1);
1140 1123 } else
1141 1124 break;
1142 1125 }
1143 1126 }
1144 1127 return (0);
1145 1128 }
1146 1129
1147 1130 void
1148 1131 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1149 1132 {
1150 1133 struct LAPIC *lapic = vlapic->apic_page;
1151 1134 uint32_t *irrptr, *isrptr;
1152 1135 int idx;
1153 1136
1154 1137 KASSERT(vector >= 16 && vector < 256, ("invalid vector %d", vector));
1155 1138
1156 1139 if (vlapic->ops.intr_accepted)
1157 1140 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1158 1141
1159 1142 /*
1160 1143 * clear the ready bit for vector being accepted in irr
1161 1144 * and set the vector as in service in isr.
1162 1145 */
1163 1146 idx = (vector / 32) * 4;
1164 1147
1165 1148 irrptr = &lapic->irr0;
1166 1149 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1167 1150 VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1168 1151
1169 1152 isrptr = &lapic->isr0;
1170 1153 isrptr[idx] |= 1 << (vector % 32);
1171 1154 VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1172 1155
1173 1156 /*
1174 1157 * The only way a fresh vector could be accepted into ISR is if it was
1175 1158 * of a higher priority than the current PPR. With that vector now
1176 1159 * in-service, the PPR must be raised.
1177 1160 */
1178 1161 vlapic_raise_ppr(vlapic, vector);
1179 1162
1180 1163 #ifdef __ISRVEC_DEBUG
1181 1164 vlapic_isrstk_accept(vlapic, vector);
1182 1165 #endif
1183 1166 }
1184 1167
1185 1168 void
1186 1169 vlapic_svr_write_handler(struct vlapic *vlapic)
1187 1170 {
1188 1171 struct LAPIC *lapic;
1189 1172 uint32_t old, new, changed;
1190 1173
1191 1174 lapic = vlapic->apic_page;
1192 1175
1193 1176 new = lapic->svr;
1194 1177 old = vlapic->svr_last;
1195 1178 vlapic->svr_last = new;
1196 1179
1197 1180 changed = old ^ new;
1198 1181 if ((changed & APIC_SVR_ENABLE) != 0) {
1199 1182 if ((new & APIC_SVR_ENABLE) == 0) {
1200 1183 /*
1201 1184 * The apic is now disabled so stop the apic timer
1202 1185 * and mask all the LVT entries.
1203 1186 */
1204 1187 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1205 1188 VLAPIC_TIMER_LOCK(vlapic);
1206 1189 callout_stop(&vlapic->callout);
1207 1190 VLAPIC_TIMER_UNLOCK(vlapic);
1208 1191 vlapic_mask_lvts(vlapic);
1209 1192 } else {
1210 1193 /*
1211 1194 * The apic is now enabled so restart the apic timer
1212 1195 * if it is configured in periodic mode.
1213 1196 */
1214 1197 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1215 1198 if (vlapic_periodic_timer(vlapic))
1216 1199 vlapic_icrtmr_write_handler(vlapic);
1217 1200 }
1218 1201 }
1219 1202 }
1220 1203
1221 1204 int
1222 1205 vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1223 1206 uint64_t *data)
1224 1207 {
1225 1208 struct LAPIC *lapic = vlapic->apic_page;
1226 1209 uint32_t *reg;
1227 1210 int i;
1228 1211
1229 1212 /* Ignore MMIO accesses in x2APIC mode */
1230 1213 if (x2apic(vlapic) && mmio_access) {
1231 1214 VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
1232 1215 offset);
1233 1216 *data = 0;
1234 1217 goto done;
1235 1218 }
1236 1219
1237 1220 if (!x2apic(vlapic) && !mmio_access) {
1238 1221 /*
1239 1222 * XXX Generate GP fault for MSR accesses in xAPIC mode
1240 1223 */
1241 1224 VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
1242 1225 "xAPIC mode", offset);
1243 1226 *data = 0;
1244 1227 goto done;
1245 1228 }
1246 1229
1247 1230 if (offset > sizeof (*lapic)) {
1248 1231 *data = 0;
1249 1232 goto done;
1250 1233 }
1251 1234
1252 1235 offset &= ~3;
1253 1236 switch (offset) {
1254 1237 case APIC_OFFSET_ID:
1255 1238 *data = lapic->id;
1256 1239 break;
1257 1240 case APIC_OFFSET_VER:
1258 1241 *data = lapic->version;
1259 1242 break;
1260 1243 case APIC_OFFSET_TPR:
1261 1244 *data = vlapic_get_tpr(vlapic);
1262 1245 break;
1263 1246 case APIC_OFFSET_APR:
1264 1247 *data = lapic->apr;
1265 1248 break;
1266 1249 case APIC_OFFSET_PPR:
1267 1250 *data = lapic->ppr;
1268 1251 break;
1269 1252 case APIC_OFFSET_EOI:
1270 1253 *data = lapic->eoi;
1271 1254 break;
1272 1255 case APIC_OFFSET_LDR:
1273 1256 *data = lapic->ldr;
1274 1257 break;
1275 1258 case APIC_OFFSET_DFR:
1276 1259 *data = lapic->dfr;
1277 1260 break;
1278 1261 case APIC_OFFSET_SVR:
1279 1262 *data = lapic->svr;
1280 1263 break;
1281 1264 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1282 1265 i = (offset - APIC_OFFSET_ISR0) >> 2;
1283 1266 reg = &lapic->isr0;
1284 1267 *data = *(reg + i);
1285 1268 break;
1286 1269 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1287 1270 i = (offset - APIC_OFFSET_TMR0) >> 2;
1288 1271 reg = &lapic->tmr0;
1289 1272 *data = *(reg + i);
1290 1273 break;
1291 1274 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1292 1275 i = (offset - APIC_OFFSET_IRR0) >> 2;
1293 1276 reg = &lapic->irr0;
1294 1277 *data = atomic_load_acq_int(reg + i);
1295 1278 break;
1296 1279 case APIC_OFFSET_ESR:
1297 1280 *data = lapic->esr;
1298 1281 break;
1299 1282 case APIC_OFFSET_ICR_LOW:
1300 1283 *data = lapic->icr_lo;
1301 1284 if (x2apic(vlapic))
1302 1285 *data |= (uint64_t)lapic->icr_hi << 32;
1303 1286 break;
1304 1287 case APIC_OFFSET_ICR_HI:
1305 1288 *data = lapic->icr_hi;
1306 1289 break;
1307 1290 case APIC_OFFSET_CMCI_LVT:
1308 1291 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1309 1292 *data = vlapic_get_lvt(vlapic, offset);
1310 1293 #ifdef INVARIANTS
1311 1294 reg = vlapic_get_lvtptr(vlapic, offset);
1312 1295 KASSERT(*data == *reg, ("inconsistent lvt value at "
1313 1296 "offset %#lx: %#lx/%#x", offset, *data, *reg));
1314 1297 #endif
1315 1298 break;
1316 1299 case APIC_OFFSET_TIMER_ICR:
1317 1300 *data = lapic->icr_timer;
1318 1301 break;
1319 1302 case APIC_OFFSET_TIMER_CCR:
1320 1303 *data = vlapic_get_ccr(vlapic);
1321 1304 break;
1322 1305 case APIC_OFFSET_TIMER_DCR:
1323 1306 *data = lapic->dcr_timer;
1324 1307 break;
1325 1308 case APIC_OFFSET_SELF_IPI:
1326 1309 /*
1327 1310 * XXX generate a GP fault if vlapic is in x2apic mode
1328 1311 */
1329 1312 *data = 0;
1330 1313 break;
1331 1314 case APIC_OFFSET_RRR:
1332 1315 default:
1333 1316 *data = 0;
1334 1317 break;
1335 1318 }
1336 1319 done:
1337 1320 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1338 1321 return (0);
1339 1322 }
1340 1323
1341 1324 int
1342 1325 vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1343 1326 uint64_t data)
1344 1327 {
1345 1328 struct LAPIC *lapic = vlapic->apic_page;
1346 1329 uint32_t *regptr;
1347 1330 int retval;
1348 1331
1349 1332 #ifdef __FreeBSD__
1350 1333 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1351 1334 ("vlapic_write: invalid offset %#lx", offset));
1352 1335 #else
1353 1336 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1354 1337 ("vlapic_write: invalid offset %lx", offset));
1355 1338 #endif
1356 1339
1357 1340 VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
1358 1341 offset, data);
1359 1342
1360 1343 if (offset > sizeof (*lapic))
1361 1344 return (0);
1362 1345
1363 1346 /* Ignore MMIO accesses in x2APIC mode */
1364 1347 if (x2apic(vlapic) && mmio_access) {
1365 1348 VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
1366 1349 "in x2APIC mode", data, offset);
1367 1350 return (0);
1368 1351 }
1369 1352
1370 1353 /*
1371 1354 * XXX Generate GP fault for MSR accesses in xAPIC mode
1372 1355 */
1373 1356 if (!x2apic(vlapic) && !mmio_access) {
1374 1357 VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
1375 1358 "in xAPIC mode", data, offset);
1376 1359 return (0);
1377 1360 }
1378 1361
1379 1362 retval = 0;
1380 1363 switch (offset) {
1381 1364 case APIC_OFFSET_ID:
1382 1365 lapic->id = data;
1383 1366 vlapic_id_write_handler(vlapic);
1384 1367 break;
1385 1368 case APIC_OFFSET_TPR:
1386 1369 vlapic_set_tpr(vlapic, data & 0xff);
1387 1370 break;
1388 1371 case APIC_OFFSET_EOI:
1389 1372 vlapic_process_eoi(vlapic);
1390 1373 break;
1391 1374 case APIC_OFFSET_LDR:
1392 1375 lapic->ldr = data;
1393 1376 vlapic_ldr_write_handler(vlapic);
1394 1377 break;
1395 1378 case APIC_OFFSET_DFR:
1396 1379 lapic->dfr = data;
1397 1380 vlapic_dfr_write_handler(vlapic);
1398 1381 break;
1399 1382 case APIC_OFFSET_SVR:
1400 1383 lapic->svr = data;
1401 1384 vlapic_svr_write_handler(vlapic);
1402 1385 break;
1403 1386 case APIC_OFFSET_ICR_LOW:
1404 1387 lapic->icr_lo = data;
1405 1388 if (x2apic(vlapic))
1406 1389 lapic->icr_hi = data >> 32;
1407 1390 retval = vlapic_icrlo_write_handler(vlapic);
1408 1391 break;
1409 1392 case APIC_OFFSET_ICR_HI:
1410 1393 lapic->icr_hi = data;
1411 1394 break;
1412 1395 case APIC_OFFSET_CMCI_LVT:
1413 1396 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1414 1397 regptr = vlapic_get_lvtptr(vlapic, offset);
1415 1398 *regptr = data;
1416 1399 vlapic_lvt_write_handler(vlapic, offset);
1417 1400 break;
1418 1401 case APIC_OFFSET_TIMER_ICR:
1419 1402 lapic->icr_timer = data;
1420 1403 vlapic_icrtmr_write_handler(vlapic);
1421 1404 break;
1422 1405
1423 1406 case APIC_OFFSET_TIMER_DCR:
1424 1407 lapic->dcr_timer = data;
1425 1408 vlapic_dcr_write_handler(vlapic);
1426 1409 break;
1427 1410
1428 1411 case APIC_OFFSET_ESR:
1429 1412 vlapic_esr_write_handler(vlapic);
1430 1413 break;
1431 1414
1432 1415 case APIC_OFFSET_SELF_IPI:
1433 1416 if (x2apic(vlapic))
1434 1417 vlapic_self_ipi_handler(vlapic, data);
1435 1418 break;
1436 1419
1437 1420 case APIC_OFFSET_VER:
1438 1421 case APIC_OFFSET_APR:
1439 1422 case APIC_OFFSET_PPR:
1440 1423 case APIC_OFFSET_RRR:
1441 1424 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1442 1425 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
|
↓ open down ↓ |
333 lines elided |
↑ open up ↑ |
1443 1426 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1444 1427 case APIC_OFFSET_TIMER_CCR:
1445 1428 default:
1446 1429 // Read only.
1447 1430 break;
1448 1431 }
1449 1432
1450 1433 return (retval);
1451 1434 }
1452 1435
1453 -static void
1436 +void
1454 1437 vlapic_reset(struct vlapic *vlapic)
1455 1438 {
1456 - struct LAPIC *lapic;
1439 + struct LAPIC *lapic = vlapic->apic_page;
1440 + uint32_t *isrptr, *tmrptr, *irrptr;
1457 1441
1458 - lapic = vlapic->apic_page;
1459 - bzero(lapic, sizeof (struct LAPIC));
1442 + /* Reset any timer-related state first */
1443 + VLAPIC_TIMER_LOCK(vlapic);
1444 + callout_stop(&vlapic->callout);
1445 + lapic->icr_timer = 0;
1446 + lapic->ccr_timer = 0;
1447 + VLAPIC_TIMER_UNLOCK(vlapic);
1448 + lapic->dcr_timer = 0;
1449 + vlapic_dcr_write_handler(vlapic);
1460 1450
1451 + /*
1452 + * Sync any APIC acceleration (APICv/AVIC) state into the APIC page so
1453 + * it is not leftover after the reset. This is performed after the APIC
1454 + * timer has been stopped, in case it happened to fire just prior to
1455 + * being deactivated.
1456 + */
1457 + if (vlapic->ops.sync_state) {
1458 + (*vlapic->ops.sync_state)(vlapic);
1459 + }
1460 +
1461 1461 lapic->id = vlapic_get_id(vlapic);
1462 1462 lapic->version = VLAPIC_VERSION;
1463 1463 lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1464 +
1465 + lapic->tpr = 0;
1466 + lapic->apr = 0;
1467 + lapic->ppr = 0;
1468 +
1469 +#ifdef __ISRVEC_DEBUG
1470 + /* With the PPR cleared, the isrvec tracking should be reset too */
1471 + vlapic->isrvec_stk_top = 0;
1472 +#endif
1473 +
1474 + lapic->eoi = 0;
1475 + lapic->ldr = 0;
1464 1476 lapic->dfr = 0xffffffff;
1465 1477 lapic->svr = APIC_SVR_VECTOR;
1466 - vlapic_mask_lvts(vlapic);
1478 + vlapic->svr_last = lapic->svr;
1467 1479
1468 - lapic->dcr_timer = 0;
1469 - vlapic_dcr_write_handler(vlapic);
1480 + isrptr = &lapic->isr0;
1481 + tmrptr = &lapic->tmr0;
1482 + irrptr = &lapic->irr0;
1483 + for (uint_t i = 0; i < 8; i++) {
1484 + atomic_store_rel_int(&isrptr[i * 4], 0);
1485 + atomic_store_rel_int(&tmrptr[i * 4], 0);
1486 + atomic_store_rel_int(&irrptr[i * 4], 0);
1487 + }
1470 1488
1471 - if (vlapic->vcpuid == 0)
1472 - vlapic->boot_state = BS_RUNNING; /* BSP */
1473 - else
1474 - vlapic->boot_state = BS_INIT; /* AP */
1489 + lapic->esr = 0;
1490 + vlapic->esr_pending = 0;
1491 + lapic->icr_lo = 0;
1492 + lapic->icr_hi = 0;
1475 1493
1476 - vlapic->svr_last = lapic->svr;
1494 + lapic->lvt_cmci = 0;
1495 + lapic->lvt_timer = 0;
1496 + lapic->lvt_thermal = 0;
1497 + lapic->lvt_pcint = 0;
1498 + lapic->lvt_lint0 = 0;
1499 + lapic->lvt_lint1 = 0;
1500 + lapic->lvt_error = 0;
1501 + vlapic_mask_lvts(vlapic);
1477 1502 }
1478 1503
1479 1504 void
1480 1505 vlapic_init(struct vlapic *vlapic)
1481 1506 {
1482 1507 KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1483 1508 KASSERT(vlapic->vcpuid >= 0 &&
1484 1509 vlapic->vcpuid < vm_get_maxcpus(vlapic->vm),
1485 1510 ("vlapic_init: vcpuid is not initialized"));
1486 1511 KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1487 1512 "initialized"));
1488 1513
1489 1514 /*
1490 1515 * If the vlapic is configured in x2apic mode then it will be
1491 1516 * accessed in the critical section via the MSR emulation code.
1492 1517 *
1493 1518 * Therefore the timer mutex must be a spinlock because blockable
1494 1519 * mutexes cannot be acquired in a critical section.
1495 1520 */
1496 1521 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1497 1522 callout_init(&vlapic->callout, 1);
1498 1523
1499 1524 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1500 1525
1501 1526 if (vlapic->vcpuid == 0)
1502 1527 vlapic->msr_apicbase |= APICBASE_BSP;
1503 1528
1504 1529 vlapic_reset(vlapic);
1505 1530 }
1506 1531
1507 1532 void
1508 1533 vlapic_cleanup(struct vlapic *vlapic)
1509 1534 {
1510 1535
1511 1536 callout_drain(&vlapic->callout);
1512 1537 }
1513 1538
1514 1539 uint64_t
1515 1540 vlapic_get_apicbase(struct vlapic *vlapic)
1516 1541 {
1517 1542
1518 1543 return (vlapic->msr_apicbase);
1519 1544 }
1520 1545
1521 1546 int
1522 1547 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1523 1548 {
1524 1549
1525 1550 if (vlapic->msr_apicbase != new) {
1526 1551 VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
1527 1552 "not supported", vlapic->msr_apicbase, new);
1528 1553 return (-1);
1529 1554 }
1530 1555
1531 1556 return (0);
1532 1557 }
1533 1558
1534 1559 void
1535 1560 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1536 1561 {
1537 1562 struct vlapic *vlapic;
1538 1563 struct LAPIC *lapic;
1539 1564
1540 1565 vlapic = vm_lapic(vm, vcpuid);
1541 1566
1542 1567 if (state == X2APIC_DISABLED)
1543 1568 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1544 1569 else
1545 1570 vlapic->msr_apicbase |= APICBASE_X2APIC;
1546 1571
1547 1572 /*
1548 1573 * Reset the local APIC registers whose values are mode-dependent.
1549 1574 *
1550 1575 * XXX this works because the APIC mode can be changed only at vcpu
1551 1576 * initialization time.
1552 1577 */
1553 1578 lapic = vlapic->apic_page;
1554 1579 lapic->id = vlapic_get_id(vlapic);
1555 1580 if (x2apic(vlapic)) {
1556 1581 lapic->ldr = x2apic_ldr(vlapic);
1557 1582 lapic->dfr = 0;
1558 1583 } else {
1559 1584 lapic->ldr = 0;
1560 1585 lapic->dfr = 0xffffffff;
1561 1586 }
1562 1587
1563 1588 if (state == X2APIC_ENABLED) {
1564 1589 if (vlapic->ops.enable_x2apic_mode)
1565 1590 (*vlapic->ops.enable_x2apic_mode)(vlapic);
1566 1591 }
1567 1592 }
1568 1593
1569 1594 void
1570 1595 vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1571 1596 int delmode, int vec)
1572 1597 {
1573 1598 bool lowprio;
1574 1599 int vcpuid;
1575 1600 cpuset_t dmask;
1576 1601
1577 1602 if (delmode != IOART_DELFIXED &&
1578 1603 delmode != IOART_DELLOPRI &&
1579 1604 delmode != IOART_DELEXINT) {
1580 1605 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1581 1606 return;
1582 1607 }
1583 1608 lowprio = (delmode == IOART_DELLOPRI);
1584 1609
1585 1610 /*
1586 1611 * We don't provide any virtual interrupt redirection hardware so
1587 1612 * all interrupts originating from the ioapic or MSI specify the
1588 1613 * 'dest' in the legacy xAPIC format.
1589 1614 */
1590 1615 vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1591 1616
1592 1617 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1593 1618 vcpuid--;
1594 1619 CPU_CLR(vcpuid, &dmask);
1595 1620 if (delmode == IOART_DELEXINT) {
1596 1621 vm_inject_extint(vm, vcpuid);
1597 1622 } else {
1598 1623 lapic_set_intr(vm, vcpuid, vec, level);
1599 1624 }
1600 1625 }
1601 1626 }
1602 1627
1603 1628 void
1604 1629 vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1605 1630 {
1606 1631 /*
1607 1632 * Post an interrupt to the vcpu currently running on 'hostcpu'.
1608 1633 *
1609 1634 * This is done by leveraging features like Posted Interrupts (Intel)
1610 1635 * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1611 1636 *
1612 1637 * If neither of these features are available then fallback to
1613 1638 * sending an IPI to 'hostcpu'.
1614 1639 */
1615 1640 if (vlapic->ops.post_intr)
1616 1641 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1617 1642 else
1618 1643 ipi_cpu(hostcpu, ipinum);
1619 1644 }
1620 1645
1621 1646 bool
1622 1647 vlapic_enabled(struct vlapic *vlapic)
1623 1648 {
1624 1649 struct LAPIC *lapic = vlapic->apic_page;
1625 1650
1626 1651 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1627 1652 (lapic->svr & APIC_SVR_ENABLE) != 0)
1628 1653 return (true);
1629 1654 else
1630 1655 return (false);
1631 1656 }
1632 1657
1633 1658 #ifndef __FreeBSD__
1634 1659 void
1635 1660 vlapic_localize_resources(struct vlapic *vlapic)
1636 1661 {
1637 1662 vmm_glue_callout_localize(&vlapic->callout);
1638 1663 }
1639 1664 #endif /* __FreeBSD */
1640 1665
1641 1666 #ifdef __ISRVEC_DEBUG
1642 1667 static void
1643 1668 vlapic_isrstk_eoi(struct vlapic *vlapic, int vector)
1644 1669 {
1645 1670 if (vlapic->isrvec_stk_top <= 0) {
1646 1671 panic("invalid vlapic isrvec_stk_top %d",
1647 1672 vlapic->isrvec_stk_top);
1648 1673 }
1649 1674 vlapic->isrvec_stk_top--;
1650 1675 vlapic_isrstk_verify(vlapic);
1651 1676 }
1652 1677
1653 1678 static void
1654 1679 vlapic_isrstk_accept(struct vlapic *vlapic, int vector)
1655 1680 {
1656 1681 int stk_top;
1657 1682
1658 1683 vlapic->isrvec_stk_top++;
1659 1684
1660 1685 stk_top = vlapic->isrvec_stk_top;
1661 1686 if (stk_top >= ISRVEC_STK_SIZE)
1662 1687 panic("isrvec_stk_top overflow %d", stk_top);
1663 1688
1664 1689 vlapic->isrvec_stk[stk_top] = vector;
1665 1690 vlapic_isrstk_verify(vlapic);
1666 1691 }
1667 1692
1668 1693 static void
1669 1694 vlapic_isrstk_dump(const struct vlapic *vlapic)
1670 1695 {
1671 1696 int i;
1672 1697 uint32_t *isrptr;
1673 1698
1674 1699 isrptr = &vlapic->apic_page->isr0;
1675 1700 for (i = 0; i < 8; i++)
1676 1701 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
1677 1702
1678 1703 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
1679 1704 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
1680 1705 }
1681 1706
1682 1707 static void
1683 1708 vlapic_isrstk_verify(const struct vlapic *vlapic)
1684 1709 {
1685 1710 int i, lastprio, curprio, vector, idx;
1686 1711 uint32_t *isrptr;
1687 1712
1688 1713 /*
1689 1714 * Note: The value at index 0 in isrvec_stk is always 0.
1690 1715 *
1691 1716 * It is a placeholder for the value of ISR vector when no bits are set
1692 1717 * in the ISRx registers.
1693 1718 */
1694 1719 if (vlapic->isrvec_stk_top == 0 && vlapic->isrvec_stk[0] != 0) {
1695 1720 panic("isrvec_stk is corrupted: %d", vlapic->isrvec_stk[0]);
1696 1721 }
1697 1722
1698 1723 /*
1699 1724 * Make sure that the priority of the nested interrupts is
1700 1725 * always increasing.
1701 1726 */
1702 1727 lastprio = -1;
1703 1728 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
1704 1729 curprio = PRIO(vlapic->isrvec_stk[i]);
1705 1730 if (curprio <= lastprio) {
1706 1731 vlapic_isrstk_dump(vlapic);
1707 1732 panic("isrvec_stk does not satisfy invariant");
1708 1733 }
1709 1734 lastprio = curprio;
1710 1735 }
1711 1736
1712 1737 /*
1713 1738 * Make sure that each bit set in the ISRx registers has a
1714 1739 * corresponding entry on the isrvec stack.
1715 1740 */
1716 1741 i = 1;
1717 1742 isrptr = &vlapic->apic_page->isr0;
1718 1743 for (vector = 0; vector < 256; vector++) {
1719 1744 idx = (vector / 32) * 4;
1720 1745 if (isrptr[idx] & (1 << (vector % 32))) {
1721 1746 if (i > vlapic->isrvec_stk_top ||
1722 1747 vlapic->isrvec_stk[i] != vector) {
1723 1748 vlapic_isrstk_dump(vlapic);
1724 1749 panic("ISR and isrvec_stk out of sync");
1725 1750 }
1726 1751 i++;
1727 1752 }
1728 1753 }
1729 1754 }
1730 1755 #endif
|
↓ open down ↓ |
244 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX