--- old/usr/src/uts/intel/io/vmm/intel/vmx_support.S	Fri Mar 22 11:58:07 2024
+++ new/usr/src/uts/intel/io/vmm/intel/vmx_support.S	Fri Mar 22 11:58:07 2024
@@ -36,7 +36,7 @@
  *
  * Copyright 2013 Pluribus Networks Inc.
  * Copyright 2018 Joyent, Inc.
- * Copyright 2022 MNX Cloud, Inc.
+ * Copyright 2024 MNX Cloud, Inc.
  */
 
 #include <sys/asm_linkage.h>
@@ -226,7 +226,7 @@
 	 * NOTE: If RSB mitigations are disabled (see cpuid.c), this call is
 	 * entirely a NOP.
 	 */
-	call	x86_rsb_stuff
+	call	x86_rsb_stuff_vmexit
 
 	/*
 	 * This will return to the caller of 'vmx_enter_guest()' with a return
--- old/usr/src/uts/intel/ml/retpoline.S	Fri Mar 22 11:58:07 2024
+++ new/usr/src/uts/intel/ml/retpoline.S	Fri Mar 22 11:58:07 2024
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2019 Joyent, Inc.
+ * Copyright 2024 MNX Cloud, Inc.
  */
 
 	.file	"retpoline.s"
@@ -119,18 +120,19 @@
 	RETPOLINE_MKJUMP(r15)
 
 	/*
-	 * The x86_rsb_stuff function is called from pretty arbitrary
-	 * contexts. It's much easier for us to save and restore all the
-	 * registers we touch rather than clobber them for callers. You must
-	 * preserve this property or the system will panic at best.
+	 * The x86_rsb_stuff{,_vmexit} functions can be called from pretty
+	 * arbitrary contexts. It's much easier for us to save and restore all
+	 * the registers we touch rather than clobber them for callers. You
+	 * must preserve this property or the system will panic at best.
+	 *
+	 * The two entry points are because the need to RSB stuff on Intel
+	 * depends greatly on factors that are different in the VMEXIT case,
+	 * vs. the other switching cases.  See cpuid.c's cpuid_patch_rsb()
+	 * for details.
 	 */
-	ENTRY(x86_rsb_stuff)
-	/*
-	 * These nops are present so we can patch a ret instruction if we need
-	 * to disable RSB stuffing because enhanced IBRS is present or we're
-	 * disabling mitigations.
-	 */
+	ENTRY(x86_rsb_stuff_vmexit)
 	nop
+	ALTENTRY(x86_rsb_stuff)
 	nop
 	pushq	%rdi
 	pushq	%rax
@@ -154,6 +156,7 @@
 	popq	%rdi
 	ret
 	SET_SIZE(x86_rsb_stuff)
+	SET_SIZE(x86_rsb_stuff_vmexit)
 
 #elif defined(__i386)
 
--- old/usr/src/uts/intel/os/cpuid.c	Fri Mar 22 11:58:07 2024
+++ new/usr/src/uts/intel/os/cpuid.c	Fri Mar 22 11:58:07 2024
@@ -1675,7 +1675,8 @@
 	"avx512_bf16",
 	"auto_ibrs",
 	"rfds_no",
-	"rfds_clear"
+	"rfds_clear",
+	"pbrsb_no"
 };
 
 boolean_t
@@ -2968,12 +2969,22 @@
 }
 
 /*
- * We default to enabling RSB mitigations.
+ * We default to enabling Return Stack Buffer (RSB) mitigations.
  *
- * NOTE: We used to skip RSB mitigations with eIBRS, but developments around
- * post-barrier RSB guessing suggests we should enable RSB mitigations always
- * unless specifically instructed not to.
+ * We used to skip RSB mitigations with Intel eIBRS, but developments around
+ * post-barrier RSB (PBRSB) guessing suggests we should enable Intel RSB
+ * mitigations always unless explicitly bypassed, or unless hardware indicates
+ * the bug has been fixed. Intel also says that machines without eIBRS do not
+ * have VMEXIT problems with PBRSB. Basically, if we're Intel and have eIBRS,
+ * we must stuff the RSB in both context switches AND in VMEXIT, unless the
+ * hardware says the PBRSB bug is fixed.  If we're Intel but without eIBRS
+ * (i.e. using retpolines), we must stuff the RSB in context switches, but we
+ * do not have to for VMEXIT.
  *
+ * See (pardon broken URL)  https://www.intel.com/content/www/us/en/developer \
+ * /articles/technical/software-security-guidance/advisory-guidance
+ * /post-barrier-return-stack-buffer-predictions.html
+ *
  * AMD indicates that when Automatic IBRS is enabled we do not need to implement
  * return stack buffer clearing for VMEXIT as it takes care of it. The manual
  * also states that as long as SMEP and we maintain at least one page between
@@ -2982,17 +2993,39 @@
  * present.
  */
 static void
-cpuid_patch_rsb(x86_spectrev2_mitigation_t mit)
+cpuid_patch_rsb(x86_spectrev2_mitigation_t mit, bool intel_pbrsb_no)
 {
 	const uint8_t ret = RET_INSTR;
 	uint8_t *stuff = (uint8_t *)x86_rsb_stuff;
+	uint8_t *vmx_stuff = (uint8_t *)x86_rsb_stuff_vmexit;
 
 	switch (mit) {
 	case X86_SPECTREV2_AUTO_IBRS:
 	case X86_SPECTREV2_DISABLED:
+		/* Don't bother with any RSB stuffing! */
 		*stuff = ret;
+		*vmx_stuff = ret;
 		break;
+	case X86_SPECTREV2_RETPOLINE:
+		/*
+		 * The Intel document on Post-Barrier RSB says that processors
+		 * without eIBRS do not have PBRSB problems upon VMEXIT.
+		 */
+		VERIFY(!intel_pbrsb_no);
+		VERIFY3U(*stuff, !=, ret);
+		*vmx_stuff = ret;
+		break;
 	default:
+		/*
+		 * eIBRS is all that's left.  If CPU claims PBRSB is fixed,
+		 * don't use the RSB mitigation in either case.
+		 */
+		if (intel_pbrsb_no) {
+			/* CPU claims PBRSB problems are fixed. */
+			*stuff = ret;
+			*vmx_stuff = ret;
+		}
+		VERIFY3U(*stuff, ==, *vmx_stuff);
 		break;
 	}
 }
@@ -3267,6 +3300,10 @@
 					add_x86_feature(featureset,
 					    X86FSET_RFDS_CLEAR);
 				}
+				if (reg & IA32_ARCH_CAP_PBRSB_NO) {
+					add_x86_feature(featureset,
+					    X86FSET_PBRSB_NO);
+				}
 			}
 			no_trap();
 		}
@@ -3327,10 +3364,17 @@
 	}
 
 	cpuid_patch_retpolines(v2mit);
-	cpuid_patch_rsb(v2mit);
 	x86_spectrev2_mitigation = v2mit;
 	membar_producer();
 
+	/*
+	 * Return-stack buffer clearing may need a software-sequence. Discover
+	 * and patch as appropriate, after setting the SPECTREv2 global
+	 * mitigation level.
+	 */
+	cpuid_patch_rsb(v2mit, is_x86_feature(featureset, X86FSET_PBRSB_NO));
+	membar_producer();
+
 	/*
 	 * We need to determine what changes are required for mitigating L1TF
 	 * and MDS. If the CPU suffers from either of them, then SMT exclusion
--- old/usr/src/uts/intel/sys/x86_archext.h	Fri Mar 22 11:58:07 2024
+++ new/usr/src/uts/intel/sys/x86_archext.h	Fri Mar 22 11:58:07 2024
@@ -934,6 +934,7 @@
 #define	X86FSET_AUTO_IBRS	108
 #define	X86FSET_RFDS_NO		109
 #define	X86FSET_RFDS_CLEAR	110
+#define	X86FSET_PBRSB_NO	111
 
 /*
  * Intel Deep C-State invariant TSC in leaf 0x80000007.
@@ -1586,7 +1587,7 @@
 
 #if defined(_KERNEL) || defined(_KMEMUSER)
 
-#define	NUM_X86_FEATURES	111
+#define	NUM_X86_FEATURES	112
 extern uchar_t x86_featureset[];
 
 extern void free_x86_featureset(void *featureset);
@@ -1609,6 +1610,7 @@
  */
 extern void (*spec_uarch_flush)(void);
 extern void x86_rsb_stuff(void);
+extern void x86_rsb_stuff_vmexit(void);
 extern void x86_md_clear(void);
 
 #endif
