import Player 16.1.0 module sources

import Player 16.0.0 module sources
import Player 15.5.5 module sources
2020-11-20 09:13:14 +01:00 · 2020-09-15 20:45:48 +02:00 · 2020-05-30 02:30:43 +02:00 · 2020-03-14 09:59:24 +01:00 · 2019-11-14 07:56:57 +01:00 · 2019-09-21 00:11:26 +02:00
53 changed files with 2661 additions and 1539 deletions
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -136,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998,2015,2017 VMware, Inc. All rights reserved.
+# Copyright (C) 1998,2015,2017,2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -72,7 +72,7 @@ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')

 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
-C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o
+C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o crosspage.o
 C_TARGETS_VMCORE := moduleloop.o
 C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -447,7 +447,7 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 *
 * MonLoaderCallout_GetSharedUserPage --
 *
- *      Gets a user-provided shared page's MPN for a VCPU.
+ *      Gets a shared page's MPN for a VCPU.
 *
 * Returns:
 *      An MPN or INVALID_MPN.
@@ -478,7 +478,14 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   switch (subIndex) {
+   case MONLOADER_CROSS_PAGE_DATA_IDX:
+      return HostIF_GetCrossPageDataMPN(ctx->vm->crosspage[vcpu]);
+   case MONLOADER_CROSS_PAGE_CODE_IDX:
+      return HostIF_GetCrossPageCodeMPN();
+   default:
+      return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   }
 }


@@ -0,0 +1,609 @@
+/*********************************************************
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * crosspage.c --
+ *
+ *    Cross page code and data.
+ *
+ *    The cross code page contains world switch code and interrupt/exception
+ *    handlers in support of world switch.  A separate data page is also mapped
+ *    in both the vmm and vmmon address spaces to describe the host and monitor
+ *    state.
+ *
+ *    Both the monitor and the vmmon driver call into the cross page to switch
+ *    worlds.  The world switch from host->vmm is able to refer directly to the
+ *    various symbols defined in CrossPage_CodePage.  On the vmm->host path
+ *    the VmmToHost entry point is stored in the data page.  Data references
+ *    must be handled as offsets from the base of the data page.
+ *
+ *    Use of inline assembly in C code is delicate as the compiler has many
+ *    possible reasons to emit unexpected instructions surrounding any assembly
+ *    in a C function.  Frame pointers are inconvenient to disable on a per-file
+ *    basis given structure of the Linux Makefile so they are being emitted
+ *    in the C wrapper function.  NORETURN and NOT_REACHED_MINIMAL are used to
+ *    omit a function epilogue.  The output/clobber lists of the assembly blocks
+ *    are not used as the compiler needs not compensate for the register use of
+ *    these hand-written assembly functions.
+ *    Each assembly function begins with a label denoting the proper entry point
+ *    for that function.  Only a handful of the functions need to be exported
+ *    out of this file, see crosspage.h.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#      include <linux/objtool.h>
+#   elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
+#      include <linux/frame.h>
+#   endif
+#endif
+
+#include "modulecall.h"
+#include "iocontrols.h"
+#include "segs.h"
+#include "x86_basic_defs.h"
+#include "vm_basic_defs.h"
+#include "vm_idt_x86.h"
+#include "crosspage.h"
+
+#ifdef __APPLE__
+/*
+ * The Mac compiler expects symbols to have a _ prefix.
+ * This arranges for any symbol being exported out of the assembly code
+ * to be accessible.
+ */
+#define ASM_PREFIX "_"
+/*
+ * OSX uses a segment,section style syntax for specifying output sections.
+ * https://developer.apple.com/library/archive/documentation/Performance/
+ * Conceptual/CodeFootprint/Articles/MachOOverview.html
+ */
+#define ASM_SECTION "__TEXT,cross"
+#else
+/*
+ * This is a COFF grouped section identifier.  This ensures the crosspage code
+ * is merged with the normal .text area but as a separate unit so it can have
+ * its own alignment properties.
+ * https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
+ *
+ * The Linux build will handle the section based on its contents (code).
+ */
+#define ASM_SECTION ".text$cross"
+#define ASM_PREFIX
+#endif
+
+#define EXPORTED_ASM_SYMBOL(fn) ".global " ASM_PREFIX #fn "\n"   \
+                                ASM_PREFIX #fn ":\n"
+
+/*
+ * Tag the crosspage code C wrapper with the crosspage section and page
+ * alignment.
+ */
+#define CPCODE __attribute__((section(ASM_SECTION)))   \
+               __attribute__((aligned(PAGE_SIZE)))
+
+/* No special handling is needed for the data. */
+#define CPDATA
+
+#define VMMDATALA(off) (LPN_2_LA(CROSS_PAGE_DATA_START) + (off))
+
+#define NOT_REACHED_MINIMAL __builtin_unreachable
+
+#ifdef ANNOTATE_INTRA_FUNCTION_CALL
+#define ANNOTATE_ASM_CALL_STR2(x) #x
+#define ANNOTATE_ASM_CALL_STR(x) ANNOTATE_ASM_CALL_STR2(x) "\n"
+#define ANNOTATE_ASM_CALL ANNOTATE_ASM_CALL_STR(ANNOTATE_INTRA_FUNCTION_CALL)
+#else
+#define ANNOTATE_ASM_CALL
+#endif
+
+void VmmToHost(void);
+
+CPDATA const VMCrossPageData cpDataTemplate = {
+   .version        = CROSSPAGE_VERSION,
+   .vmmonVersion   = VMMON_VERSION,
+
+   .monRSP         = VMM_LRET_STACK_TOP,
+   .monSS          = SYSTEM_DATA_SELECTOR,
+   .monDS          = SYSTEM_DATA_SELECTOR,
+   .monES          = SYSTEM_DATA_SELECTOR,
+
+   .vmmToHostLA    = (VA)VmmToHost,
+   .wsCR0          = CR0_PE | CR0_MP | CR0_EM | CR0_NE | CR0_WP | CR0_PG,
+   .wsCR4          = CR4_PAE | CR4_OSFXSR,
+
+   .monTask.rsp[0] = VMM_STACK_TOP,                   /* Monitor stack. */
+   .monTask.rsp[1] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL1 is not used. */
+   .monTask.rsp[2] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL2 is not used. */
+
+   .monTask.ist[IST_NONE]                 = 0, /* No stack switch. */
+   .monTask.ist[IST_VMM_DF]               = DF_STACK_TOP,
+   .monTask.ist[IST_VMM_NMI]              = NMI_STACK_TOP,
+   .monTask.ist[IST_VMM_MCE]              = MC_STACK_TOP,
+
+   .monTask.IOMapBase = sizeof(Task64),
+
+   .monGDTR.limit  = GDT_LIMIT,
+   .monGDTR.offset = GDT_START_VA,
+
+   .shadowDR[6].ureg64 = DR6_DEFAULT,
+   .shadowDR[7].ureg64 = DR7_ONES,
+
+   /*
+    * switchHostIDT and switchHostIDTR are initialized in vmmon.
+    * switchMonIDTR is static and can be fully initialized at compile-time.
+    * switchMonIDT is mostly static with all non-present entries except for
+    * the 4 handled exceptions which are populated by the crosspage setup
+    * code in task.c
+    */
+   .switchMonIDTR  = { sizeof(Gate64) * NUM_EXCEPTIONS - 1,
+                       VMMDATALA(offsetof(VMCrossPageData, switchMonIDT)) },
+};
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * CrossPage_CodePage --
+ *
+ *      This function only serves as an anchor and wrapper for the code that
+ *      runs in the world switch's cross page.  The function must not be called
+ *      directly.  Its address may be used to locate the cross page code.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+CPCODE void
+CrossPage_CodePage(void)
+{
+   __asm__ __volatile__ (
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchDBHandler --
+    *
+    *      DB handler that operates during worldswitch (in both directions).
+    *
+    *      This handler is expected to be used in two cases:
+    *      1) If the host kernel has set the DR7_GD (guard) bit, when switch
+    *         code attempts to save debug registers, a #DB will be thrown.  In
+    *         this case, the handler sets wsException[EXC_DB] and returns.
+    *      2) When stress-testing the worldswitch code, RFLAGS.TF is set across
+    *         most switch paths.  This causes a #DB to be thrown before each
+    *         instruction is executed.  In this case, the handler must set
+    *         RFLAGS.RF in the return frame to guarantee progress while
+    *         RFLAGS.TF is set.  For added stress, the handler induces a
+    *         simulated #NMI.
+    *
+    *      In case (1), returning is accomplished via a simulated iret in order
+    *      to preserve any NMI-blocking.  In case (2), the inherent
+    *      incompatibility of RFLAGS.RF for single-stepping and simulating an
+    *      iret via popf and lretq necessitates the use of an actual iret.  As
+    *      a result, there is no strong NMI-blocking guarantee when
+    *      stress-testing the switch path.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_DB] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchDBHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionDB], %%rax\n"
+   "movb         $1,               (%%rax)\n" /* log EXC_DB */
+   "popq         %%rax\n"
+   "testl        %[EFLAGS_TF],     16(%%rsp)\n" /* check trap flag */
+   "jz           return_without_enabling_nmi\n"
+   "orl          %[EFLAGS_RF],     16(%%rsp)\n"
+   "int          $2\n"
+   "iretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchUDHandler --
+    *
+    *      UD handler that operates during worldswitch (in both directions).
+    *
+    *      It sets the wsException[EXC_UD] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_UD] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchUDHandler)
+   "pushq        %%rax\n"
+   "pushq        %%rbx\n"
+   "pushq        %%rcx\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "movl         %[wsExceptionUD],      %%ecx\n"    /* log EXC_UD */
+   "movb         $1,                    (%%rax, %%rcx)\n"
+   /* Check if the exception came from a monitor RIP. */
+   "cmpq         %[MONITOR_MINIMUM_VA], 24(%%rsp)\n"
+   "jae          monitor_context\n"
+   /* Check if the exception came from the crosspage code. */
+   "movq         24(%%rsp),              %%rbx\n"
+   "andq         %[PageAlignMask],       %%rbx\n"
+   "movl         %[crosspageCodeLA],     %%ecx\n"
+   "cmpq         (%%rax, %%rcx),         %%rbx\n"
+   "je           monitor_context\n"
+   /*
+    * Linux halts the processor on host #UD2.  Act similarly.  PR 1281662.
+    * Note that this can escape due to an #NMI, #MC, or #DB.
+    */
+   "cli\n"
+   "hlt\n"
+   /* Advance 2 bytes to skip past the #UD2 instruction. */
+   "monitor_context:\n"
+   "movq         24(%%rsp), %%rbx\n"
+   "movl         %[wsUD2],  %%ecx\n"
+   "movq         %%rbx,     (%%rax, %%rcx)\n"
+   "addq         $2,        24(%%rsp)\n"
+   "popq         %%rcx\n"
+   "popq         %%rbx\n"
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchNMIHandler --
+    *
+    *      NMI handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It 'simply' sets the wsException[EXC_NMI] flag then returns,
+    *      leaving further NMI delivery inhibited.
+    *
+    *      As long as we don't execute 'iret', the CPU will not allow further
+    *      NMIs to be delivered; this is how the CPU protects itself from
+    *      runaway NMIs eating up the stack and triple-faulting.
+    *
+    *      See Vol 3, 5.5.1, Handling multiple NMIs.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_NMI] = TRUE
+    *      further NMI delivery inhibited
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchNMIHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionNMI], %%rax\n"
+   "movb         $1,                (%%rax)\n" /* log EXC_NMI */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchMCEHandler --
+    *
+    *      MCE handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It simply sets the wsException[EXC_MC] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_MC] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchMCEHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionMC], %%rax\n"
+   "movb         $1,              (%%rax)\n" /* log EXC_MC */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchHandlerReturns --
+    *
+    *      Shared code to return without enabling NMIs.  Switch handlers call
+    *      directly to these labels as needed.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "return_without_enabling_nmi:\n"
+   "pushq        %%rbp\n"
+   "pushq        %%rax\n"
+   "movq         40(%%rsp), %%rbp\n"
+   "subq         $32, %%rbp\n"
+   "movq         24(%%rsp), %%rax\n"
+   "movq         %%rax, 24(%%rbp)\n"
+   "movq         16(%%rsp), %%rax\n"
+   "movq         %%rax, 16(%%rbp)\n"
+   "movq         32(%%rsp), %%rax\n"
+   "movq         %%rax, 8(%%rbp)\n"
+   "movq         8(%%rsp),  %%rax\n"
+   "movq         %%rax, (%%rbp)\n"
+   "popq         %%rax\n"
+   "movq         %%rbp, %%rsp\n"
+   "popq         %%rbp\n"
+   "popfq\n"
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * HostToVmm --
+    *
+    *      Switch from host to monitor.
+    *
+    * Entered with:
+    *      RCX = crosspage host address
+    *      (RSP) = return to host address
+    *
+    * Must preserve:
+    *      RBX, RSI, RDI, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(HostToVmm)
+   /* Create an lret frame on the host stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold host %rsp */
+
+   /* Start from the "empty" host context and save. */
+   "leaq            %c[hostContextEmpty](%%rcx), %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* host %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rdi\n"
+   "pushq           %%rsi\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" monitor context and load. */
+   "leaq            %c[monContextFull](%%rcx), %%rsp\n"
+   "popq            %%rsi\n" /* monitor %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+   "popq            %%rax\n" /* monitor %rsp */
+   "popw            %%dx\n"  /* monitor %ss */
+   /* Load the cross GDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%rsi, %%cr3\n"
+   /* Reload the monitor's %ss into %ds/%es. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * VmmToHost --
+    *
+    *      Switch from monitor to host.
+    *
+    * Must preserve:
+    *      RBX, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(VmmToHost)
+   "movq            %c[VMMCROSSPAGE] + %c[crosspageDataLA], %%rcx\n"
+   /* Create an lret frame on the monitor stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold monitor %rsp */
+
+   /* Start from the "empty" monitor context and save. */
+   "leaq            %c[VMMCROSSPAGE] + %c[monContextEmpty], %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* monitor %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" host context and load. */
+   "leaq            %c[hostContextFull](%%rcx), %%rsp\n"
+   "popq            %%r9\n" /* host %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%rsi\n" /* Not ABI-required. */
+   "popq            %%rdi\n" /* Not ABI-required. */
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+
+   "popq            %%rax\n" /* host %rsp */
+   "popw            %%dx\n"  /* host %ss */
+   /* Load the cross GDT and IDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   "lidtq           %c[switchHostIDTR](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%r9,  %%cr3\n"
+   /* Reload the host's %ss into %ds/%es.  Technically wrong. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   /* Microsoft RTL/codegen assumes EFLAGS<DF> = 0. */
+   "cld\n"
+   "lretq\n"
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchExcGetCrossPageData --
+    *
+    *      Common function for the exception handlers to locate the data
+    *      crosspage so they can record their respective events.  In order to
+    *      reach this code an exception had to vector through the IDT.  The IDT
+    *      is known to be in the data page.  Therefore, the data page can be
+    *      found by accessing IDTR and rounding down to page alignment.
+    *
+    * Input:
+    *     None
+    *
+    * Output:
+    *     %rax = Page aligned address of the current crosspage data area.
+    *
+    * Note:
+    *     %rax (return value) and %rflags are destroyed, all other registers
+    *     are preserved.  Since this is only called by exception handlers, the
+    *     CPU has already saved %rflags so no additional handling is required.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "SwitchExcGetCrossPageData:\n"
+   "subq            $0x10,            %%rsp\n"
+   "sidt            0(%%rsp)\n"
+   "movq            2(%%rsp),         %%rax\n" /* DTR.offset */
+   "addq            $0x10,            %%rsp\n"
+   "andq            %[PageAlignMask], %%rax\n"
+   "ret\n"
+
+   EXPORTED_ASM_SYMBOL(CrossPage_CodeEnd)
+
+   : /* No output list, this is not really C code. */
+   : [MONITOR_MINIMUM_VA] "i" (MONITOR_LINEAR_START),
+     [PageAlignMask]      "i" (~PAGE_MASK),
+     [EFLAGS_TF]          "i" (EFLAGS_TF),
+     [EFLAGS_RF]          "i" (EFLAGS_RF),
+     [VMMCROSSPAGE]       "i" (CROSS_PAGE_DATA_START * PAGE_SIZE),
+     [wsUD2]              "i" (offsetof(VMCrossPageData, wsUD2)),
+
+     [wsExceptionDB]      "i" (offsetof(VMCrossPageData, wsException[EXC_DB])),
+     [wsExceptionUD]      "i" (offsetof(VMCrossPageData, wsException[EXC_UD])),
+     [wsExceptionNMI]     "i" (offsetof(VMCrossPageData, wsException[EXC_NMI])),
+     [wsExceptionMC]      "i" (offsetof(VMCrossPageData, wsException[EXC_MC])),
+
+     [hostContextEmpty]   "i" (offsetof(VMCrossPageData, hostDS)),
+     [monContextEmpty]    "i" (offsetof(VMCrossPageData, monDS)),
+
+     [hostContextFull]    "i" (offsetof(VMCrossPageData, hostCR3)),
+     [monContextFull]     "i" (offsetof(VMCrossPageData, monCR3)),
+
+     [crosspageDataLA]    "i" (offsetof(VMCrossPageData, crosspageDataLA)),
+     [crosspageCodeLA]    "i" (offsetof(VMCrossPageData, crosspageCodeLA)),
+     [crossGDTHKLADesc]   "i" (offsetof(VMCrossPageData, crossGDTHKLADesc)),
+     [switchHostIDTR]     "i" (offsetof(VMCrossPageData, switchHostIDTR))
+   );
+   NOT_REACHED_MINIMAL();
+}
+
+#ifdef STACK_FRAME_NON_STANDARD
+STACK_FRAME_NON_STANDARD(CrossPage_CodePage);
+#endif
+
+/*
+ * The crosspage data must not exceed a single page.
+ * The code area also must fit in a single page but that size isn't available
+ * until link time.
+ */
+MY_ASSERTS(CROSSPAGE_DATA,
+   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
+)
@@ -0,0 +1,43 @@
+/*********************************************************
+ * Copyright (C) 2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _CROSSPAGE_H_
+#define _CROSSPAGE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct VMCrossPageData;
+
+extern void HostToVmm(struct VMCrossPageData *crosspageWIN, void *unused1,
+                      void *unused2, struct VMCrossPageData *crosspageSYSV);
+extern const VMCrossPageData cpDataTemplate;
+
+extern void SwitchDBHandler(void);
+extern void SwitchUDHandler(void);
+extern void SwitchNMIHandler(void);
+extern void SwitchMCEHandler(void);
+extern void CrossPage_CodePage(void);
+extern void CrossPage_CodeEnd(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
@@ -68,7 +68,9 @@ typedef struct HostIFContigMemMap {

 EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
-EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);
+
+EXTERN MPN   HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData);
+EXTERN MPN   HostIF_GetCrossPageCodeMPN(void);

 EXTERN void *HostIF_AllocPage(void);
 EXTERN void  HostIF_FreePage(void *ptr);
@@ -101,13 +103,14 @@ EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 EXTERN void  HostIF_APICInit(VMDriver *vm);
 EXTERN uint8 HostIF_GetMonitorIPIVector(void);
 EXTERN uint8 HostIF_GetHVIPIVector(void);
+EXTERN uint8 HostIF_GetPerfCtrVector(void);
 EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);

 EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
                                  Vcpuid vcpuid,
                                  uint64 *args);

-EXTERN int   HostIF_SemaphoreSignal(uint64 *args);
+EXTERN int   HostIF_SemaphoreSignal(VMDriver *vm, uint64 *args);

 EXTERN void  HostIF_SemaphoreForceWakeup(VMDriver *vm, const VCPUSet *vcs);
 EXTERN void  HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,6 +66,8 @@
 #include "x86paging_64.h"
 #include "memtrack.h"
 #include "monLoader.h"
+#include "segs.h"
+#include "crosspage.h"

 #ifdef LINUX_GDT_IS_RO
 #   include <asm/desc.h>
@@ -588,7 +590,13 @@ Bool
 Task_Initialize(void)
 {
   unsigned i;
+   VA cpStart = (VA)CrossPage_CodePage;
+   VA cpEnd   = (VA)CrossPage_CodeEnd;

+   if (VA_2_VPN(cpStart) != VA_2_VPN(cpEnd)) {
+      Warning("Crosspage code validation failed.\n");
+      return FALSE;
+   }
   ASSERT_ON_COMPILE(sizeof(Atomic_uint64) == sizeof(MPN));
   for (i = 0; i < ARRAYSIZE(hvRootPage); i++) {
      Atomic_Write64(&hvRootPage[i], INVALID_MPN);
@@ -828,7 +836,7 @@ TaskSwitchPTPLookupVPN(VMDriver *vm, VPN vpn)
 *    before/after doing a BackToHost.
 *
 *    For each populated patch, performs a page walk from L4 to the patch's
- *    level.  The lowest allowed level is L3.  Invalid patch levels, collisions
+ *    level.  The lowest allowed level is L2.  Invalid patch levels, collisions
 *    at the patch's level with existing PTEs and non-present PTEs during the
 *    walk all result in failure.
 *
@@ -847,6 +855,7 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
      VMMPageTablePatch *patch = &cpData->vmmPTP[i];
      unsigned l4idx = PT_LPN_2_L4OFF(patch->lpn);
      unsigned l3idx = PT_LPN_2_L3OFF(patch->lpn);
+      unsigned l2idx = PT_LPN_2_L2OFF(patch->lpn);
      PT_L4E pte = 0;
      MA ma = cpData->monCR3 + l4idx * sizeof pte; /* PTE machine address. */

@@ -886,6 +895,34 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
               return FALSE;
            }
            break;
+         case 2:
+            ASSERT(patch->pteIdx == l2idx);
+            if (!PTE_PRESENT(pte)) {
+               return FALSE; /* Terminate page walk, L4 not present. */
+            }
+            ma = MPN_2_MA(LM_PTE_2_PFN(pte)) + l3idx * sizeof pte;
+            if (HostIF_ReadPhysical(vm, ma, PtrToVA64(&pte), TRUE,
+                                    sizeof pte) != 0) {
+               return FALSE;
+            }
+            if (!PTE_PRESENT(pte)) {
+               return FALSE; /* Terminate page walk, L3 not present. */
+            }
+            ma = MPN_2_MA(LM_PTE_2_PFN(pte)) + l2idx * sizeof pte;
+            if (HostIF_ReadPhysical(vm, ma, PtrToVA64(&pte), TRUE,
+                                    sizeof pte) != 0) {
+               return FALSE;
+            }
+            if (pte != 0) {
+               return FALSE; /* Collision. */
+            }
+
+            if (HostIF_WritePhysical(vm, ma, PtrToVA64(&patch->pte), TRUE,
+                                     sizeof patch->pte) != 0) {
+               return FALSE;
+            }
+            break;
+
         default:
            return FALSE; /* Invalid level. */
            break;
@@ -945,11 +982,14 @@ TaskVerifyPTPatches(VMDriver *vm,
                    VMCrossPageData *cpData)
 {
   LPN64 xgCPN = LA_2_LPN(cpData->crossGDTLA);
-   LPN64 xpCPN = LA_2_LPN(cpData->crossPageLA);
-   MPN   xpMPN = MA_2_MPN(cpData->crosspageMA);
+   LPN64 xpCPN = LA_2_LPN(cpData->crosspageDataLA);
+   MPN   xpMPN = MA_2_MPN(cpData->crosspageDataMA);
   MA    cr3   = cpData->monCR3;
+   LPN64 xpCodeCPN = LA_2_LPN(cpData->crosspageCodeLA);
+   MPN   xpCodeMPN = HostIF_GetCrossPageCodeMPN();
   return TaskVerifyPTMap(vm, cr3, xpCPN, xpMPN) &&
-          TaskVerifyPTMap(vm, cr3, xgCPN, crossGDTMPN);
+          TaskVerifyPTMap(vm, cr3, xgCPN, crossGDTMPN) &&
+          TaskVerifyPTMap(vm, cr3, xpCodeCPN, xpCodeMPN);
 }


@@ -1172,11 +1212,10 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
 /*
 *-----------------------------------------------------------------------------
 *
- * TaskPopulateHostSwitchIDTE --
+ * TaskPopulateSwitchIDTE --
 *
- *      Populates a host interrupt descriptor in the crosspage, setting its
- *      handler to the associated gate stub, given a host code segment selector
- *      and the crosspage.
+ *      Populates an interrupt descriptor in the crosspage, setting its
+ *      handler to the associated gate stub given a code segment selector.
 *
 * Results:
 *      None
@@ -1188,17 +1227,13 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
 */

 static void
-TaskPopulateHostSwitchIDTE(unsigned idx, Selector cs, VMCrossPage *cp)
+TaskPopulateSwitchIDTE(unsigned idx, Selector cs, Gate64 *idt, VA handlerVA)
 {
   Gate64 *entry;
-   VA64 handlerVA;

   ASSERT_ON_COMPILE(sizeof *entry == 16);
   ASSERT(idx < NUM_EXCEPTIONS);
-
-   handlerVA = PtrToVA64(&cp->crosspageCode.gateStubs[idx * CP_STUB_SIZE]);
-
-   entry = &cp->crosspageData.switchHostIDT[idx];
+   entry = &idt[idx];

   entry->offset_0_15  = LOWORD(handlerVA);
   entry->offset_16_31 = HIWORD(handlerVA);
@@ -1212,13 +1247,21 @@ TaskPopulateHostSwitchIDTE(unsigned idx, Selector cs, VMCrossPage *cp)
   entry->reserved1    = 0;
 }

+static VA
+TaskVmmCodeVA(void (*handler)(void))
+{
+   VA baseVA = VPN_2_VA(CROSS_PAGE_CODE_START);
+   VA offs = (VA)handler % PAGE_SIZE;
+
+   return baseVA + offs;
+}

 /*
 *-----------------------------------------------------------------------------
 *
- *  TaskInitHostSwitchIDT --
+ *  TaskInitSwitchIDT --
 *
- *      Initializes the contents of the host-context switch IDT and IDTR.
+ *      Initializes the contents of the switch IDTs and IDTR.
 *      Uses the cross page's host kernel linear address and host kernel CS.
 *
 * Results:
@@ -1231,18 +1274,28 @@ TaskPopulateHostSwitchIDTE(unsigned idx, Selector cs, VMCrossPage *cp)
 */

 static void
-TaskInitHostSwitchIDT(VMCrossPage *crosspage)
+TaskInitSwitchIDTs(VMCrossPageData *cpData)
 {
-   VMCrossPageData *cpData = &crosspage->crosspageData;
-   const Selector   cs     = cpData->hostInitial64CS;
+   const Selector hostCS = cpData->hostInitial64CS;
+   Gate64 *switchHostIDT = cpData->switchHostIDT;
+   Gate64 *switchMonIDT  = cpData->switchMonIDT;

   cpData->switchHostIDTR.limit = sizeof cpData->switchHostIDT - 1;
   cpData->switchHostIDTR.offset = PtrToVA64(&cpData->switchHostIDT);

-   TaskPopulateHostSwitchIDTE(EXC_DB,  cs, crosspage);
-   TaskPopulateHostSwitchIDTE(EXC_NMI, cs, crosspage);
-   TaskPopulateHostSwitchIDTE(EXC_UD,  cs, crosspage);
-   TaskPopulateHostSwitchIDTE(EXC_MC,  cs, crosspage);
+   TaskPopulateSwitchIDTE(EXC_DB,  hostCS, switchHostIDT, (VA)SwitchDBHandler);
+   TaskPopulateSwitchIDTE(EXC_NMI, hostCS, switchHostIDT, (VA)SwitchNMIHandler);
+   TaskPopulateSwitchIDTE(EXC_UD,  hostCS, switchHostIDT, (VA)SwitchUDHandler);
+   TaskPopulateSwitchIDTE(EXC_MC,  hostCS, switchHostIDT, (VA)SwitchMCEHandler);
+
+   TaskPopulateSwitchIDTE(EXC_DB,  SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchDBHandler));
+   TaskPopulateSwitchIDTE(EXC_NMI, SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchNMIHandler));
+   TaskPopulateSwitchIDTE(EXC_UD,  SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchUDHandler));
+   TaskPopulateSwitchIDTE(EXC_MC,  SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchMCEHandler));
 }


@@ -1554,7 +1607,7 @@ TaskCreatePTPatch(VMDriver        *vm,
 /*
 *-----------------------------------------------------------------------------
 *
- * TaskCreatePTPatches
+ * TaskCreatePTPatches --
 *
 *      Creates VMM mappings for the crosspage and crossGDT that match
 *      those from the vmmon address space, so that they are mapped at
@@ -1569,16 +1622,19 @@ TaskCreatePTPatch(VMDriver        *vm,
 */

 static Bool
-TaskCreatePTPatches(VMDriver    *vm,
-                    VMCrossPage *crosspage,
-                    LPN64        monStartLPN,
-                    LPN64        monEndLPN,
-                    uint16      *numPages)
+TaskCreatePTPatches(VMDriver        *vm,
+                    VMCrossPageData *cpData,
+                    LPN64            monStartLPN,
+                    LPN64            monEndLPN,
+                    uint16          *numPages)
 {
-   VMCrossPageData * const cpData = &crosspage->crosspageData;
-   LPN64                   xgCPN  = LA_2_LPN(cpData->crossGDTLA);
-   LPN64                   xpCPN  = LA_2_LPN(cpData->crossPageLA);
-   MPN                     xpMPN  = MA_2_MPN(cpData->crosspageMA);
+   LPN64                   xgCPN     = LA_2_LPN(cpData->crossGDTLA);
+   LPN64                   xpCPN     = LA_2_LPN(cpData->crosspageDataLA);
+   MPN                     xpMPN     = MA_2_MPN(cpData->crosspageDataMA);
+   LPN64                   xpCodeCPN = LA_2_LPN(cpData->crosspageCodeLA);
+   MPN                     xpCodeMPN = HostIF_GetCrossPageCodeMPN();
+
+   ASSERT(xpCodeMPN != INVALID_MPN);
   /*
    * Set up patches that the BackToHost code will use to map the
    * crosspage and crossGDT at their crossover addresses (HKLA for
@@ -1587,7 +1643,9 @@ TaskCreatePTPatches(VMDriver    *vm,
   if (TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
                         xpCPN, xpMPN, numPages) &&
       TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
-                         xgCPN, crossGDTMPN, numPages)) {
+                         xgCPN, crossGDTMPN, numPages) &&
+       TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
+                         xpCodeCPN, xpCodeMPN, numPages)) {
      TaskFixupPTPatches(vm, cpData);
      return TRUE;
   }
@@ -1612,9 +1670,9 @@ TaskCreatePTPatches(VMDriver    *vm,
 */

 Bool
-Task_InitCrosspage(VMDriver *vm,               // IN
-                   LPN monStartLPN,            // IN
-                   LPN monEndLPN,              // IN
+Task_InitCrosspage(VMDriver     *vm,           // IN
+                   LPN           monStartLPN,  // IN
+                   LPN           monEndLPN,    // IN
                   PerVcpuPages *perVcpuPages) // IN
 {
   Vcpuid vcpuid;
@@ -1626,31 +1684,19 @@ Task_InitCrosspage(VMDriver *vm,               // IN

   ASSERT(0 < vm->numVCPUs && vm->numVCPUs <= MAX_VCPUS);
   for (vcpuid = 0; vcpuid < vm->numVCPUs; vcpuid++) {
-      VA64             crossPageUserAddr = perVcpuPages[vcpuid].crosspage;
-      VMCrossPage     *p = HostIF_MapCrossPage(vm, crossPageUserAddr);
-      VMCrossPageData *cpData;
-      MPN              crossPageMPN;
+      VMCrossPageData *cpData = vm->crosspage[vcpuid];
+      MPN              crossPageDataMPN;

-      if (p == NULL) {
-         return FALSE;
-      }
-      cpData = &p->crosspageData;
-
-      HostIF_VMLock(vm, 38);
-      if (HostIF_LookupUserMPN(vm, crossPageUserAddr, &crossPageMPN) !=
-          PAGE_LOOKUP_SUCCESS ||
-          crossPageMPN == 0) {
-         HostIF_VMUnlock(vm, 38);
-         return FALSE;
-      }
-      HostIF_VMUnlock(vm, 38);
+      crossPageDataMPN = HostIF_GetCrossPageDataMPN(cpData);
+      ASSERT(crossPageDataMPN != INVALID_MPN);
+      memcpy(cpData, &cpDataTemplate, sizeof *cpData);

      /*
       * The version of the crosspage must be the first four bytes of the
       * crosspage.  See the declaration of VMCrossPage in modulecall.h.
       */

-      ASSERT_ON_COMPILE(offsetof(VMCrossPage, crosspageData.version) == 0);
+      ASSERT_ON_COMPILE(offsetof(VMCrossPageData, version) == 0);
      ASSERT_ON_COMPILE(sizeof(cpData->version) == sizeof(uint32));

      /* cpData->version is VMX's version; CROSSPAGE_VERSION is vmmon's. */
@@ -1660,12 +1706,9 @@ Task_InitCrosspage(VMDriver *vm,               // IN
                 (int)CROSSPAGE_VERSION, cpData->version);
         return FALSE;
      }
-      if (!pseudoTSC.initialized) {
-         Warning("%s*: PseudoTSC has not been initialized\n", __FUNCTION__);
-         return FALSE;
-      }
-      cpData->crosspageMA = MPN_2_MA(crossPageMPN);
-      cpData->crossPageLA = (LA64)(uintptr_t)p;
+      cpData->crosspageDataMA = MPN_2_MA(crossPageDataMPN);
+      cpData->crosspageDataLA = (LA64)(uintptr_t)cpData;
+      cpData->crosspageCodeLA = (LA64)CrossPage_CodePage;
      cpData->crossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.offset = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.limit  = sizeof(CrossGDT) - 1;
@@ -1675,7 +1718,8 @@ Task_InitCrosspage(VMDriver *vm,               // IN
      cpData->monCR3 = MPN_2_MA(vm->ptRootMpns[vcpuid]);

      HostIF_VMLock(vm, 39);
-      if (!TaskCreatePTPatches(vm, p, monStartLPN, monEndLPN, &numPTPPages)) {
+      if (!TaskCreatePTPatches(vm, cpData, monStartLPN, monEndLPN,
+                               &numPTPPages)) {
         HostIF_VMUnlock(vm, 39);
         Warning("%s: Could not create page table patches for VCPU %d\n",
                 __FUNCTION__, vcpuid);
@@ -1713,9 +1757,8 @@ Task_InitCrosspage(VMDriver *vm,               // IN
      cpData->timerIntrTS           = MAX_ABSOLUTE_TS;
      cpData->hstTimerExpiry        = MAX_ABSOLUTE_TS;
      cpData->monTimerExpiry        = MAX_ABSOLUTE_TS;
-      vm->crosspage[vcpuid]         = p;

-      TaskInitHostSwitchIDT(p);
+      TaskInitSwitchIDTs(cpData);
   }
   /*
    * Store the number of pages allocated for this VM's page table patches so
@@ -1870,16 +1913,16 @@ TaskDisableTF(void)


 static INLINE Bool
-TaskGotException(const VMCrossPage *crosspage, unsigned exc)
+TaskGotException(const VMCrossPageData *crosspageData, unsigned exc)
 {
-   return crosspage->crosspageData.wsException[exc];
+   return crosspageData->wsException[exc];
 }


 static INLINE void
-TaskSetException(VMCrossPage *crosspage, unsigned exc, Bool v)
+TaskSetException(VMCrossPageData *crosspageData, unsigned exc, Bool v)
 {
-   crosspage->crosspageData.wsException[exc] = v;
+   crosspageData->wsException[exc] = v;
 }


@@ -1905,7 +1948,7 @@ TaskSetException(VMCrossPage *crosspage, unsigned exc, Bool v)
 */

 static INLINE_SINGLE_CALLER void
-TaskSaveDebugRegisters(VMCrossPage *crosspage)
+TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
 {
   Bool saveGotDB;

@@ -1913,13 +1956,13 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
   do {                                                 \
      uintptr_t drReg;                                  \
      GET_DR##n(drReg);                                 \
-      crosspage->crosspageData.hostDR[n] = drReg;       \
+      crosspageData->hostDR[n] = drReg;                 \
   } while (0)

   /* Hardware contains the host's %dr7, %dr6, %dr3, %dr2, %dr1, %dr0 */
-   crosspage->crosspageData.hostDRInHW = ((1 << 7) | (1 << 6) |
-                                          (1 << 3) | (1 << 2) |
-                                          (1 << 1) | (1 << 0));
+   crosspageData->hostDRInHW = ((1 << 7) | (1 << 6) |
+                                (1 << 3) | (1 << 2) |
+                                (1 << 1) | (1 << 0));

   /*
    * Save DR7 since we need to disable debug breakpoints during the world
@@ -1927,8 +1970,8 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * SwitchDBHandler simply IRETs after setting crosspage gotDB flag.
    */

-   saveGotDB = TaskGotException(crosspage, EXC_DB);
-   TaskSetException(crosspage, EXC_DB, FALSE);
+   saveGotDB = TaskGotException(crosspageData, EXC_DB);
+   TaskSetException(crosspageData, EXC_DB, FALSE);
   COMPILER_MEM_BARRIER();      /* Prevent hoisting #UD-raising instructions. */
   SAVE_DR(7);

@@ -1936,7 +1979,7 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * In all cases, DR7 shouldn't have the GD bit set.
    */

-   TS_ASSERT(!(crosspage->crosspageData.hostDR[7] & DR7_GD));
+   TS_ASSERT(!(crosspageData->hostDR[7] & DR7_GD));

   /*
    * Save DR6 in order to accommodate the ICEBP instruction and other stuff
@@ -1961,15 +2004,15 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * SAVE_DR(6) can raise #DB.
    */

-   if (TaskGotException(crosspage, EXC_DB) &&
-       (crosspage->crosspageData.hostDR[6] & DR6_BD)) {
-      crosspage->crosspageData.hostDR[6] -= DR6_BD;
-      crosspage->crosspageData.hostDR[7] |= DR7_GD;
+   if (TaskGotException(crosspageData, EXC_DB) &&
+       (crosspageData->hostDR[6] & DR6_BD)) {
+      crosspageData->hostDR[6] -= DR6_BD;
+      crosspageData->hostDR[7] |= DR7_GD;
      SET_DR7(DR7_DEFAULT);

      /* HW: %dr7 and %dr6 are the guest, %dr3, %dr2, %dr1, %dr0 are host */
-      crosspage->crosspageData.hostDRInHW = ((1 << 3) | (1 << 2) |
-                                             (1 << 1) | (1 << 0));
+      crosspageData->hostDRInHW = ((1 << 3) | (1 << 2) |
+                                   (1 << 1) | (1 << 0));
   }

   /*
@@ -1977,20 +2020,20 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * coincidentally trip during the switch.
    */

-   else if (crosspage->crosspageData.hostDR[7] & DR7_ENABLED) {
+   else if (crosspageData->hostDR[7] & DR7_ENABLED) {
      SET_DR7(DR7_DEFAULT);          // no #DB here, just simple set
      /* HW: %dr7 = guest, %dr6, %dr3, %dr2, %dr1, %dr0 = host */
-      crosspage->crosspageData.hostDRInHW = ((1 << 6) | (1 << 3) | (1 << 2) |
-                                             (1 << 1) | (1 << 0));
+      crosspageData->hostDRInHW = ((1 << 6) | (1 << 3) | (1 << 2) |
+                                   (1 << 1) | (1 << 0));
   }

-   TaskSetException(crosspage, EXC_DB, saveGotDB);
+   TaskSetException(crosspageData, EXC_DB, saveGotDB);

   /*
    * hostDR[6,7] have host contents in them now.
    */

-   crosspage->crosspageData.hostDRSaved = 0xC0;
+   crosspageData->hostDRSaved = 0xC0;
 #undef SAVE_DR
 }

@@ -2259,14 +2302,10 @@ TaskUpdatePTSCParameters(VMDriver *vm,
 *
 *      Wrapper that calls code to switch from the host to the monitor.
 *
- *      The basic idea is to do a (*(crosspage->hostToVmm))(crosspage)
+ *      The basic idea is to do a HostToVmm(crosspage)
 *      but it's complicated because we must have a common call format
 *      between GCC and MSC.
 *
- *      Since we have complete control over what GCC does with asm volatile,
- *      this amounts to having GCC do exactly what MSC does.
- *      For 64-bit hosts, we pass the parameter in RCX.
- *
 *      For 64-bit GCC, the callee is expected to preserve
 *      RBX,RBP,RSP,R12..R15, whereas MSC expects the callee to preserve
 *      RBX,RSI,RDI,RBP,RSP,R12..R15.  So for simplicity, we have the
@@ -2293,48 +2332,24 @@ TaskUpdatePTSCParameters(VMDriver *vm,
 */

 static INLINE_SINGLE_CALLER void
-TaskSwitchToMonitor(VMCrossPage *crosspage)
+TaskSwitchToMonitor(VMCrossPageData *crosspageData)
 {
-   const void *codePtr = ((void *)&crosspage->crosspageCode.toVmmFunc);
-
-#if defined(__GNUC__)
   /*
-    * Pass the crosspage pointer in RCX just like 64-bit MSC does.
-    * Tell GCC that the worldswitch preserves RBX,RSI,RDI,RBP,RSP,
-    * R12..R15 just like the MSC 64-bit calling convention.
+    * Arrange for RCX to contain the crosspage pointer using the MS ABI (first
+    * argument) and with the SYSV ABI (4th argument).
    */
-
-   {
-      uint64 raxGetsWiped, rcxGetsWiped;
-
-      __asm__ __volatile__("call *%%rax"
-                           : "=a" (raxGetsWiped),
-                             "=c" (rcxGetsWiped)
-                           : "0" (codePtr),
-                             "1" (crosspage)
-                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
-   }
-#elif defined(_MSC_VER)
-   /*
-    * The 64-bit calling convention is to pass the argument in RCX and that
-    * the called function must preserve RBX,RSI,RDI,RBP,RSP,R12..R15.
-    */
-#pragma warning(suppress: 4055) // Cast of data pointer to function pointer.
-   (*(void (*)(VMCrossPage *))codePtr)(crosspage);
-#else
-#error No compiler defined for TaskSwitchToMonitor
-#endif
+   HostToVmm(crosspageData, NULL, NULL, crosspageData);
 }


 static void
-TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
+TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
 {
   static Bool testSwitchNMI = TRUE; /* test only first time through */

   /*
    * Test the DB,NMI,MCE handlers to make sure they can set the
-    * flags.  This is calling the handlers in switchNMI.S.
+    * flags.  This is calling the handlers in crosspage.c.
    */

   if (vmx86_debug && testSwitchNMI) {
@@ -2343,8 +2358,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      testSwitchNMI = FALSE;

      /*
-       * RAISE_INTERRUPT calls Switch{32,64}DBHandler in switchNMI.S
-       * (depending on host bitsize).
+       * RAISE_INTERRUPT calls SwitchDBHandler in crosspage.c.
       */

      gotSave = TaskGotException(crosspage, EXC_DB);
@@ -2354,8 +2368,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      TaskSetException(crosspage, EXC_DB, gotSave);

      /*
-       * RAISE_INTERRUPT calls Switch{32,64}NMIHandler in switchNMI.S
-       * (depending on host bitsize).
+       * RAISE_INTERRUPT calls SwitchNMIHandler in crosspage.c.
       */
      gotSave = TaskGotException(crosspage, EXC_NMI);
      TaskSetException(crosspage, EXC_NMI, FALSE);
@@ -2366,7 +2379,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      /*
       * Test the LRETQ in the 64-bit mini NMI handler to make sure
       * it works with any 16-byte offset of the stack pointer.
-       * The INT 2 calls Switch64NMIHandler in switchNMI.S.
+       * The INT 2 calls SwitchNMIHandler in crosspage.c.
       */
      {
         uint64 v1, v2;
@@ -2395,8 +2408,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      TaskSetException(crosspage, EXC_NMI, gotSave);

      /*
-       * RAISE_INTERRUPT calls Switch{32,64}MCEHandler in switchNMI.S
-       * (depending on host bitsize).
+       * RAISE_INTERRUPT calls SwitchMCEHandler in crosspage.c.
       */

      gotSave = TaskGotException(crosspage, EXC_MC);
@@ -2425,10 +2437,10 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
 */

 static INLINE Bool
-TaskShouldRetryWorldSwitch(VMCrossPage *crosspage)
+TaskShouldRetryWorldSwitch(VMCrossPageData *crosspage)
 {
-   Bool result = crosspage->crosspageData.retryWorldSwitch;
-   crosspage->crosspageData.retryWorldSwitch = FALSE;
+   Bool result = crosspage->retryWorldSwitch;
+   crosspage->retryWorldSwitch = FALSE;
   return result;
 }

@@ -2447,16 +2459,13 @@ TaskShouldRetryWorldSwitch(VMCrossPage *crosspage)
 *      state.
 *
 * Results:
- *      None.
- *
- * Side effects:
- *      Jump to the monitor. Has no direct effect on the host-visible
- *      state except that it might generate an interrupt.
+ *      TRUE - If a #UD did not occur during the context switch.
+ *      FALSE - Otherwise.
 *
 *-----------------------------------------------------------------------------
 */

-void
+Bool
 Task_Switch(VMDriver *vm,  // IN
            Vcpuid vcpuid) // IN
 {
@@ -2474,10 +2483,11 @@ Task_Switch(VMDriver *vm,  // IN
   Bool lint1NMI;
   Bool pcNMI;
   Bool thermalNMI;
-   VMCrossPage *crosspage;
+   VMCrossPageData *crosspage;
   uint32 pCPU;
   MPN hvRootMPN;
   Descriptor *tempGDTBase;
+   Bool switchOk = TRUE;
 #ifdef CYCLE_SPEC_CTRL
   Bool specCtrlEqual = FALSE;
   uint64 readSpecCtrlValue = 0;
@@ -2486,8 +2496,9 @@ Task_Switch(VMDriver *vm,  // IN

   ASSERT(vcpuid < vm->numVCPUs);
   crosspage = vm->crosspage[vcpuid];
+   ASSERT(crosspage != NULL);

-   ASSERT_ON_COMPILE(sizeof(VMCrossPage) == PAGE_SIZE);
+   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
   TaskDisableNMI(&vm->hostAPIC, &lint0NMI, &lint1NMI, &pcNMI, &thermalNMI);
   SAVE_FLAGS(flags);
   CLEAR_INTERRUPTS();
@@ -2506,13 +2517,13 @@ Task_Switch(VMDriver *vm,  // IN
    * crosspages arguments when doing this though, see bug 820257.
    */
   if (hvRootMPN == INVALID_MPN && CPUID_HostSupportsHV()) {
-      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
-      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
-      crosspage->crosspageData.pcpuNum = pCPU;
+      crosspage->userCallType = MODULECALL_USERCALL_NONE;
+      crosspage->moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
+      crosspage->pcpuNum = pCPU;
   } else if (USE_TEMPORARY_GDT && tempGDTBase == NULL) {
-      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
-      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_TMP_GDT;
-      crosspage->crosspageData.pcpuNum = pCPU;
+      crosspage->userCallType = MODULECALL_USERCALL_NONE;
+      crosspage->moduleCallType = MODULECALL_ALLOC_TMP_GDT;
+      crosspage->pcpuNum = pCPU;
   } else {
      do {
         uintptr_t cr0reg, cr2reg, cr3reg, cr4reg;
@@ -2523,7 +2534,7 @@ Task_Switch(VMDriver *vm,  // IN

         Atomic_Write32(&vm->currentHostCpu[vcpuid], pCPU);

-         TaskUpdatePTSCParameters(vm, &crosspage->crosspageData, vcpuid);
+         TaskUpdatePTSCParameters(vm, crosspage, vcpuid);

         /*
          * Disable PEBS if it is supported and enabled.  Do this while on the
@@ -2554,7 +2565,7 @@ Task_Switch(VMDriver *vm,  // IN
          */

         TaskSaveIDT(&hostIDT);
-         TaskLoadIDT(&crosspage->crosspageData.switchHostIDTR);
+         TaskLoadIDT(&crosspage->switchHostIDTR);
         TaskTestCrossPageExceptionHandlers(crosspage);

         if (CPUID_HostSupportsVT()) {
@@ -2580,21 +2591,21 @@ Task_Switch(VMDriver *vm,  // IN
         GET_CR0(cr0reg);
         GET_CR4(cr4reg);
         GET_CR3(cr3reg);
-         crosspage->crosspageData.hostCR3 = cr3reg;
+         crosspage->hostCR3 = cr3reg;

         /*
          * Any reserved bits in CR0 must be preserved when we switch
          * to the VMM. [See PR 291004.]  (On the other hand, Intel
          * recommends that we clear any reserved CR4 bits.)
          */
-         crosspage->crosspageData.wsCR0 &= ~CR0_RESERVED;
-         crosspage->crosspageData.wsCR0 |= (cr0reg & CR0_RESERVED);
+         crosspage->wsCR0 &= ~CR0_RESERVED;
+         crosspage->wsCR0 |= (cr0reg & CR0_RESERVED);

         /*
          * CR4.VMXE must be enabled to support VMX in the monitor.
          */
         if (CPUID_HostSupportsVT()) {
-            crosspage->crosspageData.wsCR4 |= CR4_VMXE;
+            crosspage->wsCR4 |= CR4_VMXE;
         }

         /*
@@ -2602,8 +2613,8 @@ Task_Switch(VMDriver *vm,  // IN
          * host's values.  CR4.PCIDE will be cleared once we're in the monitor,
          * running on a CR3 with a PCID field of 0.
          */
-         crosspage->crosspageData.wsCR4 =
-            (crosspage->crosspageData.wsCR4 & ~(CR4_MCE | CR4_PCIDE)) |
+         crosspage->wsCR4 =
+            (crosspage->wsCR4 & ~(CR4_MCE | CR4_PCIDE)) |
            (cr4reg & (CR4_MCE | CR4_PCIDE));

         /*
@@ -2611,15 +2622,15 @@ Task_Switch(VMDriver *vm,  // IN
          * switching to the monitor's CR4 ensures that global pages are
          * flushed.
          */
-         ASSERT((crosspage->crosspageData.wsCR4 & CR4_PGE) == 0);
+         ASSERT((crosspage->wsCR4 & CR4_PGE) == 0);

         /*
          * Load the world-switch CR0 and CR4.  We can't load the monitor's
          * CR3 yet, because the current code isn't mapped into the
          * monitor's address space.
          */
-         SET_CR0((uintptr_t)crosspage->crosspageData.wsCR0);
-         SET_CR4((uintptr_t)crosspage->crosspageData.wsCR4);
+         SET_CR0((uintptr_t)crosspage->wsCR0);
+         SET_CR4((uintptr_t)crosspage->wsCR4);

         TaskSaveDebugRegisters(crosspage);

@@ -2708,23 +2719,22 @@ Task_Switch(VMDriver *vm,  // IN
            currentSpecCtrlValue = (currentSpecCtrlValue + 1) % 4;
            X86MSR_SetMSR(MSR_SPEC_CTRL, currentSpecCtrlValue);
 #endif
-            crosspage->crosspageData.specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
+            crosspage->specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
         }

-         DEBUG_ONLY(crosspage->crosspageData.monTinyStack[0] = 0xDEADBEEF;)
-         DEBUG_ONLY(crosspage->crosspageData.hostTinyStack[0] = 0xDEADBEEF;)
+         DEBUG_ONLY(crosspage->monTinyStack[0] = 0xDEADBEEF;)
+         DEBUG_ONLY(crosspage->hostTinyStack[0] = 0xDEADBEEF;)
         /* Running in host context prior to TaskSwitchToMonitor() */
         TaskSwitchToMonitor(crosspage);
         /* Running in host context after to TaskSwitchToMonitor() */

-         TS_ASSERT(crosspage->crosspageData.monTinyStack[0] == 0xDEADBEEF);
-         TS_ASSERT(crosspage->crosspageData.hostTinyStack[0] == 0xDEADBEEF);
+         TS_ASSERT(crosspage->monTinyStack[0] == 0xDEADBEEF);
+         TS_ASSERT(crosspage->hostTinyStack[0] == 0xDEADBEEF);

 #ifdef CYCLE_SPEC_CTRL
         if (CPUID_HostSupportsSpecCtrl()) {
            readSpecCtrlValue = X86MSR_GetMSR(MSR_SPEC_CTRL);
-            specCtrlEqual = readSpecCtrlValue ==
-                            crosspage->crosspageData.specCtrl;
+            specCtrlEqual = readSpecCtrlValue == crosspage->specCtrl;
            /* Do not leak cycling SPEC_CTRL value back to host. */
            X86MSR_SetMSR(MSR_SPEC_CTRL, 0);
         }
@@ -2762,10 +2772,10 @@ Task_Switch(VMDriver *vm,  // IN
          * CR2 always has to be restored.  CR8 never has to be restored.
          */
         SET_CR2(cr2reg);
-         if (crosspage->crosspageData.wsCR0 != cr0reg) {
+         if (crosspage->wsCR0 != cr0reg) {
            SET_CR0(cr0reg);
         }
-         if (crosspage->crosspageData.wsCR4 != cr4reg) {
+         if (crosspage->wsCR4 != cr4reg) {
            SET_CR4(cr4reg);
         } else if ((cr4reg & CR4_PCIDE) != 0) {
            /*
@@ -2813,7 +2823,7 @@ Task_Switch(VMDriver *vm,  // IN
            TaskDisableTF();
         }

-         TaskRestoreDebugRegisters(&crosspage->crosspageData);
+         TaskRestoreDebugRegisters(crosspage);

         ASSERT_NO_INTERRUPTS();

@@ -2832,7 +2842,7 @@ Task_Switch(VMDriver *vm,  // IN
            X86MSR_SetMSR(MSR_RTIT_CTL, ptMSR);
         }

-         TaskUpdateLatestPTSC(vm, &crosspage->crosspageData);
+         TaskUpdateLatestPTSC(vm, crosspage);
         Atomic_Write32(&vm->currentHostCpu[vcpuid], INVALID_PCPU);

         /*
@@ -2864,12 +2874,13 @@ Task_Switch(VMDriver *vm,  // IN
            RAISE_INTERRUPT(EXC_MC);
         }
         if (UNLIKELY(TaskGotException(crosspage, EXC_UD))) {
-            Warning("#UD occurred on switch back to host; dumping core.\n");
+            switchOk = FALSE;
+            Warning("#UD occurred on switch back to host.\n");
         }
 #ifdef CYCLE_SPEC_CTRL
         if (UNLIKELY(!specCtrlEqual)) {
            Warning("SpecCtrl not equal: expected %"FMT64"x, got %"FMT64"x\n",
-                    crosspage->crosspageData.specCtrl, readSpecCtrlValue);
+                    crosspage->specCtrl, readSpecCtrlValue);
         }
 #endif
         /*
@@ -2877,8 +2888,8 @@ Task_Switch(VMDriver *vm,  // IN
          * received during worldswitch.  Here is the more generic case
          * of forwarding NMIs received while executing the VMM/guest.
          */
-         if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR &&
-             crosspage->crosspageData.args[0] == EXC_NMI) {
+         if (crosspage->moduleCallType == MODULECALL_INTR &&
+             crosspage->args[0] == EXC_NMI) {
            /*
             * If VMM was interrupted by an NMI, do the INT 2 so the
             * host will handle it, but then return immediately to the
@@ -2888,12 +2899,12 @@ Task_Switch(VMDriver *vm,  // IN
             * done the EOI.
             */
            RAISE_INTERRUPT(EXC_NMI);
-            crosspage->crosspageData.retryWorldSwitch = TRUE;
+            crosspage->retryWorldSwitch = TRUE;
         }
      } while (UNLIKELY(TaskShouldRetryWorldSwitch(crosspage)));
   }

-   if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR) {
+   if (crosspage->moduleCallType == MODULECALL_INTR) {

      /*
       * Newer versions of Window expect EFLAGS_AC to be set when handling an
@@ -2912,15 +2923,15 @@ Task_Switch(VMDriver *vm,  // IN
       * what bad things btw).
       */
 #ifdef _WIN64
-      if (crosspage->crosspageData.args[0] <= 0xFF &&
-          (crosspage->crosspageData.args[0] >= 0x14 ||
-           crosspage->crosspageData.args[0] == EXC_MC)) {
-         RAISE_INTERRUPT((unsigned char)crosspage->crosspageData.args[0]);
+      if (crosspage->args[0] <= 0xFF &&
+          (crosspage->args[0] >= 0x14 ||
+           crosspage->args[0] == EXC_MC)) {
+         RAISE_INTERRUPT((unsigned char)crosspage->args[0]);
      } else {
         Warning("%s: Received Unexpected Interrupt: 0x%"FMT64"X\n",
-                 __FUNCTION__, crosspage->crosspageData.args[0]);
+                 __FUNCTION__, crosspage->args[0]);
         Panic("Received Unexpected Interrupt: 0x%"FMT64"X\n",
-               crosspage->crosspageData.args[0]);
+               crosspage->args[0]);
      }
 #else
      /*
@@ -2934,7 +2945,7 @@ Task_Switch(VMDriver *vm,  // IN
 #define IRQ_INT16(_x) IRQ_INT8(_x); IRQ_INT8(_x + 8)
 #define IRQ_INT32(_x) IRQ_INT16(_x); IRQ_INT16(_x + 16)

-      switch (crosspage->crosspageData.args[0]) {
+      switch (crosspage->args[0]) {
         // These are the general IO interrupts
         // It would be nice to generate this dynamically, but see Note2 above.

@@ -2967,13 +2978,14 @@ Task_Switch(VMDriver *vm,  // IN
          */

         Warning("%s: Received Unexpected Interrupt: 0x%"FMT64"X\n",
-                 __FUNCTION__, crosspage->crosspageData.args[0]);
+                 __FUNCTION__, crosspage->args[0]);
         Panic("Received Unexpected Interrupt: 0x%"FMT64"X\n",
-               crosspage->crosspageData.args[0]);
+               crosspage->args[0]);
      }
 #endif
   }

   RESTORE_FLAGS(flags);
   TaskRestoreNMI(&vm->hostAPIC, lint0NMI, lint1NMI, pcNMI, thermalNMI);
+   return switchOk;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@ struct MonLoaderHeader;
 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
 extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
                               PerVcpuPages *perVcpuPages);
-extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
+extern Bool Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
 extern MPN Task_GetHVRootPageForPCPU(uint32 pCPU);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -112,8 +112,9 @@ static unsigned globalFastClockRate;
 HostIFContigMemMap *hvIOBitmap;

 typedef struct {
-   Atomic_uint32 index;
-   MSRQuery *query;
+   Atomic_uint32 *index; // OUT: array of cpu counters for queries.
+   MSRQuery *query;  // IN/OUT: array of query items
+   uint32 numItems;  // IN
 } Vmx86GetMSRData;

 static Bool hostUsesNX;
@@ -123,6 +124,20 @@ typedef struct NXData {
   Atomic_uint32 hasNX;
 } NXData;

+/*
+ * A structure for holding MSR indexes, values for MSR uniformity checks.
+ */
+typedef struct VMX86MSRCacheInfo {
+   uint32 msrIndex;
+   uint64 msrValue;
+} VMX86MSRCacheInfo;
+
+static VMX86MSRCacheInfo msrUniformityCacheInfo[] = {
+#define MSR_FEAT(member) {member, CONST64(0)},
+   MSR_FEAT(IA32_MSR_ARCH_CAPABILITIES)
+#undef MSR_FEAT
+};
+
 /*
 *----------------------------------------------------------------------
 *
@@ -567,6 +582,66 @@ Vmx86_Calloc(size_t numElements, // IN
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86AllocCrossPages --
+ *
+ *      Allocate numVCPUs pages suitable to be used as the VCPU's
+ *      crosspage area.
+ *
+ * Results:
+ *      TRUE if the required crosspages are allocated successfully.
+ *      FALSE otherwise.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static Bool
+Vmx86AllocCrossPages(VMDriver *vm)
+{
+   Vcpuid v;
+
+   for (v = 0; v < vm->numVCPUs; v++) {
+      MPN unused;
+
+      UNUSED_VARIABLE(unused);
+      vm->crosspage[v] = HostIF_AllocKernelPages(1, &unused);
+
+      if (vm->crosspage[v] == NULL) {
+         return FALSE;
+      }
+      memset(vm->crosspage[v], 0, PAGE_SIZE);
+   }
+   return TRUE;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86FreeCrossPages --
+ *
+ *      Free the crosspages allocated for the given VM.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static void
+Vmx86FreeCrossPages(VMDriver *vm)
+{
+   Vcpuid v;
+
+   if (vm->crosspage != NULL) {
+      for (v = 0; v < vm->numVCPUs; v++) {
+         if (vm->crosspage[v] != NULL) {
+            HostIF_FreeKernelPages(1, vm->crosspage[v]);
+         }
+      }
+   }
+}
+
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -749,6 +824,7 @@ Vmx86FreeAllVMResources(VMDriver *vm)

      Vmx86_SetHostClockRate(vm, 0);

+      Vmx86FreeCrossPages(vm);
      if (vm->ptpTracker != NULL) {
         Task_SwitchPTPPageCleanup(vm);
      }
@@ -1197,6 +1273,15 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
      goto error;
   }

+   if (!pseudoTSC.initialized) {
+      Warning("%s: PseudoTSC has not been initialized\n", __FUNCTION__);
+      goto error;
+   }
+
+   if (!Vmx86AllocCrossPages(vm)) {
+      Warning("Failed to allocate cross pages.\n");
+      goto error;
+   }
   /*
    * Initialize the driver's part of the cross-over page used to
    * talk to the monitor.
@@ -1638,12 +1723,12 @@ Vmx86_MonTimerIPI(void)
      VCPUSet_Empty(&expiredVCPUs);

      for (v = 0; v < vm->numVCPUs; v++) {
-         VMCrossPage *crosspage = vm->crosspage[v];
+         VMCrossPageData *crosspage = vm->crosspage[v];

-         if (!crosspage) {
+         if (crosspage == NULL) {
            continue;  // VCPU is not initialized yet
         }
-         expiry = crosspage->crosspageData.monTimerExpiry;
+         expiry = crosspage->monTimerExpiry;
         if (expiry != 0 && expiry <= pNow) {
            VCPUSet_Include(&expiredVCPUs, v);
            hasWork = TRUE;
@@ -2779,45 +2864,47 @@ Vmx86_CheckPseudoTSC(uint64 *lastTSC, // IN/OUT: last/current value of the TSC
 static void
 Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 {
+   uint32 i;
   Vmx86GetMSRData *data = (Vmx86GetMSRData *)clientData;
-   MSRQuery *query;
-   uint32 index;
-   int err;
+   ASSERT(data && data->index && data->query);

-   ASSERT(data);
-   query = data->query;
-   ASSERT(query);
+   for (i = 0; i < data->numItems; ++i) {
+      uint32 index;
+      int err;
+      Atomic_uint32 *cpus = &data->index[i];
+      MSRQuery *query = &data->query[i];

-   index = Atomic_ReadInc32(&data->index);
-   if (index >= query->numLogicalCPUs) {
-      return;
-   }
-
-   query->logicalCPUs[index].tag = HostIF_GetCurrentPCPU();
-
-   /*
-    * We treat BIOS_SIGN_ID (microcode version) specially on Intel,
-    * where the preferred read sequence involves a macro.
-    */
-
-   if (CPUID_GetVendor() == CPUID_VENDOR_INTEL &&
-       query->msrNum == MSR_BIOS_SIGN_ID) {
-      /* safe to read: MSR_BIOS_SIGN_ID architectural since Pentium Pro */
-      query->logicalCPUs[index].msrVal = INTEL_MICROCODE_VERSION();
-      err = 0;
-   } else {
-      /*
-       * Try to enable HV any time these MSRs are queried.  We have seen
-       * buggy formware that forgets to re-enable HV after waking from
-       * deep sleep. [PR 1020692]
-       */
-      if (query->msrNum == MSR_FEATCTL || query->msrNum == MSR_VM_CR) {
-         Vmx86EnableHVOnCPU();
+      index = Atomic_ReadInc32(cpus);
+      if (index >= query->numLogicalCPUs) {
+         continue;
      }
-      err = HostIF_SafeRDMSR(query->msrNum, &query->logicalCPUs[index].msrVal);
-   }

-   query->logicalCPUs[index].implemented = (err == 0) ? 1 : 0;
+      query->logicalCPUs[index].tag = HostIF_GetCurrentPCPU();
+
+      /*
+       * We treat BIOS_SIGN_ID (microcode version) specially on Intel,
+       * where the preferred read sequence involves a macro.
+       */
+      if (CPUID_GetVendor() == CPUID_VENDOR_INTEL &&
+          query->msrNum == MSR_BIOS_SIGN_ID) {
+         /* safe to read: MSR_BIOS_SIGN_ID architectural since Pentium Pro */
+         query->logicalCPUs[index].msrVal = INTEL_MICROCODE_VERSION();
+         err = 0;
+      } else {
+         /*
+          * Try to enable HV any time these MSRs are queried.  We have seen
+          * buggy firmware that forgets to re-enable HV after waking from
+          * deep sleep. [PR 1020692]
+          */
+         if (query->msrNum == MSR_FEATCTL || query->msrNum == MSR_VM_CR) {
+            Vmx86EnableHVOnCPU();
+         }
+         err =
+            HostIF_SafeRDMSR(query->msrNum, &query->logicalCPUs[index].msrVal);
+      }
+
+      query->logicalCPUs[index].implemented = (err == 0) ? 1 : 0;
+   }
 }


@@ -2826,7 +2913,7 @@ Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 *
 * Vmx86_GetAllMSRs --
 *
- *      Collect MSR value on all logical CPUs.
+ *      Collect MSR value on number of logical CPUs requested.
 *
 *      The caller is responsible for ensuring that the requested MSR is valid
 *      on all logical CPUs.
@@ -2848,24 +2935,40 @@ Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 Bool
 Vmx86_GetAllMSRs(MSRQuery *query) // IN/OUT
 {
+   unsigned i, cpu;
+   Atomic_uint32 index;
   Vmx86GetMSRData data;
+   data.index = &index;
+   data.numItems = 1;

-   Atomic_Write32(&data.index, 0);
+   /* Check MSR uniformity cache first. */
+   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
+      if (msrUniformityCacheInfo[i].msrIndex == query->msrNum) {
+         for (cpu = 0; cpu < query->numLogicalCPUs; cpu++) {
+            query->logicalCPUs[cpu].msrVal = msrUniformityCacheInfo[i].msrValue;
+            query->logicalCPUs[cpu].implemented = 1;
+            query->logicalCPUs[cpu].tag = cpu;
+         }
+         return TRUE;
+      }
+   }
+
+   Atomic_Write32(data.index, 0);
   data.query = query;

   HostIF_CallOnEachCPU(Vmx86GetMSR, &data);

   /*
-    * At this point, Atomic_Read32(&data.index) is the number of logical CPUs
+    * At this point, Atomic_Read32(data.index) is the number of logical CPUs
    * who replied.
    */

-   if (Atomic_Read32(&data.index) > query->numLogicalCPUs) {
+   if (Atomic_Read32(data.index) > query->numLogicalCPUs) {
      return FALSE;
   }

-   ASSERT(Atomic_Read32(&data.index) <= query->numLogicalCPUs);
-   query->numLogicalCPUs = Atomic_Read32(&data.index);
+   ASSERT(Atomic_Read32(data.index) <= query->numLogicalCPUs);
+   query->numLogicalCPUs = Atomic_Read32(data.index);

   return TRUE;
 }
@@ -3408,16 +3511,13 @@ Vmx86_GetMonitorContext(VMDriver *vm,       // IN: The VM instance.
                        Vcpuid vcpuid,      // IN: VCPU in question.
                        Context64 *context) // OUT: context.
 {
-   VMCrossPage *crosspage;
   VMCrossPageData *cpData;
-   if (vcpuid >= vm->numVCPUs) {
+
+   if (vcpuid >= vm->numVCPUs || vm->crosspage[vcpuid] == NULL) {
      return FALSE;
   }
-   crosspage = vm->crosspage[vcpuid];
-   if (!crosspage) {
-      return FALSE;
-   }
-   cpData = &crosspage->crosspageData;
+   cpData = vm->crosspage[vcpuid];
+
   memset(context, 0, sizeof *context);
   context->es  = cpData->monES;
   context->ss  = cpData->monSS;
@@ -3494,3 +3594,137 @@ Vmx86_CreateHVIOBitmap(void)
   memset(hvIOBitmap->addr, 0xff, SVM_VMCB_IO_BITMAP_SIZE);
   return TRUE;
 }
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86RegisterCPU --
+ *
+ *      Registers each logical CPU by incrementing a counter.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      counter value pointed by 'data' is incremented by one.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static void
+Vmx86RegisterCPU(void *data) // IN: *data
+{
+   Atomic_uint32 *numLogicalCPUs = data;
+   ASSERT(numLogicalCPUs);
+   Atomic_Inc32(numLogicalCPUs);
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86_CheckMSRUniformity --
+ *
+ *      Provides basic hardware MSR feature checks for x86 hosted platform. VMM
+ *      requires and prefers uniformity of certain MSRs. This function iterates
+ *      through a list of MSR features (i.e. msrUniformityCacheInfo), checking
+ *      uniformity for MSR value on each logical CPU. A Uniformity check is
+ *      ignored for MSRs are that are not available for the target architecture
+ *      or cpu family. If MSRs are non uniform then, a common bit field is
+ *      calculated by taking the intersection of MSR values across cpu(s).
+ *
+ * Results:
+ *      Returns TRUE if MSR uniformity checks complete successfully, FALSE
+ *      otherwise.
+ *
+ * Side effects:
+ *      updates msrUniformityCacheInfo cache with MSR values.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+Bool
+Vmx86_CheckMSRUniformity(void)
+{
+   Vmx86GetMSRData data;
+   uint32 i, j;
+   uint32 numQueries = ARRAYSIZE(msrUniformityCacheInfo);
+   Atomic_uint32 numLogicalCPUs;
+   Atomic_uint32 *cpuCounters;
+   uint32 numPCPUs = 0;
+   MSRQuery *query = NULL;
+
+   Atomic_Write32(&numLogicalCPUs, 0);
+   /*
+    * Calculates number of logical CPUs by counting and then uses this
+    * information to set up MSR queries; will be executed on each logical CPU.
+    */
+   HostIF_CallOnEachCPU(Vmx86RegisterCPU, &numLogicalCPUs);
+   numPCPUs = Atomic_Read32(&numLogicalCPUs);
+   ASSERT(numPCPUs > 0);
+   query = Vmx86_Calloc(numQueries,
+                        sizeof(MSRQuery) + sizeof(MSRReply) * numPCPUs, TRUE);
+   if (query == NULL) {
+      Warning("Fatal, not enough memory for MSR feature uniformity checks");
+      return FALSE;
+   }
+
+   cpuCounters = Vmx86_Calloc(numQueries, sizeof(Atomic_uint32), TRUE);
+   if (cpuCounters == NULL) {
+      Vmx86_Free(query);
+      Warning("Fatal, not enough memory for MSR feature uniformity checks");
+      return FALSE;
+   }
+   data.query = query;
+   data.index = cpuCounters;
+   data.numItems = numQueries;
+
+   /*
+    * Enumerates a MSR list and initializes MSR data structure before the
+    * actual (safe) MSR query takes place. The Nested loop tests a MSR for
+    * uniformity on all logical processors.
+    */
+   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
+      query = &data.query[i];
+      Atomic_Write32(&data.index[i], 0);
+      query->msrNum = msrUniformityCacheInfo[i].msrIndex;
+      query->numLogicalCPUs = numPCPUs;
+   }
+
+   /* perform once, a multi MSR query for MSRs in uniformity check list. */
+   HostIF_CallOnEachCPU(Vmx86GetMSR, &data);
+
+   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
+      uint32 msrIndex = msrUniformityCacheInfo[i].msrIndex;
+      query = &data.query[i];
+      ASSERT(Atomic_Read32(&data.index[i]) == numPCPUs);
+      msrUniformityCacheInfo[i].msrValue = query->logicalCPUs[0].msrVal;
+      if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
+         /*
+          * MSR_ARCH_CAPABILITIES_RSBA bit 1 represents lack of feature while 0
+          * represents presence. Therefore, bit is flipped for calculating the
+          * least common set and flipped again on the final value for resetting.
+          */
+         msrUniformityCacheInfo[i].msrValue ^= MSR_ARCH_CAPABILITIES_RSBA;
+      }
+      for (j = 1; j < numPCPUs; j++) {
+         uint64 msrValuePCPU = query->logicalCPUs[j].msrVal;
+         if (msrValuePCPU != query->logicalCPUs[0].msrVal) {
+            if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
+               msrValuePCPU ^= MSR_ARCH_CAPABILITIES_RSBA;
+            }
+            msrUniformityCacheInfo[i].msrValue &= msrValuePCPU;
+            Warning("Found a mismatch on MSR feature 0x%x; logical cpu%u "
+                    "value = 0x%llx, but logical cpu%u value = 0x%llx\n",
+                    msrIndex, j, msrValuePCPU, 0, query->logicalCPUs[0].msrVal);
+         }
+      }
+      if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
+         msrUniformityCacheInfo[i].msrValue ^= MSR_ARCH_CAPABILITIES_RSBA;
+      }
+   }
+   Vmx86_Free(cpuCounters);
+   Vmx86_Free(query);
+   return TRUE;
+}
+
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -68,7 +68,7 @@ typedef struct VMDriver {
   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
-   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
+   struct VMCrossPageData **crosspage;        /* numVCPUs-sized array. */
   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
@@ -175,6 +175,7 @@ extern void Vmx86_UpdateMemInfo(VMDriver *curVM,
 extern void Vmx86_Add2MonPageTable(VMDriver *vm, VPN vpn, MPN mpn,
                                   Bool readOnly);
 extern Bool Vmx86_GetAllMSRs(MSRQuery *query);
+extern Bool Vmx86_CheckMSRUniformity(void);
 extern void Vmx86_FlushVMCSAllCPUs(MA vmcs);
 extern void Vmx86_MonTimerIPI(void);
 extern void Vmx86_InitIDList(void);
@@ -0,0 +1,319 @@
+/*********************************************************
+ * Copyright (C) 2007-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _ADDRLAYOUT_H_
+#define _ADDRLAYOUT_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "address_defs.h"
+#include "x86desc.h"
+#include "x86types.h"
+#include "vmm_constants.h"
+
+#define DIRECT_EXEC_USER_RPL    3
+#define BINARY_TRANSLATION_RPL  1
+typedef char x86_PAGE[PAGE_SIZE];
+
+#define MAX_VADDR                    CONST64U(0xffffffffffffffff)
+#define MONITOR_SIZE                 (64 * 1024 * 1024)
+#define MONITOR_LINEAR_START         (MAX_VADDR - MONITOR_SIZE + 1)
+#define MONITOR_LINEAR_END           (MAX_VADDR)
+#define NUM_MONITOR_PAGES            (MONITOR_SIZE / PAGE_SIZE)
+#define MONITOR_BASE_VPN             (MONITOR_LINEAR_START >> PAGE_SHIFT)
+
+#define MONITOR_AS_START             MONITOR_BASE_VPN
+#define MONITOR_AS_LEN               NUM_MONITOR_PAGES
+
+/*
+ * Regions and items necessarily overlap.  Build parallel structures for each.
+ * These 'field' structures are used to determine page numbers and offsets.
+ * AddrLayout contains zero-length regions used for alignment-checking.  (These
+ * are used by gcc as Windows prohibits zero-length arrays.)
+ */
+
+/* Item layout. */
+typedef struct {
+#if defined __GNUC__
+#define REGION(name, length) x86_PAGE name##_REGION_MARKER_FIELD[0];
+#else
+#define REGION(name, length)
+#endif
+#define ITEM(name, length) x86_PAGE name##_FIELD[length];
+#include "addrlayout_table.h"
+} AddrLayout;
+#undef REGION
+#undef ITEM
+#define FIELD_PAGE_NUMBER(name) PAGE_NUMBER(&(((AddrLayout *)0)->name##_FIELD))
+
+
+/* Region layout. */
+typedef struct {
+#define REGION(name, length) x86_PAGE name##_REGION_FIELD[length];
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+} RegionAddrLayout;
+#undef REGION
+#undef ITEM
+#define REGION_FIELD_PAGE_NUMBER(name) \
+   PAGE_NUMBER(&(((RegionAddrLayout *)0)->name##_REGION_FIELD))
+
+#if defined __GNUC__
+/*
+ * The sum of sizes of items in a region must add up to the region's size.
+ * AddrLayout contains zero-sized region markers at their correct offsets.
+ * RegionAddrLayout contains full-sized regions (but no items).  Comparing the
+ * marker and region offsets ensures that all items sizes in the previous
+ * region sum to the size of the previous region.  To check the final region
+ * (as it has no successor) we assert the sizes of the tables match.
+ *
+ * We also assert 2MB region granularity.
+ *
+ * This checking is only performed when using gcc.
+ */
+#define REGION(name, length) static INLINE void name(void) { \
+   ASSERT_ON_COMPILE(FIELD_PAGE_NUMBER(name##_REGION_MARKER) == \
+                     REGION_FIELD_PAGE_NUMBER(name)); \
+   ASSERT_ON_COMPILE((length % 512) == 0); \
+   ASSERT_ON_COMPILE(sizeof(AddrLayout) == sizeof(RegionAddrLayout)); \
+   ASSERT_ON_COMPILE(sizeof(RegionAddrLayout) == MONITOR_SIZE); \
+}
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+/*
+ * Sizes.
+ */
+
+enum {
+#define REGION(name, length) name##_REGION_LEN = length,
+#define ITEM(name, length) name##_LEN = length,
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+/*
+ * Starts.
+ */
+
+#if defined __GNUC__
+/* Windows compiler and gdb do not accept 64-bit enumerations, but gcc
+ * does.  These are used by geninfo.c.
+ */
+enum {
+#define REGION(name, length) _asm_##name##_REGION_START = \
+    0ULL + MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) _asm_##name##_START = \
+    0ULL + MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+enum {
+#define REGION(name, length) name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+#else
+#define REGION(name, length) \
+   static const uint64 name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name);
+#define ITEM(name, length) \
+   static const uint64 name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name);
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+enum {
+#define REGION(name, length) name##_REGION_start_page = \
+   REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_start_page = \
+   FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+
+/* Derive the bootstrap start address from table contents. */
+#define MONITOR_BOOTSTRAP_START_LA   VPN_2_VA(BS_TXT_START)
+
+#ifndef VMX86_SERVER
+#define SWITCH_PAGE_TABLE_LEN          12
+#endif
+
+#define VMM_STACK_TOP_HW          VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+/*
+ * VMM_STKTOP_HW_LEN needs to contain an ExcFrame64ForCopy (72 bytes) and
+ * some more bytes for a "no man's land" region.  We make sure the "no
+ * man's land" region is 0 on BT_Resume and HV_Resume.  Because we
+ * check "no man's land" in 8-byte chunks, VMM_STKTOP_HW_LEN must be
+ * divisible by 8.
+ */
+#define VMM_STKTOP_HW_LEN          168
+#define VMM_STKTOP_HW_LRET_LEN     (VMM_STKTOP_HW_LEN + sizeof(LretFrame64))
+#define VMM_STKTOP_LRET_OFFSET     (PAGE_SIZE - VMM_STKTOP_HW_LRET_LEN)
+
+#define MON_STACK_BASE            VPN_2_VA(MON_STACK_PAGES_START)
+#define MON_STACK_TOP             VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+#define DF_STACK_BASE              VPN_2_VA(DF_STACK_PAGES_START)
+#define DF_STACK_TOP               VPN_2_VA(DF_STACK_PAGES_START + \
+                                            DF_STACK_PAGES_LEN)
+#define MC_STACK_BASE              VPN_2_VA(MC_STACK_PAGES_START)
+#define MC_STACK_TOP               VPN_2_VA(MC_STACK_PAGES_START + \
+                                            MC_STACK_PAGES_LEN)
+#define NMI_STACK_BASE             VPN_2_VA(NMI_STACK_PAGES_START)
+#define NMI_STACK_TOP              VPN_2_VA(NMI_STACK_PAGES_START + \
+                                            NMI_STACK_PAGES_LEN)
+
+#define NUM_MON_PTABS   MON_PAGE_TABLE_L1_LEN
+
+#define MON_START_L5OFF PT_LA_2_L5OFF(MONITOR_LINEAR_START)
+#define MON_START_L4OFF PT_LA_2_L4OFF(MONITOR_LINEAR_START)
+#define MON_START_L3OFF PT_LA_2_L3OFF(MONITOR_LINEAR_START)
+#define MON_START_L2OFF PT_LA_2_L2OFF(MONITOR_LINEAR_START)
+
+/*
+ * Allocated wired pages for the monitor.
+ * Currently we allocate wired pages for the:
+ *  0) The shared area between user and monitor;
+ *  a) GDT of the monitor
+ *  a1) BT (CP1) stack
+ *  b) The monitor's page table with one page directory under 4 GB
+ *      and two page table pages.
+ *  c) Pages for the monitor's stack.
+ *  d) The BT area pages
+ *  e) The cross driver/monitor page.
+ *  f) The physical memory array for the machine.
+ *  g) The SMRAM memory
+ *
+ * We allocate the shared area separately since it must be read/write
+ * in the monitor.   See SharedArea_PowerOn().
+ */
+
+#define MON_BOOTSTRAP_PAGE_TABLE_PAGES (BOOTSTRAP_REGION_LEN     /  \
+                                        PAE_PTES_PER_PGTBL)
+#ifdef VMX86_SERVER
+#define NUM_MONWIRED_PAGES     0
+
+#define MON_PAGE_TABLE_ALLOC_PAGES 0
+#define NUM_MONWIRED_BOOTSTRAP_PAGES 0
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN  + \
+                                    DF_STACK_PAGES_LEN   + \
+                                    MC_STACK_PAGES_LEN   + \
+                                    NMI_STACK_PAGES_LEN  + \
+                                    HV_CURRENT_VMCB_LEN  + \
+                                    IDT_NORMAL_LEN       + \
+                                    GDT_AND_TASK_LEN)
+
+#else
+#define NUM_MONWIRED_PAGES    (CROSS_PAGE_CODE_LEN   + \
+                               CROSS_PAGE_DATA_LEN   + \
+                               MON_PAGE_TABLE_L4_LEN + \
+                               MON_PAGE_TABLE_L3_LEN + \
+                               MON_PAGE_TABLE_L2_LEN)
+
+#define MON_PAGE_TABLE_ALLOC_PAGES ((MONITOR_READONLY_REGION_LEN + \
+                                     MONITOR_DATA_REGION_LEN     + \
+                                     MONITOR_MISC_REGION_LEN)    / \
+                                     PAE_PTES_PER_PGTBL)
+
+#define NUM_MONWIRED_BOOTSTRAP_PAGES MON_BOOTSTRAP_PAGE_TABLE_PAGES
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN + \
+                                    DF_STACK_PAGES_LEN  + \
+                                    MC_STACK_PAGES_LEN  + \
+                                    NMI_STACK_PAGES_LEN + \
+                                    GDT_AND_TASK_LEN    + \
+                                    IDT_NORMAL_LEN      + \
+                                    HV_CURRENT_VMCB_LEN + \
+                                    SWITCH_PAGE_TABLE_LEN)
+#endif
+
+#define MON_PAGE_TABLE_L5        VPN_2_VA(MON_PAGE_TABLE_L5_START)
+#define MON_PAGE_TABLE_L4        VPN_2_VA(MON_PAGE_TABLE_L4_START)
+#define VMM_STACK_TOP            (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LEN)
+#define VMM_LRET_STACK_TOP       (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LRET_LEN)
+/*
+ * Mark out a guard page for the VMM stack, this is not in the addrlayout_table
+ * explicitly because it overlaps the last page of the MONITOR_READONLY region.
+ */
+#define VMM_STACK_GUARD_START    MON_STACK_PAGES_START
+#define VMM_STACK_GUARD_LEN      PAGE_SIZE
+
+/*
+ * Following functions return true if the range [va, va+len) is within
+ * range of a particular monitor stack.  (Don't adjust without
+ * considering uint64 overflow when va is very high.)
+ */
+static INLINE Bool
+AddrLayout_InMonStack(VA64 va, size_t len)
+{
+   return VPN_2_VA(MON_STACK_PAGES_START) <= va &&
+          va <= VMM_STACK_TOP_HW - len;
+}
+
+static INLINE Bool
+AddrLayout_InNMIStack(VA64 va, size_t len)
+{
+   return va >= NMI_STACK_BASE && va <= NMI_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InDFStack(VA64 va, size_t len)
+{
+   return va >= DF_STACK_BASE && va <= DF_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InMCStack(VA64 va, size_t len)
+{
+   return va >= MC_STACK_BASE && va <= MC_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InAMonitorStack(VA va, size_t len)
+{
+   return AddrLayout_InMonStack(va, len)  ||
+          AddrLayout_InNMIStack(va, len)  ||
+          AddrLayout_InDFStack(va, len)   ||
+          AddrLayout_InMCStack(va, len);
+}
+
+#endif
@@ -0,0 +1,144 @@
+/*********************************************************
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * addrlayout_table.h --
+ *
+ * Named elements in the monitor's address space of fixed size and location.
+ * All units are 4 kilobyte pages.  Also used for vmmlayout output.
+ *
+ * The monitor's address space is populated three different ways:
+ * 1) For elements of a fixed size and location, at compile-time.
+ * 2) For well-known program sections and the shared area, at link-time.
+ * 3) Dynamically, by the allocator, during bootstrapping and monitor power-on.
+ *
+ * This file describes all of the monitor's main 64MB address space.  All of
+ * (1) is described specifically.  Ranges for (2) and (3) are accounted for,
+ * but their contents are described generally here.
+ *
+ * The monitor's address space is organized into container regions which are an
+ * integer number of 2MB "large" pages.  Within a container region, items
+ * comprised of one or more 4KB pages may be described.
+ *
+ * All container regions and items are described by a name and a size in pages.
+ * No two items may have the same name nor may two regions.
+ *
+ * REGION(region-name, number-of-pages)
+ *   ITEM(item-name,   number-of-pages)     // first item in region
+ *   ITEM(item-name,   number-of-pages)     // second item in region
+ * <...>
+ */
+
+
+/*
+ * Monitor .rodata/.text.  Mapped large (reducing TLB pressure) and read-only.
+ */
+REGION(MONITOR_READONLY,        1024)
+  ITEM(MONITOR_READONLY_LINKER, 1024) // Used by linker.
+
+/*
+ * Monitor .data/.bss.  Mapped large (reducing TLB pressure) and read-write.
+ */
+REGION(MONITOR_DATA,            512)
+  ITEM(MON_STACK_PAGES,           8) // Monitor stack.
+  ITEM(IDT_NORMAL,                1) // Normal IDT (used for all but SVM).
+  ITEM(MONITOR_DATA,            503) // Used by linker and TC.
+
+/*
+ * The monitor's translation cache.  This object starts at the page boundary
+ * following the end of the used space in MONITOR_DATA and extends through
+ * the TC_REGION.
+ */
+REGION(TC_REGION,              1024)
+  ITEM(TC_BLOCK,               1024)
+
+/*
+ * The monitor's pages for architectural state, stacks, page tables, shared
+ * pages, and dynamic allocation.  This region also contains the linked shared
+ * area and other contents.
+ * Mapped small (allowing non-present mappings and sharing) and read-write.
+ */
+REGION(MONITOR_MISC,           5632)
+#ifdef VMX86_SERVER
+  ITEM(GUARD_PAGE,                2) // Reserved (for symmetry with hosted).
+#else
+  ITEM(CROSS_PAGE_DATA,           1) // Cross page data (RW).
+  ITEM(CROSS_PAGE_CODE,           1) // Cross page code (RX).
+#endif
+  ITEM(GDT_AND_TASK,              1) // GDT and Task State Segment.
+
+  ITEM(MON_PAGE_TABLE_L5,         1) // Monitor page root, if 5-Level PT used.
+  ITEM(MON_PAGE_TABLE_L4,         1) // Monitor page root, if 4-Level PT used.
+  ITEM(MON_PAGE_TABLE_L3,         1) // Monitor L3 page table.
+  ITEM(MON_PAGE_TABLE_L2,         1) // Monitor L2 page table.
+  ITEM(MON_PAGE_TABLE_L1,        32) // Monitor L1 page tables.
+
+  ITEM(HOST_APIC,                 1) // Physical APIC.
+  ITEM(GUEST_APIC,                1) // Guest APIC.
+
+  ITEM(DF_GUARD_PAGE,             1) // Double Fault stack guard page.
+  ITEM(DF_STACK_PAGES,            1) // Double Fault stack; need ~768 bytes.
+
+  ITEM(MC_GUARD_PAGE,             1) // Machine Check stack guard page.
+  ITEM(MC_STACK_PAGES,            1) // Machine Check stack.
+
+  ITEM(NMI_GUARD_PAGE,            1) // NMI stack guard page.
+  ITEM(NMI_STACK_PAGES,           4) // NMI stack.  Profiling requires a larger
+                                     // stack than MC/DF.
+
+  ITEM(HV_SWITCH,                 1) // SVM switch page
+  ITEM(HV_CURRENT_VMCB,           1) // SVM current VMCB
+  ITEM(HV_STD_NATIVE_VMCB,        1) // SVM/VT standard VMCB/VMCS
+  ITEM(HV_AUX_NATIVE_VMCB,        1) // SVM/VT auxiliary VMCB/VMCS
+  ITEM(VHV_GUEST_VMCB,            1) // VSVM/VVT guest VMCB/VMCS
+  ITEM(HV_SEV_VMSA,               1) // SVM SEV-ES state save area
+  ITEM(VPROBE_MON_RELOC,          1) // VProbe monitor reloc page.
+  ITEM(GART_LIST_PAGES,          48) // PhysMem vmm gart list pages.
+  ITEM(GART_ALT_LIST_PAGES,      48) // Alternate vmm gart pages
+                                     // (for invalidation).
+  ITEM(GART_BF_PAGES,             8) // Gart Bloom filter memory pages.
+  ITEM(VVT_GUEST_VIRT_APIC,       1) // Inner guest virtual APIC page.
+  ITEM(HT_STATE_MAP,              8) // Information used by the SecureHT module
+  ITEM(SHARED_RW_DATA,         5462) // R/W shared data, including the shared
+                                     // area and stat vars. Any remaining pages
+                                     // are left to the dynamic allocator, which
+                                     // extends throughout the entire reclaimed
+                                     // bootstrap region.
+/*
+ * Bootstrap-used space.  This space is reclaimed when bootstrap is done, and
+ * is used by the allocator thereafter.  The allocator starts at the first
+ * unused page in the SHARED_RW_DATA item from the MONITOR_MISC region and
+ * extends through to the end of the VMM address space once the bootstrap region
+ * is reclaimed.
+ */
+REGION(BOOTSTRAP,              8192)
+  ITEM(BS_TXT,                   17) // bootstrap's .text
+  ITEM(BS_RODATA,                10) // bootstrap's .rodata
+  ITEM(BS_DATA,                  10) // bootstrap's .data
+  ITEM(BS_BSS,                   10) // bootstrap's .bss
+  ITEM(BS_HEADER,                 1) // MonLoaderHeader mapping
+  ITEM(BS_VCPU_L1PT_RANGE,       32) // VCPU L1PT mappings
+  ITEM(BS_VCPU_L1PT_PT,           1) // L1PT that maps a VCPU's L1 page tables
+  ITEM(BS_INIT_POOL,              1) // pool for initializing VMM pages
+  ITEM(BS_DYNAMIC_ALLOC,       1880) // bs_alloc's dynamic allocator pages
+  ITEM(IDT_BOOTSTRAP,             1) // bootstrap IDT
+  ITEM(IDT_BOOTSTRAP_STUBS,       2) // bootstrap IDT gate stubs
+  ITEM(BS_PER_VM_VMX,           300) // vmm64's shared_per_vm_vmx
+  ITEM(BS_PER_VCPU,               8) // vmm64's shared_per_vcpu for VCPU 0
+  ITEM(BS_PER_VCPU_VMX,         136) // vmm64's shared_per_vcpu_vmx for VCPU 0
+  ITEM(VMM_MODULES,            5783) // ~22.6MB for unlinked VMM modules
@@ -148,7 +148,7 @@ PtrToVA64(void const *ptr) // IN
 * the NT specific VMX86_DRIVER_VERSION.
 */

-#define VMMON_VERSION           (392 << 16 | 0)
+#define VMMON_VERSION           (401 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -248,6 +248,7 @@ enum IOCTLCmd {
   IOCTLCMD(GET_IPI_VECTORS),
   IOCTLCMD(SEND_IPI),
   IOCTLCMD(SEND_ONE_IPI),
+   IOCTLCMD(GET_SWITCH_ERROR_ADDR),

   /*
    * Keep host-specific calls at the end so they can be undefined
@@ -316,6 +317,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
+#define IOCTL_VMX86_GET_SWITCH_ERROR_ADDR VMIOCTL_BUFFERED(GET_SWITCH_ERROR_ADDR)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
 #define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
@@ -531,6 +533,8 @@ typedef struct IPIVectors {
 */
 typedef struct VMCreateBlock {
   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
+   VA64               vmmonData;     // IN: User VA of a userlevel scratch area
+                                     //     required by the Linux vmmon
   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
   uint32             numVCPUs;      // IN: Number of VCPUs.
   uint16             vmid;          // OUT: VM ID for the created VM.
@@ -563,7 +567,6 @@ typedef struct VMStatVarsRegistrationBlock {
 } VMStatVarsRegistrationBlock;

 typedef struct {
-   VA64   crosspage; // IN: User VA of VCPU crosspage.
   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
 } PerVcpuPages;

@@ -579,6 +582,15 @@ typedef struct VMProcessBootstrapBlock {
   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

+/*
+ * Arguments for retrieving the switch error address.
+ */
+typedef struct VMSwitchErrorArgs {
+   Vcpuid vcpuid;  // IN: The VCPU of interest.
+   uint64 addr;    // OUT: The code address that a failure was detected at, or 0
+                   //      if no failure has occurred.
+} VMSwitchErrorArgs;
+
 static INLINE size_t
 GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
 {
@@ -42,8 +42,6 @@
 #include "mon_assert.h"

 #define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
-#define CP_STUB_SIZE     16     /* A relative jmp instruction (5 bytes)
-                                   padded to the next 16 byte boundary. */

 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
@@ -66,12 +64,14 @@
   MC(ALLOC_ANON_LOW_PAGE)                                                    \
   MC(GET_MON_IPI_VECTOR)                                                     \
   MC(GET_HV_IPI_VECTOR)                                                      \
+   MC(GET_PERF_CTR_VECTOR)                                                    \
   MC(GET_HOST_TIMER_VECTORS)                                                 \
   MC(BOOTSTRAP_CLEANUP)                                                      \
   MC(GET_SHARED_AREA)                                                        \
   MC(GET_STAT_VARS)                                                          \
   MC(GET_NUM_PTP_PAGES)                                                      \
   MC(GET_HV_IO_BITMAP)                                                       \
+   MC(GET_MSR)                                                                \

 /*
 *----------------------------------------------------------------------
@@ -109,8 +109,8 @@ typedef enum ModuleCallType {
 */

 #if defined(VMX86_UCCOST) && !defined(VMX86_SERVER)
-#define UCTIMESTAMP(cp, stamp) \
-             do { (cp)->ucTimeStamps[UCCOST_ ## stamp] = RDTSC(); } while (0)
+#define UCTIMESTAMP(ptr, stamp) \
+             do { (ptr)[UCCOST_ ## stamp] = RDTSC(); } while (0)
 #else
 #define UCTIMESTAMP(cp, stamp)
 #endif
@@ -130,7 +130,7 @@ typedef struct UCCostResults {
 } UCCostResults;

 typedef enum UCCostStamp {
-#define UC(x) UCCOST_ ## x,
+#define UC(x, y) UCCOST_ ## x,
 #include "uccostTable.h"
   UCCOST_MAX
 } UCCostStamp;
@@ -144,12 +144,12 @@ typedef enum UCCostStamp {
 * MAX_SWITCH_PT_PATCHES
 *
 *   This is the maximum number of patches that must be placed into
- *   the monitor page tables so that the host GDT and the crosspage
- *   can be accessed during worldswitch.
+ *   the monitor page tables so that the host GDT, crosspage data, and
+ *   crosspage code can be accessed during worldswitch.
 *
 *----------------------------------------------------------------------
 */
-#define MAX_SWITCH_PT_PATCHES 2
+#define MAX_SWITCH_PT_PATCHES 3

 /*----------------------------------------------------------------------
 *
@@ -236,16 +236,23 @@ VMMPageTablePatch;
 * - the largest stack use instantaneously possible by #DB handling code
 * - one high-water uint32 used to detect stack overflows when debugging
 *
- * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+ * A breakdown of the worst-case exception handler stack usage (SwitchUDHandler)
+ * is: 5 * uint64 (Hardware) + 1 * uint64 (RAX) + 1 * uint64 (RBX) +
+ *     1 * uint64 (RCX) + 1 * uint64 (call) + 2 * uint64 (sidt) = 11 * uint64
+ * This is a slight over-estimate of the possible usage at any time but there
+ * is plenty of space available in the cross page data area.
+ *
+ * 264 (11 * sizeof(uint64) * 3) bytes is slightly more than enough as of
+ * 2020/06/14.
 */
-#define TINY_STACK_SIZE      184
+#define TINY_STACK_SIZE      264

 /*
 *----------------------------------------------------------------------
 *
 * VMCrossPageData --
 *
- *      Data which is stored on the VMCrossPage.
+ *      Data which is stored on the cross page.
 *
 *----------------------------------------------------------------------
 */
@@ -294,11 +301,13 @@ struct VMCrossPageData {
   uint16   monES;  /* Not saved/restored in assembly switch */
   uint16   monPad;

-   uint64   crosspageMA;
+   uint64   crosspageDataMA;

   uint64   hostDR[8];
-   LA64     crossPageLA;       // where host/PTP map the cross page
+   LA64     crosspageDataLA;   // where host/PTP map the cross data page
   LA64     crossGDTLA;        // where host/PTP map the cross GDT
+   LA64     crosspageCodeLA;   // where host/PTP map the cross code page
+   LA64     vmmToHostLA;       // where host has placed the VmmToHost function
   uint16   hostInitial64CS;
   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
@@ -337,6 +346,7 @@ struct VMCrossPageData {

 #if !defined(VMX86_SERVER)
   uint64 ucTimeStamps[UCCOST_MAX];
+   uint8  _ucPad[8];
 #endif

   SwitchedMSRState switchedMSRState;
@@ -399,71 +409,19 @@ struct VMCrossPageData {
 #include "vmware_pack_end.h"
 VMCrossPageData;

-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPageCode --
- *
- *      Code which is stored on the VMCrossPage.
- *
- *----------------------------------------------------------------------
- */
-
-#define CODE_BLOCK_SIZE (PAGE_SIZE - sizeof(VMCrossPageData) - \
-                         (2 + NUM_EXCEPTIONS) * CP_STUB_SIZE)
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPageCode {
-   uint8         toVmmFunc[CP_STUB_SIZE];    // Fixed-position stubs to jump
-   uint8         toHostFunc[CP_STUB_SIZE];   // to world switch functions
-   uint8         gateStubs[NUM_EXCEPTIONS *
-                           CP_STUB_SIZE];    // Stubs for crossIDT.
-   uint8         codeBlock[CODE_BLOCK_SIZE]; // Code for worldswitch and
-                                             // fault handling.
-}
-#include "vmware_pack_end.h"
-VMCrossPageCode;
-
-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPage --
- *
- *      Data structure shared between the monitor and the module
- *      that is used for crossing between the two.
- *      Accessible as vm->crosspage (kernel module) and CROSS_PAGE
- *      (monitor)
- *
- *      Must be exactly one page long.
- *
- *----------------------------------------------------------------------
- */
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPage {
-   VMCrossPageData crosspageData;
-   VMCrossPageCode crosspageCode;
-}
-#include "vmware_pack_end.h"
-VMCrossPage;
-
-#define CROSSPAGE_VERSION_BASE 0xc0e /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc12 /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE             ((VMCrossPage * const)VPN_2_VA(CROSS_PAGE_START))
-#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
+#define CROSS_PAGE             ((VMCrossPageData *) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
+#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData * const) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
 #endif

 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
-#define MX_WAITNORMAL          1  // Must equal one; see linux module code.
-#define MX_WAITERROR           0  // Use MX_ISWAITERROR() to test for error.
+#define MX_WAITNORMAL          1
+#define MX_WAITERROR           0

-// Any zero or negative value denotes error.
-#define MX_ISWAITERROR(e)      ((e) <= MX_WAITERROR)
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -296,9 +296,10 @@ typedef enum MonLoaderError {
 * Values above shared area subindices for sharing of MonLoaderHeader
 * and htSchedStateMap.
 */
-#define MONLOADER_HEADER_IDX 6
-#define MONLOADER_HT_MAP_IDX 7
-
+#define MONLOADER_HEADER_IDX          6
+#define MONLOADER_HT_MAP_IDX          7
+#define MONLOADER_CROSS_PAGE_CODE_IDX 8
+#define MONLOADER_CROSS_PAGE_DATA_IDX 9

 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
                                 void *args, unsigned *line, Vcpuid *vcpu);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,7 +66,6 @@ typedef enum PCIPassthru_IntrType {
   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)

 typedef struct FPTIntrProxyInfo {
-   uint32            adapterIndex;
   uint32            vectorIndex;
   MonitorIdemAction actionID;
 } FPTIntrProxyInfo;
@@ -76,9 +75,15 @@ typedef struct UPTIntrProxyInfo {
   MonitorIdemAction actionID;
 } UPTIntrProxyInfo;

+typedef struct MPTIntrProxyInfo {
+   uint32            adapterIndex;
+   MonitorIdemAction actionID;
+} MPTIntrProxyInfo;
+
 typedef union PCIPassthru_IntrProxyInfo {
   FPTIntrProxyInfo fpt;
   UPTIntrProxyInfo upt;
+   MPTIntrProxyInfo mpt;
 } PCIPassthru_IntrProxyInfo;

 typedef enum PCIPassthru_ErrorType {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014, 2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -133,13 +133,19 @@
 */
 #define PERFCTR_AMD_SMI_COUNT                                  0x2b

-/* AMD Data Cache Events */
+/*
+ * AMD Data Cache Events
+ * Family <  17H: 0x45 - Unified TLB hit
+ *                0x46 - Unified TLB miss
+ * Family >= 17H: 0x45 - L1 DTLB miss (L2 DTLB hit or miss)
+ *                0x46 - Tablewalker
+ */
 #define PERFCTR_AMD_DATA_CACHE_ACCESSES                        0x40
 #define PERFCTR_AMD_DATA_CACHE_MISSES                          0x41
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_L2_OR_SYSTEM       0x42
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_SYSTEM             0x43
 #define PERFCTR_AMD_DATA_CACHE_LINES_EVICTED                   0x44
-#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT               0x45
+#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT_OR_MISS       0x45
 #define PERFCTR_AMD_L1_DTLB_AND_L2_DTLB_MISS                   0x46
 #define PERFCTR_AMD_MISALIGNED_ACCESSES                        0x47
 #define PERFCTR_AMD_PREFETCH_INSTRS_DISPATCHED                 0x4b
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -125,7 +125,7 @@ PTSC_MHz(void)

 #if defined(VM_X86_64) || defined(VM_ARM_64)

-/* 
+/*
 * Conversions to/from cycles.  Note that the conversions operate on
 * signed values, so be careful when taking the difference of two
 * VmAbsoluteTS (which is unsigned) that that value is not out of range
@@ -197,13 +197,13 @@ VmAbsoluteTS PTSC_Get(void);

 #endif

-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HasSynchronizedTSCs --
 *
 *      Returns TRUE iff the platform TSCs are known to be synchronized.
- * 
+ *
 *-----------------------------------------------------------------------------
 */

@@ -214,7 +214,7 @@ PTSC_HasSynchronizedTSCs(void)
 }


-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HostAdjustedTSCs --
@@ -231,4 +231,37 @@ PTSC_HostAdjustedTSCs(void)
   return ptscInfo.hwTSCsAdjusted;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PTSC_AdvanceTimer --
+ *
+ *      Advance '*deadline' in 'period' increments such that it is
+ *      greater than 'now'.  Return the number of ticks incremented.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE uint64
+PTSC_AdvanceTimer(VmAbsoluteTS now,
+                  VmIntervalTS period,
+                  VmAbsoluteTS *deadline)
+{
+   VmAbsoluteTS d = *deadline;
+   if (now >= d) {
+      uint64 count = 1;
+      d += period;
+      if (UNLIKELY(now >= d)) {
+         uint64 t = (now - d) / period + 1;
+         VmIntervalTS diff = t * period;
+         count += t;
+         d += diff;
+      }
+      *deadline = d;
+      return count;
+   }
+   return 0;
+}
+
 #endif /* ifndef _PTSC_H_ */
@@ -0,0 +1,176 @@
+/*********************************************************
+ * Copyright (C) 2018-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * segs.h --
+ *
+ *      Describes segment descriptors and structures containing them for both
+ *      the monitor (in hosted and ESX) and the vmkernel.
+ */
+
+#ifndef _SEGS_H_
+#define _SEGS_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "x86types.h"
+#include "x86segdescrs.h"
+#include "x86sel.h"
+#include "addrlayout.h"
+
+
+/*
+ * Segment map of the monitor.
+ *
+ * The GDT and Task State Segment reside consecutively on one page.  The
+ * monitor segments are placed at the end of the GDT.  The vmkernel can use all
+ * lower-numbered segments for user-mode as well as higher-numbered segments
+ * (though the vmkernel should not use monitor-private segments).  The high
+ * segment placement ensures that there is no selector-overlap with hosted
+ * kernel segments (the hosted world switch code can be a bit faster then, as
+ * it can use a single cross GDT).
+ */
+
+#define GDT_SIZE              (sizeof(Descriptor) * NUM_VALID_SEGMENTS)
+#define GDT_LIMIT             (GDT_SIZE - 1)
+
+#define IRB_SIZE              32 /* Interrupt redirection bitmap. */
+#define TSS_SIZE              (sizeof(Task64) + IRB_SIZE)
+
+#define GDT_START_VA          (VPN_2_VA(GDT_AND_TASK_START))
+#define TASK_START_VA         (VPN_2_VA(GDT_AND_TASK_START) + GDT_SIZE)
+
+/*
+ * vmkBoot uses some of the lower-numbered segments, as do host kernels on
+ * hosted.  User segments could start earlier than 32; see bug #1904257.
+ * Descriptor is used to describe boot, user and kernel segments.
+ * Descriptor64 (twice as large) is used for task segments.
+ */
+#define NUM_BOOT_SEGMENTS     32
+#define NUM_USER_SEGMENTS     (FIRST_SYSTEM_SEGMENT - FIRST_USER_SEGMENT)
+#define NUM_SYSTEM_SEGMENTS   2
+#define NUM_TASK_SEGMENTS     2
+
+#define FIRST_USER_SEGMENT    NUM_BOOT_SEGMENTS
+#define FIRST_SYSTEM_SEGMENT  (PAGE_SIZE / sizeof(Descriptor) - \
+                               NUM_SYSTEM_SEGMENTS            - \
+                               NUM_TASK_SEGMENTS * 2          - \
+                               TSS_SIZE / sizeof(Descriptor))
+
+#define GDT_USER_TLS_MIN      USER_TLS_1_SEGMENT
+#define GDT_USER_TLS_MAX      USER_TLS_3_SEGMENT
+#define USER_TLS_COUNT        ((USER_TLS_3_SEGMENT - USER_TLS_1_SEGMENT) + 1)
+
+#define FOREACH_USER_TLS_INDEX(_i)              \
+   {                                            \
+      unsigned _i;                              \
+      for (_i = 0; _i < USER_TLS_COUNT; _i++) { \
+
+#define FOREACH_USER_TLS_INDEX_DONE             \
+      }                                         \
+   }
+
+#define NULL_LDTR             0
+
+/*
+ * The descriptor after SYSTEM_CODE_SEGMENT (loaded into %cs) must be
+ * appropriate for %ss because of the syscall instruction for 64-bit
+ * user worlds.  Thus SYSTEM_DATA_SEGMENT is directly after it.
+ */
+typedef enum VmwSegs {
+   NULL_SEGMENT             = 0,
+   /* (... reserved for host operating system or vmkBoot segments). */
+
+   USER32_CODE_SEGMENT      = FIRST_USER_SEGMENT,
+   USER_DATA_SEGMENT,
+   USER64_SYSRET_SEGMENT,
+   USER64_STACK_SEGMENT,
+   USER64_CODE_SEGMENT,
+   USER_TLS_1_SEGMENT,
+   USER_TLS_2_SEGMENT,
+   USER_TLS_3_SEGMENT,
+
+   AFTER_LAST_USER_SEGMENT,
+
+   SYSTEM_CODE_SEGMENT      = FIRST_SYSTEM_SEGMENT,
+   SYSTEM_DATA_SEGMENT,
+   VMKERNEL_TASK_SEGMENT,
+   VMKERNEL_TASK_SEGMENT_HI,
+   MONITOR_TASK_SEGMENT,
+   MONITOR_TASK_SEGMENT_HI,
+
+   NUM_VALID_SEGMENTS
+} VmwSegs;
+
+#define GDT_SYSTEM_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_SYSTEM_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_USER_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 3)
+#define GDT_USER_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 3)
+
+/* Selectors used statically in code or in assembly must be unchecked. */
+#define SYSTEM_NULL_SELECTOR    GDT_SYSTEM_SEL(NULL)
+#define USER32_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER32_CODE)
+#define USER_DATA_SELECTOR      GDT_USER_SEL_UNCHECKED(USER_DATA)
+#define USER64_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER64_CODE)
+#define USER64_SYSRET_SELECTOR  GDT_USER_SEL(USER64_SYSRET)
+#define SYSTEM_CODE_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_CODE)
+#define SYSTEM_DATA_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_DATA)
+#define MONITOR_TASK_SELECTOR   GDT_SYSTEM_SEL(MONITOR_TASK)
+#define VMKERNEL_TASK_SELECTOR  GDT_SYSTEM_SEL(VMKERNEL_TASK)
+
+/*
+ * The VMM GDT is comprised of many segment descriptors with one initial
+ * Task State Segment system descriptor.  The VMM Task State Segment is
+ * on the same page sequentially after its GDT.
+ */
+#pragma pack(push, 1)
+typedef struct StaticGDTAndTSS {
+   Descriptor   empty[NUM_BOOT_SEGMENTS + NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS];
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+   Task64       monTSS;
+   uint8        TSSIRBitmap[IRB_SIZE];
+} StaticGDTAndTSS;
+#pragma pack(pop)
+
+#pragma pack(push, 1)
+typedef struct VmkernelGDT {
+   Descriptor   bootSegs[NUM_BOOT_SEGMENTS];
+   Descriptor   userSegs[NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS]; /* VMM/VMK-shared. */
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+} VmkernelGDT;
+#pragma pack(pop)
+
+MY_ASSERTS(segs,
+           ASSERT_ON_COMPILE(SYSTEM_CODE_SEGMENT + 1 == SYSTEM_DATA_SEGMENT);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT - FIRST_USER_SEGMENT <=
+                             NUM_USER_SEGMENTS);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT <= FIRST_SYSTEM_SEGMENT);
+)
+
+#endif /* _SEGS_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -20,15 +20,12 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-UC(CALL_START)
-UC(BEGIN_BACK_TO_HOST)
-UC(SWITCHED_TO_MODULE)
-UC(VMX_SELECT_RETURN)
-UC(VMX_HANDLER_START)
-UC(VMX_SIGNAL)
-UC(MODULE_SIGNAL)
-UC(SWITCHING_TO_MONITOR)
-UC(DONE_BACK_TO_HOST)
-UC(CALL_END)
+UC(CALL_START,             FALSE)
+UC(BEGIN_BACK_TO_HOST,     FALSE)
+UC(SWITCHED_TO_MODULE,     TRUE)
+UC(VMX_HANDLER_START,      FALSE)
+UC(SWITCHING_TO_MONITOR,   TRUE)
+UC(DONE_BACK_TO_HOST,      FALSE)
+UC(CALL_END,               FALSE)

 #undef UC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,5 +31,6 @@
 * failing DeviceIoControl on Windows.
 */
 #define USERCALL_VMX86ALLOCERR (-8192)
+#define USERCALL_SWITCHERR     (-8193)

-#endif 
+#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -60,7 +60,7 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define MAX_SMALL_VM_VCPUS 128

 /* Supported limit. */
-#define MAX_SUPPORTED_VCPUS   256
+#define MAX_SUPPORTED_VCPUS   (FEATURE_STATE_IS_ENABLED(MonsterVM) ? 768 : 256)

 #if defined __cplusplus
 } // extern "C"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,10 +44,11 @@
 #   include "str.h"     /* Str_Snprintf */
 #   define VCS_SNPRINTF Str_Snprintf
 #elif defined MONITOR_APP
-#   include <stdio.h>   /* libc snprintf */
 #   if defined WIN32
-#      define VCS_SNPRINTF _snprintf
+#      include "str.h"     /* Str_Snprintf */
+#      define VCS_SNPRINTF Str_Snprintf
 #   else
+#      include <stdio.h>   /* libc snprintf */
 #      define VCS_SNPRINTF snprintf
 #   endif
 #elif defined VMM || defined VMKERNEL
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016-2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -417,31 +417,13 @@ _BUILD_GET_CR(_GET_CR8, 8);
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
 {
-#ifdef VM_X86_64
   _lgdt(dtr);
-#else
-   /*
-    * timetools 32-bit targets use an old version of MSVC that doesn't support
-    * the _lgdt intrinsic. Luckily, 32-bit still supports inline asm.
-    */
-   __asm mov eax, dtr
-   __asm lgdt [eax]
-#endif
 }

 static INLINE void
 _Get_GDT(_GETSET_DTR_TYPE *dtr)
 {
-#ifdef VM_X86_64
   _sgdt(dtr);
-#else
-   /*
-    * timetools 32-bit targets use an old version of MSVC that doesn't support
-    * the _sgdt intrinsic. Luckily, 32-bit still supports inline asm.
-    */
-   __asm mov eax, dtr
-   __asm sgdt [eax]
-#endif
 }

 static INLINE void
@@ -466,141 +448,6 @@ _Get_IDT(_GETSET_DTR_TYPE *dtr)

 #define RESTORE_FLAGS(x) __writeeflags(x)

-
-#ifdef VM_X86_32
-
-#define SET_TR(_tr)      _Set_TR(_tr)
-#define SET_LDT(_tr)     _Set_LDT(_tr)
-
-#define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
-#define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)
-
-#define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
-#define CLTS()           __asm clts
-
-#define FNCLEX()         __asm fnclex
-
-#define TLB_INVALIDATE_PAGE(_addr) {  \
-   void *_a = (_addr); \
-   __asm mov eax, _a __asm invlpg [eax] \
-}
-
-#define RAISE_INTERRUPT(_x)  {__asm int _x }
-#define RETURN_FROM_INT()   {__asm iretd }
-
-
-static INLINE void
-SET_DS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ds, ax
-}
-
-static INLINE void
-SET_ES(Selector val)
-{
-   __asm mov ax, val
-   __asm mov es, ax
-}
-
-static INLINE void
-SET_FS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov fs, ax
-}
-
-static INLINE void
-SET_GS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov gs, ax
-}
-
-static INLINE void
-SET_SS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ss, ax
-}
-
-static INLINE Selector
-GET_FS(void)
-{
-   Selector _v;
-   __asm mov _v,fs
-   return _v;
-}
-
-static INLINE Selector
-GET_GS(void)
-{
-   Selector _v;
-   __asm mov _v,gs
-   return _v;
-}
-
-static INLINE Selector
-GET_DS(void)
-{
-   Selector _v;
-   __asm mov _v,ds
-   return _v;
-}
-
-static INLINE Selector
-GET_ES(void)
-{
-   Selector _v;
-   __asm mov _v,es
-   return _v;
-}
-
-static INLINE Selector
-GET_SS(void)
-{
-   Selector _v;
-   __asm mov _v,ss
-   return _v;
-}
-
-static INLINE Selector
-GET_CS(void)
-{
-   Selector _v;
-   __asm mov _v,cs
-   return _v;
-}
-
-static INLINE void
-_Set_LDT(Selector val)
-{
-   __asm lldt val
-}
-
-static INLINE void
-_Set_TR(Selector val)
-{
-   __asm ltr val
-}
-
-static INLINE Selector
-_Get_LDT(void)
-{
-   Selector sel;
-   __asm sldt sel
-   return sel;
-}
-
-static INLINE Selector
-_Get_TR(void)
-{
-   Selector sel;
-   __asm str sel
-   return sel;
-}
-
-#endif /* !VM_X86_32 */
 #endif /* !__GNUC__ && !_MSC_VER */


@@ -680,7 +527,7 @@ INTERRUPTS_ENABLED(void)
 #endif


-#if defined (__GNUC__) || (defined (_MSC_VER) && defined (VM_X86_32))
+#if defined (__GNUC__)
 static INLINE unsigned
 CURRENT_CPL(void)
 {
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -145,64 +152,6 @@ typedef ALIGNED(16) struct Atomic_uint128 {
 } Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock And [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -499,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -552,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -588,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -632,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -655,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -782,17 +772,9 @@ PAUSE(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -840,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -853,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,41 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef ULM
-void _fxsave64(void *save);
-void _fxsave(void *save);
-void _fxrstor64(const void *load);
-void _fxrstor(const void *load);
-void _xsave64(void *save, uint64 mask);
-void _xsave(void *save, uint64 mask);
-void _xsaveopt64(void *save, uint64 mask);
-void _xsavec(void *save, uint64 mask);
-void _xrstor64(const void *load, uint64 mask);
-void _xrstor(const void *load, uint64 mask);
-#endif /* ULM */
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#ifdef ULM
-#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
-                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
-#endif /* ULM */
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -358,11 +259,7 @@ SFENCE(void)
   );
 #elif defined _MSC_VER
   _ReadWriteBarrier();
-#if defined VM_X86_32
-   __asm sfence;
-#else
   _mm_sfence();
-#endif
   _ReadWriteBarrier();
 #else
 #error No compiler defined for SFENCE
@@ -688,22 +688,28 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-3.3 (circa 2003).  Enforce this here, various things below
+ * gcc-4.1 (circa 2006).  Enforce this here, various things below
 * this line depend upon it.
 *
- * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
- * The various linux kernel modules may use older (gcc-3.3) compilers.
+ * Current oldest compilers:
+ * - guest tools: 4.1.2 (freebsd/solaris)
+ * - buildhost compiler: 4.4.3
+ * - hosted kernel modules: 4.5
+ *
+ * SWIG's preprocessor is exempt.
 */
-#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
-#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
+#ifndef SWIG
+#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
+#error "gcc version is too old, need gcc-4.1 or better"
+#endif
 #endif

 /*
- * Similarly, we require a compiler that is at least vc90 (vs2008).
+ * Similarly, we require a compiler that is at least vs2012.
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1500
-#error "cl.exe version is too old, need vc90 or better"
+#if defined _MSC_VER && _MSC_VER < 1700
+#error "cl.exe version is too old, need vs2012 or better"
 #endif


@@ -0,0 +1,88 @@
+/*********************************************************
+ * Copyright (C) 2012,2017-2018,2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vm_idt_x86.h --
+ *
+ *	Definitions for IDT use in x86/x86-64 ESX and hosted VMware Products.
+ */
+
+#ifndef _VM_IDT_X86_H_
+#define _VM_IDT_X86_H_
+
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMCORE
+#include "includeCheck.h"
+
+#if defined __cplusplus
+extern "C" {
+#endif
+
+/*
+ * User worlds are the only worlds that expect to receive
+ * interrupts on a stack specified in the %rsp entries of
+ * the TSS.
+ */
+#define TSS_RSP0 0
+#define TSS_RSP1 1
+#define TSS_RSP2 2
+
+/*
+ * The monitor and the vmkernel use gate stub handlers of a single size in
+ * their IDTs.  The only exception is the #PF gate (#14) which is twice as
+ * long in the monitor IDT, to copy CR2.
+ */
+#define IDT_STUB_SIZE              16
+
+#define IST_NONE                    0
+/* IST entries for the monitor. */
+#define IST_VMM_DF                  1
+#define IST_VMM_NMI                 2
+#define IST_VMM_MCE                 3
+#define MAX_VMM_IST                 3
+/* IST entries for the vmkernel. */
+#define IST_VMK_MCE                 5
+#define IST_VMK_DF                  6
+#define IST_VMK_NMI                 7
+
+
+static INLINE int
+IDT_MonitorISTForVector(int v)
+{
+   /*
+    * For non-SVM execution, the monitor uses the IST for #DF, #NMI, and #MC.
+    */
+   if (v == EXC_DF) {
+      return IST_VMM_DF;
+   } else if (v == EXC_NMI) {
+      return IST_VMM_NMI;
+   } else if (v == EXC_MC) {
+      return IST_VMM_MCE;
+   }
+   return IST_NONE;
+}
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
+#endif /* _VM_IDT_X86_H_ */
@@ -32,6 +32,7 @@
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMX
 #include "includeCheck.h"
+#include "community_source.h"

 /*
 * Page remapping definitions.
@@ -97,10 +98,12 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

+#define MDEF_DRAINER
+
 #define VMMEM_SERVICES_DEFS                                       \
   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
-   MDEF(VMMEM_SERVICES_TYPE_DIRTY_CLEAR, DirtyDrainer_ClearDirty)
+   MDEF_DRAINER

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -99,6 +99,7 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
+#define CR4_CET        0x00800000
 #define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

@@ -203,6 +204,7 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_CP           21  // Control Protection exception.
 #define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

@@ -210,7 +212,7 @@
 #define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
                                (1u << EXC_NP) | (1u << EXC_SS) | \
                                (1u << EXC_GP) | (1u << EXC_PF) | \
-                                (1u << EXC_AC))
+                                (1u << EXC_AC) | (1u << EXC_CP))

 /*
 * eflag/rflag definitions.
@@ -220,24 +222,24 @@

 typedef enum x86_FLAGS {
   EFLAGS_NONE         = 0,
-   EFLAGS_CF           = (1 << 0),     /* User */ 
-   EFLAGS_SET          = (1 << 1),                                  
-   EFLAGS_PF           = (1 << 2),     /* User */ 
-   EFLAGS_AF           = (1 << 4),     /* User */ 
-   EFLAGS_ZF           = (1 << 6),     /* User */ 
-   EFLAGS_SF           = (1 << 7),     /* User */ 
-   EFLAGS_TF           = (1 << 8),     /* Priv */ 
-   EFLAGS_IF           = (1 << 9),     /* Priv */ 
-   EFLAGS_DF           = (1 << 10),    /* User */ 
-   EFLAGS_OF           = (1 << 11),    /* User */ 
-   EFLAGS_NT           = (1 << 14),    /* Priv */ 
-   EFLAGS_RF           = (1 << 16),    /* Priv */ 
-   EFLAGS_VM           = (1 << 17),    /* Priv */ 
-   EFLAGS_AC           = (1 << 18),    /* Priv */ 
-   EFLAGS_VIF          = (1 << 19),    /* Priv */ 
-   EFLAGS_VIP          = (1 << 20),    /* Priv */ 
+   EFLAGS_CF           = (1 << 0),     /* User */
+   EFLAGS_SET          = (1 << 1),
+   EFLAGS_PF           = (1 << 2),     /* User */
+   EFLAGS_AF           = (1 << 4),     /* User */
+   EFLAGS_ZF           = (1 << 6),     /* User */
+   EFLAGS_SF           = (1 << 7),     /* User */
+   EFLAGS_TF           = (1 << 8),     /* Priv */
+   EFLAGS_IF           = (1 << 9),     /* Priv */
+   EFLAGS_DF           = (1 << 10),    /* User */
+   EFLAGS_OF           = (1 << 11),    /* User */
+   EFLAGS_NT           = (1 << 14),    /* Priv */
+   EFLAGS_RF           = (1 << 16),    /* Priv */
+   EFLAGS_VM           = (1 << 17),    /* Priv */
+   EFLAGS_AC           = (1 << 18),    /* Priv */
+   EFLAGS_VIF          = (1 << 19),    /* Priv */
+   EFLAGS_VIP          = (1 << 20),    /* Priv */
   EFLAGS_ID           = (1 << 21),    /* Priv */
-   
+
   EFLAGS_IOPL         = 3 << EFLAGS_IOPL_SHIFT,
   EFLAGS_ARITH        = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_OF),
@@ -249,7 +251,7 @@ typedef enum x86_FLAGS {
   EFLAGS_ALL          = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_DF | EFLAGS_OF | EFLAGS_TF |
                          EFLAGS_IF | EFLAGS_IOPL | EFLAGS_NT | EFLAGS_RF |
-                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP | 
+                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP |
                          EFLAGS_ID),
   EFLAGS_ALL_16       = EFLAGS_ALL & 0xffff,
   EFLAGS_REAL_32      = (EFLAGS_ALL & ~(EFLAGS_VIP | EFLAGS_VIF | EFLAGS_VM)),
@@ -141,7 +141,7 @@ CPUIDQuery;
   CPUIDLEVEL(TRUE,  18,  0x18,       8, 17) \
   CPUIDLEVEL(TRUE,  1A,  0x1a,       0, 17) \
   CPUIDLEVEL(TRUE,  1B,  0x1b,       2, 17) \
-   CPUIDLEVEL(TRUE,  1F,  0x1f,       6, 17) \
+   CPUIDLEVEL(FALSE, 1F,  0x1f,       6, 17) \
   CPUIDLEVEL(FALSE, 400, 0x40000000, 0,  0) \
   CPUIDLEVEL(FALSE, 401, 0x40000001, 0,  0) \
   CPUIDLEVEL(FALSE, 402, 0x40000002, 0,  0) \
@@ -481,7 +481,7 @@ FIELD(  6,  0, EDX, 16, 16, HW_FEEDBACK_INDEX,                   NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_7                                            \
-FIELD(  7,  0, EAX,  0, 32, LEAF_7_MAX_SUBLEVEL,                 YES, FUT ) \
+FIELD(  7,  0, EAX,  0, 32, LEAF_7_MAX_SUBLEVEL,                 YES,  18 ) \
 FLAG(   7,  0, EBX,  0,  1, FSGSBASE,                            YES,   9 ) \
 FLAG(   7,  0, EBX,  1,  1, TSC_ADJUST,                          ANY,  11 ) \
 FLAG(   7,  0, EBX,  2,  1, SGX,                                 ANY,  17 ) \
@@ -530,18 +530,20 @@ FLAG(   7,  0, ECX, 14,  1, AVX512VPOPCNTDQ,                     YES,  16 ) \
 FLAG(   7,  0, ECX, 16,  1, VA57,                                NO,    0 ) \
 FIELD(  7,  0, ECX, 17,  5, MAWA,                                NO,    0 ) \
 FLAG(   7,  0, ECX, 22,  1, RDPID,                               YES,  17 ) \
-FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            YES, FUT ) \
-FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             YES, FUT ) \
-FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           YES, FUT ) \
+FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            YES,  18 ) \
+FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             YES,  18 ) \
+FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           YES,  18 ) \
 FLAG(   7,  0, ECX, 29,  1, ENQCMD,                              NO,    0 ) \
 FLAG(   7,  0, ECX, 30,  1, SGX_LC,                              ANY,  17 ) \
 FLAG(   7,  0, EDX,  2,  1, AVX512QVNNIW,                        YES,  16 ) \
 FLAG(   7,  0, EDX,  3,  1, AVX512QFMAPS,                        YES,  16 ) \
-FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   YES, FUT ) \
-FLAG(   7,  0, EDX,  8,  1, AVX512VP2INTERSECT,                  YES, FUT ) \
+FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   YES,  18 ) \
+FLAG(   7,  0, EDX,  8,  1, AVX512VP2INTERSECT,                  YES,  18 ) \
 FLAG(   7,  0, EDX, 10,  1, MDCLEAR,                             YES,   9 ) \
 FLAG(   7,  0, EDX, 13,  1, TSX_MICROCODE_UPDATE,                NO,    0 ) \
+FLAG(   7,  0, EDX, 14,  1, SERIALIZE,                           NO,    0 ) \
 FLAG(   7,  0, EDX, 15,  1, HYBRID,                              NO,    0 ) \
+FLAG(   7,  0, EDX, 16,  1, TSXLDTRK,                            NO,    0 ) \
 FLAG(   7,  0, EDX, 18,  1, PCONFIG,                             NO,    0 ) \
 FLAG(   7,  0, EDX, 20,  1, CET_IBT,                             NO,    0 ) \
 FLAG(   7,  0, EDX, 26,  1, IBRSIBPB,                            ANY,   9 ) \
@@ -550,7 +552,7 @@ FLAG(   7,  0, EDX, 28,  1, FCMD,                                YES,   9 ) \
 FLAG(   7,  0, EDX, 29,  1, ARCH_CAPABILITIES,                   ANY,   9 ) \
 FLAG(   7,  0, EDX, 30,  1, CORE_CAPABILITIES,                   NO,    0 ) \
 FLAG(   7,  0, EDX, 31,  1, SSBD,                                YES,   9 ) \
-FLAG(   7,  1, EAX,  5,  1, AVX512BF16,                          YES, FUT )
+FLAG(   7,  1, EAX,  5,  1, AVX512BF16,                          YES,  18 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_9                                            \
@@ -789,11 +791,11 @@ FIELD( 1B,  0, EDX,  0, 32, PCONFIG_TARGET_ID3,                  NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_1F                                           \
-FIELD( 1F,  0, EAX,  0,  5, TOPOLOGY_V2_MASK_WIDTH,              NO,    0 ) \
-FIELD( 1F,  0, EBX,  0, 16, TOPOLOGY_V2_CPUS_SHARING_LEVEL,      NO,    0 ) \
-FIELD( 1F,  0, ECX,  0,  8, TOPOLOGY_V2_LEVEL_NUMBER,            NO,    0 ) \
-FIELD( 1F,  0, ECX,  8,  8, TOPOLOGY_V2_LEVEL_TYPE,              NO,    0 ) \
-FIELD( 1F,  0, EDX,  0, 32, TOPOLOGY_V2_X2APIC_ID,               NO,    0 )
+FIELD( 1F,  0, EAX,  0,  5, TOPOLOGY_V2_MASK_WIDTH,              NA,    0 ) \
+FIELD( 1F,  0, EBX,  0, 16, TOPOLOGY_V2_CPUS_SHARING_LEVEL,      NA,    0 ) \
+FIELD( 1F,  0, ECX,  0,  8, TOPOLOGY_V2_LEVEL_NUMBER,            NA,    0 ) \
+FIELD( 1F,  0, ECX,  8,  8, TOPOLOGY_V2_LEVEL_TYPE,              NA,    0 ) \
+FIELD( 1F,  0, EDX,  0, 32, TOPOLOGY_V2_X2APIC_ID,               NA,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_400                                          \
@@ -907,9 +909,6 @@ FIELD( 80,  0, EBX,  0, 32, LEAF80_VENDOR1,                      NA,    0 ) \
 FIELD( 80,  0, ECX,  0, 32, LEAF80_VENDOR3,                      NA,    0 ) \
 FIELD( 80,  0, EDX,  0, 32, LEAF80_VENDOR2,                      NA,    0 )

-#define CPUID_81_ECX_14 \
-FLAG(  81,  0, ECX, 14,  1, LEAF81ECX_RSVD1,                     NO,    0 )
-
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_81                                           \
 FIELD( 81,  0, EAX,  0,  4, LEAF81_STEPPING,                     ANY,   4 ) \
@@ -934,13 +933,12 @@ FLAG(  81,  0, ECX, 10,  1, IBS,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 11,  1, XOP,                                 YES,   8 ) \
 FLAG(  81,  0, ECX, 12,  1, SKINIT,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 13,  1, WATCHDOG,                            NO,    0 ) \
-CPUID_81_ECX_14 \
 FLAG(  81,  0, ECX, 15,  1, LWP,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 16,  1, FMA4,                                YES,   8 ) \
 FLAG(  81,  0, ECX, 17,  1, TCE,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 19,  1, NODEID_MSR,                          NO,    0 ) \
 FLAG(  81,  0, ECX, 21,  1, TBM,                                 YES,   9 ) \
-FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                            NO,    0 ) \
+FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                            ANY,  18 ) \
 FLAG(  81,  0, ECX, 23,  1, PERFCORE,                            ANY,   4 ) \
 FLAG(  81,  0, ECX, 24,  1, PERFNB,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 26,  1, DATABK,                              NO,    0 ) \
@@ -1067,21 +1065,29 @@ FIELD( 88,  0, EAX, 16,  8, GUEST_PHYS_ADDR_SZ,                  YES,   8 ) \
 FLAG(  88,  0, EBX,  0,  1, CLZERO,                              YES,  14 ) \
 FLAG(  88,  0, EBX,  1,  1, IRPERF,                              NO,    0 ) \
 FLAG(  88,  0, EBX,  2,  1, XSAVE_ERR_PTR,                       NO,    0 ) \
+FLAG(  88,  0, EBX,  3,  1, INVLPGB,                             NO,    0 ) \
 FLAG(  88,  0, EBX,  4,  1, RDPRU,                               NO,    0 ) \
 FLAG(  88,  0, EBX,  6,  1, MBE,                                 NO,    0 ) \
+FLAG(  88,  0, EBX,  8,  1, MCOMMIT,                             NO,    0 ) \
 FLAG(  88,  0, EBX,  9,  1, WBNOINVD,                            YES,  17 ) \
+FLAG(  88,  0, EBX, 10,  1, LBREXTN,                             NO,    0 ) \
 FLAG(  88,  0, EBX, 12,  1, LEAF88_IBPB,                         ANY,   9 ) \
+FLAG(  88,  0, EBX, 13,  1, WBINVD_INT,                          NO,    0 ) \
 FLAG(  88,  0, EBX, 14,  1, LEAF88_IBRS,                         NO,    0 ) \
 FLAG(  88,  0, EBX, 15,  1, LEAF88_STIBP,                        NO,    0 ) \
 FLAG(  88,  0, EBX, 16,  1, LEAF88_IBRS_ALWAYS,                  NO,    0 ) \
 FLAG(  88,  0, EBX, 17,  1, LEAF88_STIBP_ALWAYS,                 NO,    0 ) \
 FLAG(  88,  0, EBX, 18,  1, LEAF88_PREFER_IBRS,                  NO,    0 ) \
+FLAG(  88,  0, EBX, 19,  1, LEAF88_IBRS_SAME_MODE,               NO,    0 ) \
+FLAG(  88,  0, EBX, 20,  1, LMSLE_UNSUPPORTED,                   NO,    0 ) \
+FLAG(  88,  0, EBX, 23,  1, PPIN,                                NO,    0 ) \
 FLAG(  88,  0, EBX, 24,  1, LEAF88_SSBD_SPEC_CTRL,               ANY,   9 ) \
 FLAG(  88,  0, EBX, 25,  1, LEAF88_SSBD_VIRT_SPEC_CTRL,          ANY,   9 ) \
 FLAG(  88,  0, EBX, 26,  1, LEAF88_SSBD_NOT_NEEDED,              NO,    0 ) \
 FIELD( 88,  0, ECX,  0,  8, LEAF88_CORE_COUNT,                   YES,   4 ) \
 FIELD( 88,  0, ECX, 12,  4, APICID_COREID_SIZE,                  YES,   7 ) \
 FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                        NO,    0 ) \
+FIELD( 88,  0, EDX,  0, 16, INVLPGB_MAX,                         NO,    0 ) \
 FIELD( 88,  0, EDX, 16,  8, RDPRU_MAX,                           NO,    0 )

 #define CPUID_8A_EDX_11 \
@@ -1115,7 +1121,9 @@ FLAG(  8A,  0, EDX, 16,  1, SVM_VGIF,                            NO,    0 ) \
 FLAG(  8A,  0, EDX, 17,  1, SVM_GMET,                            YES,  17 ) \
 FIELD( 8A,  0, EDX, 18,  2, SVMEDX_RSVD3,                        NO,    0 ) \
 FLAG(  8A,  0, EDX, 20,  1, SVM_GUEST_SPEC_CTRL,                 NO,    0 ) \
-FIELD( 8A,  0, EDX, 21, 11, SVMEDX_RSVD4,                        NO,    0 )
+FIELD( 8A,  0, EDX, 21,  3, SVMEDX_RSVD4,                        NO,    0 ) \
+FLAG(  8A,  0, EDX, 24,  1, SVM_TLB_CTL,                         NO,    0 ) \
+FIELD( 8A,  0, EDX, 25,  7, SVMEDX_RSVD5,                        NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_819                                          \
@@ -1206,18 +1214,23 @@ FIELD(81E,  0, EBX,  8,  2, CORES_PER_COMPUTE_UNIT,              NA,    0 ) \
 FIELD(81E,  0, ECX,  0,  8, NODEID_VAL,                          NA,    0 ) \
 FIELD(81E,  0, ECX,  8,  3, NODES_PER_PKG,                       NA,    0 )

-#define CPUID_81F_EAX_10 \
-FLAG( 81F,  0, EAX, 10,  1, SVMEAX_RSVD1,                        NO,    0 )
-
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_81F                                          \
 FLAG( 81F,  0, EAX,  0,  1, SME,                                 NO,    0 ) \
 FLAG( 81F,  0, EAX,  1,  1, SEV,                                 YES,  17 ) \
 FLAG( 81F,  0, EAX,  2,  1, PAGE_FLUSH_MSR,                      NO,    0 ) \
 FLAG( 81F,  0, EAX,  3,  1, SEV_ES,                              YES,  17 ) \
-CPUID_81F_EAX_10 \
+FLAG( 81F,  0, EAX,  4,  1, SEV_SNP,                             NO,    0 ) \
+FLAG( 81F,  0, EAX,  5,  1, VMPL,                                NO,    0 ) \
+FLAG( 81F,  0, EAX, 10,  1, SEV_HEC,                             NO,    0 ) \
+FLAG( 81F,  0, EAX, 11,  1, SEV_64BIT_REQ,                       NO,    0 ) \
+FLAG( 81F,  0, EAX, 12,  1, SEV_RESTR_INJECTION,                 NO,    0 ) \
+FLAG( 81F,  0, EAX, 13,  1, SEV_ALT_INJECTION,                   NO,    0 ) \
+FLAG( 81F,  0, EAX, 14,  1, SEV_DEBUG_SWAP,                      NO,    0 ) \
+FLAG( 81F,  0, EAX, 15,  1, SEV_NO_HOST_IBS,                     NO,    0 ) \
 FIELD(81F,  0, EBX,  0,  6, SME_PAGE_TABLE_BIT_NUM,              YES,  17 ) \
 FIELD(81F,  0, EBX,  6,  6, SME_PHYS_ADDR_SPACE_REDUCTION,       NO,    0 ) \
+FIELD(81F,  0, EBX, 12,  4, NUM_VMPL,                            NO,    0 ) \
 FIELD(81F,  0, ECX,  0, 32, NUM_ENCRYPTED_GUESTS,                NO,    0 ) \
 FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,31 +39,6 @@
 #include "x86cpuid.h"


-/*
- * The declarations for the intrinsic functions were taken from MSDN.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#ifdef _MSC_VER
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * __cpuid has been supported since VS2003
- * __cpuidex has been supported since VS2008
- */
-void __cpuid(int regs[4], int eax);
-void __cpuidex(int regs[4], int eax, int ecx);
-#pragma intrinsic(__cpuid)
-#pragma intrinsic(__cpuidex)
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
-
 #ifdef __GNUC__ // {

 /*
@@ -113,6 +113,20 @@ MSRQuery;
 #define MSR_TSC_AUX           0xc0000103
 #define MSR_BD_TSC_RATIO      0xc0000104

+/* CET MSRs */
+#define MSR_U_CET                            0x6a0
+#define MSR_S_CET                            0x6a2
+#define MSR_CET_SH_STK_EN                         (1ULL << 0)
+#define MSR_CET_WR_SHSTK_EN                       (1ULL << 1)
+#define MSR_PL0_SSP                          0x6a4
+#define MSR_PL1_SSP                          0x6a5
+#define MSR_PL2_SSP                          0x6a6
+#define MSR_PL3_SSP                          0x6a7
+#define MSR_ISST_ADDR                        0x6a8
+
+#define MSR_TEST_CTRL                        0x33
+#define MSR_TEST_CTRL_SPLIT_LOCK_DETECT           (1ULL << 29)
+
 #define IA32_MSR_ARCH_CAPABILITIES           0x10a
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
@@ -480,6 +494,8 @@ MSRQuery;
 #define MSR_K8_SYSCFG_MTRRTOM2EN         (1ULL<<21)
 #define MSR_K8_SYSCFG_TOM2FORCEMEMTYPEWB (1ULL<<22)
 #define MSR_K8_SYSCFG_SMEE               (1ULL<<23)
+#define MSR_K8_SYSCFG_SNPE               (1ULL<<24)
+#define MSR_K8_SYSCFG_VMPLE              (1ULL<<25)

 #define MSR_K8_TOPMEM2       0xc001001d

@@ -521,13 +537,22 @@ MSRQuery;
 /* SEV related MSRs. */
 #define MSR_VMPAGE_FLUSH           0xc001011e
 #define MSR_GHCB_PA                0xc0010130
+#define MSR_GHCB_PA_FUNCTION_MASK       0xfff
 #define MSR_GHCB_PA_SEVINFO_HV              1
 #define MSR_GHCB_PA_SEVINFO_REQ             2
 #define MSR_GHCB_PA_AP_JUMP_TABLE           3
+#define MSR_GHCB_PA_CPUID_REQ               4
+#define MSR_GHCB_PA_CPUID_RESP              5
+#define MSR_GHCB_PA_TERMINATE             256
 #define MSR_SEV_STATUS             0xc0010131

 #define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
 #define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
+#define MSR_SEV_STATUS_SEV_SNP_EN  0x0000000000000004ULL // SEV-SNP is enabled
+
+/* SEV-SNP (Secure Nested Paging) MSRs. */
+#define MSR_RMP_BASE              0xc0010132 // Address of first byte of RMP
+#define MSR_RMP_END               0xc0010133 // Address of last byte of RMP

 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)
@@ -796,7 +821,6 @@ X86MSR_SetMSR(uint32 cx, uint64 value)
 }
 #endif
 #elif defined _MSC_VER // !__GNUC__ && _MSC_VER
-#ifdef _WIN64
 unsigned __int64  __readmsr(unsigned long);
 void              __writemsr(unsigned long, unsigned __int64);
 #pragma intrinsic(__readmsr, __writemsr)
@@ -811,37 +835,6 @@ X86MSR_SetMSR(uint32 cx, uint64 value)
 {
   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
 }
-#else // !__GNUC__ && _MSC_VER && !_WIN64
-#pragma warning( disable : 4035)
-static INLINE uint64
-X86MSR_GetMSR(uint32 input)
-{
-   __asm push ecx
-   __asm mov  ecx, input
-   /* 0x0f 0x32 -> edx:eax = rdmsr[ecx] */
-   __asm _emit 0x0f __asm _emit 0x32
-   __asm pop ecx
-}
-
-static INLINE void
-X86MSR_SetMSR(uint32 input, uint64 value)
-{
-      uint32 hival = (uint32)((value) >> 32);
-      uint32 loval = (uint32)value;
-      __asm push edx
-      __asm push ecx
-      __asm push eax
-      __asm mov  eax, loval
-      __asm mov  edx, hival
-      __asm mov  ecx, input
-      /* 0x0f 0x30 -> wrmsr[ecx] = edx:eax */
-      __asm _emit 0x0f __asm _emit 0x30
-      __asm pop  eax
-      __asm pop  ecx
-      __asm pop  edx
-}
-#pragma warning (default: 4035)
-#endif
 #else // !__GNUC__ && !_MSC_VER
 #error No compiler defined for RDMSR/WRMSR.
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,6 +40,7 @@
 #include "x86types.h"
 #include "vm_pagetable.h"
 #include "x86paging_common.h"
+#include "vm_assert.h"

 #define LM_PTE_PFN_MASK      CONST64U(0xffffffffff000)
 #define LM_PTE_2_PFN(_pte)   (((_pte) & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
@@ -136,4 +137,95 @@ LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
 #define VA64_CANONICAL_HOLE_START   (CONST64U(1) << (VA64_IMPL_BITS - 1))
 #define VA64_CANONICAL_HOLE_LEN  VA64_CANONICAL_MASK - VA64_CANONICAL_HOLE_START

+static INLINE Bool
+x86IsCanonicalC(VA64 va)
+{
+   return (va & VA64_CANONICAL_MASK) == 0 ||
+          (va & VA64_CANONICAL_MASK) == VA64_CANONICAL_MASK;
+}
+
+#if defined(VM_X86_64) && defined(__GNUC__)
+static INLINE Bool
+x86IsCanonicalAsm(VA64 va)
+{
+   Bool out;
+   /*
+    * sarq $48, %0: Move 17 bits from position 63:47 into 15:0 and CF,
+    *  sign-extending as we do so.
+    * adcl $0, %0: Then add (0 + CF) to the shifted value.  The result is zero
+    *  iff (CF == 1 && bits 31:0 == 0xffffffff) or (CF == 0 && bits 31:0 == 0).
+    *  That is, if original bits 47 and higher were all 1s or all 0s.
+    * setz %1: Create the boolean.
+    *
+    * The chain of AdcLong macros will emit one adc instruction; gcc
+    * has no obvious way to force the size of a quad %0 to a long.
+    */
+#define AdcLong(reg64, reg32) \
+   ".ifc  %0, %%" #reg64 "\n" \
+   " adcl $0, %%" #reg32 "\n" \
+   ".endif \n"
+
+   asm ("sarq $48, %0 \n"
+        AdcLong(rax, eax)
+        AdcLong(rcx, ecx)
+        AdcLong(rdx, edx)
+        AdcLong(rbx, ebx)
+        AdcLong(rsp, esp)
+        AdcLong(rbp, ebp)
+        AdcLong(rsi, esi)
+        AdcLong(rdi, edi)
+        AdcLong(r8,  r8d)
+        AdcLong(r9,  r9d)
+        AdcLong(r10, r10d)
+        AdcLong(r11, r11d)
+        AdcLong(r12, r12d)
+        AdcLong(r13, r13d)
+        AdcLong(r14, r14d)
+        AdcLong(r15, r15d)
+        "setz %1" : "+r"(va), "=r"(out));
+   return out;
+#undef AdcLong
+}
+#endif
+
+static INLINE Bool
+x86_IsCanonical(VA64 va)
+{
+#if defined(VM_X86_64) && defined(__GNUC__)
+   if (__builtin_constant_p(va)) {
+      return x86IsCanonicalC(va);
+   } else {
+      return x86IsCanonicalAsm(va);
+   }
+#else
+   return x86IsCanonicalC(va);
+#endif
+}
+
+static INLINE Bool
+x86_IsCanonicalRange(VA64 va, unsigned size)
+{
+   /*
+    * The check is simple as long as the size is less
+    * than the number of implemented bits.
+    *
+    * The only case we don't handle is one where the VA goes from a
+    * high canonical address and wraps to a non-canonical address
+    * (e.g. 0x00008000_00000000) or higher.  Our test would falsely
+    * consider this canonical.
+    */
+   ASSERT_ON_COMPILE(sizeof(size) * 8 < VA64_IMPL_BITS);
+
+   /*
+    * VA64_CANONICAL_MASK is the lowest canonical address with the
+    * upper bits all set.
+    *
+    * VA64_CANONICAL_HOLE_START is one higher than the highest valid
+    * canonical address with the upper bits all cleared.  Note that we
+    * access up to (va + size - 1), not (va + size), so <= is correct.
+    */
+   return va >= VA64_CANONICAL_MASK ||
+          va + size <= VA64_CANONICAL_HOLE_START;
+}
+
 #endif /* _X86PAGING_64_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,7 @@
 #define PF_RSVD         0x0008
 #define PF_ID           0x0010
 #define PF_PK           0x0020
+#define PF_SS           0x0040
 #define PF_SGX          0x8000

 /*
@@ -109,4 +110,12 @@ typedef struct {
   uint64   la;      /* Used for single address invalidation.  */
 } InvpcidArg;

+typedef enum {
+   PM_FLAT,        /* non paging mode                                      */
+   PM_2_LEVEL,     /* classic 32 bit non-pae style paging mode.            */
+   PM_3_LEVEL,     /* classic 32 bit pae style paging mode.                */
+   PM_4_LEVEL,     /* long mode style paging mode (including compat mode). */
+   PM_NUM     = 4, /* number of different paging modes.                    */
+} PagingMode;
+
 #endif /* _X86PAGING_COMMON_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,6 +39,7 @@
 extern "C" {
 #endif

+#include "vm_assert.h"

 #define SELECTOR_GDT             0
 #define SELECTOR_LDT             1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -95,9 +95,10 @@
 *       _V  -- virtualized
 *       _NV -- not virtualized
 *   _access is one of:
- *       _NA -- no access
- *       _RW -- read/write access
- *       _RO -- read-only access
+ *       _NA  -- no access
+ *       _RW  -- read/write access
+ *       _RO  -- read-only access
+ *       _URW -- read/write acess for the ULM only
 *   _sticky is one of:
 *       _S  -- sticky
 *       _NS -- non-sticky
@@ -109,9 +110,9 @@
 VMCS_SET_START(16)
 /* 16-bit control fields. */
 VMCS_GROUP_START(16, CTL)
-VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA,  _S)
-VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA, _NS)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV,  _NA,  _S)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _URW, _NS)
 VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
 VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
 VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
@@ -328,16 +329,16 @@ VMCS_GROUP_END(64, DATA)

 /* 64-bit guest state. */
 VMCS_GROUP_START(64, GUEST)
-VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA, _NS)
-VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA, _NS)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _URW, _NS)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V,  _NA, _NS)
 VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
 VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
 VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
@@ -403,24 +404,24 @@ VMCS_SET_END(64)
 VMCS_SET_START(32)
 /* 32-bit control fields. */
 VMCS_GROUP_START(32, CTL)
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA, _NS)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA,  _S)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA,  _S)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _URW,  _S)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _URW, _NS)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _URW,  _S)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V,  _NA,  _S)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
 VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
 VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
@@ -551,14 +552,14 @@ VMCS_SET_END(32)
 VMCS_SET_START(NAT)
 /* natural-width control fields. */
 VMCS_GROUP_START(NAT, CTL)
-VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
 VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
 VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
@@ -622,26 +623,26 @@ VMCS_GROUP_END(NAT, DATA)

 /* natural-width guest state. */
 VMCS_GROUP_START(NAT, GUEST)
-VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW, _NS)
-VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW, _NS)
-VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW, _NS)
-VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA, _NS)
-VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _URW, _NS)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _URW, _NS)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V,  _RW, _NS)
 VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
 VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
 VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
@@ -658,18 +659,18 @@ VMCS_GROUP_END(NAT, GUEST)

 /* natural-width host state. */
 VMCS_GROUP_START(NAT, HOST)
-VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
 VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
 VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
@@ -23,6 +23,7 @@

 #include <linux/file.h>
 #include <linux/highmem.h>
+#include <linux/miscdevice.h>
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/slab.h>
@@ -36,8 +37,8 @@

 #include "usercalldefs.h"

-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
-#error Linux kernels before 2.6.32 are not supported
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0)
+#error Linux kernels before 3.10 are not supported
 #endif

 #include <asm/io.h>
@@ -96,7 +97,22 @@ static int LinuxDriver_Close(struct inode *inode, struct file *filp);

 static unsigned int LinuxDriverEstimateTSCkHz(void);

-static struct file_operations vmuser_fops;
+static const struct file_operations vmuser_fops = {
+   .owner = THIS_MODULE,
+   .open = LinuxDriver_Open,
+   .release = LinuxDriver_Close,
+   .unlocked_ioctl = LinuxDriver_Ioctl,
+   .compat_ioctl = LinuxDriver_Ioctl,
+};
+
+#ifndef VMX86_DEVEL
+static struct miscdevice vmmon_miscdev = {
+   .name = "vmmon",
+   .minor = MISC_DYNAMIC_MINOR,
+   .fops = &vmuser_fops,
+};
+#endif
+
 static struct timer_list tscTimer;
 static Atomic_uint32 tsckHz;
 static VmTimeStart tsckHzStartTime;
@@ -251,9 +267,8 @@ LinuxDriverInitTSCkHz(void)
 *      linux module entry point. Called by /sbin/insmod command
 *
 * Results:
- *      registers a device driver for a major # that depends
- *      on the uid. Add yourself to that list.  List is now in
- *      private/driver-private.c.
+ *      Release: registers a device driver with a misc minor node.
+ *      Devel: registers for a major number with user-created node.
 *
 *----------------------------------------------------------------------
 */
@@ -280,46 +295,43 @@ init_module(void)
   if (!Vmx86_CreateHVIOBitmap()) {
      return -ENOMEM;
   }
+
+   if (!Vmx86_CheckMSRUniformity()) {
+      return -EPERM;
+   }
+
   linuxState.fastClockThread = NULL;
   linuxState.fastClockRate = 0;

-   /*
-    * Initialize the file_operations structure. Because this code is always
-    * compiled as a module, this is fine to do it here and not in a static
-    * initializer.
-    */
-
-   memset(&vmuser_fops, 0, sizeof vmuser_fops);
-   vmuser_fops.owner = THIS_MODULE;
-   vmuser_fops.unlocked_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.compat_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.open = LinuxDriver_Open;
-   vmuser_fops.release = LinuxDriver_Close;
-
 #ifdef VMX86_DEVEL
   devel_init_module();
-   linuxState.minor = 0;
   retval = register_chrdev(linuxState.major, linuxState.deviceName,
                            &vmuser_fops);
+   if (retval) {
+      Warning("Module %s: error registering with major=%d\n",
+              linuxState.deviceName, linuxState.major);
+   } else {
+      Log("Module %s: registered with major=%d\n",
+          linuxState.deviceName, linuxState.major);
+   }
 #else
   sprintf(linuxState.deviceName, "vmmon");
   linuxState.major = 10;
-   linuxState.minor = 165;
-   linuxState.misc.minor = linuxState.minor;
-   linuxState.misc.name = linuxState.deviceName;
-   linuxState.misc.fops = &vmuser_fops;

-   retval = misc_register(&linuxState.misc);
+   retval = misc_register(&vmmon_miscdev);
+   if (retval) {
+      Warning("Module %s: error registering misc device %s\n",
+              linuxState.deviceName, vmmon_miscdev.name);
+   } else {
+      Log("Module %s: registered as misc device %s\n", linuxState.deviceName,
+          vmmon_miscdev.name);
+   }
 #endif

   if (retval) {
-      Warning("Module %s: error registering with major=%d minor=%d\n",
-              linuxState.deviceName, linuxState.major, linuxState.minor);
      Vmx86_CleanupHVIOBitmap();
      return -ENOENT;
   }
-   Log("Module %s: registered with major=%d minor=%d\n",
-       linuxState.deviceName, linuxState.major, linuxState.minor);

   HostIF_InitUptime();
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
@@ -357,7 +369,7 @@ cleanup_module(void)
 #ifdef VMX86_DEVEL
   unregister_chrdev(linuxState.major, linuxState.deviceName);
 #else
-   misc_deregister(&linuxState.misc);
+   misc_deregister(&vmmon_miscdev);
 #endif

   Log("Module %s: unloaded\n", linuxState.deviceName);
@@ -787,6 +799,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      } else {
         device->vm = vm;
         args.vmid = vm->userID;
+         vm->vmhost->vmmonData = (void *__user)args.vmmonData;
         retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args);
      }
      break;
@@ -1080,6 +1093,23 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_SWITCH_ERROR_ADDR: {
+      VMSwitchErrorArgs args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->crosspage == NULL ||
+          vm->crosspage[args.vcpuid] == NULL) {
+         retval = -EINVAL;
+         break;
+      }
+      args.addr = vm->crosspage[args.vcpuid]->wsUD2;
+      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      break;
+   }
+
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
      retval = LinuxDriverEstimateTSCkHz();
      break;
@@ -1227,7 +1257,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
-#ifdef VMX86_DEVEL
+#if VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -23,7 +23,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
@@ -61,8 +60,6 @@ typedef struct Device {

 typedef struct VMXLinuxState {
   int major;
-   int minor;
-   struct miscdevice misc;
   char deviceName[VM_DEVICE_NAME_SIZE];
   char buf[LINUXLOG_BUFFER_SIZE];
   Device *head;
@@ -72,6 +72,7 @@
 #include "memDefaults.h"
 #include "vcpuid.h"
 #include "x86svm.h"
+#include "crosspage.h"

 #include "pgtbl.h"
 #include "versioned_atomic.h"
@@ -188,6 +189,7 @@ static void UnlockEntry(void *clientData, MemTrackEntry *entryPtr);
 #define SPURIOUS_APIC_VECTOR 0
 #endif
 static const uint8 monitorIPIVector = SPURIOUS_APIC_VECTOR;
+
 /*
 * POSTED_INTR_VECTOR is defined on kernels after 3.10 when built with
 * KVM support.
@@ -197,6 +199,15 @@ static const uint8 monitorIPIVector = SPURIOUS_APIC_VECTOR;
 #endif
 static const uint8 hvIPIVector = POSTED_INTR_VECTOR;

+/*
+ * perfCtrVector may leak, choose highest priority interrupt that is
+ * consistently ack'd by all current linux kernels.
+ */
+#ifndef ERROR_APIC_VECTOR
+#define ERROR_APIC_VECTOR 0xFE
+#endif
+#define PERF_CTR_VECTOR ERROR_APIC_VECTOR
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -615,38 +626,6 @@ HostIF_FastClockUnlock(int callerID) // IN
 }


-/*
- *----------------------------------------------------------------------
- *
- * MapCrossPage & UnmapCrossPage
- *
- *    Both x86-64 and ia32 need to map crosspage to an executable
- *    virtual address. We use the vmap interface instead of kmap
- *    due to bug 43907.
- *
- * Side effects:
- *
- *    UnmapCrossPage assumes that the page has been refcounted up
- *    so it takes care of the put_page.
- *
- *----------------------------------------------------------------------
- */
-static void *
-MapCrossPage(struct page *p)  // IN:
-{
-   return vmap(&p, 1, VM_MAP, VM_PAGE_KERNEL_EXEC);
-}
-
-
-static void
-UnmapCrossPage(struct page *p,  // IN:
-               void *va)        // IN:
-{
-   vunmap(va);
-   put_page(p);
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -974,11 +953,8 @@ static void
 HostIFFreeVMHost(VMHost *vmhost) // IN:
 {
   ASSERT(vmhost->lockedPages == NULL &&
-          vmhost->AWEPages    == NULL &&
-          vmhost->crosspagePagesCount == 0);
-   Vmx86_Free(vmhost->crosspagePages);
+          vmhost->AWEPages    == NULL);
   Vmx86_Free(vmhost->vcpuSemaTask);
-   vmhost->crosspagePages = NULL;
   vmhost->vcpuSemaTask   = NULL;
   HostIF_FreeKernelMem(vmhost);
 }
@@ -1007,14 +983,14 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
   if (vmhost == NULL) {
      return NULL;
   }
-   if ((vmhost->crosspagePages =
-        Vmx86_Calloc(numVCPUs, sizeof *vmhost->crosspagePages, TRUE)) != NULL &&
-       (vmhost->vcpuSemaTask =
-        Vmx86_Calloc(numVCPUs, sizeof *vmhost->vcpuSemaTask, TRUE))   != NULL) {
-      return vmhost;
+   vmhost->vcpuSemaTask = Vmx86_Calloc(numVCPUs, sizeof *vmhost->vcpuSemaTask,
+                                       TRUE);
+   if (vmhost->vcpuSemaTask == NULL) {
+      HostIFFreeVMHost(vmhost);
+      return NULL;
   }
-   HostIFFreeVMHost(vmhost);
-   return NULL;
+
+   return vmhost;
 }


@@ -1519,20 +1495,12 @@ UnlockEntry(void *clientData,         // IN:
 void
 HostIF_FreeAllResources(VMDriver *vm) // IN
 {
-   unsigned int cnt;
-
   HostIFHostMemCleanup(vm);
   if (vm->memtracker) {
      MemTrack_Cleanup(vm->memtracker, UnlockEntry, vm);
      vm->memtracker = NULL;
   }
   if (vm->vmhost) {
-      ASSERT(vm->vmhost->crosspagePagesCount <= vm->numVCPUs);
-      for (cnt = vm->vmhost->crosspagePagesCount; cnt > 0; ) {
-         struct page* p = vm->vmhost->crosspagePages[--cnt];
-         UnmapCrossPage(p, vm->crosspage[cnt]);
-      }
-      vm->vmhost->crosspagePagesCount = 0;
      HostIFFreeVMHost(vm->vmhost);
      vm->vmhost = NULL;
   }
@@ -1739,7 +1707,10 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
   lockedPages += global_page_state(NR_PAGETABLE);
 #endif
   /* NR_SLAB_* moved from zone to node in 4.13. */
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)
+   lockedPages += global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
   lockedPages += global_node_page_state(NR_SLAB_UNRECLAIMABLE);
 #else
   lockedPages += global_page_state(NR_SLAB_UNRECLAIMABLE);
@@ -2154,49 +2125,36 @@ HostIF_CopyToUser(VA64 dst,         // OUT
 /*
 *-----------------------------------------------------------------------------
 *
- * HostIF_MapCrossPage --
+ * HostIFGetCrossPageMPNWork --
+ * HostIF_GetCrossPageCodeMPN --
+ * HostIF_GetCrossPageDataMPN --
 *
- *    Map the cross page in the kernel address space given a user VA.  The
- *    kernel mapping must not overlap the monitor's 64MB VA space (bug 32922).
- *    In practice, the host kernel does not return mappings in that range.
- *    This is checked in TaskCreatePTPatch() which gracefully fails if it must.
- *
- * Results:
- *    The kernel virtual address on success
- *    NULL on failure
- *
- * Side effects:
- *    None
 *
 *-----------------------------------------------------------------------------
 */

-void *
-HostIF_MapCrossPage(VMDriver *vm, // IN
-                    VA64 uAddr)   // IN
+static MPN
+HostIFGetCrossPageMPNWork(void *va)
 {
-   void *p = VA64ToPtr(uAddr);
-   struct page *page;
-   VA           vPgAddr;
-   VA           ret;
+   struct page *pg = vmalloc_to_page(va);

-   if (HostIFGetUserPages(p, &page, 1)) {
-      return NULL;
+   if (pg != NULL) {
+      return page_to_pfn(pg);
   }
-   vPgAddr = (VA) MapCrossPage(page);
-   HostIF_VMLock(vm, 27);
-   if (vm->vmhost->crosspagePagesCount >= vm->numVCPUs) {
-      HostIF_VMUnlock(vm, 27);
-      UnmapCrossPage(page, (void*)vPgAddr);

-      return NULL;
-   }
-   vm->vmhost->crosspagePages[vm->vmhost->crosspagePagesCount++] = page;
-   HostIF_VMUnlock(vm, 27);
+   return INVALID_MPN;
+}

-   ret = vPgAddr | (((VA)p) & (PAGE_SIZE - 1));
+MPN
+HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData)
+{
+   return HostIFGetCrossPageMPNWork(crosspageData);
+}

-   return (void*)ret;
+MPN
+HostIF_GetCrossPageCodeMPN(void)
+{
+   return HostIFGetCrossPageMPNWork((void *)CrossPage_CodePage);
 }


@@ -2389,16 +2347,23 @@ HostIF_VMLockIsHeld(VMDriver *vm) // IN
 static Bool
 isVAReadable(VA r)  // IN:
 {
-   mm_segment_t old_fs;
   uint32 dummy;
   int ret;

+   r = APICR_TO_ADDR(r, APICR_VERSION);
+#ifdef HAVE_GET_KERNEL_NOFAULT
+   ret = get_kernel_nofault(dummy, (void *)r);
+#else
+   {
+   mm_segment_t old_fs;
+
   old_fs = get_fs();
   set_fs(KERNEL_DS);
-   r = APICR_TO_ADDR(r, APICR_VERSION);
+
   ret = HostIF_CopyFromUser(&dummy, r, sizeof dummy);
   set_fs(old_fs);
-
+   }
+#endif
   return ret == 0;
 }
 #endif
@@ -2530,6 +2495,32 @@ HostIF_GetHVIPIVector(void)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_GetPerfCtrVector --
+ *
+ *      Return a vector the monitor can use for perf counters.
+ *      This needs to be a high priority interrupt that is always ack'd
+ *      by the kernel because they can leak in edge cases.
+ *
+ * Results:
+ *     The vector for perf counter events.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint8
+HostIF_GetPerfCtrVector(void)
+{
+   ASSERT_ON_COMPILE(PERF_CTR_VECTOR == 0xFE);
+   return PERF_CTR_VECTOR;
+}
+
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -2538,8 +2529,10 @@ HostIF_GetHVIPIVector(void)
 *    Perform the semaphore wait (P) operation, possibly blocking.
 *
 * Result:
- *    1 (which equals MX_WAITNORMAL) if success,
- *    negated error code otherwise.
+ *      On success: MX_WAITINTERRUPTED (3) if interrupted by a Unix signal.
+ *                  MX_WAITTIMEDOUT (2) if the timeout occurred.
+ *                  MX_WAITNORMAL (1) if the event was raised.
+ *      On error: MX_WAITERROR (0) on generic error.
 *
 * Side-effects:
 *    None
@@ -2553,11 +2546,12 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
                     uint64 *args)   // IN:
 {
   struct file *file;
-   mm_segment_t old_fs;
   int res;
   int waitFD = args[0];
   int timeoutms = args[2];
   uint64 value;
+   struct poll_wqueues table;
+   unsigned int mask;

   ASSERT(vcpuid < vm->numVCPUs);

@@ -2565,47 +2559,46 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   if (file == NULL) {
      return MX_WAITERROR;
   }
-
-   old_fs = get_fs();
-   set_fs(KERNEL_DS);
-
-   {
-      struct poll_wqueues table;
-      unsigned int mask;
-
-      poll_initwait(&table);
-      current->state = TASK_INTERRUPTIBLE;
-      mask = file->f_op->poll(file, &table.pt);
-      if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
-         vm->vmhost->vcpuSemaTask[vcpuid] = current;
-         schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
-         vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
-      }
-      current->state = TASK_RUNNING;
-      poll_freewait(&table);
+   if (file->f_op->poll == NULL) {
+      fput(file);
+      Warning("%s: File poll operation is NULL\n", __func__);
+      return MX_WAITERROR;
   }

+   poll_initwait(&table);
+   current->state = TASK_INTERRUPTIBLE;
+   mask = file->f_op->poll(file, &table.pt);
+   if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
+      vm->vmhost->vcpuSemaTask[vcpuid] = current;
+      schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
+      vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
+   }
+   current->state = TASK_RUNNING;
+   poll_freewait(&table);
+
   /*
    * Userland only writes in multiples of sizeof(uint64). This will allow
    * the code to happily deal with a pipe or an eventfd. We only care about
    * reading no bytes (EAGAIN - non blocking fd) or sizeof(uint64).
+    *
+    * Upstream Linux changed the function parameter types/ordering in 4.14.0.
    */
-
-   res = file->f_op->read(file, (char *) &value, sizeof value, &file->f_pos);
-
-   if (res == sizeof value) {
-      res = MX_WAITNORMAL;
-   } else {
-      if (res == 0) {
-         res = -EBADF;
-      }
-   }
-
-   set_fs(old_fs);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+   res = kernel_read(file, file->f_pos, (char *)&value, sizeof value);
+#else
+   res = kernel_read(file, &value, sizeof value, &file->f_pos);
+#endif
   fput(file);

+   if (res == sizeof value) {
+      return MX_WAITNORMAL;
+   }
+   if (res == 0) {
+      res = -EBADF;
+   }
+
   /*
-    * Handle benign errors:
+    * Handle various errors:
    * EAGAIN is MX_WAITTIMEDOUT.
    * The signal-related errors are all mapped into MX_WAITINTERRUPTED.
    */
@@ -2624,6 +2617,10 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   case -EBADF:
      res = MX_WAITERROR;
      break;
+   default:
+      Warning("Unexpected error: %s err=%d\n", __FUNCTION__, res);
+      res = MX_WAITERROR;
+      break;
   }
   return res;
 }
@@ -2677,7 +2674,6 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
 *      On error: MX_WAITINTERRUPTED (3) if interrupted by a Unix signal (we
 *                   can block on a preemptive kernel).
 *                MX_WAITERROR (0) on generic error.
- *                Negated system error (< 0).
 *
 * Side-effects:
 *      None
@@ -2686,57 +2682,64 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
 */

 int
-HostIF_SemaphoreSignal(uint64 *args)  // IN:
+HostIF_SemaphoreSignal(VMDriver *vm,  // IN:
+                       uint64 *args)  // IN:
 {
-   struct file *file;
-   mm_segment_t old_fs;
   int res;
   int signalFD = args[1];
-   uint64 value = 1;  // make an eventfd happy should it be there
+   struct file *file;

   file = vmware_fget(signalFD);
-   if (!file) {
-      return MX_WAITERROR;
+   if (file == NULL) {
+      return FALSE;
   }

-   old_fs = get_fs();
-   set_fs(KERNEL_DS);
-
   /*
    * Always write sizeof(uint64) bytes. This works fine for eventfd and
    * pipes. The data written is formatted to make an eventfd happy should
    * it be present.
    */
-
-   res = file->f_op->write(file, (char *) &value, sizeof value, &file->f_pos);
-
-   if (res == sizeof value) {
-      res = MX_WAITNORMAL;
+   if (file->f_op->write != NULL) {
+      /*
+       * PR2650348 - The kernel_write function signature changed in the vanilla
+       * kernel in 4.14.0 and the signature update was backported before that
+       * point in a kernel used by OpenSUSE 15.1 so call the function directly
+       * instead of via the wrapper.
+       *
+       * Kernel 5.10 removed the set_fs functionality which means a user pointer
+       * must be passed into the write function or a -EFAULT will be returned.
+       */
+      res = file->f_op->write(file, vm->vmhost->vmmonData, sizeof (uint64),
+                              &file->f_pos);
+   } else {
+      Warning("%s: File write operation is NULL\n", __func__);
+      res = -ENOTSUPP;
   }
-
-   set_fs(old_fs);
   fput(file);

+   if (res == sizeof (uint64)) {
+      return MX_WAITNORMAL;
+   }
+
   /*
-    * Handle benign errors:
+    * Handle various errors:
    * EAGAIN is MX_WAITTIMEDOUT.
    * The signal-related errors are all mapped into MX_WAITINTERRUPTED.
    */
-
   switch (res) {
   case -EAGAIN:
-      // The pipe is full, so it is already signalled. Success.
-      res = MX_WAITNORMAL;
-      break;
+      /* The pipe is full, so it is already signalled. Success. */
+      return MX_WAITNORMAL;
   case -EINTR:
   case -ERESTART:
   case -ERESTARTSYS:
   case -ERESTARTNOINTR:
   case -ERESTARTNOHAND:
-      res = MX_WAITINTERRUPTED;
-      break;
+      return MX_WAITINTERRUPTED;
+   default:
+      Warning("Unexpected error: %s err=%d\n", __FUNCTION__, res);
+      return MX_WAITERROR;
   }
-   return res;
 }


@@ -3261,12 +3264,9 @@ static int
 HostIFFastClockThread(void *unused)  // IN:
 {
   int res;
-   mm_segment_t oldFS;
   unsigned int rate = 0;
   unsigned int prevRate = 0;

-   oldFS = get_fs();
-   set_fs(KERNEL_DS);
   allow_signal(SIGKILL);

   while ((rate = linuxState.fastClockRate) > MIN_RATE) {
@@ -3289,8 +3289,6 @@ HostIFFastClockThread(void *unused)  // IN:
   }

 out:
-   set_fs(oldFS);
-
   /*
    * Do not exit thread until we are told to do so.
    */
@@ -79,8 +79,6 @@ typedef struct VMHost {
    */
   Mutex vmMutex;

-   unsigned int         crosspagePagesCount;
-   struct page        **crosspagePages;      /* ptr to numVCPUs-sized array */
   struct task_struct **vcpuSemaTask;        /* ptr to numVCPUs-sized array */

   /*
@@ -94,6 +92,11 @@ typedef struct VMHost {
    * as pages for "AWE" guest memory.
    */
   struct PhysTracker  *AWEPages;
+   /*
+    * Pointer to a userlevel 64-bit area containing the value 1.
+    * This is used for HostIF_SemaphoreSignal.
+    */
+   void *__user vmmonData;
 } VMHost;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -75,12 +75,13 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
   uint64           retval    = MODULECALL_USERRETURN;
   VMCrossPageData *crosspage;
   int              bailValue = 0;
+   Bool             switchOk = TRUE;

   ASSERT(vcpuid < vm->numVCPUs);
   if (vm->crosspage[vcpuid] == NULL) {
      return USERCALL_VMX86ALLOCERR;
   }
-   crosspage = &vm->crosspage[vcpuid]->crosspageData;
+   crosspage = vm->crosspage[vcpuid];

   /*
    * Check if we were interrupted by signal.
@@ -95,9 +96,9 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
       * Task_Switch changes the world to the monitor.
       * The monitor is waiting in the BackToHost routine.
       */
-      UCTIMESTAMP(crosspage, SWITCHING_TO_MONITOR);
-      Task_Switch(vm, vcpuid);
-      UCTIMESTAMP(crosspage, SWITCHED_TO_MODULE);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHING_TO_MONITOR);
+      switchOk = Task_Switch(vm, vcpuid);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHED_TO_MODULE);

      /*
       * Wake up anything that was waiting for this vcpu to run
@@ -119,6 +120,11 @@ skipTaskSwitch:;

      retval = MODULECALL_USERRETURN;

+      if (UNLIKELY(!switchOk)) {
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
+
      if (crosspage->userCallType != MODULECALL_USERCALL_NONE) {
         /*
          * This is the main user call path.  Handle by returning
@@ -165,7 +171,6 @@ skipTaskSwitch:;

      case MODULECALL_SEMAWAIT: {
         retval = HostIF_SemaphoreWait(vm, vcpuid, crosspage->args);
-
         if (retval == MX_WAITINTERRUPTED) {
            crosspage->moduleCallInterrupted = TRUE;
            bailValue = USERCALL_RESTART;
@@ -175,7 +180,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_SEMASIGNAL: {
-         retval = HostIF_SemaphoreSignal(crosspage->args);
+         retval = HostIF_SemaphoreSignal(vm, crosspage->args);

         if (retval == MX_WAITINTERRUPTED) {
             crosspage->moduleCallInterrupted = TRUE;
@@ -283,6 +288,10 @@ skipTaskSwitch:;
         retval = HostIF_GetHVIPIVector();
      } break;

+      case MODULECALL_GET_PERF_CTR_VECTOR: {
+         retval = HostIF_GetPerfCtrVector();
+      } break;
+
      case MODULECALL_GET_HOST_TIMER_VECTORS: {
         uint8 v0, v1;
         HostIF_GetTimerVectors(&v0, &v1);
@@ -321,6 +330,19 @@ skipTaskSwitch:;
                                                   hvIOBitmap->mpn;
      } break;

+      case MODULECALL_GET_MSR: {
+         /*
+          * Allocate a MSRQuery request on stack with only one MSRReply slot
+          * since this MSR will be queried from a MSR cache or on a single pcpu.
+          */
+         uint8 req[sizeof(MSRQuery) + sizeof(MSRReply)];
+         MSRQuery *query = (MSRQuery *)&req[0];
+         query->msrNum = (uint32)crosspage->args[0];
+         query->numLogicalCPUs = 1;
+         retval = Vmx86_GetAllMSRs(query);
+         crosspage->args[0] = query->logicalCPUs[0].msrVal;
+      } break;
+
      case MODULECALL_ALLOC_CONTIG_PAGES: {
         PageCnt pages = crosspage->args[0];
         HostIFContigMemMap *alloc;
@@ -340,7 +362,13 @@ skipTaskSwitch:;
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
-      ASSERT(retval == (uint64)((uint32)retval));
+
+      if (retval != (uint64)((uint32)retval)) {
+         Warning("Unexpected error in modulecall %u (%llu)\n",
+                 crosspage->moduleCallType, retval);
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
      crosspage->retval = (uint32)retval;
 #if defined(__linux__)
      cond_resched(); // Other kernels are preemptible
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -136,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002,2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -4327,7 +4327,7 @@ LookupTypeClass(unsigned short typeValue) // IN: ethernet type
 *----------------------------------------------------------------------
 */

-#if _WIN32
+#ifdef _WIN32
 _Use_decl_annotations_
 #endif
 void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -81,7 +81,9 @@ extern unsigned int  vnet_max_qlen;
 #   define skb_frag_page(frag) (frag)->page
 #   define skb_frag_size(frag) (frag)->size
 #endif
-#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0)
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0) && \
+    !(defined(CONFIG_SUSE_VERSION) && CONFIG_SUSE_VERSION == 15 && \
+      defined(CONFIG_SUSE_PATCHLEVEL) && CONFIG_SUSE_PATCHLEVEL >= 2)
 #   define skb_frag_off(frag) (frag)->page_offset
 #endif

@@ -559,7 +561,12 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
      return -EINVAL;
   }

+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
   csum = csum_and_copy_to_user(skb->data + offset, curr, len, 0, &err);
+#else
+   csum = csum_and_copy_to_user(skb->data + offset, curr, len);
+   err = (csum == 0) ? -EFAULT : 0;
+#endif
   if (err) {
      return err;
   }
@@ -573,8 +580,14 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
 	 const void *vaddr;

 	 vaddr = kmap(skb_frag_page(frag));
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
 	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
 					 curr, skb_frag_size(frag), 0, &err);
+#else
+	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
+					 curr, skb_frag_size(frag));
+         err = (tmpCsum == 0) ? -EFAULT : 0;
+#endif
 	 kunmap(skb_frag_page(frag));

 	 if (err) {
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -145,64 +152,6 @@ typedef ALIGNED(16) struct Atomic_uint128 {
 } Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock And [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -499,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -552,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -588,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -632,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -655,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -782,17 +772,9 @@ PAUSE(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -840,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -853,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,41 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef ULM
-void _fxsave64(void *save);
-void _fxsave(void *save);
-void _fxrstor64(const void *load);
-void _fxrstor(const void *load);
-void _xsave64(void *save, uint64 mask);
-void _xsave(void *save, uint64 mask);
-void _xsaveopt64(void *save, uint64 mask);
-void _xsavec(void *save, uint64 mask);
-void _xrstor64(const void *load, uint64 mask);
-void _xrstor(const void *load, uint64 mask);
-#endif /* ULM */
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#ifdef ULM
-#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
-                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
-#endif /* ULM */
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -358,11 +259,7 @@ SFENCE(void)
   );
 #elif defined _MSC_VER
   _ReadWriteBarrier();
-#if defined VM_X86_32
-   __asm sfence;
-#else
   _mm_sfence();
-#endif
   _ReadWriteBarrier();
 #else
 #error No compiler defined for SFENCE
@@ -688,22 +688,28 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-3.3 (circa 2003).  Enforce this here, various things below
+ * gcc-4.1 (circa 2006).  Enforce this here, various things below
 * this line depend upon it.
 *
- * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
- * The various linux kernel modules may use older (gcc-3.3) compilers.
+ * Current oldest compilers:
+ * - guest tools: 4.1.2 (freebsd/solaris)
+ * - buildhost compiler: 4.4.3
+ * - hosted kernel modules: 4.5
+ *
+ * SWIG's preprocessor is exempt.
 */
-#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
-#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
+#ifndef SWIG
+#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
+#error "gcc version is too old, need gcc-4.1 or better"
+#endif
 #endif

 /*
- * Similarly, we require a compiler that is at least vc90 (vs2008).
+ * Similarly, we require a compiler that is at least vs2012.
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1500
-#error "cl.exe version is too old, need vc90 or better"
+#if defined _MSC_VER && _MSC_VER < 1700
+#error "cl.exe version is too old, need vs2012 or better"
 #endif


@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2005-2012,2014-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2005-2012,2014-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,8 +35,6 @@
 #endif
 #endif

-#include <linux/pci_ids.h>
-
 /* LSILogic 53C1030 Parallel SCSI controller
 * LSILogic SAS1068 SAS controller
 */
@@ -55,10 +53,7 @@
 *    VMware HD Audio codec
 *    VMware HD Audio controller
 */
-#ifndef PCI_VENDOR_ID_VMWARE
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
-#endif
-
 #define PCI_DEVICE_ID_VMWARE_SVGA3              0x0406
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
@@ -76,11 +71,7 @@
 #define PCI_DEVICE_ID_VMWARE_1394               0x0780
 #define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
 #define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
-
-#ifndef PCI_DEVICE_ID_VMWARE_VMXNET3
 #define PCI_DEVICE_ID_VMWARE_VMXNET3            0x07B0
-#endif
-
 #define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
 #define PCI_DEVICE_ID_VMWARE_82574              0x07D0
 #define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
@@ -353,7 +344,7 @@
 #define MAX_NVDIMM 64

 /************* vRDMA implementation limits ******************************/
-#define MAX_VRDMA_DEVICES 1
+#define MAX_VRDMA_DEVICES 10

 /************* QAT implementation limits ********************/
 #define MAX_QAT_PCI_DEVICES 4
Author	SHA1	Message	Date
Michal Kubecek	31c032e989	import Player 16.1.0 module sources	2020-11-20 09:13:14 +01:00
Michal Kubecek	a84c5005fe	import Player 16.0.0 module sources	2020-09-15 20:45:48 +02:00
Michal Kubecek	7fd1a58863	import Player 15.5.5 module sources	2020-05-30 02:30:43 +02:00
Michal Kubecek	cbd3374fd0	import Player 15.5.2 module sources	2020-03-14 09:59:24 +01:00
Michal Kubecek	87feab5f03	import Player 15.5.1 module sources	2019-11-14 07:56:57 +01:00
Michal Kubecek	6daa6921e7	import Player 15.5.0 module sources	2019-09-21 00:11:26 +02:00
Michal Kubecek	193db5638c	Merge branch 'master' into player	2019-09-21 00:10:31 +02:00
Michal Kubecek	1f8ca44e74	import Player 15.1.0 module sources	2019-05-15 07:06:02 +02:00
Michal Kubecek	3686288fcb	import Player 15.0.1 module sources	2018-11-12 07:48:28 +01:00
Michal Kubecek	3d8f309653	import Player 15.0.0 module sources	2018-10-02 19:09:36 +02:00
Michal Kubecek	56b2ba7686	import Player 14.1.3 module sources	2018-08-20 07:28:17 +02:00
Michal Kubecek	7c61516e73	Merge branch 'master' into player	2018-05-27 16:13:48 +02:00
Michal Kubecek	7ee6cd4520	import Player 14.1.2 module sources	2018-05-23 18:14:05 +02:00
Michal Kubecek	5791987f56	Merge branch 'master' into player	2018-04-02 15:37:18 +02:00
Michal Kubecek	0b2713797e	Merge branch 'master' into player	2018-02-04 21:50:37 +01:00
Michal Kubecek	3e445144a1	import Player 14.1.1 module sources	2018-01-12 13:51:34 +01:00
Michal Kubecek	82f569a212	import Player 14.1.0 module sources	2017-12-25 09:47:38 +01:00
Michal Kubecek	455d4d9368	import Player 14.0.0 module sources	2017-09-30 20:31:15 +02:00
Michal Kubecek	027dd56418	import Player 12.5.7 module sources	2017-06-30 19:11:22 +02:00
Michal Kubecek	0e634ffbf1	import Player 12.5.6 module sources	2017-05-31 10:06:51 +02:00
Michal Kubecek	c62c78e908	import Player 12.5.5 module sources	2017-05-31 09:20:00 +02:00
Michal Kubecek	4f648692e4	import Player 12.5.2 module sources	2017-05-31 09:19:26 +02:00