vmmon: use standard definition of MSR_TSX_FORCE_ABORT if available

The MSR_TSX_FORCE_ABORT macro is defined in mainline since commit 52f64909409c ("x86: Add TSX Force Abort CPUID/MSR") in v5.1-rc1.
vmnet: convert VNetBridge debugging code to ktime_get_ns()
2020-05-30 02:48:55 +02:00 · 2020-05-30 02:48:43 +02:00 · 2020-05-30 02:48:11 +02:00 · 2020-05-30 02:47:57 +02:00 · 2020-05-30 02:47:50 +02:00 · 2020-05-30 02:47:42 +02:00
58 changed files with 1605 additions and 2670 deletions
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -136,8 +136,6 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

-# Add Spectre options when available
-
 include $(SRCROOT)/Makefile.kernel

 else
@@ -31,7 +31,7 @@ $(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, \
 		$(SRCROOT)/bootstrap/*.c)))

 clean:
-	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \
+	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions .cache.mk \
 		Module.symvers Modules.symvers Module.markers modules.order \
 		$(foreach dir,linux/ common/ vmcore/ bootstrap/ \
 		./,$(addprefix $(dir),.*.cmd .*.o.flags *.o)))
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998,2015,2017,2020 VMware, Inc. All rights reserved.
+# Copyright (C) 1998,2015,2017 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -72,7 +72,7 @@ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')

 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
-C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o crosspage.o
+C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o
 C_TARGETS_VMCORE := moduleloop.o
 C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -447,7 +447,7 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 *
 * MonLoaderCallout_GetSharedUserPage --
 *
- *      Gets a shared page's MPN for a VCPU.
+ *      Gets a user-provided shared page's MPN for a VCPU.
 *
 * Returns:
 *      An MPN or INVALID_MPN.
@@ -478,14 +478,7 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   switch (subIndex) {
-   case MONLOADER_CROSS_PAGE_DATA_IDX:
-      return HostIF_GetCrossPageDataMPN(ctx->vm->crosspage[vcpu]);
-   case MONLOADER_CROSS_PAGE_CODE_IDX:
-      return HostIF_GetCrossPageCodeMPN();
-   default:
-      return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
-   }
+   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
 }


@@ -1,609 +0,0 @@
-/*********************************************************
- * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * crosspage.c --
- *
- *    Cross page code and data.
- *
- *    The cross code page contains world switch code and interrupt/exception
- *    handlers in support of world switch.  A separate data page is also mapped
- *    in both the vmm and vmmon address spaces to describe the host and monitor
- *    state.
- *
- *    Both the monitor and the vmmon driver call into the cross page to switch
- *    worlds.  The world switch from host->vmm is able to refer directly to the
- *    various symbols defined in CrossPage_CodePage.  On the vmm->host path
- *    the VmmToHost entry point is stored in the data page.  Data references
- *    must be handled as offsets from the base of the data page.
- *
- *    Use of inline assembly in C code is delicate as the compiler has many
- *    possible reasons to emit unexpected instructions surrounding any assembly
- *    in a C function.  Frame pointers are inconvenient to disable on a per-file
- *    basis given structure of the Linux Makefile so they are being emitted
- *    in the C wrapper function.  NORETURN and NOT_REACHED_MINIMAL are used to
- *    omit a function epilogue.  The output/clobber lists of the assembly blocks
- *    are not used as the compiler needs not compensate for the register use of
- *    these hand-written assembly functions.
- *    Each assembly function begins with a label denoting the proper entry point
- *    for that function.  Only a handful of the functions need to be exported
- *    out of this file, see crosspage.h.
- */
-
-#ifdef __linux__
-#   include "driver-config.h"
-#   if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
-#      include <linux/objtool.h>
-#   elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
-#      include <linux/frame.h>
-#   endif
-#endif
-
-#include "modulecall.h"
-#include "iocontrols.h"
-#include "segs.h"
-#include "x86_basic_defs.h"
-#include "vm_basic_defs.h"
-#include "vm_idt_x86.h"
-#include "crosspage.h"
-
-#ifdef __APPLE__
-/*
- * The Mac compiler expects symbols to have a _ prefix.
- * This arranges for any symbol being exported out of the assembly code
- * to be accessible.
- */
-#define ASM_PREFIX "_"
-/*
- * OSX uses a segment,section style syntax for specifying output sections.
- * https://developer.apple.com/library/archive/documentation/Performance/
- * Conceptual/CodeFootprint/Articles/MachOOverview.html
- */
-#define ASM_SECTION "__TEXT,cross"
-#else
-/*
- * This is a COFF grouped section identifier.  This ensures the crosspage code
- * is merged with the normal .text area but as a separate unit so it can have
- * its own alignment properties.
- * https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
- *
- * The Linux build will handle the section based on its contents (code).
- */
-#define ASM_SECTION ".text$cross"
-#define ASM_PREFIX
-#endif
-
-#define EXPORTED_ASM_SYMBOL(fn) ".global " ASM_PREFIX #fn "\n"   \
-                                ASM_PREFIX #fn ":\n"
-
-/*
- * Tag the crosspage code C wrapper with the crosspage section and page
- * alignment.
- */
-#define CPCODE __attribute__((section(ASM_SECTION)))   \
-               __attribute__((aligned(PAGE_SIZE)))
-
-/* No special handling is needed for the data. */
-#define CPDATA
-
-#define VMMDATALA(off) (LPN_2_LA(CROSS_PAGE_DATA_START) + (off))
-
-#define NOT_REACHED_MINIMAL __builtin_unreachable
-
-#ifdef ANNOTATE_INTRA_FUNCTION_CALL
-#define ANNOTATE_ASM_CALL_STR2(x) #x
-#define ANNOTATE_ASM_CALL_STR(x) ANNOTATE_ASM_CALL_STR2(x) "\n"
-#define ANNOTATE_ASM_CALL ANNOTATE_ASM_CALL_STR(ANNOTATE_INTRA_FUNCTION_CALL)
-#else
-#define ANNOTATE_ASM_CALL
-#endif
-
-void VmmToHost(void);
-
-CPDATA const VMCrossPageData cpDataTemplate = {
-   .version        = CROSSPAGE_VERSION,
-   .vmmonVersion   = VMMON_VERSION,
-
-   .monRSP         = VMM_LRET_STACK_TOP,
-   .monSS          = SYSTEM_DATA_SELECTOR,
-   .monDS          = SYSTEM_DATA_SELECTOR,
-   .monES          = SYSTEM_DATA_SELECTOR,
-
-   .vmmToHostLA    = (VA)VmmToHost,
-   .wsCR0          = CR0_PE | CR0_MP | CR0_EM | CR0_NE | CR0_WP | CR0_PG,
-   .wsCR4          = CR4_PAE | CR4_OSFXSR,
-
-   .monTask.rsp[0] = VMM_STACK_TOP,                   /* Monitor stack. */
-   .monTask.rsp[1] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL1 is not used. */
-   .monTask.rsp[2] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL2 is not used. */
-
-   .monTask.ist[IST_NONE]                 = 0, /* No stack switch. */
-   .monTask.ist[IST_VMM_DF]               = DF_STACK_TOP,
-   .monTask.ist[IST_VMM_NMI]              = NMI_STACK_TOP,
-   .monTask.ist[IST_VMM_MCE]              = MC_STACK_TOP,
-
-   .monTask.IOMapBase = sizeof(Task64),
-
-   .monGDTR.limit  = GDT_LIMIT,
-   .monGDTR.offset = GDT_START_VA,
-
-   .shadowDR[6].ureg64 = DR6_DEFAULT,
-   .shadowDR[7].ureg64 = DR7_ONES,
-
-   /*
-    * switchHostIDT and switchHostIDTR are initialized in vmmon.
-    * switchMonIDTR is static and can be fully initialized at compile-time.
-    * switchMonIDT is mostly static with all non-present entries except for
-    * the 4 handled exceptions which are populated by the crosspage setup
-    * code in task.c
-    */
-   .switchMonIDTR  = { sizeof(Gate64) * NUM_EXCEPTIONS - 1,
-                       VMMDATALA(offsetof(VMCrossPageData, switchMonIDT)) },
-};
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * CrossPage_CodePage --
- *
- *      This function only serves as an anchor and wrapper for the code that
- *      runs in the world switch's cross page.  The function must not be called
- *      directly.  Its address may be used to locate the cross page code.
- *
- *-----------------------------------------------------------------------------
- */
-
-CPCODE void
-CrossPage_CodePage(void)
-{
-   __asm__ __volatile__ (
-
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * SwitchDBHandler --
-    *
-    *      DB handler that operates during worldswitch (in both directions).
-    *
-    *      This handler is expected to be used in two cases:
-    *      1) If the host kernel has set the DR7_GD (guard) bit, when switch
-    *         code attempts to save debug registers, a #DB will be thrown.  In
-    *         this case, the handler sets wsException[EXC_DB] and returns.
-    *      2) When stress-testing the worldswitch code, RFLAGS.TF is set across
-    *         most switch paths.  This causes a #DB to be thrown before each
-    *         instruction is executed.  In this case, the handler must set
-    *         RFLAGS.RF in the return frame to guarantee progress while
-    *         RFLAGS.TF is set.  For added stress, the handler induces a
-    *         simulated #NMI.
-    *
-    *      In case (1), returning is accomplished via a simulated iret in order
-    *      to preserve any NMI-blocking.  In case (2), the inherent
-    *      incompatibility of RFLAGS.RF for single-stepping and simulating an
-    *      iret via popf and lretq necessitates the use of an actual iret.  As
-    *      a result, there is no strong NMI-blocking guarantee when
-    *      stress-testing the switch path.
-    *
-    * Input:
-    *
-    *       0(%rsp) = previous RIP
-    *       8(%rsp) = previous CS
-    *      16(%rsp) = previous RFLAGS
-    *      24(%rsp) = previous RSP
-    *      32(%rsp) = previous SS
-    *
-    * Output:
-    *
-    *      wsException[EXC_DB] = TRUE
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   EXPORTED_ASM_SYMBOL(SwitchDBHandler)
-   "pushq        %%rax\n"
-   ANNOTATE_ASM_CALL
-   "call         SwitchExcGetCrossPageData\n"
-   "addq         %[wsExceptionDB], %%rax\n"
-   "movb         $1,               (%%rax)\n" /* log EXC_DB */
-   "popq         %%rax\n"
-   "testl        %[EFLAGS_TF],     16(%%rsp)\n" /* check trap flag */
-   "jz           return_without_enabling_nmi\n"
-   "orl          %[EFLAGS_RF],     16(%%rsp)\n"
-   "int          $2\n"
-   "iretq\n"
-
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * SwitchUDHandler --
-    *
-    *      UD handler that operates during worldswitch (in both directions).
-    *
-    *      It sets the wsException[EXC_UD] flag then returns.
-    *
-    * Input:
-    *
-    *       0(%rsp) = previous RIP
-    *       8(%rsp) = previous CS
-    *      16(%rsp) = previous RFLAGS
-    *      24(%rsp) = previous RSP
-    *      32(%rsp) = previous SS
-    *
-    * Output:
-    *
-    *      wsException[EXC_UD] = TRUE
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   EXPORTED_ASM_SYMBOL(SwitchUDHandler)
-   "pushq        %%rax\n"
-   "pushq        %%rbx\n"
-   "pushq        %%rcx\n"
-   ANNOTATE_ASM_CALL
-   "call         SwitchExcGetCrossPageData\n"
-   "movl         %[wsExceptionUD],      %%ecx\n"    /* log EXC_UD */
-   "movb         $1,                    (%%rax, %%rcx)\n"
-   /* Check if the exception came from a monitor RIP. */
-   "cmpq         %[MONITOR_MINIMUM_VA], 24(%%rsp)\n"
-   "jae          monitor_context\n"
-   /* Check if the exception came from the crosspage code. */
-   "movq         24(%%rsp),              %%rbx\n"
-   "andq         %[PageAlignMask],       %%rbx\n"
-   "movl         %[crosspageCodeLA],     %%ecx\n"
-   "cmpq         (%%rax, %%rcx),         %%rbx\n"
-   "je           monitor_context\n"
-   /*
-    * Linux halts the processor on host #UD2.  Act similarly.  PR 1281662.
-    * Note that this can escape due to an #NMI, #MC, or #DB.
-    */
-   "cli\n"
-   "hlt\n"
-   /* Advance 2 bytes to skip past the #UD2 instruction. */
-   "monitor_context:\n"
-   "movq         24(%%rsp), %%rbx\n"
-   "movl         %[wsUD2],  %%ecx\n"
-   "movq         %%rbx,     (%%rax, %%rcx)\n"
-   "addq         $2,        24(%%rsp)\n"
-   "popq         %%rcx\n"
-   "popq         %%rbx\n"
-   "popq         %%rax\n"
-   "jmp          return_without_enabling_nmi\n"
-
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * SwitchNMIHandler --
-    *
-    *      NMI handler that operates during worldswitch (in both
-    *      directions).
-    *
-    *      It 'simply' sets the wsException[EXC_NMI] flag then returns,
-    *      leaving further NMI delivery inhibited.
-    *
-    *      As long as we don't execute 'iret', the CPU will not allow further
-    *      NMIs to be delivered; this is how the CPU protects itself from
-    *      runaway NMIs eating up the stack and triple-faulting.
-    *
-    *      See Vol 3, 5.5.1, Handling multiple NMIs.
-    *
-    * Input:
-    *
-    *       0(%rsp) = previous RIP
-    *       8(%rsp) = previous CS
-    *      16(%rsp) = previous RFLAGS
-    *      24(%rsp) = previous RSP
-    *      32(%rsp) = previous SS
-    *
-    * Output:
-    *
-    *      wsException[EXC_NMI] = TRUE
-    *      further NMI delivery inhibited
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   EXPORTED_ASM_SYMBOL(SwitchNMIHandler)
-   "pushq        %%rax\n"
-   ANNOTATE_ASM_CALL
-   "call         SwitchExcGetCrossPageData\n"
-   "addq         %[wsExceptionNMI], %%rax\n"
-   "movb         $1,                (%%rax)\n" /* log EXC_NMI */
-   "popq         %%rax\n"
-   "jmp          return_without_enabling_nmi\n"
-
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * SwitchMCEHandler --
-    *
-    *      MCE handler that operates during worldswitch (in both
-    *      directions).
-    *
-    *      It simply sets the wsException[EXC_MC] flag then returns.
-    *
-    * Input:
-    *
-    *       0(%rsp) = previous RIP
-    *       8(%rsp) = previous CS
-    *      16(%rsp) = previous RFLAGS
-    *      24(%rsp) = previous RSP
-    *      32(%rsp) = previous SS
-    *
-    * Output:
-    *
-    *      wsException[EXC_MC] = TRUE
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   EXPORTED_ASM_SYMBOL(SwitchMCEHandler)
-   "pushq        %%rax\n"
-   ANNOTATE_ASM_CALL
-   "call         SwitchExcGetCrossPageData\n"
-   "addq         %[wsExceptionMC], %%rax\n"
-   "movb         $1,              (%%rax)\n" /* log EXC_MC */
-   "popq         %%rax\n"
-   "jmp          return_without_enabling_nmi\n"
-
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * SwitchHandlerReturns --
-    *
-    *      Shared code to return without enabling NMIs.  Switch handlers call
-    *      directly to these labels as needed.
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   "return_without_enabling_nmi:\n"
-   "pushq        %%rbp\n"
-   "pushq        %%rax\n"
-   "movq         40(%%rsp), %%rbp\n"
-   "subq         $32, %%rbp\n"
-   "movq         24(%%rsp), %%rax\n"
-   "movq         %%rax, 24(%%rbp)\n"
-   "movq         16(%%rsp), %%rax\n"
-   "movq         %%rax, 16(%%rbp)\n"
-   "movq         32(%%rsp), %%rax\n"
-   "movq         %%rax, 8(%%rbp)\n"
-   "movq         8(%%rsp),  %%rax\n"
-   "movq         %%rax, (%%rbp)\n"
-   "popq         %%rax\n"
-   "movq         %%rbp, %%rsp\n"
-   "popq         %%rbp\n"
-   "popfq\n"
-   "lretq\n"
-
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * HostToVmm --
-    *
-    *      Switch from host to monitor.
-    *
-    * Entered with:
-    *      RCX = crosspage host address
-    *      (RSP) = return to host address
-    *
-    * Must preserve:
-    *      RBX, RSI, RDI, RBP, RSP, R12..R15
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   EXPORTED_ASM_SYMBOL(HostToVmm)
-   /* Create an lret frame on the host stack. */
-   "pushq           (%%rsp)\n"
-   "mov             %%cs,  8(%%rsp)\n"
-   "movq            %%rsp, %%rax\n" /* Temporarily hold host %rsp */
-
-   /* Start from the "empty" host context and save. */
-   "leaq            %c[hostContextEmpty](%%rcx), %%rsp\n"
-   /* Note that only %ss is stored, not %ds/%es. */
-   "movw            %%ss,  %%dx\n"
-   "pushw           %%dx\n"
-   "pushq           %%rax\n" /* host %rsp */
-   "pushq           %%r15\n"
-   "pushq           %%r14\n"
-   "pushq           %%r13\n"
-   "pushq           %%r12\n"
-   "pushq           %%rdi\n"
-   "pushq           %%rsi\n"
-   "pushq           %%rbp\n"
-   "pushq           %%rbx\n"
-   "movq            %%cr3, %%rax\n"
-   "pushq           %%rax\n"
-
-   /* Start from the "full" monitor context and load. */
-   "leaq            %c[monContextFull](%%rcx), %%rsp\n"
-   "popq            %%rsi\n" /* monitor %cr3 */
-   "popq            %%rbx\n"
-   "popq            %%rbp\n"
-   "popq            %%r12\n"
-   "popq            %%r13\n"
-   "popq            %%r14\n"
-   "popq            %%r15\n"
-   "popq            %%rax\n" /* monitor %rsp */
-   "popw            %%dx\n"  /* monitor %ss */
-   /* Load the cross GDT before reloading segments, %cr3 */
-   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
-   /* Load %cr3 last to flush the TLB after all stack ops. */
-   "movq            %%rsi, %%cr3\n"
-   /* Reload the monitor's %ss into %ds/%es. */
-   "movw            %%dx,  %%ds\n"
-   "movw            %%dx,  %%es\n"
-   "movw            %%dx,  %%ss\n"
-   "movq            %%rax, %%rsp\n"
-
-   "lretq\n"
-
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * VmmToHost --
-    *
-    *      Switch from monitor to host.
-    *
-    * Must preserve:
-    *      RBX, RBP, RSP, R12..R15
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   EXPORTED_ASM_SYMBOL(VmmToHost)
-   "movq            %c[VMMCROSSPAGE] + %c[crosspageDataLA], %%rcx\n"
-   /* Create an lret frame on the monitor stack. */
-   "pushq           (%%rsp)\n"
-   "mov             %%cs,  8(%%rsp)\n"
-   "movq            %%rsp, %%rax\n" /* Temporarily hold monitor %rsp */
-
-   /* Start from the "empty" monitor context and save. */
-   "leaq            %c[VMMCROSSPAGE] + %c[monContextEmpty], %%rsp\n"
-   /* Note that only %ss is stored, not %ds/%es. */
-   "movw            %%ss,  %%dx\n"
-   "pushw           %%dx\n"
-   "pushq           %%rax\n" /* monitor %rsp */
-   "pushq           %%r15\n"
-   "pushq           %%r14\n"
-   "pushq           %%r13\n"
-   "pushq           %%r12\n"
-   "pushq           %%rbp\n"
-   "pushq           %%rbx\n"
-   "movq            %%cr3, %%rax\n"
-   "pushq           %%rax\n"
-
-   /* Start from the "full" host context and load. */
-   "leaq            %c[hostContextFull](%%rcx), %%rsp\n"
-   "popq            %%r9\n" /* host %cr3 */
-   "popq            %%rbx\n"
-   "popq            %%rbp\n"
-   "popq            %%rsi\n" /* Not ABI-required. */
-   "popq            %%rdi\n" /* Not ABI-required. */
-   "popq            %%r12\n"
-   "popq            %%r13\n"
-   "popq            %%r14\n"
-   "popq            %%r15\n"
-
-   "popq            %%rax\n" /* host %rsp */
-   "popw            %%dx\n"  /* host %ss */
-   /* Load the cross GDT and IDT before reloading segments, %cr3 */
-   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
-   "lidtq           %c[switchHostIDTR](%%rcx)\n"
-   /* Load %cr3 last to flush the TLB after all stack ops. */
-   "movq            %%r9,  %%cr3\n"
-   /* Reload the host's %ss into %ds/%es.  Technically wrong. */
-   "movw            %%dx,  %%ds\n"
-   "movw            %%dx,  %%es\n"
-   "movw            %%dx,  %%ss\n"
-   "movq            %%rax, %%rsp\n"
-
-   /* Microsoft RTL/codegen assumes EFLAGS<DF> = 0. */
-   "cld\n"
-   "lretq\n"
-
-   /*
-    *---------------------------------------------------------------------------
-    *
-    * SwitchExcGetCrossPageData --
-    *
-    *      Common function for the exception handlers to locate the data
-    *      crosspage so they can record their respective events.  In order to
-    *      reach this code an exception had to vector through the IDT.  The IDT
-    *      is known to be in the data page.  Therefore, the data page can be
-    *      found by accessing IDTR and rounding down to page alignment.
-    *
-    * Input:
-    *     None
-    *
-    * Output:
-    *     %rax = Page aligned address of the current crosspage data area.
-    *
-    * Note:
-    *     %rax (return value) and %rflags are destroyed, all other registers
-    *     are preserved.  Since this is only called by exception handlers, the
-    *     CPU has already saved %rflags so no additional handling is required.
-    *
-    *---------------------------------------------------------------------------
-    */
-
-   ".p2align 4\n"
-   "SwitchExcGetCrossPageData:\n"
-   "subq            $0x10,            %%rsp\n"
-   "sidt            0(%%rsp)\n"
-   "movq            2(%%rsp),         %%rax\n" /* DTR.offset */
-   "addq            $0x10,            %%rsp\n"
-   "andq            %[PageAlignMask], %%rax\n"
-   "ret\n"
-
-   EXPORTED_ASM_SYMBOL(CrossPage_CodeEnd)
-
-   : /* No output list, this is not really C code. */
-   : [MONITOR_MINIMUM_VA] "i" (MONITOR_LINEAR_START),
-     [PageAlignMask]      "i" (~PAGE_MASK),
-     [EFLAGS_TF]          "i" (EFLAGS_TF),
-     [EFLAGS_RF]          "i" (EFLAGS_RF),
-     [VMMCROSSPAGE]       "i" (CROSS_PAGE_DATA_START * PAGE_SIZE),
-     [wsUD2]              "i" (offsetof(VMCrossPageData, wsUD2)),
-
-     [wsExceptionDB]      "i" (offsetof(VMCrossPageData, wsException[EXC_DB])),
-     [wsExceptionUD]      "i" (offsetof(VMCrossPageData, wsException[EXC_UD])),
-     [wsExceptionNMI]     "i" (offsetof(VMCrossPageData, wsException[EXC_NMI])),
-     [wsExceptionMC]      "i" (offsetof(VMCrossPageData, wsException[EXC_MC])),
-
-     [hostContextEmpty]   "i" (offsetof(VMCrossPageData, hostDS)),
-     [monContextEmpty]    "i" (offsetof(VMCrossPageData, monDS)),
-
-     [hostContextFull]    "i" (offsetof(VMCrossPageData, hostCR3)),
-     [monContextFull]     "i" (offsetof(VMCrossPageData, monCR3)),
-
-     [crosspageDataLA]    "i" (offsetof(VMCrossPageData, crosspageDataLA)),
-     [crosspageCodeLA]    "i" (offsetof(VMCrossPageData, crosspageCodeLA)),
-     [crossGDTHKLADesc]   "i" (offsetof(VMCrossPageData, crossGDTHKLADesc)),
-     [switchHostIDTR]     "i" (offsetof(VMCrossPageData, switchHostIDTR))
-   );
-   NOT_REACHED_MINIMAL();
-}
-
-#ifdef STACK_FRAME_NON_STANDARD
-STACK_FRAME_NON_STANDARD(CrossPage_CodePage);
-#endif
-
-/*
- * The crosspage data must not exceed a single page.
- * The code area also must fit in a single page but that size isn't available
- * until link time.
- */
-MY_ASSERTS(CROSSPAGE_DATA,
-   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
-)
@@ -1,43 +0,0 @@
-/*********************************************************
- * Copyright (C) 2020 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _CROSSPAGE_H_
-#define _CROSSPAGE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct VMCrossPageData;
-
-extern void HostToVmm(struct VMCrossPageData *crosspageWIN, void *unused1,
-                      void *unused2, struct VMCrossPageData *crosspageSYSV);
-extern const VMCrossPageData cpDataTemplate;
-
-extern void SwitchDBHandler(void);
-extern void SwitchUDHandler(void);
-extern void SwitchNMIHandler(void);
-extern void SwitchMCEHandler(void);
-extern void CrossPage_CodePage(void);
-extern void CrossPage_CodeEnd(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
@@ -68,9 +68,7 @@ typedef struct HostIFContigMemMap {

 EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
-
-EXTERN MPN   HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData);
-EXTERN MPN   HostIF_GetCrossPageCodeMPN(void);
+EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);

 EXTERN void *HostIF_AllocPage(void);
 EXTERN void  HostIF_FreePage(void *ptr);
@@ -103,14 +101,13 @@ EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 EXTERN void  HostIF_APICInit(VMDriver *vm);
 EXTERN uint8 HostIF_GetMonitorIPIVector(void);
 EXTERN uint8 HostIF_GetHVIPIVector(void);
-EXTERN uint8 HostIF_GetPerfCtrVector(void);
 EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);

 EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
                                  Vcpuid vcpuid,
                                  uint64 *args);

-EXTERN int   HostIF_SemaphoreSignal(VMDriver *vm, uint64 *args);
+EXTERN int   HostIF_SemaphoreSignal(uint64 *args);

 EXTERN void  HostIF_SemaphoreForceWakeup(VMDriver *vm, const VCPUSet *vcs);
 EXTERN void  HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,8 +66,6 @@
 #include "x86paging_64.h"
 #include "memtrack.h"
 #include "monLoader.h"
-#include "segs.h"
-#include "crosspage.h"

 #ifdef LINUX_GDT_IS_RO
 #   include <asm/desc.h>
@@ -590,13 +588,7 @@ Bool
 Task_Initialize(void)
 {
   unsigned i;
-   VA cpStart = (VA)CrossPage_CodePage;
-   VA cpEnd   = (VA)CrossPage_CodeEnd;

-   if (VA_2_VPN(cpStart) != VA_2_VPN(cpEnd)) {
-      Warning("Crosspage code validation failed.\n");
-      return FALSE;
-   }
   ASSERT_ON_COMPILE(sizeof(Atomic_uint64) == sizeof(MPN));
   for (i = 0; i < ARRAYSIZE(hvRootPage); i++) {
      Atomic_Write64(&hvRootPage[i], INVALID_MPN);
@@ -836,7 +828,7 @@ TaskSwitchPTPLookupVPN(VMDriver *vm, VPN vpn)
 *    before/after doing a BackToHost.
 *
 *    For each populated patch, performs a page walk from L4 to the patch's
- *    level.  The lowest allowed level is L2.  Invalid patch levels, collisions
+ *    level.  The lowest allowed level is L3.  Invalid patch levels, collisions
 *    at the patch's level with existing PTEs and non-present PTEs during the
 *    walk all result in failure.
 *
@@ -855,7 +847,6 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
      VMMPageTablePatch *patch = &cpData->vmmPTP[i];
      unsigned l4idx = PT_LPN_2_L4OFF(patch->lpn);
      unsigned l3idx = PT_LPN_2_L3OFF(patch->lpn);
-      unsigned l2idx = PT_LPN_2_L2OFF(patch->lpn);
      PT_L4E pte = 0;
      MA ma = cpData->monCR3 + l4idx * sizeof pte; /* PTE machine address. */

@@ -895,34 +886,6 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
               return FALSE;
            }
            break;
-         case 2:
-            ASSERT(patch->pteIdx == l2idx);
-            if (!PTE_PRESENT(pte)) {
-               return FALSE; /* Terminate page walk, L4 not present. */
-            }
-            ma = MPN_2_MA(LM_PTE_2_PFN(pte)) + l3idx * sizeof pte;
-            if (HostIF_ReadPhysical(vm, ma, PtrToVA64(&pte), TRUE,
-                                    sizeof pte) != 0) {
-               return FALSE;
-            }
-            if (!PTE_PRESENT(pte)) {
-               return FALSE; /* Terminate page walk, L3 not present. */
-            }
-            ma = MPN_2_MA(LM_PTE_2_PFN(pte)) + l2idx * sizeof pte;
-            if (HostIF_ReadPhysical(vm, ma, PtrToVA64(&pte), TRUE,
-                                    sizeof pte) != 0) {
-               return FALSE;
-            }
-            if (pte != 0) {
-               return FALSE; /* Collision. */
-            }
-
-            if (HostIF_WritePhysical(vm, ma, PtrToVA64(&patch->pte), TRUE,
-                                     sizeof patch->pte) != 0) {
-               return FALSE;
-            }
-            break;
-
         default:
            return FALSE; /* Invalid level. */
            break;
@@ -982,14 +945,11 @@ TaskVerifyPTPatches(VMDriver *vm,
                    VMCrossPageData *cpData)
 {
   LPN64 xgCPN = LA_2_LPN(cpData->crossGDTLA);
-   LPN64 xpCPN = LA_2_LPN(cpData->crosspageDataLA);
-   MPN   xpMPN = MA_2_MPN(cpData->crosspageDataMA);
+   LPN64 xpCPN = LA_2_LPN(cpData->crossPageLA);
+   MPN   xpMPN = MA_2_MPN(cpData->crosspageMA);
   MA    cr3   = cpData->monCR3;
-   LPN64 xpCodeCPN = LA_2_LPN(cpData->crosspageCodeLA);
-   MPN   xpCodeMPN = HostIF_GetCrossPageCodeMPN();
   return TaskVerifyPTMap(vm, cr3, xpCPN, xpMPN) &&
-          TaskVerifyPTMap(vm, cr3, xgCPN, crossGDTMPN) &&
-          TaskVerifyPTMap(vm, cr3, xpCodeCPN, xpCodeMPN);
+          TaskVerifyPTMap(vm, cr3, xgCPN, crossGDTMPN);
 }


@@ -1212,10 +1172,11 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
 /*
 *-----------------------------------------------------------------------------
 *
- * TaskPopulateSwitchIDTE --
+ * TaskPopulateHostSwitchIDTE --
 *
- *      Populates an interrupt descriptor in the crosspage, setting its
- *      handler to the associated gate stub given a code segment selector.
+ *      Populates a host interrupt descriptor in the crosspage, setting its
+ *      handler to the associated gate stub, given a host code segment selector
+ *      and the crosspage.
 *
 * Results:
 *      None
@@ -1227,13 +1188,17 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
 */

 static void
-TaskPopulateSwitchIDTE(unsigned idx, Selector cs, Gate64 *idt, VA handlerVA)
+TaskPopulateHostSwitchIDTE(unsigned idx, Selector cs, VMCrossPage *cp)
 {
   Gate64 *entry;
+   VA64 handlerVA;

   ASSERT_ON_COMPILE(sizeof *entry == 16);
   ASSERT(idx < NUM_EXCEPTIONS);
-   entry = &idt[idx];
+
+   handlerVA = PtrToVA64(&cp->crosspageCode.gateStubs[idx * CP_STUB_SIZE]);
+
+   entry = &cp->crosspageData.switchHostIDT[idx];

   entry->offset_0_15  = LOWORD(handlerVA);
   entry->offset_16_31 = HIWORD(handlerVA);
@@ -1247,21 +1212,13 @@ TaskPopulateSwitchIDTE(unsigned idx, Selector cs, Gate64 *idt, VA handlerVA)
   entry->reserved1    = 0;
 }

-static VA
-TaskVmmCodeVA(void (*handler)(void))
-{
-   VA baseVA = VPN_2_VA(CROSS_PAGE_CODE_START);
-   VA offs = (VA)handler % PAGE_SIZE;
-
-   return baseVA + offs;
-}

 /*
 *-----------------------------------------------------------------------------
 *
- *  TaskInitSwitchIDT --
+ *  TaskInitHostSwitchIDT --
 *
- *      Initializes the contents of the switch IDTs and IDTR.
+ *      Initializes the contents of the host-context switch IDT and IDTR.
 *      Uses the cross page's host kernel linear address and host kernel CS.
 *
 * Results:
@@ -1274,28 +1231,18 @@ TaskVmmCodeVA(void (*handler)(void))
 */

 static void
-TaskInitSwitchIDTs(VMCrossPageData *cpData)
+TaskInitHostSwitchIDT(VMCrossPage *crosspage)
 {
-   const Selector hostCS = cpData->hostInitial64CS;
-   Gate64 *switchHostIDT = cpData->switchHostIDT;
-   Gate64 *switchMonIDT  = cpData->switchMonIDT;
+   VMCrossPageData *cpData = &crosspage->crosspageData;
+   const Selector   cs     = cpData->hostInitial64CS;

   cpData->switchHostIDTR.limit = sizeof cpData->switchHostIDT - 1;
   cpData->switchHostIDTR.offset = PtrToVA64(&cpData->switchHostIDT);

-   TaskPopulateSwitchIDTE(EXC_DB,  hostCS, switchHostIDT, (VA)SwitchDBHandler);
-   TaskPopulateSwitchIDTE(EXC_NMI, hostCS, switchHostIDT, (VA)SwitchNMIHandler);
-   TaskPopulateSwitchIDTE(EXC_UD,  hostCS, switchHostIDT, (VA)SwitchUDHandler);
-   TaskPopulateSwitchIDTE(EXC_MC,  hostCS, switchHostIDT, (VA)SwitchMCEHandler);
-
-   TaskPopulateSwitchIDTE(EXC_DB,  SYSTEM_CODE_SELECTOR, switchMonIDT,
-                          TaskVmmCodeVA(SwitchDBHandler));
-   TaskPopulateSwitchIDTE(EXC_NMI, SYSTEM_CODE_SELECTOR, switchMonIDT,
-                          TaskVmmCodeVA(SwitchNMIHandler));
-   TaskPopulateSwitchIDTE(EXC_UD,  SYSTEM_CODE_SELECTOR, switchMonIDT,
-                          TaskVmmCodeVA(SwitchUDHandler));
-   TaskPopulateSwitchIDTE(EXC_MC,  SYSTEM_CODE_SELECTOR, switchMonIDT,
-                          TaskVmmCodeVA(SwitchMCEHandler));
+   TaskPopulateHostSwitchIDTE(EXC_DB,  cs, crosspage);
+   TaskPopulateHostSwitchIDTE(EXC_NMI, cs, crosspage);
+   TaskPopulateHostSwitchIDTE(EXC_UD,  cs, crosspage);
+   TaskPopulateHostSwitchIDTE(EXC_MC,  cs, crosspage);
 }


@@ -1607,7 +1554,7 @@ TaskCreatePTPatch(VMDriver        *vm,
 /*
 *-----------------------------------------------------------------------------
 *
- * TaskCreatePTPatches --
+ * TaskCreatePTPatches
 *
 *      Creates VMM mappings for the crosspage and crossGDT that match
 *      those from the vmmon address space, so that they are mapped at
@@ -1622,19 +1569,16 @@ TaskCreatePTPatch(VMDriver        *vm,
 */

 static Bool
-TaskCreatePTPatches(VMDriver        *vm,
-                    VMCrossPageData *cpData,
-                    LPN64            monStartLPN,
-                    LPN64            monEndLPN,
-                    uint16          *numPages)
+TaskCreatePTPatches(VMDriver    *vm,
+                    VMCrossPage *crosspage,
+                    LPN64        monStartLPN,
+                    LPN64        monEndLPN,
+                    uint16      *numPages)
 {
-   LPN64                   xgCPN     = LA_2_LPN(cpData->crossGDTLA);
-   LPN64                   xpCPN     = LA_2_LPN(cpData->crosspageDataLA);
-   MPN                     xpMPN     = MA_2_MPN(cpData->crosspageDataMA);
-   LPN64                   xpCodeCPN = LA_2_LPN(cpData->crosspageCodeLA);
-   MPN                     xpCodeMPN = HostIF_GetCrossPageCodeMPN();
-
-   ASSERT(xpCodeMPN != INVALID_MPN);
+   VMCrossPageData * const cpData = &crosspage->crosspageData;
+   LPN64                   xgCPN  = LA_2_LPN(cpData->crossGDTLA);
+   LPN64                   xpCPN  = LA_2_LPN(cpData->crossPageLA);
+   MPN                     xpMPN  = MA_2_MPN(cpData->crosspageMA);
   /*
    * Set up patches that the BackToHost code will use to map the
    * crosspage and crossGDT at their crossover addresses (HKLA for
@@ -1643,9 +1587,7 @@ TaskCreatePTPatches(VMDriver        *vm,
   if (TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
                         xpCPN, xpMPN, numPages) &&
       TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
-                         xgCPN, crossGDTMPN, numPages) &&
-       TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
-                         xpCodeCPN, xpCodeMPN, numPages)) {
+                         xgCPN, crossGDTMPN, numPages)) {
      TaskFixupPTPatches(vm, cpData);
      return TRUE;
   }
@@ -1670,9 +1612,9 @@ TaskCreatePTPatches(VMDriver        *vm,
 */

 Bool
-Task_InitCrosspage(VMDriver     *vm,           // IN
-                   LPN           monStartLPN,  // IN
-                   LPN           monEndLPN,    // IN
+Task_InitCrosspage(VMDriver *vm,               // IN
+                   LPN monStartLPN,            // IN
+                   LPN monEndLPN,              // IN
                   PerVcpuPages *perVcpuPages) // IN
 {
   Vcpuid vcpuid;
@@ -1684,19 +1626,31 @@ Task_InitCrosspage(VMDriver     *vm,           // IN

   ASSERT(0 < vm->numVCPUs && vm->numVCPUs <= MAX_VCPUS);
   for (vcpuid = 0; vcpuid < vm->numVCPUs; vcpuid++) {
-      VMCrossPageData *cpData = vm->crosspage[vcpuid];
-      MPN              crossPageDataMPN;
+      VA64             crossPageUserAddr = perVcpuPages[vcpuid].crosspage;
+      VMCrossPage     *p = HostIF_MapCrossPage(vm, crossPageUserAddr);
+      VMCrossPageData *cpData;
+      MPN              crossPageMPN;

-      crossPageDataMPN = HostIF_GetCrossPageDataMPN(cpData);
-      ASSERT(crossPageDataMPN != INVALID_MPN);
-      memcpy(cpData, &cpDataTemplate, sizeof *cpData);
+      if (p == NULL) {
+         return FALSE;
+      }
+      cpData = &p->crosspageData;
+
+      HostIF_VMLock(vm, 38);
+      if (HostIF_LookupUserMPN(vm, crossPageUserAddr, &crossPageMPN) !=
+          PAGE_LOOKUP_SUCCESS ||
+          crossPageMPN == 0) {
+         HostIF_VMUnlock(vm, 38);
+         return FALSE;
+      }
+      HostIF_VMUnlock(vm, 38);

      /*
       * The version of the crosspage must be the first four bytes of the
       * crosspage.  See the declaration of VMCrossPage in modulecall.h.
       */

-      ASSERT_ON_COMPILE(offsetof(VMCrossPageData, version) == 0);
+      ASSERT_ON_COMPILE(offsetof(VMCrossPage, crosspageData.version) == 0);
      ASSERT_ON_COMPILE(sizeof(cpData->version) == sizeof(uint32));

      /* cpData->version is VMX's version; CROSSPAGE_VERSION is vmmon's. */
@@ -1706,9 +1660,12 @@ Task_InitCrosspage(VMDriver     *vm,           // IN
                 (int)CROSSPAGE_VERSION, cpData->version);
         return FALSE;
      }
-      cpData->crosspageDataMA = MPN_2_MA(crossPageDataMPN);
-      cpData->crosspageDataLA = (LA64)(uintptr_t)cpData;
-      cpData->crosspageCodeLA = (LA64)CrossPage_CodePage;
+      if (!pseudoTSC.initialized) {
+         Warning("%s*: PseudoTSC has not been initialized\n", __FUNCTION__);
+         return FALSE;
+      }
+      cpData->crosspageMA = MPN_2_MA(crossPageMPN);
+      cpData->crossPageLA = (LA64)(uintptr_t)p;
      cpData->crossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.offset = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.limit  = sizeof(CrossGDT) - 1;
@@ -1718,8 +1675,7 @@ Task_InitCrosspage(VMDriver     *vm,           // IN
      cpData->monCR3 = MPN_2_MA(vm->ptRootMpns[vcpuid]);

      HostIF_VMLock(vm, 39);
-      if (!TaskCreatePTPatches(vm, cpData, monStartLPN, monEndLPN,
-                               &numPTPPages)) {
+      if (!TaskCreatePTPatches(vm, p, monStartLPN, monEndLPN, &numPTPPages)) {
         HostIF_VMUnlock(vm, 39);
         Warning("%s: Could not create page table patches for VCPU %d\n",
                 __FUNCTION__, vcpuid);
@@ -1757,8 +1713,9 @@ Task_InitCrosspage(VMDriver     *vm,           // IN
      cpData->timerIntrTS           = MAX_ABSOLUTE_TS;
      cpData->hstTimerExpiry        = MAX_ABSOLUTE_TS;
      cpData->monTimerExpiry        = MAX_ABSOLUTE_TS;
+      vm->crosspage[vcpuid]         = p;

-      TaskInitSwitchIDTs(cpData);
+      TaskInitHostSwitchIDT(p);
   }
   /*
    * Store the number of pages allocated for this VM's page table patches so
@@ -1913,16 +1870,16 @@ TaskDisableTF(void)


 static INLINE Bool
-TaskGotException(const VMCrossPageData *crosspageData, unsigned exc)
+TaskGotException(const VMCrossPage *crosspage, unsigned exc)
 {
-   return crosspageData->wsException[exc];
+   return crosspage->crosspageData.wsException[exc];
 }


 static INLINE void
-TaskSetException(VMCrossPageData *crosspageData, unsigned exc, Bool v)
+TaskSetException(VMCrossPage *crosspage, unsigned exc, Bool v)
 {
-   crosspageData->wsException[exc] = v;
+   crosspage->crosspageData.wsException[exc] = v;
 }


@@ -1948,7 +1905,7 @@ TaskSetException(VMCrossPageData *crosspageData, unsigned exc, Bool v)
 */

 static INLINE_SINGLE_CALLER void
-TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
+TaskSaveDebugRegisters(VMCrossPage *crosspage)
 {
   Bool saveGotDB;

@@ -1956,13 +1913,13 @@ TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
   do {                                                 \
      uintptr_t drReg;                                  \
      GET_DR##n(drReg);                                 \
-      crosspageData->hostDR[n] = drReg;                 \
+      crosspage->crosspageData.hostDR[n] = drReg;       \
   } while (0)

   /* Hardware contains the host's %dr7, %dr6, %dr3, %dr2, %dr1, %dr0 */
-   crosspageData->hostDRInHW = ((1 << 7) | (1 << 6) |
-                                (1 << 3) | (1 << 2) |
-                                (1 << 1) | (1 << 0));
+   crosspage->crosspageData.hostDRInHW = ((1 << 7) | (1 << 6) |
+                                          (1 << 3) | (1 << 2) |
+                                          (1 << 1) | (1 << 0));

   /*
    * Save DR7 since we need to disable debug breakpoints during the world
@@ -1970,8 +1927,8 @@ TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
    * SwitchDBHandler simply IRETs after setting crosspage gotDB flag.
    */

-   saveGotDB = TaskGotException(crosspageData, EXC_DB);
-   TaskSetException(crosspageData, EXC_DB, FALSE);
+   saveGotDB = TaskGotException(crosspage, EXC_DB);
+   TaskSetException(crosspage, EXC_DB, FALSE);
   COMPILER_MEM_BARRIER();      /* Prevent hoisting #UD-raising instructions. */
   SAVE_DR(7);

@@ -1979,7 +1936,7 @@ TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
    * In all cases, DR7 shouldn't have the GD bit set.
    */

-   TS_ASSERT(!(crosspageData->hostDR[7] & DR7_GD));
+   TS_ASSERT(!(crosspage->crosspageData.hostDR[7] & DR7_GD));

   /*
    * Save DR6 in order to accommodate the ICEBP instruction and other stuff
@@ -2004,15 +1961,15 @@ TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
    * SAVE_DR(6) can raise #DB.
    */

-   if (TaskGotException(crosspageData, EXC_DB) &&
-       (crosspageData->hostDR[6] & DR6_BD)) {
-      crosspageData->hostDR[6] -= DR6_BD;
-      crosspageData->hostDR[7] |= DR7_GD;
+   if (TaskGotException(crosspage, EXC_DB) &&
+       (crosspage->crosspageData.hostDR[6] & DR6_BD)) {
+      crosspage->crosspageData.hostDR[6] -= DR6_BD;
+      crosspage->crosspageData.hostDR[7] |= DR7_GD;
      SET_DR7(DR7_DEFAULT);

      /* HW: %dr7 and %dr6 are the guest, %dr3, %dr2, %dr1, %dr0 are host */
-      crosspageData->hostDRInHW = ((1 << 3) | (1 << 2) |
-                                   (1 << 1) | (1 << 0));
+      crosspage->crosspageData.hostDRInHW = ((1 << 3) | (1 << 2) |
+                                             (1 << 1) | (1 << 0));
   }

   /*
@@ -2020,20 +1977,20 @@ TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
    * coincidentally trip during the switch.
    */

-   else if (crosspageData->hostDR[7] & DR7_ENABLED) {
+   else if (crosspage->crosspageData.hostDR[7] & DR7_ENABLED) {
      SET_DR7(DR7_DEFAULT);          // no #DB here, just simple set
      /* HW: %dr7 = guest, %dr6, %dr3, %dr2, %dr1, %dr0 = host */
-      crosspageData->hostDRInHW = ((1 << 6) | (1 << 3) | (1 << 2) |
-                                   (1 << 1) | (1 << 0));
+      crosspage->crosspageData.hostDRInHW = ((1 << 6) | (1 << 3) | (1 << 2) |
+                                             (1 << 1) | (1 << 0));
   }

-   TaskSetException(crosspageData, EXC_DB, saveGotDB);
+   TaskSetException(crosspage, EXC_DB, saveGotDB);

   /*
    * hostDR[6,7] have host contents in them now.
    */

-   crosspageData->hostDRSaved = 0xC0;
+   crosspage->crosspageData.hostDRSaved = 0xC0;
 #undef SAVE_DR
 }

@@ -2302,10 +2259,14 @@ TaskUpdatePTSCParameters(VMDriver *vm,
 *
 *      Wrapper that calls code to switch from the host to the monitor.
 *
- *      The basic idea is to do a HostToVmm(crosspage)
+ *      The basic idea is to do a (*(crosspage->hostToVmm))(crosspage)
 *      but it's complicated because we must have a common call format
 *      between GCC and MSC.
 *
+ *      Since we have complete control over what GCC does with asm volatile,
+ *      this amounts to having GCC do exactly what MSC does.
+ *      For 64-bit hosts, we pass the parameter in RCX.
+ *
 *      For 64-bit GCC, the callee is expected to preserve
 *      RBX,RBP,RSP,R12..R15, whereas MSC expects the callee to preserve
 *      RBX,RSI,RDI,RBP,RSP,R12..R15.  So for simplicity, we have the
@@ -2332,24 +2293,59 @@ TaskUpdatePTSCParameters(VMDriver *vm,
 */

 static INLINE_SINGLE_CALLER void
-TaskSwitchToMonitor(VMCrossPageData *crosspageData)
+TaskSwitchToMonitor(VMCrossPage *crosspage)
 {
+   const void *codePtr = ((void *)&crosspage->crosspageCode.toVmmFunc);
+
+#if defined(__GNUC__)
   /*
-    * Arrange for RCX to contain the crosspage pointer using the MS ABI (first
-    * argument) and with the SYSV ABI (4th argument).
+    * Pass the crosspage pointer in RCX just like 64-bit MSC does.
+    * Tell GCC that the worldswitch preserves RBX,RSI,RDI,RBP,RSP,
+    * R12..R15 just like the MSC 64-bit calling convention.
    */
-   HostToVmm(crosspageData, NULL, NULL, crosspageData);
+
+   {
+      uint64 raxGetsWiped, rcxGetsWiped;
+
+#ifdef CALL_NOSPEC
+      __asm__ __volatile__(CALL_NOSPEC
+                           : "=a" (raxGetsWiped),
+                             "=c" (rcxGetsWiped)
+                           : "0" (codePtr),
+                             "1" (crosspage),
+                             THUNK_TARGET(codePtr)
+                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
+#else
+      __asm__ __volatile__("call *%%rax"
+                           : "=a" (raxGetsWiped),
+                             "=c" (rcxGetsWiped)
+                           : "0" (codePtr),
+                             "1" (crosspage)
+                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
+#endif
+
+   }
+#elif defined(_MSC_VER)
+   /*
+    * The 64-bit calling convention is to pass the argument in RCX and that
+    * the called function must preserve RBX,RSI,RDI,RBP,RSP,R12..R15.
+    */
+#pragma warning(suppress: 4055) // Cast of data pointer to function pointer.
+   (*(void (*)(VMCrossPage *))codePtr)(crosspage);
+#else
+#error No compiler defined for TaskSwitchToMonitor
+#endif
 }


 static void
-TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
+TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
 {
   static Bool testSwitchNMI = TRUE; /* test only first time through */

   /*
    * Test the DB,NMI,MCE handlers to make sure they can set the
-    * flags.  This is calling the handlers in crosspage.c.
+    * flags.  This is calling the handlers in switchNMI.S.
    */

   if (vmx86_debug && testSwitchNMI) {
@@ -2358,7 +2354,8 @@ TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
      testSwitchNMI = FALSE;

      /*
-       * RAISE_INTERRUPT calls SwitchDBHandler in crosspage.c.
+       * RAISE_INTERRUPT calls Switch{32,64}DBHandler in switchNMI.S
+       * (depending on host bitsize).
       */

      gotSave = TaskGotException(crosspage, EXC_DB);
@@ -2368,7 +2365,8 @@ TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
      TaskSetException(crosspage, EXC_DB, gotSave);

      /*
-       * RAISE_INTERRUPT calls SwitchNMIHandler in crosspage.c.
+       * RAISE_INTERRUPT calls Switch{32,64}NMIHandler in switchNMI.S
+       * (depending on host bitsize).
       */
      gotSave = TaskGotException(crosspage, EXC_NMI);
      TaskSetException(crosspage, EXC_NMI, FALSE);
@@ -2379,7 +2377,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
      /*
       * Test the LRETQ in the 64-bit mini NMI handler to make sure
       * it works with any 16-byte offset of the stack pointer.
-       * The INT 2 calls SwitchNMIHandler in crosspage.c.
+       * The INT 2 calls Switch64NMIHandler in switchNMI.S.
       */
      {
         uint64 v1, v2;
@@ -2408,7 +2406,8 @@ TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
      TaskSetException(crosspage, EXC_NMI, gotSave);

      /*
-       * RAISE_INTERRUPT calls SwitchMCEHandler in crosspage.c.
+       * RAISE_INTERRUPT calls Switch{32,64}MCEHandler in switchNMI.S
+       * (depending on host bitsize).
       */

      gotSave = TaskGotException(crosspage, EXC_MC);
@@ -2437,10 +2436,10 @@ TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
 */

 static INLINE Bool
-TaskShouldRetryWorldSwitch(VMCrossPageData *crosspage)
+TaskShouldRetryWorldSwitch(VMCrossPage *crosspage)
 {
-   Bool result = crosspage->retryWorldSwitch;
-   crosspage->retryWorldSwitch = FALSE;
+   Bool result = crosspage->crosspageData.retryWorldSwitch;
+   crosspage->crosspageData.retryWorldSwitch = FALSE;
   return result;
 }

@@ -2459,13 +2458,16 @@ TaskShouldRetryWorldSwitch(VMCrossPageData *crosspage)
 *      state.
 *
 * Results:
- *      TRUE - If a #UD did not occur during the context switch.
- *      FALSE - Otherwise.
+ *      None.
+ *
+ * Side effects:
+ *      Jump to the monitor. Has no direct effect on the host-visible
+ *      state except that it might generate an interrupt.
 *
 *-----------------------------------------------------------------------------
 */

-Bool
+void
 Task_Switch(VMDriver *vm,  // IN
            Vcpuid vcpuid) // IN
 {
@@ -2483,11 +2485,10 @@ Task_Switch(VMDriver *vm,  // IN
   Bool lint1NMI;
   Bool pcNMI;
   Bool thermalNMI;
-   VMCrossPageData *crosspage;
+   VMCrossPage *crosspage;
   uint32 pCPU;
   MPN hvRootMPN;
   Descriptor *tempGDTBase;
-   Bool switchOk = TRUE;
 #ifdef CYCLE_SPEC_CTRL
   Bool specCtrlEqual = FALSE;
   uint64 readSpecCtrlValue = 0;
@@ -2496,9 +2497,8 @@ Task_Switch(VMDriver *vm,  // IN

   ASSERT(vcpuid < vm->numVCPUs);
   crosspage = vm->crosspage[vcpuid];
-   ASSERT(crosspage != NULL);

-   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
+   ASSERT_ON_COMPILE(sizeof(VMCrossPage) == PAGE_SIZE);
   TaskDisableNMI(&vm->hostAPIC, &lint0NMI, &lint1NMI, &pcNMI, &thermalNMI);
   SAVE_FLAGS(flags);
   CLEAR_INTERRUPTS();
@@ -2517,13 +2517,13 @@ Task_Switch(VMDriver *vm,  // IN
    * crosspages arguments when doing this though, see bug 820257.
    */
   if (hvRootMPN == INVALID_MPN && CPUID_HostSupportsHV()) {
-      crosspage->userCallType = MODULECALL_USERCALL_NONE;
-      crosspage->moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
-      crosspage->pcpuNum = pCPU;
+      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
+      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
+      crosspage->crosspageData.pcpuNum = pCPU;
   } else if (USE_TEMPORARY_GDT && tempGDTBase == NULL) {
-      crosspage->userCallType = MODULECALL_USERCALL_NONE;
-      crosspage->moduleCallType = MODULECALL_ALLOC_TMP_GDT;
-      crosspage->pcpuNum = pCPU;
+      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
+      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_TMP_GDT;
+      crosspage->crosspageData.pcpuNum = pCPU;
   } else {
      do {
         uintptr_t cr0reg, cr2reg, cr3reg, cr4reg;
@@ -2534,7 +2534,7 @@ Task_Switch(VMDriver *vm,  // IN

         Atomic_Write32(&vm->currentHostCpu[vcpuid], pCPU);

-         TaskUpdatePTSCParameters(vm, crosspage, vcpuid);
+         TaskUpdatePTSCParameters(vm, &crosspage->crosspageData, vcpuid);

         /*
          * Disable PEBS if it is supported and enabled.  Do this while on the
@@ -2565,7 +2565,7 @@ Task_Switch(VMDriver *vm,  // IN
          */

         TaskSaveIDT(&hostIDT);
-         TaskLoadIDT(&crosspage->switchHostIDTR);
+         TaskLoadIDT(&crosspage->crosspageData.switchHostIDTR);
         TaskTestCrossPageExceptionHandlers(crosspage);

         if (CPUID_HostSupportsVT()) {
@@ -2591,21 +2591,21 @@ Task_Switch(VMDriver *vm,  // IN
         GET_CR0(cr0reg);
         GET_CR4(cr4reg);
         GET_CR3(cr3reg);
-         crosspage->hostCR3 = cr3reg;
+         crosspage->crosspageData.hostCR3 = cr3reg;

         /*
          * Any reserved bits in CR0 must be preserved when we switch
          * to the VMM. [See PR 291004.]  (On the other hand, Intel
          * recommends that we clear any reserved CR4 bits.)
          */
-         crosspage->wsCR0 &= ~CR0_RESERVED;
-         crosspage->wsCR0 |= (cr0reg & CR0_RESERVED);
+         crosspage->crosspageData.wsCR0 &= ~CR0_RESERVED;
+         crosspage->crosspageData.wsCR0 |= (cr0reg & CR0_RESERVED);

         /*
          * CR4.VMXE must be enabled to support VMX in the monitor.
          */
         if (CPUID_HostSupportsVT()) {
-            crosspage->wsCR4 |= CR4_VMXE;
+            crosspage->crosspageData.wsCR4 |= CR4_VMXE;
         }

         /*
@@ -2613,8 +2613,8 @@ Task_Switch(VMDriver *vm,  // IN
          * host's values.  CR4.PCIDE will be cleared once we're in the monitor,
          * running on a CR3 with a PCID field of 0.
          */
-         crosspage->wsCR4 =
-            (crosspage->wsCR4 & ~(CR4_MCE | CR4_PCIDE)) |
+         crosspage->crosspageData.wsCR4 =
+            (crosspage->crosspageData.wsCR4 & ~(CR4_MCE | CR4_PCIDE)) |
            (cr4reg & (CR4_MCE | CR4_PCIDE));

         /*
@@ -2622,15 +2622,15 @@ Task_Switch(VMDriver *vm,  // IN
          * switching to the monitor's CR4 ensures that global pages are
          * flushed.
          */
-         ASSERT((crosspage->wsCR4 & CR4_PGE) == 0);
+         ASSERT((crosspage->crosspageData.wsCR4 & CR4_PGE) == 0);

         /*
          * Load the world-switch CR0 and CR4.  We can't load the monitor's
          * CR3 yet, because the current code isn't mapped into the
          * monitor's address space.
          */
-         SET_CR0((uintptr_t)crosspage->wsCR0);
-         SET_CR4((uintptr_t)crosspage->wsCR4);
+         SET_CR0((uintptr_t)crosspage->crosspageData.wsCR0);
+         SET_CR4((uintptr_t)crosspage->crosspageData.wsCR4);

         TaskSaveDebugRegisters(crosspage);

@@ -2719,22 +2719,23 @@ Task_Switch(VMDriver *vm,  // IN
            currentSpecCtrlValue = (currentSpecCtrlValue + 1) % 4;
            X86MSR_SetMSR(MSR_SPEC_CTRL, currentSpecCtrlValue);
 #endif
-            crosspage->specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
+            crosspage->crosspageData.specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
         }

-         DEBUG_ONLY(crosspage->monTinyStack[0] = 0xDEADBEEF;)
-         DEBUG_ONLY(crosspage->hostTinyStack[0] = 0xDEADBEEF;)
+         DEBUG_ONLY(crosspage->crosspageData.monTinyStack[0] = 0xDEADBEEF;)
+         DEBUG_ONLY(crosspage->crosspageData.hostTinyStack[0] = 0xDEADBEEF;)
         /* Running in host context prior to TaskSwitchToMonitor() */
         TaskSwitchToMonitor(crosspage);
         /* Running in host context after to TaskSwitchToMonitor() */

-         TS_ASSERT(crosspage->monTinyStack[0] == 0xDEADBEEF);
-         TS_ASSERT(crosspage->hostTinyStack[0] == 0xDEADBEEF);
+         TS_ASSERT(crosspage->crosspageData.monTinyStack[0] == 0xDEADBEEF);
+         TS_ASSERT(crosspage->crosspageData.hostTinyStack[0] == 0xDEADBEEF);

 #ifdef CYCLE_SPEC_CTRL
         if (CPUID_HostSupportsSpecCtrl()) {
            readSpecCtrlValue = X86MSR_GetMSR(MSR_SPEC_CTRL);
-            specCtrlEqual = readSpecCtrlValue == crosspage->specCtrl;
+            specCtrlEqual = readSpecCtrlValue ==
+                            crosspage->crosspageData.specCtrl;
            /* Do not leak cycling SPEC_CTRL value back to host. */
            X86MSR_SetMSR(MSR_SPEC_CTRL, 0);
         }
@@ -2772,10 +2773,10 @@ Task_Switch(VMDriver *vm,  // IN
          * CR2 always has to be restored.  CR8 never has to be restored.
          */
         SET_CR2(cr2reg);
-         if (crosspage->wsCR0 != cr0reg) {
+         if (crosspage->crosspageData.wsCR0 != cr0reg) {
            SET_CR0(cr0reg);
         }
-         if (crosspage->wsCR4 != cr4reg) {
+         if (crosspage->crosspageData.wsCR4 != cr4reg) {
            SET_CR4(cr4reg);
         } else if ((cr4reg & CR4_PCIDE) != 0) {
            /*
@@ -2823,7 +2824,7 @@ Task_Switch(VMDriver *vm,  // IN
            TaskDisableTF();
         }

-         TaskRestoreDebugRegisters(crosspage);
+         TaskRestoreDebugRegisters(&crosspage->crosspageData);

         ASSERT_NO_INTERRUPTS();

@@ -2842,7 +2843,7 @@ Task_Switch(VMDriver *vm,  // IN
            X86MSR_SetMSR(MSR_RTIT_CTL, ptMSR);
         }

-         TaskUpdateLatestPTSC(vm, crosspage);
+         TaskUpdateLatestPTSC(vm, &crosspage->crosspageData);
         Atomic_Write32(&vm->currentHostCpu[vcpuid], INVALID_PCPU);

         /*
@@ -2874,13 +2875,12 @@ Task_Switch(VMDriver *vm,  // IN
            RAISE_INTERRUPT(EXC_MC);
         }
         if (UNLIKELY(TaskGotException(crosspage, EXC_UD))) {
-            switchOk = FALSE;
-            Warning("#UD occurred on switch back to host.\n");
+            Warning("#UD occurred on switch back to host; dumping core.\n");
         }
 #ifdef CYCLE_SPEC_CTRL
         if (UNLIKELY(!specCtrlEqual)) {
            Warning("SpecCtrl not equal: expected %"FMT64"x, got %"FMT64"x\n",
-                    crosspage->specCtrl, readSpecCtrlValue);
+                    crosspage->crosspageData.specCtrl, readSpecCtrlValue);
         }
 #endif
         /*
@@ -2888,8 +2888,8 @@ Task_Switch(VMDriver *vm,  // IN
          * received during worldswitch.  Here is the more generic case
          * of forwarding NMIs received while executing the VMM/guest.
          */
-         if (crosspage->moduleCallType == MODULECALL_INTR &&
-             crosspage->args[0] == EXC_NMI) {
+         if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR &&
+             crosspage->crosspageData.args[0] == EXC_NMI) {
            /*
             * If VMM was interrupted by an NMI, do the INT 2 so the
             * host will handle it, but then return immediately to the
@@ -2899,12 +2899,12 @@ Task_Switch(VMDriver *vm,  // IN
             * done the EOI.
             */
            RAISE_INTERRUPT(EXC_NMI);
-            crosspage->retryWorldSwitch = TRUE;
+            crosspage->crosspageData.retryWorldSwitch = TRUE;
         }
      } while (UNLIKELY(TaskShouldRetryWorldSwitch(crosspage)));
   }

-   if (crosspage->moduleCallType == MODULECALL_INTR) {
+   if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR) {

      /*
       * Newer versions of Window expect EFLAGS_AC to be set when handling an
@@ -2923,15 +2923,15 @@ Task_Switch(VMDriver *vm,  // IN
       * what bad things btw).
       */
 #ifdef _WIN64
-      if (crosspage->args[0] <= 0xFF &&
-          (crosspage->args[0] >= 0x14 ||
-           crosspage->args[0] == EXC_MC)) {
-         RAISE_INTERRUPT((unsigned char)crosspage->args[0]);
+      if (crosspage->crosspageData.args[0] <= 0xFF &&
+          (crosspage->crosspageData.args[0] >= 0x14 ||
+           crosspage->crosspageData.args[0] == EXC_MC)) {
+         RAISE_INTERRUPT((unsigned char)crosspage->crosspageData.args[0]);
      } else {
         Warning("%s: Received Unexpected Interrupt: 0x%"FMT64"X\n",
-                 __FUNCTION__, crosspage->args[0]);
+                 __FUNCTION__, crosspage->crosspageData.args[0]);
         Panic("Received Unexpected Interrupt: 0x%"FMT64"X\n",
-               crosspage->args[0]);
+               crosspage->crosspageData.args[0]);
      }
 #else
      /*
@@ -2945,7 +2945,7 @@ Task_Switch(VMDriver *vm,  // IN
 #define IRQ_INT16(_x) IRQ_INT8(_x); IRQ_INT8(_x + 8)
 #define IRQ_INT32(_x) IRQ_INT16(_x); IRQ_INT16(_x + 16)

-      switch (crosspage->args[0]) {
+      switch (crosspage->crosspageData.args[0]) {
         // These are the general IO interrupts
         // It would be nice to generate this dynamically, but see Note2 above.

@@ -2978,14 +2978,13 @@ Task_Switch(VMDriver *vm,  // IN
          */

         Warning("%s: Received Unexpected Interrupt: 0x%"FMT64"X\n",
-                 __FUNCTION__, crosspage->args[0]);
+                 __FUNCTION__, crosspage->crosspageData.args[0]);
         Panic("Received Unexpected Interrupt: 0x%"FMT64"X\n",
-               crosspage->args[0]);
+               crosspage->crosspageData.args[0]);
      }
 #endif
   }

   RESTORE_FLAGS(flags);
   TaskRestoreNMI(&vm->hostAPIC, lint0NMI, lint1NMI, pcNMI, thermalNMI);
-   return switchOk;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015,2019-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@ struct MonLoaderHeader;
 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
 extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
                               PerVcpuPages *perVcpuPages);
-extern Bool Task_Switch(VMDriver *vm, Vcpuid vcpuid);
+extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
 extern MPN Task_GetHVRootPageForPCPU(uint32 pCPU);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -112,9 +112,8 @@ static unsigned globalFastClockRate;
 HostIFContigMemMap *hvIOBitmap;

 typedef struct {
-   Atomic_uint32 *index; // OUT: array of cpu counters for queries.
-   MSRQuery *query;  // IN/OUT: array of query items
-   uint32 numItems;  // IN
+   Atomic_uint32 index;
+   MSRQuery *query;
 } Vmx86GetMSRData;

 static Bool hostUsesNX;
@@ -124,20 +123,6 @@ typedef struct NXData {
   Atomic_uint32 hasNX;
 } NXData;

-/*
- * A structure for holding MSR indexes, values for MSR uniformity checks.
- */
-typedef struct VMX86MSRCacheInfo {
-   uint32 msrIndex;
-   uint64 msrValue;
-} VMX86MSRCacheInfo;
-
-static VMX86MSRCacheInfo msrUniformityCacheInfo[] = {
-#define MSR_FEAT(member) {member, CONST64(0)},
-   MSR_FEAT(IA32_MSR_ARCH_CAPABILITIES)
-#undef MSR_FEAT
-};
-
 /*
 *----------------------------------------------------------------------
 *
@@ -582,66 +567,6 @@ Vmx86_Calloc(size_t numElements, // IN
 }


-/*
- *-----------------------------------------------------------------------------
- *
- * Vmx86AllocCrossPages --
- *
- *      Allocate numVCPUs pages suitable to be used as the VCPU's
- *      crosspage area.
- *
- * Results:
- *      TRUE if the required crosspages are allocated successfully.
- *      FALSE otherwise.
- *
- *-----------------------------------------------------------------------------
- */
-
-static Bool
-Vmx86AllocCrossPages(VMDriver *vm)
-{
-   Vcpuid v;
-
-   for (v = 0; v < vm->numVCPUs; v++) {
-      MPN unused;
-
-      UNUSED_VARIABLE(unused);
-      vm->crosspage[v] = HostIF_AllocKernelPages(1, &unused);
-
-      if (vm->crosspage[v] == NULL) {
-         return FALSE;
-      }
-      memset(vm->crosspage[v], 0, PAGE_SIZE);
-   }
-   return TRUE;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * Vmx86FreeCrossPages --
- *
- *      Free the crosspages allocated for the given VM.
- *
- *-----------------------------------------------------------------------------
- */
-
-static void
-Vmx86FreeCrossPages(VMDriver *vm)
-{
-   Vcpuid v;
-
-   if (vm->crosspage != NULL) {
-      for (v = 0; v < vm->numVCPUs; v++) {
-         if (vm->crosspage[v] != NULL) {
-            HostIF_FreeKernelPages(1, vm->crosspage[v]);
-         }
-      }
-   }
-}
-
-
 /*
 *-----------------------------------------------------------------------------
 *
@@ -824,7 +749,6 @@ Vmx86FreeAllVMResources(VMDriver *vm)

      Vmx86_SetHostClockRate(vm, 0);

-      Vmx86FreeCrossPages(vm);
      if (vm->ptpTracker != NULL) {
         Task_SwitchPTPPageCleanup(vm);
      }
@@ -1273,15 +1197,6 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
      goto error;
   }

-   if (!pseudoTSC.initialized) {
-      Warning("%s: PseudoTSC has not been initialized\n", __FUNCTION__);
-      goto error;
-   }
-
-   if (!Vmx86AllocCrossPages(vm)) {
-      Warning("Failed to allocate cross pages.\n");
-      goto error;
-   }
   /*
    * Initialize the driver's part of the cross-over page used to
    * talk to the monitor.
@@ -1723,12 +1638,12 @@ Vmx86_MonTimerIPI(void)
      VCPUSet_Empty(&expiredVCPUs);

      for (v = 0; v < vm->numVCPUs; v++) {
-         VMCrossPageData *crosspage = vm->crosspage[v];
+         VMCrossPage *crosspage = vm->crosspage[v];

-         if (crosspage == NULL) {
+         if (!crosspage) {
            continue;  // VCPU is not initialized yet
         }
-         expiry = crosspage->monTimerExpiry;
+         expiry = crosspage->crosspageData.monTimerExpiry;
         if (expiry != 0 && expiry <= pNow) {
            VCPUSet_Include(&expiredVCPUs, v);
            hasWork = TRUE;
@@ -2864,47 +2779,45 @@ Vmx86_CheckPseudoTSC(uint64 *lastTSC, // IN/OUT: last/current value of the TSC
 static void
 Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 {
-   uint32 i;
   Vmx86GetMSRData *data = (Vmx86GetMSRData *)clientData;
-   ASSERT(data && data->index && data->query);
+   MSRQuery *query;
+   uint32 index;
+   int err;

-   for (i = 0; i < data->numItems; ++i) {
-      uint32 index;
-      int err;
-      Atomic_uint32 *cpus = &data->index[i];
-      MSRQuery *query = &data->query[i];
+   ASSERT(data);
+   query = data->query;
+   ASSERT(query);

-      index = Atomic_ReadInc32(cpus);
-      if (index >= query->numLogicalCPUs) {
-         continue;
-      }
-
-      query->logicalCPUs[index].tag = HostIF_GetCurrentPCPU();
-
-      /*
-       * We treat BIOS_SIGN_ID (microcode version) specially on Intel,
-       * where the preferred read sequence involves a macro.
-       */
-      if (CPUID_GetVendor() == CPUID_VENDOR_INTEL &&
-          query->msrNum == MSR_BIOS_SIGN_ID) {
-         /* safe to read: MSR_BIOS_SIGN_ID architectural since Pentium Pro */
-         query->logicalCPUs[index].msrVal = INTEL_MICROCODE_VERSION();
-         err = 0;
-      } else {
-         /*
-          * Try to enable HV any time these MSRs are queried.  We have seen
-          * buggy firmware that forgets to re-enable HV after waking from
-          * deep sleep. [PR 1020692]
-          */
-         if (query->msrNum == MSR_FEATCTL || query->msrNum == MSR_VM_CR) {
-            Vmx86EnableHVOnCPU();
-         }
-         err =
-            HostIF_SafeRDMSR(query->msrNum, &query->logicalCPUs[index].msrVal);
-      }
-
-      query->logicalCPUs[index].implemented = (err == 0) ? 1 : 0;
+   index = Atomic_ReadInc32(&data->index);
+   if (index >= query->numLogicalCPUs) {
+      return;
   }
+
+   query->logicalCPUs[index].tag = HostIF_GetCurrentPCPU();
+
+   /*
+    * We treat BIOS_SIGN_ID (microcode version) specially on Intel,
+    * where the preferred read sequence involves a macro.
+    */
+
+   if (CPUID_GetVendor() == CPUID_VENDOR_INTEL &&
+       query->msrNum == MSR_BIOS_SIGN_ID) {
+      /* safe to read: MSR_BIOS_SIGN_ID architectural since Pentium Pro */
+      query->logicalCPUs[index].msrVal = INTEL_MICROCODE_VERSION();
+      err = 0;
+   } else {
+      /*
+       * Try to enable HV any time these MSRs are queried.  We have seen
+       * buggy formware that forgets to re-enable HV after waking from
+       * deep sleep. [PR 1020692]
+       */
+      if (query->msrNum == MSR_FEATCTL || query->msrNum == MSR_VM_CR) {
+         Vmx86EnableHVOnCPU();
+      }
+      err = HostIF_SafeRDMSR(query->msrNum, &query->logicalCPUs[index].msrVal);
+   }
+
+   query->logicalCPUs[index].implemented = (err == 0) ? 1 : 0;
 }


@@ -2913,7 +2826,7 @@ Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 *
 * Vmx86_GetAllMSRs --
 *
- *      Collect MSR value on number of logical CPUs requested.
+ *      Collect MSR value on all logical CPUs.
 *
 *      The caller is responsible for ensuring that the requested MSR is valid
 *      on all logical CPUs.
@@ -2935,40 +2848,24 @@ Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 Bool
 Vmx86_GetAllMSRs(MSRQuery *query) // IN/OUT
 {
-   unsigned i, cpu;
-   Atomic_uint32 index;
   Vmx86GetMSRData data;
-   data.index = &index;
-   data.numItems = 1;

-   /* Check MSR uniformity cache first. */
-   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
-      if (msrUniformityCacheInfo[i].msrIndex == query->msrNum) {
-         for (cpu = 0; cpu < query->numLogicalCPUs; cpu++) {
-            query->logicalCPUs[cpu].msrVal = msrUniformityCacheInfo[i].msrValue;
-            query->logicalCPUs[cpu].implemented = 1;
-            query->logicalCPUs[cpu].tag = cpu;
-         }
-         return TRUE;
-      }
-   }
-
-   Atomic_Write32(data.index, 0);
+   Atomic_Write32(&data.index, 0);
   data.query = query;

   HostIF_CallOnEachCPU(Vmx86GetMSR, &data);

   /*
-    * At this point, Atomic_Read32(data.index) is the number of logical CPUs
+    * At this point, Atomic_Read32(&data.index) is the number of logical CPUs
    * who replied.
    */

-   if (Atomic_Read32(data.index) > query->numLogicalCPUs) {
+   if (Atomic_Read32(&data.index) > query->numLogicalCPUs) {
      return FALSE;
   }

-   ASSERT(Atomic_Read32(data.index) <= query->numLogicalCPUs);
-   query->numLogicalCPUs = Atomic_Read32(data.index);
+   ASSERT(Atomic_Read32(&data.index) <= query->numLogicalCPUs);
+   query->numLogicalCPUs = Atomic_Read32(&data.index);

   return TRUE;
 }
@@ -3511,13 +3408,16 @@ Vmx86_GetMonitorContext(VMDriver *vm,       // IN: The VM instance.
                        Vcpuid vcpuid,      // IN: VCPU in question.
                        Context64 *context) // OUT: context.
 {
+   VMCrossPage *crosspage;
   VMCrossPageData *cpData;
-
-   if (vcpuid >= vm->numVCPUs || vm->crosspage[vcpuid] == NULL) {
+   if (vcpuid >= vm->numVCPUs) {
      return FALSE;
   }
-   cpData = vm->crosspage[vcpuid];
-
+   crosspage = vm->crosspage[vcpuid];
+   if (!crosspage) {
+      return FALSE;
+   }
+   cpData = &crosspage->crosspageData;
   memset(context, 0, sizeof *context);
   context->es  = cpData->monES;
   context->ss  = cpData->monSS;
@@ -3594,137 +3494,3 @@ Vmx86_CreateHVIOBitmap(void)
   memset(hvIOBitmap->addr, 0xff, SVM_VMCB_IO_BITMAP_SIZE);
   return TRUE;
 }
-
-/*
- *-----------------------------------------------------------------------------
- *
- * Vmx86RegisterCPU --
- *
- *      Registers each logical CPU by incrementing a counter.
- *
- * Results:
- *      None.
- *
- * Side effects:
- *      counter value pointed by 'data' is incremented by one.
- *
- *-----------------------------------------------------------------------------
- */
-
-static void
-Vmx86RegisterCPU(void *data) // IN: *data
-{
-   Atomic_uint32 *numLogicalCPUs = data;
-   ASSERT(numLogicalCPUs);
-   Atomic_Inc32(numLogicalCPUs);
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * Vmx86_CheckMSRUniformity --
- *
- *      Provides basic hardware MSR feature checks for x86 hosted platform. VMM
- *      requires and prefers uniformity of certain MSRs. This function iterates
- *      through a list of MSR features (i.e. msrUniformityCacheInfo), checking
- *      uniformity for MSR value on each logical CPU. A Uniformity check is
- *      ignored for MSRs are that are not available for the target architecture
- *      or cpu family. If MSRs are non uniform then, a common bit field is
- *      calculated by taking the intersection of MSR values across cpu(s).
- *
- * Results:
- *      Returns TRUE if MSR uniformity checks complete successfully, FALSE
- *      otherwise.
- *
- * Side effects:
- *      updates msrUniformityCacheInfo cache with MSR values.
- *
- *-----------------------------------------------------------------------------
- */
-
-Bool
-Vmx86_CheckMSRUniformity(void)
-{
-   Vmx86GetMSRData data;
-   uint32 i, j;
-   uint32 numQueries = ARRAYSIZE(msrUniformityCacheInfo);
-   Atomic_uint32 numLogicalCPUs;
-   Atomic_uint32 *cpuCounters;
-   uint32 numPCPUs = 0;
-   MSRQuery *query = NULL;
-
-   Atomic_Write32(&numLogicalCPUs, 0);
-   /*
-    * Calculates number of logical CPUs by counting and then uses this
-    * information to set up MSR queries; will be executed on each logical CPU.
-    */
-   HostIF_CallOnEachCPU(Vmx86RegisterCPU, &numLogicalCPUs);
-   numPCPUs = Atomic_Read32(&numLogicalCPUs);
-   ASSERT(numPCPUs > 0);
-   query = Vmx86_Calloc(numQueries,
-                        sizeof(MSRQuery) + sizeof(MSRReply) * numPCPUs, TRUE);
-   if (query == NULL) {
-      Warning("Fatal, not enough memory for MSR feature uniformity checks");
-      return FALSE;
-   }
-
-   cpuCounters = Vmx86_Calloc(numQueries, sizeof(Atomic_uint32), TRUE);
-   if (cpuCounters == NULL) {
-      Vmx86_Free(query);
-      Warning("Fatal, not enough memory for MSR feature uniformity checks");
-      return FALSE;
-   }
-   data.query = query;
-   data.index = cpuCounters;
-   data.numItems = numQueries;
-
-   /*
-    * Enumerates a MSR list and initializes MSR data structure before the
-    * actual (safe) MSR query takes place. The Nested loop tests a MSR for
-    * uniformity on all logical processors.
-    */
-   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
-      query = &data.query[i];
-      Atomic_Write32(&data.index[i], 0);
-      query->msrNum = msrUniformityCacheInfo[i].msrIndex;
-      query->numLogicalCPUs = numPCPUs;
-   }
-
-   /* perform once, a multi MSR query for MSRs in uniformity check list. */
-   HostIF_CallOnEachCPU(Vmx86GetMSR, &data);
-
-   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
-      uint32 msrIndex = msrUniformityCacheInfo[i].msrIndex;
-      query = &data.query[i];
-      ASSERT(Atomic_Read32(&data.index[i]) == numPCPUs);
-      msrUniformityCacheInfo[i].msrValue = query->logicalCPUs[0].msrVal;
-      if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
-         /*
-          * MSR_ARCH_CAPABILITIES_RSBA bit 1 represents lack of feature while 0
-          * represents presence. Therefore, bit is flipped for calculating the
-          * least common set and flipped again on the final value for resetting.
-          */
-         msrUniformityCacheInfo[i].msrValue ^= MSR_ARCH_CAPABILITIES_RSBA;
-      }
-      for (j = 1; j < numPCPUs; j++) {
-         uint64 msrValuePCPU = query->logicalCPUs[j].msrVal;
-         if (msrValuePCPU != query->logicalCPUs[0].msrVal) {
-            if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
-               msrValuePCPU ^= MSR_ARCH_CAPABILITIES_RSBA;
-            }
-            msrUniformityCacheInfo[i].msrValue &= msrValuePCPU;
-            Warning("Found a mismatch on MSR feature 0x%x; logical cpu%u "
-                    "value = 0x%llx, but logical cpu%u value = 0x%llx\n",
-                    msrIndex, j, msrValuePCPU, 0, query->logicalCPUs[0].msrVal);
-         }
-      }
-      if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
-         msrUniformityCacheInfo[i].msrValue ^= MSR_ARCH_CAPABILITIES_RSBA;
-      }
-   }
-   Vmx86_Free(cpuCounters);
-   Vmx86_Free(query);
-   return TRUE;
-}
-
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -68,7 +68,7 @@ typedef struct VMDriver {
   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
-   struct VMCrossPageData **crosspage;        /* numVCPUs-sized array. */
+   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
@@ -175,7 +175,6 @@ extern void Vmx86_UpdateMemInfo(VMDriver *curVM,
 extern void Vmx86_Add2MonPageTable(VMDriver *vm, VPN vpn, MPN mpn,
                                   Bool readOnly);
 extern Bool Vmx86_GetAllMSRs(MSRQuery *query);
-extern Bool Vmx86_CheckMSRUniformity(void);
 extern void Vmx86_FlushVMCSAllCPUs(MA vmcs);
 extern void Vmx86_MonTimerIPI(void);
 extern void Vmx86_InitIDList(void);
@@ -1,319 +0,0 @@
-/*********************************************************
- * Copyright (C) 2007-2020 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _ADDRLAYOUT_H_
-#define _ADDRLAYOUT_H_
-
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_VMX
-#include "includeCheck.h"
-
-#include "vm_basic_types.h"
-#include "vm_basic_defs.h"
-#include "address_defs.h"
-#include "x86desc.h"
-#include "x86types.h"
-#include "vmm_constants.h"
-
-#define DIRECT_EXEC_USER_RPL    3
-#define BINARY_TRANSLATION_RPL  1
-typedef char x86_PAGE[PAGE_SIZE];
-
-#define MAX_VADDR                    CONST64U(0xffffffffffffffff)
-#define MONITOR_SIZE                 (64 * 1024 * 1024)
-#define MONITOR_LINEAR_START         (MAX_VADDR - MONITOR_SIZE + 1)
-#define MONITOR_LINEAR_END           (MAX_VADDR)
-#define NUM_MONITOR_PAGES            (MONITOR_SIZE / PAGE_SIZE)
-#define MONITOR_BASE_VPN             (MONITOR_LINEAR_START >> PAGE_SHIFT)
-
-#define MONITOR_AS_START             MONITOR_BASE_VPN
-#define MONITOR_AS_LEN               NUM_MONITOR_PAGES
-
-/*
- * Regions and items necessarily overlap.  Build parallel structures for each.
- * These 'field' structures are used to determine page numbers and offsets.
- * AddrLayout contains zero-length regions used for alignment-checking.  (These
- * are used by gcc as Windows prohibits zero-length arrays.)
- */
-
-/* Item layout. */
-typedef struct {
-#if defined __GNUC__
-#define REGION(name, length) x86_PAGE name##_REGION_MARKER_FIELD[0];
-#else
-#define REGION(name, length)
-#endif
-#define ITEM(name, length) x86_PAGE name##_FIELD[length];
-#include "addrlayout_table.h"
-} AddrLayout;
-#undef REGION
-#undef ITEM
-#define FIELD_PAGE_NUMBER(name) PAGE_NUMBER(&(((AddrLayout *)0)->name##_FIELD))
-
-
-/* Region layout. */
-typedef struct {
-#define REGION(name, length) x86_PAGE name##_REGION_FIELD[length];
-#define ITEM(name, length)
-#include "addrlayout_table.h"
-} RegionAddrLayout;
-#undef REGION
-#undef ITEM
-#define REGION_FIELD_PAGE_NUMBER(name) \
-   PAGE_NUMBER(&(((RegionAddrLayout *)0)->name##_REGION_FIELD))
-
-#if defined __GNUC__
-/*
- * The sum of sizes of items in a region must add up to the region's size.
- * AddrLayout contains zero-sized region markers at their correct offsets.
- * RegionAddrLayout contains full-sized regions (but no items).  Comparing the
- * marker and region offsets ensures that all items sizes in the previous
- * region sum to the size of the previous region.  To check the final region
- * (as it has no successor) we assert the sizes of the tables match.
- *
- * We also assert 2MB region granularity.
- *
- * This checking is only performed when using gcc.
- */
-#define REGION(name, length) static INLINE void name(void) { \
-   ASSERT_ON_COMPILE(FIELD_PAGE_NUMBER(name##_REGION_MARKER) == \
-                     REGION_FIELD_PAGE_NUMBER(name)); \
-   ASSERT_ON_COMPILE((length % 512) == 0); \
-   ASSERT_ON_COMPILE(sizeof(AddrLayout) == sizeof(RegionAddrLayout)); \
-   ASSERT_ON_COMPILE(sizeof(RegionAddrLayout) == MONITOR_SIZE); \
-}
-#define ITEM(name, length)
-#include "addrlayout_table.h"
-#undef REGION
-#undef ITEM
-#endif
-
-/*
- * Sizes.
- */
-
-enum {
-#define REGION(name, length) name##_REGION_LEN = length,
-#define ITEM(name, length) name##_LEN = length,
-#include "addrlayout_table.h"
-};
-#undef REGION
-#undef ITEM
-
-/*
- * Starts.
- */
-
-#if defined __GNUC__
-/* Windows compiler and gdb do not accept 64-bit enumerations, but gcc
- * does.  These are used by geninfo.c.
- */
-enum {
-#define REGION(name, length) _asm_##name##_REGION_START = \
-    0ULL + MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
-#define ITEM(name, length) _asm_##name##_START = \
-    0ULL + MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
-#include "addrlayout_table.h"
-};
-#undef REGION
-#undef ITEM
-
-enum {
-#define REGION(name, length) name##_REGION_START = \
-   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
-#define ITEM(name, length) name##_START = \
-   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
-#include "addrlayout_table.h"
-};
-#undef REGION
-#undef ITEM
-
-#else
-#define REGION(name, length) \
-   static const uint64 name##_REGION_START = \
-   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name);
-#define ITEM(name, length) \
-   static const uint64 name##_START = \
-   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name);
-#include "addrlayout_table.h"
-#undef REGION
-#undef ITEM
-#endif
-
-enum {
-#define REGION(name, length) name##_REGION_start_page = \
-   REGION_FIELD_PAGE_NUMBER(name),
-#define ITEM(name, length) name##_start_page = \
-   FIELD_PAGE_NUMBER(name),
-#include "addrlayout_table.h"
-};
-#undef REGION
-#undef ITEM
-
-
-/* Derive the bootstrap start address from table contents. */
-#define MONITOR_BOOTSTRAP_START_LA   VPN_2_VA(BS_TXT_START)
-
-#ifndef VMX86_SERVER
-#define SWITCH_PAGE_TABLE_LEN          12
-#endif
-
-#define VMM_STACK_TOP_HW          VPN_2_VA(MON_STACK_PAGES_START + \
-                                           MON_STACK_PAGES_LEN)
-/*
- * VMM_STKTOP_HW_LEN needs to contain an ExcFrame64ForCopy (72 bytes) and
- * some more bytes for a "no man's land" region.  We make sure the "no
- * man's land" region is 0 on BT_Resume and HV_Resume.  Because we
- * check "no man's land" in 8-byte chunks, VMM_STKTOP_HW_LEN must be
- * divisible by 8.
- */
-#define VMM_STKTOP_HW_LEN          168
-#define VMM_STKTOP_HW_LRET_LEN     (VMM_STKTOP_HW_LEN + sizeof(LretFrame64))
-#define VMM_STKTOP_LRET_OFFSET     (PAGE_SIZE - VMM_STKTOP_HW_LRET_LEN)
-
-#define MON_STACK_BASE            VPN_2_VA(MON_STACK_PAGES_START)
-#define MON_STACK_TOP             VPN_2_VA(MON_STACK_PAGES_START + \
-                                           MON_STACK_PAGES_LEN)
-#define DF_STACK_BASE              VPN_2_VA(DF_STACK_PAGES_START)
-#define DF_STACK_TOP               VPN_2_VA(DF_STACK_PAGES_START + \
-                                            DF_STACK_PAGES_LEN)
-#define MC_STACK_BASE              VPN_2_VA(MC_STACK_PAGES_START)
-#define MC_STACK_TOP               VPN_2_VA(MC_STACK_PAGES_START + \
-                                            MC_STACK_PAGES_LEN)
-#define NMI_STACK_BASE             VPN_2_VA(NMI_STACK_PAGES_START)
-#define NMI_STACK_TOP              VPN_2_VA(NMI_STACK_PAGES_START + \
-                                            NMI_STACK_PAGES_LEN)
-
-#define NUM_MON_PTABS   MON_PAGE_TABLE_L1_LEN
-
-#define MON_START_L5OFF PT_LA_2_L5OFF(MONITOR_LINEAR_START)
-#define MON_START_L4OFF PT_LA_2_L4OFF(MONITOR_LINEAR_START)
-#define MON_START_L3OFF PT_LA_2_L3OFF(MONITOR_LINEAR_START)
-#define MON_START_L2OFF PT_LA_2_L2OFF(MONITOR_LINEAR_START)
-
-/*
- * Allocated wired pages for the monitor.
- * Currently we allocate wired pages for the:
- *  0) The shared area between user and monitor;
- *  a) GDT of the monitor
- *  a1) BT (CP1) stack
- *  b) The monitor's page table with one page directory under 4 GB
- *      and two page table pages.
- *  c) Pages for the monitor's stack.
- *  d) The BT area pages
- *  e) The cross driver/monitor page.
- *  f) The physical memory array for the machine.
- *  g) The SMRAM memory
- *
- * We allocate the shared area separately since it must be read/write
- * in the monitor.   See SharedArea_PowerOn().
- */
-
-#define MON_BOOTSTRAP_PAGE_TABLE_PAGES (BOOTSTRAP_REGION_LEN     /  \
-                                        PAE_PTES_PER_PGTBL)
-#ifdef VMX86_SERVER
-#define NUM_MONWIRED_PAGES     0
-
-#define MON_PAGE_TABLE_ALLOC_PAGES 0
-#define NUM_MONWIRED_BOOTSTRAP_PAGES 0
-
-#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN  + \
-                                    DF_STACK_PAGES_LEN   + \
-                                    MC_STACK_PAGES_LEN   + \
-                                    NMI_STACK_PAGES_LEN  + \
-                                    HV_CURRENT_VMCB_LEN  + \
-                                    IDT_NORMAL_LEN       + \
-                                    GDT_AND_TASK_LEN)
-
-#else
-#define NUM_MONWIRED_PAGES    (CROSS_PAGE_CODE_LEN   + \
-                               CROSS_PAGE_DATA_LEN   + \
-                               MON_PAGE_TABLE_L4_LEN + \
-                               MON_PAGE_TABLE_L3_LEN + \
-                               MON_PAGE_TABLE_L2_LEN)
-
-#define MON_PAGE_TABLE_ALLOC_PAGES ((MONITOR_READONLY_REGION_LEN + \
-                                     MONITOR_DATA_REGION_LEN     + \
-                                     MONITOR_MISC_REGION_LEN)    / \
-                                     PAE_PTES_PER_PGTBL)
-
-#define NUM_MONWIRED_BOOTSTRAP_PAGES MON_BOOTSTRAP_PAGE_TABLE_PAGES
-
-#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN + \
-                                    DF_STACK_PAGES_LEN  + \
-                                    MC_STACK_PAGES_LEN  + \
-                                    NMI_STACK_PAGES_LEN + \
-                                    GDT_AND_TASK_LEN    + \
-                                    IDT_NORMAL_LEN      + \
-                                    HV_CURRENT_VMCB_LEN + \
-                                    SWITCH_PAGE_TABLE_LEN)
-#endif
-
-#define MON_PAGE_TABLE_L5        VPN_2_VA(MON_PAGE_TABLE_L5_START)
-#define MON_PAGE_TABLE_L4        VPN_2_VA(MON_PAGE_TABLE_L4_START)
-#define VMM_STACK_TOP            (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LEN)
-#define VMM_LRET_STACK_TOP       (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LRET_LEN)
-/*
- * Mark out a guard page for the VMM stack, this is not in the addrlayout_table
- * explicitly because it overlaps the last page of the MONITOR_READONLY region.
- */
-#define VMM_STACK_GUARD_START    MON_STACK_PAGES_START
-#define VMM_STACK_GUARD_LEN      PAGE_SIZE
-
-/*
- * Following functions return true if the range [va, va+len) is within
- * range of a particular monitor stack.  (Don't adjust without
- * considering uint64 overflow when va is very high.)
- */
-static INLINE Bool
-AddrLayout_InMonStack(VA64 va, size_t len)
-{
-   return VPN_2_VA(MON_STACK_PAGES_START) <= va &&
-          va <= VMM_STACK_TOP_HW - len;
-}
-
-static INLINE Bool
-AddrLayout_InNMIStack(VA64 va, size_t len)
-{
-   return va >= NMI_STACK_BASE && va <= NMI_STACK_TOP - len;
-}
-
-static INLINE Bool
-AddrLayout_InDFStack(VA64 va, size_t len)
-{
-   return va >= DF_STACK_BASE && va <= DF_STACK_TOP - len;
-}
-
-static INLINE Bool
-AddrLayout_InMCStack(VA64 va, size_t len)
-{
-   return va >= MC_STACK_BASE && va <= MC_STACK_TOP - len;
-}
-
-static INLINE Bool
-AddrLayout_InAMonitorStack(VA va, size_t len)
-{
-   return AddrLayout_InMonStack(va, len)  ||
-          AddrLayout_InNMIStack(va, len)  ||
-          AddrLayout_InDFStack(va, len)   ||
-          AddrLayout_InMCStack(va, len);
-}
-
-#endif
@@ -1,144 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * addrlayout_table.h --
- *
- * Named elements in the monitor's address space of fixed size and location.
- * All units are 4 kilobyte pages.  Also used for vmmlayout output.
- *
- * The monitor's address space is populated three different ways:
- * 1) For elements of a fixed size and location, at compile-time.
- * 2) For well-known program sections and the shared area, at link-time.
- * 3) Dynamically, by the allocator, during bootstrapping and monitor power-on.
- *
- * This file describes all of the monitor's main 64MB address space.  All of
- * (1) is described specifically.  Ranges for (2) and (3) are accounted for,
- * but their contents are described generally here.
- *
- * The monitor's address space is organized into container regions which are an
- * integer number of 2MB "large" pages.  Within a container region, items
- * comprised of one or more 4KB pages may be described.
- *
- * All container regions and items are described by a name and a size in pages.
- * No two items may have the same name nor may two regions.
- *
- * REGION(region-name, number-of-pages)
- *   ITEM(item-name,   number-of-pages)     // first item in region
- *   ITEM(item-name,   number-of-pages)     // second item in region
- * <...>
- */
-
-
-/*
- * Monitor .rodata/.text.  Mapped large (reducing TLB pressure) and read-only.
- */
-REGION(MONITOR_READONLY,        1024)
-  ITEM(MONITOR_READONLY_LINKER, 1024) // Used by linker.
-
-/*
- * Monitor .data/.bss.  Mapped large (reducing TLB pressure) and read-write.
- */
-REGION(MONITOR_DATA,            512)
-  ITEM(MON_STACK_PAGES,           8) // Monitor stack.
-  ITEM(IDT_NORMAL,                1) // Normal IDT (used for all but SVM).
-  ITEM(MONITOR_DATA,            503) // Used by linker and TC.
-
-/*
- * The monitor's translation cache.  This object starts at the page boundary
- * following the end of the used space in MONITOR_DATA and extends through
- * the TC_REGION.
- */
-REGION(TC_REGION,              1024)
-  ITEM(TC_BLOCK,               1024)
-
-/*
- * The monitor's pages for architectural state, stacks, page tables, shared
- * pages, and dynamic allocation.  This region also contains the linked shared
- * area and other contents.
- * Mapped small (allowing non-present mappings and sharing) and read-write.
- */
-REGION(MONITOR_MISC,           5632)
-#ifdef VMX86_SERVER
-  ITEM(GUARD_PAGE,                2) // Reserved (for symmetry with hosted).
-#else
-  ITEM(CROSS_PAGE_DATA,           1) // Cross page data (RW).
-  ITEM(CROSS_PAGE_CODE,           1) // Cross page code (RX).
-#endif
-  ITEM(GDT_AND_TASK,              1) // GDT and Task State Segment.
-
-  ITEM(MON_PAGE_TABLE_L5,         1) // Monitor page root, if 5-Level PT used.
-  ITEM(MON_PAGE_TABLE_L4,         1) // Monitor page root, if 4-Level PT used.
-  ITEM(MON_PAGE_TABLE_L3,         1) // Monitor L3 page table.
-  ITEM(MON_PAGE_TABLE_L2,         1) // Monitor L2 page table.
-  ITEM(MON_PAGE_TABLE_L1,        32) // Monitor L1 page tables.
-
-  ITEM(HOST_APIC,                 1) // Physical APIC.
-  ITEM(GUEST_APIC,                1) // Guest APIC.
-
-  ITEM(DF_GUARD_PAGE,             1) // Double Fault stack guard page.
-  ITEM(DF_STACK_PAGES,            1) // Double Fault stack; need ~768 bytes.
-
-  ITEM(MC_GUARD_PAGE,             1) // Machine Check stack guard page.
-  ITEM(MC_STACK_PAGES,            1) // Machine Check stack.
-
-  ITEM(NMI_GUARD_PAGE,            1) // NMI stack guard page.
-  ITEM(NMI_STACK_PAGES,           4) // NMI stack.  Profiling requires a larger
-                                     // stack than MC/DF.
-
-  ITEM(HV_SWITCH,                 1) // SVM switch page
-  ITEM(HV_CURRENT_VMCB,           1) // SVM current VMCB
-  ITEM(HV_STD_NATIVE_VMCB,        1) // SVM/VT standard VMCB/VMCS
-  ITEM(HV_AUX_NATIVE_VMCB,        1) // SVM/VT auxiliary VMCB/VMCS
-  ITEM(VHV_GUEST_VMCB,            1) // VSVM/VVT guest VMCB/VMCS
-  ITEM(HV_SEV_VMSA,               1) // SVM SEV-ES state save area
-  ITEM(VPROBE_MON_RELOC,          1) // VProbe monitor reloc page.
-  ITEM(GART_LIST_PAGES,          48) // PhysMem vmm gart list pages.
-  ITEM(GART_ALT_LIST_PAGES,      48) // Alternate vmm gart pages
-                                     // (for invalidation).
-  ITEM(GART_BF_PAGES,             8) // Gart Bloom filter memory pages.
-  ITEM(VVT_GUEST_VIRT_APIC,       1) // Inner guest virtual APIC page.
-  ITEM(HT_STATE_MAP,              8) // Information used by the SecureHT module
-  ITEM(SHARED_RW_DATA,         5462) // R/W shared data, including the shared
-                                     // area and stat vars. Any remaining pages
-                                     // are left to the dynamic allocator, which
-                                     // extends throughout the entire reclaimed
-                                     // bootstrap region.
-/*
- * Bootstrap-used space.  This space is reclaimed when bootstrap is done, and
- * is used by the allocator thereafter.  The allocator starts at the first
- * unused page in the SHARED_RW_DATA item from the MONITOR_MISC region and
- * extends through to the end of the VMM address space once the bootstrap region
- * is reclaimed.
- */
-REGION(BOOTSTRAP,              8192)
-  ITEM(BS_TXT,                   17) // bootstrap's .text
-  ITEM(BS_RODATA,                10) // bootstrap's .rodata
-  ITEM(BS_DATA,                  10) // bootstrap's .data
-  ITEM(BS_BSS,                   10) // bootstrap's .bss
-  ITEM(BS_HEADER,                 1) // MonLoaderHeader mapping
-  ITEM(BS_VCPU_L1PT_RANGE,       32) // VCPU L1PT mappings
-  ITEM(BS_VCPU_L1PT_PT,           1) // L1PT that maps a VCPU's L1 page tables
-  ITEM(BS_INIT_POOL,              1) // pool for initializing VMM pages
-  ITEM(BS_DYNAMIC_ALLOC,       1880) // bs_alloc's dynamic allocator pages
-  ITEM(IDT_BOOTSTRAP,             1) // bootstrap IDT
-  ITEM(IDT_BOOTSTRAP_STUBS,       2) // bootstrap IDT gate stubs
-  ITEM(BS_PER_VM_VMX,           300) // vmm64's shared_per_vm_vmx
-  ITEM(BS_PER_VCPU,               8) // vmm64's shared_per_vcpu for VCPU 0
-  ITEM(BS_PER_VCPU_VMX,         136) // vmm64's shared_per_vcpu_vmx for VCPU 0
-  ITEM(VMM_MODULES,            5783) // ~22.6MB for unlinked VMM modules
@@ -0,0 +1,30 @@
+#ifndef __COMPAT_POLL_H__
+#define __COMPAT_POLL_H__
+
+#include <linux/poll.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0))
+
+#ifndef __poll_t
+typedef unsigned int __poll_t;
+#endif
+
+static inline __poll_t compat_vfs_poll(struct file *file,
+				       struct poll_table_struct *pt)
+{
+	if (unlikely(!file->f_op->poll))
+		return DEFAULT_POLLMASK;
+	return file->f_op->poll(file, pt);
+}
+
+#else
+
+static inline __poll_t compat_vfs_poll(struct file *file,
+				       struct poll_table_struct *pt)
+{
+	return vfs_poll(file, pt);
+}
+
+#endif
+
+#endif /* __COMPAT_POLL_H__ */
@@ -148,7 +148,7 @@ PtrToVA64(void const *ptr) // IN
 * the NT specific VMX86_DRIVER_VERSION.
 */

-#define VMMON_VERSION           (401 << 16 | 0)
+#define VMMON_VERSION           (392 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -248,7 +248,6 @@ enum IOCTLCmd {
   IOCTLCMD(GET_IPI_VECTORS),
   IOCTLCMD(SEND_IPI),
   IOCTLCMD(SEND_ONE_IPI),
-   IOCTLCMD(GET_SWITCH_ERROR_ADDR),

   /*
    * Keep host-specific calls at the end so they can be undefined
@@ -317,7 +316,6 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
-#define IOCTL_VMX86_GET_SWITCH_ERROR_ADDR VMIOCTL_BUFFERED(GET_SWITCH_ERROR_ADDR)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
 #define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
@@ -533,8 +531,6 @@ typedef struct IPIVectors {
 */
 typedef struct VMCreateBlock {
   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
-   VA64               vmmonData;     // IN: User VA of a userlevel scratch area
-                                     //     required by the Linux vmmon
   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
   uint32             numVCPUs;      // IN: Number of VCPUs.
   uint16             vmid;          // OUT: VM ID for the created VM.
@@ -567,6 +563,7 @@ typedef struct VMStatVarsRegistrationBlock {
 } VMStatVarsRegistrationBlock;

 typedef struct {
+   VA64   crosspage; // IN: User VA of VCPU crosspage.
   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
 } PerVcpuPages;

@@ -582,15 +579,6 @@ typedef struct VMProcessBootstrapBlock {
   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

-/*
- * Arguments for retrieving the switch error address.
- */
-typedef struct VMSwitchErrorArgs {
-   Vcpuid vcpuid;  // IN: The VCPU of interest.
-   uint64 addr;    // OUT: The code address that a failure was detected at, or 0
-                   //      if no failure has occurred.
-} VMSwitchErrorArgs;
-
 static INLINE size_t
 GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
 {
@@ -42,6 +42,8 @@
 #include "mon_assert.h"

 #define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
+#define CP_STUB_SIZE     16     /* A relative jmp instruction (5 bytes)
+                                   padded to the next 16 byte boundary. */

 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
@@ -64,14 +66,12 @@
   MC(ALLOC_ANON_LOW_PAGE)                                                    \
   MC(GET_MON_IPI_VECTOR)                                                     \
   MC(GET_HV_IPI_VECTOR)                                                      \
-   MC(GET_PERF_CTR_VECTOR)                                                    \
   MC(GET_HOST_TIMER_VECTORS)                                                 \
   MC(BOOTSTRAP_CLEANUP)                                                      \
   MC(GET_SHARED_AREA)                                                        \
   MC(GET_STAT_VARS)                                                          \
   MC(GET_NUM_PTP_PAGES)                                                      \
   MC(GET_HV_IO_BITMAP)                                                       \
-   MC(GET_MSR)                                                                \

 /*
 *----------------------------------------------------------------------
@@ -109,8 +109,8 @@ typedef enum ModuleCallType {
 */

 #if defined(VMX86_UCCOST) && !defined(VMX86_SERVER)
-#define UCTIMESTAMP(ptr, stamp) \
-             do { (ptr)[UCCOST_ ## stamp] = RDTSC(); } while (0)
+#define UCTIMESTAMP(cp, stamp) \
+             do { (cp)->ucTimeStamps[UCCOST_ ## stamp] = RDTSC(); } while (0)
 #else
 #define UCTIMESTAMP(cp, stamp)
 #endif
@@ -130,7 +130,7 @@ typedef struct UCCostResults {
 } UCCostResults;

 typedef enum UCCostStamp {
-#define UC(x, y) UCCOST_ ## x,
+#define UC(x) UCCOST_ ## x,
 #include "uccostTable.h"
   UCCOST_MAX
 } UCCostStamp;
@@ -144,12 +144,12 @@ typedef enum UCCostStamp {
 * MAX_SWITCH_PT_PATCHES
 *
 *   This is the maximum number of patches that must be placed into
- *   the monitor page tables so that the host GDT, crosspage data, and
- *   crosspage code can be accessed during worldswitch.
+ *   the monitor page tables so that the host GDT and the crosspage
+ *   can be accessed during worldswitch.
 *
 *----------------------------------------------------------------------
 */
-#define MAX_SWITCH_PT_PATCHES 3
+#define MAX_SWITCH_PT_PATCHES 2

 /*----------------------------------------------------------------------
 *
@@ -236,23 +236,16 @@ VMMPageTablePatch;
 * - the largest stack use instantaneously possible by #DB handling code
 * - one high-water uint32 used to detect stack overflows when debugging
 *
- * A breakdown of the worst-case exception handler stack usage (SwitchUDHandler)
- * is: 5 * uint64 (Hardware) + 1 * uint64 (RAX) + 1 * uint64 (RBX) +
- *     1 * uint64 (RCX) + 1 * uint64 (call) + 2 * uint64 (sidt) = 11 * uint64
- * This is a slight over-estimate of the possible usage at any time but there
- * is plenty of space available in the cross page data area.
- *
- * 264 (11 * sizeof(uint64) * 3) bytes is slightly more than enough as of
- * 2020/06/14.
+ * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
 */
-#define TINY_STACK_SIZE      264
+#define TINY_STACK_SIZE      184

 /*
 *----------------------------------------------------------------------
 *
 * VMCrossPageData --
 *
- *      Data which is stored on the cross page.
+ *      Data which is stored on the VMCrossPage.
 *
 *----------------------------------------------------------------------
 */
@@ -301,13 +294,11 @@ struct VMCrossPageData {
   uint16   monES;  /* Not saved/restored in assembly switch */
   uint16   monPad;

-   uint64   crosspageDataMA;
+   uint64   crosspageMA;

   uint64   hostDR[8];
-   LA64     crosspageDataLA;   // where host/PTP map the cross data page
+   LA64     crossPageLA;       // where host/PTP map the cross page
   LA64     crossGDTLA;        // where host/PTP map the cross GDT
-   LA64     crosspageCodeLA;   // where host/PTP map the cross code page
-   LA64     vmmToHostLA;       // where host has placed the VmmToHost function
   uint16   hostInitial64CS;
   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
@@ -346,7 +337,6 @@ struct VMCrossPageData {

 #if !defined(VMX86_SERVER)
   uint64 ucTimeStamps[UCCOST_MAX];
-   uint8  _ucPad[8];
 #endif

   SwitchedMSRState switchedMSRState;
@@ -409,19 +399,71 @@ struct VMCrossPageData {
 #include "vmware_pack_end.h"
 VMCrossPageData;

-#define CROSSPAGE_VERSION_BASE 0xc12 /* increment by 1 */
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VMCrossPageCode --
+ *
+ *      Code which is stored on the VMCrossPage.
+ *
+ *----------------------------------------------------------------------
+ */
+
+#define CODE_BLOCK_SIZE (PAGE_SIZE - sizeof(VMCrossPageData) - \
+                         (2 + NUM_EXCEPTIONS) * CP_STUB_SIZE)
+
+typedef
+#include "vmware_pack_begin.h"
+struct VMCrossPageCode {
+   uint8         toVmmFunc[CP_STUB_SIZE];    // Fixed-position stubs to jump
+   uint8         toHostFunc[CP_STUB_SIZE];   // to world switch functions
+   uint8         gateStubs[NUM_EXCEPTIONS *
+                           CP_STUB_SIZE];    // Stubs for crossIDT.
+   uint8         codeBlock[CODE_BLOCK_SIZE]; // Code for worldswitch and
+                                             // fault handling.
+}
+#include "vmware_pack_end.h"
+VMCrossPageCode;
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VMCrossPage --
+ *
+ *      Data structure shared between the monitor and the module
+ *      that is used for crossing between the two.
+ *      Accessible as vm->crosspage (kernel module) and CROSS_PAGE
+ *      (monitor)
+ *
+ *      Must be exactly one page long.
+ *
+ *----------------------------------------------------------------------
+ */
+
+typedef
+#include "vmware_pack_begin.h"
+struct VMCrossPage {
+   VMCrossPageData crosspageData;
+   VMCrossPageCode crosspageCode;
+}
+#include "vmware_pack_end.h"
+VMCrossPage;
+
+#define CROSSPAGE_VERSION_BASE 0xc0e /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE             ((VMCrossPageData *) \
-                                VPN_2_VA(CROSS_PAGE_DATA_START))
-#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData * const) \
-                                VPN_2_VA(CROSS_PAGE_DATA_START))
+#define CROSS_PAGE             ((VMCrossPage * const)VPN_2_VA(CROSS_PAGE_START))
+#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
 #endif

 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
-#define MX_WAITNORMAL          1
-#define MX_WAITERROR           0
+#define MX_WAITNORMAL          1  // Must equal one; see linux module code.
+#define MX_WAITERROR           0  // Use MX_ISWAITERROR() to test for error.

+// Any zero or negative value denotes error.
+#define MX_ISWAITERROR(e)      ((e) <= MX_WAITERROR)
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -296,10 +296,9 @@ typedef enum MonLoaderError {
 * Values above shared area subindices for sharing of MonLoaderHeader
 * and htSchedStateMap.
 */
-#define MONLOADER_HEADER_IDX          6
-#define MONLOADER_HT_MAP_IDX          7
-#define MONLOADER_CROSS_PAGE_CODE_IDX 8
-#define MONLOADER_CROSS_PAGE_DATA_IDX 9
+#define MONLOADER_HEADER_IDX 6
+#define MONLOADER_HT_MAP_IDX 7
+

 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
                                 void *args, unsigned *line, Vcpuid *vcpu);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,6 +66,7 @@ typedef enum PCIPassthru_IntrType {
   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)

 typedef struct FPTIntrProxyInfo {
+   uint32            adapterIndex;
   uint32            vectorIndex;
   MonitorIdemAction actionID;
 } FPTIntrProxyInfo;
@@ -75,15 +76,9 @@ typedef struct UPTIntrProxyInfo {
   MonitorIdemAction actionID;
 } UPTIntrProxyInfo;

-typedef struct MPTIntrProxyInfo {
-   uint32            adapterIndex;
-   MonitorIdemAction actionID;
-} MPTIntrProxyInfo;
-
 typedef union PCIPassthru_IntrProxyInfo {
   FPTIntrProxyInfo fpt;
   UPTIntrProxyInfo upt;
-   MPTIntrProxyInfo mpt;
 } PCIPassthru_IntrProxyInfo;

 typedef enum PCIPassthru_ErrorType {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014, 2019-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -133,19 +133,13 @@
 */
 #define PERFCTR_AMD_SMI_COUNT                                  0x2b

-/*
- * AMD Data Cache Events
- * Family <  17H: 0x45 - Unified TLB hit
- *                0x46 - Unified TLB miss
- * Family >= 17H: 0x45 - L1 DTLB miss (L2 DTLB hit or miss)
- *                0x46 - Tablewalker
- */
+/* AMD Data Cache Events */
 #define PERFCTR_AMD_DATA_CACHE_ACCESSES                        0x40
 #define PERFCTR_AMD_DATA_CACHE_MISSES                          0x41
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_L2_OR_SYSTEM       0x42
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_SYSTEM             0x43
 #define PERFCTR_AMD_DATA_CACHE_LINES_EVICTED                   0x44
-#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT_OR_MISS       0x45
+#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT               0x45
 #define PERFCTR_AMD_L1_DTLB_AND_L2_DTLB_MISS                   0x46
 #define PERFCTR_AMD_MISALIGNED_ACCESSES                        0x47
 #define PERFCTR_AMD_PREFETCH_INSTRS_DISPATCHED                 0x4b
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2017,2019-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -125,7 +125,7 @@ PTSC_MHz(void)

 #if defined(VM_X86_64) || defined(VM_ARM_64)

-/*
+/* 
 * Conversions to/from cycles.  Note that the conversions operate on
 * signed values, so be careful when taking the difference of two
 * VmAbsoluteTS (which is unsigned) that that value is not out of range
@@ -197,13 +197,13 @@ VmAbsoluteTS PTSC_Get(void);

 #endif

-/*
+/* 
 *-----------------------------------------------------------------------------
 *
 * PTSC_HasSynchronizedTSCs --
 *
 *      Returns TRUE iff the platform TSCs are known to be synchronized.
- *
+ * 
 *-----------------------------------------------------------------------------
 */

@@ -214,7 +214,7 @@ PTSC_HasSynchronizedTSCs(void)
 }


-/*
+/* 
 *-----------------------------------------------------------------------------
 *
 * PTSC_HostAdjustedTSCs --
@@ -231,37 +231,4 @@ PTSC_HostAdjustedTSCs(void)
   return ptscInfo.hwTSCsAdjusted;
 }

-
-/*
- *----------------------------------------------------------------------
- *
- * PTSC_AdvanceTimer --
- *
- *      Advance '*deadline' in 'period' increments such that it is
- *      greater than 'now'.  Return the number of ticks incremented.
- *
- *----------------------------------------------------------------------
- */
-
-static INLINE uint64
-PTSC_AdvanceTimer(VmAbsoluteTS now,
-                  VmIntervalTS period,
-                  VmAbsoluteTS *deadline)
-{
-   VmAbsoluteTS d = *deadline;
-   if (now >= d) {
-      uint64 count = 1;
-      d += period;
-      if (UNLIKELY(now >= d)) {
-         uint64 t = (now - d) / period + 1;
-         VmIntervalTS diff = t * period;
-         count += t;
-         d += diff;
-      }
-      *deadline = d;
-      return count;
-   }
-   return 0;
-}
-
 #endif /* ifndef _PTSC_H_ */
@@ -1,176 +0,0 @@
-/*********************************************************
- * Copyright (C) 2018-2020 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * segs.h --
- *
- *      Describes segment descriptors and structures containing them for both
- *      the monitor (in hosted and ESX) and the vmkernel.
- */
-
-#ifndef _SEGS_H_
-#define _SEGS_H_
-
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-#include "includeCheck.h"
-
-#include "vm_basic_types.h"
-#include "vm_basic_defs.h"
-#include "x86types.h"
-#include "x86segdescrs.h"
-#include "x86sel.h"
-#include "addrlayout.h"
-
-
-/*
- * Segment map of the monitor.
- *
- * The GDT and Task State Segment reside consecutively on one page.  The
- * monitor segments are placed at the end of the GDT.  The vmkernel can use all
- * lower-numbered segments for user-mode as well as higher-numbered segments
- * (though the vmkernel should not use monitor-private segments).  The high
- * segment placement ensures that there is no selector-overlap with hosted
- * kernel segments (the hosted world switch code can be a bit faster then, as
- * it can use a single cross GDT).
- */
-
-#define GDT_SIZE              (sizeof(Descriptor) * NUM_VALID_SEGMENTS)
-#define GDT_LIMIT             (GDT_SIZE - 1)
-
-#define IRB_SIZE              32 /* Interrupt redirection bitmap. */
-#define TSS_SIZE              (sizeof(Task64) + IRB_SIZE)
-
-#define GDT_START_VA          (VPN_2_VA(GDT_AND_TASK_START))
-#define TASK_START_VA         (VPN_2_VA(GDT_AND_TASK_START) + GDT_SIZE)
-
-/*
- * vmkBoot uses some of the lower-numbered segments, as do host kernels on
- * hosted.  User segments could start earlier than 32; see bug #1904257.
- * Descriptor is used to describe boot, user and kernel segments.
- * Descriptor64 (twice as large) is used for task segments.
- */
-#define NUM_BOOT_SEGMENTS     32
-#define NUM_USER_SEGMENTS     (FIRST_SYSTEM_SEGMENT - FIRST_USER_SEGMENT)
-#define NUM_SYSTEM_SEGMENTS   2
-#define NUM_TASK_SEGMENTS     2
-
-#define FIRST_USER_SEGMENT    NUM_BOOT_SEGMENTS
-#define FIRST_SYSTEM_SEGMENT  (PAGE_SIZE / sizeof(Descriptor) - \
-                               NUM_SYSTEM_SEGMENTS            - \
-                               NUM_TASK_SEGMENTS * 2          - \
-                               TSS_SIZE / sizeof(Descriptor))
-
-#define GDT_USER_TLS_MIN      USER_TLS_1_SEGMENT
-#define GDT_USER_TLS_MAX      USER_TLS_3_SEGMENT
-#define USER_TLS_COUNT        ((USER_TLS_3_SEGMENT - USER_TLS_1_SEGMENT) + 1)
-
-#define FOREACH_USER_TLS_INDEX(_i)              \
-   {                                            \
-      unsigned _i;                              \
-      for (_i = 0; _i < USER_TLS_COUNT; _i++) { \
-
-#define FOREACH_USER_TLS_INDEX_DONE             \
-      }                                         \
-   }
-
-#define NULL_LDTR             0
-
-/*
- * The descriptor after SYSTEM_CODE_SEGMENT (loaded into %cs) must be
- * appropriate for %ss because of the syscall instruction for 64-bit
- * user worlds.  Thus SYSTEM_DATA_SEGMENT is directly after it.
- */
-typedef enum VmwSegs {
-   NULL_SEGMENT             = 0,
-   /* (... reserved for host operating system or vmkBoot segments). */
-
-   USER32_CODE_SEGMENT      = FIRST_USER_SEGMENT,
-   USER_DATA_SEGMENT,
-   USER64_SYSRET_SEGMENT,
-   USER64_STACK_SEGMENT,
-   USER64_CODE_SEGMENT,
-   USER_TLS_1_SEGMENT,
-   USER_TLS_2_SEGMENT,
-   USER_TLS_3_SEGMENT,
-
-   AFTER_LAST_USER_SEGMENT,
-
-   SYSTEM_CODE_SEGMENT      = FIRST_SYSTEM_SEGMENT,
-   SYSTEM_DATA_SEGMENT,
-   VMKERNEL_TASK_SEGMENT,
-   VMKERNEL_TASK_SEGMENT_HI,
-   MONITOR_TASK_SEGMENT,
-   MONITOR_TASK_SEGMENT_HI,
-
-   NUM_VALID_SEGMENTS
-} VmwSegs;
-
-#define GDT_SYSTEM_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 0)
-#define GDT_SYSTEM_SEL_UNCHECKED(x) \
-   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 0)
-#define GDT_USER_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 3)
-#define GDT_USER_SEL_UNCHECKED(x) \
-   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 3)
-
-/* Selectors used statically in code or in assembly must be unchecked. */
-#define SYSTEM_NULL_SELECTOR    GDT_SYSTEM_SEL(NULL)
-#define USER32_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER32_CODE)
-#define USER_DATA_SELECTOR      GDT_USER_SEL_UNCHECKED(USER_DATA)
-#define USER64_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER64_CODE)
-#define USER64_SYSRET_SELECTOR  GDT_USER_SEL(USER64_SYSRET)
-#define SYSTEM_CODE_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_CODE)
-#define SYSTEM_DATA_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_DATA)
-#define MONITOR_TASK_SELECTOR   GDT_SYSTEM_SEL(MONITOR_TASK)
-#define VMKERNEL_TASK_SELECTOR  GDT_SYSTEM_SEL(VMKERNEL_TASK)
-
-/*
- * The VMM GDT is comprised of many segment descriptors with one initial
- * Task State Segment system descriptor.  The VMM Task State Segment is
- * on the same page sequentially after its GDT.
- */
-#pragma pack(push, 1)
-typedef struct StaticGDTAndTSS {
-   Descriptor   empty[NUM_BOOT_SEGMENTS + NUM_USER_SEGMENTS];
-   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS];
-   Descriptor64 vmkTask;
-   Descriptor64 monTask;
-   Task64       monTSS;
-   uint8        TSSIRBitmap[IRB_SIZE];
-} StaticGDTAndTSS;
-#pragma pack(pop)
-
-#pragma pack(push, 1)
-typedef struct VmkernelGDT {
-   Descriptor   bootSegs[NUM_BOOT_SEGMENTS];
-   Descriptor   userSegs[NUM_USER_SEGMENTS];
-   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS]; /* VMM/VMK-shared. */
-   Descriptor64 vmkTask;
-   Descriptor64 monTask;
-} VmkernelGDT;
-#pragma pack(pop)
-
-MY_ASSERTS(segs,
-           ASSERT_ON_COMPILE(SYSTEM_CODE_SEGMENT + 1 == SYSTEM_DATA_SEGMENT);
-           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT - FIRST_USER_SEGMENT <=
-                             NUM_USER_SEGMENTS);
-           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT <= FIRST_SYSTEM_SEGMENT);
-)
-
-#endif /* _SEGS_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -20,12 +20,15 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-UC(CALL_START,             FALSE)
-UC(BEGIN_BACK_TO_HOST,     FALSE)
-UC(SWITCHED_TO_MODULE,     TRUE)
-UC(VMX_HANDLER_START,      FALSE)
-UC(SWITCHING_TO_MONITOR,   TRUE)
-UC(DONE_BACK_TO_HOST,      FALSE)
-UC(CALL_END,               FALSE)
+UC(CALL_START)
+UC(BEGIN_BACK_TO_HOST)
+UC(SWITCHED_TO_MODULE)
+UC(VMX_SELECT_RETURN)
+UC(VMX_HANDLER_START)
+UC(VMX_SIGNAL)
+UC(MODULE_SIGNAL)
+UC(SWITCHING_TO_MONITOR)
+UC(DONE_BACK_TO_HOST)
+UC(CALL_END)

 #undef UC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008,2019-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,6 +31,5 @@
 * failing DeviceIoControl on Windows.
 */
 #define USERCALL_VMX86ALLOCERR (-8192)
-#define USERCALL_SWITCHERR     (-8193)

-#endif
+#endif 
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -60,7 +60,7 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define MAX_SMALL_VM_VCPUS 128

 /* Supported limit. */
-#define MAX_SUPPORTED_VCPUS   (FEATURE_STATE_IS_ENABLED(MonsterVM) ? 768 : 256)
+#define MAX_SUPPORTED_VCPUS   256

 #if defined __cplusplus
 } // extern "C"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,11 +44,10 @@
 #   include "str.h"     /* Str_Snprintf */
 #   define VCS_SNPRINTF Str_Snprintf
 #elif defined MONITOR_APP
+#   include <stdio.h>   /* libc snprintf */
 #   if defined WIN32
-#      include "str.h"     /* Str_Snprintf */
-#      define VCS_SNPRINTF Str_Snprintf
+#      define VCS_SNPRINTF _snprintf
 #   else
-#      include <stdio.h>   /* libc snprintf */
 #      define VCS_SNPRINTF snprintf
 #   endif
 #elif defined VMM || defined VMKERNEL
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -417,13 +417,31 @@ _BUILD_GET_CR(_GET_CR8, 8);
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
 {
+#ifdef VM_X86_64
   _lgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _lgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm lgdt [eax]
+#endif
 }

 static INLINE void
 _Get_GDT(_GETSET_DTR_TYPE *dtr)
 {
+#ifdef VM_X86_64
   _sgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _sgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm sgdt [eax]
+#endif
 }

 static INLINE void
@@ -448,6 +466,141 @@ _Get_IDT(_GETSET_DTR_TYPE *dtr)

 #define RESTORE_FLAGS(x) __writeeflags(x)

+
+#ifdef VM_X86_32
+
+#define SET_TR(_tr)      _Set_TR(_tr)
+#define SET_LDT(_tr)     _Set_LDT(_tr)
+
+#define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
+#define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)
+
+#define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
+#define CLTS()           __asm clts
+
+#define FNCLEX()         __asm fnclex
+
+#define TLB_INVALIDATE_PAGE(_addr) {  \
+   void *_a = (_addr); \
+   __asm mov eax, _a __asm invlpg [eax] \
+}
+
+#define RAISE_INTERRUPT(_x)  {__asm int _x }
+#define RETURN_FROM_INT()   {__asm iretd }
+
+
+static INLINE void
+SET_DS(Selector val)
+{
+   __asm mov ax, val
+   __asm mov ds, ax
+}
+
+static INLINE void
+SET_ES(Selector val)
+{
+   __asm mov ax, val
+   __asm mov es, ax
+}
+
+static INLINE void
+SET_FS(Selector val)
+{
+   __asm mov ax, val
+   __asm mov fs, ax
+}
+
+static INLINE void
+SET_GS(Selector val)
+{
+   __asm mov ax, val
+   __asm mov gs, ax
+}
+
+static INLINE void
+SET_SS(Selector val)
+{
+   __asm mov ax, val
+   __asm mov ss, ax
+}
+
+static INLINE Selector
+GET_FS(void)
+{
+   Selector _v;
+   __asm mov _v,fs
+   return _v;
+}
+
+static INLINE Selector
+GET_GS(void)
+{
+   Selector _v;
+   __asm mov _v,gs
+   return _v;
+}
+
+static INLINE Selector
+GET_DS(void)
+{
+   Selector _v;
+   __asm mov _v,ds
+   return _v;
+}
+
+static INLINE Selector
+GET_ES(void)
+{
+   Selector _v;
+   __asm mov _v,es
+   return _v;
+}
+
+static INLINE Selector
+GET_SS(void)
+{
+   Selector _v;
+   __asm mov _v,ss
+   return _v;
+}
+
+static INLINE Selector
+GET_CS(void)
+{
+   Selector _v;
+   __asm mov _v,cs
+   return _v;
+}
+
+static INLINE void
+_Set_LDT(Selector val)
+{
+   __asm lldt val
+}
+
+static INLINE void
+_Set_TR(Selector val)
+{
+   __asm ltr val
+}
+
+static INLINE Selector
+_Get_LDT(void)
+{
+   Selector sel;
+   __asm sldt sel
+   return sel;
+}
+
+static INLINE Selector
+_Get_TR(void)
+{
+   Selector sel;
+   __asm str sel
+   return sel;
+}
+
+#endif /* !VM_X86_32 */
 #endif /* !__GNUC__ && !_MSC_VER */


@@ -527,7 +680,7 @@ INTERRUPTS_ENABLED(void)
 #endif


-#if defined (__GNUC__)
+#if defined (__GNUC__) || (defined (_MSC_VER) && defined (VM_X86_32))
 static INLINE unsigned
 CURRENT_CPL(void)
 {
@@ -67,6 +67,7 @@ extern "C" {
 #if defined (VMKPANIC) 
 #include "vmk_assert.h"
 #else /* !VMKPANIC */
+#include <linux/kernel.h>
 #define _ASSERT_PANIC(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
 #define _ASSERT_PANIC_BUG(bug, name) \
@@ -107,7 +108,7 @@ NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
 } while(0)

 #else
-NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+#define Panic panic
 #endif

 void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
@@ -41,13 +41,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#pragma warning(push)
-#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
-#include <intrin.h>
-#pragma warning(pop)
-#endif
-
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -152,6 +145,64 @@ typedef ALIGNED(16) struct Atomic_uint128 {
 } Atomic_uint128;
 #endif

+/*
+ * Prototypes for msft atomics.  These are defined & inlined by the
+ * compiler so no function definition is needed.  The prototypes are
+ * needed for C++.
+ *
+ * The declarations for the intrinsic functions were taken from ntddk.h
+ * in the DDK. The declarations must match otherwise the 64-bit C++
+ * compiler will complain about second linkage of the intrinsic functions.
+ * We define the intrinsic using the basic types corresponding to the
+ * Windows typedefs. This avoids having to include windows header files
+ * to get to the windows types.
+ */
+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#ifdef __cplusplus
+extern "C" {
+#endif
+long  _InterlockedExchange(long volatile*, long);
+long  _InterlockedCompareExchange(long volatile*, long, long);
+long  _InterlockedExchangeAdd(long volatile*, long);
+long  _InterlockedDecrement(long volatile*);
+long  _InterlockedIncrement(long volatile*);
+__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
+#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
+#pragma intrinsic(_InterlockedIncrement)
+#pragma intrinsic(_InterlockedCompareExchange64)
+
+# if _MSC_VER >= 1600
+char     _InterlockedExchange8(char volatile *, char);
+char     _InterlockedCompareExchange8(char volatile *, char, char);
+#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
+#endif
+
+#if defined VM_X86_64
+long     _InterlockedAnd(long volatile*, long);
+__int64  _InterlockedAnd64(__int64 volatile*, __int64);
+long     _InterlockedOr(long volatile*, long);
+__int64  _InterlockedOr64(__int64 volatile*, __int64);
+long     _InterlockedXor(long volatile*, long);
+__int64  _InterlockedXor64(__int64 volatile*, __int64);
+__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
+__int64  _InterlockedIncrement64(__int64 volatile*);
+__int64  _InterlockedDecrement64(__int64 volatile*);
+__int64  _InterlockedExchange64(__int64 volatile*, __int64);
+#if !defined _WIN64
+#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
+#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
+#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
+#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
+#endif /* !_WIN64 */
+#endif /* __x86_64__ */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _MSC_VER */
+
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -204,6 +255,15 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

+#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
+uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
+                              uint8 val);
+
+uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
+                                     uint8 newVal,
+                                     uint8 oldVal);
+#endif
+
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -345,8 +405,19 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER
+#elif defined _MSC_VER && _MSC_VER >= 1600
   return _InterlockedExchange8((volatile char *)&var->value, val);
+#elif defined _MSC_VER && defined __i386__
+#pragma warning(push)
+#pragma warning(disable : 4035)         // disable no-return warning
+   {
+      __asm movzx eax, val
+      __asm mov ebx, var
+      __asm xchg [ebx]Atomic_uint8.value, al
+   }
+#pragma warning(pop)
+#elif defined _MSC_VER && defined __x86_64__
+   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -429,9 +500,23 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER
+#elif defined _MSC_VER && _MSC_VER >= 1600
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
+#elif defined _MSC_VER && defined __i386__
+#pragma warning(push)
+#pragma warning(disable : 4035)         // disable no-return warning
+   {
+      __asm mov al, oldVal
+      __asm mov ebx, var
+      __asm mov cl, newVal
+      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
+      __asm movzx eax, al
+      // eax is the return value, this is documented to work - edward
+   }
+#pragma warning(pop)
+#elif defined _MSC_VER && defined __x86_64__
+   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1227,7 +1312,13 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
+#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
+#else
+   __asm mov eax, val
+   __asm mov ebx, var
+   __asm lock And [ebx]Atomic_uint32.value, eax
+#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1286,7 +1377,13 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
+#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
+#else
+   __asm mov eax, val
+   __asm mov ebx, var
+   __asm lock Or [ebx]Atomic_uint32.value, eax
+#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1345,7 +1442,13 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
+#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
+#else
+   __asm mov eax, val
+   __asm mov ebx, var
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
+#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2084,13 +2187,18 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
-   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
-   return _InterlockedCompareExchange64((__int64 *)&var->value,
-                                        (__int64)255,  // Unlikely value to
-                                        (__int64)255); // not dirty cache
+#   pragma warning(push)
+#   pragma warning(disable : 4035)      // disable no-return warning
+   {
+      __asm mov ecx, var
+      __asm mov edx, ecx
+      __asm mov eax, ebx
+      __asm lock cmpxchg8b [ecx]
+      // edx:eax is the return value; this is documented to work. --mann
+   }
+#   pragma warning(pop)
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,13 +35,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#pragma warning(push)
-#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
-#include <intrin.h>
-#pragma warning(pop)
-#endif
-
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -506,7 +499,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" ((uint64)val)
+      : "r" (val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -559,7 +552,6 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
-   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -596,7 +588,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
+      : "+r" (tmpDst), "+r" (count), "+r" (val)
      :
      : "cc", "memory");
 #else
@@ -640,13 +632,22 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_ANY
+#ifdef VM_X86_64
   __stosw((uint16*)dst, val, count);
-#else
+#elif defined(VM_ARM_32)
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
+#else
+   __asm { pushf;
+           mov ax, val;
+           mov ecx, count;
+           mov edi, dst;
+           cld;
+           rep stosw;
+           popf;
+   }
 #endif
   return dst;
 }
@@ -654,13 +655,22 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_ANY
+#ifdef VM_X86_64
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#else
+#elif defined(VM_ARM_32)
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
+#else
+   __asm { pushf;
+           mov eax, val;
+           mov ecx, count;
+           mov edi, dst;
+           cld;
+           rep stosd;
+           popf;
+   }
 #endif
   return dst;
 }
@@ -772,9 +782,17 @@ PAUSE(void)
 #endif
 }
 #elif defined(_MSC_VER)
+#ifdef VM_X86_64
 {
   _mm_pause();
 }
+#else /* VM_X86_64 */
+#pragma warning( disable : 4035)
+{
+   __asm _emit 0xf3 __asm _emit 0x90
+}
+#pragma warning (default: 4035)
+#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -822,11 +840,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_ANY
+#ifdef VM_X86_64
 {
   return __rdtsc();
 }
-#else
+#elif defined(VM_ARM_32)
 {
   /*
    * We need to do more inverstagetion here to find
@@ -835,7 +853,13 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#endif /* VM_X86_ANY */
+#else
+#pragma warning( disable : 4035)
+{
+   __asm _emit 0x0f __asm _emit 0x31
+}
+#pragma warning (default: 4035)
+#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,6 +39,41 @@
 #error "This file is x86-64 only!"
 #endif

+#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+uint64 _umul128(uint64 multiplier, uint64 multiplicand,
+                uint64 *highProduct);
+int64 _mul128(int64 multiplier, int64 multiplicand,
+              int64 *highProduct);
+uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
+#ifdef ULM
+void _fxsave64(void *save);
+void _fxsave(void *save);
+void _fxrstor64(const void *load);
+void _fxrstor(const void *load);
+void _xsave64(void *save, uint64 mask);
+void _xsave(void *save, uint64 mask);
+void _xsaveopt64(void *save, uint64 mask);
+void _xsavec(void *save, uint64 mask);
+void _xrstor64(const void *load, uint64 mask);
+void _xrstor(const void *load, uint64 mask);
+#endif /* ULM */
+#ifdef __cplusplus
+}
+#endif
+
+#pragma intrinsic(_umul128, _mul128, __shiftright128)
+
+#ifdef ULM
+#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
+                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
+#endif /* ULM */
+
+#endif // _MSC_VER
+
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,6 +38,97 @@
 #error "Should be included only in x86 builds"
 #endif

+/*
+ * x86-64 windows doesn't support inline asm so we have to use these
+ * intrinsic functions defined in the compiler.  Not all of these are well
+ * documented.  There is an array in the compiler dll (c1.dll) which has
+ * an array of the names of all the intrinsics minus the leading
+ * underscore.  Searching around in the ntddk.h file can also be helpful.
+ *
+ * The declarations for the intrinsic functions were taken from the DDK.
+ * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
+ * will complain about second linkage of the intrinsic functions.
+ * We define the intrinsic using the basic types corresponding to the
+ * Windows typedefs. This avoids having to include windows header files
+ * to get to the windows types.
+ */
+#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * It seems x86 & x86-64 windows still implements these intrinsic
+ * functions.  The documentation for the x86-64 suggest the
+ * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
+ * __inbyte/__outbyte aren't supported on x86.
+ */
+int            _inp(unsigned short);
+unsigned short _inpw(unsigned short);
+unsigned long  _inpd(unsigned short);
+
+int            _outp(unsigned short, int);
+unsigned short _outpw(unsigned short, unsigned short);
+unsigned long  _outpd(uint16, unsigned long);
+#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
+
+/*
+ * Prevents compiler from re-ordering reads, writes and reads&writes.
+ * These functions do not add any instructions thus only affect
+ * the compiler ordering.
+ *
+ * See:
+ * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
+ * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
+ */
+void _ReadBarrier(void);
+void _WriteBarrier(void);
+void _ReadWriteBarrier(void);
+#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
+
+void _mm_mfence(void);
+void _mm_lfence(void);
+#pragma intrinsic(_mm_mfence, _mm_lfence)
+
+long _InterlockedXor(long volatile *, long);
+#pragma intrinsic(_InterlockedXor)
+
+unsigned int __getcallerseflags(void);
+#pragma intrinsic(__getcallerseflags)
+
+#ifdef VM_X86_64
+/*
+ * intrinsic functions only supported by x86-64 windows as of 2k3sp1
+ */
+unsigned __int64 __rdtsc(void);
+void             __stosw(unsigned short *, unsigned short, size_t);
+void             __stosd(unsigned long *, unsigned long, size_t);
+void             _mm_pause(void);
+#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
+
+unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
+unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
+#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
+#endif /* VM_X86_64 */
+
+unsigned char  _BitScanForward(unsigned long *, unsigned long);
+unsigned char  _BitScanReverse(unsigned long *, unsigned long);
+#pragma intrinsic(_BitScanForward, _BitScanReverse)
+
+unsigned char  _bittest(const long *, long);
+unsigned char  _bittestandset(long *, long);
+unsigned char  _bittestandreset(long *, long);
+unsigned char  _bittestandcomplement(long *, long);
+#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
+#ifdef VM_X86_64
+unsigned char  _bittestandset64(__int64 *, __int64);
+unsigned char  _bittestandreset64(__int64 *, __int64);
+#pragma intrinsic(_bittestandset64, _bittestandreset64)
+#endif // VM_X86_64
+#ifdef __cplusplus
+}
+#endif
+#endif // _MSC_VER
+
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -88,6 +179,9 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

+#define GET_CURRENT_EIP(_eip) \
+      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
+
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -97,35 +191,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE uint8
+static INLINE  uint8
 INB(uint16 port)
 {
-   return __inbyte(port);
+   return (uint8)_inp(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   __outbyte(port, value);
+   _outp(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return __inword(port);
+   return _inpw(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   __outword(port, value);
+   _outpw(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return __indword(port);
+   return _inpd(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   __outdword(port, value);
+   _outpd(port, value);
 }

 #ifndef VM_X86_64
@@ -133,6 +227,11 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

+#define GET_CURRENT_EIP(_eip) do { \
+   __asm call NEAR PTR $+5 \
+   __asm pop eax \
+   __asm mov _eip, eax \
+} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -259,7 +358,11 @@ SFENCE(void)
   );
 #elif defined _MSC_VER
   _ReadWriteBarrier();
+#if defined VM_X86_32
+   __asm sfence;
+#else
   _mm_sfence();
+#endif
   _ReadWriteBarrier();
 #else
 #error No compiler defined for SFENCE
@@ -688,28 +688,22 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-4.1 (circa 2006).  Enforce this here, various things below
+ * gcc-3.3 (circa 2003).  Enforce this here, various things below
 * this line depend upon it.
 *
- * Current oldest compilers:
- * - guest tools: 4.1.2 (freebsd/solaris)
- * - buildhost compiler: 4.4.3
- * - hosted kernel modules: 4.5
- *
- * SWIG's preprocessor is exempt.
+ * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
+ * The various linux kernel modules may use older (gcc-3.3) compilers.
 */
-#ifndef SWIG
-#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
-#error "gcc version is too old, need gcc-4.1 or better"
-#endif
+#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
+#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
 #endif

 /*
- * Similarly, we require a compiler that is at least vs2012.
+ * Similarly, we require a compiler that is at least vc90 (vs2008).
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1700
-#error "cl.exe version is too old, need vs2012 or better"
+#if defined _MSC_VER && _MSC_VER < 1500
+#error "cl.exe version is too old, need vc90 or better"
 #endif


@@ -1,88 +0,0 @@
-/*********************************************************
- * Copyright (C) 2012,2017-2018,2020 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * vm_idt_x86.h --
- *
- *	Definitions for IDT use in x86/x86-64 ESX and hosted VMware Products.
- */
-
-#ifndef _VM_IDT_X86_H_
-#define _VM_IDT_X86_H_
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_VMCORE
-#include "includeCheck.h"
-
-#if defined __cplusplus
-extern "C" {
-#endif
-
-/*
- * User worlds are the only worlds that expect to receive
- * interrupts on a stack specified in the %rsp entries of
- * the TSS.
- */
-#define TSS_RSP0 0
-#define TSS_RSP1 1
-#define TSS_RSP2 2
-
-/*
- * The monitor and the vmkernel use gate stub handlers of a single size in
- * their IDTs.  The only exception is the #PF gate (#14) which is twice as
- * long in the monitor IDT, to copy CR2.
- */
-#define IDT_STUB_SIZE              16
-
-#define IST_NONE                    0
-/* IST entries for the monitor. */
-#define IST_VMM_DF                  1
-#define IST_VMM_NMI                 2
-#define IST_VMM_MCE                 3
-#define MAX_VMM_IST                 3
-/* IST entries for the vmkernel. */
-#define IST_VMK_MCE                 5
-#define IST_VMK_DF                  6
-#define IST_VMK_NMI                 7
-
-
-static INLINE int
-IDT_MonitorISTForVector(int v)
-{
-   /*
-    * For non-SVM execution, the monitor uses the IST for #DF, #NMI, and #MC.
-    */
-   if (v == EXC_DF) {
-      return IST_VMM_DF;
-   } else if (v == EXC_NMI) {
-      return IST_VMM_NMI;
-   } else if (v == EXC_MC) {
-      return IST_VMM_MCE;
-   }
-   return IST_NONE;
-}
-
-#if defined __cplusplus
-} // extern "C"
-#endif
-
-#endif /* _VM_IDT_X86_H_ */
@@ -32,7 +32,6 @@
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMX
 #include "includeCheck.h"
-#include "community_source.h"

 /*
 * Page remapping definitions.
@@ -98,12 +97,10 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

-#define MDEF_DRAINER
-
 #define VMMEM_SERVICES_DEFS                                       \
   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
-   MDEF_DRAINER
+   MDEF(VMMEM_SERVICES_TYPE_DIRTY_CLEAR, DirtyDrainer_ClearDirty)

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,8 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#include <asm/processor-flags.h>
+
 #define X86_MAX_INSTR_LEN  15   /* Max byte length of an x86 instruction. */

 #define NUM_IDT_VECTORS 256
@@ -75,7 +77,9 @@
 #define CR3_PDB_MASK   0xfffff000
 #define CR3_IGNORE     0xFFF
 #define PAE_CR3_IGNORE 0x1F
+#ifndef CR3_PCID_MASK
 #define CR3_PCID_MASK  0xFFF
+#endif
 #define CR3_NO_FLUSH   (1ULL << 63)

 #define CR4_VME        0x00000001
@@ -99,7 +103,6 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
-#define CR4_CET        0x00800000
 #define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

@@ -204,7 +207,6 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
-#define EXC_CP           21  // Control Protection exception.
 #define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

@@ -212,7 +214,7 @@
 #define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
                                (1u << EXC_NP) | (1u << EXC_SS) | \
                                (1u << EXC_GP) | (1u << EXC_PF) | \
-                                (1u << EXC_AC) | (1u << EXC_CP))
+                                (1u << EXC_AC))

 /*
 * eflag/rflag definitions.
@@ -222,24 +224,24 @@

 typedef enum x86_FLAGS {
   EFLAGS_NONE         = 0,
-   EFLAGS_CF           = (1 << 0),     /* User */
-   EFLAGS_SET          = (1 << 1),
-   EFLAGS_PF           = (1 << 2),     /* User */
-   EFLAGS_AF           = (1 << 4),     /* User */
-   EFLAGS_ZF           = (1 << 6),     /* User */
-   EFLAGS_SF           = (1 << 7),     /* User */
-   EFLAGS_TF           = (1 << 8),     /* Priv */
-   EFLAGS_IF           = (1 << 9),     /* Priv */
-   EFLAGS_DF           = (1 << 10),    /* User */
-   EFLAGS_OF           = (1 << 11),    /* User */
-   EFLAGS_NT           = (1 << 14),    /* Priv */
-   EFLAGS_RF           = (1 << 16),    /* Priv */
-   EFLAGS_VM           = (1 << 17),    /* Priv */
-   EFLAGS_AC           = (1 << 18),    /* Priv */
-   EFLAGS_VIF          = (1 << 19),    /* Priv */
-   EFLAGS_VIP          = (1 << 20),    /* Priv */
+   EFLAGS_CF           = (1 << 0),     /* User */ 
+   EFLAGS_SET          = (1 << 1),                                  
+   EFLAGS_PF           = (1 << 2),     /* User */ 
+   EFLAGS_AF           = (1 << 4),     /* User */ 
+   EFLAGS_ZF           = (1 << 6),     /* User */ 
+   EFLAGS_SF           = (1 << 7),     /* User */ 
+   EFLAGS_TF           = (1 << 8),     /* Priv */ 
+   EFLAGS_IF           = (1 << 9),     /* Priv */ 
+   EFLAGS_DF           = (1 << 10),    /* User */ 
+   EFLAGS_OF           = (1 << 11),    /* User */ 
+   EFLAGS_NT           = (1 << 14),    /* Priv */ 
+   EFLAGS_RF           = (1 << 16),    /* Priv */ 
+   EFLAGS_VM           = (1 << 17),    /* Priv */ 
+   EFLAGS_AC           = (1 << 18),    /* Priv */ 
+   EFLAGS_VIF          = (1 << 19),    /* Priv */ 
+   EFLAGS_VIP          = (1 << 20),    /* Priv */ 
   EFLAGS_ID           = (1 << 21),    /* Priv */
-
+   
   EFLAGS_IOPL         = 3 << EFLAGS_IOPL_SHIFT,
   EFLAGS_ARITH        = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_OF),
@@ -251,7 +253,7 @@ typedef enum x86_FLAGS {
   EFLAGS_ALL          = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_DF | EFLAGS_OF | EFLAGS_TF |
                          EFLAGS_IF | EFLAGS_IOPL | EFLAGS_NT | EFLAGS_RF |
-                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP |
+                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP | 
                          EFLAGS_ID),
   EFLAGS_ALL_16       = EFLAGS_ALL & 0xffff,
   EFLAGS_REAL_32      = (EFLAGS_ALL & ~(EFLAGS_VIP | EFLAGS_VIF | EFLAGS_VM)),
@@ -141,7 +141,7 @@ CPUIDQuery;
   CPUIDLEVEL(TRUE,  18,  0x18,       8, 17) \
   CPUIDLEVEL(TRUE,  1A,  0x1a,       0, 17) \
   CPUIDLEVEL(TRUE,  1B,  0x1b,       2, 17) \
-   CPUIDLEVEL(FALSE, 1F,  0x1f,       6, 17) \
+   CPUIDLEVEL(TRUE,  1F,  0x1f,       6, 17) \
   CPUIDLEVEL(FALSE, 400, 0x40000000, 0,  0) \
   CPUIDLEVEL(FALSE, 401, 0x40000001, 0,  0) \
   CPUIDLEVEL(FALSE, 402, 0x40000002, 0,  0) \
@@ -481,7 +481,7 @@ FIELD(  6,  0, EDX, 16, 16, HW_FEEDBACK_INDEX,                   NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_7                                            \
-FIELD(  7,  0, EAX,  0, 32, LEAF_7_MAX_SUBLEVEL,                 YES,  18 ) \
+FIELD(  7,  0, EAX,  0, 32, LEAF_7_MAX_SUBLEVEL,                 YES, FUT ) \
 FLAG(   7,  0, EBX,  0,  1, FSGSBASE,                            YES,   9 ) \
 FLAG(   7,  0, EBX,  1,  1, TSC_ADJUST,                          ANY,  11 ) \
 FLAG(   7,  0, EBX,  2,  1, SGX,                                 ANY,  17 ) \
@@ -530,20 +530,18 @@ FLAG(   7,  0, ECX, 14,  1, AVX512VPOPCNTDQ,                     YES,  16 ) \
 FLAG(   7,  0, ECX, 16,  1, VA57,                                NO,    0 ) \
 FIELD(  7,  0, ECX, 17,  5, MAWA,                                NO,    0 ) \
 FLAG(   7,  0, ECX, 22,  1, RDPID,                               YES,  17 ) \
-FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            YES,  18 ) \
-FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             YES,  18 ) \
-FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           YES,  18 ) \
+FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            YES, FUT ) \
+FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             YES, FUT ) \
+FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           YES, FUT ) \
 FLAG(   7,  0, ECX, 29,  1, ENQCMD,                              NO,    0 ) \
 FLAG(   7,  0, ECX, 30,  1, SGX_LC,                              ANY,  17 ) \
 FLAG(   7,  0, EDX,  2,  1, AVX512QVNNIW,                        YES,  16 ) \
 FLAG(   7,  0, EDX,  3,  1, AVX512QFMAPS,                        YES,  16 ) \
-FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   YES,  18 ) \
-FLAG(   7,  0, EDX,  8,  1, AVX512VP2INTERSECT,                  YES,  18 ) \
+FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   YES, FUT ) \
+FLAG(   7,  0, EDX,  8,  1, AVX512VP2INTERSECT,                  YES, FUT ) \
 FLAG(   7,  0, EDX, 10,  1, MDCLEAR,                             YES,   9 ) \
 FLAG(   7,  0, EDX, 13,  1, TSX_MICROCODE_UPDATE,                NO,    0 ) \
-FLAG(   7,  0, EDX, 14,  1, SERIALIZE,                           NO,    0 ) \
 FLAG(   7,  0, EDX, 15,  1, HYBRID,                              NO,    0 ) \
-FLAG(   7,  0, EDX, 16,  1, TSXLDTRK,                            NO,    0 ) \
 FLAG(   7,  0, EDX, 18,  1, PCONFIG,                             NO,    0 ) \
 FLAG(   7,  0, EDX, 20,  1, CET_IBT,                             NO,    0 ) \
 FLAG(   7,  0, EDX, 26,  1, IBRSIBPB,                            ANY,   9 ) \
@@ -552,7 +550,7 @@ FLAG(   7,  0, EDX, 28,  1, FCMD,                                YES,   9 ) \
 FLAG(   7,  0, EDX, 29,  1, ARCH_CAPABILITIES,                   ANY,   9 ) \
 FLAG(   7,  0, EDX, 30,  1, CORE_CAPABILITIES,                   NO,    0 ) \
 FLAG(   7,  0, EDX, 31,  1, SSBD,                                YES,   9 ) \
-FLAG(   7,  1, EAX,  5,  1, AVX512BF16,                          YES,  18 )
+FLAG(   7,  1, EAX,  5,  1, AVX512BF16,                          YES, FUT )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_9                                            \
@@ -791,11 +789,11 @@ FIELD( 1B,  0, EDX,  0, 32, PCONFIG_TARGET_ID3,                  NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_1F                                           \
-FIELD( 1F,  0, EAX,  0,  5, TOPOLOGY_V2_MASK_WIDTH,              NA,    0 ) \
-FIELD( 1F,  0, EBX,  0, 16, TOPOLOGY_V2_CPUS_SHARING_LEVEL,      NA,    0 ) \
-FIELD( 1F,  0, ECX,  0,  8, TOPOLOGY_V2_LEVEL_NUMBER,            NA,    0 ) \
-FIELD( 1F,  0, ECX,  8,  8, TOPOLOGY_V2_LEVEL_TYPE,              NA,    0 ) \
-FIELD( 1F,  0, EDX,  0, 32, TOPOLOGY_V2_X2APIC_ID,               NA,    0 )
+FIELD( 1F,  0, EAX,  0,  5, TOPOLOGY_V2_MASK_WIDTH,              NO,    0 ) \
+FIELD( 1F,  0, EBX,  0, 16, TOPOLOGY_V2_CPUS_SHARING_LEVEL,      NO,    0 ) \
+FIELD( 1F,  0, ECX,  0,  8, TOPOLOGY_V2_LEVEL_NUMBER,            NO,    0 ) \
+FIELD( 1F,  0, ECX,  8,  8, TOPOLOGY_V2_LEVEL_TYPE,              NO,    0 ) \
+FIELD( 1F,  0, EDX,  0, 32, TOPOLOGY_V2_X2APIC_ID,               NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_400                                          \
@@ -909,6 +907,9 @@ FIELD( 80,  0, EBX,  0, 32, LEAF80_VENDOR1,                      NA,    0 ) \
 FIELD( 80,  0, ECX,  0, 32, LEAF80_VENDOR3,                      NA,    0 ) \
 FIELD( 80,  0, EDX,  0, 32, LEAF80_VENDOR2,                      NA,    0 )

+#define CPUID_81_ECX_14 \
+FLAG(  81,  0, ECX, 14,  1, LEAF81ECX_RSVD1,                     NO,    0 )
+
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_81                                           \
 FIELD( 81,  0, EAX,  0,  4, LEAF81_STEPPING,                     ANY,   4 ) \
@@ -933,12 +934,13 @@ FLAG(  81,  0, ECX, 10,  1, IBS,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 11,  1, XOP,                                 YES,   8 ) \
 FLAG(  81,  0, ECX, 12,  1, SKINIT,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 13,  1, WATCHDOG,                            NO,    0 ) \
+CPUID_81_ECX_14 \
 FLAG(  81,  0, ECX, 15,  1, LWP,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 16,  1, FMA4,                                YES,   8 ) \
 FLAG(  81,  0, ECX, 17,  1, TCE,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 19,  1, NODEID_MSR,                          NO,    0 ) \
 FLAG(  81,  0, ECX, 21,  1, TBM,                                 YES,   9 ) \
-FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                            ANY,  18 ) \
+FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                            NO,    0 ) \
 FLAG(  81,  0, ECX, 23,  1, PERFCORE,                            ANY,   4 ) \
 FLAG(  81,  0, ECX, 24,  1, PERFNB,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 26,  1, DATABK,                              NO,    0 ) \
@@ -1065,29 +1067,21 @@ FIELD( 88,  0, EAX, 16,  8, GUEST_PHYS_ADDR_SZ,                  YES,   8 ) \
 FLAG(  88,  0, EBX,  0,  1, CLZERO,                              YES,  14 ) \
 FLAG(  88,  0, EBX,  1,  1, IRPERF,                              NO,    0 ) \
 FLAG(  88,  0, EBX,  2,  1, XSAVE_ERR_PTR,                       NO,    0 ) \
-FLAG(  88,  0, EBX,  3,  1, INVLPGB,                             NO,    0 ) \
 FLAG(  88,  0, EBX,  4,  1, RDPRU,                               NO,    0 ) \
 FLAG(  88,  0, EBX,  6,  1, MBE,                                 NO,    0 ) \
-FLAG(  88,  0, EBX,  8,  1, MCOMMIT,                             NO,    0 ) \
 FLAG(  88,  0, EBX,  9,  1, WBNOINVD,                            YES,  17 ) \
-FLAG(  88,  0, EBX, 10,  1, LBREXTN,                             NO,    0 ) \
 FLAG(  88,  0, EBX, 12,  1, LEAF88_IBPB,                         ANY,   9 ) \
-FLAG(  88,  0, EBX, 13,  1, WBINVD_INT,                          NO,    0 ) \
 FLAG(  88,  0, EBX, 14,  1, LEAF88_IBRS,                         NO,    0 ) \
 FLAG(  88,  0, EBX, 15,  1, LEAF88_STIBP,                        NO,    0 ) \
 FLAG(  88,  0, EBX, 16,  1, LEAF88_IBRS_ALWAYS,                  NO,    0 ) \
 FLAG(  88,  0, EBX, 17,  1, LEAF88_STIBP_ALWAYS,                 NO,    0 ) \
 FLAG(  88,  0, EBX, 18,  1, LEAF88_PREFER_IBRS,                  NO,    0 ) \
-FLAG(  88,  0, EBX, 19,  1, LEAF88_IBRS_SAME_MODE,               NO,    0 ) \
-FLAG(  88,  0, EBX, 20,  1, LMSLE_UNSUPPORTED,                   NO,    0 ) \
-FLAG(  88,  0, EBX, 23,  1, PPIN,                                NO,    0 ) \
 FLAG(  88,  0, EBX, 24,  1, LEAF88_SSBD_SPEC_CTRL,               ANY,   9 ) \
 FLAG(  88,  0, EBX, 25,  1, LEAF88_SSBD_VIRT_SPEC_CTRL,          ANY,   9 ) \
 FLAG(  88,  0, EBX, 26,  1, LEAF88_SSBD_NOT_NEEDED,              NO,    0 ) \
 FIELD( 88,  0, ECX,  0,  8, LEAF88_CORE_COUNT,                   YES,   4 ) \
 FIELD( 88,  0, ECX, 12,  4, APICID_COREID_SIZE,                  YES,   7 ) \
 FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                        NO,    0 ) \
-FIELD( 88,  0, EDX,  0, 16, INVLPGB_MAX,                         NO,    0 ) \
 FIELD( 88,  0, EDX, 16,  8, RDPRU_MAX,                           NO,    0 )

 #define CPUID_8A_EDX_11 \
@@ -1121,9 +1115,7 @@ FLAG(  8A,  0, EDX, 16,  1, SVM_VGIF,                            NO,    0 ) \
 FLAG(  8A,  0, EDX, 17,  1, SVM_GMET,                            YES,  17 ) \
 FIELD( 8A,  0, EDX, 18,  2, SVMEDX_RSVD3,                        NO,    0 ) \
 FLAG(  8A,  0, EDX, 20,  1, SVM_GUEST_SPEC_CTRL,                 NO,    0 ) \
-FIELD( 8A,  0, EDX, 21,  3, SVMEDX_RSVD4,                        NO,    0 ) \
-FLAG(  8A,  0, EDX, 24,  1, SVM_TLB_CTL,                         NO,    0 ) \
-FIELD( 8A,  0, EDX, 25,  7, SVMEDX_RSVD5,                        NO,    0 )
+FIELD( 8A,  0, EDX, 21, 11, SVMEDX_RSVD4,                        NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_819                                          \
@@ -1214,23 +1206,18 @@ FIELD(81E,  0, EBX,  8,  2, CORES_PER_COMPUTE_UNIT,              NA,    0 ) \
 FIELD(81E,  0, ECX,  0,  8, NODEID_VAL,                          NA,    0 ) \
 FIELD(81E,  0, ECX,  8,  3, NODES_PER_PKG,                       NA,    0 )

+#define CPUID_81F_EAX_10 \
+FLAG( 81F,  0, EAX, 10,  1, SVMEAX_RSVD1,                        NO,    0 )
+
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_81F                                          \
 FLAG( 81F,  0, EAX,  0,  1, SME,                                 NO,    0 ) \
 FLAG( 81F,  0, EAX,  1,  1, SEV,                                 YES,  17 ) \
 FLAG( 81F,  0, EAX,  2,  1, PAGE_FLUSH_MSR,                      NO,    0 ) \
 FLAG( 81F,  0, EAX,  3,  1, SEV_ES,                              YES,  17 ) \
-FLAG( 81F,  0, EAX,  4,  1, SEV_SNP,                             NO,    0 ) \
-FLAG( 81F,  0, EAX,  5,  1, VMPL,                                NO,    0 ) \
-FLAG( 81F,  0, EAX, 10,  1, SEV_HEC,                             NO,    0 ) \
-FLAG( 81F,  0, EAX, 11,  1, SEV_64BIT_REQ,                       NO,    0 ) \
-FLAG( 81F,  0, EAX, 12,  1, SEV_RESTR_INJECTION,                 NO,    0 ) \
-FLAG( 81F,  0, EAX, 13,  1, SEV_ALT_INJECTION,                   NO,    0 ) \
-FLAG( 81F,  0, EAX, 14,  1, SEV_DEBUG_SWAP,                      NO,    0 ) \
-FLAG( 81F,  0, EAX, 15,  1, SEV_NO_HOST_IBS,                     NO,    0 ) \
+CPUID_81F_EAX_10 \
 FIELD(81F,  0, EBX,  0,  6, SME_PAGE_TABLE_BIT_NUM,              YES,  17 ) \
 FIELD(81F,  0, EBX,  6,  6, SME_PHYS_ADDR_SPACE_REDUCTION,       NO,    0 ) \
-FIELD(81F,  0, EBX, 12,  4, NUM_VMPL,                            NO,    0 ) \
 FIELD(81F,  0, ECX,  0, 32, NUM_ENCRYPTED_GUESTS,                NO,    0 ) \
 FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,6 +39,31 @@
 #include "x86cpuid.h"


+/*
+ * The declarations for the intrinsic functions were taken from MSDN.
+ * We define the intrinsic using the basic types corresponding to the
+ * Windows typedefs. This avoids having to include windows header files
+ * to get to the windows types.
+ */
+#ifdef _MSC_VER
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * __cpuid has been supported since VS2003
+ * __cpuidex has been supported since VS2008
+ */
+void __cpuid(int regs[4], int eax);
+void __cpuidex(int regs[4], int eax, int ecx);
+#pragma intrinsic(__cpuid)
+#pragma intrinsic(__cpuidex)
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _MSC_VER */
+
+
 #ifdef __GNUC__ // {

 /*
@@ -24,6 +24,7 @@

 #ifndef _X86MSR_H_
 #define _X86MSR_H_
+#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX

@@ -113,20 +114,6 @@ MSRQuery;
 #define MSR_TSC_AUX           0xc0000103
 #define MSR_BD_TSC_RATIO      0xc0000104

-/* CET MSRs */
-#define MSR_U_CET                            0x6a0
-#define MSR_S_CET                            0x6a2
-#define MSR_CET_SH_STK_EN                         (1ULL << 0)
-#define MSR_CET_WR_SHSTK_EN                       (1ULL << 1)
-#define MSR_PL0_SSP                          0x6a4
-#define MSR_PL1_SSP                          0x6a5
-#define MSR_PL2_SSP                          0x6a6
-#define MSR_PL3_SSP                          0x6a7
-#define MSR_ISST_ADDR                        0x6a8
-
-#define MSR_TEST_CTRL                        0x33
-#define MSR_TEST_CTRL_SPLIT_LOCK_DETECT           (1ULL << 29)
-
 #define IA32_MSR_ARCH_CAPABILITIES           0x10a
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
@@ -151,7 +138,9 @@ MSRQuery;
 #define MSR_TSX_CTRL_RTM_DISABLE                  (1ULL << 0)
 #define MSR_TSX_CTRL_CPUID_CLEAR                  (1ULL << 1)

+#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
+#endif

 /* Intel Core Architecture and later: use only architected counters. */
 #define IA32_MSR_PERF_CAPABILITIES                   0x345
@@ -486,7 +475,9 @@ MSRQuery;
 #define MSR_K7_HWCR_SSEDIS         0x00008000ULL // Disable SSE bit
 #define MSR_K7_HWCR_MONMWAITUSEREN 0x00000400ULL // Enable MONITOR/MWAIT CPL>0
 #define MSR_K7_HWCR_TLBFFDIS       0x00000040ULL // Disable TLB Flush Filter
+#ifndef MSR_K7_HWCR_SMMLOCK
 #define MSR_K7_HWCR_SMMLOCK        0x00000001ULL // Lock SMM environment
+#endif

 #ifndef MSR_K8_SYSCFG
 #define MSR_K8_SYSCFG        0xc0010010
@@ -494,8 +485,6 @@ MSRQuery;
 #define MSR_K8_SYSCFG_MTRRTOM2EN         (1ULL<<21)
 #define MSR_K8_SYSCFG_TOM2FORCEMEMTYPEWB (1ULL<<22)
 #define MSR_K8_SYSCFG_SMEE               (1ULL<<23)
-#define MSR_K8_SYSCFG_SNPE               (1ULL<<24)
-#define MSR_K8_SYSCFG_VMPLE              (1ULL<<25)

 #define MSR_K8_TOPMEM2       0xc001001d

@@ -537,22 +526,13 @@ MSRQuery;
 /* SEV related MSRs. */
 #define MSR_VMPAGE_FLUSH           0xc001011e
 #define MSR_GHCB_PA                0xc0010130
-#define MSR_GHCB_PA_FUNCTION_MASK       0xfff
 #define MSR_GHCB_PA_SEVINFO_HV              1
 #define MSR_GHCB_PA_SEVINFO_REQ             2
 #define MSR_GHCB_PA_AP_JUMP_TABLE           3
-#define MSR_GHCB_PA_CPUID_REQ               4
-#define MSR_GHCB_PA_CPUID_RESP              5
-#define MSR_GHCB_PA_TERMINATE             256
 #define MSR_SEV_STATUS             0xc0010131

 #define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
 #define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
-#define MSR_SEV_STATUS_SEV_SNP_EN  0x0000000000000004ULL // SEV-SNP is enabled
-
-/* SEV-SNP (Secure Nested Paging) MSRs. */
-#define MSR_RMP_BASE              0xc0010132 // Address of first byte of RMP
-#define MSR_RMP_END               0xc0010133 // Address of last byte of RMP

 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)
@@ -658,7 +638,9 @@ MSRQuery;
 #define MSR_HYPERV_GUESTOSID_OS_WINNT_DERIVATIVE 4ULL

 /* MSR for forcing RTM abort to recover PMC3 (see PR 2333817) */
+#ifndef MSR_TSX_FORCE_ABORT
 #define MSR_TSX_FORCE_ABORT                      0x0000010f
+#endif
 #define MSR_TSX_FORCE_ABORT_RTM_BIT_INDEX        0

 /*
@@ -712,7 +694,11 @@ typedef unsigned char MTRRType;
 /*
 * MISC_FEATURES_ENABLES bits
 */
+#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
+#endif



@@ -821,6 +807,7 @@ X86MSR_SetMSR(uint32 cx, uint64 value)
 }
 #endif
 #elif defined _MSC_VER // !__GNUC__ && _MSC_VER
+#ifdef _WIN64
 unsigned __int64  __readmsr(unsigned long);
 void              __writemsr(unsigned long, unsigned __int64);
 #pragma intrinsic(__readmsr, __writemsr)
@@ -835,6 +822,37 @@ X86MSR_SetMSR(uint32 cx, uint64 value)
 {
   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
 }
+#else // !__GNUC__ && _MSC_VER && !_WIN64
+#pragma warning( disable : 4035)
+static INLINE uint64
+X86MSR_GetMSR(uint32 input)
+{
+   __asm push ecx
+   __asm mov  ecx, input
+   /* 0x0f 0x32 -> edx:eax = rdmsr[ecx] */
+   __asm _emit 0x0f __asm _emit 0x32
+   __asm pop ecx
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 input, uint64 value)
+{
+      uint32 hival = (uint32)((value) >> 32);
+      uint32 loval = (uint32)value;
+      __asm push edx
+      __asm push ecx
+      __asm push eax
+      __asm mov  eax, loval
+      __asm mov  edx, hival
+      __asm mov  ecx, input
+      /* 0x0f 0x30 -> wrmsr[ecx] = edx:eax */
+      __asm _emit 0x0f __asm _emit 0x30
+      __asm pop  eax
+      __asm pop  ecx
+      __asm pop  edx
+}
+#pragma warning (default: 4035)
+#endif
 #else // !__GNUC__ && !_MSC_VER
 #error No compiler defined for RDMSR/WRMSR.
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,7 +40,6 @@
 #include "x86types.h"
 #include "vm_pagetable.h"
 #include "x86paging_common.h"
-#include "vm_assert.h"

 #define LM_PTE_PFN_MASK      CONST64U(0xffffffffff000)
 #define LM_PTE_2_PFN(_pte)   (((_pte) & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
@@ -137,95 +136,4 @@ LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
 #define VA64_CANONICAL_HOLE_START   (CONST64U(1) << (VA64_IMPL_BITS - 1))
 #define VA64_CANONICAL_HOLE_LEN  VA64_CANONICAL_MASK - VA64_CANONICAL_HOLE_START

-static INLINE Bool
-x86IsCanonicalC(VA64 va)
-{
-   return (va & VA64_CANONICAL_MASK) == 0 ||
-          (va & VA64_CANONICAL_MASK) == VA64_CANONICAL_MASK;
-}
-
-#if defined(VM_X86_64) && defined(__GNUC__)
-static INLINE Bool
-x86IsCanonicalAsm(VA64 va)
-{
-   Bool out;
-   /*
-    * sarq $48, %0: Move 17 bits from position 63:47 into 15:0 and CF,
-    *  sign-extending as we do so.
-    * adcl $0, %0: Then add (0 + CF) to the shifted value.  The result is zero
-    *  iff (CF == 1 && bits 31:0 == 0xffffffff) or (CF == 0 && bits 31:0 == 0).
-    *  That is, if original bits 47 and higher were all 1s or all 0s.
-    * setz %1: Create the boolean.
-    *
-    * The chain of AdcLong macros will emit one adc instruction; gcc
-    * has no obvious way to force the size of a quad %0 to a long.
-    */
-#define AdcLong(reg64, reg32) \
-   ".ifc  %0, %%" #reg64 "\n" \
-   " adcl $0, %%" #reg32 "\n" \
-   ".endif \n"
-
-   asm ("sarq $48, %0 \n"
-        AdcLong(rax, eax)
-        AdcLong(rcx, ecx)
-        AdcLong(rdx, edx)
-        AdcLong(rbx, ebx)
-        AdcLong(rsp, esp)
-        AdcLong(rbp, ebp)
-        AdcLong(rsi, esi)
-        AdcLong(rdi, edi)
-        AdcLong(r8,  r8d)
-        AdcLong(r9,  r9d)
-        AdcLong(r10, r10d)
-        AdcLong(r11, r11d)
-        AdcLong(r12, r12d)
-        AdcLong(r13, r13d)
-        AdcLong(r14, r14d)
-        AdcLong(r15, r15d)
-        "setz %1" : "+r"(va), "=r"(out));
-   return out;
-#undef AdcLong
-}
-#endif
-
-static INLINE Bool
-x86_IsCanonical(VA64 va)
-{
-#if defined(VM_X86_64) && defined(__GNUC__)
-   if (__builtin_constant_p(va)) {
-      return x86IsCanonicalC(va);
-   } else {
-      return x86IsCanonicalAsm(va);
-   }
-#else
-   return x86IsCanonicalC(va);
-#endif
-}
-
-static INLINE Bool
-x86_IsCanonicalRange(VA64 va, unsigned size)
-{
-   /*
-    * The check is simple as long as the size is less
-    * than the number of implemented bits.
-    *
-    * The only case we don't handle is one where the VA goes from a
-    * high canonical address and wraps to a non-canonical address
-    * (e.g. 0x00008000_00000000) or higher.  Our test would falsely
-    * consider this canonical.
-    */
-   ASSERT_ON_COMPILE(sizeof(size) * 8 < VA64_IMPL_BITS);
-
-   /*
-    * VA64_CANONICAL_MASK is the lowest canonical address with the
-    * upper bits all set.
-    *
-    * VA64_CANONICAL_HOLE_START is one higher than the highest valid
-    * canonical address with the upper bits all cleared.  Note that we
-    * access up to (va + size - 1), not (va + size), so <= is correct.
-    */
-   return va >= VA64_CANONICAL_MASK ||
-          va + size <= VA64_CANONICAL_HOLE_START;
-}
-
 #endif /* _X86PAGING_64_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,7 +92,6 @@
 #define PF_RSVD         0x0008
 #define PF_ID           0x0010
 #define PF_PK           0x0020
-#define PF_SS           0x0040
 #define PF_SGX          0x8000

 /*
@@ -110,12 +109,4 @@ typedef struct {
   uint64   la;      /* Used for single address invalidation.  */
 } InvpcidArg;

-typedef enum {
-   PM_FLAT,        /* non paging mode                                      */
-   PM_2_LEVEL,     /* classic 32 bit non-pae style paging mode.            */
-   PM_3_LEVEL,     /* classic 32 bit pae style paging mode.                */
-   PM_4_LEVEL,     /* long mode style paging mode (including compat mode). */
-   PM_NUM     = 4, /* number of different paging modes.                    */
-} PagingMode;
-
 #endif /* _X86PAGING_COMMON_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014,2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,7 +39,6 @@
 extern "C" {
 #endif

-#include "vm_assert.h"

 #define SELECTOR_GDT             0
 #define SELECTOR_LDT             1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -95,10 +95,9 @@
 *       _V  -- virtualized
 *       _NV -- not virtualized
 *   _access is one of:
- *       _NA  -- no access
- *       _RW  -- read/write access
- *       _RO  -- read-only access
- *       _URW -- read/write acess for the ULM only
+ *       _NA -- no access
+ *       _RW -- read/write access
+ *       _RO -- read-only access
 *   _sticky is one of:
 *       _S  -- sticky
 *       _NS -- non-sticky
@@ -110,9 +109,9 @@
 VMCS_SET_START(16)
 /* 16-bit control fields. */
 VMCS_GROUP_START(16, CTL)
-VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _URW,  _S)
-VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV,  _NA,  _S)
-VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _URW, _NS)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
 VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
 VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
@@ -329,16 +328,16 @@ VMCS_GROUP_END(64, DATA)

 /* 64-bit guest state. */
 VMCS_GROUP_START(64, GUEST)
-VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V,  _NA, _NS)
-VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V,  _NA, _NS)
-VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V,  _NA, _NS)
-VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _URW, _NS)
-VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V,  _NA, _NS)
-VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA, _NS)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
 VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
 VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
@@ -404,24 +403,24 @@ VMCS_SET_END(64)
 VMCS_SET_START(32)
 /* 32-bit control fields. */
 VMCS_GROUP_START(32, CTL)
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _URW,  _S)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _URW,  _S)
-VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _URW,  _S)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _URW,  _S)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _URW,  _S)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _URW, _NS)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _URW,  _S)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _URW,  _S)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V,  _NA,  _S)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _URW,  _S)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA,  _S)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA, _NS)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA,  _S)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA,  _S)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
 VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
 VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
@@ -552,14 +551,14 @@ VMCS_SET_END(32)
 VMCS_SET_START(NAT)
 /* natural-width control fields. */
 VMCS_GROUP_START(NAT, CTL)
-VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _URW,  _S)
-VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _URW,  _S)
-VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
 VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
 VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
@@ -623,26 +622,26 @@ VMCS_GROUP_END(NAT, DATA)

 /* natural-width guest state. */
 VMCS_GROUP_START(NAT, GUEST)
-VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _URW, _NS)
-VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _URW, _NS)
-VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V,  _RW, _NS)
-VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW, _NS)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW, _NS)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA, _NS)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW, _NS)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW, _NS)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW, _NS)
 VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
 VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
 VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
@@ -659,18 +658,18 @@ VMCS_GROUP_END(NAT, GUEST)

 /* natural-width host state. */
 VMCS_GROUP_START(NAT, HOST)
-VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _URW,  _S)
-VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V,  _NA,  _S)
-VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _URW,  _S)
-VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
 VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
 VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
@@ -23,7 +23,6 @@

 #include <linux/file.h>
 #include <linux/highmem.h>
-#include <linux/miscdevice.h>
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/slab.h>
@@ -37,8 +36,8 @@

 #include "usercalldefs.h"

-#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0)
-#error Linux kernels before 3.10 are not supported
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
+#error Linux kernels before 2.6.32 are not supported
 #endif

 #include <asm/io.h>
@@ -97,22 +96,7 @@ static int LinuxDriver_Close(struct inode *inode, struct file *filp);

 static unsigned int LinuxDriverEstimateTSCkHz(void);

-static const struct file_operations vmuser_fops = {
-   .owner = THIS_MODULE,
-   .open = LinuxDriver_Open,
-   .release = LinuxDriver_Close,
-   .unlocked_ioctl = LinuxDriver_Ioctl,
-   .compat_ioctl = LinuxDriver_Ioctl,
-};
-
-#ifndef VMX86_DEVEL
-static struct miscdevice vmmon_miscdev = {
-   .name = "vmmon",
-   .minor = MISC_DYNAMIC_MINOR,
-   .fops = &vmuser_fops,
-};
-#endif
-
+static struct file_operations vmuser_fops;
 static struct timer_list tscTimer;
 static Atomic_uint32 tsckHz;
 static VmTimeStart tsckHzStartTime;
@@ -267,8 +251,9 @@ LinuxDriverInitTSCkHz(void)
 *      linux module entry point. Called by /sbin/insmod command
 *
 * Results:
- *      Release: registers a device driver with a misc minor node.
- *      Devel: registers for a major number with user-created node.
+ *      registers a device driver for a major # that depends
+ *      on the uid. Add yourself to that list.  List is now in
+ *      private/driver-private.c.
 *
 *----------------------------------------------------------------------
 */
@@ -295,43 +280,46 @@ init_module(void)
   if (!Vmx86_CreateHVIOBitmap()) {
      return -ENOMEM;
   }
-
-   if (!Vmx86_CheckMSRUniformity()) {
-      return -EPERM;
-   }
-
   linuxState.fastClockThread = NULL;
   linuxState.fastClockRate = 0;

+   /*
+    * Initialize the file_operations structure. Because this code is always
+    * compiled as a module, this is fine to do it here and not in a static
+    * initializer.
+    */
+
+   memset(&vmuser_fops, 0, sizeof vmuser_fops);
+   vmuser_fops.owner = THIS_MODULE;
+   vmuser_fops.unlocked_ioctl = LinuxDriver_Ioctl;
+   vmuser_fops.compat_ioctl = LinuxDriver_Ioctl;
+   vmuser_fops.open = LinuxDriver_Open;
+   vmuser_fops.release = LinuxDriver_Close;
+
 #ifdef VMX86_DEVEL
   devel_init_module();
+   linuxState.minor = 0;
   retval = register_chrdev(linuxState.major, linuxState.deviceName,
                            &vmuser_fops);
-   if (retval) {
-      Warning("Module %s: error registering with major=%d\n",
-              linuxState.deviceName, linuxState.major);
-   } else {
-      Log("Module %s: registered with major=%d\n",
-          linuxState.deviceName, linuxState.major);
-   }
 #else
   sprintf(linuxState.deviceName, "vmmon");
   linuxState.major = 10;
+   linuxState.minor = 165;
+   linuxState.misc.minor = linuxState.minor;
+   linuxState.misc.name = linuxState.deviceName;
+   linuxState.misc.fops = &vmuser_fops;

-   retval = misc_register(&vmmon_miscdev);
-   if (retval) {
-      Warning("Module %s: error registering misc device %s\n",
-              linuxState.deviceName, vmmon_miscdev.name);
-   } else {
-      Log("Module %s: registered as misc device %s\n", linuxState.deviceName,
-          vmmon_miscdev.name);
-   }
+   retval = misc_register(&linuxState.misc);
 #endif

   if (retval) {
+      Warning("Module %s: error registering with major=%d minor=%d\n",
+              linuxState.deviceName, linuxState.major, linuxState.minor);
      Vmx86_CleanupHVIOBitmap();
      return -ENOENT;
   }
+   Log("Module %s: registered with major=%d minor=%d\n",
+       linuxState.deviceName, linuxState.major, linuxState.minor);

   HostIF_InitUptime();
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
@@ -369,7 +357,7 @@ cleanup_module(void)
 #ifdef VMX86_DEVEL
   unregister_chrdev(linuxState.major, linuxState.deviceName);
 #else
-   misc_deregister(&vmmon_miscdev);
+   misc_deregister(&linuxState.misc);
 #endif

   Log("Module %s: unloaded\n", linuxState.deviceName);
@@ -676,7 +664,7 @@ LinuxDriverReadTSC(void *data,   // OUT: TSC values
 *-----------------------------------------------------------------------------
 */

-__attribute__((always_inline)) static Bool
+__always_inline static Bool
 LinuxDriverSyncReadTSCs(uint64 *delta) // OUT: TSC max - TSC min
 {
   TSCDelta tscDelta;
@@ -799,7 +787,6 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      } else {
         device->vm = vm;
         args.vmid = vm->userID;
-         vm->vmhost->vmmonData = (void *__user)args.vmmonData;
         retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args);
      }
      break;
@@ -1093,23 +1080,6 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

-   case IOCTL_VMX86_GET_SWITCH_ERROR_ADDR: {
-      VMSwitchErrorArgs args;
-
-      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
-      if (retval != 0) {
-         break;
-      }
-      if (args.vcpuid >= vm->numVCPUs || vm->crosspage == NULL ||
-          vm->crosspage[args.vcpuid] == NULL) {
-         retval = -EINVAL;
-         break;
-      }
-      args.addr = vm->crosspage[args.vcpuid]->wsUD2;
-      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
-      break;
-   }
-
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
      retval = LinuxDriverEstimateTSCkHz();
      break;
@@ -1257,7 +1227,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
-#if VMX86_DEVEL
+#ifdef VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -23,6 +23,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
@@ -60,6 +61,8 @@ typedef struct Device {

 typedef struct VMXLinuxState {
   int major;
+   int minor;
+   struct miscdevice misc;
   char deviceName[VM_DEVICE_NAME_SIZE];
   char buf[LINUXLOG_BUFFER_SIZE];
   Device *head;
@@ -72,10 +72,10 @@
 #include "memDefaults.h"
 #include "vcpuid.h"
 #include "x86svm.h"
-#include "crosspage.h"

 #include "pgtbl.h"
 #include "versioned_atomic.h"
+#include "compat_poll.h"

 #if !defined(CONFIG_HIGH_RES_TIMERS)
 #error CONFIG_HIGH_RES_TIMERS required for acceptable performance
@@ -189,7 +189,6 @@ static void UnlockEntry(void *clientData, MemTrackEntry *entryPtr);
 #define SPURIOUS_APIC_VECTOR 0
 #endif
 static const uint8 monitorIPIVector = SPURIOUS_APIC_VECTOR;
-
 /*
 * POSTED_INTR_VECTOR is defined on kernels after 3.10 when built with
 * KVM support.
@@ -199,15 +198,6 @@ static const uint8 monitorIPIVector = SPURIOUS_APIC_VECTOR;
 #endif
 static const uint8 hvIPIVector = POSTED_INTR_VECTOR;

-/*
- * perfCtrVector may leak, choose highest priority interrupt that is
- * consistently ack'd by all current linux kernels.
- */
-#ifndef ERROR_APIC_VECTOR
-#define ERROR_APIC_VECTOR 0xFE
-#endif
-#define PERF_CTR_VECTOR ERROR_APIC_VECTOR
-
 /*
 *-----------------------------------------------------------------------------
 *
@@ -626,6 +616,38 @@ HostIF_FastClockUnlock(int callerID) // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MapCrossPage & UnmapCrossPage
+ *
+ *    Both x86-64 and ia32 need to map crosspage to an executable
+ *    virtual address. We use the vmap interface instead of kmap
+ *    due to bug 43907.
+ *
+ * Side effects:
+ *
+ *    UnmapCrossPage assumes that the page has been refcounted up
+ *    so it takes care of the put_page.
+ *
+ *----------------------------------------------------------------------
+ */
+static void *
+MapCrossPage(struct page *p)  // IN:
+{
+   return vmap(&p, 1, VM_MAP, VM_PAGE_KERNEL_EXEC);
+}
+
+
+static void
+UnmapCrossPage(struct page *p,  // IN:
+               void *va)        // IN:
+{
+   vunmap(va);
+   put_page(p);
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -953,8 +975,11 @@ static void
 HostIFFreeVMHost(VMHost *vmhost) // IN:
 {
   ASSERT(vmhost->lockedPages == NULL &&
-          vmhost->AWEPages    == NULL);
+          vmhost->AWEPages    == NULL &&
+          vmhost->crosspagePagesCount == 0);
+   Vmx86_Free(vmhost->crosspagePages);
   Vmx86_Free(vmhost->vcpuSemaTask);
+   vmhost->crosspagePages = NULL;
   vmhost->vcpuSemaTask   = NULL;
   HostIF_FreeKernelMem(vmhost);
 }
@@ -983,14 +1008,14 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
   if (vmhost == NULL) {
      return NULL;
   }
-   vmhost->vcpuSemaTask = Vmx86_Calloc(numVCPUs, sizeof *vmhost->vcpuSemaTask,
-                                       TRUE);
-   if (vmhost->vcpuSemaTask == NULL) {
-      HostIFFreeVMHost(vmhost);
-      return NULL;
+   if ((vmhost->crosspagePages =
+        Vmx86_Calloc(numVCPUs, sizeof *vmhost->crosspagePages, TRUE)) != NULL &&
+       (vmhost->vcpuSemaTask =
+        Vmx86_Calloc(numVCPUs, sizeof *vmhost->vcpuSemaTask, TRUE))   != NULL) {
+      return vmhost;
   }
-
-   return vmhost;
+   HostIFFreeVMHost(vmhost);
+   return NULL;
 }


@@ -1495,12 +1520,20 @@ UnlockEntry(void *clientData,         // IN:
 void
 HostIF_FreeAllResources(VMDriver *vm) // IN
 {
+   unsigned int cnt;
+
   HostIFHostMemCleanup(vm);
   if (vm->memtracker) {
      MemTrack_Cleanup(vm->memtracker, UnlockEntry, vm);
      vm->memtracker = NULL;
   }
   if (vm->vmhost) {
+      ASSERT(vm->vmhost->crosspagePagesCount <= vm->numVCPUs);
+      for (cnt = vm->vmhost->crosspagePagesCount; cnt > 0; ) {
+         struct page* p = vm->vmhost->crosspagePages[--cnt];
+         UnmapCrossPage(p, vm->crosspage[cnt]);
+      }
+      vm->vmhost->crosspagePagesCount = 0;
      HostIFFreeVMHost(vm->vmhost);
      vm->vmhost = NULL;
   }
@@ -1707,10 +1740,7 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
   lockedPages += global_page_state(NR_PAGETABLE);
 #endif
   /* NR_SLAB_* moved from zone to node in 4.13. */
-
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)
-   lockedPages += global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B);
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
   lockedPages += global_node_page_state(NR_SLAB_UNRECLAIMABLE);
 #else
   lockedPages += global_page_state(NR_SLAB_UNRECLAIMABLE);
@@ -2125,36 +2155,49 @@ HostIF_CopyToUser(VA64 dst,         // OUT
 /*
 *-----------------------------------------------------------------------------
 *
- * HostIFGetCrossPageMPNWork --
- * HostIF_GetCrossPageCodeMPN --
- * HostIF_GetCrossPageDataMPN --
+ * HostIF_MapCrossPage --
 *
+ *    Map the cross page in the kernel address space given a user VA.  The
+ *    kernel mapping must not overlap the monitor's 64MB VA space (bug 32922).
+ *    In practice, the host kernel does not return mappings in that range.
+ *    This is checked in TaskCreatePTPatch() which gracefully fails if it must.
+ *
+ * Results:
+ *    The kernel virtual address on success
+ *    NULL on failure
+ *
+ * Side effects:
+ *    None
 *
 *-----------------------------------------------------------------------------
 */

-static MPN
-HostIFGetCrossPageMPNWork(void *va)
+void *
+HostIF_MapCrossPage(VMDriver *vm, // IN
+                    VA64 uAddr)   // IN
 {
-   struct page *pg = vmalloc_to_page(va);
+   void *p = VA64ToPtr(uAddr);
+   struct page *page;
+   VA           vPgAddr;
+   VA           ret;

-   if (pg != NULL) {
-      return page_to_pfn(pg);
+   if (HostIFGetUserPages(p, &page, 1)) {
+      return NULL;
   }
+   vPgAddr = (VA) MapCrossPage(page);
+   HostIF_VMLock(vm, 27);
+   if (vm->vmhost->crosspagePagesCount >= vm->numVCPUs) {
+      HostIF_VMUnlock(vm, 27);
+      UnmapCrossPage(page, (void*)vPgAddr);

-   return INVALID_MPN;
-}
+      return NULL;
+   }
+   vm->vmhost->crosspagePages[vm->vmhost->crosspagePagesCount++] = page;
+   HostIF_VMUnlock(vm, 27);

-MPN
-HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData)
-{
-   return HostIFGetCrossPageMPNWork(crosspageData);
-}
+   ret = vPgAddr | (((VA)p) & (PAGE_SIZE - 1));

-MPN
-HostIF_GetCrossPageCodeMPN(void)
-{
-   return HostIFGetCrossPageMPNWork((void *)CrossPage_CodePage);
+   return (void*)ret;
 }


@@ -2347,23 +2390,16 @@ HostIF_VMLockIsHeld(VMDriver *vm) // IN
 static Bool
 isVAReadable(VA r)  // IN:
 {
+   mm_segment_t old_fs;
   uint32 dummy;
   int ret;

-   r = APICR_TO_ADDR(r, APICR_VERSION);
-#ifdef HAVE_GET_KERNEL_NOFAULT
-   ret = get_kernel_nofault(dummy, (void *)r);
-#else
-   {
-   mm_segment_t old_fs;
-
   old_fs = get_fs();
   set_fs(KERNEL_DS);
-
+   r = APICR_TO_ADDR(r, APICR_VERSION);
   ret = HostIF_CopyFromUser(&dummy, r, sizeof dummy);
   set_fs(old_fs);
-   }
-#endif
+
   return ret == 0;
 }
 #endif
@@ -2495,32 +2531,6 @@ HostIF_GetHVIPIVector(void)
 }


-/*
- *----------------------------------------------------------------------
- *
- * HostIF_GetPerfCtrVector --
- *
- *      Return a vector the monitor can use for perf counters.
- *      This needs to be a high priority interrupt that is always ack'd
- *      by the kernel because they can leak in edge cases.
- *
- * Results:
- *     The vector for perf counter events.
- *
- * Side effects:
- *      None.
- *
- *----------------------------------------------------------------------
- */
-
-uint8
-HostIF_GetPerfCtrVector(void)
-{
-   ASSERT_ON_COMPILE(PERF_CTR_VECTOR == 0xFE);
-   return PERF_CTR_VECTOR;
-}
-
-
 /*
 *-----------------------------------------------------------------------------
 *
@@ -2529,10 +2539,8 @@ HostIF_GetPerfCtrVector(void)
 *    Perform the semaphore wait (P) operation, possibly blocking.
 *
 * Result:
- *      On success: MX_WAITINTERRUPTED (3) if interrupted by a Unix signal.
- *                  MX_WAITTIMEDOUT (2) if the timeout occurred.
- *                  MX_WAITNORMAL (1) if the event was raised.
- *      On error: MX_WAITERROR (0) on generic error.
+ *    1 (which equals MX_WAITNORMAL) if success,
+ *    negated error code otherwise.
 *
 * Side-effects:
 *    None
@@ -2546,12 +2554,11 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
                     uint64 *args)   // IN:
 {
   struct file *file;
+   mm_segment_t old_fs;
   int res;
   int waitFD = args[0];
   int timeoutms = args[2];
   uint64 value;
-   struct poll_wqueues table;
-   unsigned int mask;

   ASSERT(vcpuid < vm->numVCPUs);

@@ -2559,46 +2566,47 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   if (file == NULL) {
      return MX_WAITERROR;
   }
-   if (file->f_op->poll == NULL) {
-      fput(file);
-      Warning("%s: File poll operation is NULL\n", __func__);
-      return MX_WAITERROR;
-   }

-   poll_initwait(&table);
-   current->state = TASK_INTERRUPTIBLE;
-   mask = file->f_op->poll(file, &table.pt);
-   if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
-      vm->vmhost->vcpuSemaTask[vcpuid] = current;
-      schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
-      vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
+   old_fs = get_fs();
+   set_fs(KERNEL_DS);
+
+   {
+      struct poll_wqueues table;
+      unsigned int mask;
+
+      poll_initwait(&table);
+      current->state = TASK_INTERRUPTIBLE;
+      mask = compat_vfs_poll(file, &table.pt);
+      if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
+         vm->vmhost->vcpuSemaTask[vcpuid] = current;
+         schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
+         vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
+      }
+      current->state = TASK_RUNNING;
+      poll_freewait(&table);
   }
-   current->state = TASK_RUNNING;
-   poll_freewait(&table);

   /*
    * Userland only writes in multiples of sizeof(uint64). This will allow
    * the code to happily deal with a pipe or an eventfd. We only care about
    * reading no bytes (EAGAIN - non blocking fd) or sizeof(uint64).
-    *
-    * Upstream Linux changed the function parameter types/ordering in 4.14.0.
    */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
-   res = kernel_read(file, file->f_pos, (char *)&value, sizeof value);
-#else
-   res = kernel_read(file, &value, sizeof value, &file->f_pos);
-#endif
-   fput(file);
+
+   res = file->f_op->read(file, (char *) &value, sizeof value, &file->f_pos);

   if (res == sizeof value) {
-      return MX_WAITNORMAL;
-   }
-   if (res == 0) {
-      res = -EBADF;
+      res = MX_WAITNORMAL;
+   } else {
+      if (res == 0) {
+         res = -EBADF;
+      }
   }

+   set_fs(old_fs);
+   fput(file);
+
   /*
-    * Handle various errors:
+    * Handle benign errors:
    * EAGAIN is MX_WAITTIMEDOUT.
    * The signal-related errors are all mapped into MX_WAITINTERRUPTED.
    */
@@ -2617,10 +2625,6 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   case -EBADF:
      res = MX_WAITERROR;
      break;
-   default:
-      Warning("Unexpected error: %s err=%d\n", __FUNCTION__, res);
-      res = MX_WAITERROR;
-      break;
   }
   return res;
 }
@@ -2674,6 +2678,7 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
 *      On error: MX_WAITINTERRUPTED (3) if interrupted by a Unix signal (we
 *                   can block on a preemptive kernel).
 *                MX_WAITERROR (0) on generic error.
+ *                Negated system error (< 0).
 *
 * Side-effects:
 *      None
@@ -2682,64 +2687,57 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
 */

 int
-HostIF_SemaphoreSignal(VMDriver *vm,  // IN:
-                       uint64 *args)  // IN:
+HostIF_SemaphoreSignal(uint64 *args)  // IN:
 {
+   struct file *file;
+   mm_segment_t old_fs;
   int res;
   int signalFD = args[1];
-   struct file *file;
+   uint64 value = 1;  // make an eventfd happy should it be there

   file = vmware_fget(signalFD);
-   if (file == NULL) {
-      return FALSE;
+   if (!file) {
+      return MX_WAITERROR;
   }

+   old_fs = get_fs();
+   set_fs(KERNEL_DS);
+
   /*
    * Always write sizeof(uint64) bytes. This works fine for eventfd and
    * pipes. The data written is formatted to make an eventfd happy should
    * it be present.
    */
-   if (file->f_op->write != NULL) {
-      /*
-       * PR2650348 - The kernel_write function signature changed in the vanilla
-       * kernel in 4.14.0 and the signature update was backported before that
-       * point in a kernel used by OpenSUSE 15.1 so call the function directly
-       * instead of via the wrapper.
-       *
-       * Kernel 5.10 removed the set_fs functionality which means a user pointer
-       * must be passed into the write function or a -EFAULT will be returned.
-       */
-      res = file->f_op->write(file, vm->vmhost->vmmonData, sizeof (uint64),
-                              &file->f_pos);
-   } else {
-      Warning("%s: File write operation is NULL\n", __func__);
-      res = -ENOTSUPP;
+
+   res = file->f_op->write(file, (char *) &value, sizeof value, &file->f_pos);
+
+   if (res == sizeof value) {
+      res = MX_WAITNORMAL;
   }
+
+   set_fs(old_fs);
   fput(file);

-   if (res == sizeof (uint64)) {
-      return MX_WAITNORMAL;
-   }
-
   /*
-    * Handle various errors:
+    * Handle benign errors:
    * EAGAIN is MX_WAITTIMEDOUT.
    * The signal-related errors are all mapped into MX_WAITINTERRUPTED.
    */
+
   switch (res) {
   case -EAGAIN:
-      /* The pipe is full, so it is already signalled. Success. */
-      return MX_WAITNORMAL;
+      // The pipe is full, so it is already signalled. Success.
+      res = MX_WAITNORMAL;
+      break;
   case -EINTR:
   case -ERESTART:
   case -ERESTARTSYS:
   case -ERESTARTNOINTR:
   case -ERESTARTNOHAND:
-      return MX_WAITINTERRUPTED;
-   default:
-      Warning("Unexpected error: %s err=%d\n", __FUNCTION__, res);
-      return MX_WAITERROR;
+      res = MX_WAITINTERRUPTED;
+      break;
   }
+   return res;
 }


@@ -3264,9 +3262,12 @@ static int
 HostIFFastClockThread(void *unused)  // IN:
 {
   int res;
+   mm_segment_t oldFS;
   unsigned int rate = 0;
   unsigned int prevRate = 0;

+   oldFS = get_fs();
+   set_fs(KERNEL_DS);
   allow_signal(SIGKILL);

   while ((rate = linuxState.fastClockRate) > MIN_RATE) {
@@ -3289,6 +3290,8 @@ HostIFFastClockThread(void *unused)  // IN:
   }

 out:
+   set_fs(oldFS);
+
   /*
    * Do not exit thread until we are told to do so.
    */
@@ -79,6 +79,8 @@ typedef struct VMHost {
    */
   Mutex vmMutex;

+   unsigned int         crosspagePagesCount;
+   struct page        **crosspagePages;      /* ptr to numVCPUs-sized array */
   struct task_struct **vcpuSemaTask;        /* ptr to numVCPUs-sized array */

   /*
@@ -92,11 +94,6 @@ typedef struct VMHost {
    * as pages for "AWE" guest memory.
    */
   struct PhysTracker  *AWEPages;
-   /*
-    * Pointer to a userlevel 64-bit area containing the value 1.
-    * This is used for HostIF_SemaphoreSignal.
-    */
-   void *__user vmmonData;
 } VMHost;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -75,13 +75,12 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
   uint64           retval    = MODULECALL_USERRETURN;
   VMCrossPageData *crosspage;
   int              bailValue = 0;
-   Bool             switchOk = TRUE;

   ASSERT(vcpuid < vm->numVCPUs);
   if (vm->crosspage[vcpuid] == NULL) {
      return USERCALL_VMX86ALLOCERR;
   }
-   crosspage = vm->crosspage[vcpuid];
+   crosspage = &vm->crosspage[vcpuid]->crosspageData;

   /*
    * Check if we were interrupted by signal.
@@ -96,9 +95,9 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
       * Task_Switch changes the world to the monitor.
       * The monitor is waiting in the BackToHost routine.
       */
-      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHING_TO_MONITOR);
-      switchOk = Task_Switch(vm, vcpuid);
-      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHED_TO_MODULE);
+      UCTIMESTAMP(crosspage, SWITCHING_TO_MONITOR);
+      Task_Switch(vm, vcpuid);
+      UCTIMESTAMP(crosspage, SWITCHED_TO_MODULE);

      /*
       * Wake up anything that was waiting for this vcpu to run
@@ -120,11 +119,6 @@ skipTaskSwitch:;

      retval = MODULECALL_USERRETURN;

-      if (UNLIKELY(!switchOk)) {
-         bailValue = USERCALL_SWITCHERR;
-         goto bailOut;
-      }
-
      if (crosspage->userCallType != MODULECALL_USERCALL_NONE) {
         /*
          * This is the main user call path.  Handle by returning
@@ -171,6 +165,7 @@ skipTaskSwitch:;

      case MODULECALL_SEMAWAIT: {
         retval = HostIF_SemaphoreWait(vm, vcpuid, crosspage->args);
+
         if (retval == MX_WAITINTERRUPTED) {
            crosspage->moduleCallInterrupted = TRUE;
            bailValue = USERCALL_RESTART;
@@ -180,7 +175,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_SEMASIGNAL: {
-         retval = HostIF_SemaphoreSignal(vm, crosspage->args);
+         retval = HostIF_SemaphoreSignal(crosspage->args);

         if (retval == MX_WAITINTERRUPTED) {
             crosspage->moduleCallInterrupted = TRUE;
@@ -288,10 +283,6 @@ skipTaskSwitch:;
         retval = HostIF_GetHVIPIVector();
      } break;

-      case MODULECALL_GET_PERF_CTR_VECTOR: {
-         retval = HostIF_GetPerfCtrVector();
-      } break;
-
      case MODULECALL_GET_HOST_TIMER_VECTORS: {
         uint8 v0, v1;
         HostIF_GetTimerVectors(&v0, &v1);
@@ -330,19 +321,6 @@ skipTaskSwitch:;
                                                   hvIOBitmap->mpn;
      } break;

-      case MODULECALL_GET_MSR: {
-         /*
-          * Allocate a MSRQuery request on stack with only one MSRReply slot
-          * since this MSR will be queried from a MSR cache or on a single pcpu.
-          */
-         uint8 req[sizeof(MSRQuery) + sizeof(MSRReply)];
-         MSRQuery *query = (MSRQuery *)&req[0];
-         query->msrNum = (uint32)crosspage->args[0];
-         query->numLogicalCPUs = 1;
-         retval = Vmx86_GetAllMSRs(query);
-         crosspage->args[0] = query->logicalCPUs[0].msrVal;
-      } break;
-
      case MODULECALL_ALLOC_CONTIG_PAGES: {
         PageCnt pages = crosspage->args[0];
         HostIFContigMemMap *alloc;
@@ -362,13 +340,7 @@ skipTaskSwitch:;
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
-
-      if (retval != (uint64)((uint32)retval)) {
-         Warning("Unexpected error in modulecall %u (%llu)\n",
-                 crosspage->moduleCallType, retval);
-         bailValue = USERCALL_SWITCHERR;
-         goto bailOut;
-      }
+      ASSERT(retval == (uint64)((uint32)retval));
      crosspage->retval = (uint32)retval;
 #if defined(__linux__)
      cond_resched(); // Other kernels are preemptible
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -136,8 +136,6 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

-# Add Spectre options when available
-
 include $(SRCROOT)/Makefile.kernel

 else
@@ -32,6 +32,6 @@ $(DRIVER)-y := driver.o hub.o userif.o netif.o bridge.o procfs.o smac_compat.o \
 ####

 clean:
-	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko \
+	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko .cache.mk \
 	       .tmp_versions Module.symvers Modules.symvers Module.markers \
 	       modules.order $($(DRIVER)-y) .*.cmd .*.o.flags
@@ -66,7 +66,7 @@
 #endif

 #if LOGLEVEL >= 4
-static struct timeval vnetTime;
+static u64 vnetTime;
 #endif

 typedef struct VNetBridge VNetBridge;
@@ -693,7 +693,7 @@ VNetBridgeReceiveFromVNet(VNetJack        *this, // IN: jack

 	 netif_rx_ni(clone);
 #	 if LOGLEVEL >= 4
-	 do_gettimeofday(&vnetTime);
+	 vnetTime = ktime_get_ns();
 #	 endif
      }
   }
@@ -1490,12 +1490,11 @@ VNetBridgeReceiveFromDev(struct sk_buff *skb,         // IN: packet to receive

 #  if LOGLEVEL >= 4
   {
-      struct timeval now;
-      do_gettimeofday(&now);
+      u64 now;
+
+      now = ktime_get_ns();
      LOG(3, (KERN_DEBUG "bridge-%s: time %d\n",
-	      bridge->name,
-	      (int)((now.tv_sec * 1000000 + now.tv_usec)
-                    - (vnetTime.tv_sec * 1000000 + vnetTime.tv_usec))));
+	      bridge->name, (int)((now - vnetTime) / NSEC_PER_USEC)));
   }
 #  endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002,2018,2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -4327,7 +4327,7 @@ LookupTypeClass(unsigned short typeValue) // IN: ethernet type
 *----------------------------------------------------------------------
 */

-#ifdef _WIN32
+#if _WIN32
 _Use_decl_annotations_
 #endif
 void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017,2019,2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -81,9 +81,7 @@ extern unsigned int  vnet_max_qlen;
 #   define skb_frag_page(frag) (frag)->page
 #   define skb_frag_size(frag) (frag)->size
 #endif
-#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0) && \
-    !(defined(CONFIG_SUSE_VERSION) && CONFIG_SUSE_VERSION == 15 && \
-      defined(CONFIG_SUSE_PATCHLEVEL) && CONFIG_SUSE_PATCHLEVEL >= 2)
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0)
 #   define skb_frag_off(frag) (frag)->page_offset
 #endif

@@ -561,12 +559,7 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
      return -EINVAL;
   }

-#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
   csum = csum_and_copy_to_user(skb->data + offset, curr, len, 0, &err);
-#else
-   csum = csum_and_copy_to_user(skb->data + offset, curr, len);
-   err = (csum == 0) ? -EFAULT : 0;
-#endif
   if (err) {
      return err;
   }
@@ -580,14 +573,8 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
 	 const void *vaddr;

 	 vaddr = kmap(skb_frag_page(frag));
-#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
 	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
 					 curr, skb_frag_size(frag), 0, &err);
-#else
-	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
-					 curr, skb_frag_size(frag));
-         err = (tmpCsum == 0) ? -EFAULT : 0;
-#endif
 	 kunmap(skb_frag_page(frag));

 	 if (err) {
@@ -41,13 +41,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#pragma warning(push)
-#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
-#include <intrin.h>
-#pragma warning(pop)
-#endif
-
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -152,6 +145,64 @@ typedef ALIGNED(16) struct Atomic_uint128 {
 } Atomic_uint128;
 #endif

+/*
+ * Prototypes for msft atomics.  These are defined & inlined by the
+ * compiler so no function definition is needed.  The prototypes are
+ * needed for C++.
+ *
+ * The declarations for the intrinsic functions were taken from ntddk.h
+ * in the DDK. The declarations must match otherwise the 64-bit C++
+ * compiler will complain about second linkage of the intrinsic functions.
+ * We define the intrinsic using the basic types corresponding to the
+ * Windows typedefs. This avoids having to include windows header files
+ * to get to the windows types.
+ */
+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#ifdef __cplusplus
+extern "C" {
+#endif
+long  _InterlockedExchange(long volatile*, long);
+long  _InterlockedCompareExchange(long volatile*, long, long);
+long  _InterlockedExchangeAdd(long volatile*, long);
+long  _InterlockedDecrement(long volatile*);
+long  _InterlockedIncrement(long volatile*);
+__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
+#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
+#pragma intrinsic(_InterlockedIncrement)
+#pragma intrinsic(_InterlockedCompareExchange64)
+
+# if _MSC_VER >= 1600
+char     _InterlockedExchange8(char volatile *, char);
+char     _InterlockedCompareExchange8(char volatile *, char, char);
+#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
+#endif
+
+#if defined VM_X86_64
+long     _InterlockedAnd(long volatile*, long);
+__int64  _InterlockedAnd64(__int64 volatile*, __int64);
+long     _InterlockedOr(long volatile*, long);
+__int64  _InterlockedOr64(__int64 volatile*, __int64);
+long     _InterlockedXor(long volatile*, long);
+__int64  _InterlockedXor64(__int64 volatile*, __int64);
+__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
+__int64  _InterlockedIncrement64(__int64 volatile*);
+__int64  _InterlockedDecrement64(__int64 volatile*);
+__int64  _InterlockedExchange64(__int64 volatile*, __int64);
+#if !defined _WIN64
+#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
+#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
+#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
+#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
+#endif /* !_WIN64 */
+#endif /* __x86_64__ */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _MSC_VER */
+
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -204,6 +255,15 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

+#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
+uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
+                              uint8 val);
+
+uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
+                                     uint8 newVal,
+                                     uint8 oldVal);
+#endif
+
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -345,8 +405,19 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER
+#elif defined _MSC_VER && _MSC_VER >= 1600
   return _InterlockedExchange8((volatile char *)&var->value, val);
+#elif defined _MSC_VER && defined __i386__
+#pragma warning(push)
+#pragma warning(disable : 4035)         // disable no-return warning
+   {
+      __asm movzx eax, val
+      __asm mov ebx, var
+      __asm xchg [ebx]Atomic_uint8.value, al
+   }
+#pragma warning(pop)
+#elif defined _MSC_VER && defined __x86_64__
+   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -429,9 +500,23 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER
+#elif defined _MSC_VER && _MSC_VER >= 1600
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
+#elif defined _MSC_VER && defined __i386__
+#pragma warning(push)
+#pragma warning(disable : 4035)         // disable no-return warning
+   {
+      __asm mov al, oldVal
+      __asm mov ebx, var
+      __asm mov cl, newVal
+      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
+      __asm movzx eax, al
+      // eax is the return value, this is documented to work - edward
+   }
+#pragma warning(pop)
+#elif defined _MSC_VER && defined __x86_64__
+   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1227,7 +1312,13 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
+#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
+#else
+   __asm mov eax, val
+   __asm mov ebx, var
+   __asm lock And [ebx]Atomic_uint32.value, eax
+#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1286,7 +1377,13 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
+#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
+#else
+   __asm mov eax, val
+   __asm mov ebx, var
+   __asm lock Or [ebx]Atomic_uint32.value, eax
+#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1345,7 +1442,13 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
+#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
+#else
+   __asm mov eax, val
+   __asm mov ebx, var
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
+#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2084,13 +2187,18 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
-   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
-   return _InterlockedCompareExchange64((__int64 *)&var->value,
-                                        (__int64)255,  // Unlikely value to
-                                        (__int64)255); // not dirty cache
+#   pragma warning(push)
+#   pragma warning(disable : 4035)      // disable no-return warning
+   {
+      __asm mov ecx, var
+      __asm mov edx, ecx
+      __asm mov eax, ebx
+      __asm lock cmpxchg8b [ecx]
+      // edx:eax is the return value; this is documented to work. --mann
+   }
+#   pragma warning(pop)
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,13 +35,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#pragma warning(push)
-#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
-#include <intrin.h>
-#pragma warning(pop)
-#endif
-
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -506,7 +499,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" ((uint64)val)
+      : "r" (val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -559,7 +552,6 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
-   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -596,7 +588,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
+      : "+r" (tmpDst), "+r" (count), "+r" (val)
      :
      : "cc", "memory");
 #else
@@ -640,13 +632,22 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_ANY
+#ifdef VM_X86_64
   __stosw((uint16*)dst, val, count);
-#else
+#elif defined(VM_ARM_32)
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
+#else
+   __asm { pushf;
+           mov ax, val;
+           mov ecx, count;
+           mov edi, dst;
+           cld;
+           rep stosw;
+           popf;
+   }
 #endif
   return dst;
 }
@@ -654,13 +655,22 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_ANY
+#ifdef VM_X86_64
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#else
+#elif defined(VM_ARM_32)
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
+#else
+   __asm { pushf;
+           mov eax, val;
+           mov ecx, count;
+           mov edi, dst;
+           cld;
+           rep stosd;
+           popf;
+   }
 #endif
   return dst;
 }
@@ -772,9 +782,17 @@ PAUSE(void)
 #endif
 }
 #elif defined(_MSC_VER)
+#ifdef VM_X86_64
 {
   _mm_pause();
 }
+#else /* VM_X86_64 */
+#pragma warning( disable : 4035)
+{
+   __asm _emit 0xf3 __asm _emit 0x90
+}
+#pragma warning (default: 4035)
+#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -822,11 +840,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_ANY
+#ifdef VM_X86_64
 {
   return __rdtsc();
 }
-#else
+#elif defined(VM_ARM_32)
 {
   /*
    * We need to do more inverstagetion here to find
@@ -835,7 +853,13 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#endif /* VM_X86_ANY */
+#else
+#pragma warning( disable : 4035)
+{
+   __asm _emit 0x0f __asm _emit 0x31
+}
+#pragma warning (default: 4035)
+#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,6 +39,41 @@
 #error "This file is x86-64 only!"
 #endif

+#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+uint64 _umul128(uint64 multiplier, uint64 multiplicand,
+                uint64 *highProduct);
+int64 _mul128(int64 multiplier, int64 multiplicand,
+              int64 *highProduct);
+uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
+#ifdef ULM
+void _fxsave64(void *save);
+void _fxsave(void *save);
+void _fxrstor64(const void *load);
+void _fxrstor(const void *load);
+void _xsave64(void *save, uint64 mask);
+void _xsave(void *save, uint64 mask);
+void _xsaveopt64(void *save, uint64 mask);
+void _xsavec(void *save, uint64 mask);
+void _xrstor64(const void *load, uint64 mask);
+void _xrstor(const void *load, uint64 mask);
+#endif /* ULM */
+#ifdef __cplusplus
+}
+#endif
+
+#pragma intrinsic(_umul128, _mul128, __shiftright128)
+
+#ifdef ULM
+#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
+                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
+#endif /* ULM */
+
+#endif // _MSC_VER
+
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,6 +38,97 @@
 #error "Should be included only in x86 builds"
 #endif

+/*
+ * x86-64 windows doesn't support inline asm so we have to use these
+ * intrinsic functions defined in the compiler.  Not all of these are well
+ * documented.  There is an array in the compiler dll (c1.dll) which has
+ * an array of the names of all the intrinsics minus the leading
+ * underscore.  Searching around in the ntddk.h file can also be helpful.
+ *
+ * The declarations for the intrinsic functions were taken from the DDK.
+ * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
+ * will complain about second linkage of the intrinsic functions.
+ * We define the intrinsic using the basic types corresponding to the
+ * Windows typedefs. This avoids having to include windows header files
+ * to get to the windows types.
+ */
+#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * It seems x86 & x86-64 windows still implements these intrinsic
+ * functions.  The documentation for the x86-64 suggest the
+ * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
+ * __inbyte/__outbyte aren't supported on x86.
+ */
+int            _inp(unsigned short);
+unsigned short _inpw(unsigned short);
+unsigned long  _inpd(unsigned short);
+
+int            _outp(unsigned short, int);
+unsigned short _outpw(unsigned short, unsigned short);
+unsigned long  _outpd(uint16, unsigned long);
+#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
+
+/*
+ * Prevents compiler from re-ordering reads, writes and reads&writes.
+ * These functions do not add any instructions thus only affect
+ * the compiler ordering.
+ *
+ * See:
+ * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
+ * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
+ */
+void _ReadBarrier(void);
+void _WriteBarrier(void);
+void _ReadWriteBarrier(void);
+#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
+
+void _mm_mfence(void);
+void _mm_lfence(void);
+#pragma intrinsic(_mm_mfence, _mm_lfence)
+
+long _InterlockedXor(long volatile *, long);
+#pragma intrinsic(_InterlockedXor)
+
+unsigned int __getcallerseflags(void);
+#pragma intrinsic(__getcallerseflags)
+
+#ifdef VM_X86_64
+/*
+ * intrinsic functions only supported by x86-64 windows as of 2k3sp1
+ */
+unsigned __int64 __rdtsc(void);
+void             __stosw(unsigned short *, unsigned short, size_t);
+void             __stosd(unsigned long *, unsigned long, size_t);
+void             _mm_pause(void);
+#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
+
+unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
+unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
+#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
+#endif /* VM_X86_64 */
+
+unsigned char  _BitScanForward(unsigned long *, unsigned long);
+unsigned char  _BitScanReverse(unsigned long *, unsigned long);
+#pragma intrinsic(_BitScanForward, _BitScanReverse)
+
+unsigned char  _bittest(const long *, long);
+unsigned char  _bittestandset(long *, long);
+unsigned char  _bittestandreset(long *, long);
+unsigned char  _bittestandcomplement(long *, long);
+#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
+#ifdef VM_X86_64
+unsigned char  _bittestandset64(__int64 *, __int64);
+unsigned char  _bittestandreset64(__int64 *, __int64);
+#pragma intrinsic(_bittestandset64, _bittestandreset64)
+#endif // VM_X86_64
+#ifdef __cplusplus
+}
+#endif
+#endif // _MSC_VER
+
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -88,6 +179,9 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

+#define GET_CURRENT_EIP(_eip) \
+      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
+
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -97,35 +191,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE uint8
+static INLINE  uint8
 INB(uint16 port)
 {
-   return __inbyte(port);
+   return (uint8)_inp(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   __outbyte(port, value);
+   _outp(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return __inword(port);
+   return _inpw(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   __outword(port, value);
+   _outpw(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return __indword(port);
+   return _inpd(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   __outdword(port, value);
+   _outpd(port, value);
 }

 #ifndef VM_X86_64
@@ -133,6 +227,11 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

+#define GET_CURRENT_EIP(_eip) do { \
+   __asm call NEAR PTR $+5 \
+   __asm pop eax \
+   __asm mov _eip, eax \
+} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -259,7 +358,11 @@ SFENCE(void)
   );
 #elif defined _MSC_VER
   _ReadWriteBarrier();
+#if defined VM_X86_32
+   __asm sfence;
+#else
   _mm_sfence();
+#endif
   _ReadWriteBarrier();
 #else
 #error No compiler defined for SFENCE
@@ -688,28 +688,22 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-4.1 (circa 2006).  Enforce this here, various things below
+ * gcc-3.3 (circa 2003).  Enforce this here, various things below
 * this line depend upon it.
 *
- * Current oldest compilers:
- * - guest tools: 4.1.2 (freebsd/solaris)
- * - buildhost compiler: 4.4.3
- * - hosted kernel modules: 4.5
- *
- * SWIG's preprocessor is exempt.
+ * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
+ * The various linux kernel modules may use older (gcc-3.3) compilers.
 */
-#ifndef SWIG
-#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
-#error "gcc version is too old, need gcc-4.1 or better"
-#endif
+#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
+#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
 #endif

 /*
- * Similarly, we require a compiler that is at least vs2012.
+ * Similarly, we require a compiler that is at least vc90 (vs2008).
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1700
-#error "cl.exe version is too old, need vs2012 or better"
+#if defined _MSC_VER && _MSC_VER < 1500
+#error "cl.exe version is too old, need vc90 or better"
 #endif


@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2005-2012,2014-2020 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2005-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,8 @@
 #endif
 #endif

+#include <linux/pci_ids.h>
+
 /* LSILogic 53C1030 Parallel SCSI controller
 * LSILogic SAS1068 SAS controller
 */
@@ -53,7 +55,10 @@
 *    VMware HD Audio codec
 *    VMware HD Audio controller
 */
+#ifndef PCI_VENDOR_ID_VMWARE
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
+#endif
+
 #define PCI_DEVICE_ID_VMWARE_SVGA3              0x0406
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
@@ -71,7 +76,11 @@
 #define PCI_DEVICE_ID_VMWARE_1394               0x0780
 #define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
 #define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
+
+#ifndef PCI_DEVICE_ID_VMWARE_VMXNET3
 #define PCI_DEVICE_ID_VMWARE_VMXNET3            0x07B0
+#endif
+
 #define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
 #define PCI_DEVICE_ID_VMWARE_82574              0x07D0
 #define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
@@ -344,7 +353,7 @@
 #define MAX_NVDIMM 64

 /************* vRDMA implementation limits ******************************/
-#define MAX_VRDMA_DEVICES 10
+#define MAX_VRDMA_DEVICES 1

 /************* QAT implementation limits ********************/
 #define MAX_QAT_PCI_DEVICES 4
Author	SHA1	Message	Date
Michal Kubecek	e02b071950	vmmon: use standard definition of MSR_TSX_FORCE_ABORT if available The MSR_TSX_FORCE_ABORT macro is defined in mainline since commit 52f64909409c ("x86: Add TSX Force Abort CPUID/MSR") in v5.1-rc1.	2020-05-30 02:48:55 +02:00
Michal Kubecek	f98d4d9c34	vmnet: convert VNetBridge debugging code to ktime_get_ns() The do_gettimeofday() helper was removed by commit e4b92b108c6c ("timekeeping: remove obsolete time accessors") in v5.0-rc1 and since commit c766d1472c70 ("y2038: hide timeval/timespec/itimerval/itimerspec types") in v5.6-rc3, struct timeval should no longer be used in kernel code. Convert the do_gettimeofday() relics in VNetBridge (which are only compiled with LOGLEVEL >= 4) completely to ktime based interface.	2020-05-30 02:48:43 +02:00
Michal Kubecek	c21a12d6ff	vmmon: check presence of file_operations::poll() Commit 11c5ad0ec441 ("eventpoll: switch to ->poll_mask") in v4.18-rc1, made eventpoll switch from ->poll() to ->poll_mask(), resulting in a null pointer dereference as vmmon calls the callback directly without checking its availability. Rather than calling the callback directly (which would result in null pointer dereference), use vfs_poll() wrapper. As this wrapper is only available since 4.18-rc1 cycle as well, provide a copy to use when building against older kernels. Note: even if the commit revealing the problem was reverted in 4.18-rc3 (and is unlikely to go back in the same form) so that vmmon no longer crashes with 4.18 kernel, calling the callback without a check is a bad practice so let's keep the patch in place.	2020-05-30 02:48:11 +02:00
Michal Kubecek	2bb5a1991c	vmmon: fix indirect call with retpoline build Build against kernel with retpoline support issues warning objtool: Task_Switch()+0x425: indirect call found in RETPOLINE build This is because an indirect call in TaskSwitchToMonitor() is encoded using inline assembler so that it bypasses retpoline generation. For this purpose, macro CALL_NOSPEC exists since v4.15-rc8 (and has been backported into some distribution kernels with the rest of retpoline support). Use the macro if available and fallback to the original code if not.	2020-05-30 02:47:57 +02:00
Michal Kubecek	e384440ab1	vmmon: fix always_inline attribute usage Function declared with __attribute__((always_inline)) should also be declared as inline, otherwise gcc issues a warning "always_inline function might not be inlinable". It's just cosmetic but getting rid of known harmless warnings makes it easier to spot actual problems. Use the __always_inline macro for LinuxDriverSyncReadTSCs() as this is how always inline functions should be declared in kernel code.	2020-05-30 02:47:50 +02:00
Michal Kubecek	05049f29d7	vmmon: use standard definition of MSR_K7_HWCR_SMMLOCK if available The MSR_K7_HWCR_SMMLOCK macro is defined in mainline since commit 18c71ce9c882 ("x86/CPU/AMD: Add the Secure Encrypted Virtualization CPU feature") in v4.16-rc1.	2020-05-30 02:47:42 +02:00
Theo Bertacchini	240df2c940	modules: remove .cache.mk on make clean Remove .cache.mk inside vmmon-only and vmnet-only when executing "make clean". This file can cause issues when upgrading gcc as make will still look for includes inside older gcc includes directory. File .cache.mk with cache of generated variables was created by build since kernel v4.15-rc1, commit 3298b690b21c ("kbuild: Add a cache for generated variables") until the feature was removed in v4.18-rc1, commit e08d6de4e532 ("kbuild: remove kbuild cache").	2020-05-30 02:46:10 +02:00
Michal Kubecek	5d69da38fe	vmmon: quick workaround for objtool warnings As discussed in https://bugzilla.suse.com/show_bug.cgi?id=1059674 the reason for multiple objtool warnings is the fact that vmmon module defines its own Panic() function which never returns. While it is marked as such which is used by the compiler for optimization, there is no way to find this from object file. While this seems innocuous, it might result in problems with unwinder later. The quickest way around is to replace vmmon's own Panic() with standard kernel panic() until a cleaner solution is found.	2020-05-30 02:46:01 +02:00
Michal Kubecek	48563763f7	vmmon: use standard definition of CR3_PCID_MASK if available The CR3_PCID_MASK is defined in mainline asm/processor-flags.h since commit 6c690ee1039b ("x86/mm: Split read_cr3() into read_cr3_pa() and __read_cr3()") in v4.13-rc1.	2020-05-30 02:45:45 +02:00
Michal Kubecek	e8af2c1e83	vmmon: use standard definition of MSR_MISC_FEATURES_ENABLES if available The MSR_MISC_FEATURES_ENABLES macro is defined in mainline since commit ab6d9468631a ("x86/msr: Rename MISC_FEATURE_ENABLES to MISC_FEATURES_ENABLES") in v4.12-rc1. The MSR_MISC_FEATURES_ENABLES_CPUID_FAULT macro is defined in mainline since commit e9ea1e7f53b8 ("x86/arch_prctl: Add ARCH_[GET\|SET]_CPUID") in v4.12-rc1.	2020-05-30 02:45:35 +02:00
Michal Kubecek	bad750a0d8	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE_VMXNET3 if available The PCI_DEVICE_ID_VMWARE_VMXNET3 macro is defined in mainline pci_ids.h since commit b1226c7db1d9 ("vmxnet3: Move PCI Id to pci_ids.h") in v4.10-rc1.	2020-05-30 02:44:33 +02:00
Michal Kubecek	c1263a00b2	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE if available The PCI_VENDOR_ID_VMWARE macro is defined in mainline pci_ids.h since commit 94e57fea6202 ("PCI: Move PCI_VENDOR_ID_VMWARE to pci_ids.h") in v3.18-rc1.	2020-05-30 02:44:04 +02:00
Michal Kubecek	992ae1d5a1	modules: do not override CC command line options Workstation/Player 15.5.0 tries to guess the right retpoline (Spectre) related compiler command line options, resulting in objtool warnings like warning: objtool: .text.__x86_indirect_thunk_rbp+0x0: unsupported intra-function call warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate e.g. in vmmon-only/common/memtrack.o (but also many others). This seems wrong; out-of-tree modules should be built with the same compiler options that were used to build the kernel they are built against. Distribution kernel devel packages usually make sure this is the way so let them do their job.	2020-05-30 02:43:52 +02:00
Michal Kubecek	06283a2606	vmmon: fix wrong VMX86_DEVEL test If VMX86_DEVEL macro is not defined, preprocessor warns when it is tested with "#if" in vmmon-only/linux/driver.c. As VMX86_DEVEL is tested with "#ifdef" everywhere else, change this one test to match the rest.	2020-05-30 02:43:42 +02:00