import Player 16.1.0 module sources

import Player 16.0.0 module sources
import Player 15.5.5 module sources
2020-11-20 09:13:14 +01:00 · 2020-09-15 20:45:48 +02:00 · 2020-05-30 02:30:43 +02:00 · 2020-03-14 09:59:24 +01:00 · 2019-11-14 07:56:57 +01:00 · 2019-09-21 00:11:26 +02:00
62 changed files with 3406 additions and 2659 deletions
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -136,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998,2015,2017 VMware, Inc. All rights reserved.
+# Copyright (C) 1998,2015,2017,2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -72,7 +72,7 @@ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')

 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
-C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o
+C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o crosspage.o
 C_TARGETS_VMCORE := moduleloop.o
 C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -447,7 +447,7 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 *
 * MonLoaderCallout_GetSharedUserPage --
 *
- *      Gets a user-provided shared page's MPN for a VCPU.
+ *      Gets a shared page's MPN for a VCPU.
 *
 * Returns:
 *      An MPN or INVALID_MPN.
@@ -478,7 +478,14 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   switch (subIndex) {
+   case MONLOADER_CROSS_PAGE_DATA_IDX:
+      return HostIF_GetCrossPageDataMPN(ctx->vm->crosspage[vcpu]);
+   case MONLOADER_CROSS_PAGE_CODE_IDX:
+      return HostIF_GetCrossPageCodeMPN();
+   default:
+      return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   }
 }


@@ -0,0 +1,609 @@
+/*********************************************************
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * crosspage.c --
+ *
+ *    Cross page code and data.
+ *
+ *    The cross code page contains world switch code and interrupt/exception
+ *    handlers in support of world switch.  A separate data page is also mapped
+ *    in both the vmm and vmmon address spaces to describe the host and monitor
+ *    state.
+ *
+ *    Both the monitor and the vmmon driver call into the cross page to switch
+ *    worlds.  The world switch from host->vmm is able to refer directly to the
+ *    various symbols defined in CrossPage_CodePage.  On the vmm->host path
+ *    the VmmToHost entry point is stored in the data page.  Data references
+ *    must be handled as offsets from the base of the data page.
+ *
+ *    Use of inline assembly in C code is delicate as the compiler has many
+ *    possible reasons to emit unexpected instructions surrounding any assembly
+ *    in a C function.  Frame pointers are inconvenient to disable on a per-file
+ *    basis given structure of the Linux Makefile so they are being emitted
+ *    in the C wrapper function.  NORETURN and NOT_REACHED_MINIMAL are used to
+ *    omit a function epilogue.  The output/clobber lists of the assembly blocks
+ *    are not used as the compiler needs not compensate for the register use of
+ *    these hand-written assembly functions.
+ *    Each assembly function begins with a label denoting the proper entry point
+ *    for that function.  Only a handful of the functions need to be exported
+ *    out of this file, see crosspage.h.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#      include <linux/objtool.h>
+#   elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
+#      include <linux/frame.h>
+#   endif
+#endif
+
+#include "modulecall.h"
+#include "iocontrols.h"
+#include "segs.h"
+#include "x86_basic_defs.h"
+#include "vm_basic_defs.h"
+#include "vm_idt_x86.h"
+#include "crosspage.h"
+
+#ifdef __APPLE__
+/*
+ * The Mac compiler expects symbols to have a _ prefix.
+ * This arranges for any symbol being exported out of the assembly code
+ * to be accessible.
+ */
+#define ASM_PREFIX "_"
+/*
+ * OSX uses a segment,section style syntax for specifying output sections.
+ * https://developer.apple.com/library/archive/documentation/Performance/
+ * Conceptual/CodeFootprint/Articles/MachOOverview.html
+ */
+#define ASM_SECTION "__TEXT,cross"
+#else
+/*
+ * This is a COFF grouped section identifier.  This ensures the crosspage code
+ * is merged with the normal .text area but as a separate unit so it can have
+ * its own alignment properties.
+ * https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
+ *
+ * The Linux build will handle the section based on its contents (code).
+ */
+#define ASM_SECTION ".text$cross"
+#define ASM_PREFIX
+#endif
+
+#define EXPORTED_ASM_SYMBOL(fn) ".global " ASM_PREFIX #fn "\n"   \
+                                ASM_PREFIX #fn ":\n"
+
+/*
+ * Tag the crosspage code C wrapper with the crosspage section and page
+ * alignment.
+ */
+#define CPCODE __attribute__((section(ASM_SECTION)))   \
+               __attribute__((aligned(PAGE_SIZE)))
+
+/* No special handling is needed for the data. */
+#define CPDATA
+
+#define VMMDATALA(off) (LPN_2_LA(CROSS_PAGE_DATA_START) + (off))
+
+#define NOT_REACHED_MINIMAL __builtin_unreachable
+
+#ifdef ANNOTATE_INTRA_FUNCTION_CALL
+#define ANNOTATE_ASM_CALL_STR2(x) #x
+#define ANNOTATE_ASM_CALL_STR(x) ANNOTATE_ASM_CALL_STR2(x) "\n"
+#define ANNOTATE_ASM_CALL ANNOTATE_ASM_CALL_STR(ANNOTATE_INTRA_FUNCTION_CALL)
+#else
+#define ANNOTATE_ASM_CALL
+#endif
+
+void VmmToHost(void);
+
+CPDATA const VMCrossPageData cpDataTemplate = {
+   .version        = CROSSPAGE_VERSION,
+   .vmmonVersion   = VMMON_VERSION,
+
+   .monRSP         = VMM_LRET_STACK_TOP,
+   .monSS          = SYSTEM_DATA_SELECTOR,
+   .monDS          = SYSTEM_DATA_SELECTOR,
+   .monES          = SYSTEM_DATA_SELECTOR,
+
+   .vmmToHostLA    = (VA)VmmToHost,
+   .wsCR0          = CR0_PE | CR0_MP | CR0_EM | CR0_NE | CR0_WP | CR0_PG,
+   .wsCR4          = CR4_PAE | CR4_OSFXSR,
+
+   .monTask.rsp[0] = VMM_STACK_TOP,                   /* Monitor stack. */
+   .monTask.rsp[1] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL1 is not used. */
+   .monTask.rsp[2] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL2 is not used. */
+
+   .monTask.ist[IST_NONE]                 = 0, /* No stack switch. */
+   .monTask.ist[IST_VMM_DF]               = DF_STACK_TOP,
+   .monTask.ist[IST_VMM_NMI]              = NMI_STACK_TOP,
+   .monTask.ist[IST_VMM_MCE]              = MC_STACK_TOP,
+
+   .monTask.IOMapBase = sizeof(Task64),
+
+   .monGDTR.limit  = GDT_LIMIT,
+   .monGDTR.offset = GDT_START_VA,
+
+   .shadowDR[6].ureg64 = DR6_DEFAULT,
+   .shadowDR[7].ureg64 = DR7_ONES,
+
+   /*
+    * switchHostIDT and switchHostIDTR are initialized in vmmon.
+    * switchMonIDTR is static and can be fully initialized at compile-time.
+    * switchMonIDT is mostly static with all non-present entries except for
+    * the 4 handled exceptions which are populated by the crosspage setup
+    * code in task.c
+    */
+   .switchMonIDTR  = { sizeof(Gate64) * NUM_EXCEPTIONS - 1,
+                       VMMDATALA(offsetof(VMCrossPageData, switchMonIDT)) },
+};
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * CrossPage_CodePage --
+ *
+ *      This function only serves as an anchor and wrapper for the code that
+ *      runs in the world switch's cross page.  The function must not be called
+ *      directly.  Its address may be used to locate the cross page code.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+CPCODE void
+CrossPage_CodePage(void)
+{
+   __asm__ __volatile__ (
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchDBHandler --
+    *
+    *      DB handler that operates during worldswitch (in both directions).
+    *
+    *      This handler is expected to be used in two cases:
+    *      1) If the host kernel has set the DR7_GD (guard) bit, when switch
+    *         code attempts to save debug registers, a #DB will be thrown.  In
+    *         this case, the handler sets wsException[EXC_DB] and returns.
+    *      2) When stress-testing the worldswitch code, RFLAGS.TF is set across
+    *         most switch paths.  This causes a #DB to be thrown before each
+    *         instruction is executed.  In this case, the handler must set
+    *         RFLAGS.RF in the return frame to guarantee progress while
+    *         RFLAGS.TF is set.  For added stress, the handler induces a
+    *         simulated #NMI.
+    *
+    *      In case (1), returning is accomplished via a simulated iret in order
+    *      to preserve any NMI-blocking.  In case (2), the inherent
+    *      incompatibility of RFLAGS.RF for single-stepping and simulating an
+    *      iret via popf and lretq necessitates the use of an actual iret.  As
+    *      a result, there is no strong NMI-blocking guarantee when
+    *      stress-testing the switch path.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_DB] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchDBHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionDB], %%rax\n"
+   "movb         $1,               (%%rax)\n" /* log EXC_DB */
+   "popq         %%rax\n"
+   "testl        %[EFLAGS_TF],     16(%%rsp)\n" /* check trap flag */
+   "jz           return_without_enabling_nmi\n"
+   "orl          %[EFLAGS_RF],     16(%%rsp)\n"
+   "int          $2\n"
+   "iretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchUDHandler --
+    *
+    *      UD handler that operates during worldswitch (in both directions).
+    *
+    *      It sets the wsException[EXC_UD] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_UD] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchUDHandler)
+   "pushq        %%rax\n"
+   "pushq        %%rbx\n"
+   "pushq        %%rcx\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "movl         %[wsExceptionUD],      %%ecx\n"    /* log EXC_UD */
+   "movb         $1,                    (%%rax, %%rcx)\n"
+   /* Check if the exception came from a monitor RIP. */
+   "cmpq         %[MONITOR_MINIMUM_VA], 24(%%rsp)\n"
+   "jae          monitor_context\n"
+   /* Check if the exception came from the crosspage code. */
+   "movq         24(%%rsp),              %%rbx\n"
+   "andq         %[PageAlignMask],       %%rbx\n"
+   "movl         %[crosspageCodeLA],     %%ecx\n"
+   "cmpq         (%%rax, %%rcx),         %%rbx\n"
+   "je           monitor_context\n"
+   /*
+    * Linux halts the processor on host #UD2.  Act similarly.  PR 1281662.
+    * Note that this can escape due to an #NMI, #MC, or #DB.
+    */
+   "cli\n"
+   "hlt\n"
+   /* Advance 2 bytes to skip past the #UD2 instruction. */
+   "monitor_context:\n"
+   "movq         24(%%rsp), %%rbx\n"
+   "movl         %[wsUD2],  %%ecx\n"
+   "movq         %%rbx,     (%%rax, %%rcx)\n"
+   "addq         $2,        24(%%rsp)\n"
+   "popq         %%rcx\n"
+   "popq         %%rbx\n"
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchNMIHandler --
+    *
+    *      NMI handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It 'simply' sets the wsException[EXC_NMI] flag then returns,
+    *      leaving further NMI delivery inhibited.
+    *
+    *      As long as we don't execute 'iret', the CPU will not allow further
+    *      NMIs to be delivered; this is how the CPU protects itself from
+    *      runaway NMIs eating up the stack and triple-faulting.
+    *
+    *      See Vol 3, 5.5.1, Handling multiple NMIs.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_NMI] = TRUE
+    *      further NMI delivery inhibited
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchNMIHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionNMI], %%rax\n"
+   "movb         $1,                (%%rax)\n" /* log EXC_NMI */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchMCEHandler --
+    *
+    *      MCE handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It simply sets the wsException[EXC_MC] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_MC] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchMCEHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionMC], %%rax\n"
+   "movb         $1,              (%%rax)\n" /* log EXC_MC */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchHandlerReturns --
+    *
+    *      Shared code to return without enabling NMIs.  Switch handlers call
+    *      directly to these labels as needed.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "return_without_enabling_nmi:\n"
+   "pushq        %%rbp\n"
+   "pushq        %%rax\n"
+   "movq         40(%%rsp), %%rbp\n"
+   "subq         $32, %%rbp\n"
+   "movq         24(%%rsp), %%rax\n"
+   "movq         %%rax, 24(%%rbp)\n"
+   "movq         16(%%rsp), %%rax\n"
+   "movq         %%rax, 16(%%rbp)\n"
+   "movq         32(%%rsp), %%rax\n"
+   "movq         %%rax, 8(%%rbp)\n"
+   "movq         8(%%rsp),  %%rax\n"
+   "movq         %%rax, (%%rbp)\n"
+   "popq         %%rax\n"
+   "movq         %%rbp, %%rsp\n"
+   "popq         %%rbp\n"
+   "popfq\n"
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * HostToVmm --
+    *
+    *      Switch from host to monitor.
+    *
+    * Entered with:
+    *      RCX = crosspage host address
+    *      (RSP) = return to host address
+    *
+    * Must preserve:
+    *      RBX, RSI, RDI, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(HostToVmm)
+   /* Create an lret frame on the host stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold host %rsp */
+
+   /* Start from the "empty" host context and save. */
+   "leaq            %c[hostContextEmpty](%%rcx), %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* host %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rdi\n"
+   "pushq           %%rsi\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" monitor context and load. */
+   "leaq            %c[monContextFull](%%rcx), %%rsp\n"
+   "popq            %%rsi\n" /* monitor %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+   "popq            %%rax\n" /* monitor %rsp */
+   "popw            %%dx\n"  /* monitor %ss */
+   /* Load the cross GDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%rsi, %%cr3\n"
+   /* Reload the monitor's %ss into %ds/%es. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * VmmToHost --
+    *
+    *      Switch from monitor to host.
+    *
+    * Must preserve:
+    *      RBX, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(VmmToHost)
+   "movq            %c[VMMCROSSPAGE] + %c[crosspageDataLA], %%rcx\n"
+   /* Create an lret frame on the monitor stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold monitor %rsp */
+
+   /* Start from the "empty" monitor context and save. */
+   "leaq            %c[VMMCROSSPAGE] + %c[monContextEmpty], %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* monitor %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" host context and load. */
+   "leaq            %c[hostContextFull](%%rcx), %%rsp\n"
+   "popq            %%r9\n" /* host %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%rsi\n" /* Not ABI-required. */
+   "popq            %%rdi\n" /* Not ABI-required. */
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+
+   "popq            %%rax\n" /* host %rsp */
+   "popw            %%dx\n"  /* host %ss */
+   /* Load the cross GDT and IDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   "lidtq           %c[switchHostIDTR](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%r9,  %%cr3\n"
+   /* Reload the host's %ss into %ds/%es.  Technically wrong. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   /* Microsoft RTL/codegen assumes EFLAGS<DF> = 0. */
+   "cld\n"
+   "lretq\n"
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchExcGetCrossPageData --
+    *
+    *      Common function for the exception handlers to locate the data
+    *      crosspage so they can record their respective events.  In order to
+    *      reach this code an exception had to vector through the IDT.  The IDT
+    *      is known to be in the data page.  Therefore, the data page can be
+    *      found by accessing IDTR and rounding down to page alignment.
+    *
+    * Input:
+    *     None
+    *
+    * Output:
+    *     %rax = Page aligned address of the current crosspage data area.
+    *
+    * Note:
+    *     %rax (return value) and %rflags are destroyed, all other registers
+    *     are preserved.  Since this is only called by exception handlers, the
+    *     CPU has already saved %rflags so no additional handling is required.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "SwitchExcGetCrossPageData:\n"
+   "subq            $0x10,            %%rsp\n"
+   "sidt            0(%%rsp)\n"
+   "movq            2(%%rsp),         %%rax\n" /* DTR.offset */
+   "addq            $0x10,            %%rsp\n"
+   "andq            %[PageAlignMask], %%rax\n"
+   "ret\n"
+
+   EXPORTED_ASM_SYMBOL(CrossPage_CodeEnd)
+
+   : /* No output list, this is not really C code. */
+   : [MONITOR_MINIMUM_VA] "i" (MONITOR_LINEAR_START),
+     [PageAlignMask]      "i" (~PAGE_MASK),
+     [EFLAGS_TF]          "i" (EFLAGS_TF),
+     [EFLAGS_RF]          "i" (EFLAGS_RF),
+     [VMMCROSSPAGE]       "i" (CROSS_PAGE_DATA_START * PAGE_SIZE),
+     [wsUD2]              "i" (offsetof(VMCrossPageData, wsUD2)),
+
+     [wsExceptionDB]      "i" (offsetof(VMCrossPageData, wsException[EXC_DB])),
+     [wsExceptionUD]      "i" (offsetof(VMCrossPageData, wsException[EXC_UD])),
+     [wsExceptionNMI]     "i" (offsetof(VMCrossPageData, wsException[EXC_NMI])),
+     [wsExceptionMC]      "i" (offsetof(VMCrossPageData, wsException[EXC_MC])),
+
+     [hostContextEmpty]   "i" (offsetof(VMCrossPageData, hostDS)),
+     [monContextEmpty]    "i" (offsetof(VMCrossPageData, monDS)),
+
+     [hostContextFull]    "i" (offsetof(VMCrossPageData, hostCR3)),
+     [monContextFull]     "i" (offsetof(VMCrossPageData, monCR3)),
+
+     [crosspageDataLA]    "i" (offsetof(VMCrossPageData, crosspageDataLA)),
+     [crosspageCodeLA]    "i" (offsetof(VMCrossPageData, crosspageCodeLA)),
+     [crossGDTHKLADesc]   "i" (offsetof(VMCrossPageData, crossGDTHKLADesc)),
+     [switchHostIDTR]     "i" (offsetof(VMCrossPageData, switchHostIDTR))
+   );
+   NOT_REACHED_MINIMAL();
+}
+
+#ifdef STACK_FRAME_NON_STANDARD
+STACK_FRAME_NON_STANDARD(CrossPage_CodePage);
+#endif
+
+/*
+ * The crosspage data must not exceed a single page.
+ * The code area also must fit in a single page but that size isn't available
+ * until link time.
+ */
+MY_ASSERTS(CROSSPAGE_DATA,
+   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
+)
@@ -0,0 +1,43 @@
+/*********************************************************
+ * Copyright (C) 2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _CROSSPAGE_H_
+#define _CROSSPAGE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct VMCrossPageData;
+
+extern void HostToVmm(struct VMCrossPageData *crosspageWIN, void *unused1,
+                      void *unused2, struct VMCrossPageData *crosspageSYSV);
+extern const VMCrossPageData cpDataTemplate;
+
+extern void SwitchDBHandler(void);
+extern void SwitchUDHandler(void);
+extern void SwitchNMIHandler(void);
+extern void SwitchMCEHandler(void);
+extern void CrossPage_CodePage(void);
+extern void CrossPage_CodeEnd(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,9 +59,18 @@

 #define CROSSCALL_SLEEP_US 1000

+typedef struct HostIFContigMemMap {
+   MPN mpn;
+   void *addr;
+   PageCnt pages;
+   struct HostIFContigMemMap *next;
+} HostIFContigMemMap;
+
 EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
-EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);
+
+EXTERN MPN   HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData);
+EXTERN MPN   HostIF_GetCrossPageCodeMPN(void);

 EXTERN void *HostIF_AllocPage(void);
 EXTERN void  HostIF_FreePage(void *ptr);
@@ -82,22 +91,26 @@ EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
 EXTERN void *HostIF_AllocKernelPages(PageCnt numPages, MPN *mpns);
 EXTERN void  HostIF_FreeKernelPages(PageCnt numPages, void *ptr);
+EXTERN HostIFContigMemMap *HostIF_AllocContigPages(VMDriver *vm,
+                                                   PageCnt numPages);
+EXTERN void  HostIF_FreeContigPages(VMDriver *vm, HostIFContigMemMap *mapping);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
 EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 #endif

-EXTERN Bool  HostIF_APICInit(VMDriver *vm, Bool setVMPtr, Bool probe);
+EXTERN void  HostIF_APICInit(VMDriver *vm);
 EXTERN uint8 HostIF_GetMonitorIPIVector(void);
 EXTERN uint8 HostIF_GetHVIPIVector(void);
+EXTERN uint8 HostIF_GetPerfCtrVector(void);
 EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);

 EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
                                  Vcpuid vcpuid,
                                  uint64 *args);

-EXTERN int   HostIF_SemaphoreSignal(uint64 *args);
+EXTERN int   HostIF_SemaphoreSignal(VMDriver *vm, uint64 *args);

 EXTERN void  HostIF_SemaphoreForceWakeup(VMDriver *vm, const VCPUSet *vcs);
 EXTERN void  HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,6 +66,8 @@
 #include "x86paging_64.h"
 #include "memtrack.h"
 #include "monLoader.h"
+#include "segs.h"
+#include "crosspage.h"

 #ifdef LINUX_GDT_IS_RO
 #   include <asm/desc.h>
@@ -588,7 +590,13 @@ Bool
 Task_Initialize(void)
 {
   unsigned i;
+   VA cpStart = (VA)CrossPage_CodePage;
+   VA cpEnd   = (VA)CrossPage_CodeEnd;

+   if (VA_2_VPN(cpStart) != VA_2_VPN(cpEnd)) {
+      Warning("Crosspage code validation failed.\n");
+      return FALSE;
+   }
   ASSERT_ON_COMPILE(sizeof(Atomic_uint64) == sizeof(MPN));
   for (i = 0; i < ARRAYSIZE(hvRootPage); i++) {
      Atomic_Write64(&hvRootPage[i], INVALID_MPN);
@@ -828,7 +836,7 @@ TaskSwitchPTPLookupVPN(VMDriver *vm, VPN vpn)
 *    before/after doing a BackToHost.
 *
 *    For each populated patch, performs a page walk from L4 to the patch's
- *    level.  The lowest allowed level is L3.  Invalid patch levels, collisions
+ *    level.  The lowest allowed level is L2.  Invalid patch levels, collisions
 *    at the patch's level with existing PTEs and non-present PTEs during the
 *    walk all result in failure.
 *
@@ -847,6 +855,7 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
      VMMPageTablePatch *patch = &cpData->vmmPTP[i];
      unsigned l4idx = PT_LPN_2_L4OFF(patch->lpn);
      unsigned l3idx = PT_LPN_2_L3OFF(patch->lpn);
+      unsigned l2idx = PT_LPN_2_L2OFF(patch->lpn);
      PT_L4E pte = 0;
      MA ma = cpData->monCR3 + l4idx * sizeof pte; /* PTE machine address. */

@@ -886,6 +895,34 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
               return FALSE;
            }
            break;
+         case 2:
+            ASSERT(patch->pteIdx == l2idx);
+            if (!PTE_PRESENT(pte)) {
+               return FALSE; /* Terminate page walk, L4 not present. */
+            }
+            ma = MPN_2_MA(LM_PTE_2_PFN(pte)) + l3idx * sizeof pte;
+            if (HostIF_ReadPhysical(vm, ma, PtrToVA64(&pte), TRUE,
+                                    sizeof pte) != 0) {
+               return FALSE;
+            }
+            if (!PTE_PRESENT(pte)) {
+               return FALSE; /* Terminate page walk, L3 not present. */
+            }
+            ma = MPN_2_MA(LM_PTE_2_PFN(pte)) + l2idx * sizeof pte;
+            if (HostIF_ReadPhysical(vm, ma, PtrToVA64(&pte), TRUE,
+                                    sizeof pte) != 0) {
+               return FALSE;
+            }
+            if (pte != 0) {
+               return FALSE; /* Collision. */
+            }
+
+            if (HostIF_WritePhysical(vm, ma, PtrToVA64(&patch->pte), TRUE,
+                                     sizeof patch->pte) != 0) {
+               return FALSE;
+            }
+            break;
+
         default:
            return FALSE; /* Invalid level. */
            break;
@@ -945,11 +982,14 @@ TaskVerifyPTPatches(VMDriver *vm,
                    VMCrossPageData *cpData)
 {
   LPN64 xgCPN = LA_2_LPN(cpData->crossGDTLA);
-   LPN64 xpCPN = LA_2_LPN(cpData->crossPageLA);
-   MPN   xpMPN = MA_2_MPN(cpData->crosspageMA);
+   LPN64 xpCPN = LA_2_LPN(cpData->crosspageDataLA);
+   MPN   xpMPN = MA_2_MPN(cpData->crosspageDataMA);
   MA    cr3   = cpData->monCR3;
+   LPN64 xpCodeCPN = LA_2_LPN(cpData->crosspageCodeLA);
+   MPN   xpCodeMPN = HostIF_GetCrossPageCodeMPN();
   return TaskVerifyPTMap(vm, cr3, xpCPN, xpMPN) &&
-          TaskVerifyPTMap(vm, cr3, xgCPN, crossGDTMPN);
+          TaskVerifyPTMap(vm, cr3, xgCPN, crossGDTMPN) &&
+          TaskVerifyPTMap(vm, cr3, xpCodeCPN, xpCodeMPN);
 }


@@ -1172,11 +1212,10 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
 /*
 *-----------------------------------------------------------------------------
 *
- * TaskPopulateHostSwitchIDTE --
+ * TaskPopulateSwitchIDTE --
 *
- *      Populates a host interrupt descriptor in the crosspage, setting its
- *      handler to the associated gate stub, given a host code segment selector
- *      and the crosspage.
+ *      Populates an interrupt descriptor in the crosspage, setting its
+ *      handler to the associated gate stub given a code segment selector.
 *
 * Results:
 *      None
@@ -1188,17 +1227,13 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
 */

 static void
-TaskPopulateHostSwitchIDTE(unsigned idx, Selector cs, VMCrossPage *cp)
+TaskPopulateSwitchIDTE(unsigned idx, Selector cs, Gate64 *idt, VA handlerVA)
 {
   Gate64 *entry;
-   VA64 handlerVA;

   ASSERT_ON_COMPILE(sizeof *entry == 16);
   ASSERT(idx < NUM_EXCEPTIONS);
-
-   handlerVA = PtrToVA64(&cp->crosspageCode.gateStubs[idx * CP_STUB_SIZE]);
-
-   entry = &cp->crosspageData.switchHostIDT[idx];
+   entry = &idt[idx];

   entry->offset_0_15  = LOWORD(handlerVA);
   entry->offset_16_31 = HIWORD(handlerVA);
@@ -1212,13 +1247,21 @@ TaskPopulateHostSwitchIDTE(unsigned idx, Selector cs, VMCrossPage *cp)
   entry->reserved1    = 0;
 }

+static VA
+TaskVmmCodeVA(void (*handler)(void))
+{
+   VA baseVA = VPN_2_VA(CROSS_PAGE_CODE_START);
+   VA offs = (VA)handler % PAGE_SIZE;
+
+   return baseVA + offs;
+}

 /*
 *-----------------------------------------------------------------------------
 *
- *  TaskInitHostSwitchIDT --
+ *  TaskInitSwitchIDT --
 *
- *      Initializes the contents of the host-context switch IDT and IDTR.
+ *      Initializes the contents of the switch IDTs and IDTR.
 *      Uses the cross page's host kernel linear address and host kernel CS.
 *
 * Results:
@@ -1231,18 +1274,28 @@ TaskPopulateHostSwitchIDTE(unsigned idx, Selector cs, VMCrossPage *cp)
 */

 static void
-TaskInitHostSwitchIDT(VMCrossPage *crosspage)
+TaskInitSwitchIDTs(VMCrossPageData *cpData)
 {
-   VMCrossPageData *cpData = &crosspage->crosspageData;
-   const Selector   cs     = cpData->hostInitial64CS;
+   const Selector hostCS = cpData->hostInitial64CS;
+   Gate64 *switchHostIDT = cpData->switchHostIDT;
+   Gate64 *switchMonIDT  = cpData->switchMonIDT;

   cpData->switchHostIDTR.limit = sizeof cpData->switchHostIDT - 1;
   cpData->switchHostIDTR.offset = PtrToVA64(&cpData->switchHostIDT);

-   TaskPopulateHostSwitchIDTE(EXC_DB,  cs, crosspage);
-   TaskPopulateHostSwitchIDTE(EXC_NMI, cs, crosspage);
-   TaskPopulateHostSwitchIDTE(EXC_UD,  cs, crosspage);
-   TaskPopulateHostSwitchIDTE(EXC_MC,  cs, crosspage);
+   TaskPopulateSwitchIDTE(EXC_DB,  hostCS, switchHostIDT, (VA)SwitchDBHandler);
+   TaskPopulateSwitchIDTE(EXC_NMI, hostCS, switchHostIDT, (VA)SwitchNMIHandler);
+   TaskPopulateSwitchIDTE(EXC_UD,  hostCS, switchHostIDT, (VA)SwitchUDHandler);
+   TaskPopulateSwitchIDTE(EXC_MC,  hostCS, switchHostIDT, (VA)SwitchMCEHandler);
+
+   TaskPopulateSwitchIDTE(EXC_DB,  SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchDBHandler));
+   TaskPopulateSwitchIDTE(EXC_NMI, SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchNMIHandler));
+   TaskPopulateSwitchIDTE(EXC_UD,  SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchUDHandler));
+   TaskPopulateSwitchIDTE(EXC_MC,  SYSTEM_CODE_SELECTOR, switchMonIDT,
+                          TaskVmmCodeVA(SwitchMCEHandler));
 }


@@ -1554,7 +1607,7 @@ TaskCreatePTPatch(VMDriver        *vm,
 /*
 *-----------------------------------------------------------------------------
 *
- * TaskCreatePTPatches
+ * TaskCreatePTPatches --
 *
 *      Creates VMM mappings for the crosspage and crossGDT that match
 *      those from the vmmon address space, so that they are mapped at
@@ -1569,16 +1622,19 @@ TaskCreatePTPatch(VMDriver        *vm,
 */

 static Bool
-TaskCreatePTPatches(VMDriver    *vm,
-                    VMCrossPage *crosspage,
-                    LPN64        monStartLPN,
-                    LPN64        monEndLPN,
-                    uint16      *numPages)
+TaskCreatePTPatches(VMDriver        *vm,
+                    VMCrossPageData *cpData,
+                    LPN64            monStartLPN,
+                    LPN64            monEndLPN,
+                    uint16          *numPages)
 {
-   VMCrossPageData * const cpData = &crosspage->crosspageData;
-   LPN64                   xgCPN  = LA_2_LPN(cpData->crossGDTLA);
-   LPN64                   xpCPN  = LA_2_LPN(cpData->crossPageLA);
-   MPN                     xpMPN  = MA_2_MPN(cpData->crosspageMA);
+   LPN64                   xgCPN     = LA_2_LPN(cpData->crossGDTLA);
+   LPN64                   xpCPN     = LA_2_LPN(cpData->crosspageDataLA);
+   MPN                     xpMPN     = MA_2_MPN(cpData->crosspageDataMA);
+   LPN64                   xpCodeCPN = LA_2_LPN(cpData->crosspageCodeLA);
+   MPN                     xpCodeMPN = HostIF_GetCrossPageCodeMPN();
+
+   ASSERT(xpCodeMPN != INVALID_MPN);
   /*
    * Set up patches that the BackToHost code will use to map the
    * crosspage and crossGDT at their crossover addresses (HKLA for
@@ -1587,7 +1643,9 @@ TaskCreatePTPatches(VMDriver    *vm,
   if (TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
                         xpCPN, xpMPN, numPages) &&
       TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
-                         xgCPN, crossGDTMPN, numPages)) {
+                         xgCPN, crossGDTMPN, numPages) &&
+       TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN,
+                         xpCodeCPN, xpCodeMPN, numPages)) {
      TaskFixupPTPatches(vm, cpData);
      return TRUE;
   }
@@ -1612,9 +1670,9 @@ TaskCreatePTPatches(VMDriver    *vm,
 */

 Bool
-Task_InitCrosspage(VMDriver *vm,               // IN
-                   LPN monStartLPN,            // IN
-                   LPN monEndLPN,              // IN
+Task_InitCrosspage(VMDriver     *vm,           // IN
+                   LPN           monStartLPN,  // IN
+                   LPN           monEndLPN,    // IN
                   PerVcpuPages *perVcpuPages) // IN
 {
   Vcpuid vcpuid;
@@ -1626,31 +1684,19 @@ Task_InitCrosspage(VMDriver *vm,               // IN

   ASSERT(0 < vm->numVCPUs && vm->numVCPUs <= MAX_VCPUS);
   for (vcpuid = 0; vcpuid < vm->numVCPUs; vcpuid++) {
-      VA64             crossPageUserAddr = perVcpuPages[vcpuid].crosspage;
-      VMCrossPage     *p = HostIF_MapCrossPage(vm, crossPageUserAddr);
-      VMCrossPageData *cpData;
-      MPN              crossPageMPN;
+      VMCrossPageData *cpData = vm->crosspage[vcpuid];
+      MPN              crossPageDataMPN;

-      if (p == NULL) {
-         return FALSE;
-      }
-      cpData = &p->crosspageData;
-
-      HostIF_VMLock(vm, 38);
-      if (HostIF_LookupUserMPN(vm, crossPageUserAddr, &crossPageMPN) !=
-          PAGE_LOOKUP_SUCCESS ||
-          crossPageMPN == 0) {
-         HostIF_VMUnlock(vm, 38);
-         return FALSE;
-      }
-      HostIF_VMUnlock(vm, 38);
+      crossPageDataMPN = HostIF_GetCrossPageDataMPN(cpData);
+      ASSERT(crossPageDataMPN != INVALID_MPN);
+      memcpy(cpData, &cpDataTemplate, sizeof *cpData);

      /*
       * The version of the crosspage must be the first four bytes of the
       * crosspage.  See the declaration of VMCrossPage in modulecall.h.
       */

-      ASSERT_ON_COMPILE(offsetof(VMCrossPage, crosspageData.version) == 0);
+      ASSERT_ON_COMPILE(offsetof(VMCrossPageData, version) == 0);
      ASSERT_ON_COMPILE(sizeof(cpData->version) == sizeof(uint32));

      /* cpData->version is VMX's version; CROSSPAGE_VERSION is vmmon's. */
@@ -1660,12 +1706,9 @@ Task_InitCrosspage(VMDriver *vm,               // IN
                 (int)CROSSPAGE_VERSION, cpData->version);
         return FALSE;
      }
-      if (!pseudoTSC.initialized) {
-         Warning("%s*: PseudoTSC has not been initialized\n", __FUNCTION__);
-         return FALSE;
-      }
-      cpData->crosspageMA = MPN_2_MA(crossPageMPN);
-      cpData->crossPageLA = (LA64)(uintptr_t)p;
+      cpData->crosspageDataMA = MPN_2_MA(crossPageDataMPN);
+      cpData->crosspageDataLA = (LA64)(uintptr_t)cpData;
+      cpData->crosspageCodeLA = (LA64)CrossPage_CodePage;
      cpData->crossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.offset = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.limit  = sizeof(CrossGDT) - 1;
@@ -1675,7 +1718,8 @@ Task_InitCrosspage(VMDriver *vm,               // IN
      cpData->monCR3 = MPN_2_MA(vm->ptRootMpns[vcpuid]);

      HostIF_VMLock(vm, 39);
-      if (!TaskCreatePTPatches(vm, p, monStartLPN, monEndLPN, &numPTPPages)) {
+      if (!TaskCreatePTPatches(vm, cpData, monStartLPN, monEndLPN,
+                               &numPTPPages)) {
         HostIF_VMUnlock(vm, 39);
         Warning("%s: Could not create page table patches for VCPU %d\n",
                 __FUNCTION__, vcpuid);
@@ -1713,9 +1757,8 @@ Task_InitCrosspage(VMDriver *vm,               // IN
      cpData->timerIntrTS           = MAX_ABSOLUTE_TS;
      cpData->hstTimerExpiry        = MAX_ABSOLUTE_TS;
      cpData->monTimerExpiry        = MAX_ABSOLUTE_TS;
-      vm->crosspage[vcpuid]         = p;

-      TaskInitHostSwitchIDT(p);
+      TaskInitSwitchIDTs(cpData);
   }
   /*
    * Store the number of pages allocated for this VM's page table patches so
@@ -1870,16 +1913,16 @@ TaskDisableTF(void)


 static INLINE Bool
-TaskGotException(const VMCrossPage *crosspage, unsigned exc)
+TaskGotException(const VMCrossPageData *crosspageData, unsigned exc)
 {
-   return crosspage->crosspageData.wsException[exc];
+   return crosspageData->wsException[exc];
 }


 static INLINE void
-TaskSetException(VMCrossPage *crosspage, unsigned exc, Bool v)
+TaskSetException(VMCrossPageData *crosspageData, unsigned exc, Bool v)
 {
-   crosspage->crosspageData.wsException[exc] = v;
+   crosspageData->wsException[exc] = v;
 }


@@ -1905,7 +1948,7 @@ TaskSetException(VMCrossPage *crosspage, unsigned exc, Bool v)
 */

 static INLINE_SINGLE_CALLER void
-TaskSaveDebugRegisters(VMCrossPage *crosspage)
+TaskSaveDebugRegisters(VMCrossPageData *crosspageData)
 {
   Bool saveGotDB;

@@ -1913,13 +1956,13 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
   do {                                                 \
      uintptr_t drReg;                                  \
      GET_DR##n(drReg);                                 \
-      crosspage->crosspageData.hostDR[n] = drReg;       \
+      crosspageData->hostDR[n] = drReg;                 \
   } while (0)

   /* Hardware contains the host's %dr7, %dr6, %dr3, %dr2, %dr1, %dr0 */
-   crosspage->crosspageData.hostDRInHW = ((1 << 7) | (1 << 6) |
-                                          (1 << 3) | (1 << 2) |
-                                          (1 << 1) | (1 << 0));
+   crosspageData->hostDRInHW = ((1 << 7) | (1 << 6) |
+                                (1 << 3) | (1 << 2) |
+                                (1 << 1) | (1 << 0));

   /*
    * Save DR7 since we need to disable debug breakpoints during the world
@@ -1927,8 +1970,8 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * SwitchDBHandler simply IRETs after setting crosspage gotDB flag.
    */

-   saveGotDB = TaskGotException(crosspage, EXC_DB);
-   TaskSetException(crosspage, EXC_DB, FALSE);
+   saveGotDB = TaskGotException(crosspageData, EXC_DB);
+   TaskSetException(crosspageData, EXC_DB, FALSE);
   COMPILER_MEM_BARRIER();      /* Prevent hoisting #UD-raising instructions. */
   SAVE_DR(7);

@@ -1936,7 +1979,7 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * In all cases, DR7 shouldn't have the GD bit set.
    */

-   TS_ASSERT(!(crosspage->crosspageData.hostDR[7] & DR7_GD));
+   TS_ASSERT(!(crosspageData->hostDR[7] & DR7_GD));

   /*
    * Save DR6 in order to accommodate the ICEBP instruction and other stuff
@@ -1961,15 +2004,15 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * SAVE_DR(6) can raise #DB.
    */

-   if (TaskGotException(crosspage, EXC_DB) &&
-       (crosspage->crosspageData.hostDR[6] & DR6_BD)) {
-      crosspage->crosspageData.hostDR[6] -= DR6_BD;
-      crosspage->crosspageData.hostDR[7] |= DR7_GD;
+   if (TaskGotException(crosspageData, EXC_DB) &&
+       (crosspageData->hostDR[6] & DR6_BD)) {
+      crosspageData->hostDR[6] -= DR6_BD;
+      crosspageData->hostDR[7] |= DR7_GD;
      SET_DR7(DR7_DEFAULT);

      /* HW: %dr7 and %dr6 are the guest, %dr3, %dr2, %dr1, %dr0 are host */
-      crosspage->crosspageData.hostDRInHW = ((1 << 3) | (1 << 2) |
-                                             (1 << 1) | (1 << 0));
+      crosspageData->hostDRInHW = ((1 << 3) | (1 << 2) |
+                                   (1 << 1) | (1 << 0));
   }

   /*
@@ -1977,20 +2020,20 @@ TaskSaveDebugRegisters(VMCrossPage *crosspage)
    * coincidentally trip during the switch.
    */

-   else if (crosspage->crosspageData.hostDR[7] & DR7_ENABLED) {
+   else if (crosspageData->hostDR[7] & DR7_ENABLED) {
      SET_DR7(DR7_DEFAULT);          // no #DB here, just simple set
      /* HW: %dr7 = guest, %dr6, %dr3, %dr2, %dr1, %dr0 = host */
-      crosspage->crosspageData.hostDRInHW = ((1 << 6) | (1 << 3) | (1 << 2) |
-                                             (1 << 1) | (1 << 0));
+      crosspageData->hostDRInHW = ((1 << 6) | (1 << 3) | (1 << 2) |
+                                   (1 << 1) | (1 << 0));
   }

-   TaskSetException(crosspage, EXC_DB, saveGotDB);
+   TaskSetException(crosspageData, EXC_DB, saveGotDB);

   /*
    * hostDR[6,7] have host contents in them now.
    */

-   crosspage->crosspageData.hostDRSaved = 0xC0;
+   crosspageData->hostDRSaved = 0xC0;
 #undef SAVE_DR
 }

@@ -2259,14 +2302,10 @@ TaskUpdatePTSCParameters(VMDriver *vm,
 *
 *      Wrapper that calls code to switch from the host to the monitor.
 *
- *      The basic idea is to do a (*(crosspage->hostToVmm))(crosspage)
+ *      The basic idea is to do a HostToVmm(crosspage)
 *      but it's complicated because we must have a common call format
 *      between GCC and MSC.
 *
- *      Since we have complete control over what GCC does with asm volatile,
- *      this amounts to having GCC do exactly what MSC does.
- *      For 64-bit hosts, we pass the parameter in RCX.
- *
 *      For 64-bit GCC, the callee is expected to preserve
 *      RBX,RBP,RSP,R12..R15, whereas MSC expects the callee to preserve
 *      RBX,RSI,RDI,RBP,RSP,R12..R15.  So for simplicity, we have the
@@ -2293,48 +2332,24 @@ TaskUpdatePTSCParameters(VMDriver *vm,
 */

 static INLINE_SINGLE_CALLER void
-TaskSwitchToMonitor(VMCrossPage *crosspage)
+TaskSwitchToMonitor(VMCrossPageData *crosspageData)
 {
-   const void *codePtr = ((void *)&crosspage->crosspageCode.toVmmFunc);
-
-#if defined(__GNUC__)
   /*
-    * Pass the crosspage pointer in RCX just like 64-bit MSC does.
-    * Tell GCC that the worldswitch preserves RBX,RSI,RDI,RBP,RSP,
-    * R12..R15 just like the MSC 64-bit calling convention.
+    * Arrange for RCX to contain the crosspage pointer using the MS ABI (first
+    * argument) and with the SYSV ABI (4th argument).
    */
-
-   {
-      uint64 raxGetsWiped, rcxGetsWiped;
-
-      __asm__ __volatile__("call *%%rax"
-                           : "=a" (raxGetsWiped),
-                             "=c" (rcxGetsWiped)
-                           : "0" (codePtr),
-                             "1" (crosspage)
-                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
-   }
-#elif defined(_MSC_VER)
-   /*
-    * The 64-bit calling convention is to pass the argument in RCX and that
-    * the called function must preserve RBX,RSI,RDI,RBP,RSP,R12..R15.
-    */
-#pragma warning(suppress: 4055) // Cast of data pointer to function pointer.
-   (*(void (*)(VMCrossPage *))codePtr)(crosspage);
-#else
-#error No compiler defined for TaskSwitchToMonitor
-#endif
+   HostToVmm(crosspageData, NULL, NULL, crosspageData);
 }


 static void
-TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
+TaskTestCrossPageExceptionHandlers(VMCrossPageData *crosspage)
 {
   static Bool testSwitchNMI = TRUE; /* test only first time through */

   /*
    * Test the DB,NMI,MCE handlers to make sure they can set the
-    * flags.  This is calling the handlers in switchNMI.S.
+    * flags.  This is calling the handlers in crosspage.c.
    */

   if (vmx86_debug && testSwitchNMI) {
@@ -2343,8 +2358,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      testSwitchNMI = FALSE;

      /*
-       * RAISE_INTERRUPT calls Switch{32,64}DBHandler in switchNMI.S
-       * (depending on host bitsize).
+       * RAISE_INTERRUPT calls SwitchDBHandler in crosspage.c.
       */

      gotSave = TaskGotException(crosspage, EXC_DB);
@@ -2354,8 +2368,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      TaskSetException(crosspage, EXC_DB, gotSave);

      /*
-       * RAISE_INTERRUPT calls Switch{32,64}NMIHandler in switchNMI.S
-       * (depending on host bitsize).
+       * RAISE_INTERRUPT calls SwitchNMIHandler in crosspage.c.
       */
      gotSave = TaskGotException(crosspage, EXC_NMI);
      TaskSetException(crosspage, EXC_NMI, FALSE);
@@ -2366,7 +2379,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      /*
       * Test the LRETQ in the 64-bit mini NMI handler to make sure
       * it works with any 16-byte offset of the stack pointer.
-       * The INT 2 calls Switch64NMIHandler in switchNMI.S.
+       * The INT 2 calls SwitchNMIHandler in crosspage.c.
       */
      {
         uint64 v1, v2;
@@ -2395,8 +2408,7 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
      TaskSetException(crosspage, EXC_NMI, gotSave);

      /*
-       * RAISE_INTERRUPT calls Switch{32,64}MCEHandler in switchNMI.S
-       * (depending on host bitsize).
+       * RAISE_INTERRUPT calls SwitchMCEHandler in crosspage.c.
       */

      gotSave = TaskGotException(crosspage, EXC_MC);
@@ -2425,10 +2437,10 @@ TaskTestCrossPageExceptionHandlers(VMCrossPage *crosspage)
 */

 static INLINE Bool
-TaskShouldRetryWorldSwitch(VMCrossPage *crosspage)
+TaskShouldRetryWorldSwitch(VMCrossPageData *crosspage)
 {
-   Bool result = crosspage->crosspageData.retryWorldSwitch;
-   crosspage->crosspageData.retryWorldSwitch = FALSE;
+   Bool result = crosspage->retryWorldSwitch;
+   crosspage->retryWorldSwitch = FALSE;
   return result;
 }

@@ -2447,16 +2459,13 @@ TaskShouldRetryWorldSwitch(VMCrossPage *crosspage)
 *      state.
 *
 * Results:
- *      None.
- *
- * Side effects:
- *      Jump to the monitor. Has no direct effect on the host-visible
- *      state except that it might generate an interrupt.
+ *      TRUE - If a #UD did not occur during the context switch.
+ *      FALSE - Otherwise.
 *
 *-----------------------------------------------------------------------------
 */

-void
+Bool
 Task_Switch(VMDriver *vm,  // IN
            Vcpuid vcpuid) // IN
 {
@@ -2474,10 +2483,11 @@ Task_Switch(VMDriver *vm,  // IN
   Bool lint1NMI;
   Bool pcNMI;
   Bool thermalNMI;
-   VMCrossPage *crosspage;
+   VMCrossPageData *crosspage;
   uint32 pCPU;
   MPN hvRootMPN;
   Descriptor *tempGDTBase;
+   Bool switchOk = TRUE;
 #ifdef CYCLE_SPEC_CTRL
   Bool specCtrlEqual = FALSE;
   uint64 readSpecCtrlValue = 0;
@@ -2486,8 +2496,9 @@ Task_Switch(VMDriver *vm,  // IN

   ASSERT(vcpuid < vm->numVCPUs);
   crosspage = vm->crosspage[vcpuid];
+   ASSERT(crosspage != NULL);

-   ASSERT_ON_COMPILE(sizeof(VMCrossPage) == PAGE_SIZE);
+   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
   TaskDisableNMI(&vm->hostAPIC, &lint0NMI, &lint1NMI, &pcNMI, &thermalNMI);
   SAVE_FLAGS(flags);
   CLEAR_INTERRUPTS();
@@ -2506,13 +2517,13 @@ Task_Switch(VMDriver *vm,  // IN
    * crosspages arguments when doing this though, see bug 820257.
    */
   if (hvRootMPN == INVALID_MPN && CPUID_HostSupportsHV()) {
-      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
-      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
-      crosspage->crosspageData.pcpuNum = pCPU;
+      crosspage->userCallType = MODULECALL_USERCALL_NONE;
+      crosspage->moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
+      crosspage->pcpuNum = pCPU;
   } else if (USE_TEMPORARY_GDT && tempGDTBase == NULL) {
-      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
-      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_TMP_GDT;
-      crosspage->crosspageData.pcpuNum = pCPU;
+      crosspage->userCallType = MODULECALL_USERCALL_NONE;
+      crosspage->moduleCallType = MODULECALL_ALLOC_TMP_GDT;
+      crosspage->pcpuNum = pCPU;
   } else {
      do {
         uintptr_t cr0reg, cr2reg, cr3reg, cr4reg;
@@ -2523,7 +2534,7 @@ Task_Switch(VMDriver *vm,  // IN

         Atomic_Write32(&vm->currentHostCpu[vcpuid], pCPU);

-         TaskUpdatePTSCParameters(vm, &crosspage->crosspageData, vcpuid);
+         TaskUpdatePTSCParameters(vm, crosspage, vcpuid);

         /*
          * Disable PEBS if it is supported and enabled.  Do this while on the
@@ -2554,7 +2565,7 @@ Task_Switch(VMDriver *vm,  // IN
          */

         TaskSaveIDT(&hostIDT);
-         TaskLoadIDT(&crosspage->crosspageData.switchHostIDTR);
+         TaskLoadIDT(&crosspage->switchHostIDTR);
         TaskTestCrossPageExceptionHandlers(crosspage);

         if (CPUID_HostSupportsVT()) {
@@ -2580,21 +2591,21 @@ Task_Switch(VMDriver *vm,  // IN
         GET_CR0(cr0reg);
         GET_CR4(cr4reg);
         GET_CR3(cr3reg);
-         crosspage->crosspageData.hostCR3 = cr3reg;
+         crosspage->hostCR3 = cr3reg;

         /*
          * Any reserved bits in CR0 must be preserved when we switch
          * to the VMM. [See PR 291004.]  (On the other hand, Intel
          * recommends that we clear any reserved CR4 bits.)
          */
-         crosspage->crosspageData.wsCR0 &= ~CR0_RESERVED;
-         crosspage->crosspageData.wsCR0 |= (cr0reg & CR0_RESERVED);
+         crosspage->wsCR0 &= ~CR0_RESERVED;
+         crosspage->wsCR0 |= (cr0reg & CR0_RESERVED);

         /*
          * CR4.VMXE must be enabled to support VMX in the monitor.
          */
         if (CPUID_HostSupportsVT()) {
-            crosspage->crosspageData.wsCR4 |= CR4_VMXE;
+            crosspage->wsCR4 |= CR4_VMXE;
         }

         /*
@@ -2602,8 +2613,8 @@ Task_Switch(VMDriver *vm,  // IN
          * host's values.  CR4.PCIDE will be cleared once we're in the monitor,
          * running on a CR3 with a PCID field of 0.
          */
-         crosspage->crosspageData.wsCR4 =
-            (crosspage->crosspageData.wsCR4 & ~(CR4_MCE | CR4_PCIDE)) |
+         crosspage->wsCR4 =
+            (crosspage->wsCR4 & ~(CR4_MCE | CR4_PCIDE)) |
            (cr4reg & (CR4_MCE | CR4_PCIDE));

         /*
@@ -2611,15 +2622,15 @@ Task_Switch(VMDriver *vm,  // IN
          * switching to the monitor's CR4 ensures that global pages are
          * flushed.
          */
-         ASSERT((crosspage->crosspageData.wsCR4 & CR4_PGE) == 0);
+         ASSERT((crosspage->wsCR4 & CR4_PGE) == 0);

         /*
          * Load the world-switch CR0 and CR4.  We can't load the monitor's
          * CR3 yet, because the current code isn't mapped into the
          * monitor's address space.
          */
-         SET_CR0((uintptr_t)crosspage->crosspageData.wsCR0);
-         SET_CR4((uintptr_t)crosspage->crosspageData.wsCR4);
+         SET_CR0((uintptr_t)crosspage->wsCR0);
+         SET_CR4((uintptr_t)crosspage->wsCR4);

         TaskSaveDebugRegisters(crosspage);

@@ -2708,23 +2719,22 @@ Task_Switch(VMDriver *vm,  // IN
            currentSpecCtrlValue = (currentSpecCtrlValue + 1) % 4;
            X86MSR_SetMSR(MSR_SPEC_CTRL, currentSpecCtrlValue);
 #endif
-            crosspage->crosspageData.specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
+            crosspage->specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
         }

-         DEBUG_ONLY(crosspage->crosspageData.monTinyStack[0] = 0xDEADBEEF;)
-         DEBUG_ONLY(crosspage->crosspageData.hostTinyStack[0] = 0xDEADBEEF;)
+         DEBUG_ONLY(crosspage->monTinyStack[0] = 0xDEADBEEF;)
+         DEBUG_ONLY(crosspage->hostTinyStack[0] = 0xDEADBEEF;)
         /* Running in host context prior to TaskSwitchToMonitor() */
         TaskSwitchToMonitor(crosspage);
         /* Running in host context after to TaskSwitchToMonitor() */

-         TS_ASSERT(crosspage->crosspageData.monTinyStack[0] == 0xDEADBEEF);
-         TS_ASSERT(crosspage->crosspageData.hostTinyStack[0] == 0xDEADBEEF);
+         TS_ASSERT(crosspage->monTinyStack[0] == 0xDEADBEEF);
+         TS_ASSERT(crosspage->hostTinyStack[0] == 0xDEADBEEF);

 #ifdef CYCLE_SPEC_CTRL
         if (CPUID_HostSupportsSpecCtrl()) {
            readSpecCtrlValue = X86MSR_GetMSR(MSR_SPEC_CTRL);
-            specCtrlEqual = readSpecCtrlValue ==
-                            crosspage->crosspageData.specCtrl;
+            specCtrlEqual = readSpecCtrlValue == crosspage->specCtrl;
            /* Do not leak cycling SPEC_CTRL value back to host. */
            X86MSR_SetMSR(MSR_SPEC_CTRL, 0);
         }
@@ -2762,10 +2772,10 @@ Task_Switch(VMDriver *vm,  // IN
          * CR2 always has to be restored.  CR8 never has to be restored.
          */
         SET_CR2(cr2reg);
-         if (crosspage->crosspageData.wsCR0 != cr0reg) {
+         if (crosspage->wsCR0 != cr0reg) {
            SET_CR0(cr0reg);
         }
-         if (crosspage->crosspageData.wsCR4 != cr4reg) {
+         if (crosspage->wsCR4 != cr4reg) {
            SET_CR4(cr4reg);
         } else if ((cr4reg & CR4_PCIDE) != 0) {
            /*
@@ -2813,7 +2823,7 @@ Task_Switch(VMDriver *vm,  // IN
            TaskDisableTF();
         }

-         TaskRestoreDebugRegisters(&crosspage->crosspageData);
+         TaskRestoreDebugRegisters(crosspage);

         ASSERT_NO_INTERRUPTS();

@@ -2832,7 +2842,7 @@ Task_Switch(VMDriver *vm,  // IN
            X86MSR_SetMSR(MSR_RTIT_CTL, ptMSR);
         }

-         TaskUpdateLatestPTSC(vm, &crosspage->crosspageData);
+         TaskUpdateLatestPTSC(vm, crosspage);
         Atomic_Write32(&vm->currentHostCpu[vcpuid], INVALID_PCPU);

         /*
@@ -2864,12 +2874,13 @@ Task_Switch(VMDriver *vm,  // IN
            RAISE_INTERRUPT(EXC_MC);
         }
         if (UNLIKELY(TaskGotException(crosspage, EXC_UD))) {
-            Warning("#UD occurred on switch back to host; dumping core.\n");
+            switchOk = FALSE;
+            Warning("#UD occurred on switch back to host.\n");
         }
 #ifdef CYCLE_SPEC_CTRL
         if (UNLIKELY(!specCtrlEqual)) {
            Warning("SpecCtrl not equal: expected %"FMT64"x, got %"FMT64"x\n",
-                    crosspage->crosspageData.specCtrl, readSpecCtrlValue);
+                    crosspage->specCtrl, readSpecCtrlValue);
         }
 #endif
         /*
@@ -2877,8 +2888,8 @@ Task_Switch(VMDriver *vm,  // IN
          * received during worldswitch.  Here is the more generic case
          * of forwarding NMIs received while executing the VMM/guest.
          */
-         if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR &&
-             crosspage->crosspageData.args[0] == EXC_NMI) {
+         if (crosspage->moduleCallType == MODULECALL_INTR &&
+             crosspage->args[0] == EXC_NMI) {
            /*
             * If VMM was interrupted by an NMI, do the INT 2 so the
             * host will handle it, but then return immediately to the
@@ -2888,12 +2899,12 @@ Task_Switch(VMDriver *vm,  // IN
             * done the EOI.
             */
            RAISE_INTERRUPT(EXC_NMI);
-            crosspage->crosspageData.retryWorldSwitch = TRUE;
+            crosspage->retryWorldSwitch = TRUE;
         }
      } while (UNLIKELY(TaskShouldRetryWorldSwitch(crosspage)));
   }

-   if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR) {
+   if (crosspage->moduleCallType == MODULECALL_INTR) {

      /*
       * Newer versions of Window expect EFLAGS_AC to be set when handling an
@@ -2912,15 +2923,15 @@ Task_Switch(VMDriver *vm,  // IN
       * what bad things btw).
       */
 #ifdef _WIN64
-      if (crosspage->crosspageData.args[0] <= 0xFF &&
-          (crosspage->crosspageData.args[0] >= 0x14 ||
-           crosspage->crosspageData.args[0] == EXC_MC)) {
-         RAISE_INTERRUPT((unsigned char)crosspage->crosspageData.args[0]);
+      if (crosspage->args[0] <= 0xFF &&
+          (crosspage->args[0] >= 0x14 ||
+           crosspage->args[0] == EXC_MC)) {
+         RAISE_INTERRUPT((unsigned char)crosspage->args[0]);
      } else {
         Warning("%s: Received Unexpected Interrupt: 0x%"FMT64"X\n",
-                 __FUNCTION__, crosspage->crosspageData.args[0]);
+                 __FUNCTION__, crosspage->args[0]);
         Panic("Received Unexpected Interrupt: 0x%"FMT64"X\n",
-               crosspage->crosspageData.args[0]);
+               crosspage->args[0]);
      }
 #else
      /*
@@ -2934,7 +2945,7 @@ Task_Switch(VMDriver *vm,  // IN
 #define IRQ_INT16(_x) IRQ_INT8(_x); IRQ_INT8(_x + 8)
 #define IRQ_INT32(_x) IRQ_INT16(_x); IRQ_INT16(_x + 16)

-      switch (crosspage->crosspageData.args[0]) {
+      switch (crosspage->args[0]) {
         // These are the general IO interrupts
         // It would be nice to generate this dynamically, but see Note2 above.

@@ -2967,13 +2978,14 @@ Task_Switch(VMDriver *vm,  // IN
          */

         Warning("%s: Received Unexpected Interrupt: 0x%"FMT64"X\n",
-                 __FUNCTION__, crosspage->crosspageData.args[0]);
+                 __FUNCTION__, crosspage->args[0]);
         Panic("Received Unexpected Interrupt: 0x%"FMT64"X\n",
-               crosspage->crosspageData.args[0]);
+               crosspage->args[0]);
      }
 #endif
   }

   RESTORE_FLAGS(flags);
   TaskRestoreNMI(&vm->hostAPIC, lint0NMI, lint1NMI, pcNMI, thermalNMI);
+   return switchOk;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@ struct MonLoaderHeader;
 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
 extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
                               PerVcpuPages *perVcpuPages);
-extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
+extern Bool Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
 extern MPN Task_GetHVRootPageForPCPU(uint32 pCPU);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -108,9 +108,13 @@ static int vmIDsUnused;
 /* Max rate requested for fast clock by any virtual machine. */
 static unsigned globalFastClockRate;

+/* 3 physically contiguous pages for the I/O bitmap.  SVM only. */
+HostIFContigMemMap *hvIOBitmap;
+
 typedef struct {
-   Atomic_uint32 index;
-   MSRQuery *query;
+   Atomic_uint32 *index; // OUT: array of cpu counters for queries.
+   MSRQuery *query;  // IN/OUT: array of query items
+   uint32 numItems;  // IN
 } Vmx86GetMSRData;

 static Bool hostUsesNX;
@@ -120,6 +124,20 @@ typedef struct NXData {
   Atomic_uint32 hasNX;
 } NXData;

+/*
+ * A structure for holding MSR indexes, values for MSR uniformity checks.
+ */
+typedef struct VMX86MSRCacheInfo {
+   uint32 msrIndex;
+   uint64 msrValue;
+} VMX86MSRCacheInfo;
+
+static VMX86MSRCacheInfo msrUniformityCacheInfo[] = {
+#define MSR_FEAT(member) {member, CONST64(0)},
+   MSR_FEAT(IA32_MSR_ARCH_CAPABILITIES)
+#undef MSR_FEAT
+};
+
 /*
 *----------------------------------------------------------------------
 *
@@ -564,6 +582,66 @@ Vmx86_Calloc(size_t numElements, // IN
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86AllocCrossPages --
+ *
+ *      Allocate numVCPUs pages suitable to be used as the VCPU's
+ *      crosspage area.
+ *
+ * Results:
+ *      TRUE if the required crosspages are allocated successfully.
+ *      FALSE otherwise.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static Bool
+Vmx86AllocCrossPages(VMDriver *vm)
+{
+   Vcpuid v;
+
+   for (v = 0; v < vm->numVCPUs; v++) {
+      MPN unused;
+
+      UNUSED_VARIABLE(unused);
+      vm->crosspage[v] = HostIF_AllocKernelPages(1, &unused);
+
+      if (vm->crosspage[v] == NULL) {
+         return FALSE;
+      }
+      memset(vm->crosspage[v], 0, PAGE_SIZE);
+   }
+   return TRUE;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86FreeCrossPages --
+ *
+ *      Free the crosspages allocated for the given VM.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static void
+Vmx86FreeCrossPages(VMDriver *vm)
+{
+   Vcpuid v;
+
+   if (vm->crosspage != NULL) {
+      for (v = 0; v < vm->numVCPUs; v++) {
+         if (vm->crosspage[v] != NULL) {
+            HostIF_FreeKernelPages(1, vm->crosspage[v]);
+         }
+      }
+   }
+}
+
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -687,6 +765,38 @@ Vmx86_CleanupVMMPages(VMDriver *vm)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86CleanupContigMappings --
+ *
+ *     Frees all allocations from HostIF_AllocContigPages that are associated
+ *     with the given vm.
+ *
+ * Results:
+ *      None
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+static void
+Vmx86CleanupContigMappings(VMDriver *vm)
+{
+   HostIFContigMemMap *m, *next;
+
+   HostIF_VMLock(vm, 48);
+   for (m = vm->contigMappings; m != NULL; m = next) {
+      next = m->next;
+      HostIF_FreeContigPages(vm, m);
+   }
+   HostIF_VMUnlock(vm, 48);
+   vm->contigMappings = NULL;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -714,6 +824,7 @@ Vmx86FreeAllVMResources(VMDriver *vm)

      Vmx86_SetHostClockRate(vm, 0);

+      Vmx86FreeCrossPages(vm);
      if (vm->ptpTracker != NULL) {
         Task_SwitchPTPPageCleanup(vm);
      }
@@ -732,7 +843,9 @@ Vmx86FreeAllVMResources(VMDriver *vm)
         StatVarsVmmon_Cleanup(vm->statVars);
         vm->statVars = NULL;
      }
-
+      if (vm->contigMappings != NULL) {
+         Vmx86CleanupContigMappings(vm);
+      }
      HostIF_FreeAllResources(vm);

      Vmx86FreeVMDriver(vm);
@@ -1160,6 +1273,15 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
      goto error;
   }

+   if (!pseudoTSC.initialized) {
+      Warning("%s: PseudoTSC has not been initialized\n", __FUNCTION__);
+      goto error;
+   }
+
+   if (!Vmx86AllocCrossPages(vm)) {
+      Warning("Failed to allocate cross pages.\n");
+      goto error;
+   }
   /*
    * Initialize the driver's part of the cross-over page used to
    * talk to the monitor.
@@ -1601,12 +1723,12 @@ Vmx86_MonTimerIPI(void)
      VCPUSet_Empty(&expiredVCPUs);

      for (v = 0; v < vm->numVCPUs; v++) {
-         VMCrossPage *crosspage = vm->crosspage[v];
+         VMCrossPageData *crosspage = vm->crosspage[v];

-         if (!crosspage) {
+         if (crosspage == NULL) {
            continue;  // VCPU is not initialized yet
         }
-         expiry = crosspage->crosspageData.monTimerExpiry;
+         expiry = crosspage->monTimerExpiry;
         if (expiry != 0 && expiry <= pNow) {
            VCPUSet_Include(&expiredVCPUs, v);
            hasWork = TRUE;
@@ -2042,7 +2164,9 @@ Vmx86_AllocLowPage(VMDriver *vm,      // IN: VMDriver
      return INVALID_MPN;
   }

+   HostIF_VMLock(vm, 49);
   mpn = HostIF_AllocLowPage(vm);
+   HostIF_VMUnlock(vm, 49);

   if (mpn == INVALID_MPN) {
      Vmx86UnreserveFreePages(vm, 1);
@@ -2740,45 +2864,47 @@ Vmx86_CheckPseudoTSC(uint64 *lastTSC, // IN/OUT: last/current value of the TSC
 static void
 Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 {
+   uint32 i;
   Vmx86GetMSRData *data = (Vmx86GetMSRData *)clientData;
-   MSRQuery *query;
-   uint32 index;
-   int err;
+   ASSERT(data && data->index && data->query);

-   ASSERT(data);
-   query = data->query;
-   ASSERT(query);
+   for (i = 0; i < data->numItems; ++i) {
+      uint32 index;
+      int err;
+      Atomic_uint32 *cpus = &data->index[i];
+      MSRQuery *query = &data->query[i];

-   index = Atomic_ReadInc32(&data->index);
-   if (index >= query->numLogicalCPUs) {
-      return;
-   }
-
-   query->logicalCPUs[index].tag = HostIF_GetCurrentPCPU();
-
-   /*
-    * We treat BIOS_SIGN_ID (microcode version) specially on Intel,
-    * where the preferred read sequence involves a macro.
-    */
-
-   if (CPUID_GetVendor() == CPUID_VENDOR_INTEL &&
-       query->msrNum == MSR_BIOS_SIGN_ID) {
-      /* safe to read: MSR_BIOS_SIGN_ID architectural since Pentium Pro */
-      query->logicalCPUs[index].msrVal = INTEL_MICROCODE_VERSION();
-      err = 0;
-   } else {
-      /*
-       * Try to enable HV any time these MSRs are queried.  We have seen
-       * buggy formware that forgets to re-enable HV after waking from
-       * deep sleep. [PR 1020692]
-       */
-      if (query->msrNum == MSR_FEATCTL || query->msrNum == MSR_VM_CR) {
-         Vmx86EnableHVOnCPU();
+      index = Atomic_ReadInc32(cpus);
+      if (index >= query->numLogicalCPUs) {
+         continue;
      }
-      err = HostIF_SafeRDMSR(query->msrNum, &query->logicalCPUs[index].msrVal);
-   }

-   query->logicalCPUs[index].implemented = (err == 0) ? 1 : 0;
+      query->logicalCPUs[index].tag = HostIF_GetCurrentPCPU();
+
+      /*
+       * We treat BIOS_SIGN_ID (microcode version) specially on Intel,
+       * where the preferred read sequence involves a macro.
+       */
+      if (CPUID_GetVendor() == CPUID_VENDOR_INTEL &&
+          query->msrNum == MSR_BIOS_SIGN_ID) {
+         /* safe to read: MSR_BIOS_SIGN_ID architectural since Pentium Pro */
+         query->logicalCPUs[index].msrVal = INTEL_MICROCODE_VERSION();
+         err = 0;
+      } else {
+         /*
+          * Try to enable HV any time these MSRs are queried.  We have seen
+          * buggy firmware that forgets to re-enable HV after waking from
+          * deep sleep. [PR 1020692]
+          */
+         if (query->msrNum == MSR_FEATCTL || query->msrNum == MSR_VM_CR) {
+            Vmx86EnableHVOnCPU();
+         }
+         err =
+            HostIF_SafeRDMSR(query->msrNum, &query->logicalCPUs[index].msrVal);
+      }
+
+      query->logicalCPUs[index].implemented = (err == 0) ? 1 : 0;
+   }
 }


@@ -2787,7 +2913,7 @@ Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 *
 * Vmx86_GetAllMSRs --
 *
- *      Collect MSR value on all logical CPUs.
+ *      Collect MSR value on number of logical CPUs requested.
 *
 *      The caller is responsible for ensuring that the requested MSR is valid
 *      on all logical CPUs.
@@ -2809,24 +2935,40 @@ Vmx86GetMSR(void *clientData) // IN/OUT: A Vmx86GetMSRData *
 Bool
 Vmx86_GetAllMSRs(MSRQuery *query) // IN/OUT
 {
+   unsigned i, cpu;
+   Atomic_uint32 index;
   Vmx86GetMSRData data;
+   data.index = &index;
+   data.numItems = 1;

-   Atomic_Write32(&data.index, 0);
+   /* Check MSR uniformity cache first. */
+   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
+      if (msrUniformityCacheInfo[i].msrIndex == query->msrNum) {
+         for (cpu = 0; cpu < query->numLogicalCPUs; cpu++) {
+            query->logicalCPUs[cpu].msrVal = msrUniformityCacheInfo[i].msrValue;
+            query->logicalCPUs[cpu].implemented = 1;
+            query->logicalCPUs[cpu].tag = cpu;
+         }
+         return TRUE;
+      }
+   }
+
+   Atomic_Write32(data.index, 0);
   data.query = query;

   HostIF_CallOnEachCPU(Vmx86GetMSR, &data);

   /*
-    * At this point, Atomic_Read32(&data.index) is the number of logical CPUs
+    * At this point, Atomic_Read32(data.index) is the number of logical CPUs
    * who replied.
    */

-   if (Atomic_Read32(&data.index) > query->numLogicalCPUs) {
+   if (Atomic_Read32(data.index) > query->numLogicalCPUs) {
      return FALSE;
   }

-   ASSERT(Atomic_Read32(&data.index) <= query->numLogicalCPUs);
-   query->numLogicalCPUs = Atomic_Read32(&data.index);
+   ASSERT(Atomic_Read32(data.index) <= query->numLogicalCPUs);
+   query->numLogicalCPUs = Atomic_Read32(data.index);

   return TRUE;
 }
@@ -3369,16 +3511,13 @@ Vmx86_GetMonitorContext(VMDriver *vm,       // IN: The VM instance.
                        Vcpuid vcpuid,      // IN: VCPU in question.
                        Context64 *context) // OUT: context.
 {
-   VMCrossPage *crosspage;
   VMCrossPageData *cpData;
-   if (vcpuid >= vm->numVCPUs) {
+
+   if (vcpuid >= vm->numVCPUs || vm->crosspage[vcpuid] == NULL) {
      return FALSE;
   }
-   crosspage = vm->crosspage[vcpuid];
-   if (!crosspage) {
-      return FALSE;
-   }
-   cpData = &crosspage->crosspageData;
+   cpData = vm->crosspage[vcpuid];
+
   memset(context, 0, sizeof *context);
   context->es  = cpData->monES;
   context->ss  = cpData->monSS;
@@ -3393,3 +3532,199 @@ Vmx86_GetMonitorContext(VMDriver *vm,       // IN: The VM instance.
   context->rip = cpData->monRIP;
   return TRUE;
 }
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_CleanupHVIOBitmap --
+ *
+ *      Free any resources that were allocated for the HV I/O bitmap.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+Vmx86_CleanupHVIOBitmap(void)
+{
+   if (hvIOBitmap != NULL) {
+      HostIF_FreeContigPages(NULL, hvIOBitmap);
+      hvIOBitmap = NULL;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_CreateHVIOBitmap --
+ *
+ *      Called on driver load to create and initialize the host wide SVM I/O
+ *      bitmap.  This item is a physically contiguous region of
+ *      SVM_VMCB_IO_BITMAP_PAGES pages and is initialized to all-bits-set.
+ *
+ * Results:
+ *      TRUE on success or FALSE on failure.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+Vmx86_CreateHVIOBitmap(void)
+{
+   if (!CPUID_HostSupportsSVM()) {
+      return TRUE;
+   }
+   if (vmx86_apple) {
+      /*
+       * This function is not called on MacOS.  No supported MacOS system is
+       * available for AMD so that platform has no need to create the SVM I/O
+       * bitmap.
+       */
+      return TRUE;
+   }
+   hvIOBitmap = HostIF_AllocContigPages(NULL, SVM_VMCB_IO_BITMAP_PAGES);
+   if (hvIOBitmap == NULL) {
+      Warning("Failed to allocate SVM I/O bitmap.\n");
+      return FALSE;
+   }
+   memset(hvIOBitmap->addr, 0xff, SVM_VMCB_IO_BITMAP_SIZE);
+   return TRUE;
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86RegisterCPU --
+ *
+ *      Registers each logical CPU by incrementing a counter.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      counter value pointed by 'data' is incremented by one.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static void
+Vmx86RegisterCPU(void *data) // IN: *data
+{
+   Atomic_uint32 *numLogicalCPUs = data;
+   ASSERT(numLogicalCPUs);
+   Atomic_Inc32(numLogicalCPUs);
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86_CheckMSRUniformity --
+ *
+ *      Provides basic hardware MSR feature checks for x86 hosted platform. VMM
+ *      requires and prefers uniformity of certain MSRs. This function iterates
+ *      through a list of MSR features (i.e. msrUniformityCacheInfo), checking
+ *      uniformity for MSR value on each logical CPU. A Uniformity check is
+ *      ignored for MSRs are that are not available for the target architecture
+ *      or cpu family. If MSRs are non uniform then, a common bit field is
+ *      calculated by taking the intersection of MSR values across cpu(s).
+ *
+ * Results:
+ *      Returns TRUE if MSR uniformity checks complete successfully, FALSE
+ *      otherwise.
+ *
+ * Side effects:
+ *      updates msrUniformityCacheInfo cache with MSR values.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+Bool
+Vmx86_CheckMSRUniformity(void)
+{
+   Vmx86GetMSRData data;
+   uint32 i, j;
+   uint32 numQueries = ARRAYSIZE(msrUniformityCacheInfo);
+   Atomic_uint32 numLogicalCPUs;
+   Atomic_uint32 *cpuCounters;
+   uint32 numPCPUs = 0;
+   MSRQuery *query = NULL;
+
+   Atomic_Write32(&numLogicalCPUs, 0);
+   /*
+    * Calculates number of logical CPUs by counting and then uses this
+    * information to set up MSR queries; will be executed on each logical CPU.
+    */
+   HostIF_CallOnEachCPU(Vmx86RegisterCPU, &numLogicalCPUs);
+   numPCPUs = Atomic_Read32(&numLogicalCPUs);
+   ASSERT(numPCPUs > 0);
+   query = Vmx86_Calloc(numQueries,
+                        sizeof(MSRQuery) + sizeof(MSRReply) * numPCPUs, TRUE);
+   if (query == NULL) {
+      Warning("Fatal, not enough memory for MSR feature uniformity checks");
+      return FALSE;
+   }
+
+   cpuCounters = Vmx86_Calloc(numQueries, sizeof(Atomic_uint32), TRUE);
+   if (cpuCounters == NULL) {
+      Vmx86_Free(query);
+      Warning("Fatal, not enough memory for MSR feature uniformity checks");
+      return FALSE;
+   }
+   data.query = query;
+   data.index = cpuCounters;
+   data.numItems = numQueries;
+
+   /*
+    * Enumerates a MSR list and initializes MSR data structure before the
+    * actual (safe) MSR query takes place. The Nested loop tests a MSR for
+    * uniformity on all logical processors.
+    */
+   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
+      query = &data.query[i];
+      Atomic_Write32(&data.index[i], 0);
+      query->msrNum = msrUniformityCacheInfo[i].msrIndex;
+      query->numLogicalCPUs = numPCPUs;
+   }
+
+   /* perform once, a multi MSR query for MSRs in uniformity check list. */
+   HostIF_CallOnEachCPU(Vmx86GetMSR, &data);
+
+   for (i = 0; i < ARRAYSIZE(msrUniformityCacheInfo); ++i) {
+      uint32 msrIndex = msrUniformityCacheInfo[i].msrIndex;
+      query = &data.query[i];
+      ASSERT(Atomic_Read32(&data.index[i]) == numPCPUs);
+      msrUniformityCacheInfo[i].msrValue = query->logicalCPUs[0].msrVal;
+      if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
+         /*
+          * MSR_ARCH_CAPABILITIES_RSBA bit 1 represents lack of feature while 0
+          * represents presence. Therefore, bit is flipped for calculating the
+          * least common set and flipped again on the final value for resetting.
+          */
+         msrUniformityCacheInfo[i].msrValue ^= MSR_ARCH_CAPABILITIES_RSBA;
+      }
+      for (j = 1; j < numPCPUs; j++) {
+         uint64 msrValuePCPU = query->logicalCPUs[j].msrVal;
+         if (msrValuePCPU != query->logicalCPUs[0].msrVal) {
+            if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
+               msrValuePCPU ^= MSR_ARCH_CAPABILITIES_RSBA;
+            }
+            msrUniformityCacheInfo[i].msrValue &= msrValuePCPU;
+            Warning("Found a mismatch on MSR feature 0x%x; logical cpu%u "
+                    "value = 0x%llx, but logical cpu%u value = 0x%llx\n",
+                    msrIndex, j, msrValuePCPU, 0, query->logicalCPUs[0].msrVal);
+         }
+      }
+      if (msrIndex == IA32_MSR_ARCH_CAPABILITIES) {
+         msrUniformityCacheInfo[i].msrValue ^= MSR_ARCH_CAPABILITIES_RSBA;
+      }
+   }
+   Vmx86_Free(cpuCounters);
+   Vmx86_Free(query);
+   return TRUE;
+}
+
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -48,6 +48,10 @@ typedef struct TSCDelta {

 struct VmmBlobInfo;

+struct HostIFContigMemMap;
+
+extern struct HostIFContigMemMap *hvIOBitmap;
+
 /*
 * VMDriver - the main data structure for the driver side of a
 *            virtual machine.
@@ -64,7 +68,7 @@ typedef struct VMDriver {
   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
-   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
+   struct VMCrossPageData **crosspage;        /* numVCPUs-sized array. */
   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
@@ -79,6 +83,12 @@ typedef struct VMDriver {
   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
   PageCnt                 numPTPPages;      /* Num PTP pages allocated. */
+   /*
+    * List of physically contiguous allocations associated with this VM.
+    * Access is protected by the VM lock.
+    */
+   struct HostIFContigMemMap     *contigMappings;
+
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -165,9 +175,12 @@ extern void Vmx86_UpdateMemInfo(VMDriver *curVM,
 extern void Vmx86_Add2MonPageTable(VMDriver *vm, VPN vpn, MPN mpn,
                                   Bool readOnly);
 extern Bool Vmx86_GetAllMSRs(MSRQuery *query);
+extern Bool Vmx86_CheckMSRUniformity(void);
 extern void Vmx86_FlushVMCSAllCPUs(MA vmcs);
 extern void Vmx86_MonTimerIPI(void);
 extern void Vmx86_InitIDList(void);
+extern Bool Vmx86_CreateHVIOBitmap(void);
+extern void Vmx86_CleanupHVIOBitmap(void);
 extern Bool Vmx86_GetPageRoot(VMDriver *vm, Vcpuid vcpuid, MPN *mpn);
 extern void Vmx86_Open(void);
 extern void Vmx86_Close(void);
@@ -0,0 +1,319 @@
+/*********************************************************
+ * Copyright (C) 2007-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _ADDRLAYOUT_H_
+#define _ADDRLAYOUT_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "address_defs.h"
+#include "x86desc.h"
+#include "x86types.h"
+#include "vmm_constants.h"
+
+#define DIRECT_EXEC_USER_RPL    3
+#define BINARY_TRANSLATION_RPL  1
+typedef char x86_PAGE[PAGE_SIZE];
+
+#define MAX_VADDR                    CONST64U(0xffffffffffffffff)
+#define MONITOR_SIZE                 (64 * 1024 * 1024)
+#define MONITOR_LINEAR_START         (MAX_VADDR - MONITOR_SIZE + 1)
+#define MONITOR_LINEAR_END           (MAX_VADDR)
+#define NUM_MONITOR_PAGES            (MONITOR_SIZE / PAGE_SIZE)
+#define MONITOR_BASE_VPN             (MONITOR_LINEAR_START >> PAGE_SHIFT)
+
+#define MONITOR_AS_START             MONITOR_BASE_VPN
+#define MONITOR_AS_LEN               NUM_MONITOR_PAGES
+
+/*
+ * Regions and items necessarily overlap.  Build parallel structures for each.
+ * These 'field' structures are used to determine page numbers and offsets.
+ * AddrLayout contains zero-length regions used for alignment-checking.  (These
+ * are used by gcc as Windows prohibits zero-length arrays.)
+ */
+
+/* Item layout. */
+typedef struct {
+#if defined __GNUC__
+#define REGION(name, length) x86_PAGE name##_REGION_MARKER_FIELD[0];
+#else
+#define REGION(name, length)
+#endif
+#define ITEM(name, length) x86_PAGE name##_FIELD[length];
+#include "addrlayout_table.h"
+} AddrLayout;
+#undef REGION
+#undef ITEM
+#define FIELD_PAGE_NUMBER(name) PAGE_NUMBER(&(((AddrLayout *)0)->name##_FIELD))
+
+
+/* Region layout. */
+typedef struct {
+#define REGION(name, length) x86_PAGE name##_REGION_FIELD[length];
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+} RegionAddrLayout;
+#undef REGION
+#undef ITEM
+#define REGION_FIELD_PAGE_NUMBER(name) \
+   PAGE_NUMBER(&(((RegionAddrLayout *)0)->name##_REGION_FIELD))
+
+#if defined __GNUC__
+/*
+ * The sum of sizes of items in a region must add up to the region's size.
+ * AddrLayout contains zero-sized region markers at their correct offsets.
+ * RegionAddrLayout contains full-sized regions (but no items).  Comparing the
+ * marker and region offsets ensures that all items sizes in the previous
+ * region sum to the size of the previous region.  To check the final region
+ * (as it has no successor) we assert the sizes of the tables match.
+ *
+ * We also assert 2MB region granularity.
+ *
+ * This checking is only performed when using gcc.
+ */
+#define REGION(name, length) static INLINE void name(void) { \
+   ASSERT_ON_COMPILE(FIELD_PAGE_NUMBER(name##_REGION_MARKER) == \
+                     REGION_FIELD_PAGE_NUMBER(name)); \
+   ASSERT_ON_COMPILE((length % 512) == 0); \
+   ASSERT_ON_COMPILE(sizeof(AddrLayout) == sizeof(RegionAddrLayout)); \
+   ASSERT_ON_COMPILE(sizeof(RegionAddrLayout) == MONITOR_SIZE); \
+}
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+/*
+ * Sizes.
+ */
+
+enum {
+#define REGION(name, length) name##_REGION_LEN = length,
+#define ITEM(name, length) name##_LEN = length,
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+/*
+ * Starts.
+ */
+
+#if defined __GNUC__
+/* Windows compiler and gdb do not accept 64-bit enumerations, but gcc
+ * does.  These are used by geninfo.c.
+ */
+enum {
+#define REGION(name, length) _asm_##name##_REGION_START = \
+    0ULL + MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) _asm_##name##_START = \
+    0ULL + MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+enum {
+#define REGION(name, length) name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+#else
+#define REGION(name, length) \
+   static const uint64 name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name);
+#define ITEM(name, length) \
+   static const uint64 name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name);
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+enum {
+#define REGION(name, length) name##_REGION_start_page = \
+   REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_start_page = \
+   FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+
+/* Derive the bootstrap start address from table contents. */
+#define MONITOR_BOOTSTRAP_START_LA   VPN_2_VA(BS_TXT_START)
+
+#ifndef VMX86_SERVER
+#define SWITCH_PAGE_TABLE_LEN          12
+#endif
+
+#define VMM_STACK_TOP_HW          VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+/*
+ * VMM_STKTOP_HW_LEN needs to contain an ExcFrame64ForCopy (72 bytes) and
+ * some more bytes for a "no man's land" region.  We make sure the "no
+ * man's land" region is 0 on BT_Resume and HV_Resume.  Because we
+ * check "no man's land" in 8-byte chunks, VMM_STKTOP_HW_LEN must be
+ * divisible by 8.
+ */
+#define VMM_STKTOP_HW_LEN          168
+#define VMM_STKTOP_HW_LRET_LEN     (VMM_STKTOP_HW_LEN + sizeof(LretFrame64))
+#define VMM_STKTOP_LRET_OFFSET     (PAGE_SIZE - VMM_STKTOP_HW_LRET_LEN)
+
+#define MON_STACK_BASE            VPN_2_VA(MON_STACK_PAGES_START)
+#define MON_STACK_TOP             VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+#define DF_STACK_BASE              VPN_2_VA(DF_STACK_PAGES_START)
+#define DF_STACK_TOP               VPN_2_VA(DF_STACK_PAGES_START + \
+                                            DF_STACK_PAGES_LEN)
+#define MC_STACK_BASE              VPN_2_VA(MC_STACK_PAGES_START)
+#define MC_STACK_TOP               VPN_2_VA(MC_STACK_PAGES_START + \
+                                            MC_STACK_PAGES_LEN)
+#define NMI_STACK_BASE             VPN_2_VA(NMI_STACK_PAGES_START)
+#define NMI_STACK_TOP              VPN_2_VA(NMI_STACK_PAGES_START + \
+                                            NMI_STACK_PAGES_LEN)
+
+#define NUM_MON_PTABS   MON_PAGE_TABLE_L1_LEN
+
+#define MON_START_L5OFF PT_LA_2_L5OFF(MONITOR_LINEAR_START)
+#define MON_START_L4OFF PT_LA_2_L4OFF(MONITOR_LINEAR_START)
+#define MON_START_L3OFF PT_LA_2_L3OFF(MONITOR_LINEAR_START)
+#define MON_START_L2OFF PT_LA_2_L2OFF(MONITOR_LINEAR_START)
+
+/*
+ * Allocated wired pages for the monitor.
+ * Currently we allocate wired pages for the:
+ *  0) The shared area between user and monitor;
+ *  a) GDT of the monitor
+ *  a1) BT (CP1) stack
+ *  b) The monitor's page table with one page directory under 4 GB
+ *      and two page table pages.
+ *  c) Pages for the monitor's stack.
+ *  d) The BT area pages
+ *  e) The cross driver/monitor page.
+ *  f) The physical memory array for the machine.
+ *  g) The SMRAM memory
+ *
+ * We allocate the shared area separately since it must be read/write
+ * in the monitor.   See SharedArea_PowerOn().
+ */
+
+#define MON_BOOTSTRAP_PAGE_TABLE_PAGES (BOOTSTRAP_REGION_LEN     /  \
+                                        PAE_PTES_PER_PGTBL)
+#ifdef VMX86_SERVER
+#define NUM_MONWIRED_PAGES     0
+
+#define MON_PAGE_TABLE_ALLOC_PAGES 0
+#define NUM_MONWIRED_BOOTSTRAP_PAGES 0
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN  + \
+                                    DF_STACK_PAGES_LEN   + \
+                                    MC_STACK_PAGES_LEN   + \
+                                    NMI_STACK_PAGES_LEN  + \
+                                    HV_CURRENT_VMCB_LEN  + \
+                                    IDT_NORMAL_LEN       + \
+                                    GDT_AND_TASK_LEN)
+
+#else
+#define NUM_MONWIRED_PAGES    (CROSS_PAGE_CODE_LEN   + \
+                               CROSS_PAGE_DATA_LEN   + \
+                               MON_PAGE_TABLE_L4_LEN + \
+                               MON_PAGE_TABLE_L3_LEN + \
+                               MON_PAGE_TABLE_L2_LEN)
+
+#define MON_PAGE_TABLE_ALLOC_PAGES ((MONITOR_READONLY_REGION_LEN + \
+                                     MONITOR_DATA_REGION_LEN     + \
+                                     MONITOR_MISC_REGION_LEN)    / \
+                                     PAE_PTES_PER_PGTBL)
+
+#define NUM_MONWIRED_BOOTSTRAP_PAGES MON_BOOTSTRAP_PAGE_TABLE_PAGES
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN + \
+                                    DF_STACK_PAGES_LEN  + \
+                                    MC_STACK_PAGES_LEN  + \
+                                    NMI_STACK_PAGES_LEN + \
+                                    GDT_AND_TASK_LEN    + \
+                                    IDT_NORMAL_LEN      + \
+                                    HV_CURRENT_VMCB_LEN + \
+                                    SWITCH_PAGE_TABLE_LEN)
+#endif
+
+#define MON_PAGE_TABLE_L5        VPN_2_VA(MON_PAGE_TABLE_L5_START)
+#define MON_PAGE_TABLE_L4        VPN_2_VA(MON_PAGE_TABLE_L4_START)
+#define VMM_STACK_TOP            (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LEN)
+#define VMM_LRET_STACK_TOP       (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LRET_LEN)
+/*
+ * Mark out a guard page for the VMM stack, this is not in the addrlayout_table
+ * explicitly because it overlaps the last page of the MONITOR_READONLY region.
+ */
+#define VMM_STACK_GUARD_START    MON_STACK_PAGES_START
+#define VMM_STACK_GUARD_LEN      PAGE_SIZE
+
+/*
+ * Following functions return true if the range [va, va+len) is within
+ * range of a particular monitor stack.  (Don't adjust without
+ * considering uint64 overflow when va is very high.)
+ */
+static INLINE Bool
+AddrLayout_InMonStack(VA64 va, size_t len)
+{
+   return VPN_2_VA(MON_STACK_PAGES_START) <= va &&
+          va <= VMM_STACK_TOP_HW - len;
+}
+
+static INLINE Bool
+AddrLayout_InNMIStack(VA64 va, size_t len)
+{
+   return va >= NMI_STACK_BASE && va <= NMI_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InDFStack(VA64 va, size_t len)
+{
+   return va >= DF_STACK_BASE && va <= DF_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InMCStack(VA64 va, size_t len)
+{
+   return va >= MC_STACK_BASE && va <= MC_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InAMonitorStack(VA va, size_t len)
+{
+   return AddrLayout_InMonStack(va, len)  ||
+          AddrLayout_InNMIStack(va, len)  ||
+          AddrLayout_InDFStack(va, len)   ||
+          AddrLayout_InMCStack(va, len);
+}
+
+#endif
@@ -0,0 +1,144 @@
+/*********************************************************
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * addrlayout_table.h --
+ *
+ * Named elements in the monitor's address space of fixed size and location.
+ * All units are 4 kilobyte pages.  Also used for vmmlayout output.
+ *
+ * The monitor's address space is populated three different ways:
+ * 1) For elements of a fixed size and location, at compile-time.
+ * 2) For well-known program sections and the shared area, at link-time.
+ * 3) Dynamically, by the allocator, during bootstrapping and monitor power-on.
+ *
+ * This file describes all of the monitor's main 64MB address space.  All of
+ * (1) is described specifically.  Ranges for (2) and (3) are accounted for,
+ * but their contents are described generally here.
+ *
+ * The monitor's address space is organized into container regions which are an
+ * integer number of 2MB "large" pages.  Within a container region, items
+ * comprised of one or more 4KB pages may be described.
+ *
+ * All container regions and items are described by a name and a size in pages.
+ * No two items may have the same name nor may two regions.
+ *
+ * REGION(region-name, number-of-pages)
+ *   ITEM(item-name,   number-of-pages)     // first item in region
+ *   ITEM(item-name,   number-of-pages)     // second item in region
+ * <...>
+ */
+
+
+/*
+ * Monitor .rodata/.text.  Mapped large (reducing TLB pressure) and read-only.
+ */
+REGION(MONITOR_READONLY,        1024)
+  ITEM(MONITOR_READONLY_LINKER, 1024) // Used by linker.
+
+/*
+ * Monitor .data/.bss.  Mapped large (reducing TLB pressure) and read-write.
+ */
+REGION(MONITOR_DATA,            512)
+  ITEM(MON_STACK_PAGES,           8) // Monitor stack.
+  ITEM(IDT_NORMAL,                1) // Normal IDT (used for all but SVM).
+  ITEM(MONITOR_DATA,            503) // Used by linker and TC.
+
+/*
+ * The monitor's translation cache.  This object starts at the page boundary
+ * following the end of the used space in MONITOR_DATA and extends through
+ * the TC_REGION.
+ */
+REGION(TC_REGION,              1024)
+  ITEM(TC_BLOCK,               1024)
+
+/*
+ * The monitor's pages for architectural state, stacks, page tables, shared
+ * pages, and dynamic allocation.  This region also contains the linked shared
+ * area and other contents.
+ * Mapped small (allowing non-present mappings and sharing) and read-write.
+ */
+REGION(MONITOR_MISC,           5632)
+#ifdef VMX86_SERVER
+  ITEM(GUARD_PAGE,                2) // Reserved (for symmetry with hosted).
+#else
+  ITEM(CROSS_PAGE_DATA,           1) // Cross page data (RW).
+  ITEM(CROSS_PAGE_CODE,           1) // Cross page code (RX).
+#endif
+  ITEM(GDT_AND_TASK,              1) // GDT and Task State Segment.
+
+  ITEM(MON_PAGE_TABLE_L5,         1) // Monitor page root, if 5-Level PT used.
+  ITEM(MON_PAGE_TABLE_L4,         1) // Monitor page root, if 4-Level PT used.
+  ITEM(MON_PAGE_TABLE_L3,         1) // Monitor L3 page table.
+  ITEM(MON_PAGE_TABLE_L2,         1) // Monitor L2 page table.
+  ITEM(MON_PAGE_TABLE_L1,        32) // Monitor L1 page tables.
+
+  ITEM(HOST_APIC,                 1) // Physical APIC.
+  ITEM(GUEST_APIC,                1) // Guest APIC.
+
+  ITEM(DF_GUARD_PAGE,             1) // Double Fault stack guard page.
+  ITEM(DF_STACK_PAGES,            1) // Double Fault stack; need ~768 bytes.
+
+  ITEM(MC_GUARD_PAGE,             1) // Machine Check stack guard page.
+  ITEM(MC_STACK_PAGES,            1) // Machine Check stack.
+
+  ITEM(NMI_GUARD_PAGE,            1) // NMI stack guard page.
+  ITEM(NMI_STACK_PAGES,           4) // NMI stack.  Profiling requires a larger
+                                     // stack than MC/DF.
+
+  ITEM(HV_SWITCH,                 1) // SVM switch page
+  ITEM(HV_CURRENT_VMCB,           1) // SVM current VMCB
+  ITEM(HV_STD_NATIVE_VMCB,        1) // SVM/VT standard VMCB/VMCS
+  ITEM(HV_AUX_NATIVE_VMCB,        1) // SVM/VT auxiliary VMCB/VMCS
+  ITEM(VHV_GUEST_VMCB,            1) // VSVM/VVT guest VMCB/VMCS
+  ITEM(HV_SEV_VMSA,               1) // SVM SEV-ES state save area
+  ITEM(VPROBE_MON_RELOC,          1) // VProbe monitor reloc page.
+  ITEM(GART_LIST_PAGES,          48) // PhysMem vmm gart list pages.
+  ITEM(GART_ALT_LIST_PAGES,      48) // Alternate vmm gart pages
+                                     // (for invalidation).
+  ITEM(GART_BF_PAGES,             8) // Gart Bloom filter memory pages.
+  ITEM(VVT_GUEST_VIRT_APIC,       1) // Inner guest virtual APIC page.
+  ITEM(HT_STATE_MAP,              8) // Information used by the SecureHT module
+  ITEM(SHARED_RW_DATA,         5462) // R/W shared data, including the shared
+                                     // area and stat vars. Any remaining pages
+                                     // are left to the dynamic allocator, which
+                                     // extends throughout the entire reclaimed
+                                     // bootstrap region.
+/*
+ * Bootstrap-used space.  This space is reclaimed when bootstrap is done, and
+ * is used by the allocator thereafter.  The allocator starts at the first
+ * unused page in the SHARED_RW_DATA item from the MONITOR_MISC region and
+ * extends through to the end of the VMM address space once the bootstrap region
+ * is reclaimed.
+ */
+REGION(BOOTSTRAP,              8192)
+  ITEM(BS_TXT,                   17) // bootstrap's .text
+  ITEM(BS_RODATA,                10) // bootstrap's .rodata
+  ITEM(BS_DATA,                  10) // bootstrap's .data
+  ITEM(BS_BSS,                   10) // bootstrap's .bss
+  ITEM(BS_HEADER,                 1) // MonLoaderHeader mapping
+  ITEM(BS_VCPU_L1PT_RANGE,       32) // VCPU L1PT mappings
+  ITEM(BS_VCPU_L1PT_PT,           1) // L1PT that maps a VCPU's L1 page tables
+  ITEM(BS_INIT_POOL,              1) // pool for initializing VMM pages
+  ITEM(BS_DYNAMIC_ALLOC,       1880) // bs_alloc's dynamic allocator pages
+  ITEM(IDT_BOOTSTRAP,             1) // bootstrap IDT
+  ITEM(IDT_BOOTSTRAP_STUBS,       2) // bootstrap IDT gate stubs
+  ITEM(BS_PER_VM_VMX,           300) // vmm64's shared_per_vm_vmx
+  ITEM(BS_PER_VCPU,               8) // vmm64's shared_per_vcpu for VCPU 0
+  ITEM(BS_PER_VCPU_VMX,         136) // vmm64's shared_per_vcpu_vmx for VCPU 0
+  ITEM(VMM_MODULES,            5783) // ~22.6MB for unlinked VMM modules
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -143,9 +143,12 @@ PtrToVA64(void const *ptr) // IN
 * does not break vmcore compatibility.
 *
 * See bora/doc/vmcore details.
+ *
+ * If you increment VMMON_VERSION, you must also increment
+ * the NT specific VMX86_DRIVER_VERSION.
 */

-#define VMMON_VERSION           (385 << 16 | 0)
+#define VMMON_VERSION           (401 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -245,6 +248,7 @@ enum IOCTLCmd {
   IOCTLCMD(GET_IPI_VECTORS),
   IOCTLCMD(SEND_IPI),
   IOCTLCMD(SEND_ONE_IPI),
+   IOCTLCMD(GET_SWITCH_ERROR_ADDR),

   /*
    * Keep host-specific calls at the end so they can be undefined
@@ -253,18 +257,10 @@ enum IOCTLCmd {

 #if defined __linux__
   IOCTLCMD(SET_UID),		// VMX86_DEVEL only
-#endif
-
-#if defined __linux__ || defined __APPLE__
   IOCTLCMD(GET_ALL_CPUID),
 #endif

-#if defined _WIN32 || defined __APPLE__
-   IOCTLCMD(ALLOC_CONTIG_PAGES),
-#endif
-
 #if defined _WIN32
-   IOCTLCMD(FREE_CONTIG_PAGES),
   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows
   IOCTLCMD(CHANGE_HARD_LIMIT),         // used by vmauthd on Windows
   IOCTLCMD(READ_DISASM_PROC_BINARY),
@@ -272,6 +268,7 @@ enum IOCTLCmd {
   IOCTLCMD(SET_MEMORY_PARAMS),
   IOCTLCMD(REMEMBER_KHZ_ESTIMATE),
   IOCTLCMD(REMAP_SCATTER_LIST),
+   IOCTLCMD(REMAP_SCATTER_LIST_RO),     // map the list as read-only
   IOCTLCMD(UNMAP_SCATTER_LIST),
 #endif

@@ -281,10 +278,6 @@ enum IOCTLCmd {
   IOCTLCMD(BLUEPILL),
 #endif

-#if defined __linux__
-   IOCTLCMD(SET_HOST_SWAP_SIZE),
-#endif
-
   IOCTLCMD(GET_UNAVAIL_PERF_CTRS),
   IOCTLCMD(GET_MONITOR_CONTEXT),
   // Must be last.
@@ -324,6 +317,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
+#define IOCTL_VMX86_GET_SWITCH_ERROR_ADDR VMIOCTL_BUFFERED(GET_SWITCH_ERROR_ADDR)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
 #define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
@@ -336,8 +330,6 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UPDATE_MEM_INFO     VMIOCTL_BUFFERED(UPDATE_MEM_INFO)
 #define IOCTL_VMX86_HARD_LIMIT_MONITOR_STATUS   VMIOCTL_BUFFERED(HARD_LIMIT_MONITOR_STATUS)
 #define IOCTL_VMX86_CHANGE_HARD_LIMIT   VMIOCTL_BUFFERED(CHANGE_HARD_LIMIT)
-#define IOCTL_VMX86_ALLOC_CONTIG_PAGES  VMIOCTL_BUFFERED(ALLOC_CONTIG_PAGES)
-#define IOCTL_VMX86_FREE_CONTIG_PAGES   VMIOCTL_BUFFERED(FREE_CONTIG_PAGES)

 #define IOCTL_VMX86_GET_TOTAL_MEM_USAGE	VMIOCTL_BUFFERED(GET_TOTAL_MEM_USAGE)
 #define IOCTL_VMX86_GET_KHZ_ESTIMATE    VMIOCTL_BUFFERED(GET_KHZ_ESTIMATE)
@@ -367,20 +359,14 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_GET_PSEUDO_TSC           VMIOCTL_NEITHER(GET_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS    VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS)
 #define IOCTL_VMX86_GET_MONITOR_CONTEXT      VMIOCTL_BUFFERED(GET_MONITOR_CONTEXT)
-#define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
-#define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST    VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST_RO VMIOCTL_BUFFERED(REMAP_SCATTER_LIST_RO)
+#define IOCTL_VMX86_UNMAP_SCATTER_LIST    VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
 #endif


 #define INIT_BLOCK_MAGIC     (0x1789 + 14)

-/*
- * Flags sent into APICBASE ioctl
- */
-
-#define APIC_FLAG_DISABLE_NMI       0x00000001
-#define APIC_FLAG_PROBE             0x00000002
-#define APIC_FLAG_FORCE_ENABLE      0x00000004

 typedef
 #include "vmware_pack_begin.h"
@@ -409,10 +395,6 @@ union {
 #include "vmware_pack_end.h"
 VcpuPageRoot;

-typedef struct VMAPICInfo {
-   uint32 flags;
-} VMAPICInfo;
-
 #define VMX86_DRIVER_VCPUID_OFFSET 1000


@@ -551,6 +533,8 @@ typedef struct IPIVectors {
 */
 typedef struct VMCreateBlock {
   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
+   VA64               vmmonData;     // IN: User VA of a userlevel scratch area
+                                     //     required by the Linux vmmon
   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
   uint32             numVCPUs;      // IN: Number of VCPUs.
   uint16             vmid;          // OUT: VM ID for the created VM.
@@ -583,7 +567,6 @@ typedef struct VMStatVarsRegistrationBlock {
 } VMStatVarsRegistrationBlock;

 typedef struct {
-   VA64   crosspage; // IN: User VA of VCPU crosspage.
   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
 } PerVcpuPages;

@@ -599,6 +582,15 @@ typedef struct VMProcessBootstrapBlock {
   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

+/*
+ * Arguments for retrieving the switch error address.
+ */
+typedef struct VMSwitchErrorArgs {
+   Vcpuid vcpuid;  // IN: The VCPU of interest.
+   uint64 addr;    // OUT: The code address that a failure was detected at, or 0
+                   //      if no failure has occurred.
+} VMSwitchErrorArgs;
+
 static INLINE size_t
 GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
 {
@@ -614,39 +606,7 @@ typedef union {
   Context64 context; // OUT
 } VMMonContext;

-#if defined __linux__
-
-/*
- * Linux uses mmap(2) to allocate contiguous locked pages, and uses these
- * macros to marshall real arguments to mmap's made-up 'offset' argument.
- */
-
-#define VMMON_MAP_OFFSET_SHIFT  0
-#define VMMON_MAP_OFFSET_MASK   0x00000FFF
-#define VMMON_MAP_ORDER_SHIFT   12
-#define VMMON_MAP_ORDER_MASK    0xF
-#define VMMON_MAP_RSVD_SHIFT    16
-
-#define VMMON_MAP_RSVD(base)    \
-                ((base) >> VMMON_MAP_RSVD_SHIFT)
-#define VMMON_MAP_ORDER(base)   \
-                (((base) >> VMMON_MAP_ORDER_SHIFT) & VMMON_MAP_ORDER_MASK)
-#define VMMON_MAP_OFFSET(base)  \
-                (((base) >> VMMON_MAP_OFFSET_SHIFT) & VMMON_MAP_OFFSET_MASK)
-
-#define VMMON_MAP_BASE(order)   ((order) << VMMON_MAP_ORDER_SHIFT)
-
-#elif defined _WIN32
-/*
- * Windows uses an ioctl to allocate contiguous locked pages.
- */
-
-typedef struct VMAllocContiguousMem {
-   VA64   mpnList;  // IN: User VA of an array of 64-bit MPNs.
-   uint32 mpnCount; // IN
-   uint32 order;    // IN
-} VMAllocContiguousMem;
-#elif defined __APPLE__
+#if defined __APPLE__
 #   include "iocontrolsMacos.h"
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,8 +42,6 @@
 #include "mon_assert.h"

 #define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
-#define CP_STUB_SIZE     16     /* A relative jmp instruction (5 bytes)
-                                   padded to the next 16 byte boundary. */

 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
@@ -57,6 +55,7 @@
   MC(RELEASE_ANON_PAGES)                                                     \
   MC(LOOKUP_MPN)                                                             \
   MC(COSCHED)                                                                \
+   MC(ALLOC_CONTIG_PAGES)                                                     \
   MC(ALLOC_VMX_PAGE)                                                         \
   MC(ALLOC_TMP_GDT)                                                          \
   MC(PIN_MPN)                                                                \
@@ -65,11 +64,14 @@
   MC(ALLOC_ANON_LOW_PAGE)                                                    \
   MC(GET_MON_IPI_VECTOR)                                                     \
   MC(GET_HV_IPI_VECTOR)                                                      \
+   MC(GET_PERF_CTR_VECTOR)                                                    \
   MC(GET_HOST_TIMER_VECTORS)                                                 \
   MC(BOOTSTRAP_CLEANUP)                                                      \
   MC(GET_SHARED_AREA)                                                        \
   MC(GET_STAT_VARS)                                                          \
-   MC(GET_NUM_PTP_PAGES)
+   MC(GET_NUM_PTP_PAGES)                                                      \
+   MC(GET_HV_IO_BITMAP)                                                       \
+   MC(GET_MSR)                                                                \

 /*
 *----------------------------------------------------------------------
@@ -107,8 +109,8 @@ typedef enum ModuleCallType {
 */

 #if defined(VMX86_UCCOST) && !defined(VMX86_SERVER)
-#define UCTIMESTAMP(cp, stamp) \
-             do { (cp)->ucTimeStamps[UCCOST_ ## stamp] = RDTSC(); } while (0)
+#define UCTIMESTAMP(ptr, stamp) \
+             do { (ptr)[UCCOST_ ## stamp] = RDTSC(); } while (0)
 #else
 #define UCTIMESTAMP(cp, stamp)
 #endif
@@ -128,7 +130,7 @@ typedef struct UCCostResults {
 } UCCostResults;

 typedef enum UCCostStamp {
-#define UC(x) UCCOST_ ## x,
+#define UC(x, y) UCCOST_ ## x,
 #include "uccostTable.h"
   UCCOST_MAX
 } UCCostStamp;
@@ -142,12 +144,12 @@ typedef enum UCCostStamp {
 * MAX_SWITCH_PT_PATCHES
 *
 *   This is the maximum number of patches that must be placed into
- *   the monitor page tables so that the host GDT and the crosspage
- *   can be accessed during worldswitch.
+ *   the monitor page tables so that the host GDT, crosspage data, and
+ *   crosspage code can be accessed during worldswitch.
 *
 *----------------------------------------------------------------------
 */
-#define MAX_SWITCH_PT_PATCHES 2
+#define MAX_SWITCH_PT_PATCHES 3

 /*----------------------------------------------------------------------
 *
@@ -234,16 +236,23 @@ VMMPageTablePatch;
 * - the largest stack use instantaneously possible by #DB handling code
 * - one high-water uint32 used to detect stack overflows when debugging
 *
- * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+ * A breakdown of the worst-case exception handler stack usage (SwitchUDHandler)
+ * is: 5 * uint64 (Hardware) + 1 * uint64 (RAX) + 1 * uint64 (RBX) +
+ *     1 * uint64 (RCX) + 1 * uint64 (call) + 2 * uint64 (sidt) = 11 * uint64
+ * This is a slight over-estimate of the possible usage at any time but there
+ * is plenty of space available in the cross page data area.
+ *
+ * 264 (11 * sizeof(uint64) * 3) bytes is slightly more than enough as of
+ * 2020/06/14.
 */
-#define TINY_STACK_SIZE      184
+#define TINY_STACK_SIZE      264

 /*
 *----------------------------------------------------------------------
 *
 * VMCrossPageData --
 *
- *      Data which is stored on the VMCrossPage.
+ *      Data which is stored on the cross page.
 *
 *----------------------------------------------------------------------
 */
@@ -292,11 +301,13 @@ struct VMCrossPageData {
   uint16   monES;  /* Not saved/restored in assembly switch */
   uint16   monPad;

-   uint64   crosspageMA;
+   uint64   crosspageDataMA;

   uint64   hostDR[8];
-   LA64     crossPageLA;       // where host/PTP map the cross page
+   LA64     crosspageDataLA;   // where host/PTP map the cross data page
   LA64     crossGDTLA;        // where host/PTP map the cross GDT
+   LA64     crosspageCodeLA;   // where host/PTP map the cross code page
+   LA64     vmmToHostLA;       // where host has placed the VmmToHost function
   uint16   hostInitial64CS;
   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
@@ -335,6 +346,7 @@ struct VMCrossPageData {

 #if !defined(VMX86_SERVER)
   uint64 ucTimeStamps[UCCOST_MAX];
+   uint8  _ucPad[8];
 #endif

   SwitchedMSRState switchedMSRState;
@@ -393,76 +405,23 @@ struct VMCrossPageData {
   uint8         _pad7[4];
   uint64        wsUD2;                       // IP of ud2 instr or 0 if unset.
   uint64        specCtrl; /* host MSR_SPEC_CTRL value before world switch. */
-   uint8         _pad8[8];
 }
 #include "vmware_pack_end.h"
 VMCrossPageData;

-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPageCode --
- *
- *      Code which is stored on the VMCrossPage.
- *
- *----------------------------------------------------------------------
- */
-
-#define CODE_BLOCK_SIZE (PAGE_SIZE - sizeof(VMCrossPageData) - \
-                         (2 + NUM_EXCEPTIONS) * CP_STUB_SIZE)
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPageCode {
-   uint8         toVmmFunc[CP_STUB_SIZE];    // Fixed-position stubs to jump
-   uint8         toHostFunc[CP_STUB_SIZE];   // to world switch functions
-   uint8         gateStubs[NUM_EXCEPTIONS *
-                           CP_STUB_SIZE];    // Stubs for crossIDT.
-   uint8         codeBlock[CODE_BLOCK_SIZE]; // Code for worldswitch and
-                                             // fault handling.
-}
-#include "vmware_pack_end.h"
-VMCrossPageCode;
-
-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPage --
- *
- *      Data structure shared between the monitor and the module
- *      that is used for crossing between the two.
- *      Accessible as vm->crosspage (kernel module) and CROSS_PAGE
- *      (monitor)
- *
- *      Must be exactly one page long.
- *
- *----------------------------------------------------------------------
- */
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPage {
-   VMCrossPageData crosspageData;
-   VMCrossPageCode crosspageCode;
-}
-#include "vmware_pack_end.h"
-VMCrossPage;
-
-#define CROSSPAGE_VERSION_BASE 0xc0c /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc12 /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE             ((VMCrossPage * const)VPN_2_VA(CROSS_PAGE_START))
-#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
+#define CROSS_PAGE             ((VMCrossPageData *) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
+#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData * const) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
 #endif

 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
-#define MX_WAITNORMAL          1  // Must equal one; see linux module code.
-#define MX_WAITERROR           0  // Use MX_ISWAITERROR() to test for error.
+#define MX_WAITNORMAL          1
+#define MX_WAITERROR           0

-// Any zero or negative value denotes error.
-#define MX_ISWAITERROR(e)      ((e) <= MX_WAITERROR)
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006,2009-2011,2013-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006,2009-2011,2013-2015,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,8 +66,7 @@
   SWMSR(MSR_CSTAR)         \
   SWMSR(MSR_SFMASK)        \
   SWMSR(MSR_TSC_AUX)       \
-   SWMSR(MSR_BD_TSC_RATIO)  \
-   SWMSR(MSR_BNDCFGS)
+   SWMSR(MSR_BD_TSC_RATIO)

 /*
 *      Data structures for dealing with the context-switched MSRs that need
@@ -94,7 +93,6 @@ typedef struct SwitchedMSRValues {
 typedef struct SwitchedMSRState {
   SwitchedMSRValues smv;
   uint8             flags[NUM_SWITCHED_MSRS];
-   uint32            _pad;
 } SwitchedMSRState;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -296,9 +296,10 @@ typedef enum MonLoaderError {
 * Values above shared area subindices for sharing of MonLoaderHeader
 * and htSchedStateMap.
 */
-#define MONLOADER_HEADER_IDX 6
-#define MONLOADER_HT_MAP_IDX 7
-
+#define MONLOADER_HEADER_IDX          6
+#define MONLOADER_HT_MAP_IDX          7
+#define MONLOADER_CROSS_PAGE_CODE_IDX 8
+#define MONLOADER_CROSS_PAGE_DATA_IDX 9

 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
                                 void *args, unsigned *line, Vcpuid *vcpu);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,7 +66,6 @@ typedef enum PCIPassthru_IntrType {
   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)

 typedef struct FPTIntrProxyInfo {
-   uint32            adapterIndex;
   uint32            vectorIndex;
   MonitorIdemAction actionID;
 } FPTIntrProxyInfo;
@@ -76,9 +75,15 @@ typedef struct UPTIntrProxyInfo {
   MonitorIdemAction actionID;
 } UPTIntrProxyInfo;

+typedef struct MPTIntrProxyInfo {
+   uint32            adapterIndex;
+   MonitorIdemAction actionID;
+} MPTIntrProxyInfo;
+
 typedef union PCIPassthru_IntrProxyInfo {
   FPTIntrProxyInfo fpt;
   UPTIntrProxyInfo upt;
+   MPTIntrProxyInfo mpt;
 } PCIPassthru_IntrProxyInfo;

 typedef enum PCIPassthru_ErrorType {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014, 2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -133,13 +133,19 @@
 */
 #define PERFCTR_AMD_SMI_COUNT                                  0x2b

-/* AMD Data Cache Events */
+/*
+ * AMD Data Cache Events
+ * Family <  17H: 0x45 - Unified TLB hit
+ *                0x46 - Unified TLB miss
+ * Family >= 17H: 0x45 - L1 DTLB miss (L2 DTLB hit or miss)
+ *                0x46 - Tablewalker
+ */
 #define PERFCTR_AMD_DATA_CACHE_ACCESSES                        0x40
 #define PERFCTR_AMD_DATA_CACHE_MISSES                          0x41
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_L2_OR_SYSTEM       0x42
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_SYSTEM             0x43
 #define PERFCTR_AMD_DATA_CACHE_LINES_EVICTED                   0x44
-#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT               0x45
+#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT_OR_MISS       0x45
 #define PERFCTR_AMD_L1_DTLB_AND_L2_DTLB_MISS                   0x46
 #define PERFCTR_AMD_MISALIGNED_ACCESSES                        0x47
 #define PERFCTR_AMD_PREFETCH_INSTRS_DISPATCHED                 0x4b
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -125,7 +125,7 @@ PTSC_MHz(void)

 #if defined(VM_X86_64) || defined(VM_ARM_64)

-/* 
+/*
 * Conversions to/from cycles.  Note that the conversions operate on
 * signed values, so be careful when taking the difference of two
 * VmAbsoluteTS (which is unsigned) that that value is not out of range
@@ -197,13 +197,13 @@ VmAbsoluteTS PTSC_Get(void);

 #endif

-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HasSynchronizedTSCs --
 *
 *      Returns TRUE iff the platform TSCs are known to be synchronized.
- * 
+ *
 *-----------------------------------------------------------------------------
 */

@@ -214,7 +214,7 @@ PTSC_HasSynchronizedTSCs(void)
 }


-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HostAdjustedTSCs --
@@ -231,4 +231,37 @@ PTSC_HostAdjustedTSCs(void)
   return ptscInfo.hwTSCsAdjusted;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PTSC_AdvanceTimer --
+ *
+ *      Advance '*deadline' in 'period' increments such that it is
+ *      greater than 'now'.  Return the number of ticks incremented.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE uint64
+PTSC_AdvanceTimer(VmAbsoluteTS now,
+                  VmIntervalTS period,
+                  VmAbsoluteTS *deadline)
+{
+   VmAbsoluteTS d = *deadline;
+   if (now >= d) {
+      uint64 count = 1;
+      d += period;
+      if (UNLIKELY(now >= d)) {
+         uint64 t = (now - d) / period + 1;
+         VmIntervalTS diff = t * period;
+         count += t;
+         d += diff;
+      }
+      *deadline = d;
+      return count;
+   }
+   return 0;
+}
+
 #endif /* ifndef _PTSC_H_ */
@@ -0,0 +1,176 @@
+/*********************************************************
+ * Copyright (C) 2018-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * segs.h --
+ *
+ *      Describes segment descriptors and structures containing them for both
+ *      the monitor (in hosted and ESX) and the vmkernel.
+ */
+
+#ifndef _SEGS_H_
+#define _SEGS_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "x86types.h"
+#include "x86segdescrs.h"
+#include "x86sel.h"
+#include "addrlayout.h"
+
+
+/*
+ * Segment map of the monitor.
+ *
+ * The GDT and Task State Segment reside consecutively on one page.  The
+ * monitor segments are placed at the end of the GDT.  The vmkernel can use all
+ * lower-numbered segments for user-mode as well as higher-numbered segments
+ * (though the vmkernel should not use monitor-private segments).  The high
+ * segment placement ensures that there is no selector-overlap with hosted
+ * kernel segments (the hosted world switch code can be a bit faster then, as
+ * it can use a single cross GDT).
+ */
+
+#define GDT_SIZE              (sizeof(Descriptor) * NUM_VALID_SEGMENTS)
+#define GDT_LIMIT             (GDT_SIZE - 1)
+
+#define IRB_SIZE              32 /* Interrupt redirection bitmap. */
+#define TSS_SIZE              (sizeof(Task64) + IRB_SIZE)
+
+#define GDT_START_VA          (VPN_2_VA(GDT_AND_TASK_START))
+#define TASK_START_VA         (VPN_2_VA(GDT_AND_TASK_START) + GDT_SIZE)
+
+/*
+ * vmkBoot uses some of the lower-numbered segments, as do host kernels on
+ * hosted.  User segments could start earlier than 32; see bug #1904257.
+ * Descriptor is used to describe boot, user and kernel segments.
+ * Descriptor64 (twice as large) is used for task segments.
+ */
+#define NUM_BOOT_SEGMENTS     32
+#define NUM_USER_SEGMENTS     (FIRST_SYSTEM_SEGMENT - FIRST_USER_SEGMENT)
+#define NUM_SYSTEM_SEGMENTS   2
+#define NUM_TASK_SEGMENTS     2
+
+#define FIRST_USER_SEGMENT    NUM_BOOT_SEGMENTS
+#define FIRST_SYSTEM_SEGMENT  (PAGE_SIZE / sizeof(Descriptor) - \
+                               NUM_SYSTEM_SEGMENTS            - \
+                               NUM_TASK_SEGMENTS * 2          - \
+                               TSS_SIZE / sizeof(Descriptor))
+
+#define GDT_USER_TLS_MIN      USER_TLS_1_SEGMENT
+#define GDT_USER_TLS_MAX      USER_TLS_3_SEGMENT
+#define USER_TLS_COUNT        ((USER_TLS_3_SEGMENT - USER_TLS_1_SEGMENT) + 1)
+
+#define FOREACH_USER_TLS_INDEX(_i)              \
+   {                                            \
+      unsigned _i;                              \
+      for (_i = 0; _i < USER_TLS_COUNT; _i++) { \
+
+#define FOREACH_USER_TLS_INDEX_DONE             \
+      }                                         \
+   }
+
+#define NULL_LDTR             0
+
+/*
+ * The descriptor after SYSTEM_CODE_SEGMENT (loaded into %cs) must be
+ * appropriate for %ss because of the syscall instruction for 64-bit
+ * user worlds.  Thus SYSTEM_DATA_SEGMENT is directly after it.
+ */
+typedef enum VmwSegs {
+   NULL_SEGMENT             = 0,
+   /* (... reserved for host operating system or vmkBoot segments). */
+
+   USER32_CODE_SEGMENT      = FIRST_USER_SEGMENT,
+   USER_DATA_SEGMENT,
+   USER64_SYSRET_SEGMENT,
+   USER64_STACK_SEGMENT,
+   USER64_CODE_SEGMENT,
+   USER_TLS_1_SEGMENT,
+   USER_TLS_2_SEGMENT,
+   USER_TLS_3_SEGMENT,
+
+   AFTER_LAST_USER_SEGMENT,
+
+   SYSTEM_CODE_SEGMENT      = FIRST_SYSTEM_SEGMENT,
+   SYSTEM_DATA_SEGMENT,
+   VMKERNEL_TASK_SEGMENT,
+   VMKERNEL_TASK_SEGMENT_HI,
+   MONITOR_TASK_SEGMENT,
+   MONITOR_TASK_SEGMENT_HI,
+
+   NUM_VALID_SEGMENTS
+} VmwSegs;
+
+#define GDT_SYSTEM_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_SYSTEM_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_USER_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 3)
+#define GDT_USER_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 3)
+
+/* Selectors used statically in code or in assembly must be unchecked. */
+#define SYSTEM_NULL_SELECTOR    GDT_SYSTEM_SEL(NULL)
+#define USER32_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER32_CODE)
+#define USER_DATA_SELECTOR      GDT_USER_SEL_UNCHECKED(USER_DATA)
+#define USER64_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER64_CODE)
+#define USER64_SYSRET_SELECTOR  GDT_USER_SEL(USER64_SYSRET)
+#define SYSTEM_CODE_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_CODE)
+#define SYSTEM_DATA_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_DATA)
+#define MONITOR_TASK_SELECTOR   GDT_SYSTEM_SEL(MONITOR_TASK)
+#define VMKERNEL_TASK_SELECTOR  GDT_SYSTEM_SEL(VMKERNEL_TASK)
+
+/*
+ * The VMM GDT is comprised of many segment descriptors with one initial
+ * Task State Segment system descriptor.  The VMM Task State Segment is
+ * on the same page sequentially after its GDT.
+ */
+#pragma pack(push, 1)
+typedef struct StaticGDTAndTSS {
+   Descriptor   empty[NUM_BOOT_SEGMENTS + NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS];
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+   Task64       monTSS;
+   uint8        TSSIRBitmap[IRB_SIZE];
+} StaticGDTAndTSS;
+#pragma pack(pop)
+
+#pragma pack(push, 1)
+typedef struct VmkernelGDT {
+   Descriptor   bootSegs[NUM_BOOT_SEGMENTS];
+   Descriptor   userSegs[NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS]; /* VMM/VMK-shared. */
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+} VmkernelGDT;
+#pragma pack(pop)
+
+MY_ASSERTS(segs,
+           ASSERT_ON_COMPILE(SYSTEM_CODE_SEGMENT + 1 == SYSTEM_DATA_SEGMENT);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT - FIRST_USER_SEGMENT <=
+                             NUM_USER_SEGMENTS);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT <= FIRST_SYSTEM_SEGMENT);
+)
+
+#endif /* _SEGS_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -20,15 +20,12 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-UC(CALL_START)
-UC(BEGIN_BACK_TO_HOST)
-UC(SWITCHED_TO_MODULE)
-UC(VMX_SELECT_RETURN)
-UC(VMX_HANDLER_START)
-UC(VMX_SIGNAL)
-UC(MODULE_SIGNAL)
-UC(SWITCHING_TO_MONITOR)
-UC(DONE_BACK_TO_HOST)
-UC(CALL_END)
+UC(CALL_START,             FALSE)
+UC(BEGIN_BACK_TO_HOST,     FALSE)
+UC(SWITCHED_TO_MODULE,     TRUE)
+UC(VMX_HANDLER_START,      FALSE)
+UC(SWITCHING_TO_MONITOR,   TRUE)
+UC(DONE_BACK_TO_HOST,      FALSE)
+UC(CALL_END,               FALSE)

 #undef UC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,5 +31,6 @@
 * failing DeviceIoControl on Windows.
 */
 #define USERCALL_VMX86ALLOCERR (-8192)
+#define USERCALL_SWITCHERR     (-8193)

-#endif 
+#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,6 +59,9 @@ typedef uint32 Vcpuid;                 // VCPU number
 */
 #define MAX_SMALL_VM_VCPUS 128

+/* Supported limit. */
+#define MAX_SUPPORTED_VCPUS   (FEATURE_STATE_IS_ENABLED(MonsterVM) ? 768 : 256)
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,10 +44,11 @@
 #   include "str.h"     /* Str_Snprintf */
 #   define VCS_SNPRINTF Str_Snprintf
 #elif defined MONITOR_APP
-#   include <stdio.h>   /* libc snprintf */
 #   if defined WIN32
-#      define VCS_SNPRINTF _snprintf
+#      include "str.h"     /* Str_Snprintf */
+#      define VCS_SNPRINTF Str_Snprintf
 #   else
+#      include <stdio.h>   /* libc snprintf */
 #      define VCS_SNPRINTF snprintf
 #   endif
 #elif defined VMM || defined VMKERNEL
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016-2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -155,7 +155,6 @@ _GET_LDT(Selector * const result)
   } while (0)


-/* Checked against the Intel manual and GCC --thutt */
 #define _BUILD_SET_R(func, reg)        \
   static INLINE void                  \
   func(uintptr_t r)                   \
@@ -166,24 +165,20 @@ _GET_LDT(Selector * const result)
              : "memory");             \
   }

-/* Not yet checked against the Intel manual and GCC --slava
- *
- * 'volatile' because CRs and DRs can change without the compiler
- * knowing it (when there is a page fault, when a breakpoint occurs,
- * and moreover it seems there is no way to teach gcc that smsw
- * clobbers cr0 for example).
- *
- * The parameter is a 'uintptr_t *' so that the size of the actual
- * parameter must exactly match the size of the hardware register.
- * This prevents the use of 32-bit variables when building 64-bit
- * code.
+/*
+ * The inline asm is marked 'volatile' because CRs and DRs can change
+ * without the compiler knowing it (when there is a page fault, when a
+ * breakpoint occurs, and moreover it seems there is no way to teach
+ * gcc that smsw clobbers cr0 for example).
 */
 #define _BUILD_GET_R(func, reg)                         \
-   static INLINE void                                   \
-   func(uintptr_t *result)                              \
+   static INLINE uintptr_t                              \
+   func(void)                                           \
   {                                                    \
+      uintptr_t result;                                 \
      __asm__ __volatile__("mov %%" #reg ", %0"         \
-                           : "=r" (*result));           \
+                           : "=r" (result));            \
+      return result;                                    \
   }

 _BUILD_SET_R(_SET_CR0, cr0)
@@ -374,10 +369,10 @@ CLTS(void)
   }

 #define _BUILD_GET_DR(func, reg)                      \
-   static INLINE void                                 \
-   func(uintptr_t *result)                            \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
   {                                                  \
-      *result = __readdr(reg);                        \
+      return __readdr(reg);                           \
   }

 #define _BUILD_SET_CR(func, reg)                      \
@@ -388,10 +383,10 @@ CLTS(void)
   }

 #define _BUILD_GET_CR(func, reg)                      \
-   static INLINE void                                 \
-   func(uintptr_t *result)                            \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
   {                                                  \
-      *result = __readcr##reg();                      \
+      return __readcr##reg();                         \
   }

 _BUILD_SET_DR(_SET_DR0, 0)
@@ -422,31 +417,13 @@ _BUILD_GET_CR(_GET_CR8, 8);
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
 {
-#ifdef VM_X86_64
   _lgdt(dtr);
-#else
-   /*
-    * timetools 32-bit targets use an old version of MSVC that doesn't support
-    * the _lgdt intrinsic. Luckily, 32-bit still supports inline asm.
-    */
-   __asm mov eax, dtr
-   __asm lgdt [eax]
-#endif
 }

 static INLINE void
 _Get_GDT(_GETSET_DTR_TYPE *dtr)
 {
-#ifdef VM_X86_64
   _sgdt(dtr);
-#else
-   /*
-    * timetools 32-bit targets use an old version of MSVC that doesn't support
-    * the _sgdt intrinsic. Luckily, 32-bit still supports inline asm.
-    */
-   __asm mov eax, dtr
-   __asm sgdt [eax]
-#endif
 }

 static INLINE void
@@ -471,141 +448,6 @@ _Get_IDT(_GETSET_DTR_TYPE *dtr)

 #define RESTORE_FLAGS(x) __writeeflags(x)

-
-#ifdef VM_X86_32
-
-#define SET_TR(_tr)      _Set_TR(_tr)
-#define SET_LDT(_tr)     _Set_LDT(_tr)
-
-#define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
-#define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)
-
-#define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
-#define CLTS()           __asm clts
-
-#define FNCLEX()         __asm fnclex
-
-#define TLB_INVALIDATE_PAGE(_addr) {  \
-   void *_a = (_addr); \
-   __asm mov eax, _a __asm invlpg [eax] \
-}
-
-#define RAISE_INTERRUPT(_x)  {__asm int _x }
-#define RETURN_FROM_INT()   {__asm iretd }
-
-
-static INLINE void
-SET_DS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ds, ax
-}
-
-static INLINE void
-SET_ES(Selector val)
-{
-   __asm mov ax, val
-   __asm mov es, ax
-}
-
-static INLINE void
-SET_FS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov fs, ax
-}
-
-static INLINE void
-SET_GS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov gs, ax
-}
-
-static INLINE void
-SET_SS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ss, ax
-}
-
-static INLINE Selector
-GET_FS(void)
-{
-   Selector _v;
-   __asm mov _v,fs
-   return _v;
-}
-
-static INLINE Selector
-GET_GS(void)
-{
-   Selector _v;
-   __asm mov _v,gs
-   return _v;
-}
-
-static INLINE Selector
-GET_DS(void)
-{
-   Selector _v;
-   __asm mov _v,ds
-   return _v;
-}
-
-static INLINE Selector
-GET_ES(void)
-{
-   Selector _v;
-   __asm mov _v,es
-   return _v;
-}
-
-static INLINE Selector
-GET_SS(void)
-{
-   Selector _v;
-   __asm mov _v,ss
-   return _v;
-}
-
-static INLINE Selector
-GET_CS(void)
-{
-   Selector _v;
-   __asm mov _v,cs
-   return _v;
-}
-
-static INLINE void
-_Set_LDT(Selector val)
-{
-   __asm lldt val
-}
-
-static INLINE void
-_Set_TR(Selector val)
-{
-   __asm ltr val
-}
-
-static INLINE Selector
-_Get_LDT(void)
-{
-   Selector sel;
-   __asm sldt sel
-   return sel;
-}
-
-static INLINE Selector
-_Get_TR(void)
-{
-   Selector sel;
-   __asm str sel
-   return sel;
-}
-
-#endif /* !VM_X86_32 */
 #endif /* !__GNUC__ && !_MSC_VER */


@@ -618,7 +460,7 @@ _Get_TR(void)

 #define GET_CR_DR(regType, regNum, var) \
   do {                                 \
-      _GET_##regType##regNum(&(var));   \
+      var = _GET_##regType##regNum();   \
   } while (0)

 #define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
@@ -648,6 +490,11 @@ _Get_TR(void)
 #define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
 #define SET_CR8(expr) SET_CR_DR(CR, 8, expr)

+/*
+ * When the ULM macro-defines INTERRUPTS_ENABLED, attempting to define
+ * it as a function produces hard-to-diagnose compile-time errors.
+ */
+#if !defined(ULM) && !defined(INTERRUPTS_ENABLED)
 static INLINE Bool
 INTERRUPTS_ENABLED(void)
 {
@@ -655,6 +502,7 @@ INTERRUPTS_ENABLED(void)
   SAVE_FLAGS(flags);
   return ((flags & EFLAGS_IF) != 0);
 }
+#endif

 /*
 * [GS]ET_[GI]DT() are defined as macros wrapping a function
@@ -679,7 +527,7 @@ INTERRUPTS_ENABLED(void)
 #endif


-#if defined (__GNUC__) || (defined (_MSC_VER) && defined (VM_X86_32))
+#if defined (__GNUC__)
 static INLINE unsigned
 CURRENT_CPL(void)
 {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -207,7 +207,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +314,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -122,87 +129,29 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock And [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -3584,7 +3476,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3607,7 +3499,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -48,7 +55,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -500,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -553,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -589,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -633,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -656,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -783,17 +772,9 @@ PAUSE(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -841,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -854,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -1180,6 +1155,9 @@ RoundUpPow2_32(uint32 value)
 static INLINE unsigned
 PopCount32(uint32 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
   /*
    * Attribution:
    *     This algorithm was copied from:
@@ -1223,6 +1201,7 @@ PopCount32(uint32 value)
   value += (value >> 8);
   value += (value >> 16);
   return value & 0x0000003f;
+#endif
 }


@@ -1245,6 +1224,13 @@ PopCount32(uint32 value)
 static INLINE unsigned
 PopCount64(uint64 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
   value -= (value >> 1) & 0x5555555555555555ULL;
   value = ((value >> 2) & 0x3333333333333333ULL) +
           (value & 0x3333333333333333ULL);
@@ -1253,6 +1239,7 @@ PopCount64(uint64 value)
   value += value >> 16;
   value += value >> 32;
   return (unsigned) (value & 0xff);
+#endif
 }


@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,41 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef ULM
-void _fxsave64(void *save);
-void _fxsave(void *save);
-void _fxrstor64(const void *load);
-void _fxrstor(const void *load);
-void _xsave64(void *save, uint64 mask);
-void _xsave(void *save, uint64 mask);
-void _xsaveopt64(void *save, uint64 mask);
-void _xsavec(void *save, uint64 mask);
-void _xrstor64(const void *load, uint64 mask);
-void _xrstor(const void *load, uint64 mask);
-#endif /* ULM */
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#ifdef ULM
-#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
-                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
-#endif /* ULM */
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -358,11 +259,7 @@ SFENCE(void)
   );
 #elif defined _MSC_VER
   _ReadWriteBarrier();
-#if defined VM_X86_32
-   __asm sfence;
-#else
   _mm_sfence();
-#endif
   _ReadWriteBarrier();
 #else
 #error No compiler defined for SFENCE
@@ -438,10 +438,6 @@ void *_ReturnAddress(void);

 #define strtok_r  strtok_s

-#if (_MSC_VER < 1500)
-#define	vsnprintf _vsnprintf
-#endif
-
 typedef int uid_t;
 typedef int gid_t;

@@ -465,20 +461,9 @@ typedef int pid_t;
 #define       W_OK          2
 #define       R_OK          4

-#endif // }
+#endif // } _WIN32

-/*
- * Macro for username comparison.
- */
-
-#ifdef _WIN32 // {
-#define USERCMP(x,y)  Str_Strcasecmp(x,y)
-#else
-#define USERCMP(x,y)  strcmp(x,y)
-#endif // }
-
-
-#endif // }
+#endif // } USERLEVEL

 #ifndef va_copy

@@ -657,8 +642,10 @@ typedef int pid_t;
 #endif

 #ifdef VMM
+#define vmx86_vmm 1
 #define VMM_ONLY(x) x
 #else
+#define vmx86_vmm 0
 #define VMM_ONLY(x)
 #endif

@@ -699,7 +686,6 @@ typedef int pid_t;
 * display/printer drivers only.
 */
 #ifdef _WIN32
-#ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
 #ifdef _WIN64
 #define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
@@ -711,7 +697,6 @@ typedef int pid_t;
 #define WinDrvPrint(arg, ...)
 #define WinDrvEngPrint(arg, ...)
 #endif
-#endif
 #endif // _WIN32

 /*
@@ -801,4 +786,30 @@ typedef int pid_t;
 #define END_PTR_TO_ALIGNED_VAR \
   } while (0)

+
+/*
+ * -Wswitch means that when you pass switch an enum that it's looking for
+ * all values from that enum, and only that enum, to be accounted for.
+ * "default:;" is fine for catching values you don't care about. But today
+ * we have a bunch of code that uses internal and external enum values, or
+ * in other words combines two enums into a single variable. This cast is
+ * the workaround, but we really need to fix this mess.
+ */
+#define UNCHECKED_SWITCH__FIXME(x) switch ((uint64)(x))
+
+
+/*
+ * When clang static analyzer parses source files, it implicitly defines
+ * __clang_analyzer__ macro. We use this to define our custom macro to stop
+ * its execution for the current path of analysis by calling a function that
+ * doesn't return, making it think that it hit a failed assertion.
+ *
+ * DO NOT use to silence the analyzer! See PR2447238.
+ */
+#ifdef __clang_analyzer__
+#define VMW_CLANG_ANALYZER_NORETURN() Panic("Disable Clang static analyzer")
+#else
+#define VMW_CLANG_ANALYZER_NORETURN() ((void)0)
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -101,6 +101,7 @@

 #pragma warning (3 :4505) // unreferenced local function
 #pragma warning (disable :4018) // signed/unsigned mismatch
+#pragma warning (suppress:4619) // suppress warning next line (C4761 was removed in vs2019u4)
 #pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
 #pragma warning (disable :4305) // truncation from 'const int' to 'short'
 #pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
@@ -687,22 +688,28 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-3.3 (circa 2003).  Enforce this here, various things below
+ * gcc-4.1 (circa 2006).  Enforce this here, various things below
 * this line depend upon it.
 *
- * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
- * The various linux kernel modules may use older (gcc-3.3) compilers.
+ * Current oldest compilers:
+ * - guest tools: 4.1.2 (freebsd/solaris)
+ * - buildhost compiler: 4.4.3
+ * - hosted kernel modules: 4.5
+ *
+ * SWIG's preprocessor is exempt.
 */
-#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
-#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
+#ifndef SWIG
+#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
+#error "gcc version is too old, need gcc-4.1 or better"
+#endif
 #endif

 /*
- * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Similarly, we require a compiler that is at least vs2012.
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1400
-#error "cl.exe version is too old, need vc80 or better"
+#if defined _MSC_VER && _MSC_VER < 1700
+#error "cl.exe version is too old, need vs2012 or better"
 #endif


@@ -864,12 +871,10 @@ typedef void * UserVA;
 #endif

 #ifndef UNUSED_TYPE
-// XXX _Pragma would better but doesn't always work right now.
 #  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
 #endif

 #ifndef UNUSED_VARIABLE
-// XXX is there a better way?
 #  define UNUSED_VARIABLE(_var) (void)_var
 #endif

@@ -884,29 +889,21 @@ typedef void * UserVA;

 /*
 * ALIGNED specifies minimum alignment in "n" bytes.
+ *
+ * NOTE: __declspec(align) has limited syntax; it must essentially be
+ *       an integer literal.  Expressions, such as sizeof(), do not
+ *       work.
 */

 #ifdef __GNUC__
 #define ALIGNED(n) __attribute__((__aligned__(n)))
+#elif defined(_MSC_VER)
+#define ALIGNED(n) __declspec(align(n))
 #else
 #define ALIGNED(n)
 #endif


-/*
- * Encapsulate the syntactic differences between gcc and msvc alignment control.
- * BOUNDARY must match in the prefix and suffix.
- */
-
-#ifdef _WIN32
-#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
-#define ALIGN_SUFFIX(BOUNDRY)
-#else
-#define ALIGN_PREFIX(BOUNDRY)
-#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
-#endif
-
-
 /*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
@@ -0,0 +1,88 @@
+/*********************************************************
+ * Copyright (C) 2012,2017-2018,2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vm_idt_x86.h --
+ *
+ *	Definitions for IDT use in x86/x86-64 ESX and hosted VMware Products.
+ */
+
+#ifndef _VM_IDT_X86_H_
+#define _VM_IDT_X86_H_
+
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMCORE
+#include "includeCheck.h"
+
+#if defined __cplusplus
+extern "C" {
+#endif
+
+/*
+ * User worlds are the only worlds that expect to receive
+ * interrupts on a stack specified in the %rsp entries of
+ * the TSS.
+ */
+#define TSS_RSP0 0
+#define TSS_RSP1 1
+#define TSS_RSP2 2
+
+/*
+ * The monitor and the vmkernel use gate stub handlers of a single size in
+ * their IDTs.  The only exception is the #PF gate (#14) which is twice as
+ * long in the monitor IDT, to copy CR2.
+ */
+#define IDT_STUB_SIZE              16
+
+#define IST_NONE                    0
+/* IST entries for the monitor. */
+#define IST_VMM_DF                  1
+#define IST_VMM_NMI                 2
+#define IST_VMM_MCE                 3
+#define MAX_VMM_IST                 3
+/* IST entries for the vmkernel. */
+#define IST_VMK_MCE                 5
+#define IST_VMK_DF                  6
+#define IST_VMK_NMI                 7
+
+
+static INLINE int
+IDT_MonitorISTForVector(int v)
+{
+   /*
+    * For non-SVM execution, the monitor uses the IST for #DF, #NMI, and #MC.
+    */
+   if (v == EXC_DF) {
+      return IST_VMM_DF;
+   } else if (v == EXC_NMI) {
+      return IST_VMM_NMI;
+   } else if (v == EXC_MC) {
+      return IST_VMM_MCE;
+   }
+   return IST_NONE;
+}
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
+#endif /* _VM_IDT_X86_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,6 +32,7 @@
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMX
 #include "includeCheck.h"
+#include "community_source.h"

 /*
 * Page remapping definitions.
@@ -46,6 +47,7 @@
 #define VMMEM_ANON_IOABLE_PAGE    VMMEM_FLAG_BIT(4)
 #define VMMEM_ANON_ALL_FLAGS      MASK(5)

+#define VMMEM_GUEST_NO_FLAGS      0
 #define VMMEM_GUEST_WRITEABLE     VMMEM_FLAG_BIT(0)
 #define VMMEM_GUEST_BREAKCOW      VMMEM_FLAG_BIT(1)
 #define VMMEM_GUEST_2M_PAGE       VMMEM_FLAG_BIT(2)
@@ -96,10 +98,12 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

+#define MDEF_DRAINER
+
 #define VMMEM_SERVICES_DEFS                                       \
   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
-   MDEF(VMMEM_SERVICES_TYPE_DIRTY_CLEAR, DirtyDrainer_ClearDirty)
+   MDEF_DRAINER

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -99,6 +99,7 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
+#define CR4_CET        0x00800000
 #define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

@@ -203,6 +204,7 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_CP           21  // Control Protection exception.
 #define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

@@ -210,7 +212,7 @@
 #define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
                                (1u << EXC_NP) | (1u << EXC_SS) | \
                                (1u << EXC_GP) | (1u << EXC_PF) | \
-                                (1u << EXC_AC))
+                                (1u << EXC_AC) | (1u << EXC_CP))

 /*
 * eflag/rflag definitions.
@@ -220,24 +222,24 @@

 typedef enum x86_FLAGS {
   EFLAGS_NONE         = 0,
-   EFLAGS_CF           = (1 << 0),     /* User */ 
-   EFLAGS_SET          = (1 << 1),                                  
-   EFLAGS_PF           = (1 << 2),     /* User */ 
-   EFLAGS_AF           = (1 << 4),     /* User */ 
-   EFLAGS_ZF           = (1 << 6),     /* User */ 
-   EFLAGS_SF           = (1 << 7),     /* User */ 
-   EFLAGS_TF           = (1 << 8),     /* Priv */ 
-   EFLAGS_IF           = (1 << 9),     /* Priv */ 
-   EFLAGS_DF           = (1 << 10),    /* User */ 
-   EFLAGS_OF           = (1 << 11),    /* User */ 
-   EFLAGS_NT           = (1 << 14),    /* Priv */ 
-   EFLAGS_RF           = (1 << 16),    /* Priv */ 
-   EFLAGS_VM           = (1 << 17),    /* Priv */ 
-   EFLAGS_AC           = (1 << 18),    /* Priv */ 
-   EFLAGS_VIF          = (1 << 19),    /* Priv */ 
-   EFLAGS_VIP          = (1 << 20),    /* Priv */ 
+   EFLAGS_CF           = (1 << 0),     /* User */
+   EFLAGS_SET          = (1 << 1),
+   EFLAGS_PF           = (1 << 2),     /* User */
+   EFLAGS_AF           = (1 << 4),     /* User */
+   EFLAGS_ZF           = (1 << 6),     /* User */
+   EFLAGS_SF           = (1 << 7),     /* User */
+   EFLAGS_TF           = (1 << 8),     /* Priv */
+   EFLAGS_IF           = (1 << 9),     /* Priv */
+   EFLAGS_DF           = (1 << 10),    /* User */
+   EFLAGS_OF           = (1 << 11),    /* User */
+   EFLAGS_NT           = (1 << 14),    /* Priv */
+   EFLAGS_RF           = (1 << 16),    /* Priv */
+   EFLAGS_VM           = (1 << 17),    /* Priv */
+   EFLAGS_AC           = (1 << 18),    /* Priv */
+   EFLAGS_VIF          = (1 << 19),    /* Priv */
+   EFLAGS_VIP          = (1 << 20),    /* Priv */
   EFLAGS_ID           = (1 << 21),    /* Priv */
-   
+
   EFLAGS_IOPL         = 3 << EFLAGS_IOPL_SHIFT,
   EFLAGS_ARITH        = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_OF),
@@ -249,7 +251,7 @@ typedef enum x86_FLAGS {
   EFLAGS_ALL          = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_DF | EFLAGS_OF | EFLAGS_TF |
                          EFLAGS_IF | EFLAGS_IOPL | EFLAGS_NT | EFLAGS_RF |
-                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP | 
+                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP |
                          EFLAGS_ID),
   EFLAGS_ALL_16       = EFLAGS_ALL & 0xffff,
   EFLAGS_REAL_32      = (EFLAGS_ALL & ~(EFLAGS_VIP | EFLAGS_VIF | EFLAGS_VM)),
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -58,6 +58,7 @@ typedef struct CPUIDRegs {
 typedef union CPUIDRegsUnion {
   uint32 array[4];
   CPUIDRegs regs;
+   uint64 force8byteAlign[2]; /* See CpuidInfoNodePtr (needed on Apple Mac). */
 } CPUIDRegsUnion;

 /*
@@ -117,49 +118,57 @@ CPUIDQuery;
 * this level or parts of it to guest.
 */

-#define CPUID_CACHED_LEVELS                         \
-   CPUIDLEVEL(TRUE,  0,   0,          0,  0)        \
-   CPUIDLEVEL(TRUE,  1,   1,          0,  0)        \
-   CPUIDLEVEL(FALSE, 2,   2,          0,  0)        \
-   CPUIDLEVEL(FALSE, 4,   4,          7,  0)        \
-   CPUIDLEVEL(FALSE, 5,   5,          0,  0)        \
-   CPUIDLEVEL(TRUE,  6,   6,          0,  0)        \
-   CPUIDLEVEL(TRUE,  7,   7,          1,  0)        \
-   CPUIDLEVEL(FALSE, A,   0xA,        0,  0)        \
-   CPUIDLEVEL(FALSE, B,   0xB,        2,  0)        \
-   CPUIDLEVEL(TRUE,  D,   0xD,       10,  0)        \
-   CPUIDLEVEL(TRUE,  F,   0xF,        2, 13)        \
-   CPUIDLEVEL(TRUE,  10,  0x10,       2, 13)        \
-   CPUIDLEVEL(TRUE,  12,  0x12,       4, 13)        \
-   CPUIDLEVEL(TRUE,  14,  0x14,       2, 13)        \
-   CPUIDLEVEL(TRUE,  15,  0x15,       0, 13)        \
-   CPUIDLEVEL(TRUE,  16,  0x16,       0, 13)        \
-   CPUIDLEVEL(TRUE,  17,  0x17,       4, 14)        \
-   CPUIDLEVEL(FALSE, 400, 0x40000000, 0,  0)        \
-   CPUIDLEVEL(FALSE, 401, 0x40000001, 0,  0)        \
-   CPUIDLEVEL(FALSE, 402, 0x40000002, 0,  0)        \
-   CPUIDLEVEL(FALSE, 403, 0x40000003, 0,  0)        \
-   CPUIDLEVEL(FALSE, 404, 0x40000004, 0,  0)        \
-   CPUIDLEVEL(FALSE, 405, 0x40000005, 0,  0)        \
-   CPUIDLEVEL(FALSE, 406, 0x40000006, 0,  0)        \
-   CPUIDLEVEL(FALSE, 410, 0x40000010, 0,  0)        \
-   CPUIDLEVEL(FALSE, 80,  0x80000000, 0,  0)        \
-   CPUIDLEVEL(TRUE,  81,  0x80000001, 0,  0)        \
-   CPUIDLEVEL(FALSE, 82,  0x80000002, 0,  0)        \
-   CPUIDLEVEL(FALSE, 83,  0x80000003, 0,  0)        \
-   CPUIDLEVEL(FALSE, 84,  0x80000004, 0,  0)        \
-   CPUIDLEVEL(FALSE, 85,  0x80000005, 0,  0)        \
-   CPUIDLEVEL(FALSE, 86,  0x80000006, 0,  0)        \
-   CPUIDLEVEL(FALSE, 87,  0x80000007, 0,  0)        \
-   CPUIDLEVEL(TRUE,  88,  0x80000008, 0,  0)        \
-   CPUIDLEVEL(TRUE,  8A,  0x8000000A, 0,  0)        \
-   CPUIDLEVEL(FALSE, 819, 0x80000019, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81A, 0x8000001A, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81B, 0x8000001B, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81C, 0x8000001C, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81D, 0x8000001D, 5,  0)        \
-   CPUIDLEVEL(FALSE, 81E, 0x8000001E, 0,  0)        \
-   CPUIDLEVEL(TRUE,  81F, 0x8000001F, 0, 14)
+/*            MASKS, LVL, VAL,      CNT, HWV */
+#define CPUID_CACHED_LEVELS                  \
+   CPUIDLEVEL(TRUE,  0,   0x0,        0,  0) \
+   CPUIDLEVEL(TRUE,  1,   0x1,        0,  0) \
+   CPUIDLEVEL(FALSE, 2,   0x2,        0,  0) \
+   CPUIDLEVEL(FALSE, 4,   0x4,        7,  0) \
+   CPUIDLEVEL(FALSE, 5,   0x5,        0,  0) \
+   CPUIDLEVEL(TRUE,  6,   0x6,        0,  0) \
+   CPUIDLEVEL(TRUE,  7,   0x7,        2,  0) \
+   CPUIDLEVEL(TRUE,  9,   0x9,        0, 17) \
+   CPUIDLEVEL(FALSE, A,   0xa,        0,  0) \
+   CPUIDLEVEL(FALSE, B,   0xb,        3,  0) \
+   CPUIDLEVEL(TRUE,  D,   0xd,       10,  0) \
+   CPUIDLEVEL(TRUE,  F,   0xf,        2, 13) \
+   CPUIDLEVEL(TRUE,  10,  0x10,       4, 13) \
+   CPUIDLEVEL(TRUE,  12,  0x12,       4, 13) \
+   CPUIDLEVEL(TRUE,  14,  0x14,       2, 13) \
+   CPUIDLEVEL(TRUE,  15,  0x15,       0, 13) \
+   CPUIDLEVEL(TRUE,  16,  0x16,       0, 13) \
+   CPUIDLEVEL(TRUE,  17,  0x17,       4, 14) \
+   CPUIDLEVEL(TRUE,  18,  0x18,       8, 17) \
+   CPUIDLEVEL(TRUE,  1A,  0x1a,       0, 17) \
+   CPUIDLEVEL(TRUE,  1B,  0x1b,       2, 17) \
+   CPUIDLEVEL(FALSE, 1F,  0x1f,       6, 17) \
+   CPUIDLEVEL(FALSE, 400, 0x40000000, 0,  0) \
+   CPUIDLEVEL(FALSE, 401, 0x40000001, 0,  0) \
+   CPUIDLEVEL(FALSE, 402, 0x40000002, 0,  0) \
+   CPUIDLEVEL(FALSE, 403, 0x40000003, 0,  0) \
+   CPUIDLEVEL(FALSE, 404, 0x40000004, 0,  0) \
+   CPUIDLEVEL(FALSE, 405, 0x40000005, 0,  0) \
+   CPUIDLEVEL(FALSE, 406, 0x40000006, 0,  0) \
+   CPUIDLEVEL(FALSE, 410, 0x40000010, 0,  0) \
+   CPUIDLEVEL(FALSE, 80,  0x80000000, 0,  0) \
+   CPUIDLEVEL(TRUE,  81,  0x80000001, 0,  0) \
+   CPUIDLEVEL(FALSE, 82,  0x80000002, 0,  0) \
+   CPUIDLEVEL(FALSE, 83,  0x80000003, 0,  0) \
+   CPUIDLEVEL(FALSE, 84,  0x80000004, 0,  0) \
+   CPUIDLEVEL(FALSE, 85,  0x80000005, 0,  0) \
+   CPUIDLEVEL(FALSE, 86,  0x80000006, 0,  0) \
+   CPUIDLEVEL(FALSE, 87,  0x80000007, 0,  0) \
+   CPUIDLEVEL(TRUE,  88,  0x80000008, 0,  0) \
+   CPUIDLEVEL(TRUE,  8A,  0x8000000a, 0,  0) \
+   CPUIDLEVEL(FALSE, 819, 0x80000019, 0,  0) \
+   CPUIDLEVEL(FALSE, 81A, 0x8000001a, 0,  0) \
+   CPUIDLEVEL(FALSE, 81B, 0x8000001b, 0,  0) \
+   CPUIDLEVEL(FALSE, 81C, 0x8000001c, 0,  0) \
+   CPUIDLEVEL(FALSE, 81D, 0x8000001d, 5,  0) \
+   CPUIDLEVEL(FALSE, 81E, 0x8000001e, 0,  0) \
+   CPUIDLEVEL(TRUE,  81F, 0x8000001f, 0, 14) \
+   CPUIDLEVEL(TRUE,  820, 0x80000020, 2, 17) \
+   CPUIDLEVEL(TRUE,  821, 0x80000021, 0, 17)

 #define CPUID_ALL_LEVELS CPUID_CACHED_LEVELS

@@ -183,6 +192,7 @@ enum {
 #define CPUID_FEATURE_INFORMATION  0x01
 #define CPUID_PROCESSOR_TOPOLOGY   4
 #define CPUID_MWAIT_FEATURES       5
+#define CPUID_PMC_FEATURES         0xa
 #define CPUID_XSAVE_FEATURES       0xd
 #define CPUID_SGX_FEATURES         0x12
 #define CPUID_PT_FEATURES          0x14
@@ -453,7 +463,13 @@ FLAG(   6,  0, EAX,  9,  1, HWP_ACTIVITY_WINDOW,                 NO,    0 ) \
 FLAG(   6,  0, EAX, 10,  1, HWP_ENERGY_PERFORMANCE_PREFERENCE,   NO,    0 ) \
 FLAG(   6,  0, EAX, 11,  1, HWP_PACKAGE_LEVEL_REQUEST,           NO,    0 ) \
 FLAG(   6,  0, EAX, 13,  1, HDC,                                 NO,    0 ) \
+FLAG(   6,  0, EAX, 14,  1, TURBO_BOOST_MAX_3,                   NO,    0 ) \
+FLAG(   6,  0, EAX, 15,  1, HWP_CAPABILITIES,                    NO,    0 ) \
+FLAG(   6,  0, EAX, 16,  1, HWP_PECI,                            NO,    0 ) \
+FLAG(   6,  0, EAX, 17,  1, HWP_FLEXIBLE,                        NO,    0 ) \
+FLAG(   6,  0, EAX, 18,  1, HWP_FAST_ACCESS,                     NO,    0 ) \
 FLAG(   6,  0, EAX, 19,  1, HW_FEEDBACK,                         NO,    0 ) \
+FLAG(   6,  0, EAX, 20,  1, HWP_IGNORE_IDLE_REQUEST,             NO,    0 ) \
 FIELD(  6,  0, EBX,  0,  4, NUM_INTR_THRESHOLDS,                 NO,    0 ) \
 FLAG(   6,  0, ECX,  0,  1, HW_COORD_FEEDBACK,                   NO,    0 ) \
 FLAG(   6,  0, ECX,  1,  1, ACNT2,                               ANY,  13 ) \
@@ -463,12 +479,9 @@ FLAG(   6,  0, EDX,  1,  1, ENERGY_CAP_REPORTING,                NO,    0 ) \
 FIELD(  6,  0, EDX,  8,  4, HW_FEEDBACK_SIZE,                    NO,    0 ) \
 FIELD(  6,  0, EDX, 16, 16, HW_FEEDBACK_INDEX,                   NO,    0 )

-/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
-#define CPUID_7_EDX_10 \
-FLAG(   7,  0, EDX, 10,  1, LEAF7_RSVD,                          NO,    0 )
-
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_7                                            \
+FIELD(  7,  0, EAX,  0, 32, LEAF_7_MAX_SUBLEVEL,                 YES,  18 ) \
 FLAG(   7,  0, EBX,  0,  1, FSGSBASE,                            YES,   9 ) \
 FLAG(   7,  0, EBX,  1,  1, TSC_ADJUST,                          ANY,  11 ) \
 FLAG(   7,  0, EBX,  2,  1, SGX,                                 ANY,  17 ) \
@@ -507,6 +520,7 @@ FLAG(   7,  0, ECX,  3,  1, PKU,                                 YES,  13 ) \
 FLAG(   7,  0, ECX,  4,  1, OSPKE,                               ANY,  13 ) \
 FLAG(   7,  0, ECX,  5,  1, WAITPKG,                             NO,    0 ) \
 FLAG(   7,  0, ECX,  6,  1, AVX512VBMI2,                         YES,  17 ) \
+FLAG(   7,  0, ECX,  7,  1, CET_SS,                              NO,    0 ) \
 FLAG(   7,  0, ECX,  8,  1, GFNI,                                YES,  17 ) \
 FLAG(   7,  0, ECX,  9,  1, VAES,                                YES,  17 ) \
 FLAG(   7,  0, ECX, 10,  1, VPCLMULQDQ,                          YES,  17 ) \
@@ -516,21 +530,33 @@ FLAG(   7,  0, ECX, 14,  1, AVX512VPOPCNTDQ,                     YES,  16 ) \
 FLAG(   7,  0, ECX, 16,  1, VA57,                                NO,    0 ) \
 FIELD(  7,  0, ECX, 17,  5, MAWA,                                NO,    0 ) \
 FLAG(   7,  0, ECX, 22,  1, RDPID,                               YES,  17 ) \
-FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            NO,    0 ) \
-FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             NO,    0 ) \
-FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           NO,    0 ) \
+FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            YES,  18 ) \
+FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             YES,  18 ) \
+FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           YES,  18 ) \
+FLAG(   7,  0, ECX, 29,  1, ENQCMD,                              NO,    0 ) \
 FLAG(   7,  0, ECX, 30,  1, SGX_LC,                              ANY,  17 ) \
 FLAG(   7,  0, EDX,  2,  1, AVX512QVNNIW,                        YES,  16 ) \
 FLAG(   7,  0, EDX,  3,  1, AVX512QFMAPS,                        YES,  16 ) \
-FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   NO,    0 ) \
-CPUID_7_EDX_10 \
+FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   YES,  18 ) \
+FLAG(   7,  0, EDX,  8,  1, AVX512VP2INTERSECT,                  YES,  18 ) \
+FLAG(   7,  0, EDX, 10,  1, MDCLEAR,                             YES,   9 ) \
 FLAG(   7,  0, EDX, 13,  1, TSX_MICROCODE_UPDATE,                NO,    0 ) \
+FLAG(   7,  0, EDX, 14,  1, SERIALIZE,                           NO,    0 ) \
+FLAG(   7,  0, EDX, 15,  1, HYBRID,                              NO,    0 ) \
+FLAG(   7,  0, EDX, 16,  1, TSXLDTRK,                            NO,    0 ) \
 FLAG(   7,  0, EDX, 18,  1, PCONFIG,                             NO,    0 ) \
+FLAG(   7,  0, EDX, 20,  1, CET_IBT,                             NO,    0 ) \
 FLAG(   7,  0, EDX, 26,  1, IBRSIBPB,                            ANY,   9 ) \
 FLAG(   7,  0, EDX, 27,  1, STIBP,                               YES,   9 ) \
 FLAG(   7,  0, EDX, 28,  1, FCMD,                                YES,   9 ) \
 FLAG(   7,  0, EDX, 29,  1, ARCH_CAPABILITIES,                   ANY,   9 ) \
-FLAG(   7,  0, EDX, 31,  1, SSBD,                                YES,   9 )
+FLAG(   7,  0, EDX, 30,  1, CORE_CAPABILITIES,                   NO,    0 ) \
+FLAG(   7,  0, EDX, 31,  1, SSBD,                                YES,   9 ) \
+FLAG(   7,  1, EAX,  5,  1, AVX512BF16,                          YES,  18 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_9                                            \
+FIELD(  9,  0, EAX,  0, 32, IA32_PLATFORM_DCA_CAP_VAL,           NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_A                                            \
@@ -574,7 +600,7 @@ FLAG(   D,  0, EAX,  6,  1, XCR0_MASTER_ZMM_H,                   YES,  13 ) \
 FLAG(   D,  0, EAX,  7,  1, XCR0_MASTER_HI16_ZMM,                YES,  13 ) \
 FLAG(   D,  0, EAX,  8,  1, XCR0_MASTER_XSS,                     NO,    0 ) \
 FLAG(   D,  0, EAX,  9,  1, XCR0_MASTER_PKRU,                    YES,  13 ) \
-FIELD(  D,  0, EAX,  10,22, XCR0_MASTER_LOWER,                   NO,    0 ) \
+FIELD(  D,  0, EAX, 10, 22, XCR0_MASTER_LOWER,                   NO,    0 ) \
 FIELD(  D,  0, EBX,  0, 32, XSAVE_ENABLED_SIZE,                  ANY,   8 ) \
 FIELD(  D,  0, ECX,  0, 32, XSAVE_MAX_SIZE,                      YES,   8 ) \
 FIELD(  D,  0, EDX,  0, 29, XCR0_MASTER_UPPER,                   NO,    0 ) \
@@ -588,7 +614,7 @@ FIELD(  D,  1, EBX,  0, 32, XSAVES_ENABLED_SIZE,                 ANY,  13 ) \
 FIELD(  D,  1, ECX,  0,  7, XSS_XCR0_USED0,                      NO,    0 ) \
 FLAG(   D,  1, ECX,  8,  1, XSS_PT,                              NO,    0 ) \
 FIELD(  D,  1, ECX,  9,  1, XSS_XCR0_USED1,                      NO,    0 ) \
-FIELD(  D,  1, ECX,  10,22, XSS_RSVD0,                           NO,    0 ) \
+FIELD(  D,  1, ECX, 10, 22, XSS_RSVD0,                           NO,    0 ) \
 FIELD(  D,  1, EDX,  0, 32, XSS_RSVD1,                           NO,    0 ) \
 FIELD(  D,  2, EAX,  0, 32, XSAVE_YMM_SIZE,                      YES,   8 ) \
 FIELD(  D,  2, EBX,  0, 32, XSAVE_YMM_OFFSET,                    YES,   8 ) \
@@ -637,11 +663,8 @@ FIELD(  D,  9, EBX,  0, 32, XSAVE_PKRU_OFFSET,                   YES,  13 ) \
 FLAG(   D,  9, ECX,  0,  1, XSAVE_PKRU_SUP_BY_XSS,               NO,    0 ) \
 FLAG(   D,  9, ECX,  1,  1, XSAVE_PKRU_ALIGN,                    YES,  13 ) \
 FIELD(  D,  9, ECX,  2, 30, XSAVE_PKRU_RSVD1,                    NO,    0 ) \
-FIELD(  D,  9, EDX,  0, 32, XSAVE_PKRU_RSVD2,                    NO,    0 ) \
-FIELD(  D, 62, EAX,  0, 32, XSAVE_LWP_SIZE,                      NO,    0 ) \
-FIELD(  D, 62, EBX,  0, 32, XSAVE_LWP_OFFSET,                    NO,    0 ) \
-FIELD(  D, 62, ECX,  0, 32, XSAVE_LWP_RSVD1,                     NO,    0 ) \
-FIELD(  D, 62, EDX,  0, 32, XSAVE_LWP_RSVD2,                     NO,    0 )
+FIELD(  D,  9, EDX,  0, 32, XSAVE_PKRU_RSVD2,                    NO,    0 )
+/* D, 62: AMD LWP leaf on BD, PD, SR. Dropped in Zen. Never referenced. */

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_F                                            \
@@ -675,10 +698,10 @@ FIELD( 12,  1, EAX,  0, 32, SECS_ATTRIBUTES0,                    ANY,  17 ) \
 FIELD( 12,  1, EBX,  0, 32, SECS_ATTRIBUTES1,                    ANY,  17 ) \
 FIELD( 12,  1, ECX,  0, 32, SECS_ATTRIBUTES2,                    ANY,  17 ) \
 FIELD( 12,  1, EDX,  0, 32, SECS_ATTRIBUTES3,                    ANY,  17 ) \
-FIELD( 12,  2, EAX,  0, 15, EPC00_VALID,                         ANY,  17 ) \
+FIELD( 12,  2, EAX,  0,  4, EPC00_VALID,                         ANY,  17 ) \
 FIELD( 12,  2, EAX, 12, 20, EPC00_BASE_LOW,                      ANY,  17 ) \
 FIELD( 12,  2, EBX,  0, 20, EPC00_BASE_HIGH,                     ANY,  17 ) \
-FIELD( 12,  2, ECX,  0, 15, EPC00_PROTECTED,                     ANY,  17 ) \
+FIELD( 12,  2, ECX,  0,  4, EPC00_PROTECTED,                     ANY,  17 ) \
 FIELD( 12,  2, ECX, 12, 20, EPC00_SIZE_LOW,                      ANY,  17 ) \
 FIELD( 12,  2, EDX,  0, 20, EPC00_SIZE_HIGH,                     ANY,  17 ) \
 FIELD( 12,  3, EAX,  0,  4, EPC01_VALID,                         NO,    0 ) \
@@ -711,6 +734,7 @@ FIELD( 14,  1, EBX, 16, 16, PT_AVAIL_PSB_FREQ_ENCS,             YES, FUT ) \
 #define CPUID_FIELD_DATA_LEVEL_15                                           \
 FIELD( 15,  0, EAX,  0, 32, DENOM_TSC_TO_CORE_CRYSTAL_CLK,       NO,    0 ) \
 FIELD( 15,  0, EBX,  0, 32, NUMER_TSC_TO_CORE_CRYSTAL_CLK,       NO,    0 ) \
+FIELD( 15,  0, ECX,  0, 32, CORE_CRYSTAL_CLK_FREQ,               NO,    0 ) \

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_16                                           \
@@ -739,7 +763,42 @@ FIELD( 17,  3, ECX,  0, 32, SOC_VENDOR_BRAND_STRING_3_2,         NO,    0 ) \
 FIELD( 17,  3, EDX,  0, 32, SOC_VENDOR_BRAND_STRING_3_3,         NO,    0 ) \

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
-#define CPUID_FIELD_DATA_LEVEL_400                                         \
+#define CPUID_FIELD_DATA_LEVEL_18                                           \
+FIELD( 18,  0, EAX,  0, 32, TLB_INFO_MAX_SUBLEAF,                NO,    0 ) \
+FLAG(  18,  0, EBX,  0,  1, TLB_INFO_LEVEL_SIZE_4K,              NO,    0 ) \
+FLAG(  18,  0, EBX,  1,  1, TLB_INFO_LEVEL_SIZE_2M,              NO,    0 ) \
+FLAG(  18,  0, EBX,  2,  1, TLB_INFO_LEVEL_SIZE_4M,              NO,    0 ) \
+FLAG(  18,  0, EBX,  3,  1, TLB_INFO_LEVEL_SIZE_1G,              NO,    0 ) \
+FIELD( 18,  0, EBX,  8,  3, TLB_INFO_PARTITIONING,               NO,    0 ) \
+FIELD( 18,  0, EBX, 16, 16, TLB_INFO_NUM_WAYS,                   NO,    0 ) \
+FIELD( 18,  0, ECX,  0, 32, TLB_INFO_NUM_SETS,                   NO,    0 ) \
+FIELD( 18,  0, EDX,  0,  5, TLB_INFO_TYPE,                       NO,    0 ) \
+FIELD( 18,  0, EDX,  5,  3, TLB_INFO_LEVEL,                      NO,    0 ) \
+FLAG(  18,  0, EDX,  8,  1, TLB_INFO_FULLY_ASSOCIATIVE,          NO,    0 ) \
+FIELD( 18,  0, EDX, 14, 12, TLB_INFO_MAX_ADDRESSABLE_IDS,        NO,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_1A                                           \
+FIELD( 1A,  0, EAX,  0, 24, NATIVE_MODEL_ID,                     NO,    0 ) \
+FIELD( 1A,  0, EAX, 24,  8, CORE_TYPE,                           NO,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_1B                                           \
+FIELD( 1B,  0, EAX,  0, 12, PCONFIG_SUBLEAF_TYPE,                NO,    0 ) \
+FIELD( 1B,  0, EBX,  0, 32, PCONFIG_TARGET_ID1,                  NO,    0 ) \
+FIELD( 1B,  0, ECX,  0, 32, PCONFIG_TARGET_ID2,                  NO,    0 ) \
+FIELD( 1B,  0, EDX,  0, 32, PCONFIG_TARGET_ID3,                  NO,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_1F                                           \
+FIELD( 1F,  0, EAX,  0,  5, TOPOLOGY_V2_MASK_WIDTH,              NA,    0 ) \
+FIELD( 1F,  0, EBX,  0, 16, TOPOLOGY_V2_CPUS_SHARING_LEVEL,      NA,    0 ) \
+FIELD( 1F,  0, ECX,  0,  8, TOPOLOGY_V2_LEVEL_NUMBER,            NA,    0 ) \
+FIELD( 1F,  0, ECX,  8,  8, TOPOLOGY_V2_LEVEL_TYPE,              NA,    0 ) \
+FIELD( 1F,  0, EDX,  0, 32, TOPOLOGY_V2_X2APIC_ID,               NA,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_400                                          \
 FIELD(400,  0, EAX,  0, 32, MAX_HYP_LEVEL,                       NA,    0 ) \
 FIELD(400,  0, EBX,  0, 32, HYPERVISOR_VENDOR0,                  NA,    0 ) \
 FIELD(400,  0, ECX,  0, 32, HYPERVISOR_VENDOR1,                  NA,    0 ) \
@@ -852,14 +911,12 @@ FIELD( 80,  0, EDX,  0, 32, LEAF80_VENDOR2,                      NA,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_81                                           \
-FIELD( 81,  0, EAX,  0, 32, UNKNOWN81EAX,                        ANY,   4 ) \
 FIELD( 81,  0, EAX,  0,  4, LEAF81_STEPPING,                     ANY,   4 ) \
 FIELD( 81,  0, EAX,  4,  4, LEAF81_MODEL,                        ANY,   4 ) \
 FIELD( 81,  0, EAX,  8,  4, LEAF81_FAMILY,                       ANY,   4 ) \
 FIELD( 81,  0, EAX, 12,  2, LEAF81_TYPE,                         ANY,   4 ) \
 FIELD( 81,  0, EAX, 16,  4, LEAF81_EXTENDED_MODEL,               ANY,   4 ) \
 FIELD( 81,  0, EAX, 20,  8, LEAF81_EXTENDED_FAMILY,              ANY,   4 ) \
-FIELD( 81,  0, EBX,  0, 32, UNKNOWN81EBX,                        ANY,   4 ) \
 FIELD( 81,  0, EBX,  0, 16, LEAF81_BRAND_ID,                     ANY,   4 ) \
 FIELD( 81,  0, EBX, 16, 16, UNDEF,                               ANY,   4 ) \
 FLAG(  81,  0, ECX,  0,  1, LAHF64,                              YES,   4 ) \
@@ -881,13 +938,14 @@ FLAG(  81,  0, ECX, 16,  1, FMA4,                                YES,   8 ) \
 FLAG(  81,  0, ECX, 17,  1, TCE,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 19,  1, NODEID_MSR,                          NO,    0 ) \
 FLAG(  81,  0, ECX, 21,  1, TBM,                                 YES,   9 ) \
-FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                            NO,    0 ) \
+FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                            ANY,  18 ) \
 FLAG(  81,  0, ECX, 23,  1, PERFCORE,                            ANY,   4 ) \
 FLAG(  81,  0, ECX, 24,  1, PERFNB,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 26,  1, DATABK,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 27,  1, PERFTSC,                             NO,    0 ) \
 FLAG(  81,  0, ECX, 28,  1, PERFL3,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 29,  1, MWAITX,                              NO,    0 ) \
+FLAG(  81,  0, ECX, 30,  1, ADDR_MASK_EXT,                       NO,    0 ) \
 FLAG(  81,  0, EDX,  0,  1, LEAF81_FPU,                          YES,   4 ) \
 FLAG(  81,  0, EDX,  1,  1, LEAF81_VME,                          YES,   4 ) \
 FLAG(  81,  0, EDX,  2,  1, LEAF81_DE,                           YES,   4 ) \
@@ -982,6 +1040,7 @@ FLAG(  87,  0, EBX,  1,  1, SUCCOR,                              NA,    0 ) \
 FLAG(  87,  0, EBX,  2,  1, HWA,                                 NA,    0 ) \
 FLAG(  87,  0, EBX,  3,  1, SCALABLE_MCA,                        NA,    0 ) \
 FLAG(  87,  0, EBX,  4,  1, PFEH_SUPPORT_PRESENT,                NA,    0 ) \
+FIELD( 87,  0, ECX,  0, 32, POWER_SAMPLE_TIME_RATIO,             NA,    0 ) \
 FLAG(  87,  0, EDX,  0,  1, TS,                                  NA,    0 ) \
 FLAG(  87,  0, EDX,  1,  1, FID,                                 NA,    0 ) \
 FLAG(  87,  0, EDX,  2,  1, VID,                                 NA,    0 ) \
@@ -991,7 +1050,12 @@ FLAG(  87,  0, EDX,  5,  1, STC,                                 NA,    0 ) \
 FLAG(  87,  0, EDX,  6,  1, 100MHZSTEPS,                         NA,    0 ) \
 FLAG(  87,  0, EDX,  7,  1, HWPSTATE,                            NA,    0 ) \
 FLAG(  87,  0, EDX,  8,  1, TSC_INVARIANT,                       NA,    0 ) \
-FLAG(  87,  0, EDX,  9,  1, CORE_PERF_BOOST,                     NA,    0 )
+FLAG(  87,  0, EDX,  9,  1, CORE_PERF_BOOST,                     NA,    0 ) \
+FLAG(  87,  0, EDX, 10,  1, EFFECTIVE_FREQUENCY,                 NA,    0 ) \
+FLAG(  87,  0, EDX, 11,  1, PROC_FEEDBACK_INTERFACE,             NA,    0 ) \
+FLAG(  87,  0, EDX, 12,  1, PROC_POWER_REPORTING,                NA,    0 ) \
+FLAG(  87,  0, EDX, 13,  1, CONNECTED_STANDBY,                   NA,    0 ) \
+FLAG(  87,  0, EDX, 14,  1, RAPL,                                NA,    0 )

 /*    LEVEL, REG, POS, SIZE, NAME,                          MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_88                                           \
@@ -1001,26 +1065,35 @@ FIELD( 88,  0, EAX, 16,  8, GUEST_PHYS_ADDR_SZ,                  YES,   8 ) \
 FLAG(  88,  0, EBX,  0,  1, CLZERO,                              YES,  14 ) \
 FLAG(  88,  0, EBX,  1,  1, IRPERF,                              NO,    0 ) \
 FLAG(  88,  0, EBX,  2,  1, XSAVE_ERR_PTR,                       NO,    0 ) \
-FLAG(  88,  0, EBX,  9,  1, WBNOINVD,                            YES, FUT ) \
+FLAG(  88,  0, EBX,  3,  1, INVLPGB,                             NO,    0 ) \
+FLAG(  88,  0, EBX,  4,  1, RDPRU,                               NO,    0 ) \
+FLAG(  88,  0, EBX,  6,  1, MBE,                                 NO,    0 ) \
+FLAG(  88,  0, EBX,  8,  1, MCOMMIT,                             NO,    0 ) \
+FLAG(  88,  0, EBX,  9,  1, WBNOINVD,                            YES,  17 ) \
+FLAG(  88,  0, EBX, 10,  1, LBREXTN,                             NO,    0 ) \
 FLAG(  88,  0, EBX, 12,  1, LEAF88_IBPB,                         ANY,   9 ) \
+FLAG(  88,  0, EBX, 13,  1, WBINVD_INT,                          NO,    0 ) \
 FLAG(  88,  0, EBX, 14,  1, LEAF88_IBRS,                         NO,    0 ) \
 FLAG(  88,  0, EBX, 15,  1, LEAF88_STIBP,                        NO,    0 ) \
 FLAG(  88,  0, EBX, 16,  1, LEAF88_IBRS_ALWAYS,                  NO,    0 ) \
 FLAG(  88,  0, EBX, 17,  1, LEAF88_STIBP_ALWAYS,                 NO,    0 ) \
 FLAG(  88,  0, EBX, 18,  1, LEAF88_PREFER_IBRS,                  NO,    0 ) \
+FLAG(  88,  0, EBX, 19,  1, LEAF88_IBRS_SAME_MODE,               NO,    0 ) \
+FLAG(  88,  0, EBX, 20,  1, LMSLE_UNSUPPORTED,                   NO,    0 ) \
+FLAG(  88,  0, EBX, 23,  1, PPIN,                                NO,    0 ) \
 FLAG(  88,  0, EBX, 24,  1, LEAF88_SSBD_SPEC_CTRL,               ANY,   9 ) \
 FLAG(  88,  0, EBX, 25,  1, LEAF88_SSBD_VIRT_SPEC_CTRL,          ANY,   9 ) \
 FLAG(  88,  0, EBX, 26,  1, LEAF88_SSBD_NOT_NEEDED,              NO,    0 ) \
 FIELD( 88,  0, ECX,  0,  8, LEAF88_CORE_COUNT,                   YES,   4 ) \
 FIELD( 88,  0, ECX, 12,  4, APICID_COREID_SIZE,                  YES,   7 ) \
-FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                        NO,    0 )
+FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                        NO,    0 ) \
+FIELD( 88,  0, EDX,  0, 16, INVLPGB_MAX,                         NO,    0 ) \
+FIELD( 88,  0, EDX, 16,  8, RDPRU_MAX,                           NO,    0 )

 #define CPUID_8A_EDX_11 \
 FLAG(  8A,  0, EDX, 11,  1, SVMEDX_RSVD1,                        NO,    0 )
 #define CPUID_8A_EDX_14 \
 FLAG(  8A,  0, EDX, 14,  1, SVMEDX_RSVD2,                        NO,    0 )
-#define CPUID_8A_EDX_17 \
-FLAG(  8A,  0, EDX, 17,  1, SVMEDX_RSVD3,                        NO,    0 )

 /*    LEVEL, REG, POS, SIZE, NAME,                          MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_8A                                           \
@@ -1045,8 +1118,12 @@ FLAG(  8A,  0, EDX, 13,  1, SVM_AVIC,                            NO,    0 ) \
 CPUID_8A_EDX_14 \
 FLAG(  8A,  0, EDX, 15,  1, SVM_V_VMSAVE_VMLOAD,                 NO,    0 ) \
 FLAG(  8A,  0, EDX, 16,  1, SVM_VGIF,                            NO,    0 ) \
-CPUID_8A_EDX_17 \
-FIELD( 8A,  0, EDX, 18, 14, SVMEDX_RSVD,                         NO,    0 )
+FLAG(  8A,  0, EDX, 17,  1, SVM_GMET,                            YES,  17 ) \
+FIELD( 8A,  0, EDX, 18,  2, SVMEDX_RSVD3,                        NO,    0 ) \
+FLAG(  8A,  0, EDX, 20,  1, SVM_GUEST_SPEC_CTRL,                 NO,    0 ) \
+FIELD( 8A,  0, EDX, 21,  3, SVMEDX_RSVD4,                        NO,    0 ) \
+FLAG(  8A,  0, EDX, 24,  1, SVM_TLB_CTL,                         NO,    0 ) \
+FIELD( 8A,  0, EDX, 25,  7, SVMEDX_RSVD5,                        NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_819                                          \
@@ -1143,14 +1220,27 @@ FLAG( 81F,  0, EAX,  0,  1, SME,                                 NO,    0 ) \
 FLAG( 81F,  0, EAX,  1,  1, SEV,                                 YES,  17 ) \
 FLAG( 81F,  0, EAX,  2,  1, PAGE_FLUSH_MSR,                      NO,    0 ) \
 FLAG( 81F,  0, EAX,  3,  1, SEV_ES,                              YES,  17 ) \
+FLAG( 81F,  0, EAX,  4,  1, SEV_SNP,                             NO,    0 ) \
+FLAG( 81F,  0, EAX,  5,  1, VMPL,                                NO,    0 ) \
+FLAG( 81F,  0, EAX, 10,  1, SEV_HEC,                             NO,    0 ) \
+FLAG( 81F,  0, EAX, 11,  1, SEV_64BIT_REQ,                       NO,    0 ) \
+FLAG( 81F,  0, EAX, 12,  1, SEV_RESTR_INJECTION,                 NO,    0 ) \
+FLAG( 81F,  0, EAX, 13,  1, SEV_ALT_INJECTION,                   NO,    0 ) \
+FLAG( 81F,  0, EAX, 14,  1, SEV_DEBUG_SWAP,                      NO,    0 ) \
+FLAG( 81F,  0, EAX, 15,  1, SEV_NO_HOST_IBS,                     NO,    0 ) \
 FIELD(81F,  0, EBX,  0,  6, SME_PAGE_TABLE_BIT_NUM,              YES,  17 ) \
 FIELD(81F,  0, EBX,  6,  6, SME_PHYS_ADDR_SPACE_REDUCTION,       NO,    0 ) \
+FIELD(81F,  0, EBX, 12,  4, NUM_VMPL,                            NO,    0 ) \
 FIELD(81F,  0, ECX,  0, 32, NUM_ENCRYPTED_GUESTS,                NO,    0 ) \
 FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )

-#define INTEL_CPUID_FIELD_DATA
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_820                                          \
+FLAG( 820,  0, EBX,  1,  1, LEAF820_MBE,                         NO,    0 ) \
+FIELD(820,  1, EAX,  0, 32, CAPACITY_MASK_LEN,                   NO,    0 ) \
+FIELD(820,  1, EDX,  0, 32, NUM_SERVICE_CLASSES,                 NO,    0 )

-#define AMD_CPUID_FIELD_DATA
+#define CPUID_FIELD_DATA_LEVEL_821

 #define CPUID_FIELD_DATA                                              \
   CPUID_FIELD_DATA_LEVEL_0                                           \
@@ -1160,6 +1250,7 @@ FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )
   CPUID_FIELD_DATA_LEVEL_5                                           \
   CPUID_FIELD_DATA_LEVEL_6                                           \
   CPUID_FIELD_DATA_LEVEL_7                                           \
+   CPUID_FIELD_DATA_LEVEL_9                                           \
   CPUID_FIELD_DATA_LEVEL_A                                           \
   CPUID_FIELD_DATA_LEVEL_B                                           \
   CPUID_FIELD_DATA_LEVEL_D                                           \
@@ -1170,6 +1261,10 @@ FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )
   CPUID_FIELD_DATA_LEVEL_15                                          \
   CPUID_FIELD_DATA_LEVEL_16                                          \
   CPUID_FIELD_DATA_LEVEL_17                                          \
+   CPUID_FIELD_DATA_LEVEL_18                                          \
+   CPUID_FIELD_DATA_LEVEL_1A                                          \
+   CPUID_FIELD_DATA_LEVEL_1B                                          \
+   CPUID_FIELD_DATA_LEVEL_1F                                          \
   CPUID_FIELD_DATA_LEVEL_400                                         \
   CPUID_FIELD_DATA_LEVEL_401                                         \
   CPUID_FIELD_DATA_LEVEL_402                                         \
@@ -1195,8 +1290,8 @@ FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )
   CPUID_FIELD_DATA_LEVEL_81D                                         \
   CPUID_FIELD_DATA_LEVEL_81E                                         \
   CPUID_FIELD_DATA_LEVEL_81F                                         \
-   INTEL_CPUID_FIELD_DATA                                             \
-   AMD_CPUID_FIELD_DATA
+   CPUID_FIELD_DATA_LEVEL_820                                         \
+   CPUID_FIELD_DATA_LEVEL_821

 /*
 * Define all field and flag values as an enum.  The result is a full
@@ -1376,6 +1471,7 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_FAMILY_BULLDOZER       0x15  // BD PD SR EX
 #define CPUID_FAMILY_KYOTO           0x16  // Note: Jaguar microarch
 #define CPUID_FAMILY_ZEN             0x17
+#define CPUID_FAMILY_ZEN3            0x19

 /* Effective VIA CPU Families */
 #define CPUID_FAMILY_C7               6
@@ -1429,10 +1525,13 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_MODEL_ATOM_5D        0x5d  // Future Silvermont
 #define CPUID_MODEL_SKYLAKE_5E     0x5e  // Skylake-S / Kaby Lake S/H ES
 #define CPUID_MODEL_ATOM_5F        0x5f  // Denverton
+#define CPUID_MODEL_ATOM_86        0x86  // Snow Ridge
 #define CPUID_MODEL_CANNONLAKE_66  0x66  // Cannon Lake
 #define CPUID_MODEL_KNM_85         0x85  // Knights Mill
 #define CPUID_MODEL_KABYLAKE_8E    0x8e  // Kaby Lake U/Y QS
 #define CPUID_MODEL_KABYLAKE_9E    0x9e  // Kaby Lake S/H QS
+#define CPUID_MODEL_COMETLAKE_A5   0xa5  // Comet Lake S
+#define CPUID_MODEL_COMETLAKE_A6   0xa6  // Comet Lake U

 /* Intel stepping information */
 #define CPUID_STEPPING_KABYLAKE_ES     0x8  // Kaby Lake S/H/U/Y ES
@@ -1441,6 +1540,7 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_STEPPING_CASCADELAKE_A   0x5  // Cascade Lake A-step
 #define CPUID_STEPPING_CASCADELAKE_B1  0x7  // Cascade Lake B1-step
 #define CPUID_STEPPING_WHISKEYLAKE     0xB  // Whiskey Lake U
+#define CPUID_STEPPING_AMBERLAKE       0xC  // Amber Lake Y

 #define CPUID_MODEL_PIII_07    7
 #define CPUID_MODEL_PIII_08    8
@@ -1468,6 +1568,10 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_MODEL_ZEN_1F            0x1F // Max Zen model defined in BKDG
 #define CPUID_MODEL_ZEN2_30           0x30 // family == CPUID_FAMILY_ZEN
 #define CPUID_MODEL_ZEN2_3F           0x3F // Max Zen2 model
+#define CPUID_MODEL_ZEN2_70           0x70 // Ryzen3: family Zen, model Zen2
+#define CPUID_MODEL_ZEN2_7F           0x7F // Ryzen3: max model
+#define CPUID_MODEL_ZEN3_00           0x00 // family == CPUID_FAMILY_ZEN3
+#define CPUID_MODEL_ZEN3_0F           0x0F // Max Zen3 model

 /* AMD stepping information */
 #define CPUID_STEPPING_ZEN_NAPLES_B2  0x02 // Zen Naples ZP-B2
@@ -1716,6 +1820,24 @@ CPUID_MODEL_IS_WHISKEYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
          CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_WHISKEYLAKE;
 }

+static INLINE Bool
+CPUID_MODEL_IS_COMETLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_COMETLAKE_A5 ||
+           CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_COMETLAKE_A6);
+}
+
+static INLINE Bool
+CPUID_MODEL_IS_AMBERLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_8E &&
+          CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_AMBERLAKE;
+}
+
 static INLINE Bool
 CPUID_MODEL_IS_KABYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
 {
@@ -1747,6 +1869,8 @@ CPUID_UARCH_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
          CPUID_MODEL_IS_KABYLAKE(v)    ||
          CPUID_MODEL_IS_COFFEELAKE(v)  ||
          CPUID_MODEL_IS_WHISKEYLAKE(v) ||
+          CPUID_MODEL_IS_COMETLAKE(v)   ||
+          CPUID_MODEL_IS_AMBERLAKE(v)   ||
          CPUID_MODEL_IS_CASCADELAKE(v) ||
          CPUID_MODEL_IS_CANNONLAKE(v);
 }
@@ -1801,6 +1925,21 @@ CPUID_MODEL_IS_DENVERTON(uint32 v) // IN: %eax from CPUID with %eax=1.
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_5F;
 }

+static INLINE Bool
+CPUID_MODEL_IS_SNOWRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_86;
+}
+
+static INLINE Bool
+CPUID_UARCH_IS_TREMONT(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) && CPUID_MODEL_IS_SNOWRIDGE(v);
+}
+
 static INLINE Bool
 CPUID_MODEL_IS_WESTMERE(uint32 v) // IN: %eax from CPUID with %eax=1.
 {
@@ -1938,6 +2077,12 @@ CPUID_FAMILY_IS_ZEN(uint32 eax)
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN;
 }

+static INLINE Bool
+CPUID_FAMILY_IS_ZEN3(uint32 eax)
+{
+   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN3;
+}
+
 /*
 * AMD Barcelona (of either Opteron or Phenom kind).
 */
@@ -2044,8 +2189,10 @@ static INLINE Bool
 CPUID_MODEL_IS_ZEN2(uint32 eax)
 {
  return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN &&
-         (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_ZEN2_30 &&
-          CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN2_3F);
+         ((CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_ZEN2_30 &&
+           CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN2_3F) ||
+          (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_ZEN2_70 &&
+           CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN2_7F));
 }


@@ -2064,14 +2211,18 @@ CPUID_MODEL_IS_DHYANA_A(uint32 eax)
 }


+static INLINE Bool
+CPUID_MODEL_IS_ZEN3(uint32 eax)
+{
+  return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN3 &&
+         CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN3_0F;
+}
+
+
 #define CPUID_TYPE_PRIMARY     0
 #define CPUID_TYPE_OVERDRIVE   1
 #define CPUID_TYPE_SECONDARY   2

-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_NULL      0
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_DATA      1
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_INST      2
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_UNIF      3
 #define CPUID_LEAF4_CACHE_TYPE_NULL      0
 #define CPUID_LEAF4_CACHE_TYPE_DATA      1
 #define CPUID_LEAF4_CACHE_TYPE_INST      2
@@ -2079,17 +2230,15 @@ CPUID_MODEL_IS_DHYANA_A(uint32 eax)
 #define CPUID_LEAF4_CACHE_INDEXING_DIRECT  0
 #define CPUID_LEAF4_CACHE_INDEXING_COMPLEX 1

-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_SELF_INIT      0x00000100
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_FULLY_ASSOC    0x00000200
 #define CPUID_LEAF4_CACHE_SELF_INIT      0x00000100
 #define CPUID_LEAF4_CACHE_FULLY_ASSOC    0x00000200

-#define CPUID_INTEL_IDBECX_LEVEL_TYPE_INVALID   0
-#define CPUID_INTEL_IDBECX_LEVEL_TYPE_SMT       1
-#define CPUID_INTEL_IDBECX_LEVEL_TYPE_CORE      2
 #define CPUID_TOPOLOGY_LEVEL_TYPE_INVALID   0
 #define CPUID_TOPOLOGY_LEVEL_TYPE_SMT       1
 #define CPUID_TOPOLOGY_LEVEL_TYPE_CORE      2
+#define CPUID_TOPOLOGY_LEVEL_TYPE_MODULE    3
+#define CPUID_TOPOLOGY_LEVEL_TYPE_TILE      4
+#define CPUID_TOPOLOGY_LEVEL_TYPE_DIE       5

 #define CPUID_AMD_LEAF85_L1_CACHE_FULLY_ASSOC     0xff
 #define CPUID_AMD_LEAF86_L2_L3_CACHE_FULLY_ASSOC  0x0f
@@ -2103,64 +2252,13 @@ CPUID_MODEL_IS_DHYANA_A(uint32 eax)
 * places to ensure ordering.
 */

-static INLINE Bool
-CPUID_VendorRequiresFence(CpuidVendor vendor)
-{
-   return vendor == CPUID_VENDOR_AMD;
-}
-
-static INLINE Bool
-CPUID_VersionRequiresFence(uint32 version)
-{
-   return CPUID_EFFECTIVE_FAMILY(version) == CPUID_FAMILY_K8 &&
-          CPUID_EFFECTIVE_MODEL(version) < 0x40;
-}
-
-static INLINE Bool
-CPUID_ID0RequiresFence(CPUIDRegs *id0)
-{
-   if (id0->eax == 0) {
-      return FALSE;
-   }
-   return CPUID_IsVendorAMD(id0);
-}
-
-static INLINE Bool
-CPUID_ID1RequiresFence(CPUIDRegs *id1)
-{
-   return CPUID_VersionRequiresFence(id1->eax);
-}
-
 static INLINE Bool
 CPUID_RequiresFence(CpuidVendor vendor, // IN
-                    uint32 version)      // IN: %eax from CPUID with %eax=1.
+                    uint32 version)     // IN: %eax from CPUID with %eax=1.
 {
-   return CPUID_VendorRequiresFence(vendor) &&
-          CPUID_VersionRequiresFence(version);
-}
-
-
-/*
- * The following low-level functions compute the number of
- * cores per cpu.  They should be used cautiously because
- * they do not necessarily work on all types of CPUs.
- * High-level functions that are correct for all CPUs are
- * available elsewhere: see lib/cpuidInfo/cpuidInfo.c.
- */
-
-static INLINE uint32
-CPUID_IntelCoresPerPackage(uint32 v) /* %eax from CPUID with %eax=4 and %ecx=0. */
-{
-   // Note: This is not guaranteed to work on older Intel CPUs.
-   return 1 + CPUID_GET(4, EAX, LEAF4_CORE_COUNT, v);
-}
-
-
-static INLINE uint32
-CPUID_AMDCoresPerPackage(uint32 v) /* %ecx from CPUID with %eax=0x80000008. */
-{
-   // Note: This is not guaranteed to work on older AMD CPUs.
-   return 1 + CPUID_GET(0x80000008, ECX, LEAF88_CORE_COUNT, v);
+   return vendor == CPUID_VENDOR_AMD &&
+          CPUID_EFFECTIVE_FAMILY(version) == CPUID_FAMILY_K8 &&
+          CPUID_EFFECTIVE_MODEL(version) < 0x40;
 }


@@ -2188,8 +2286,8 @@ CPUID_LevelUsesEcx(uint32 level) {
   switch (level)
   {

-#define CPUIDLEVEL(t, s, v, c, h)   \
-      case v:                       \
+#define CPUIDLEVEL(t, s, v, c, h)     \
+      case v:                         \
         return c != 0;

      CPUID_ALL_LEVELS
@@ -2201,37 +2299,6 @@ CPUID_LevelUsesEcx(uint32 level) {
   }
 }

-/*
- *----------------------------------------------------------------------
- *
- * CPUID_IsValid*Subleaf --
- *
- *      Functions to determine the last subleaf for the level specified
- *
- *----------------------------------------------------------------------
- */
-
-static INLINE Bool
-CPUID_IsValidBSubleaf(uint32 ebx)  // IN: %ebx = cpuid.b.sublevel.ebx
-{
-   return ebx != 0;
-}
-
-static INLINE Bool
-CPUID_IsValid4Subleaf(uint32 eax)  // IN: %eax = cpuid.4.sublevel.eax
-{
-   return eax != 0;
-}
-
-static INLINE Bool
-CPUID_IsValid7Subleaf(uint32 eax, uint32 subleaf)  // IN: %eax = cpuid.7.0.eax
-{
-   /*
-    * cpuid.7.0.eax is the max ecx (subleaf) index
-    */
-   return subleaf <= eax;
-}
-
 /*
 *----------------------------------------------------------------------
 *
@@ -2251,8 +2318,9 @@ CPUID_SupportsMsrPlatformInfo(CpuidVendor vendor, uint32 version)
           CPUID_UARCH_IS_HASWELL(version)     ||
           CPUID_UARCH_IS_SKYLAKE(version)     ||
           CPUID_MODEL_IS_KNIGHTS_LANDING(version) ||
-           CPUID_MODEL_IS_DENVERTON(version) ||
-           CPUID_UARCH_IS_SILVERMONT(version));
+           CPUID_MODEL_IS_DENVERTON(version)   ||
+           CPUID_UARCH_IS_SILVERMONT(version)  ||
+           CPUID_UARCH_IS_TREMONT(version));
 }

 #ifdef _MSC_VER
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,46 +39,6 @@
 #include "x86cpuid.h"


-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#ifdef _MSC_VER
-#ifdef __cplusplus
-extern "C" {
-#endif
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-void __cpuid(int regs[4], int eax);
-#pragma intrinsic(__cpuid)
-
-/*
- * __cpuidex has been supported since VS2008
- */
-#if _MSC_VER >= 1500
-void __cpuidex(int regs[4], int eax, int ecx);
-#pragma intrinsic(__cpuidex)
-#endif /* _MSC_VER >= 1500 */
-#endif /* VM_X86_64 */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
-
 #ifdef __GNUC__ // {

 /*
@@ -89,34 +49,16 @@ void __cpuidex(int regs[4], int eax, int ecx);
 *
 */

-/*
- * %ebx is reserved on i386 PIC.  Apple's gcc-5493 (gcc 4.0) compiling
- * for x86_64 incorrectly errors out saying %ebx is reserved.  This is
- * Apple bug 7304232.
- */
-#if vm_x86_64 ? (defined __APPLE_CC__ && __APPLE_CC__ == 5493) : defined __PIC__
-#if vm_x86_64
-/*
- * Note that this generates movq %rbx,%rbx; cpuid; xchgq %rbx,%rbx ...
- * Unfortunately Apple's assembler does not have .ifnes, and I cannot
- * figure out how to do that with .if.   If we ever enable this code
- * on other 64bit systems, both movq & xchgq should be surrounded by
- * .ifnes \"%%rbx\", \"%q1\" & .endif
- */
-#define VM_CPUID_BLOCK  "movq %%rbx, %q1\n\t" \
-                        "cpuid\n\t"           \
-                        "xchgq %%rbx, %q1\n\t"
-#define VM_EBX_OUT(reg) "=&r"(reg)
+#if defined VM_X86_64 || !defined __PIC__
+#define VM_CPUID_BLOCK  "cpuid"
+#define VM_EBX_OUT(reg) "=b"(reg)
 #else
+/* %ebx is reserved on i386 PIC. */
 #define VM_CPUID_BLOCK  "movl %%ebx, %1\n\t" \
                        "cpuid\n\t"          \
                        "xchgl %%ebx, %1\n\t"
 #define VM_EBX_OUT(reg) "=&rm"(reg)
 #endif
-#else
-#define VM_CPUID_BLOCK  "cpuid"
-#define VM_EBX_OUT(reg) "=b"(reg)
-#endif

 static INLINE void
 __GET_CPUID(uint32     eax,  // IN
@@ -217,22 +159,6 @@ __GET_EDX_FROM_CPUID(uint32 eax) // IN
 }


-static INLINE uint32
-__GET_EAX_FROM_CPUID4(uint32 ecx) // IN
-{
-   uint32 eax;
-   uint32 ebx;
-
-   __asm__ __volatile__(
-      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
-      : "a" (4), "c" (ecx)
-      : "memory", "%edx"
-   );
-
-   return eax;
-}
-
 #undef VM_CPUID_BLOCK
 #undef VM_EBX_OUT

@@ -241,247 +167,51 @@ __GET_EAX_FROM_CPUID4(uint32 ecx) // IN
 static INLINE void
 __GET_CPUID(uint32 input, CPUIDRegs *regs)
 {
-#ifdef VM_X86_64
   __cpuid((int *)regs, input);
-#else
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-#endif
 }

-#ifdef VM_X86_64
-
-#if _MSC_VER >= 1500
-
-/*
- * __cpuidex has been supported since VS2008
- */
-
 static INLINE void
 __GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __cpuidex((int *)regs, inputEax, inputEcx);
 }

-#else // _MSC_VER >= 1500
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern void
-__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs);
-#endif // _MSC_VER >= 1500
-
-#else // VM_X86_64
-
-static INLINE void
-__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
-{
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, inputEax
-   __asm mov  ecx, inputEcx
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-}
-#endif
-
 static INLINE uint32
 __GET_EAX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.eax;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
 __GET_EBX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ebx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ebx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
 __GET_ECX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ecx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ecx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
 __GET_EDX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.edx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

-#ifdef VM_X86_64
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern uint32
-__GET_EAX_FROM_CPUID4(uint32 inputEcx);
-
-#else // VM_X86_64
-
-static INLINE uint32
-__GET_EAX_FROM_CPUID4(uint32 inputEcx)
-{
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, 4
-   __asm mov  ecx, inputEcx
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-}
-
-#endif // VM_X86_64
-
 #else // }
 #error
 #endif

 #define CPUID_FOR_SIDE_EFFECTS() ((void)__GET_EAX_FROM_CPUID(0))

-/* The first parameter is used as an rvalue and then as an lvalue. */
-#define GET_CPUID(_ax, _bx, _cx, _dx) { \
-   CPUIDRegs regs;                      \
-   __GET_CPUID(_ax, &regs);             \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
-#define GET_CPUID2(_ax, _bx, _cx, _dx) {\
-   CPUIDRegs regs;                      \
-   __GET_CPUID2(_ax, _cx, &regs);       \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,7 +24,6 @@

 #ifndef _X86MSR_H_
 #define _X86MSR_H_
-#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX

@@ -114,6 +113,20 @@ MSRQuery;
 #define MSR_TSC_AUX           0xc0000103
 #define MSR_BD_TSC_RATIO      0xc0000104

+/* CET MSRs */
+#define MSR_U_CET                            0x6a0
+#define MSR_S_CET                            0x6a2
+#define MSR_CET_SH_STK_EN                         (1ULL << 0)
+#define MSR_CET_WR_SHSTK_EN                       (1ULL << 1)
+#define MSR_PL0_SSP                          0x6a4
+#define MSR_PL1_SSP                          0x6a5
+#define MSR_PL2_SSP                          0x6a6
+#define MSR_PL3_SSP                          0x6a7
+#define MSR_ISST_ADDR                        0x6a8
+
+#define MSR_TEST_CTRL                        0x33
+#define MSR_TEST_CTRL_SPLIT_LOCK_DETECT           (1ULL << 29)
+
 #define IA32_MSR_ARCH_CAPABILITIES           0x10a
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
@@ -121,6 +134,9 @@ MSRQuery;
 #define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
 #define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)
 #define MSR_ARCH_CAPABILITIES_MDS_NO              (1ULL << 5)
+#define MSR_ARCH_CAPABILITIES_IF_PSCHANGE_MC_NO   (1ULL << 6)
+#define MSR_ARCH_CAPABILITIES_TSX_CTRL            (1ULL << 7)
+#define MSR_ARCH_CAPABILITIES_TAA_NO              (1ULL << 8)

 #define MSR_FLUSH_CMD                        0x10b
 #define MSR_FLUSH_CMD_FLUSH_L1D                   (1ULL << 0)
@@ -131,10 +147,11 @@ MSRQuery;

 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

+#define MSR_TSX_CTRL                         0x122
+#define MSR_TSX_CTRL_RTM_DISABLE                  (1ULL << 0)
+#define MSR_TSX_CTRL_CPUID_CLEAR                  (1ULL << 1)

-#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
-#endif

 /* Intel Core Architecture and later: use only architected counters. */
 #define IA32_MSR_PERF_CAPABILITIES                   0x345
@@ -151,14 +168,6 @@ MSRQuery;

 #define IA32_MSR_PEBS_ENABLE                      0x3f1

-typedef enum {
-   SL_PMC_FLAGS_NONE             = 0x00, /* No flags.                      */
-   SL_PMC_FLAGS_LBR_VA32         = 0x01, /* LBR format: 32-bit VA.         */
-   SL_PMC_FLAGS_LBR_LA64         = 0x02, /* LBR format: 64-bit LA.         */
-   SL_PMC_FLAGS_LBR_VA64         = 0x04, /* LBR format: 64-bit VA.         */
-   SL_PMC_FLAGS_LBR_PACKED_VA32  = 0x08, /* LBR format: 2x32-bit VAs.      */
-} StateLoggerPMCFlags;
-
 #define MSR_MTRR_BASE0        0x00000200
 #define MSR_MTRR_MASK0        0x00000201
 #define MSR_MTRR_BASE1        0x00000202
@@ -455,7 +464,6 @@ typedef enum {

 /* MSR_EFER bits. */
 #define MSR_EFER_SCE         0x0000000000000001ULL  /* Sys call ext'ns:  r/w */
-#define MSR_EFER_RAZ         0x00000000000000feULL  /* Read as zero          */
 #define MSR_EFER_LME         0x0000000000000100ULL  /* Long mode enable: r/w */
 #define MSR_EFER_LMA         0x0000000000000400ULL  /* Long mode active: r/o */
 #define MSR_EFER_NXE         0x0000000000000800ULL  /* No-exec enable:   r/w */
@@ -463,7 +471,11 @@ typedef enum {
 #define MSR_EFER_LMSLE       0x0000000000002000ULL  /* LM seg lim enable:r/w */
 #define MSR_EFER_FFXSR       0x0000000000004000ULL  /* Fast FXSAVE:      r/w */
 #define MSR_EFER_TCE         0x0000000000008000ULL  /* Trans. cache ext. r/w */
-#define MSR_EFER_MBZ         0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+/* Vendor specific EFER bits */
+#define MSR_EFER_INTEL_MBZ   0xffffffffffff02feULL  /* Must be zero (resrvd) */
+#define MSR_EFER_INTEL_RAZ   0x0000000000000000ULL  /* Read as zero          */
+#define MSR_EFER_AMD_MBZ     0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+#define MSR_EFER_AMD_RAZ     0x00000000000000feULL  /* Read as zero          */

 #define MSR_AMD_PATCH_LOADER 0xc0010020

@@ -481,6 +493,10 @@ typedef enum {
 #endif
 #define MSR_K8_SYSCFG_MTRRTOM2EN         (1ULL<<21)
 #define MSR_K8_SYSCFG_TOM2FORCEMEMTYPEWB (1ULL<<22)
+#define MSR_K8_SYSCFG_SMEE               (1ULL<<23)
+#define MSR_K8_SYSCFG_SNPE               (1ULL<<24)
+#define MSR_K8_SYSCFG_VMPLE              (1ULL<<25)
+
 #define MSR_K8_TOPMEM2       0xc001001d

 /* AMD "Greyhound" MSRs */
@@ -521,13 +537,22 @@ typedef enum {
 /* SEV related MSRs. */
 #define MSR_VMPAGE_FLUSH           0xc001011e
 #define MSR_GHCB_PA                0xc0010130
+#define MSR_GHCB_PA_FUNCTION_MASK       0xfff
 #define MSR_GHCB_PA_SEVINFO_HV              1
 #define MSR_GHCB_PA_SEVINFO_REQ             2
 #define MSR_GHCB_PA_AP_JUMP_TABLE           3
+#define MSR_GHCB_PA_CPUID_REQ               4
+#define MSR_GHCB_PA_CPUID_RESP              5
+#define MSR_GHCB_PA_TERMINATE             256
 #define MSR_SEV_STATUS             0xc0010131

 #define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
 #define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
+#define MSR_SEV_STATUS_SEV_SNP_EN  0x0000000000000004ULL // SEV-SNP is enabled
+
+/* SEV-SNP (Secure Nested Paging) MSRs. */
+#define MSR_RMP_BASE              0xc0010132 // Address of first byte of RMP
+#define MSR_RMP_END               0xc0010133 // Address of last byte of RMP

 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)
@@ -687,11 +712,7 @@ typedef unsigned char MTRRType;
 /*
 * MISC_FEATURES_ENABLES bits
 */
-#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
-#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
-#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
-#endif



@@ -800,7 +821,6 @@ X86MSR_SetMSR(uint32 cx, uint64 value)
 }
 #endif
 #elif defined _MSC_VER // !__GNUC__ && _MSC_VER
-#ifdef _WIN64
 unsigned __int64  __readmsr(unsigned long);
 void              __writemsr(unsigned long, unsigned __int64);
 #pragma intrinsic(__readmsr, __writemsr)
@@ -815,37 +835,6 @@ X86MSR_SetMSR(uint32 cx, uint64 value)
 {
   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
 }
-#else // !__GNUC__ && _MSC_VER && !_WIN64
-#pragma warning( disable : 4035)
-static INLINE uint64
-X86MSR_GetMSR(uint32 input)
-{
-   __asm push ecx
-   __asm mov  ecx, input
-   /* 0x0f 0x32 -> edx:eax = rdmsr[ecx] */
-   __asm _emit 0x0f __asm _emit 0x32
-   __asm pop ecx
-}
-
-static INLINE void
-X86MSR_SetMSR(uint32 input, uint64 value)
-{
-      uint32 hival = (uint32)((value) >> 32);
-      uint32 loval = (uint32)value;
-      __asm push edx
-      __asm push ecx
-      __asm push eax
-      __asm mov  eax, loval
-      __asm mov  edx, hival
-      __asm mov  ecx, input
-      /* 0x0f 0x30 -> wrmsr[ecx] = edx:eax */
-      __asm _emit 0x0f __asm _emit 0x30
-      __asm pop  eax
-      __asm pop  ecx
-      __asm pop  edx
-}
-#pragma warning (default: 4035)
-#endif
 #else // !__GNUC__ && !_MSC_VER
 #error No compiler defined for RDMSR/WRMSR.
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,6 +40,7 @@
 #include "x86types.h"
 #include "vm_pagetable.h"
 #include "x86paging_common.h"
+#include "vm_assert.h"

 #define LM_PTE_PFN_MASK      CONST64U(0xffffffffff000)
 #define LM_PTE_2_PFN(_pte)   (((_pte) & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
@@ -136,4 +137,95 @@ LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
 #define VA64_CANONICAL_HOLE_START   (CONST64U(1) << (VA64_IMPL_BITS - 1))
 #define VA64_CANONICAL_HOLE_LEN  VA64_CANONICAL_MASK - VA64_CANONICAL_HOLE_START

+static INLINE Bool
+x86IsCanonicalC(VA64 va)
+{
+   return (va & VA64_CANONICAL_MASK) == 0 ||
+          (va & VA64_CANONICAL_MASK) == VA64_CANONICAL_MASK;
+}
+
+#if defined(VM_X86_64) && defined(__GNUC__)
+static INLINE Bool
+x86IsCanonicalAsm(VA64 va)
+{
+   Bool out;
+   /*
+    * sarq $48, %0: Move 17 bits from position 63:47 into 15:0 and CF,
+    *  sign-extending as we do so.
+    * adcl $0, %0: Then add (0 + CF) to the shifted value.  The result is zero
+    *  iff (CF == 1 && bits 31:0 == 0xffffffff) or (CF == 0 && bits 31:0 == 0).
+    *  That is, if original bits 47 and higher were all 1s or all 0s.
+    * setz %1: Create the boolean.
+    *
+    * The chain of AdcLong macros will emit one adc instruction; gcc
+    * has no obvious way to force the size of a quad %0 to a long.
+    */
+#define AdcLong(reg64, reg32) \
+   ".ifc  %0, %%" #reg64 "\n" \
+   " adcl $0, %%" #reg32 "\n" \
+   ".endif \n"
+
+   asm ("sarq $48, %0 \n"
+        AdcLong(rax, eax)
+        AdcLong(rcx, ecx)
+        AdcLong(rdx, edx)
+        AdcLong(rbx, ebx)
+        AdcLong(rsp, esp)
+        AdcLong(rbp, ebp)
+        AdcLong(rsi, esi)
+        AdcLong(rdi, edi)
+        AdcLong(r8,  r8d)
+        AdcLong(r9,  r9d)
+        AdcLong(r10, r10d)
+        AdcLong(r11, r11d)
+        AdcLong(r12, r12d)
+        AdcLong(r13, r13d)
+        AdcLong(r14, r14d)
+        AdcLong(r15, r15d)
+        "setz %1" : "+r"(va), "=r"(out));
+   return out;
+#undef AdcLong
+}
+#endif
+
+static INLINE Bool
+x86_IsCanonical(VA64 va)
+{
+#if defined(VM_X86_64) && defined(__GNUC__)
+   if (__builtin_constant_p(va)) {
+      return x86IsCanonicalC(va);
+   } else {
+      return x86IsCanonicalAsm(va);
+   }
+#else
+   return x86IsCanonicalC(va);
+#endif
+}
+
+static INLINE Bool
+x86_IsCanonicalRange(VA64 va, unsigned size)
+{
+   /*
+    * The check is simple as long as the size is less
+    * than the number of implemented bits.
+    *
+    * The only case we don't handle is one where the VA goes from a
+    * high canonical address and wraps to a non-canonical address
+    * (e.g. 0x00008000_00000000) or higher.  Our test would falsely
+    * consider this canonical.
+    */
+   ASSERT_ON_COMPILE(sizeof(size) * 8 < VA64_IMPL_BITS);
+
+   /*
+    * VA64_CANONICAL_MASK is the lowest canonical address with the
+    * upper bits all set.
+    *
+    * VA64_CANONICAL_HOLE_START is one higher than the highest valid
+    * canonical address with the upper bits all cleared.  Note that we
+    * access up to (va + size - 1), not (va + size), so <= is correct.
+    */
+   return va >= VA64_CANONICAL_MASK ||
+          va + size <= VA64_CANONICAL_HOLE_START;
+}
+
 #endif /* _X86PAGING_64_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,7 @@
 #define PF_RSVD         0x0008
 #define PF_ID           0x0010
 #define PF_PK           0x0020
+#define PF_SS           0x0040
 #define PF_SGX          0x8000

 /*
@@ -109,4 +110,12 @@ typedef struct {
   uint64   la;      /* Used for single address invalidation.  */
 } InvpcidArg;

+typedef enum {
+   PM_FLAT,        /* non paging mode                                      */
+   PM_2_LEVEL,     /* classic 32 bit non-pae style paging mode.            */
+   PM_3_LEVEL,     /* classic 32 bit pae style paging mode.                */
+   PM_4_LEVEL,     /* long mode style paging mode (including compat mode). */
+   PM_NUM     = 4, /* number of different paging modes.                    */
+} PagingMode;
+
 #endif /* _X86PAGING_COMMON_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,6 +39,7 @@
 extern "C" {
 #endif

+#include "vm_assert.h"

 #define SELECTOR_GDT             0
 #define SELECTOR_LDT             1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -141,6 +141,7 @@
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)
 #define SVM_VMCB_NPTCTL_SEV_ENABLE         (1 << 1)
 #define SVM_VMCB_NPTCTL_SEV_ES_ENABLE      (1 << 2)
+#define SVM_VMCB_NPTCTL_GMET_ENABLE        (1 << 3)

 /* VMCB.virtExt */
 #define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
@@ -251,6 +252,7 @@
 #define SVM_EXITCODE_MMIO_READ           0x80000001   // SW only
 #define SVM_EXITCODE_MMIO_WRITE          0x80000002   // SW only
 #define SVM_EXITCODE_NMI_COMPLETE        0x80000003   // SW only
+#define SVM_EXITCODE_AP_JUMP_TABLE       0x80000005   // SW only
 #define SVM_EXITCODE_UNSUPPORTED         0x8000FFFF   // SW only
 #define SVM_EXITCODE_INVALID             (-1ULL)

@@ -313,6 +315,10 @@
 #define SVM_NPFEXIT_FINAL_ADDR     (1ULL << 32)    /* Rev. C */
 #define SVM_NPFEXIT_PTE_ACCESS     (1ULL << 33)    /* Rev. C */

+/* ExitInfo1 for AP jump table exits */
+#define SVM_APEXIT_SET            0x0
+#define SVM_APEXIT_GET            0x1
+
 /* Event Injection */
 #define SVM_INTINFO_VECTOR_MASK   0x000000ff
 #define SVM_INTINFO_TYPE_SHIFT    8
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -95,9 +95,10 @@
 *       _V  -- virtualized
 *       _NV -- not virtualized
 *   _access is one of:
- *       _NA -- no access
- *       _RW -- read/write access
- *       _RO -- read-only access
+ *       _NA  -- no access
+ *       _RW  -- read/write access
+ *       _RO  -- read-only access
+ *       _URW -- read/write acess for the ULM only
 *   _sticky is one of:
 *       _S  -- sticky
 *       _NS -- non-sticky
@@ -109,9 +110,9 @@
 VMCS_SET_START(16)
 /* 16-bit control fields. */
 VMCS_GROUP_START(16, CTL)
-VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA,  _S)
-VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA, _NS)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV,  _NA,  _S)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _URW, _NS)
 VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
 VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
 VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
@@ -328,16 +329,16 @@ VMCS_GROUP_END(64, DATA)

 /* 64-bit guest state. */
 VMCS_GROUP_START(64, GUEST)
-VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA, _NS)
-VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA, _NS)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _URW, _NS)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V,  _NA, _NS)
 VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
 VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
 VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
@@ -403,24 +404,24 @@ VMCS_SET_END(64)
 VMCS_SET_START(32)
 /* 32-bit control fields. */
 VMCS_GROUP_START(32, CTL)
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA, _NS)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA,  _S)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA,  _S)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _URW,  _S)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _URW, _NS)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _URW,  _S)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V,  _NA,  _S)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
 VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
 VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
@@ -551,14 +552,14 @@ VMCS_SET_END(32)
 VMCS_SET_START(NAT)
 /* natural-width control fields. */
 VMCS_GROUP_START(NAT, CTL)
-VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
 VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
 VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
@@ -622,26 +623,26 @@ VMCS_GROUP_END(NAT, DATA)

 /* natural-width guest state. */
 VMCS_GROUP_START(NAT, GUEST)
-VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW, _NS)
-VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW, _NS)
-VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW, _NS)
-VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA, _NS)
-VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _URW, _NS)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _URW, _NS)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V,  _RW, _NS)
 VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
 VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
 VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
@@ -658,18 +659,18 @@ VMCS_GROUP_END(NAT, GUEST)

 /* natural-width host state. */
 VMCS_GROUP_START(NAT, HOST)
-VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
 VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
 VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2004-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2004-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -173,8 +173,11 @@ enum {
 /*
 * Sizes of referenced fields
 */
-#define VT_VMCS_IO_BITMAP_SIZE    (2 * PAGE_SIZE)
-#define VT_VMCS_MSR_BITMAP_SIZE   (1 * PAGE_SIZE)
+#define VT_VMCS_IO_BITMAP_PAGES   (2)
+#define VT_VMCS_IO_BITMAP_SIZE    PAGES_2_BYTES(VT_VMCS_IO_BITMAP_PAGES)
+#define VT_VMCS_MSR_BITMAP_PAGES  (1)
+#define VT_VMCS_MSR_BITMAP_SIZE   PAGES_2_BYTES(VT_VMCS_MSR_BITMAP_PAGES)
+

 /*
 * Execution control bit capabilities come in pairs: a "required" bit in
@@ -552,7 +555,7 @@ enum {
 #define VT_VMCS_PENDDBG_MBZ        0xfffeaff0

 /* Exception error must-be-zero bits for VMEntry */
-#define VT_XCP_ERR_MBZ             0xffff8000
+#define VT_XCP_ERR_MBZ             0xffff0000


 /* Exit reasons. */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -23,6 +23,7 @@

 #include <linux/file.h>
 #include <linux/highmem.h>
+#include <linux/miscdevice.h>
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/slab.h>
@@ -36,8 +37,8 @@

 #include "usercalldefs.h"

-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
-#error Linux kernels before 2.6.32 are not supported
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0)
+#error Linux kernels before 3.10 are not supported
 #endif

 #include <asm/io.h>
@@ -68,8 +69,6 @@ static void LinuxDriverQueue(Device *device);
 static void LinuxDriverDequeue(Device *device);
 static Bool LinuxDriverCheckPadding(void);

-#define VMMON_UNKNOWN_SWAP_SIZE -1ULL
-
 struct VMXLinuxState linuxState;


@@ -95,22 +94,25 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd,
                       unsigned long ioarg);

 static int LinuxDriver_Close(struct inode *inode, struct file *filp);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
-static vm_fault_t LinuxDriverFault(struct vm_fault *fault);
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-static int LinuxDriverFault(struct vm_fault *fault);
-#else
-static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault);
-#endif
-static int LinuxDriverMmap(struct file *filp, struct vm_area_struct *vma);

 static unsigned int LinuxDriverEstimateTSCkHz(void);

-static struct vm_operations_struct vmuser_mops = {
-        .fault  = LinuxDriverFault
+static const struct file_operations vmuser_fops = {
+   .owner = THIS_MODULE,
+   .open = LinuxDriver_Open,
+   .release = LinuxDriver_Close,
+   .unlocked_ioctl = LinuxDriver_Ioctl,
+   .compat_ioctl = LinuxDriver_Ioctl,
 };

-static struct file_operations vmuser_fops;
+#ifndef VMX86_DEVEL
+static struct miscdevice vmmon_miscdev = {
+   .name = "vmmon",
+   .minor = MISC_DYNAMIC_MINOR,
+   .fops = &vmuser_fops,
+};
+#endif
+
 static struct timer_list tscTimer;
 static Atomic_uint32 tsckHz;
 static VmTimeStart tsckHzStartTime;
@@ -265,9 +267,8 @@ LinuxDriverInitTSCkHz(void)
 *      linux module entry point. Called by /sbin/insmod command
 *
 * Results:
- *      registers a device driver for a major # that depends
- *      on the uid. Add yourself to that list.  List is now in
- *      private/driver-private.c.
+ *      Release: registers a device driver with a misc minor node.
+ *      Devel: registers for a major number with user-created node.
 *
 *----------------------------------------------------------------------
 */
@@ -291,48 +292,46 @@ init_module(void)
      return -ENOEXEC;
   }

+   if (!Vmx86_CreateHVIOBitmap()) {
+      return -ENOMEM;
+   }
+
+   if (!Vmx86_CheckMSRUniformity()) {
+      return -EPERM;
+   }
+
   linuxState.fastClockThread = NULL;
   linuxState.fastClockRate = 0;
-   linuxState.swapSize = VMMON_UNKNOWN_SWAP_SIZE;
-
-   /*
-    * Initialize the file_operations structure. Because this code is always
-    * compiled as a module, this is fine to do it here and not in a static
-    * initializer.
-    */
-
-   memset(&vmuser_fops, 0, sizeof vmuser_fops);
-   vmuser_fops.owner = THIS_MODULE;
-   vmuser_fops.unlocked_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.compat_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.open = LinuxDriver_Open;
-   vmuser_fops.release = LinuxDriver_Close;
-   vmuser_fops.mmap = LinuxDriverMmap;

 #ifdef VMX86_DEVEL
   devel_init_module();
-   linuxState.minor = 0;
   retval = register_chrdev(linuxState.major, linuxState.deviceName,
                            &vmuser_fops);
+   if (retval) {
+      Warning("Module %s: error registering with major=%d\n",
+              linuxState.deviceName, linuxState.major);
+   } else {
+      Log("Module %s: registered with major=%d\n",
+          linuxState.deviceName, linuxState.major);
+   }
 #else
   sprintf(linuxState.deviceName, "vmmon");
   linuxState.major = 10;
-   linuxState.minor = 165;
-   linuxState.misc.minor = linuxState.minor;
-   linuxState.misc.name = linuxState.deviceName;
-   linuxState.misc.fops = &vmuser_fops;

-   retval = misc_register(&linuxState.misc);
+   retval = misc_register(&vmmon_miscdev);
+   if (retval) {
+      Warning("Module %s: error registering misc device %s\n",
+              linuxState.deviceName, vmmon_miscdev.name);
+   } else {
+      Log("Module %s: registered as misc device %s\n", linuxState.deviceName,
+          vmmon_miscdev.name);
+   }
 #endif

   if (retval) {
-      Warning("Module %s: error registering with major=%d minor=%d\n",
-              linuxState.deviceName, linuxState.major, linuxState.minor);
-
+      Vmx86_CleanupHVIOBitmap();
      return -ENOENT;
   }
-   Log("Module %s: registered with major=%d minor=%d\n",
-       linuxState.deviceName, linuxState.major, linuxState.minor);

   HostIF_InitUptime();
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
@@ -370,13 +369,14 @@ cleanup_module(void)
 #ifdef VMX86_DEVEL
   unregister_chrdev(linuxState.major, linuxState.deviceName);
 #else
-   misc_deregister(&linuxState.misc);
+   misc_deregister(&vmmon_miscdev);
 #endif

   Log("Module %s: unloaded\n", linuxState.deviceName);

   del_timer_sync(&tscTimer);

+   Vmx86_CleanupHVIOBitmap();
   Task_Terminate();
   // Make sure fastClockThread is dead
   HostIF_FastClockLock(1);
@@ -414,7 +414,6 @@ LinuxDriver_Open(struct inode *inode, // IN
   }
   memset(device, 0, sizeof *device);

-   sema_init(&device->lock4Gb, 1);
   init_rwsem(&device->vmDriverRWSema);

   filp->private_data = device;
@@ -426,104 +425,6 @@ LinuxDriver_Open(struct inode *inode, // IN
 }


-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocPages --
- *
- *    Allocate physically contiguous block of memory with specified order.
- *    Pages in the allocated block are configured so that caller can pass
- *    independent pages to the VM.
- *
- * Results:
- *    Zero on success, non-zero (error code) on failure.
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverAllocPages(unsigned int order,   // IN
-                      struct page **pg,     // OUT
-                      PageCnt size)         // IN
-{
-   struct page* page = alloc_pages(GFP_HIGHUSER, order);
-   if (page) {
-      /*
-       * Grab an extra reference on all pages except first one - first
-       * one was already refcounted by alloc_pages.
-       *
-       * Under normal situation all pages except first one in the block
-       * have refcount zero.  As we pass these pages to the VM, we must
-       * bump their count, otherwise VM will release these pages every
-       * time they would be unmapped from user's process, causing crash.
-       *
-       * Note that this depends on Linux VM internals.  It works on all
-       * kernels we care about.
-       */
-      PageCnt i, orderToNumPages = 1 << order;
-      for (i = 0; i < orderToNumPages; i++) {
-         if (i) {
-            /*
-             * Debug kernels assert that page->_count is not zero when
-             * calling get_page. We use init_page_count as a temporary
-             * workaround. PR 894174
-             */
-#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 16)
-            ASSERT(page_count(page) == 0);
-            init_page_count(page);
-#else
-            get_page(page);
-#endif
-         }
-         if (i >= size) {
-            put_page(page);
-         } else {
-            void *addr = kmap(page);
-            memset(addr, 0, PAGE_SIZE);
-            kunmap(page);
-            *pg++ = page;
-         }
-         page++;
-      }
-      return 0;
-   }
-   return -ENOMEM;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverDestructor4Gb --
- *
- *    Deallocate all directly mappable memory.
- *
- * Results:
- *    None
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static void
-LinuxDriverDestructor4Gb(Device *device) // IN
-{
-   PageCnt pg;
-   if (!device->size4Gb) {
-      return;
-   }
-   for (pg = 0; pg < device->size4Gb; pg++) {
-      put_page(device->pages4Gb[pg]);
-   }
-   device->size4Gb = 0;
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -555,194 +456,12 @@ LinuxDriver_Close(struct inode *inode, // IN

   Vmx86_Close();

-   /*
-    * Destroy all low memory allocations.
-    * We are closing the struct file here, so clearly no other process
-    * uses it anymore, and we do not need to hold the semaphore.
-    */
-
-   LinuxDriverDestructor4Gb(device);
-
   kfree(device);
   filp->private_data = NULL;

   return 0;
 }

-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverFault --
- *
- *      Callback for returning allocated page for memory mapping
- *
- * Results:
- *    NoPage:
- *      Page or page address on success, NULL or 0 on failure.
- *    Fault:
- *      Error code; 0, minor page fault.
- *
- * Side effects:
- *      None.
- *
- *-----------------------------------------------------------------------------
- */
-
-static
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
-vm_fault_t
-#else
-int
-#endif
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-LinuxDriverFault(struct vm_fault *fault)     //IN/OUT
-#else
-LinuxDriverFault(struct vm_area_struct *vma, //IN
-                 struct vm_fault *fault)     //IN/OUT
-#endif
-{
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-   struct vm_area_struct *vma = fault->vma;
-#endif
-   Device *device = (Device *) vma->vm_file->private_data;
-   PageNum pg;
-   struct page* page;
-
-   pg = fault->pgoff;
-   pg = VMMON_MAP_OFFSET(pg);
-   if (pg >= device->size4Gb) {
-      return VM_FAULT_SIGBUS;
-   }
-   page = device->pages4Gb[pg];
-   get_page(page);
-   fault->page = page;
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocContig --
- *
- *      Create mapping for contiguous memory areas.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int LinuxDriverAllocContig(Device *device,
-                                  struct vm_area_struct *vma,
-                                  unsigned long off,
-                                  PageCnt size)
-{
-   unsigned long vmaOrder = VMMON_MAP_ORDER(off);
-   PageCnt vmaAllocSize, i;
-
-   if (VMMON_MAP_RSVD(off)) {
-      /* Reserved bits set... */
-      return -EINVAL;
-   }
-   if (VMMON_MAP_OFFSET(off)) {
-      /* We do not need non-zero offsets... */
-      return -EINVAL;
-   }
-   if (size > VMMON_MAP_OFFSET_MASK + 1) {
-      /* Size is too big to fit to our window. */
-      return -ENOMEM;
-   }
-
-   /* 16 pages looks like a good limit... */
-   if (size > VMMON_MAX_LOWMEM_PAGES) {
-      return -ENOMEM;
-   }
-   /* Sorry. Only one mmap per one open. */
-   down(&device->lock4Gb);
-   if (device->size4Gb) {
-      up(&device->lock4Gb);
-      return -EINVAL;
-   }
-   vmaAllocSize = 1 << vmaOrder;
-   for (i = 0; i < size; i += vmaAllocSize) {
-      int err = LinuxDriverAllocPages(vmaOrder, device->pages4Gb + i, size - i);
-      if (err) {
-         while (i > 0) {
-            put_page(device->pages4Gb[--i]);
-         }
-         up(&device->lock4Gb);
-         return err;
-      }
-   }
-   device->size4Gb = size;
-   up(&device->lock4Gb);
-   vma->vm_ops = &vmuser_mops;
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverMmap --
- *
- *      Create mapping for lowmem or locked memory.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverMmap(struct file *filp,
-                struct vm_area_struct *vma)
-{
-   Device *device = (Device *) filp->private_data;
-   PageCnt size;
-   int err;
-
-   /* Only shared mappings */
-   if (!(vma->vm_flags & VM_SHARED)) {
-      return -EINVAL;
-   }
-   if ((vma->vm_end | vma->vm_start) & (PAGE_SIZE - 1)) {
-      return -EINVAL;
-   }
-   size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
-   if (size < 1) {
-      return -EINVAL;
-   }
-   down_read(&device->vmDriverRWSema);
-   if (device->vm) {
-      err = -EINVAL;
-   } else {
-      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
-   }
-   up_read(&device->vmDriverRWSema);
-   if (err) {
-      return err;
-   }
-   /* Clear VM_IO, otherwise SuSE's kernels refuse to do get_user_pages */
-   vma->vm_flags &= ~VM_IO;
-   return 0;
-}
-
-
 typedef Bool (*SyncFunc)(void *data, unsigned cpu);

 typedef struct {
@@ -1080,6 +799,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      } else {
         device->vm = vm;
         args.vmid = vm->userID;
+         vm->vmhost->vmmonData = (void *__user)args.vmmonData;
         retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args);
      }
      break;
@@ -1328,25 +1048,12 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

   case IOCTL_VMX86_APIC_INIT: {
-      VMAPICInfo info;
-      Bool setVMPtr;
-      Bool probe;
-
-      retval = HostIF_CopyFromUser(&info, ioarg, sizeof info);
-      if (retval != 0) {
-         break;
-      }
-      setVMPtr = ((info.flags & APIC_FLAG_DISABLE_NMI) != 0);
-      probe = ((info.flags & APIC_FLAG_PROBE) != 0);
-
      /*
-       * Kernel uses NMIs for deadlock detection - set APIC VMptr so that
-       * NMIs get disabled in the monitor.
+       * Kernel uses NMIs for deadlock detection - so we always have to find
+       * APIC so that NMIs get disabled in the monitor.
       */
-
-      setVMPtr = TRUE;
-
-      retval = HostIF_APICInit(vm, setVMPtr, probe) ? 0 : -ENODEV;
+      HostIF_APICInit(vm);
+      retval = 0;
      break;
   }

@@ -1386,6 +1093,23 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_SWITCH_ERROR_ADDR: {
+      VMSwitchErrorArgs args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->crosspage == NULL ||
+          vm->crosspage[args.vcpuid] == NULL) {
+         retval = -EINVAL;
+         break;
+      }
+      args.addr = vm->crosspage[args.vcpuid]->wsUD2;
+      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      break;
+   }
+
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
      retval = LinuxDriverEstimateTSCkHz();
      break;
@@ -1533,7 +1257,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
-#ifdef VMX86_DEVEL
+#if VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1599,17 +1323,6 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

-   case IOCTL_VMX86_SET_HOST_SWAP_SIZE: {
-      uint64 swapSize;
-      retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize);
-      if (retval != 0) {
-         Warning("Could not copy swap size from user, status %ld\n", retval);
-         break;
-      }
-      linuxState.swapSize = swapSize;
-      break;
-   }
-
   case IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS: {
      uint64 ctrs = Vmx86_GetUnavailablePerfCtrs();
      retval = HostIF_CopyToUser(ioarg, &ctrs, sizeof ctrs);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2018-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2011,2014-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -23,7 +23,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
@@ -37,9 +36,6 @@
 /* Per-instance driver state */
 struct VMDriver;

-/* 16 pages (64KB) looks as a good limit for one allocation */
-#define VMMON_MAX_LOWMEM_PAGES  16
-
 typedef struct Device {
   struct Device   *next;
   struct VMDriver *vm;
@@ -48,19 +44,10 @@ typedef struct Device {
    * avoid racing between various ioctls, and the creation
    * and removal of the VM in question. The lock is read-acquired
    * by ioctls that reference the VMDriver, and write-acquired by
-    * ioctls or device callbacks that allocate or destory the
+    * ioctls or device callbacks that allocate or destroy the
    * VMDriver.
    */
   struct rw_semaphore vmDriverRWSema;
-   /*
-    * The semaphore protect accesses to size4Gb and pages4Gb
-    * in mmap(). mmap() may happen only once, and all other
-    * accesses except cleanup are read-only, and may happen
-    * only after successful mmap.
-    */
-   struct semaphore lock4Gb;
-   PageCnt size4Gb;
-   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
 } Device;


@@ -73,19 +60,14 @@ typedef struct Device {

 typedef struct VMXLinuxState {
   int major;
-   int minor;
-   struct miscdevice misc;
   char deviceName[VM_DEVICE_NAME_SIZE];
   char buf[LINUXLOG_BUFFER_SIZE];
   Device *head;

   struct task_struct *fastClockThread;
   unsigned fastClockRate;
-   uint64 swapSize;
 } VMXLinuxState;

 extern VMXLinuxState linuxState;
-extern uint8 monitorIPIVector;
-extern uint8 hvIPIVector;

 #endif
@@ -71,6 +71,8 @@
 #include "apic.h"
 #include "memDefaults.h"
 #include "vcpuid.h"
+#include "x86svm.h"
+#include "crosspage.h"

 #include "pgtbl.h"
 #include "versioned_atomic.h"
@@ -179,8 +181,32 @@ static struct {

 static void UnlockEntry(void *clientData, MemTrackEntry *entryPtr);

-uint8 monitorIPIVector;
-uint8 hvIPIVector;
+/*
+ * SPURIOUS_APIC_VECTOR is not defined on kernels built without APIC
+ * support.
+ */
+#ifndef SPURIOUS_APIC_VECTOR
+#define SPURIOUS_APIC_VECTOR 0
+#endif
+static const uint8 monitorIPIVector = SPURIOUS_APIC_VECTOR;
+
+/*
+ * POSTED_INTR_VECTOR is defined on kernels after 3.10 when built with
+ * KVM support.
+ */
+#ifndef POSTED_INTR_VECTOR
+#define POSTED_INTR_VECTOR 0
+#endif
+static const uint8 hvIPIVector = POSTED_INTR_VECTOR;
+
+/*
+ * perfCtrVector may leak, choose highest priority interrupt that is
+ * consistently ack'd by all current linux kernels.
+ */
+#ifndef ERROR_APIC_VECTOR
+#define ERROR_APIC_VECTOR 0xFE
+#endif
+#define PERF_CTR_VECTOR ERROR_APIC_VECTOR

 /*
 *-----------------------------------------------------------------------------
@@ -600,38 +626,6 @@ HostIF_FastClockUnlock(int callerID) // IN
 }


-/*
- *----------------------------------------------------------------------
- *
- * MapCrossPage & UnmapCrossPage
- *
- *    Both x86-64 and ia32 need to map crosspage to an executable
- *    virtual address. We use the vmap interface instead of kmap
- *    due to bug 43907.
- *
- * Side effects:
- *
- *    UnmapCrossPage assumes that the page has been refcounted up
- *    so it takes care of the put_page.
- *
- *----------------------------------------------------------------------
- */
-static void *
-MapCrossPage(struct page *p)  // IN:
-{
-   return vmap(&p, 1, VM_MAP, VM_PAGE_KERNEL_EXEC);
-}
-
-
-static void
-UnmapCrossPage(struct page *p,  // IN:
-               void *va)        // IN:
-{
-   vunmap(va);
-   put_page(p);
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -923,7 +917,6 @@ HostIF_AllocLowPage(VMDriver *vm) //  IN: VM instance pointer

      mpn = (MPN)page_to_pfn(pg);

-      HostIF_VMLock(vm, 40);
      vmh = vm->vmhost;
      if (vmh->AWEPages != NULL) {
         if (PhysTrack_Test(vmh->AWEPages, mpn)) {
@@ -934,7 +927,6 @@ HostIF_AllocLowPage(VMDriver *vm) //  IN: VM instance pointer
         __free_page(pg);
         mpn = INVALID_MPN;
      }
-      HostIF_VMUnlock(vm, 40);
   }
   return mpn;
 }
@@ -961,12 +953,8 @@ static void
 HostIFFreeVMHost(VMHost *vmhost) // IN:
 {
   ASSERT(vmhost->lockedPages == NULL &&
-          vmhost->AWEPages    == NULL &&
-          vmhost->crosspagePagesCount == 0 &&
-          !vmhost->hostAPICIsMapped);
-   Vmx86_Free(vmhost->crosspagePages);
+          vmhost->AWEPages    == NULL);
   Vmx86_Free(vmhost->vcpuSemaTask);
-   vmhost->crosspagePages = NULL;
   vmhost->vcpuSemaTask   = NULL;
   HostIF_FreeKernelMem(vmhost);
 }
@@ -995,14 +983,116 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
   if (vmhost == NULL) {
      return NULL;
   }
-   if ((vmhost->crosspagePages =
-        Vmx86_Calloc(numVCPUs, sizeof *vmhost->crosspagePages, TRUE)) != NULL &&
-       (vmhost->vcpuSemaTask =
-        Vmx86_Calloc(numVCPUs, sizeof *vmhost->vcpuSemaTask, TRUE))   != NULL) {
-      return vmhost;
+   vmhost->vcpuSemaTask = Vmx86_Calloc(numVCPUs, sizeof *vmhost->vcpuSemaTask,
+                                       TRUE);
+   if (vmhost->vcpuSemaTask == NULL) {
+      HostIFFreeVMHost(vmhost);
+      return NULL;
   }
-   HostIFFreeVMHost(vmhost);
-   return NULL;
+
+   return vmhost;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_FreeContigPages --
+ *
+ *      Free an allocated memory block returned from an earlier call to
+ *      HostIF_AllocContigPages.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+HostIF_FreeContigPages(VMDriver *vm,             // IN: Driver state
+                       HostIFContigMemMap *map)  // IN: Object to free
+{
+   unsigned order = 0;
+
+   /*
+    * The only users of this API are the I/O and MSR bitmap allocations.
+    * Of those, the largest is the SVM I/O bitmap at 3 pages.
+    */
+   ASSERT(map->pages <= SVM_VMCB_IO_BITMAP_PAGES);
+   if (vm != NULL) {
+      PageCnt i;
+
+      for (i = 0; i < map->pages; i++) {
+         PhysTrack_Remove(vm->vmhost->AWEPages, map->mpn + i);
+      }
+   }
+   while (1u << order < map->pages) {
+      order++;
+   }
+   free_pages(PtrToVA64(map->addr), order);
+   HostIF_FreeKernelMem(map);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_AllocContigPages --
+ *
+ *      Allocate a range of physically contiguous memory.  Also add the pages
+ *      to the VmAnon PhysTracker if 'vm' is non-NULL.  This allows the pages
+ *      to be written to a coredump when needed.
+ *
+ * Results:
+ *      A HostIFContigMemMap object containing the start MPN of the allocated
+ *      range, or NULL on failure.
+ *
+ *----------------------------------------------------------------------
+ */
+
+HostIFContigMemMap *
+HostIF_AllocContigPages(VMDriver *vm,      // IN/OUT: driver state
+                        PageCnt numPages)  // IN: Number of pages
+{
+   HostIFContigMemMap *map;
+   unsigned order = 0;
+   /*
+    * The only users of this API are the I/O and MSR bitmap allocations.
+    * Of those, the largest is the SVM I/O bitmap at 3 pages.
+    */
+   if (numPages == 0 || numPages > SVM_VMCB_IO_BITMAP_PAGES) {
+      return NULL;
+   }
+   map = HostIF_AllocKernelMem(sizeof *map, FALSE);
+   if (map == NULL) {
+      return NULL;
+   }
+   memset(map, 0, sizeof *map);
+   while (1u << order < numPages) {
+      order++;
+   }
+   map->addr = VA64ToPtr(__get_free_pages(GFP_HIGHUSER, order));
+   if (map->addr == NULL) {
+      HostIF_FreeKernelMem(map);
+      return NULL;
+   }
+   map->pages = numPages;
+   map->mpn = MA_2_MPN(virt_to_phys(map->addr));
+
+   /*
+    * Add the allocated pages to the VMM anonymous memory phystracker.
+    * This is to ensure the pages will be written out during a coredump.
+    */
+   if (vm != NULL) {
+      PageCnt i;
+
+      ASSERT(HostIF_VMLockIsHeld(vm));
+      for (i = 0; i < map->pages; i++) {
+         PhysTrack_Add(vm->vmhost->AWEPages, map->mpn + i);
+      }
+   }
+
+   return map;
 }


@@ -1405,26 +1495,12 @@ UnlockEntry(void *clientData,         // IN:
 void
 HostIF_FreeAllResources(VMDriver *vm) // IN
 {
-   unsigned int cnt;
-
   HostIFHostMemCleanup(vm);
   if (vm->memtracker) {
      MemTrack_Cleanup(vm->memtracker, UnlockEntry, vm);
      vm->memtracker = NULL;
   }
   if (vm->vmhost) {
-      ASSERT(vm->vmhost->crosspagePagesCount <= vm->numVCPUs);
-      for (cnt = vm->vmhost->crosspagePagesCount; cnt > 0; ) {
-         struct page* p = vm->vmhost->crosspagePages[--cnt];
-         UnmapCrossPage(p, vm->crosspage[cnt]);
-      }
-      vm->vmhost->crosspagePagesCount = 0;
-      if (vm->vmhost->hostAPICIsMapped) {
-         ASSERT(vm->hostAPIC.base != NULL);
-         iounmap((void*)vm->hostAPIC.base);
-         vm->hostAPIC.base = NULL;
-         vm->vmhost->hostAPICIsMapped = FALSE;
-      }
      HostIFFreeVMHost(vm->vmhost);
      vm->vmhost = NULL;
   }
@@ -1623,7 +1699,6 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
                                    : BYTES_2_PAGES(vm->memInfo.hugePageBytes);
   PageCnt lockedPages = hugePages + reservedPages;
   PageCnt anonPages;
-   PageCnt swapPages = BYTES_2_PAGES(linuxState.swapSize);

   /* global_page_state is global_zone_page_state in 4.14. */
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
@@ -1632,7 +1707,10 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
   lockedPages += global_page_state(NR_PAGETABLE);
 #endif
   /* NR_SLAB_* moved from zone to node in 4.13. */
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)
+   lockedPages += global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
   lockedPages += global_node_page_state(NR_SLAB_UNRECLAIMABLE);
 #else
   lockedPages += global_page_state(NR_SLAB_UNRECLAIMABLE);
@@ -1650,9 +1728,6 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
   anonPages = global_page_state(NR_ANON_PAGES);
 #endif

-   if (anonPages > swapPages) {
-      lockedPages += anonPages - swapPages;
-   }
   forHost = lockedPages + LOCKED_PAGE_SLACK;
   if (forHost > totalPhysicalPages) {
      forHost = totalPhysicalPages;
@@ -2050,49 +2125,36 @@ HostIF_CopyToUser(VA64 dst,         // OUT
 /*
 *-----------------------------------------------------------------------------
 *
- * HostIF_MapCrossPage --
+ * HostIFGetCrossPageMPNWork --
+ * HostIF_GetCrossPageCodeMPN --
+ * HostIF_GetCrossPageDataMPN --
 *
- *    Map the cross page in the kernel address space given a user VA.  The
- *    kernel mapping must not overlap the monitor's 64MB VA space (bug 32922).
- *    In practice, the host kernel does not return mappings in that range.
- *    This is checked in TaskCreatePTPatch() which gracefully fails if it must.
- *
- * Results:
- *    The kernel virtual address on success
- *    NULL on failure
- *
- * Side effects:
- *    None
 *
 *-----------------------------------------------------------------------------
 */

-void *
-HostIF_MapCrossPage(VMDriver *vm, // IN
-                    VA64 uAddr)   // IN
+static MPN
+HostIFGetCrossPageMPNWork(void *va)
 {
-   void *p = VA64ToPtr(uAddr);
-   struct page *page;
-   VA           vPgAddr;
-   VA           ret;
+   struct page *pg = vmalloc_to_page(va);

-   if (HostIFGetUserPages(p, &page, 1)) {
-      return NULL;
+   if (pg != NULL) {
+      return page_to_pfn(pg);
   }
-   vPgAddr = (VA) MapCrossPage(page);
-   HostIF_VMLock(vm, 27);
-   if (vm->vmhost->crosspagePagesCount >= vm->numVCPUs) {
-      HostIF_VMUnlock(vm, 27);
-      UnmapCrossPage(page, (void*)vPgAddr);

-      return NULL;
-   }
-   vm->vmhost->crosspagePages[vm->vmhost->crosspagePagesCount++] = page;
-   HostIF_VMUnlock(vm, 27);
+   return INVALID_MPN;
+}

-   ret = vPgAddr | (((VA)p) & (PAGE_SIZE - 1));
+MPN
+HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData)
+{
+   return HostIFGetCrossPageMPNWork(crosspageData);
+}

-   return (void*)ret;
+MPN
+HostIF_GetCrossPageCodeMPN(void)
+{
+   return HostIFGetCrossPageMPNWork((void *)CrossPage_CodePage);
 }


@@ -2285,97 +2347,25 @@ HostIF_VMLockIsHeld(VMDriver *vm) // IN
 static Bool
 isVAReadable(VA r)  // IN:
 {
-   mm_segment_t old_fs;
   uint32 dummy;
   int ret;

+   r = APICR_TO_ADDR(r, APICR_VERSION);
+#ifdef HAVE_GET_KERNEL_NOFAULT
+   ret = get_kernel_nofault(dummy, (void *)r);
+#else
+   {
+   mm_segment_t old_fs;
+
   old_fs = get_fs();
   set_fs(KERNEL_DS);
-   r = APICR_TO_ADDR(r, APICR_VERSION);
+
   ret = HostIF_CopyFromUser(&dummy, r, sizeof dummy);
   set_fs(old_fs);
-
+   }
+#endif
   return ret == 0;
 }
-
-
-/*
- *----------------------------------------------------------------------
- *
- * SetVMAPICAddr --
- *
- *      Maps the host cpu's APIC.  The virtual address is stashed in
- *      the VMDriver structure.
- *
- * Results:
- *      None.
- *
- * Side effects:
- *      The VMDriver structure is updated.
- *
- *----------------------------------------------------------------------
- */
-
-static void
-SetVMAPICAddr(VMDriver *vm, // IN/OUT: driver state
-              MA ma)        // IN: host APIC's ma
-{
-   volatile void *hostapic;
-
-   ASSERT_ON_COMPILE(APICR_SIZE <= PAGE_SIZE);
-   hostapic = (volatile void *) ioremap(ma, PAGE_SIZE);
-   if (hostapic) {
-      if ((APIC_VERSIONREG(hostapic) & 0xF0) == 0x10) {
-         vm->hostAPIC.base = (volatile uint32 (*)[4]) hostapic;
-         ASSERT(vm->vmhost != NULL);
-         vm->vmhost->hostAPICIsMapped = TRUE;
-      } else {
-         iounmap((void*)hostapic);
-      }
-   }
-}
-
-
-/*
- *----------------------------------------------------------------------
- *
- * ProbeAPIC --
- *
- *      Attempts to map the host APIC.
- *
- *      Most versions of Linux already provide access to a mapped
- *      APIC.  This function is just a backup.
- *
- *      Caveat: We assume that the APIC physical address is the same
- *      on all host cpus.
- *
- * Results:
- *      TRUE if APIC was found, FALSE if not.
- *
- * Side effects:
- *      May map the APIC.
- *
- *----------------------------------------------------------------------
- */
-
-static Bool
-ProbeAPIC(VMDriver *vm,   // IN/OUT: driver state
-          Bool setVMPtr)  // IN: set a pointer to the APIC's virtual address
-{
-   MA ma = APIC_GetMA();
-
-   if (ma == (MA)-1) {
-      return FALSE;
-   }
-
-   if (setVMPtr) {
-      SetVMAPICAddr(vm, ma);
-   } else {
-      vm->hostAPIC.base = NULL;
-   }
-
-   return TRUE;
-}
 #endif


@@ -2384,75 +2374,50 @@ ProbeAPIC(VMDriver *vm,   // IN/OUT: driver state
 *
 * HostIF_APICInit --
 *
- *      Initialize APIC behavior.
- *      Attempts to map the host APIC into vm->hostAPIC.
+ *      Check if APIC is present, and is regular or x2 APIC.
 *
- *      We don't attempt to refresh the mapping after a host cpu
- *      migration.  Fortunately, hosts tend to use the same address
- *      for all APICs.
- *
- *      Most versions of Linux already provide a mapped APIC.  We
- *      have backup code to read APIC_BASE and map it, if needed.
+ *      We check for X2 APIC by checking X2APIC enable bit, and
+ *      for regular APIC by checking if APIC registers are mapped
+ *      as readable page (vs. not present for no APIC).
 *
 * Results:
- *      TRUE
+ *      None.
 *
 * Side effects:
- *      May map the host APIC.
+ *      None.
 *
 *----------------------------------------------------------------------
 */
-Bool
-HostIF_APICInit(VMDriver *vm,   // IN:
-                Bool setVMPtr,  // IN:
-                Bool probe)     // IN: force probing
+
+void
+HostIF_APICInit(VMDriver *vm)   // IN:
 {
 #if defined(CONFIG_SMP)         || defined(CONFIG_X86_UP_IOAPIC) || \
    defined(CONFIG_X86_UP_APIC) || defined(CONFIG_X86_LOCAL_APIC)
-   static Bool apicIPILogged = FALSE;
-   VA kAddr;
+   if (vm->hostAPIC.base == NULL && !vm->hostAPIC.isX2) {
+      static Bool apicIPILogged = FALSE;

-   monitorIPIVector = SPURIOUS_APIC_VECTOR;
-#if defined(POSTED_INTR_VECTOR)
-   hvIPIVector      = POSTED_INTR_VECTOR;
-#else
-   hvIPIVector      = 0;
-#endif
-
-
-   if (!apicIPILogged) {
-      Log("Monitor IPI vector: %x\n", monitorIPIVector);
-      Log("HV      IPI vector: %x\n", hvIPIVector);
-      apicIPILogged = TRUE;
-   }
-
-   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
-      if (setVMPtr) {
-         vm->hostAPIC.base = NULL;
-         vm->vmhost->hostAPICIsMapped = FALSE;
-         vm->hostAPIC.isX2 = TRUE;
+      if (!apicIPILogged) {
+         Log("Monitor IPI vector: %x\n", monitorIPIVector);
+         Log("HV      IPI vector: %x\n", hvIPIVector);
+         apicIPILogged = TRUE;
      }
-      return TRUE;
-   }

-   if (probe && ProbeAPIC(vm, setVMPtr)) {
-      return TRUE;
-   }
+      if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+         vm->hostAPIC.isX2 = TRUE;
+      } else {
+         VA kAddr;

-   /*
-    * Normal case: use Linux's pre-mapped APIC.
-    */
-   kAddr = __fix_to_virt(FIX_APIC_BASE);
-   if (!isVAReadable(kAddr)) {
-      return TRUE;
-   }
-   if (setVMPtr) {
-      vm->hostAPIC.base = (void *)kAddr;
-   } else {
-      vm->hostAPIC.base = NULL;
+         /*
+          * Old APIC: use Linux's pre-mapped APIC.
+          */
+         kAddr = __fix_to_virt(FIX_APIC_BASE);
+         if (isVAReadable(kAddr)) {
+            vm->hostAPIC.base = (void *)kAddr;
+         }
+      }
   }
 #endif
-   return TRUE;
 }


@@ -2530,6 +2495,32 @@ HostIF_GetHVIPIVector(void)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_GetPerfCtrVector --
+ *
+ *      Return a vector the monitor can use for perf counters.
+ *      This needs to be a high priority interrupt that is always ack'd
+ *      by the kernel because they can leak in edge cases.
+ *
+ * Results:
+ *     The vector for perf counter events.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint8
+HostIF_GetPerfCtrVector(void)
+{
+   ASSERT_ON_COMPILE(PERF_CTR_VECTOR == 0xFE);
+   return PERF_CTR_VECTOR;
+}
+
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -2538,8 +2529,10 @@ HostIF_GetHVIPIVector(void)
 *    Perform the semaphore wait (P) operation, possibly blocking.
 *
 * Result:
- *    1 (which equals MX_WAITNORMAL) if success,
- *    negated error code otherwise.
+ *      On success: MX_WAITINTERRUPTED (3) if interrupted by a Unix signal.
+ *                  MX_WAITTIMEDOUT (2) if the timeout occurred.
+ *                  MX_WAITNORMAL (1) if the event was raised.
+ *      On error: MX_WAITERROR (0) on generic error.
 *
 * Side-effects:
 *    None
@@ -2553,11 +2546,12 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
                     uint64 *args)   // IN:
 {
   struct file *file;
-   mm_segment_t old_fs;
   int res;
   int waitFD = args[0];
   int timeoutms = args[2];
   uint64 value;
+   struct poll_wqueues table;
+   unsigned int mask;

   ASSERT(vcpuid < vm->numVCPUs);

@@ -2565,47 +2559,46 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   if (file == NULL) {
      return MX_WAITERROR;
   }
-
-   old_fs = get_fs();
-   set_fs(KERNEL_DS);
-
-   {
-      struct poll_wqueues table;
-      unsigned int mask;
-
-      poll_initwait(&table);
-      current->state = TASK_INTERRUPTIBLE;
-      mask = file->f_op->poll(file, &table.pt);
-      if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
-         vm->vmhost->vcpuSemaTask[vcpuid] = current;
-         schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
-         vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
-      }
-      current->state = TASK_RUNNING;
-      poll_freewait(&table);
+   if (file->f_op->poll == NULL) {
+      fput(file);
+      Warning("%s: File poll operation is NULL\n", __func__);
+      return MX_WAITERROR;
   }

+   poll_initwait(&table);
+   current->state = TASK_INTERRUPTIBLE;
+   mask = file->f_op->poll(file, &table.pt);
+   if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
+      vm->vmhost->vcpuSemaTask[vcpuid] = current;
+      schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
+      vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
+   }
+   current->state = TASK_RUNNING;
+   poll_freewait(&table);
+
   /*
    * Userland only writes in multiples of sizeof(uint64). This will allow
    * the code to happily deal with a pipe or an eventfd. We only care about
    * reading no bytes (EAGAIN - non blocking fd) or sizeof(uint64).
+    *
+    * Upstream Linux changed the function parameter types/ordering in 4.14.0.
    */
-
-   res = file->f_op->read(file, (char *) &value, sizeof value, &file->f_pos);
-
-   if (res == sizeof value) {
-      res = MX_WAITNORMAL;
-   } else {
-      if (res == 0) {
-         res = -EBADF;
-      }
-   }
-
-   set_fs(old_fs);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+   res = kernel_read(file, file->f_pos, (char *)&value, sizeof value);
+#else
+   res = kernel_read(file, &value, sizeof value, &file->f_pos);
+#endif
   fput(file);

+   if (res == sizeof value) {
+      return MX_WAITNORMAL;
+   }
+   if (res == 0) {
+      res = -EBADF;
+   }
+
   /*
-    * Handle benign errors:
+    * Handle various errors:
    * EAGAIN is MX_WAITTIMEDOUT.
    * The signal-related errors are all mapped into MX_WAITINTERRUPTED.
    */
@@ -2624,6 +2617,10 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   case -EBADF:
      res = MX_WAITERROR;
      break;
+   default:
+      Warning("Unexpected error: %s err=%d\n", __FUNCTION__, res);
+      res = MX_WAITERROR;
+      break;
   }
   return res;
 }
@@ -2677,7 +2674,6 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
 *      On error: MX_WAITINTERRUPTED (3) if interrupted by a Unix signal (we
 *                   can block on a preemptive kernel).
 *                MX_WAITERROR (0) on generic error.
- *                Negated system error (< 0).
 *
 * Side-effects:
 *      None
@@ -2686,57 +2682,64 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
 */

 int
-HostIF_SemaphoreSignal(uint64 *args)  // IN:
+HostIF_SemaphoreSignal(VMDriver *vm,  // IN:
+                       uint64 *args)  // IN:
 {
-   struct file *file;
-   mm_segment_t old_fs;
   int res;
   int signalFD = args[1];
-   uint64 value = 1;  // make an eventfd happy should it be there
+   struct file *file;

   file = vmware_fget(signalFD);
-   if (!file) {
-      return MX_WAITERROR;
+   if (file == NULL) {
+      return FALSE;
   }

-   old_fs = get_fs();
-   set_fs(KERNEL_DS);
-
   /*
    * Always write sizeof(uint64) bytes. This works fine for eventfd and
    * pipes. The data written is formatted to make an eventfd happy should
    * it be present.
    */
-
-   res = file->f_op->write(file, (char *) &value, sizeof value, &file->f_pos);
-
-   if (res == sizeof value) {
-      res = MX_WAITNORMAL;
+   if (file->f_op->write != NULL) {
+      /*
+       * PR2650348 - The kernel_write function signature changed in the vanilla
+       * kernel in 4.14.0 and the signature update was backported before that
+       * point in a kernel used by OpenSUSE 15.1 so call the function directly
+       * instead of via the wrapper.
+       *
+       * Kernel 5.10 removed the set_fs functionality which means a user pointer
+       * must be passed into the write function or a -EFAULT will be returned.
+       */
+      res = file->f_op->write(file, vm->vmhost->vmmonData, sizeof (uint64),
+                              &file->f_pos);
+   } else {
+      Warning("%s: File write operation is NULL\n", __func__);
+      res = -ENOTSUPP;
   }
-
-   set_fs(old_fs);
   fput(file);

+   if (res == sizeof (uint64)) {
+      return MX_WAITNORMAL;
+   }
+
   /*
-    * Handle benign errors:
+    * Handle various errors:
    * EAGAIN is MX_WAITTIMEDOUT.
    * The signal-related errors are all mapped into MX_WAITINTERRUPTED.
    */
-
   switch (res) {
   case -EAGAIN:
-      // The pipe is full, so it is already signalled. Success.
-      res = MX_WAITNORMAL;
-      break;
+      /* The pipe is full, so it is already signalled. Success. */
+      return MX_WAITNORMAL;
   case -EINTR:
   case -ERESTART:
   case -ERESTARTSYS:
   case -ERESTARTNOINTR:
   case -ERESTARTNOHAND:
-      res = MX_WAITINTERRUPTED;
-      break;
+      return MX_WAITINTERRUPTED;
+   default:
+      Warning("Unexpected error: %s err=%d\n", __FUNCTION__, res);
+      return MX_WAITERROR;
   }
-   return res;
 }


@@ -3261,12 +3264,9 @@ static int
 HostIFFastClockThread(void *unused)  // IN:
 {
   int res;
-   mm_segment_t oldFS;
   unsigned int rate = 0;
   unsigned int prevRate = 0;

-   oldFS = get_fs();
-   set_fs(KERNEL_DS);
   allow_signal(SIGKILL);

   while ((rate = linuxState.fastClockRate) > MIN_RATE) {
@@ -3289,8 +3289,6 @@ HostIFFastClockThread(void *unused)  // IN:
   }

 out:
-   set_fs(oldFS);
-
   /*
    * Do not exit thread until we are told to do so.
    */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -79,8 +79,6 @@ typedef struct VMHost {
    */
   Mutex vmMutex;

-   unsigned int         crosspagePagesCount;
-   struct page        **crosspagePages;      /* ptr to numVCPUs-sized array */
   struct task_struct **vcpuSemaTask;        /* ptr to numVCPUs-sized array */

   /*
@@ -94,8 +92,11 @@ typedef struct VMHost {
    * as pages for "AWE" guest memory.
    */
   struct PhysTracker  *AWEPages;
-   /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */
-   Bool                 hostAPICIsMapped;
+   /*
+    * Pointer to a userlevel 64-bit area containing the value 1.
+    * This is used for HostIF_SemaphoreSignal.
+    */
+   void *__user vmmonData;
 } VMHost;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -75,12 +75,13 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
   uint64           retval    = MODULECALL_USERRETURN;
   VMCrossPageData *crosspage;
   int              bailValue = 0;
+   Bool             switchOk = TRUE;

   ASSERT(vcpuid < vm->numVCPUs);
   if (vm->crosspage[vcpuid] == NULL) {
      return USERCALL_VMX86ALLOCERR;
   }
-   crosspage = &vm->crosspage[vcpuid]->crosspageData;
+   crosspage = vm->crosspage[vcpuid];

   /*
    * Check if we were interrupted by signal.
@@ -95,9 +96,9 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
       * Task_Switch changes the world to the monitor.
       * The monitor is waiting in the BackToHost routine.
       */
-      UCTIMESTAMP(crosspage, SWITCHING_TO_MONITOR);
-      Task_Switch(vm, vcpuid);
-      UCTIMESTAMP(crosspage, SWITCHED_TO_MODULE);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHING_TO_MONITOR);
+      switchOk = Task_Switch(vm, vcpuid);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHED_TO_MODULE);

      /*
       * Wake up anything that was waiting for this vcpu to run
@@ -119,6 +120,11 @@ skipTaskSwitch:;

      retval = MODULECALL_USERRETURN;

+      if (UNLIKELY(!switchOk)) {
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
+
      if (crosspage->userCallType != MODULECALL_USERCALL_NONE) {
         /*
          * This is the main user call path.  Handle by returning
@@ -165,7 +171,6 @@ skipTaskSwitch:;

      case MODULECALL_SEMAWAIT: {
         retval = HostIF_SemaphoreWait(vm, vcpuid, crosspage->args);
-
         if (retval == MX_WAITINTERRUPTED) {
            crosspage->moduleCallInterrupted = TRUE;
            bailValue = USERCALL_RESTART;
@@ -175,7 +180,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_SEMASIGNAL: {
-         retval = HostIF_SemaphoreSignal(crosspage->args);
+         retval = HostIF_SemaphoreSignal(vm, crosspage->args);

         if (retval == MX_WAITINTERRUPTED) {
             crosspage->moduleCallInterrupted = TRUE;
@@ -283,6 +288,10 @@ skipTaskSwitch:;
         retval = HostIF_GetHVIPIVector();
      } break;

+      case MODULECALL_GET_PERF_CTR_VECTOR: {
+         retval = HostIF_GetPerfCtrVector();
+      } break;
+
      case MODULECALL_GET_HOST_TIMER_VECTORS: {
         uint8 v0, v1;
         HostIF_GetTimerVectors(&v0, &v1);
@@ -316,13 +325,53 @@ skipTaskSwitch:;
         crosspage->args[1] = vm->numPTPPages;
      } break;

+      case MODULECALL_GET_HV_IO_BITMAP: {
+         crosspage->args[0] = hvIOBitmap == NULL ? INVALID_MPN :
+                                                   hvIOBitmap->mpn;
+      } break;
+
+      case MODULECALL_GET_MSR: {
+         /*
+          * Allocate a MSRQuery request on stack with only one MSRReply slot
+          * since this MSR will be queried from a MSR cache or on a single pcpu.
+          */
+         uint8 req[sizeof(MSRQuery) + sizeof(MSRReply)];
+         MSRQuery *query = (MSRQuery *)&req[0];
+         query->msrNum = (uint32)crosspage->args[0];
+         query->numLogicalCPUs = 1;
+         retval = Vmx86_GetAllMSRs(query);
+         crosspage->args[0] = query->logicalCPUs[0].msrVal;
+      } break;
+
+      case MODULECALL_ALLOC_CONTIG_PAGES: {
+         PageCnt pages = crosspage->args[0];
+         HostIFContigMemMap *alloc;
+
+         HostIF_VMLock(vm, 47);
+         alloc = HostIF_AllocContigPages(vm, pages);
+         if (alloc != NULL) {
+            alloc->next = vm->contigMappings;
+            vm->contigMappings = alloc;
+            crosspage->args[1] = alloc->mpn;
+         } else {
+            crosspage->args[1] = INVALID_MPN;
+         }
+         HostIF_VMUnlock(vm, 47);
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
-      ASSERT(retval == (uint64)((uint32)retval));
+
+      if (retval != (uint64)((uint32)retval)) {
+         Warning("Unexpected error in modulecall %u (%llu)\n",
+                 crosspage->moduleCallType, retval);
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
      crosspage->retval = (uint32)retval;
 #if defined(__linux__)
-      cond_resched(); // Other kernels are preemptable
+      cond_resched(); // Other kernels are preemptible
 #endif
   }
 bailOut:
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -136,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -200,7 +200,7 @@ Net_GetNumAdapters(Net_AdapterCount *counts)
      } else if (Str_Strcasecmp(adapterStr, "e1000e") == 0) {
         counts->e1000e++;
      } else {
-         LOG_ONCE(("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr));
+         LOG_ONCE("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr);
      }
      free(adapterStr);
   }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002,2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -4327,7 +4327,7 @@ LookupTypeClass(unsigned short typeValue) // IN: ethernet type
 *----------------------------------------------------------------------
 */

-#if _WIN32
+#ifdef _WIN32
 _Use_decl_annotations_
 #endif
 void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -81,7 +81,9 @@ extern unsigned int  vnet_max_qlen;
 #   define skb_frag_page(frag) (frag)->page
 #   define skb_frag_size(frag) (frag)->size
 #endif
-#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0)
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0) && \
+    !(defined(CONFIG_SUSE_VERSION) && CONFIG_SUSE_VERSION == 15 && \
+      defined(CONFIG_SUSE_PATCHLEVEL) && CONFIG_SUSE_PATCHLEVEL >= 2)
 #   define skb_frag_off(frag) (frag)->page_offset
 #endif

@@ -559,7 +561,12 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
      return -EINVAL;
   }

+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
   csum = csum_and_copy_to_user(skb->data + offset, curr, len, 0, &err);
+#else
+   csum = csum_and_copy_to_user(skb->data + offset, curr, len);
+   err = (csum == 0) ? -EFAULT : 0;
+#endif
   if (err) {
      return err;
   }
@@ -573,8 +580,14 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
 	 const void *vaddr;

 	 vaddr = kmap(skb_frag_page(frag));
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
 	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
 					 curr, skb_frag_size(frag), 0, &err);
+#else
+	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
+					 curr, skb_frag_size(frag));
+         err = (tmpCsum == 0) ? -EFAULT : 0;
+#endif
 	 kunmap(skb_frag_page(frag));

 	 if (err) {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -207,7 +207,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +314,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -122,87 +129,29 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock And [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock Xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -3584,7 +3476,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3607,7 +3499,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -48,7 +55,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -500,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -553,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -589,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -633,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -656,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -783,17 +772,9 @@ PAUSE(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -841,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -854,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -1180,6 +1155,9 @@ RoundUpPow2_32(uint32 value)
 static INLINE unsigned
 PopCount32(uint32 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
   /*
    * Attribution:
    *     This algorithm was copied from:
@@ -1223,6 +1201,7 @@ PopCount32(uint32 value)
   value += (value >> 8);
   value += (value >> 16);
   return value & 0x0000003f;
+#endif
 }


@@ -1245,6 +1224,13 @@ PopCount32(uint32 value)
 static INLINE unsigned
 PopCount64(uint64 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
   value -= (value >> 1) & 0x5555555555555555ULL;
   value = ((value >> 2) & 0x3333333333333333ULL) +
           (value & 0x3333333333333333ULL);
@@ -1253,6 +1239,7 @@ PopCount64(uint64 value)
   value += value >> 16;
   value += value >> 32;
   return (unsigned) (value & 0xff);
+#endif
 }


@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,41 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef ULM
-void _fxsave64(void *save);
-void _fxsave(void *save);
-void _fxrstor64(const void *load);
-void _fxrstor(const void *load);
-void _xsave64(void *save, uint64 mask);
-void _xsave(void *save, uint64 mask);
-void _xsaveopt64(void *save, uint64 mask);
-void _xsavec(void *save, uint64 mask);
-void _xrstor64(const void *load, uint64 mask);
-void _xrstor(const void *load, uint64 mask);
-#endif /* ULM */
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#ifdef ULM
-#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
-                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
-#endif /* ULM */
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -358,11 +259,7 @@ SFENCE(void)
   );
 #elif defined _MSC_VER
   _ReadWriteBarrier();
-#if defined VM_X86_32
-   __asm sfence;
-#else
   _mm_sfence();
-#endif
   _ReadWriteBarrier();
 #else
 #error No compiler defined for SFENCE
@@ -438,10 +438,6 @@ void *_ReturnAddress(void);

 #define strtok_r  strtok_s

-#if (_MSC_VER < 1500)
-#define	vsnprintf _vsnprintf
-#endif
-
 typedef int uid_t;
 typedef int gid_t;

@@ -465,20 +461,9 @@ typedef int pid_t;
 #define       W_OK          2
 #define       R_OK          4

-#endif // }
+#endif // } _WIN32

-/*
- * Macro for username comparison.
- */
-
-#ifdef _WIN32 // {
-#define USERCMP(x,y)  Str_Strcasecmp(x,y)
-#else
-#define USERCMP(x,y)  strcmp(x,y)
-#endif // }
-
-
-#endif // }
+#endif // } USERLEVEL

 #ifndef va_copy

@@ -657,8 +642,10 @@ typedef int pid_t;
 #endif

 #ifdef VMM
+#define vmx86_vmm 1
 #define VMM_ONLY(x) x
 #else
+#define vmx86_vmm 0
 #define VMM_ONLY(x)
 #endif

@@ -699,7 +686,6 @@ typedef int pid_t;
 * display/printer drivers only.
 */
 #ifdef _WIN32
-#ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
 #ifdef _WIN64
 #define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
@@ -711,7 +697,6 @@ typedef int pid_t;
 #define WinDrvPrint(arg, ...)
 #define WinDrvEngPrint(arg, ...)
 #endif
-#endif
 #endif // _WIN32

 /*
@@ -801,4 +786,30 @@ typedef int pid_t;
 #define END_PTR_TO_ALIGNED_VAR \
   } while (0)

+
+/*
+ * -Wswitch means that when you pass switch an enum that it's looking for
+ * all values from that enum, and only that enum, to be accounted for.
+ * "default:;" is fine for catching values you don't care about. But today
+ * we have a bunch of code that uses internal and external enum values, or
+ * in other words combines two enums into a single variable. This cast is
+ * the workaround, but we really need to fix this mess.
+ */
+#define UNCHECKED_SWITCH__FIXME(x) switch ((uint64)(x))
+
+
+/*
+ * When clang static analyzer parses source files, it implicitly defines
+ * __clang_analyzer__ macro. We use this to define our custom macro to stop
+ * its execution for the current path of analysis by calling a function that
+ * doesn't return, making it think that it hit a failed assertion.
+ *
+ * DO NOT use to silence the analyzer! See PR2447238.
+ */
+#ifdef __clang_analyzer__
+#define VMW_CLANG_ANALYZER_NORETURN() Panic("Disable Clang static analyzer")
+#else
+#define VMW_CLANG_ANALYZER_NORETURN() ((void)0)
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -101,6 +101,7 @@

 #pragma warning (3 :4505) // unreferenced local function
 #pragma warning (disable :4018) // signed/unsigned mismatch
+#pragma warning (suppress:4619) // suppress warning next line (C4761 was removed in vs2019u4)
 #pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
 #pragma warning (disable :4305) // truncation from 'const int' to 'short'
 #pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
@@ -687,22 +688,28 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-3.3 (circa 2003).  Enforce this here, various things below
+ * gcc-4.1 (circa 2006).  Enforce this here, various things below
 * this line depend upon it.
 *
- * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
- * The various linux kernel modules may use older (gcc-3.3) compilers.
+ * Current oldest compilers:
+ * - guest tools: 4.1.2 (freebsd/solaris)
+ * - buildhost compiler: 4.4.3
+ * - hosted kernel modules: 4.5
+ *
+ * SWIG's preprocessor is exempt.
 */
-#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
-#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
+#ifndef SWIG
+#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
+#error "gcc version is too old, need gcc-4.1 or better"
+#endif
 #endif

 /*
- * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Similarly, we require a compiler that is at least vs2012.
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1400
-#error "cl.exe version is too old, need vc80 or better"
+#if defined _MSC_VER && _MSC_VER < 1700
+#error "cl.exe version is too old, need vs2012 or better"
 #endif


@@ -864,12 +871,10 @@ typedef void * UserVA;
 #endif

 #ifndef UNUSED_TYPE
-// XXX _Pragma would better but doesn't always work right now.
 #  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
 #endif

 #ifndef UNUSED_VARIABLE
-// XXX is there a better way?
 #  define UNUSED_VARIABLE(_var) (void)_var
 #endif

@@ -884,29 +889,21 @@ typedef void * UserVA;

 /*
 * ALIGNED specifies minimum alignment in "n" bytes.
+ *
+ * NOTE: __declspec(align) has limited syntax; it must essentially be
+ *       an integer literal.  Expressions, such as sizeof(), do not
+ *       work.
 */

 #ifdef __GNUC__
 #define ALIGNED(n) __attribute__((__aligned__(n)))
+#elif defined(_MSC_VER)
+#define ALIGNED(n) __declspec(align(n))
 #else
 #define ALIGNED(n)
 #endif


-/*
- * Encapsulate the syntactic differences between gcc and msvc alignment control.
- * BOUNDARY must match in the prefix and suffix.
- */
-
-#ifdef _WIN32
-#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
-#define ALIGN_SUFFIX(BOUNDRY)
-#else
-#define ALIGN_PREFIX(BOUNDRY)
-#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
-#endif
-
-
 /*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2005-2012,2014-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2005-2012,2014-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,8 +35,6 @@
 #endif
 #endif

-#include <linux/pci_ids.h>
-
 /* LSILogic 53C1030 Parallel SCSI controller
 * LSILogic SAS1068 SAS controller
 */
@@ -55,10 +53,7 @@
 *    VMware HD Audio codec
 *    VMware HD Audio controller
 */
-#ifndef PCI_VENDOR_ID_VMWARE
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
-#endif
-
 #define PCI_DEVICE_ID_VMWARE_SVGA3              0x0406
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
@@ -76,11 +71,7 @@
 #define PCI_DEVICE_ID_VMWARE_1394               0x0780
 #define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
 #define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
-
-#ifndef PCI_DEVICE_ID_VMWARE_VMXNET3
 #define PCI_DEVICE_ID_VMWARE_VMXNET3            0x07B0
-#endif
-
 #define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
 #define PCI_DEVICE_ID_VMWARE_82574              0x07D0
 #define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
@@ -353,7 +344,7 @@
 #define MAX_NVDIMM 64

 /************* vRDMA implementation limits ******************************/
-#define MAX_VRDMA_DEVICES 1
+#define MAX_VRDMA_DEVICES 10

 /************* QAT implementation limits ********************/
 #define MAX_QAT_PCI_DEVICES 4
Author	SHA1	Message	Date
Michal Kubecek	31c032e989	import Player 16.1.0 module sources	2020-11-20 09:13:14 +01:00
Michal Kubecek	a84c5005fe	import Player 16.0.0 module sources	2020-09-15 20:45:48 +02:00
Michal Kubecek	7fd1a58863	import Player 15.5.5 module sources	2020-05-30 02:30:43 +02:00
Michal Kubecek	cbd3374fd0	import Player 15.5.2 module sources	2020-03-14 09:59:24 +01:00
Michal Kubecek	87feab5f03	import Player 15.5.1 module sources	2019-11-14 07:56:57 +01:00
Michal Kubecek	6daa6921e7	import Player 15.5.0 module sources	2019-09-21 00:11:26 +02:00
Michal Kubecek	193db5638c	Merge branch 'master' into player	2019-09-21 00:10:31 +02:00
Michal Kubecek	1f8ca44e74	import Player 15.1.0 module sources	2019-05-15 07:06:02 +02:00
Michal Kubecek	3686288fcb	import Player 15.0.1 module sources	2018-11-12 07:48:28 +01:00
Michal Kubecek	3d8f309653	import Player 15.0.0 module sources	2018-10-02 19:09:36 +02:00
Michal Kubecek	56b2ba7686	import Player 14.1.3 module sources	2018-08-20 07:28:17 +02:00
Michal Kubecek	7c61516e73	Merge branch 'master' into player	2018-05-27 16:13:48 +02:00
Michal Kubecek	7ee6cd4520	import Player 14.1.2 module sources	2018-05-23 18:14:05 +02:00
Michal Kubecek	5791987f56	Merge branch 'master' into player	2018-04-02 15:37:18 +02:00
Michal Kubecek	0b2713797e	Merge branch 'master' into player	2018-02-04 21:50:37 +01:00
Michal Kubecek	3e445144a1	import Player 14.1.1 module sources	2018-01-12 13:51:34 +01:00
Michal Kubecek	82f569a212	import Player 14.1.0 module sources	2017-12-25 09:47:38 +01:00
Michal Kubecek	455d4d9368	import Player 14.0.0 module sources	2017-09-30 20:31:15 +02:00
Michal Kubecek	027dd56418	import Player 12.5.7 module sources	2017-06-30 19:11:22 +02:00
Michal Kubecek	0e634ffbf1	import Player 12.5.6 module sources	2017-05-31 10:06:51 +02:00
Michal Kubecek	c62c78e908	import Player 12.5.5 module sources	2017-05-31 09:20:00 +02:00
Michal Kubecek	4f648692e4	import Player 12.5.2 module sources	2017-05-31 09:19:26 +02:00